diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..e237b00c --- /dev/null +++ b/.gitmodules @@ -0,0 +1,4 @@ +[submodule "webrtc-java-media/third-party/ffmpeg"] + path = webrtc-java-media/third-party/ffmpeg + url = https://github.com/FFmpeg/FFmpeg.git + shallow = true diff --git a/docs/.vitepress/sidebar.ts b/docs/.vitepress/sidebar.ts index ed659fbf..07228b5c 100644 --- a/docs/.vitepress/sidebar.ts +++ b/docs/.vitepress/sidebar.ts @@ -22,6 +22,7 @@ function sidebarGuide(): DefaultTheme.SidebarItem[] { { text: 'Media Devices', link: '/media/media-devices' }, { text: 'Media Constraints', link: '/media/constraints' }, { text: 'Media Directionality', link: '/media/directionality' }, + { text: 'Media Files', link: '/media/media-files' }, ], }, { diff --git a/docs/guide/audio/custom-audio-source.md b/docs/guide/audio/custom-audio-source.md index bdc5d0ab..df26ff84 100644 --- a/docs/guide/audio/custom-audio-source.md +++ b/docs/guide/audio/custom-audio-source.md @@ -92,6 +92,20 @@ The call checks its arguments and throws `IllegalArgumentException` rather than The audio is handed to the track's senders on the thread that calls `pushAudio`, so call it from a single thread. A scheduled executor with one thread, as shown below, is the simplest way to do that. ::: +### Keeping the Source's Own Timing + +A chunk pushed with the call above is treated as captured at that moment. A source with timing of its own, such as a media file played alongside video, should say when the audio was captured instead: + +```java +long baseUs = SyncClock.currentTimeUs(); // once, when playback starts +long presentationUs = chunkIndex * 10_000L; // 10 ms per chunk + +audioSource.pushAudio(pcm, 16, sampleRate, channels, frameCount, + baseUs + presentationUs); +``` + +Use the same base for the audio and the video of one source, and the two stay in sync on the receiving side. See [Custom Video Source](/guide/video/custom-video-source) for the video half. Chunks must still be pushed in real time; the timestamp describes the audio, it does not schedule it. + ## Audio Format Considerations When pushing audio data, you need to consider the following parameters: diff --git a/docs/guide/examples.md b/docs/guide/examples.md index 86ee625c..ae9191cb 100644 --- a/docs/guide/examples.md +++ b/docs/guide/examples.md @@ -52,6 +52,27 @@ The [`DesktopVideoExample`](https://github.com/devopvoid/webrtc-java/blob/master This example is particularly useful for applications that need to implement screen sharing or remote desktop functionality. +## Media File + +The [`MediaFileExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/media/java/dev/onvoid/webrtc/examples/MediaFileExample.java) demonstrates how to send a media file over a peer connection, in place of a camera and a microphone. See the [Media Files](/guide/media/media-files) guide for the API it uses. + +**Key features demonstrated:** +- Opening a media file with a `MediaFileSource` +- Reading what the source contains from its `MediaInfo` +- Creating audio and video tracks from the media sources it feeds +- Adding those tracks to a peer connection +- Following playback through a `MediaPlayerListener` + +This example is useful for applications that stream pre-recorded media, or that need a dependable stand-in for a camera in testing. + +::: warning +This example needs the `webrtc-java-media` module, which is opt-in while it is being brought up on every platform, so it is built and run with the `with-media-extension` profile: + +```bash +mvn -Pwith-media-extension exec:java -D"exec.mainClass=dev.onvoid.webrtc.examples.MediaFileExample" -D"exec.args=movie.mp4" +``` +::: + ## Web Client The [`WebClientExample`](https://github.com/devopvoid/webrtc-java/blob/master/webrtc-examples/src/main/java/dev/onvoid/webrtc/examples/web/WebClientExample.java) demonstrates how to combine WebSocket signaling with WebRTC peer connections for real-time communication between web and Java clients. diff --git a/docs/guide/media/media-files.md b/docs/guide/media/media-files.md new file mode 100644 index 00000000..56874a1e --- /dev/null +++ b/docs/guide/media/media-files.md @@ -0,0 +1,232 @@ +# Media Files + +This guide explains how to send a media file over a peer connection instead of a camera and a microphone, using the `webrtc-java-media` module. It covers: + +- Adding the media module to a project +- Sending a file with `MediaFileSource` +- Reading what a source contains with `MediaReader` and `MediaInfo` +- Controlling playback and following it with a listener +- Feeding your own media sources with `MediaPlayer` + +Sending a file is a common need: a test pattern instead of a webcam, a pre-recorded briefing, a video that has to reach several participants. Without help, an application has to bring its own decoder and push I420 frames into a `CustomVideoSource` itself. The media module removes that work by decoding with [FFmpeg](https://ffmpeg.org) inside the library. + +Decoding happens entirely in native code. Frames never travel through Java: the module hands decoded pictures straight to the native side of a `CustomVideoSource`, and 10 ms chunks to a `CustomAudioSource`. They are paced in real time and carry the presentation times of the file, so what a receiver gets keeps the timing of the media rather than the timing of a Java thread. + +::: warning Opt-in while it is being brought up +The media module is not part of the default build yet, and its native library has so far been built for `windows-x86_64` only. Build it with the `with-media-extension` profile, as described below. +::: + +## Adding the Module + +The module builds FFmpeg from a submodule pinned to a release tag, so the submodule has to be present: + +```shell +git submodule update --init --depth 1 webrtc-java-media/third-party/ffmpeg +mvn install -Pwith-media-extension +``` + +Building FFmpeg needs `make` and `nasm`. On Windows they come from MSYS2: + +```shell +winget install MSYS2.MSYS2 +C:\msys64\usr\bin\bash -lc "pacman -S --needed make nasm diffutils pkgconf" +``` + +Maven still runs from an ordinary shell; the build enters MSYS2 and the Visual Studio environment on its own. The first build compiles FFmpeg, which takes a while; later builds reuse the install directory. + +Once installed, depend on it alongside `webrtc-java`: + +```xml + + dev.onvoid.webrtc + webrtc-java-media + 0.19.0-SNAPSHOT + +``` + +The classifier jar carries the module's native library together with the FFmpeg shared libraries it uses. Applications that do not use this module never download FFmpeg. + +## Sending a File + +`MediaFileSource` is the short way to do all of it. It opens the source, creates a media source for each kind of media the file actually has, and wires a player to feed them: + +```java +// Import required classes +import dev.onvoid.webrtc.media.ffmpeg.MediaFileSource; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.video.VideoTrack; +import java.nio.file.Path; +import java.util.List; + +MediaFileSource source = new MediaFileSource(Path.of("movie.mp4")); + +// Create tracks from the media sources the file feeds. +VideoTrack videoTrack = factory.createVideoTrack("video", source.getVideoSource()); +AudioTrack audioTrack = factory.createAudioTrack("audio", source.getAudioSource()); + +peerConnection.addTrack(videoTrack, List.of("stream")); +peerConnection.addTrack(audioTrack, List.of("stream")); + +// Start sending. +source.play(); +``` + +A file with no audio has no audio source, and likewise for video, so check before making a track: + +```java +if (source.getVideoSource() != null) { + VideoTrack videoTrack = factory.createVideoTrack("video", source.getVideoSource()); + peerConnection.addTrack(videoTrack, List.of("stream")); +} +``` + +::: warning A factory sends either pushed audio or captured audio +A `PeerConnectionFactory` fed from this module is sending pushed audio and cannot also send audio captured by its `AudioDeviceModule`. Give such a factory a dummy audio layer, and use a second factory if an application needs both: + +```java +AudioDeviceModule audioModule = new AudioDeviceModule(AudioLayer.kDummyAudio); +PeerConnectionFactory factory = new PeerConnectionFactory(audioModule); +``` +::: + +## Reading What a Source Contains + +`MediaReader` opens a source and reports what is in it, without playing anything: + +```java +// Import required classes +import dev.onvoid.webrtc.media.ffmpeg.MediaInfo; +import dev.onvoid.webrtc.media.ffmpeg.MediaReader; + +try (MediaReader reader = new MediaReader(Path.of("movie.mp4"))) { + MediaInfo info = reader.getInfo(); + + System.out.println("Runs " + info.getDurationUs() / 1_000_000.0 + " s"); + + if (info.hasVideo()) { + System.out.println(info.getVideoWidth() + "x" + info.getVideoHeight() + + " at " + info.getFrameRate() + " fps, " + info.getVideoCodec()); + } + if (info.hasAudio()) { + System.out.println(info.getSampleRate() + " Hz, " + + info.getChannels() + " channels, " + info.getAudioCodec()); + } +} +``` + +A `MediaFileSource` already has this, so there is no need to open the file twice: + +```java +MediaInfo info = source.getInfo(); +``` + +::: info +Duration is `0` for a source whose container does not say how long it runs, which is the case for live streams. Opening fails if a source holds nothing that can be played. +::: + +## Controlling Playback + +```java +source.play(); // start, or resume after a pause +source.pause(); // hold playback where it is +source.seek(30_000_000); // move to 30 seconds +source.setLooping(true); // start over instead of ending +source.getPositionUs(); // where playback has got to +source.getState(); // IDLE, PLAYING, PAUSED, ENDED or CLOSED +``` + +Looping keeps the timing across the seam, so a looping file works as a stand-in for a camera that never stops. A looping source never reports an end of stream. + +A seek lands on the keyframe at or before the position asked for, which is how far back the decoder has to go to produce a picture at all. How close that is to the position asked for depends on how often the media was encoded with keyframes. + +## Following Playback + +```java +// Import required classes +import dev.onvoid.webrtc.media.ffmpeg.MediaPlayerListener; +import dev.onvoid.webrtc.media.ffmpeg.MediaPlayerState; + +source.setListener(new MediaPlayerListener() { + + @Override + public void onStateChanged(MediaPlayerState state) { + System.out.println("Player state: " + state); + } + + @Override + public void onEndOfStream() { + System.out.println("The source ran out."); + } + + @Override + public void onError(String message) { + System.out.println("Playback failed: " + message); + } +}); +``` + +::: warning +Every call arrives on the player's own thread, and that thread is the one decoding the media. A listener must return promptly, and must not wait on the player. +::: + +## Feeding Your Own Media Sources + +`MediaFileSource` creates the media sources for you. An application that needs its own — to feed a source it already created, or to send only the video of a file — uses `MediaPlayer` directly: + +```java +// Import required classes +import dev.onvoid.webrtc.media.ffmpeg.MediaPlayer; +import dev.onvoid.webrtc.media.ffmpeg.MediaReader; +import dev.onvoid.webrtc.media.video.CustomVideoSource; + +CustomVideoSource videoSource = new CustomVideoSource(); + +// Video only: passing null for the audio source decodes and drops the audio. +MediaPlayer player = new MediaPlayer(new MediaReader(path), videoSource, null); + +VideoTrack videoTrack = factory.createVideoTrack("video", videoSource); + +player.play(); +``` + +::: info +The player takes over the reader it is given. That reader must not be used or closed afterwards; closing the player releases it. +::: + +## Closing + +Closing a `MediaFileSource` stops playback and releases the player along with both media sources. Release the senders a peer connection handed out, and the tracks, before that: + +```java +videoSender.dispose(); +audioSender.dispose(); +peerConnection.close(); + +source.close(); +``` + +## What Can Be Played + +The FFmpeg build is deliberately small, and carries only what this module plays: + +| | | +| --- | --- | +| **Containers** | MP4 and MOV, Matroska and WebM, AVI, MPEG-TS, FLV, WAV, MP3, Ogg, FLAC, AAC | +| **Video** | H.264, H.265/HEVC, VP8, VP9, MPEG-4, MJPEG | +| **Audio** | AAC, MP3, Opus, Vorbis, FLAC, PCM | + +Audio of any rate or layout is resampled to what WebRTC takes, which is 48 kHz 16-bit PCM in mono or stereo. Video that decodes to I420 — almost all 8-bit H.264, VP8, VP9 and MPEG-4 — reaches the encoder without being copied; anything else is converted first. + +Only local files play today. FFmpeg demuxes network sources just as well, so the same code will cover http, rtsp and rtmp once those protocols are turned on in the build. + +## Licensing + +The module uses FFmpeg under the LGPL version 2.1 or later. It is configured without `--enable-gpl` and without `--enable-nonfree`, and FFmpeg is linked dynamically and shipped as separate files inside the platform jar, so its libraries may be replaced with your own build, as the LGPL requires. The wrapper code is licensed under the Apache License 2.0 like the rest of webrtc-java. + +## Complete Example + +See `MediaFileExample` in the `webrtc-examples` module, which opens a file, reports what it contains, creates tracks, adds them to a peer connection and follows playback to the end. + +```shell +mvn -Pwith-media-extension -pl webrtc-examples compile +``` diff --git a/docs/guide/video/custom-video-source.md b/docs/guide/video/custom-video-source.md index 913958ac..9753ac13 100644 --- a/docs/guide/video/custom-video-source.md +++ b/docs/guide/video/custom-video-source.md @@ -142,6 +142,32 @@ public class VideoStreamer { } ``` +### Keeping the Source's Own Timing + +`pushFrame(VideoFrame)` treats a frame as captured at the moment of the call, so the timing that reaches the other peer is the timing of your pushing thread. For a live source such as a camera that is exactly right. For a source that has timing of its own, such as a video file, it is not: every hiccup of the executor above ends up in the stream, and audio pushed alongside drifts out of sync with the video. + +Pass the capture time instead: + +```java +// The clock capture timestamps are interpreted in. +long baseUs = SyncClock.currentTimeUs(); + +// For each frame, its own presentation time within the source. +long presentationUs = frameIndex * 1_000_000L / frameRate; + +videoSource.pushFrame(frame, baseUs + presentationUs); +``` + +Map your source's timeline onto the clock once, when playback starts, and add each frame's presentation time to that base. The frame rate that reaches the encoder then follows your timestamps rather than the jitter of the thread. + +::: warning +Frames must still be pushed in real time. A frame is encoded and sent when it arrives, so a timestamp in the future does not delay it; it only describes when the frame was meant to be shown. + +Timestamps must also advance by at least one millisecond from frame to frame. WebRTC drops a frame whose capture time does not move forward. +::: + +Push audio with timestamps from the same clock, as described in [Custom Audio Source](/guide/audio/custom-audio-source), and the two stay in sync on the receiving side. + ## Integration with Video Tracks ### Adding Sinks to Monitor Video diff --git a/pom.xml b/pom.xml index f6222127..b8e6770a 100644 --- a/pom.xml +++ b/pom.xml @@ -354,6 +354,19 @@ webrtc.macos.aarch64 + + + with-media-extension + + webrtc-java-media + + diff --git a/webrtc-examples/pom.xml b/webrtc-examples/pom.xml index 4c98d6ee..cf9437d3 100644 --- a/webrtc-examples/pom.xml +++ b/webrtc-examples/pom.xml @@ -94,4 +94,75 @@ 2.0.18 + + + + + with-media-extension + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + compile-media-example + compile + + compile + + + 17 + + ${project.basedir}/src/media/java + + ${project.build.directory}/media-classes + + + + + + + + org.codehaus.mojo + exec-maven-plugin + 3.5.0 + + + ${project.build.directory}/media-classes + + + + + + + + + ${project.groupId} + webrtc-java-media + ${project.version} + + + + \ No newline at end of file diff --git a/webrtc-examples/src/media/java/dev/onvoid/webrtc/examples/MediaFileExample.java b/webrtc-examples/src/media/java/dev/onvoid/webrtc/examples/MediaFileExample.java new file mode 100644 index 00000000..cdbc3b05 --- /dev/null +++ b/webrtc-examples/src/media/java/dev/onvoid/webrtc/examples/MediaFileExample.java @@ -0,0 +1,240 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.examples; + +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.logging.Level; +import java.util.logging.Logger; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.PeerConnectionObserver; +import dev.onvoid.webrtc.RTCConfiguration; +import dev.onvoid.webrtc.RTCIceCandidate; +import dev.onvoid.webrtc.RTCIceConnectionState; +import dev.onvoid.webrtc.RTCPeerConnection; +import dev.onvoid.webrtc.RTCRtpSender; +import dev.onvoid.webrtc.media.audio.AudioDeviceModule; +import dev.onvoid.webrtc.media.audio.AudioLayer; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.ffmpeg.MediaFileSource; +import dev.onvoid.webrtc.media.ffmpeg.MediaInfo; +import dev.onvoid.webrtc.media.ffmpeg.MediaPlayerListener; +import dev.onvoid.webrtc.media.ffmpeg.MediaPlayerState; +import dev.onvoid.webrtc.media.video.VideoTrack; + +import java.util.List; + +/** + * Example demonstrating how to send a media file over a peer connection, in + * place of a camera and a microphone. + *

+ * This example shows how to: + *

+ *

+ * Decoding happens entirely in native code. Frames are paced in real time and + * carry the presentation times of the file, so what a receiver gets keeps the + * timing of the media rather than the timing of a Java thread. + *

+ * Note: this example only sets up the local side. A real application would + * reach a remote peer through a signaling channel, as + * {@link PeerConnectionExample} shows. + *

+ * Run it with the media file to send: + * 

+ * java dev.onvoid.webrtc.examples.MediaFileExample movie.mp4
+ *
+ *

+ * Only local files play today. FFmpeg demuxes network sources just as well, + * so the same code covers http, rtsp and rtmp once those protocols are turned + * on in the build. + * + * @author Alex Andres + */ +public class MediaFileExample { + + public static void main(String[] args) { + if (args.length < 1) { + System.out.println("Usage: MediaFileExample "); + System.out.println(" for example: MediaFileExample movie.mp4"); + return; + } + + // A factory that sends pushed audio must not also be capturing from a + // microphone, so this one is given a dummy audio layer. An application + // that needs both needs a second factory. + AudioDeviceModule audioModule = new AudioDeviceModule(AudioLayer.kDummyAudio); + PeerConnectionFactory factory = new PeerConnectionFactory(audioModule); + + // Opening reads the container and picks the streams to play. A source + // with nothing playable in it fails here. + try (MediaFileSource source = new MediaFileSource(args[0])) { + printInfo(source.getInfo()); + + LocalPeer localPeer = new LocalPeer(factory, source); + + CountDownLatch finished = new CountDownLatch(1); + + source.setListener(new MediaPlayerListener() { + + @Override + public void onStateChanged(MediaPlayerState state) { + System.out.println("Player state: " + state); + } + + @Override + public void onEndOfStream() { + System.out.println("Reached the end of the source."); + finished.countDown(); + } + + @Override + public void onError(String message) { + System.out.println("Playback failed: " + message); + finished.countDown(); + } + }); + + // Playing the source over and over is one line, and is what a + // stand-in for a camera usually wants: + // source.setLooping(true); + + source.play(); + + // Report where playback has got to while it runs. + while (!finished.await(1, TimeUnit.SECONDS)) { + System.out.printf(" at %.1f s, %d video frames and %d audio chunks sent%n", + source.getPositionUs() / 1_000_000.0, + localPeer.videoFrames.get(), localPeer.audioChunks.get()); + } + + // The tracks go before the source, which disposes of the media + // sources they were made from. + localPeer.dispose(); + } + catch (Exception e) { + Logger.getLogger(MediaFileExample.class.getName()) + .log(Level.SEVERE, "Error in MediaFileExample", e); + } + finally { + factory.dispose(); + audioModule.dispose(); + } + } + + private static void printInfo(MediaInfo info) { + System.out.printf("Source runs %.3f s%n", info.getDurationUs() / 1_000_000.0); + + if (info.hasVideo()) { + System.out.printf(" video: %dx%d at %.2f fps, %s%n", info.getVideoWidth(), + info.getVideoHeight(), info.getFrameRate(), info.getVideoCodec()); + } + else { + System.out.println(" video: none"); + } + + if (info.hasAudio()) { + System.out.printf(" audio: %d Hz, %d channel(s), %s%n", info.getSampleRate(), + info.getChannels(), info.getAudioCodec()); + } + else { + System.out.println(" audio: none"); + } + } + + /** + * A peer connection carrying the tracks a media file feeds. + */ + private static class LocalPeer implements PeerConnectionObserver { + + final AtomicInteger videoFrames = new AtomicInteger(); + final AtomicInteger audioChunks = new AtomicInteger(); + + private final RTCPeerConnection peerConnection; + private final VideoTrack videoTrack; + private final AudioTrack audioTrack; + private final RTCRtpSender videoSender; + private final RTCRtpSender audioSender; + + + LocalPeer(PeerConnectionFactory factory, MediaFileSource source) { + peerConnection = factory.createPeerConnection(new RTCConfiguration(), this); + + // A source with no video has no video source, and likewise for + // audio, so each track is only made if there is something to feed + // it. + if (source.getVideoSource() != null) { + videoTrack = factory.createVideoTrack("video", source.getVideoSource()); + + // Watching the track shows what the peer connection is being + // given. A real application would not need this. + videoTrack.addSink(frame -> videoFrames.incrementAndGet()); + + videoSender = peerConnection.addTrack(videoTrack, List.of("stream")); + } + else { + videoTrack = null; + videoSender = null; + } + + if (source.getAudioSource() != null) { + audioTrack = factory.createAudioTrack("audio", source.getAudioSource()); + audioTrack.addSink((data, bits, rate, channels, frames) -> + audioChunks.incrementAndGet()); + + audioSender = peerConnection.addTrack(audioTrack, List.of("stream")); + } + else { + audioTrack = null; + audioSender = null; + } + } + + void dispose() { + // An RTCRtpSender is not owned by the peer connection, so each one + // handed out by addTrack has to be released here. Doing it before + // the connection closes is what lets go of the tracks, and with + // them the media sources the file source owns. + if (videoSender != null) { + videoSender.dispose(); + } + if (audioSender != null) { + audioSender.dispose(); + } + + peerConnection.close(); + } + + @Override + public void onIceCandidate(RTCIceCandidate candidate) { + // A real application sends this to the remote peer over its + // signaling channel. + } + + @Override + public void onIceConnectionChange(RTCIceConnectionState state) { + System.out.println("ICE connection state: " + state); + } + } +} diff --git a/webrtc-java-media/.gitignore b/webrtc-java-media/.gitignore new file mode 100644 index 00000000..023936a0 --- /dev/null +++ b/webrtc-java-media/.gitignore @@ -0,0 +1,3 @@ +# FFmpeg out-of-tree build output +/build +/libs diff --git a/webrtc-java-media/README.md b/webrtc-java-media/README.md new file mode 100644 index 00000000..ef048c1c --- /dev/null +++ b/webrtc-java-media/README.md @@ -0,0 +1,61 @@ +# webrtc-java-media + +Media extension for [webrtc-java](https://github.com/devopvoid/webrtc-java). It reads media files +and network streams with FFmpeg and feeds them into a peer connection, so an application can send +a video file the way it would send a camera. + +```java +MediaFileSource source = new MediaFileSource(Path.of("movie.mp4")); + +VideoTrack videoTrack = factory.createVideoTrack("video", source.getVideoSource()); +AudioTrack audioTrack = factory.createAudioTrack("audio", source.getAudioSource()); + +peerConnection.addTrack(videoTrack, List.of("stream")); +peerConnection.addTrack(audioTrack, List.of("stream")); + +source.play(); +``` + +## How it fits together + +Decoding happens entirely in native code. The module does not carry frames through Java: it calls +into webrtc-java's native side through the small C interface in `webrtc_java_api.h`, which hands +decoded pictures to a `CustomVideoSource` and 10 ms chunks to a `CustomAudioSource` without a copy +per frame. Frames are paced in real time and carry their own presentation times, so playback keeps +the timing of the file rather than the timing of a thread. + +A factory that sends media from this module sends pushed audio, so it cannot also send audio +captured by its `AudioDeviceModule`. Use a separate factory if you need both. + +## FFmpeg + +This module uses [FFmpeg](https://ffmpeg.org), licensed under the LGPL version 2.1 or later. The +FFmpeg source it is built from is the `third-party/ffmpeg` submodule, pinned to a release tag, and +is configured without `--enable-gpl` and without `--enable-nonfree`. + +FFmpeg is linked dynamically and its libraries ship as separate files inside the platform jar, so +you may replace them with your own build, as the LGPL requires. The wrapper code in this module is +licensed under the Apache License 2.0 like the rest of webrtc-java. + +## Building + +The submodule has to be present: + +```shell +git submodule update --init --depth 1 webrtc-java-media/third-party/ffmpeg +mvn install -Pwith-media-extension +``` + +The first build compiles FFmpeg, which takes a while; later builds reuse the install directory +(`ffmpeg.install.dir`, by default `~/ffmpeg/`). + +Building FFmpeg needs `make` and `nasm`. On Windows they come from MSYS2, which the build looks for +in `C:/msys64` unless `MSYS2_ROOT` points somewhere else: + +```shell +winget install MSYS2.MSYS2 +C:\msys64\usr\bin\bash -lc "pacman -S --needed make nasm diffutils pkgconf" +``` + +Maven still runs from an ordinary shell. The build enters MSYS2 and the Visual Studio environment +on its own, because FFmpeg's configure needs a POSIX shell that can also see `cl` and `link`. diff --git a/webrtc-java-media/pom.xml b/webrtc-java-media/pom.xml new file mode 100644 index 00000000..45c08edf --- /dev/null +++ b/webrtc-java-media/pom.xml @@ -0,0 +1,197 @@ + + + 4.0.0 + + + dev.onvoid.webrtc + webrtc-java-parent + 0.19.0-SNAPSHOT + + + webrtc-java-media + + webrtc-java-media + + Media extension for webrtc-java. Reads and decodes media files and network streams with + FFmpeg and feeds them into a peer connection. + + + + + 7.1.1 + + ${user.home}/ffmpeg/${platform.classifier} + Release + + + + + + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + ${project.build.directory}/natives + + + + + + + com.googlecode.cmake-maven-project + cmake-maven-plugin + 3.31.5-b1 + + + cmake-generate + + generate + + + src/main/cpp + ${project.build.directory}/${platform.classifier} + + + + + + + + + + + cmake-compile + + compile + + + ${cmake.config} + ${project.build.directory}/${platform.classifier} + + + + + + + + org.apache.maven.plugins + maven-jar-plugin + + + natives-jar + package + + jar + + + ${platform.classifier} + ${project.build.directory}/natives + + + ${ffmpeg.version} + LGPL-2.1-or-later + + + + + + + + + + + + dev.onvoid.webrtc + webrtc-java + ${project.version} + + + + + ${project.groupId} + ${project.artifactId} + ${project.version} + ${platform.classifier} + + + + + + + jni-check + + + + org.apache.maven.plugins + maven-surefire-plugin + + + + -Xcheck:jni + + + + + + + + windows-x86_64 + + + windows + amd64 + + + + -Ax64 + ${cmake.build.type} + + + + windows-aarch64 + + -AARM64 + ${cmake.build.type} + + + + + diff --git a/webrtc-java-media/src/main/cpp/CMakeLists.txt b/webrtc-java-media/src/main/cpp/CMakeLists.txt new file mode 100644 index 00000000..c9a8c294 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/CMakeLists.txt @@ -0,0 +1,69 @@ +cmake_minimum_required(VERSION 3.13) +project(webrtc-java-media CXX) + +set(CMAKE_CXX_STANDARD 17) +set(CMAKE_CXX_STANDARD_REQUIRED ON) + +if(UNIX AND NOT APPLE) + set(LINUX TRUE) + set(CMAKE_POSITION_INDEPENDENT_CODE ON) +endif() + +if(APPLE) + set(CMAKE_CXX_VISIBILITY_PRESET hidden) +elseif(WIN32) + # Match the runtime the rest of the project's native code uses. + set(CMAKE_CXX_FLAGS_RELEASE "/MT") + set(CMAKE_CXX_FLAGS_DEBUG "/MTd") +endif() + +find_package(JNI REQUIRED) + +add_subdirectory(dependencies/ffmpeg) + +file(GLOB SOURCES "src/*.cpp" "src/media/*.cpp") + +add_library(${PROJECT_NAME} SHARED ${SOURCES}) + +target_include_directories(${PROJECT_NAME} + PRIVATE + include + # webrtc_java_api.h: the C interface of webrtc-java's native library. + # Only the header is used. There is no link dependency between the two + # libraries; the function table is passed in at runtime. + ${CMAKE_CURRENT_SOURCE_DIR}/../../../../webrtc-jni/src/main/cpp/include + ${JNI_INCLUDE_DIRS} +) + +target_link_libraries(${PROJECT_NAME} PRIVATE ffmpeg) + +if(LINUX) + target_link_libraries(${PROJECT_NAME} PRIVATE dl pthread) +endif() + +# The Java side loads the library as +# webrtc-java-media--, the same scheme NativeLoader uses for the +# core library. +if(DEFINED OUTPUT_NAME_SUFFIX AND NOT OUTPUT_NAME_SUFFIX STREQUAL "") + set_target_properties(${PROJECT_NAME} PROPERTIES + OUTPUT_NAME "${PROJECT_NAME}-${OUTPUT_NAME_SUFFIX}") +endif() + +# +# Everything that has to end up in the platform jar goes into one directory, +# which the jar plugin picks up as the classes directory of the natives jar. +# +set(NATIVES_DIR "${CMAKE_CURRENT_BINARY_DIR}/../natives") + +add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E make_directory "${NATIVES_DIR}" + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "$" "${NATIVES_DIR}" + COMMENT "Collecting the native library for the platform jar") + +foreach(RUNTIME_LIB ${FFMPEG_RUNTIME_LIBS}) + add_custom_command(TARGET ${PROJECT_NAME} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy_if_different + "${RUNTIME_LIB}" "${NATIVES_DIR}" + COMMENT "Collecting ${RUNTIME_LIB} for the platform jar") +endforeach() diff --git a/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt new file mode 100644 index 00000000..30fe26a8 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/dependencies/ffmpeg/CMakeLists.txt @@ -0,0 +1,379 @@ +cmake_minimum_required(VERSION 3.13) +project(ffmpeg) + +# +# Builds the FFmpeg libraries this module links against, from the +# third-party/ffmpeg submodule, and exposes them as the imported target +# "ffmpeg". +# +# The build runs at CMake configure time and installs into FFMPEG_INSTALL_DIR, +# the way the WebRTC dependency of webrtc-jni does. A configure that finds the +# libraries already installed there skips the whole thing, so only the first +# build of a platform pays for it. +# +# The configuration is deliberately LGPL only: no --enable-gpl and no +# --enable-nonfree, and the libraries are shared so that they can be replaced, +# which is what the LGPL asks of us. Only the demuxers, decoders, parsers and +# protocols this module actually plays are enabled; everything else is off, to +# keep the shipped libraries small. +# + +set(FFMPEG_SOURCE_DIR "${CMAKE_CURRENT_SOURCE_DIR}/../../../../../third-party/ffmpeg") +get_filename_component(FFMPEG_SOURCE_DIR "${FFMPEG_SOURCE_DIR}" ABSOLUTE) + +if(NOT DEFINED FFMPEG_INSTALL_DIR OR FFMPEG_INSTALL_DIR STREQUAL "") + message(FATAL_ERROR "FFMPEG_INSTALL_DIR is not set") +endif() + +file(MAKE_DIRECTORY "${FFMPEG_INSTALL_DIR}") + +# The libraries this module needs, in dependency order. The order matters +# twice: for the link line, and for the order the Java side loads them in. +set(FFMPEG_LIBS avutil swresample swscale avcodec avformat) + +# +# Decides whether FFmpeg has to be built, by looking for one of its headers +# and one of its libraries in the install directory. +# +function(ffmpeg_is_installed RESULT) + if(NOT EXISTS "${FFMPEG_INSTALL_DIR}/include/libavformat/avformat.h") + set(${RESULT} FALSE PARENT_SCOPE) + return() + endif() + + file(GLOB FOUND_LIBS + "${FFMPEG_INSTALL_DIR}/lib/avformat*" + "${FFMPEG_INSTALL_DIR}/lib/libavformat*" + "${FFMPEG_INSTALL_DIR}/bin/avformat*") + + if(FOUND_LIBS) + set(${RESULT} TRUE PARENT_SCOPE) + else() + set(${RESULT} FALSE PARENT_SCOPE) + endif() +endfunction() + +ffmpeg_is_installed(FFMPEG_INSTALLED) + +if(FFMPEG_INSTALLED) + message(STATUS "FFmpeg ${FFMPEG_VERSION} found in ${FFMPEG_INSTALL_DIR}, skipping build") +else() + if(NOT EXISTS "${FFMPEG_SOURCE_DIR}/configure") + message(FATAL_ERROR + "The FFmpeg submodule is not checked out. Run:\n" + " git submodule update --init --depth 1 webrtc-java-media/third-party/ffmpeg") + endif() + + # FFmpeg's configure and its makefiles are shell scripts, so a POSIX shell + # is needed even to build with MSVC. On Windows that shell has to be + # MSYS2's: Git for Windows' bash carries neither make nor nasm, and it is + # the one a bare find_program(bash) turns up, so look for MSYS2 by path + # first and only then fall back to the PATH. + if(WIN32) + find_program(BASH_EXECUTABLE + NAMES bash + PATHS "$ENV{MSYS2_ROOT}/usr/bin" + "C:/msys64/usr/bin" + "C:/msys2/usr/bin" + "C:/tools/msys64/usr/bin" + NO_DEFAULT_PATH) + + if(NOT BASH_EXECUTABLE) + message(FATAL_ERROR + "No MSYS2 bash found, which building FFmpeg on Windows needs. Install MSYS2 " + "(winget install MSYS2.MSYS2), then inside it run:\n" + " pacman -S --needed make nasm diffutils pkgconf\n" + "Set MSYS2_ROOT if it is not installed in C:/msys64.") + endif() + + # make and nasm live inside MSYS2 and are only visible from its own + # shell, so check for them there rather than with find_program. + get_filename_component(MSYS2_BIN_DIR "${BASH_EXECUTABLE}" DIRECTORY) + + foreach(REQUIRED_TOOL make nasm) + if(NOT EXISTS "${MSYS2_BIN_DIR}/${REQUIRED_TOOL}.exe") + message(FATAL_ERROR + "MSYS2 has no '${REQUIRED_TOOL}', which building FFmpeg needs. Run inside " + "MSYS2:\n pacman -S --needed make nasm diffutils pkgconf") + endif() + endforeach() + + # configure probes the compiler, so the shell it runs in has to have cl + # and link on its PATH. vcvars is what puts them there, and vswhere + # ships with every Visual Studio since 2017. + find_program(VSWHERE_EXECUTABLE + NAMES vswhere + HINTS "$ENV{ProgramFiles\(x86\)}/Microsoft Visual Studio/Installer" + "C:/Program Files (x86)/Microsoft Visual Studio/Installer") + + if(NOT VSWHERE_EXECUTABLE) + message(FATAL_ERROR + "No 'vswhere' found, so Visual Studio could not be located. Building FFmpeg " + "on Windows needs a Visual Studio with the C++ workload.") + endif() + + execute_process( + COMMAND "${VSWHERE_EXECUTABLE}" -latest -products * -property installationPath + OUTPUT_VARIABLE VS_INSTALL_DIR + OUTPUT_STRIP_TRAILING_WHITESPACE + ) + + if(NOT VS_INSTALL_DIR) + message(FATAL_ERROR "vswhere reported no Visual Studio installation.") + endif() + + file(TO_CMAKE_PATH "${VS_INSTALL_DIR}" VS_INSTALL_DIR) + + # The cross compiler is hosted on x64 in every case this builds for. + set(FFMPEG_MSVC_HOST_ARCH x64) + + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$") + set(FFMPEG_MSVC_TARGET_ARCH arm64) + set(FFMPEG_VCVARS "${VS_INSTALL_DIR}/VC/Auxiliary/Build/vcvarsamd64_arm64.bat") + else() + set(FFMPEG_MSVC_TARGET_ARCH x64) + set(FFMPEG_VCVARS "${VS_INSTALL_DIR}/VC/Auxiliary/Build/vcvars64.bat") + endif() + + if(NOT EXISTS "${FFMPEG_VCVARS}") + message(FATAL_ERROR + "No '${FFMPEG_VCVARS}'. Visual Studio is installed without the C++ tools for " + "this target architecture.") + endif() + + file(TO_NATIVE_PATH "${FFMPEG_VCVARS}" FFMPEG_VCVARS) + else() + find_program(BASH_EXECUTABLE NAMES bash) + + if(NOT BASH_EXECUTABLE) + message(FATAL_ERROR "No 'bash' found, which FFmpeg's configure needs.") + endif() + + find_program(MAKE_EXECUTABLE NAMES make gmake) + + if(NOT MAKE_EXECUTABLE) + message(FATAL_ERROR "No 'make' found, which building FFmpeg needs.") + endif() + endif() + + # configure and make are run by a POSIX shell, which on Windows means MSYS2 + # and its /c/... form of a path rather than C:/... + macro(to_shell_path PATH_IN RESULT) + set(${RESULT} "${PATH_IN}") + + if(WIN32) + string(REGEX REPLACE "^([A-Za-z]):/" "/\\1/" ${RESULT} "${${RESULT}}") + endif() + endmacro() + + to_shell_path("${FFMPEG_INSTALL_DIR}" FFMPEG_INSTALL_DIR_SH) + to_shell_path("${FFMPEG_SOURCE_DIR}" FFMPEG_SOURCE_DIR_SH) + + set(FFMPEG_CONFIGURE_ARGS + --prefix=${FFMPEG_INSTALL_DIR_SH} + --disable-static + --enable-shared + --enable-pic + --disable-debug + --disable-doc + --disable-programs + --disable-avdevice + --disable-avfilter + --disable-postproc + --disable-network + --disable-everything + --disable-autodetect + # Containers this module reads. + --enable-demuxer=mov + --enable-demuxer=matroska + --enable-demuxer=avi + --enable-demuxer=mpegts + --enable-demuxer=flv + --enable-demuxer=wav + --enable-demuxer=mp3 + --enable-demuxer=ogg + --enable-demuxer=flac + --enable-demuxer=aac + # Video this module decodes. + --enable-decoder=h264 + --enable-decoder=hevc + --enable-decoder=vp8 + --enable-decoder=vp9 + --enable-decoder=mpeg4 + --enable-decoder=mjpeg + # Audio this module decodes. + --enable-decoder=aac + --enable-decoder=mp3 + --enable-decoder=opus + --enable-decoder=vorbis + --enable-decoder=flac + --enable-decoder=pcm_s16le + --enable-decoder=pcm_s16be + --enable-decoder=pcm_u8 + --enable-decoder=pcm_f32le + # Parsers for the decoders above. + --enable-parser=h264 + --enable-parser=hevc + --enable-parser=vp8 + --enable-parser=vp9 + --enable-parser=mpeg4video + --enable-parser=mjpeg + --enable-parser=aac + --enable-parser=mpegaudio + --enable-parser=opus + --enable-parser=vorbis + --enable-parser=flac + # Local files only for now; network protocols follow once the TLS + # backends are wired up per platform. + --enable-protocol=file + --enable-protocol=pipe + ) + + if(WIN32) + # FFmpeg builds against the MSVC runtime through its own msvc + # toolchain support, which keeps the DLLs free of an MSYS2 runtime + # dependency. configure needs Windows-style output, so cl and link + # must be on the PATH of the shell that runs it. + list(APPEND FFMPEG_CONFIGURE_ARGS --toolchain=msvc) + endif() + + if(CMAKE_SYSTEM_PROCESSOR MATCHES "^(aarch64|arm64|ARM64)$" AND WIN32) + # No assembler for this target yet; correctness first, speed later. + list(APPEND FFMPEG_CONFIGURE_ARGS --disable-asm --arch=arm64) + endif() + + if(APPLE AND CMAKE_OSX_ARCHITECTURES) + list(APPEND FFMPEG_CONFIGURE_ARGS + --extra-cflags=-arch\ ${CMAKE_OSX_ARCHITECTURES} + --extra-ldflags=-arch\ ${CMAKE_OSX_ARCHITECTURES}) + endif() + + # Build out of tree so the submodule working tree stays clean. + set(FFMPEG_BUILD_DIR "${CMAKE_CURRENT_BINARY_DIR}/ffmpeg-build") + file(MAKE_DIRECTORY "${FFMPEG_BUILD_DIR}") + to_shell_path("${FFMPEG_BUILD_DIR}" FFMPEG_BUILD_DIR_SH) + + string(REPLACE ";" " " FFMPEG_CONFIGURE_ARGS_STR "${FFMPEG_CONFIGURE_ARGS}") + + include(ProcessorCount) + ProcessorCount(BUILD_JOBS) + + if(BUILD_JOBS EQUAL 0) + set(BUILD_JOBS 1) + endif() + + # + # configure, make and make install go into one shell script rather than + # three execute_process calls, because on Windows all three have to run in + # the same MSYS2 shell, and that shell has to be entered through vcvars so + # that FFmpeg's configure can find cl and link. + # + set(FFMPEG_SHELL_PREAMBLE "") + + if(WIN32) + # MSYS2's own make, reached from inside its shell. + set(FFMPEG_MAKE make) + + # MSYS2 ships a coreutils "link" in /usr/bin that otherwise shadows + # MSVC's linker, and FFmpeg's configure then reports that the C + # compiler cannot create executables. Putting the MSVC toolchain + # first is what keeps that from happening. + set(FFMPEG_SHELL_PREAMBLE +"if [ -n \"$VCToolsInstallDir\" ]; then + export PATH=\"$(cygpath -u \"$VCToolsInstallDir\")/bin/Host${FFMPEG_MSVC_HOST_ARCH}/${FFMPEG_MSVC_TARGET_ARCH}:$PATH\" +fi +") + else() + set(FFMPEG_MAKE "${MAKE_EXECUTABLE}") + endif() + + set(FFMPEG_BUILD_SCRIPT "${CMAKE_CURRENT_BINARY_DIR}/build-ffmpeg.sh") + + file(WRITE "${FFMPEG_BUILD_SCRIPT}" +"# Generated by CMake. Builds FFmpeg ${FFMPEG_VERSION} for webrtc-java-media. +set -e +${FFMPEG_SHELL_PREAMBLE}cd \"${FFMPEG_BUILD_DIR_SH}\" +\"${FFMPEG_SOURCE_DIR_SH}/configure\" ${FFMPEG_CONFIGURE_ARGS_STR} +\"${FFMPEG_MAKE}\" -j${BUILD_JOBS} +\"${FFMPEG_MAKE}\" install +") + + if(WIN32) + to_shell_path("${FFMPEG_BUILD_SCRIPT}" FFMPEG_BUILD_SCRIPT_SH) + + # vcvars is a batch file, so entering it needs cmd, and the shell it + # hands over to has to inherit the PATH it set up. + set(FFMPEG_LAUNCHER "${CMAKE_CURRENT_BINARY_DIR}/build-ffmpeg.cmd") + + file(WRITE "${FFMPEG_LAUNCHER}" +"@echo off +call \"${FFMPEG_VCVARS}\" >nul +if errorlevel 1 exit /b 1 +set MSYS2_PATH_TYPE=inherit +\"${BASH_EXECUTABLE}\" -l \"${FFMPEG_BUILD_SCRIPT_SH}\" +") + + file(TO_NATIVE_PATH "${FFMPEG_LAUNCHER}" FFMPEG_LAUNCHER_NATIVE) + set(FFMPEG_BUILD_COMMAND cmd /c "${FFMPEG_LAUNCHER_NATIVE}") + else() + set(FFMPEG_BUILD_COMMAND "${BASH_EXECUTABLE}" "${FFMPEG_BUILD_SCRIPT}") + endif() + + message(STATUS "Building FFmpeg ${FFMPEG_VERSION} into ${FFMPEG_INSTALL_DIR} with ${BUILD_JOBS} jobs") + message(STATUS " configure ${FFMPEG_CONFIGURE_ARGS_STR}") + + execute_process( + COMMAND ${FFMPEG_BUILD_COMMAND} + WORKING_DIRECTORY "${FFMPEG_BUILD_DIR}" + RESULT_VARIABLE BUILD_RESULT + ) + + if(NOT BUILD_RESULT EQUAL 0) + message(FATAL_ERROR + "Building FFmpeg failed (${BUILD_RESULT}). The script that ran is " + "${FFMPEG_BUILD_SCRIPT}; see ${FFMPEG_BUILD_DIR}/ffbuild/config.log for what " + "configure could not find.") + endif() + + ffmpeg_is_installed(FFMPEG_INSTALLED) + + if(NOT FFMPEG_INSTALLED) + message(FATAL_ERROR + "FFmpeg reported success but nothing was installed in ${FFMPEG_INSTALL_DIR}") + endif() +endif() + +# +# Expose the installed libraries as one interface target. The import libraries +# are what the link line needs; the runtime libraries are collected separately +# because they have to travel in the platform jar. +# +add_library(ffmpeg INTERFACE) + +target_include_directories(ffmpeg INTERFACE "${FFMPEG_INSTALL_DIR}/include") + +foreach(LIB ${FFMPEG_LIBS}) + # An MSVC build of FFmpeg puts the import libraries next to the DLLs in + # bin/ and leaves only the .def files in lib/, so both have to be searched. + find_library(FFMPEG_${LIB}_LIB + NAMES ${LIB} lib${LIB} + PATHS "${FFMPEG_INSTALL_DIR}/lib" "${FFMPEG_INSTALL_DIR}/bin" + NO_DEFAULT_PATH) + + if(NOT FFMPEG_${LIB}_LIB) + message(FATAL_ERROR + "Could not find lib${LIB} in ${FFMPEG_INSTALL_DIR}/lib or ${FFMPEG_INSTALL_DIR}/bin") + endif() + + target_link_libraries(ffmpeg INTERFACE "${FFMPEG_${LIB}_LIB}") +endforeach() + +# The shared libraries to ship. On Windows the DLLs land in bin/, elsewhere in +# lib/ next to the import libraries. +file(GLOB FFMPEG_RUNTIME_LIBS + "${FFMPEG_INSTALL_DIR}/bin/*.dll" + "${FFMPEG_INSTALL_DIR}/lib/*.so.*" + "${FFMPEG_INSTALL_DIR}/lib/*.dylib") + +set(FFMPEG_RUNTIME_LIBS "${FFMPEG_RUNTIME_LIBS}" CACHE INTERNAL + "FFmpeg shared libraries that ship in the platform jar") diff --git a/webrtc-java-media/src/main/cpp/include/JNI_FFmpeg.h b/webrtc-java-media/src/main/cpp/include/JNI_FFmpeg.h new file mode 100644 index 00000000..84234f0a --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/JNI_FFmpeg.h @@ -0,0 +1,44 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_ffmpeg_FFmpeg */ + +#ifndef _Included_dev_onvoid_webrtc_media_ffmpeg_FFmpeg +#define _Included_dev_onvoid_webrtc_media_ffmpeg_FFmpeg +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_FFmpeg + * Method: version + * Signature: ()Ljava/lang/String; + */ + JNIEXPORT jstring JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_FFmpeg_version + (JNIEnv *, jclass); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_FFmpeg + * Method: license + * Signature: ()Ljava/lang/String; + */ + JNIEXPORT jstring JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_FFmpeg_license + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-java-media/src/main/cpp/include/JNI_MediaPlayer.h b/webrtc-java-media/src/main/cpp/include/JNI_MediaPlayer.h new file mode 100644 index 00000000..fd080bd9 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/JNI_MediaPlayer.h @@ -0,0 +1,92 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_ffmpeg_MediaPlayer */ + +#ifndef _Included_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer +#define _Included_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: create + * Signature: (JJJJ)J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_create + (JNIEnv *, jobject, jlong, jlong, jlong, jlong); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: start + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_start + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: suspend + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_suspend + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: seek + * Signature: (JJ)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_seek + (JNIEnv *, jclass, jlong, jlong); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: setLooping + * Signature: (JZ)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_setLooping + (JNIEnv *, jclass, jlong, jboolean); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: position + * Signature: (J)J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_position + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: state + * Signature: (J)I + */ + JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_state + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaPlayer + * Method: dispose + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_dispose + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-java-media/src/main/cpp/include/JNI_MediaReader.h b/webrtc-java-media/src/main/cpp/include/JNI_MediaReader.h new file mode 100644 index 00000000..77d8badc --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/JNI_MediaReader.h @@ -0,0 +1,52 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_media_ffmpeg_MediaReader */ + +#ifndef _Included_dev_onvoid_webrtc_media_ffmpeg_MediaReader +#define _Included_dev_onvoid_webrtc_media_ffmpeg_MediaReader +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaReader + * Method: open + * Signature: (Ljava/lang/String;)J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaReader_open + (JNIEnv *, jclass, jstring); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaReader + * Method: info + * Signature: (J)Ldev/onvoid/webrtc/media/ffmpeg/MediaInfo; + */ + JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaReader_info + (JNIEnv *, jclass, jlong); + + /* + * Class: dev_onvoid_webrtc_media_ffmpeg_MediaReader + * Method: dispose + * Signature: (J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaReader_dispose + (JNIEnv *, jclass, jlong); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h b/webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h new file mode 100644 index 00000000..14a80c91 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/AudioDecoder.h @@ -0,0 +1,106 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_AUDIO_DECODER_H_ +#define WEBRTC_JAVA_MEDIA_AUDIO_DECODER_H_ + +#include +#include + +extern "C" { +#include +#include +#include +} + +namespace ffmpeg +{ + // Decodes one audio stream into the only shape WebRTC accepts: interleaved + // signed 16-bit PCM, at most 48 kHz, mono or stereo, in chunks of exactly + // 10 ms. + // + // A source rarely arrives in that shape, so decoded audio goes through + // swresample and is then re-cut into 10 ms chunks, which is why this holds + // a buffer between calls: one decoded frame is seldom a whole number of + // chunks. + class AudioDecoder + { + public: + // The rate WebRTC works at internally, so resampling to it here + // saves it from doing the same work again later. + static constexpr int kSampleRate = 48000; + + // WebRTC takes 10 ms of audio at a time and nothing else. + static constexpr int kFramesPerChunk = kSampleRate / 100; + + AudioDecoder() = default; + ~AudioDecoder(); + + AudioDecoder(const AudioDecoder &) = delete; + AudioDecoder & operator=(const AudioDecoder &) = delete; + + // Opens a decoder for the given stream, which must outlive this + // decoder. Source audio of more than two channels is downmixed to + // stereo. Returns 0 or a negative AVERROR. + int Open(const AVStream * stream); + + void Close(); + + // Drops the decoder's state and the partial chunk being gathered, + // which is what a seek needs. + void Flush(); + + // Hands a packet to the decoder, or null to start draining at the + // end of the stream. Returns 0 or a negative AVERROR. + int SendPacket(const AVPacket * packet); + + // Takes the next 10 ms of audio, resampled and interleaved. + // + // Returns 0 when a chunk was produced, AVERROR(EAGAIN) when the + // decoder needs another packet first, AVERROR_EOF once draining + // has finished, or another negative AVERROR. The tail of a stream + // that does not fill a whole chunk is padded with silence, since a + // short chunk is not something WebRTC can be given. + int ReceiveChunk(std::vector & chunk, int64_t * timestamp_us); + + int GetChannels() const; + + private: + // Resamples one decoded frame into the pending buffer. + int Resample(const AVFrame * frame); + + // Moves one chunk out of the pending buffer. + void TakeChunk(std::vector & chunk, int64_t * timestamp_us); + + AVCodecContext * codec_context_ = nullptr; + SwrContext * swr_context_ = nullptr; + AVFrame * decoded_ = nullptr; + AVRational time_base_ = { 0, 1 }; + int channels_ = 0; + + // Resampled samples not yet handed out, interleaved. + std::vector pending_; + + // When the next chunk starts, in microseconds on the source's own + // timeline. Chunks advance it in exact 10 ms steps, so the audio + // this produces is perfectly regular whatever the container's + // packet timing looks like. + int64_t next_timestamp_us_ = 0; + bool have_timestamp_ = false; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/JavaPlayerObserver.h b/webrtc-java-media/src/main/cpp/include/media/JavaPlayerObserver.h new file mode 100644 index 00000000..187337b8 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/JavaPlayerObserver.h @@ -0,0 +1,67 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_JAVA_PLAYER_OBSERVER_H_ +#define WEBRTC_JAVA_MEDIA_JAVA_PLAYER_OBSERVER_H_ + +#include "media/MediaPlayerObserver.h" + +#include + +namespace ffmpeg +{ + // Passes a player's reports on to its Java MediaPlayer, which turns them + // into calls on whatever listener the application set. + // + // The calls arrive on the player's own thread, which the JVM knows nothing + // about, so each one attaches that thread, makes the call and detaches + // again. Attaching is not free, but these are state changes and errors, + // not frames: they happen a handful of times over a whole playback. + class JavaPlayerObserver : public MediaPlayerObserver + { + public: + // Keeps a global reference to the given Java MediaPlayer, so the + // player may outlive the call that created it. + JavaPlayerObserver(JNIEnv * env, jobject player); + ~JavaPlayerObserver() override; + + JavaPlayerObserver(const JavaPlayerObserver &) = delete; + JavaPlayerObserver & operator=(const JavaPlayerObserver &) = delete; + + void OnStateChanged(int state) override; + void OnEndOfStream() override; + void OnError(const std::string & message) override; + + private: + // Returns an environment for the calling thread, attaching it if + // it is not known to the JVM. Sets attached when it did, which is + // then the caller's to undo. + JNIEnv * Attach(bool * attached); + void Detach(bool attached); + + // A listener that throws must not be left to surface somewhere + // unrelated later, so anything pending is reported and cleared. + void ClearPendingException(JNIEnv * env); + + JavaVM * vm_ = nullptr; + jobject player_ = nullptr; + jmethodID on_state_changed_ = nullptr; + jmethodID on_end_of_stream_ = nullptr; + jmethodID on_error_ = nullptr; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/MediaPacer.h b/webrtc-java-media/src/main/cpp/include/media/MediaPacer.h new file mode 100644 index 00000000..37773ee2 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/MediaPacer.h @@ -0,0 +1,162 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_MEDIA_PACER_H_ +#define WEBRTC_JAVA_MEDIA_MEDIA_PACER_H_ + +#include "webrtc_java_api.h" + +#include +#include +#include +#include +#include +#include + +extern "C" { +#include +} + +namespace ffmpeg +{ + // Delivers decoded media to webrtc-java's custom sources in real time. + // + // WebRTC encodes and sends a frame the moment it arrives, so playback has + // to be paced rather than pushed as fast as it decodes. One thread does + // that: it sleeps until an item is due and then hands it over, which is + // the only thread that ever touches the extension API. + // + // Timing works off one mapping, made when the first item is delivered and + // kept from then on: a source's own presentation time plus a base offset + // is a time on the WebRTC clock. Every item carries that time with it, so + // the timeline that reaches the receiver is the source's own and not the + // pacing thread's, and audio stays lined up with video however much the + // thread is jostled. + // + // Video and audio are paced independently. They go to different sources + // and need no ordering between them, and keeping them independent is also + // what keeps a full video queue from starving audio. + class MediaPacer + { + public: + // The queues are what decouples decoding from playback. A second + // of audio and about a second of video is enough to ride out a + // slow decode without holding much memory. + static constexpr size_t kVideoCapacity = 60; + static constexpr size_t kAudioCapacity = 100; + + // The sources may be 0, in which case media of that kind is + // dropped rather than delivered. + MediaPacer(const webrtc_java_api * api, void * video_source, + void * audio_source); + ~MediaPacer(); + + MediaPacer(const MediaPacer &) = delete; + MediaPacer & operator=(const MediaPacer &) = delete; + + void Start(); + + // Stops the thread and drops whatever is still queued. Safe to + // call more than once. + void Stop(); + + // Hands over a decoded frame and its presentation time. Ownership + // of the frame passes to the pacer whether this succeeds or not. + // + // Blocks while the queue is full, which is the backpressure that + // keeps decoding from running ahead of playback. Returns false + // once the pacer has been stopped. + bool PushVideo(AVFrame * frame, int64_t timestamp_us); + + // Hands over one 10 ms chunk of interleaved 16-bit PCM. + bool PushAudio(std::vector && samples, int channels, + int64_t timestamp_us); + + void Pause(); + void Resume(); + + // Drops everything queued and forgets the clock mapping, so that + // the next item delivered starts a new one. This is what a seek + // needs: what is queued belongs to the position being left. + void Flush(); + + // True once everything handed over has been delivered. + bool IsDrained() const; + + // The presentation time of the item delivered last, which is where + // playback has got to. + int64_t GetPositionUs() const; + + private: + struct VideoItem + { + AVFrame * frame = nullptr; + int64_t timestamp_us = 0; + + VideoItem() = default; + VideoItem(AVFrame * f, int64_t ts) : frame(f), timestamp_us(ts) {} + ~VideoItem(); + + VideoItem(VideoItem && other) noexcept; + VideoItem & operator=(VideoItem && other) noexcept; + + VideoItem(const VideoItem &) = delete; + VideoItem & operator=(const VideoItem &) = delete; + }; + + struct AudioItem + { + std::vector samples; + int channels = 0; + int64_t timestamp_us = 0; + }; + + void Run(); + + // Both run without the lock held: the extension API delivers + // synchronously into WebRTC, which must not happen underneath a + // lock of ours. The time on the WebRTC clock is worked out by the + // caller while it still holds the lock and passed in, so that + // neither of these reads the mapping unguarded. + void DeliverVideo(VideoItem item, int64_t mapped_us); + void DeliverAudio(const AudioItem & item, int64_t mapped_us); + + const webrtc_java_api * api_; + void * video_source_; + void * audio_source_; + + std::thread thread_; + + mutable std::mutex mutex_; + std::condition_variable work_; + std::condition_variable video_space_; + std::condition_variable audio_space_; + + std::deque video_queue_; + std::deque audio_queue_; + + bool running_ = false; + bool paused_ = false; + + // Presentation time plus this is a time on the WebRTC clock. + int64_t base_us_ = 0; + bool have_base_ = false; + int64_t paused_at_us_ = 0; + int64_t position_us_ = 0; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/MediaPlayer.h b/webrtc-java-media/src/main/cpp/include/media/MediaPlayer.h new file mode 100644 index 00000000..43c8a742 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/MediaPlayer.h @@ -0,0 +1,161 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_MEDIA_PLAYER_H_ +#define WEBRTC_JAVA_MEDIA_MEDIA_PLAYER_H_ + +#include "media/AudioDecoder.h" +#include "media/MediaPacer.h" +#include "media/MediaPlayerObserver.h" +#include "media/MediaReader.h" +#include "media/VideoDecoder.h" +#include "webrtc_java_api.h" + +#include +#include +#include +#include +#include +#include +#include + +namespace ffmpeg +{ + // Kept in step with the Java MediaPlayerState enum, which is what these + // values are reported to the observer as. + enum MediaPlayerState + { + kIdle = 0, + kPlaying = 1, + kPaused = 2, + kEnded = 3, + kClosed = 4 + }; + + // Plays a media source into webrtc-java's custom media sources. + // + // One thread reads packets, decodes them and hands the result to a + // MediaPacer, which delivers it in real time. That thread is the only one + // that touches the reader or the decoders, so the commands below do not + // act directly: they leave a request behind and wake it. + // + // Decoding runs ahead of playback only as far as the pacer's queues allow. + // Once those are full the decode thread blocks, which is what keeps a + // whole file from being decoded into memory at once. + class MediaPlayer + { + public: + // The sources may be 0, in which case media of that kind is + // decoded and dropped. The observer may be null. + MediaPlayer(std::unique_ptr reader, + const webrtc_java_api * api, void * video_source, + void * audio_source); + ~MediaPlayer(); + + MediaPlayer(const MediaPlayer &) = delete; + MediaPlayer & operator=(const MediaPlayer &) = delete; + + // Takes over the observer, which then lives as long as the + // player does. Call this before Initialize: the observer is read + // without the lock while it is being called, on the assumption + // that it does not change once the thread is running. + void SetObserver(std::unique_ptr observer); + + // Opens the decoders for whichever streams the source has. + // Returns 0 or a negative AVERROR. + int Initialize(); + + void Play(); + void Pause(); + + // Moves playback to the given position, in microseconds from the + // start. What is already queued is dropped. + void Seek(int64_t position_us); + + void SetLooping(bool looping); + bool IsLooping() const; + + // Stops playback and releases the thread. The player cannot be + // used afterwards. + void Close(); + + // Where playback has got to within the current pass, in + // microseconds. A looping player starts again from zero. + int64_t GetPositionUs() const; + + int GetState() const; + + private: + void Run(); + + // Reads and decodes until the source is exhausted. Returns false + // if playback should stop, either on error or on close. + bool PumpOnce(AVPacket * packet, bool * end_of_stream); + + // Pushes everything the decoders still hold, which is what the end + // of a stream and a loop both need. + void DrainDecoders(); + + int DecodeVideo(); + int DecodeAudio(); + + void PerformSeek(int64_t position_us); + void SetState(int state); + void ReportError(const std::string & message, int error); + + std::unique_ptr reader_; + std::unique_ptr pacer_; + VideoDecoder video_decoder_; + AudioDecoder audio_decoder_; + + const webrtc_java_api * api_; + bool has_video_ = false; + bool has_audio_ = false; + + std::thread thread_; + + mutable std::mutex mutex_; + std::condition_variable command_; + + std::unique_ptr observer_; + bool running_ = false; + bool playing_ = false; + bool closing_ = false; + int state_ = kIdle; + + bool seek_pending_ = false; + int64_t seek_position_us_ = 0; + + // Set once the source has run out and everything queued has been + // delivered, so that the thread stops pumping until something + // asks it to start again. + bool ended_ = false; + + // The largest presentation time seen, used to advance the loop + // offset when the container does not say how long it runs. + int64_t max_source_pts_us_ = 0; + + std::atomic looping_{ false }; + + // Added to every presentation time handed to the pacer, so that a + // source played again keeps producing times that increase. The + // pacer needs that: WebRTC drops a frame whose capture time does + // not advance. + std::atomic loop_offset_us_{ 0 }; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/MediaPlayerObserver.h b/webrtc-java-media/src/main/cpp/include/media/MediaPlayerObserver.h new file mode 100644 index 00000000..5c9c2984 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/MediaPlayerObserver.h @@ -0,0 +1,47 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_MEDIA_PLAYER_OBSERVER_H_ +#define WEBRTC_JAVA_MEDIA_MEDIA_PLAYER_OBSERVER_H_ + +#include + +namespace ffmpeg +{ + // What a player reports while it runs. + // + // Every call arrives on the player's own thread, never on the thread that + // asked for playback, so an implementation must not block: the thread it + // holds up is the one decoding the media. + class MediaPlayerObserver + { + public: + virtual ~MediaPlayerObserver() = default; + + // The player moved to another state, as a MediaPlayerState value. + virtual void OnStateChanged(int state) = 0; + + // The source ran out. A looping player never reports this, since + // it starts again instead. + virtual void OnEndOfStream() = 0; + + // Playback stopped because something went wrong, with the message + // FFmpeg gave for it. + virtual void OnError(const std::string & message) = 0; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/MediaReader.h b/webrtc-java-media/src/main/cpp/include/media/MediaReader.h new file mode 100644 index 00000000..bf38a4c7 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/MediaReader.h @@ -0,0 +1,101 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_MEDIA_READER_H_ +#define WEBRTC_JAVA_MEDIA_MEDIA_READER_H_ + +#include + +extern "C" { +#include +} + +namespace ffmpeg +{ + // Opens a media source with libavformat and reports what it contains. + // + // A source is anything libavformat accepts: a file path today, and an + // http, rtsp or rtmp URL once those protocols are enabled in the build. + // Opening reads the container and picks the streams that will be played. + // Decoding is built on top of this and does not belong here. + class MediaReader + { + public: + MediaReader() = default; + ~MediaReader(); + + MediaReader(const MediaReader &) = delete; + MediaReader & operator=(const MediaReader &) = delete; + + // Opens the given source and selects the video and audio stream + // that are meant to be played. Returns 0, or the negative AVERROR + // libavformat reported, in which case the reader stays closed. + int Open(const std::string & url); + + // Releases the container. Does nothing on a closed reader, and + // runs from the destructor. + void Close(); + + bool IsOpen() const; + + // How long the source runs in microseconds, or 0 when the + // container does not say, as is the case for a live stream. + int64_t GetDurationUs() const; + + bool HasVideo() const; + int GetVideoWidth() const; + int GetVideoHeight() const; + + // The average frame rate the container reports, or 0 when it does + // not say. A variable frame rate source only has an average. + double GetFrameRate() const; + + // The codec name as FFmpeg spells it, never null: an unknown or + // absent codec reads as "none". It points into libavcodec's own + // static table and outlives this reader. + const char * GetVideoCodecName() const; + + bool HasAudio() const; + int GetSampleRate() const; + int GetChannels() const; + const char * GetAudioCodecName() const; + + // The streams the decoders attach to, or null when the source has + // none of that kind. They belong to the reader and die with it. + const AVStream * GetVideoStream() const; + const AVStream * GetAudioStream() const; + + int GetVideoStreamIndex() const; + int GetAudioStreamIndex() const; + + // Reads the next packet of any stream into the given packet, which + // the caller unrefs. Returns 0, AVERROR_EOF once the source is + // exhausted, or another negative AVERROR. + int ReadPacket(AVPacket * packet); + + // Moves to the keyframe at or before the given position, in + // microseconds from the start. The decoders have to be flushed + // afterwards, since what they hold belongs to the old position. + int Seek(int64_t position_us); + + private: + AVFormatContext * format_context_ = nullptr; + int video_stream_index_ = -1; + int audio_stream_index_ = -1; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h b/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h new file mode 100644 index 00000000..673f96a4 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/include/media/VideoDecoder.h @@ -0,0 +1,82 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_MEDIA_VIDEO_DECODER_H_ +#define WEBRTC_JAVA_MEDIA_VIDEO_DECODER_H_ + +extern "C" { +#include +#include +#include +} + +namespace ffmpeg +{ + // Decodes one video stream into I420, which is the only pixel format + // WebRTC takes. + // + // Almost all 8-bit H.264, VP8, VP9 and MPEG-4 content decodes to yuv420p + // already, and such a frame is handed on untouched, so the decoded picture + // reaches the encoder without a single copy. Anything else is converted + // with swscale first, which costs one copy and is the price of the format + // rather than of this design. + class VideoDecoder + { + public: + VideoDecoder() = default; + ~VideoDecoder(); + + VideoDecoder(const VideoDecoder &) = delete; + VideoDecoder & operator=(const VideoDecoder &) = delete; + + // Opens a decoder for the given stream, which must outlive this + // decoder. Returns 0 or a negative AVERROR. + int Open(const AVStream * stream); + + void Close(); + + // Drops everything the decoder holds, which is what a seek needs: + // the frames in flight belong to the position that was left. + void Flush(); + + // Hands a packet to the decoder, or null to start draining at the + // end of the stream. Returns 0 or a negative AVERROR. + int SendPacket(const AVPacket * packet); + + // Takes the next decoded frame, in I420 and owned by the caller, + // who frees it with av_frame_free. + // + // Returns 0 when a frame was produced, AVERROR(EAGAIN) when the + // decoder needs another packet first, AVERROR_EOF once draining + // has finished, or another negative AVERROR. + int ReceiveFrame(AVFrame ** frame, int64_t * timestamp_us); + + int GetWidth() const; + int GetHeight() const; + + private: + // Converts a decoded frame to I420. The returned frame is a new + // reference the caller owns. + int ConvertToI420(const AVFrame * source, AVFrame ** result); + + AVCodecContext * codec_context_ = nullptr; + SwsContext * sws_context_ = nullptr; + AVFrame * decoded_ = nullptr; + AVRational time_base_ = { 0, 1 }; + }; +} + +#endif diff --git a/webrtc-java-media/src/main/cpp/src/JNI_FFmpeg.cpp b/webrtc-java-media/src/main/cpp/src/JNI_FFmpeg.cpp new file mode 100644 index 00000000..dd8c7285 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/JNI_FFmpeg.cpp @@ -0,0 +1,37 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_FFmpeg.h" + +extern "C" { +#include +#include +} + +JNIEXPORT jstring JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_FFmpeg_version +(JNIEnv * env, jclass caller) +{ + // The version of the libraries that were actually loaded, which is not + // necessarily the one this module was built against: the LGPL lets an + // application replace them. + return env->NewStringUTF(av_version_info()); +} + +JNIEXPORT jstring JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_FFmpeg_license +(JNIEnv * env, jclass caller) +{ + return env->NewStringUTF(avformat_license()); +} diff --git a/webrtc-java-media/src/main/cpp/src/JNI_MediaPlayer.cpp b/webrtc-java-media/src/main/cpp/src/JNI_MediaPlayer.cpp new file mode 100644 index 00000000..13a6e2a6 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/JNI_MediaPlayer.cpp @@ -0,0 +1,161 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_MediaPlayer.h" +#include "media/JavaPlayerObserver.h" +#include "media/MediaPlayer.h" +#include "media/MediaReader.h" +#include "webrtc_java_api.h" + +#include +#include + +extern "C" { +#include +} + +namespace +{ + std::string ErrorMessage(int error) + { + char buffer[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + + if (av_strerror(error, buffer, sizeof(buffer)) < 0) { + return "Unknown FFmpeg error " + std::to_string(error); + } + + return buffer; + } + + void ThrowIOException(JNIEnv * env, const std::string & message) + { + jclass cls = env->FindClass("java/io/IOException"); + + if (cls != nullptr) { + env->ThrowNew(cls, message.c_str()); + env->DeleteLocalRef(cls); + } + } + + ffmpeg::MediaPlayer * PlayerOf(jlong handle) + { + return reinterpret_cast(handle); + } +} + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_create +(JNIEnv * env, jobject caller, jlong readerHandle, jlong tableAddress, + jlong videoSourceHandle, jlong audioSourceHandle) +{ + // The Java side detached the reader before this call, so it is ours now + // and has to be released here if anything goes wrong. + std::unique_ptr reader( + reinterpret_cast(readerHandle)); + + if (reader == nullptr) { + ThrowIOException(env, "The reader is closed"); + + return 0; + } + + const webrtc_java_api * api = + reinterpret_cast(tableAddress); + + if (api == nullptr) { + ThrowIOException(env, "The webrtc-java function table is not available"); + + return 0; + } + if (api->version != WEBRTC_JAVA_API_VERSION) { + ThrowIOException(env, "This module was built against webrtc-java interface version " + + std::to_string(WEBRTC_JAVA_API_VERSION) + ", but the loaded library provides " + + std::to_string(api->version)); + + return 0; + } + + auto player = std::make_unique(std::move(reader), api, + reinterpret_cast(videoSourceHandle), + reinterpret_cast(audioSourceHandle)); + + player->SetObserver(std::make_unique(env, caller)); + + int result = player->Initialize(); + + if (result < 0) { + ThrowIOException(env, "Opening the decoders failed: " + ErrorMessage(result)); + + return 0; + } + + return reinterpret_cast(player.release()); +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_start +(JNIEnv * env, jclass caller, jlong handle) +{ + if (handle != 0) { + PlayerOf(handle)->Play(); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_suspend +(JNIEnv * env, jclass caller, jlong handle) +{ + if (handle != 0) { + PlayerOf(handle)->Pause(); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_seek +(JNIEnv * env, jclass caller, jlong handle, jlong positionUs) +{ + if (handle != 0) { + PlayerOf(handle)->Seek(positionUs); + } +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_setLooping +(JNIEnv * env, jclass caller, jlong handle, jboolean looping) +{ + if (handle != 0) { + PlayerOf(handle)->SetLooping(looping == JNI_TRUE); + } +} + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_position +(JNIEnv * env, jclass caller, jlong handle) +{ + return handle != 0 ? PlayerOf(handle)->GetPositionUs() : 0; +} + +JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_state +(JNIEnv * env, jclass caller, jlong handle) +{ + // A player that is gone is closed, which is what the Java side reports + // once it has let go of the handle. + return handle != 0 ? PlayerOf(handle)->GetState() : ffmpeg::kClosed; +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaPlayer_dispose +(JNIEnv * env, jclass caller, jlong handle) +{ + // Closing twice is allowed, so a handle that is already zero is simply + // nothing to do. + if (handle != 0) { + delete PlayerOf(handle); + } +} diff --git a/webrtc-java-media/src/main/cpp/src/JNI_MediaReader.cpp b/webrtc-java-media/src/main/cpp/src/JNI_MediaReader.cpp new file mode 100644 index 00000000..3e952f7a --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/JNI_MediaReader.cpp @@ -0,0 +1,166 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_MediaReader.h" +#include "media/MediaReader.h" + +#include + +extern "C" { +#include +} + +namespace +{ + // Turns an AVERROR code into the message FFmpeg has for it, so that a + // failure to open reaches Java saying what libavformat actually objected + // to instead of a bare number. + std::string ErrorMessage(int error) + { + char buffer[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + + if (av_strerror(error, buffer, sizeof(buffer)) < 0) { + return "Unknown FFmpeg error " + std::to_string(error); + } + + return buffer; + } + + void ThrowIOException(JNIEnv * env, const std::string & message) + { + jclass cls = env->FindClass("java/io/IOException"); + + if (cls != nullptr) { + env->ThrowNew(cls, message.c_str()); + env->DeleteLocalRef(cls); + } + } + + ffmpeg::MediaReader * ReaderOf(JNIEnv * env, jlong handle) + { + if (handle == 0) { + jclass cls = env->FindClass("java/lang/IllegalStateException"); + + if (cls != nullptr) { + env->ThrowNew(cls, "MediaReader is closed"); + env->DeleteLocalRef(cls); + } + + return nullptr; + } + + return reinterpret_cast(handle); + } +} + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaReader_open +(JNIEnv * env, jclass caller, jstring source) +{ + if (source == nullptr) { + ThrowIOException(env, "Source must not be null"); + + return 0; + } + + const char * chars = env->GetStringUTFChars(source, nullptr); + + if (chars == nullptr) { + // The VM is out of memory and has already thrown. + return 0; + } + + std::string url(chars); + + env->ReleaseStringUTFChars(source, chars); + + auto reader = new ffmpeg::MediaReader(); + int result = reader->Open(url); + + if (result < 0) { + delete reader; + + ThrowIOException(env, "Opening '" + url + "' failed: " + ErrorMessage(result)); + + return 0; + } + + return reinterpret_cast(reader); +} + +JNIEXPORT jobject JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaReader_info +(JNIEnv * env, jclass caller, jlong handle) +{ + ffmpeg::MediaReader * reader = ReaderOf(env, handle); + + if (reader == nullptr) { + return nullptr; + } + + jclass cls = env->FindClass("dev/onvoid/webrtc/media/ffmpeg/MediaInfo"); + + if (cls == nullptr) { + return nullptr; + } + + jmethodID ctor = env->GetMethodID(cls, "", + "(JIIDLjava/lang/String;IILjava/lang/String;)V"); + + if (ctor == nullptr) { + env->DeleteLocalRef(cls); + + return nullptr; + } + + // A stream that is not there is reported as a null codec name and zero + // everything else, which is the contract MediaInfo documents. + jstring videoCodec = reader->HasVideo() + ? env->NewStringUTF(reader->GetVideoCodecName()) : nullptr; + jstring audioCodec = reader->HasAudio() + ? env->NewStringUTF(reader->GetAudioCodecName()) : nullptr; + + jobject info = env->NewObject(cls, ctor, + static_cast(reader->GetDurationUs()), + static_cast(reader->GetVideoWidth()), + static_cast(reader->GetVideoHeight()), + static_cast(reader->GetFrameRate()), + videoCodec, + static_cast(reader->GetSampleRate()), + static_cast(reader->GetChannels()), + audioCodec); + + if (videoCodec != nullptr) { + env->DeleteLocalRef(videoCodec); + } + if (audioCodec != nullptr) { + env->DeleteLocalRef(audioCodec); + } + + env->DeleteLocalRef(cls); + + return info; +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_ffmpeg_MediaReader_dispose +(JNIEnv * env, jclass caller, jlong handle) +{ + // Closing twice is allowed, so a handle that is already zero is simply + // nothing to do rather than an error. + if (handle == 0) { + return; + } + + delete reinterpret_cast(handle); +} diff --git a/webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp b/webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp new file mode 100644 index 00000000..c1594cac --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/AudioDecoder.cpp @@ -0,0 +1,252 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/AudioDecoder.h" + +extern "C" { +#include +#include +} + +namespace ffmpeg +{ + AudioDecoder::~AudioDecoder() + { + Close(); + } + + int AudioDecoder::Open(const AVStream * stream) + { + Close(); + + if (stream == nullptr) { + return AVERROR(EINVAL); + } + + const AVCodec * codec = avcodec_find_decoder(stream->codecpar->codec_id); + + if (codec == nullptr) { + return AVERROR_DECODER_NOT_FOUND; + } + + codec_context_ = avcodec_alloc_context3(codec); + + if (codec_context_ == nullptr) { + return AVERROR(ENOMEM); + } + + int result = avcodec_parameters_to_context(codec_context_, stream->codecpar); + + if (result < 0) { + Close(); + + return result; + } + + codec_context_->pkt_timebase = stream->time_base; + + result = avcodec_open2(codec_context_, codec, nullptr); + + if (result < 0) { + Close(); + + return result; + } + + // WebRTC takes mono or stereo, so anything wider is downmixed. The + // downmix itself is swresample's job. + channels_ = codec_context_->ch_layout.nb_channels >= 2 ? 2 : 1; + + AVChannelLayout out_layout; + + av_channel_layout_default(&out_layout, channels_); + + result = swr_alloc_set_opts2(&swr_context_, + &out_layout, AV_SAMPLE_FMT_S16, kSampleRate, + &codec_context_->ch_layout, codec_context_->sample_fmt, + codec_context_->sample_rate, 0, nullptr); + + av_channel_layout_uninit(&out_layout); + + if (result < 0) { + Close(); + + return result; + } + + result = swr_init(swr_context_); + + if (result < 0) { + Close(); + + return result; + } + + decoded_ = av_frame_alloc(); + + if (decoded_ == nullptr) { + Close(); + + return AVERROR(ENOMEM); + } + + time_base_ = stream->time_base; + + return 0; + } + + void AudioDecoder::Close() + { + if (decoded_ != nullptr) { + av_frame_free(&decoded_); + } + if (swr_context_ != nullptr) { + swr_free(&swr_context_); + } + if (codec_context_ != nullptr) { + avcodec_free_context(&codec_context_); + } + + pending_.clear(); + + time_base_ = { 0, 1 }; + channels_ = 0; + next_timestamp_us_ = 0; + have_timestamp_ = false; + } + + void AudioDecoder::Flush() + { + if (codec_context_ != nullptr) { + avcodec_flush_buffers(codec_context_); + } + + pending_.clear(); + + // The next frame decoded tells where the audio now starts. + have_timestamp_ = false; + } + + int AudioDecoder::SendPacket(const AVPacket * packet) + { + if (codec_context_ == nullptr) { + return AVERROR(EINVAL); + } + + return avcodec_send_packet(codec_context_, packet); + } + + int AudioDecoder::ReceiveChunk(std::vector & chunk, int64_t * timestamp_us) + { + if (codec_context_ == nullptr || decoded_ == nullptr) { + return AVERROR(EINVAL); + } + + const size_t wanted = static_cast(kFramesPerChunk) * channels_; + + while (pending_.size() < wanted) { + int result = avcodec_receive_frame(codec_context_, decoded_); + + if (result == AVERROR_EOF) { + if (pending_.empty()) { + return AVERROR_EOF; + } + + // The last chunk of a stream rarely lands on a 10 ms boundary, + // and WebRTC has no way to take a short one, so the remainder + // is filled with silence. + pending_.resize(wanted, 0); + + break; + } + if (result < 0) { + return result; + } + + result = Resample(decoded_); + + av_frame_unref(decoded_); + + if (result < 0) { + return result; + } + } + + TakeChunk(chunk, timestamp_us); + + return 0; + } + + int AudioDecoder::Resample(const AVFrame * frame) + { + if (!have_timestamp_) { + int64_t pts = frame->best_effort_timestamp != AV_NOPTS_VALUE + ? frame->best_effort_timestamp : frame->pts; + + next_timestamp_us_ = pts != AV_NOPTS_VALUE + ? av_rescale_q(pts, time_base_, AV_TIME_BASE_Q) : 0; + have_timestamp_ = true; + } + + // swresample holds samples back when rates differ, and those come out + // of a later call, so the room needed is the delay plus this frame. + int64_t delay = swr_get_delay(swr_context_, codec_context_->sample_rate); + int64_t capacity = av_rescale_rnd(delay + frame->nb_samples, kSampleRate, + codec_context_->sample_rate, AV_ROUND_UP); + + if (capacity <= 0) { + return 0; + } + + const size_t offset = pending_.size(); + + pending_.resize(offset + static_cast(capacity) * channels_); + + uint8_t * output = reinterpret_cast(pending_.data() + offset); + + int converted = swr_convert(swr_context_, &output, static_cast(capacity), + const_cast(frame->data), frame->nb_samples); + + if (converted < 0) { + pending_.resize(offset); + + return converted; + } + + // swresample usually produces fewer samples than the room made for it. + pending_.resize(offset + static_cast(converted) * channels_); + + return 0; + } + + void AudioDecoder::TakeChunk(std::vector & chunk, int64_t * timestamp_us) + { + const size_t wanted = static_cast(kFramesPerChunk) * channels_; + + chunk.assign(pending_.begin(), pending_.begin() + wanted); + pending_.erase(pending_.begin(), pending_.begin() + wanted); + + *timestamp_us = next_timestamp_us_; + + // Exactly 10 ms on, whatever the container's own packet timing was. + next_timestamp_us_ += 10000; + } + + int AudioDecoder::GetChannels() const + { + return channels_; + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/JavaPlayerObserver.cpp b/webrtc-java-media/src/main/cpp/src/media/JavaPlayerObserver.cpp new file mode 100644 index 00000000..02d8c160 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/JavaPlayerObserver.cpp @@ -0,0 +1,176 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/JavaPlayerObserver.h" + +namespace ffmpeg +{ + JavaPlayerObserver::JavaPlayerObserver(JNIEnv * env, jobject player) + { + if (env->GetJavaVM(&vm_) != JNI_OK) { + return; + } + + player_ = env->NewGlobalRef(player); + + if (player_ == nullptr) { + return; + } + + jclass cls = env->GetObjectClass(player_); + + if (cls == nullptr) { + return; + } + + on_state_changed_ = env->GetMethodID(cls, "onNativeStateChanged", "(I)V"); + on_end_of_stream_ = env->GetMethodID(cls, "onNativeEndOfStream", "()V"); + on_error_ = env->GetMethodID(cls, "onNativeError", "(Ljava/lang/String;)V"); + + env->DeleteLocalRef(cls); + + ClearPendingException(env); + } + + JavaPlayerObserver::~JavaPlayerObserver() + { + if (player_ == nullptr || vm_ == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (env != nullptr) { + env->DeleteGlobalRef(player_); + } + + player_ = nullptr; + + Detach(attached); + } + + void JavaPlayerObserver::OnStateChanged(int state) + { + if (on_state_changed_ == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (env != nullptr) { + env->CallVoidMethod(player_, on_state_changed_, static_cast(state)); + + ClearPendingException(env); + } + + Detach(attached); + } + + void JavaPlayerObserver::OnEndOfStream() + { + if (on_end_of_stream_ == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (env != nullptr) { + env->CallVoidMethod(player_, on_end_of_stream_); + + ClearPendingException(env); + } + + Detach(attached); + } + + void JavaPlayerObserver::OnError(const std::string & message) + { + if (on_error_ == nullptr) { + return; + } + + bool attached = false; + JNIEnv * env = Attach(&attached); + + if (env != nullptr) { + jstring text = env->NewStringUTF(message.c_str()); + + if (text != nullptr) { + env->CallVoidMethod(player_, on_error_, text); + + ClearPendingException(env); + + env->DeleteLocalRef(text); + } + else { + ClearPendingException(env); + } + } + + Detach(attached); + } + + JNIEnv * JavaPlayerObserver::Attach(bool * attached) + { + *attached = false; + + if (vm_ == nullptr) { + return nullptr; + } + + JNIEnv * env = nullptr; + + jint result = vm_->GetEnv(reinterpret_cast(&env), JNI_VERSION_1_6); + + if (result == JNI_OK) { + return env; + } + if (result != JNI_EDETACHED) { + return nullptr; + } + + // The player's own thread is not one the JVM started, so it has to be + // introduced before it can call anything. + if (vm_->AttachCurrentThreadAsDaemon(reinterpret_cast(&env), nullptr) != JNI_OK) { + return nullptr; + } + + *attached = true; + + return env; + } + + void JavaPlayerObserver::Detach(bool attached) + { + if (attached && vm_ != nullptr) { + vm_->DetachCurrentThread(); + } + } + + void JavaPlayerObserver::ClearPendingException(JNIEnv * env) + { + if (env->ExceptionCheck() == JNI_TRUE) { + // There is nowhere to throw this: the thread below is decoding, + // not running Java. Reporting and clearing keeps it from taking + // down the next unrelated JNI call instead. + env->ExceptionDescribe(); + env->ExceptionClear(); + } + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/MediaPacer.cpp b/webrtc-java-media/src/main/cpp/src/media/MediaPacer.cpp new file mode 100644 index 00000000..9d7fdba7 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/MediaPacer.cpp @@ -0,0 +1,365 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/MediaPacer.h" +#include "media/AudioDecoder.h" + +#include +#include +#include + +namespace +{ + // Runs when WebRTC drops its last reference to the pixels of a pushed + // frame, on whichever thread that happens to be. + void ReleaseVideoFrame(void * opaque) + { + AVFrame * frame = static_cast(opaque); + + av_frame_free(&frame); + } +} + +namespace ffmpeg +{ + MediaPacer::VideoItem::~VideoItem() + { + if (frame != nullptr) { + av_frame_free(&frame); + } + } + + MediaPacer::VideoItem::VideoItem(VideoItem && other) noexcept + : frame(other.frame), timestamp_us(other.timestamp_us) + { + other.frame = nullptr; + } + + MediaPacer::VideoItem & MediaPacer::VideoItem::operator=(VideoItem && other) noexcept + { + if (this != &other) { + if (frame != nullptr) { + av_frame_free(&frame); + } + + frame = other.frame; + timestamp_us = other.timestamp_us; + other.frame = nullptr; + } + + return *this; + } + + MediaPacer::MediaPacer(const webrtc_java_api * api, void * video_source, + void * audio_source) + : api_(api), + video_source_(video_source), + audio_source_(audio_source) + { + } + + MediaPacer::~MediaPacer() + { + Stop(); + } + + void MediaPacer::Start() + { + std::lock_guard lock(mutex_); + + if (running_) { + return; + } + + running_ = true; + thread_ = std::thread(&MediaPacer::Run, this); + } + + void MediaPacer::Stop() + { + { + std::lock_guard lock(mutex_); + + if (!running_ && !thread_.joinable()) { + return; + } + + running_ = false; + + // Everyone waiting has to see that nothing more is coming: the + // pacing thread, and a decode thread held up by a full queue. + work_.notify_all(); + video_space_.notify_all(); + audio_space_.notify_all(); + } + + if (thread_.joinable()) { + thread_.join(); + } + + std::lock_guard lock(mutex_); + + video_queue_.clear(); + audio_queue_.clear(); + } + + bool MediaPacer::PushVideo(AVFrame * frame, int64_t timestamp_us) + { + std::unique_lock lock(mutex_); + + video_space_.wait(lock, [this] { + return !running_ || video_queue_.size() < kVideoCapacity; + }); + + if (!running_) { + // The frame became ours on the way in, so it is ours to drop. + av_frame_free(&frame); + + return false; + } + + video_queue_.emplace_back(frame, timestamp_us); + + work_.notify_one(); + + return true; + } + + bool MediaPacer::PushAudio(std::vector && samples, int channels, + int64_t timestamp_us) + { + std::unique_lock lock(mutex_); + + audio_space_.wait(lock, [this] { + return !running_ || audio_queue_.size() < kAudioCapacity; + }); + + if (!running_) { + return false; + } + + AudioItem item; + + item.samples = std::move(samples); + item.channels = channels; + item.timestamp_us = timestamp_us; + + audio_queue_.push_back(std::move(item)); + + work_.notify_one(); + + return true; + } + + void MediaPacer::Pause() + { + std::lock_guard lock(mutex_); + + if (paused_) { + return; + } + + paused_ = true; + paused_at_us_ = api_->now_us(); + + work_.notify_all(); + } + + void MediaPacer::Resume() + { + std::lock_guard lock(mutex_); + + if (!paused_) { + return; + } + + // What is queued is due that much later than it was, so the mapping + // moves with the pause instead of the queue draining all at once. + base_us_ += api_->now_us() - paused_at_us_; + paused_ = false; + + work_.notify_all(); + } + + void MediaPacer::Flush() + { + std::lock_guard lock(mutex_); + + video_queue_.clear(); + audio_queue_.clear(); + + // The next item delivered starts a new mapping, wherever it comes from. + have_base_ = false; + + video_space_.notify_all(); + audio_space_.notify_all(); + work_.notify_all(); + } + + bool MediaPacer::IsDrained() const + { + std::lock_guard lock(mutex_); + + return video_queue_.empty() && audio_queue_.empty(); + } + + int64_t MediaPacer::GetPositionUs() const + { + std::lock_guard lock(mutex_); + + return position_us_; + } + + void MediaPacer::Run() + { + std::unique_lock lock(mutex_); + + while (running_) { + if (paused_) { + work_.wait(lock); + + continue; + } + + const int64_t now = api_->now_us(); + int64_t next_due_us = std::numeric_limits::max(); + bool delivered = false; + + // The mapping is made once, from whichever kind of media is ready + // first, and holds for both from then on. + if (!have_base_ && (!video_queue_.empty() || !audio_queue_.empty())) { + int64_t first = !video_queue_.empty() + ? video_queue_.front().timestamp_us + : audio_queue_.front().timestamp_us; + + if (!video_queue_.empty() && !audio_queue_.empty()) { + first = std::min(first, audio_queue_.front().timestamp_us); + } + + base_us_ = now - first; + have_base_ = true; + } + + if (!video_queue_.empty()) { + const int64_t due = video_queue_.front().timestamp_us + base_us_; + + if (due <= now) { + VideoItem item = std::move(video_queue_.front()); + + video_queue_.pop_front(); + position_us_ = item.timestamp_us; + + video_space_.notify_one(); + + lock.unlock(); + DeliverVideo(std::move(item), due); + lock.lock(); + + delivered = true; + } + else { + next_due_us = std::min(next_due_us, due); + } + } + + if (!audio_queue_.empty()) { + const int64_t due = audio_queue_.front().timestamp_us + base_us_; + + if (due <= now) { + AudioItem item = std::move(audio_queue_.front()); + + audio_queue_.pop_front(); + position_us_ = std::max(position_us_, item.timestamp_us); + + audio_space_.notify_one(); + + lock.unlock(); + DeliverAudio(item, due); + lock.lock(); + + delivered = true; + } + else { + next_due_us = std::min(next_due_us, due); + } + } + + if (delivered) { + // Something else may be due already. + continue; + } + + if (next_due_us == std::numeric_limits::max()) { + // Nothing queued at all; a push or a stop wakes this. + work_.wait(lock); + } + else { + work_.wait_for(lock, std::chrono::microseconds(next_due_us - now)); + } + } + } + + void MediaPacer::DeliverVideo(VideoItem item, int64_t mapped_us) + { + AVFrame * frame = item.frame; + + if (video_source_ == nullptr || frame == nullptr) { + // Nothing to deliver to; the item frees the frame as it goes out + // of scope. + return; + } + + wj_i420_frame pushed = {}; + + pushed.width = frame->width; + pushed.height = frame->height; + pushed.y = frame->data[0]; + pushed.u = frame->data[1]; + pushed.v = frame->data[2]; + pushed.stride_y = frame->linesize[0]; + pushed.stride_u = frame->linesize[1]; + pushed.stride_v = frame->linesize[2]; + pushed.rotation = 0; + pushed.timestamp_us = mapped_us; + pushed.release = ReleaseVideoFrame; + pushed.opaque = frame; + + // The frame belongs to the release callback now, which runs whether + // the push succeeds or fails, so this item must not free it as well. + item.frame = nullptr; + + api_->video_source_push(video_source_, &pushed); + } + + void MediaPacer::DeliverAudio(const AudioItem & item, int64_t mapped_us) + { + if (audio_source_ == nullptr || item.samples.empty()) { + return; + } + + wj_audio_chunk chunk = {}; + + chunk.samples = item.samples.data(); + chunk.sample_rate = AudioDecoder::kSampleRate; + chunk.channels = item.channels; + chunk.frames = AudioDecoder::kFramesPerChunk; + chunk.timestamp_us = mapped_us; + + // The samples are copied during the call, so the item may go away as + // soon as this returns. + api_->audio_source_push(audio_source_, &chunk); + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/MediaPlayer.cpp b/webrtc-java-media/src/main/cpp/src/media/MediaPlayer.cpp new file mode 100644 index 00000000..72695e87 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/MediaPlayer.cpp @@ -0,0 +1,515 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/MediaPlayer.h" + +#include +#include + +extern "C" { +#include +} + +namespace +{ + std::string ErrorText(int error) + { + char buffer[AV_ERROR_MAX_STRING_SIZE] = { 0 }; + + if (av_strerror(error, buffer, sizeof(buffer)) < 0) { + return "error " + std::to_string(error); + } + + return buffer; + } +} + +namespace ffmpeg +{ + MediaPlayer::MediaPlayer(std::unique_ptr reader, + const webrtc_java_api * api, void * video_source, void * audio_source) + : reader_(std::move(reader)), + api_(api) + { + pacer_ = std::make_unique(api, video_source, audio_source); + } + + MediaPlayer::~MediaPlayer() + { + Close(); + } + + void MediaPlayer::SetObserver(std::unique_ptr observer) + { + std::lock_guard lock(mutex_); + + observer_ = std::move(observer); + } + + int MediaPlayer::Initialize() + { + if (reader_ == nullptr || !reader_->IsOpen()) { + return AVERROR(EINVAL); + } + + if (reader_->HasVideo()) { + int result = video_decoder_.Open(reader_->GetVideoStream()); + + if (result < 0) { + return result; + } + + has_video_ = true; + } + + if (reader_->HasAudio()) { + int result = audio_decoder_.Open(reader_->GetAudioStream()); + + if (result < 0) { + return result; + } + + has_audio_ = true; + } + + if (!has_video_ && !has_audio_) { + return AVERROR_STREAM_NOT_FOUND; + } + + pacer_->Start(); + + { + std::lock_guard lock(mutex_); + + running_ = true; + thread_ = std::thread(&MediaPlayer::Run, this); + } + + return 0; + } + + void MediaPlayer::Play() + { + { + std::lock_guard lock(mutex_); + + if (closing_ || playing_) { + return; + } + + playing_ = true; + + if (ended_) { + // Playing again after the source ran out starts it over, + // which is what a listener would expect of a play button. + ended_ = false; + seek_pending_ = true; + seek_position_us_ = 0; + } + + command_.notify_all(); + } + + pacer_->Resume(); + + SetState(kPlaying); + } + + void MediaPlayer::Pause() + { + { + std::lock_guard lock(mutex_); + + if (closing_ || !playing_) { + return; + } + + playing_ = false; + } + + pacer_->Pause(); + + SetState(kPaused); + } + + void MediaPlayer::Seek(int64_t position_us) + { + { + std::lock_guard lock(mutex_); + + if (closing_) { + return; + } + + seek_pending_ = true; + seek_position_us_ = position_us; + ended_ = false; + + command_.notify_all(); + } + + // The decode thread may be held up by a full queue, and it cannot act + // on the request until it gets out of that. Dropping what is queued + // both frees it and throws away what belongs to the old position; the + // thread flushes again once the seek has actually happened. + pacer_->Flush(); + } + + void MediaPlayer::SetLooping(bool looping) + { + looping_.store(looping); + } + + bool MediaPlayer::IsLooping() const + { + return looping_.load(); + } + + void MediaPlayer::Close() + { + { + std::lock_guard lock(mutex_); + + if (closing_) { + return; + } + + closing_ = true; + playing_ = false; + + command_.notify_all(); + } + + // Stopping the pacer is what releases a decode thread waiting for room + // in a queue that nothing is draining any more. + pacer_->Stop(); + + if (thread_.joinable()) { + thread_.join(); + } + + video_decoder_.Close(); + audio_decoder_.Close(); + + if (reader_ != nullptr) { + reader_->Close(); + } + + SetState(kClosed); + } + + int64_t MediaPlayer::GetPositionUs() const + { + const int64_t position = pacer_->GetPositionUs() - loop_offset_us_.load(); + + return position > 0 ? position : 0; + } + + int MediaPlayer::GetState() const + { + std::lock_guard lock(mutex_); + + return state_; + } + + void MediaPlayer::Run() + { + AVPacket * packet = av_packet_alloc(); + + if (packet == nullptr) { + ReportError("Allocating a packet failed", AVERROR(ENOMEM)); + + return; + } + + for (;;) { + int64_t seek_to = 0; + bool do_seek = false; + + { + std::unique_lock lock(mutex_); + + command_.wait(lock, [this] { + return closing_ || seek_pending_ || (playing_ && !ended_); + }); + + if (closing_) { + break; + } + + if (seek_pending_) { + seek_pending_ = false; + seek_to = seek_position_us_; + do_seek = true; + } + } + + if (do_seek) { + PerformSeek(seek_to); + + continue; + } + + bool end_of_stream = false; + + if (!PumpOnce(packet, &end_of_stream)) { + break; + } + + if (!end_of_stream) { + continue; + } + + // Whatever the decoders still hold belongs to this pass, so it + // goes out before anything is rewound. + DrainDecoders(); + + if (looping_.load()) { + int64_t advance = reader_->GetDurationUs(); + + if (advance <= 0) { + // A container that does not say how long it runs still + // tells us where its last frame was. + advance = max_source_pts_us_ + 10000; + } + + loop_offset_us_.fetch_add(advance); + + reader_->Seek(0); + video_decoder_.Flush(); + audio_decoder_.Flush(); + + // The pacer keeps its queue and its mapping, so the seam + // between one pass and the next is not heard or seen. + continue; + } + + // Everything has been handed over, but not yet played out. + for (;;) { + std::unique_lock lock(mutex_); + + if (closing_ || pacer_->IsDrained()) { + break; + } + + command_.wait_for(lock, std::chrono::milliseconds(10)); + } + + { + std::lock_guard lock(mutex_); + + if (closing_) { + break; + } + + ended_ = true; + playing_ = false; + } + + SetState(kEnded); + + MediaPlayerObserver * observer = nullptr; + { + std::lock_guard lock(mutex_); + + observer = observer_.get(); + } + + if (observer != nullptr) { + observer->OnEndOfStream(); + } + } + + av_packet_free(&packet); + } + + bool MediaPlayer::PumpOnce(AVPacket * packet, bool * end_of_stream) + { + int result = reader_->ReadPacket(packet); + + if (result == AVERROR_EOF) { + *end_of_stream = true; + + return true; + } + if (result < 0) { + ReportError("Reading the source failed", result); + + return false; + } + + const int index = packet->stream_index; + bool video = has_video_ && index == reader_->GetVideoStreamIndex(); + bool audio = has_audio_ && index == reader_->GetAudioStreamIndex(); + + if (video) { + result = video_decoder_.SendPacket(packet); + } + else if (audio) { + result = audio_decoder_.SendPacket(packet); + } + else { + // A stream this player does not play, such as a subtitle track. + result = 0; + } + + av_packet_unref(packet); + + if (result < 0 && result != AVERROR(EAGAIN)) { + ReportError("Decoding failed", result); + + return false; + } + + if (video) { + return DecodeVideo() >= 0; + } + if (audio) { + return DecodeAudio() >= 0; + } + + return true; + } + + void MediaPlayer::DrainDecoders() + { + if (has_video_) { + video_decoder_.SendPacket(nullptr); + DecodeVideo(); + } + if (has_audio_) { + audio_decoder_.SendPacket(nullptr); + DecodeAudio(); + } + } + + int MediaPlayer::DecodeVideo() + { + for (;;) { + AVFrame * frame = nullptr; + int64_t timestamp_us = 0; + + int result = video_decoder_.ReceiveFrame(&frame, ×tamp_us); + + if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) { + return 0; + } + if (result < 0) { + ReportError("Decoding video failed", result); + + return result; + } + + if (timestamp_us > max_source_pts_us_) { + max_source_pts_us_ = timestamp_us; + } + + // The frame belongs to the pacer from here on, whether or not it + // makes it into the queue. + if (!pacer_->PushVideo(frame, timestamp_us + loop_offset_us_.load())) { + return 0; + } + } + } + + int MediaPlayer::DecodeAudio() + { + for (;;) { + std::vector chunk; + int64_t timestamp_us = 0; + + int result = audio_decoder_.ReceiveChunk(chunk, ×tamp_us); + + if (result == AVERROR(EAGAIN) || result == AVERROR_EOF) { + return 0; + } + if (result < 0) { + ReportError("Decoding audio failed", result); + + return result; + } + + if (timestamp_us > max_source_pts_us_) { + max_source_pts_us_ = timestamp_us; + } + + if (!pacer_->PushAudio(std::move(chunk), audio_decoder_.GetChannels(), + timestamp_us + loop_offset_us_.load())) { + return 0; + } + } + } + + void MediaPlayer::PerformSeek(int64_t position_us) + { + int result = reader_->Seek(position_us); + + if (result < 0) { + ReportError("Seeking failed", result); + + return; + } + + video_decoder_.Flush(); + audio_decoder_.Flush(); + + // A seek starts a timeline of its own: the pacer maps the first item + // that arrives onto the clock afresh, so the offset that kept looping + // monotonic is no longer needed and would only skew the position. + loop_offset_us_.store(0); + max_source_pts_us_ = position_us; + + pacer_->Flush(); + } + + void MediaPlayer::SetState(int state) + { + MediaPlayerObserver * observer = nullptr; + + { + std::lock_guard lock(mutex_); + + if (state_ == state) { + return; + } + + state_ = state; + observer = observer_.get(); + } + + // Called with the lock released: an observer runs Java code, which + // must never happen underneath a lock of ours. + if (observer != nullptr) { + observer->OnStateChanged(state); + } + } + + void MediaPlayer::ReportError(const std::string & message, int error) + { + MediaPlayerObserver * observer = nullptr; + + { + std::lock_guard lock(mutex_); + + playing_ = false; + observer = observer_.get(); + } + + if (observer != nullptr) { + observer->OnError(message + ": " + ErrorText(error)); + } + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/MediaReader.cpp b/webrtc-java-media/src/main/cpp/src/media/MediaReader.cpp new file mode 100644 index 00000000..32493604 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/MediaReader.cpp @@ -0,0 +1,224 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/MediaReader.h" + +extern "C" { +#include +#include +} + +namespace ffmpeg +{ + MediaReader::~MediaReader() + { + Close(); + } + + int MediaReader::Open(const std::string & url) + { + Close(); + + int result = avformat_open_input(&format_context_, url.c_str(), nullptr, nullptr); + + if (result < 0) { + // avformat_open_input frees the context and nulls the pointer + // itself when it fails, so there is nothing left to release. + format_context_ = nullptr; + + return result; + } + + result = avformat_find_stream_info(format_context_, nullptr); + + if (result < 0) { + Close(); + + return result; + } + + // A container may hold several streams of a kind, and libavformat is + // the one that knows which of them is meant to be played. A source + // with no stream of a kind reports AVERROR_STREAM_NOT_FOUND, which is + // not a failure: a file may be video only or audio only. + video_stream_index_ = av_find_best_stream(format_context_, + AVMEDIA_TYPE_VIDEO, -1, -1, nullptr, 0); + audio_stream_index_ = av_find_best_stream(format_context_, + AVMEDIA_TYPE_AUDIO, -1, -1, nullptr, 0); + + if (video_stream_index_ < 0) { + video_stream_index_ = -1; + } + if (audio_stream_index_ < 0) { + audio_stream_index_ = -1; + } + + if (video_stream_index_ < 0 && audio_stream_index_ < 0) { + // Nothing here can be played, which is worth reporting as a + // failure rather than handing back an empty reader. + Close(); + + return AVERROR_STREAM_NOT_FOUND; + } + + return 0; + } + + void MediaReader::Close() + { + if (format_context_ != nullptr) { + avformat_close_input(&format_context_); + } + + format_context_ = nullptr; + video_stream_index_ = -1; + audio_stream_index_ = -1; + } + + bool MediaReader::IsOpen() const + { + return format_context_ != nullptr; + } + + int64_t MediaReader::GetDurationUs() const + { + if (format_context_ == nullptr || format_context_->duration == AV_NOPTS_VALUE) { + return 0; + } + + // AVFormatContext::duration is already in AV_TIME_BASE units, which + // are microseconds. + return format_context_->duration; + } + + bool MediaReader::HasVideo() const + { + return GetVideoStream() != nullptr; + } + + int MediaReader::GetVideoWidth() const + { + const AVStream * stream = GetVideoStream(); + + return stream != nullptr ? stream->codecpar->width : 0; + } + + int MediaReader::GetVideoHeight() const + { + const AVStream * stream = GetVideoStream(); + + return stream != nullptr ? stream->codecpar->height : 0; + } + + double MediaReader::GetFrameRate() const + { + const AVStream * stream = GetVideoStream(); + + if (stream == nullptr || stream->avg_frame_rate.den == 0) { + return 0; + } + + return av_q2d(stream->avg_frame_rate); + } + + const char * MediaReader::GetVideoCodecName() const + { + const AVStream * stream = GetVideoStream(); + + return avcodec_get_name(stream != nullptr + ? stream->codecpar->codec_id : AV_CODEC_ID_NONE); + } + + bool MediaReader::HasAudio() const + { + return GetAudioStream() != nullptr; + } + + int MediaReader::GetSampleRate() const + { + const AVStream * stream = GetAudioStream(); + + return stream != nullptr ? stream->codecpar->sample_rate : 0; + } + + int MediaReader::GetChannels() const + { + const AVStream * stream = GetAudioStream(); + + return stream != nullptr ? stream->codecpar->ch_layout.nb_channels : 0; + } + + const char * MediaReader::GetAudioCodecName() const + { + const AVStream * stream = GetAudioStream(); + + return avcodec_get_name(stream != nullptr + ? stream->codecpar->codec_id : AV_CODEC_ID_NONE); + } + + int MediaReader::GetVideoStreamIndex() const + { + return video_stream_index_; + } + + int MediaReader::GetAudioStreamIndex() const + { + return audio_stream_index_; + } + + int MediaReader::ReadPacket(AVPacket * packet) + { + if (format_context_ == nullptr) { + return AVERROR(EINVAL); + } + + return av_read_frame(format_context_, packet); + } + + int MediaReader::Seek(int64_t position_us) + { + if (format_context_ == nullptr) { + return AVERROR(EINVAL); + } + + if (position_us < 0) { + position_us = 0; + } + + // AVSEEK_FLAG_BACKWARD lands on the keyframe at or before the target, + // so that what follows can actually be decoded. Frames between that + // keyframe and the target are decoded and dropped by the caller. + return av_seek_frame(format_context_, -1, position_us, AVSEEK_FLAG_BACKWARD); + } + + const AVStream * MediaReader::GetVideoStream() const + { + if (format_context_ == nullptr || video_stream_index_ < 0) { + return nullptr; + } + + return format_context_->streams[video_stream_index_]; + } + + const AVStream * MediaReader::GetAudioStream() const + { + if (format_context_ == nullptr || audio_stream_index_ < 0) { + return nullptr; + } + + return format_context_->streams[audio_stream_index_]; + } +} diff --git a/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp new file mode 100644 index 00000000..b3da1c21 --- /dev/null +++ b/webrtc-java-media/src/main/cpp/src/media/VideoDecoder.cpp @@ -0,0 +1,238 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "media/VideoDecoder.h" + +extern "C" { +#include +#include +} + +namespace +{ + // yuv420p is what WebRTC calls I420. yuvj420p has the same planes and + // strides and differs only in the range the values cover, which is + // metadata rather than layout, so it needs no conversion either. + bool IsI420(int format) + { + return format == AV_PIX_FMT_YUV420P || format == AV_PIX_FMT_YUVJ420P; + } +} + +namespace ffmpeg +{ + VideoDecoder::~VideoDecoder() + { + Close(); + } + + int VideoDecoder::Open(const AVStream * stream) + { + Close(); + + if (stream == nullptr) { + return AVERROR(EINVAL); + } + + const AVCodec * codec = avcodec_find_decoder(stream->codecpar->codec_id); + + if (codec == nullptr) { + return AVERROR_DECODER_NOT_FOUND; + } + + codec_context_ = avcodec_alloc_context3(codec); + + if (codec_context_ == nullptr) { + return AVERROR(ENOMEM); + } + + int result = avcodec_parameters_to_context(codec_context_, stream->codecpar); + + if (result < 0) { + Close(); + + return result; + } + + // Without this the decoder cannot put a meaningful timestamp on a + // frame, and every frame would come back with AV_NOPTS_VALUE. + codec_context_->pkt_timebase = stream->time_base; + + // 0 lets libavcodec pick a thread count for the machine. Decoding is + // the one part of playback that can genuinely use several cores. + codec_context_->thread_count = 0; + + result = avcodec_open2(codec_context_, codec, nullptr); + + if (result < 0) { + Close(); + + return result; + } + + decoded_ = av_frame_alloc(); + + if (decoded_ == nullptr) { + Close(); + + return AVERROR(ENOMEM); + } + + time_base_ = stream->time_base; + + return 0; + } + + void VideoDecoder::Close() + { + if (decoded_ != nullptr) { + av_frame_free(&decoded_); + } + if (sws_context_ != nullptr) { + sws_freeContext(sws_context_); + + sws_context_ = nullptr; + } + if (codec_context_ != nullptr) { + avcodec_free_context(&codec_context_); + } + + time_base_ = { 0, 1 }; + } + + void VideoDecoder::Flush() + { + if (codec_context_ != nullptr) { + avcodec_flush_buffers(codec_context_); + } + } + + int VideoDecoder::SendPacket(const AVPacket * packet) + { + if (codec_context_ == nullptr) { + return AVERROR(EINVAL); + } + + return avcodec_send_packet(codec_context_, packet); + } + + int VideoDecoder::ReceiveFrame(AVFrame ** frame, int64_t * timestamp_us) + { + if (codec_context_ == nullptr || decoded_ == nullptr) { + return AVERROR(EINVAL); + } + + int result = avcodec_receive_frame(codec_context_, decoded_); + + if (result < 0) { + return result; + } + + // best_effort_timestamp is what libavcodec makes of a stream whose + // packets carry no presentation time of their own. + int64_t pts = decoded_->best_effort_timestamp != AV_NOPTS_VALUE + ? decoded_->best_effort_timestamp : decoded_->pts; + + *timestamp_us = pts != AV_NOPTS_VALUE + ? av_rescale_q(pts, time_base_, AV_TIME_BASE_Q) : 0; + + if (IsI420(decoded_->format)) { + // Hand over a reference to the decoded picture rather than a copy + // of it. The caller drops that reference once WebRTC is done. + AVFrame * reference = av_frame_alloc(); + + if (reference == nullptr) { + av_frame_unref(decoded_); + + return AVERROR(ENOMEM); + } + + result = av_frame_ref(reference, decoded_); + + av_frame_unref(decoded_); + + if (result < 0) { + av_frame_free(&reference); + + return result; + } + + *frame = reference; + + return 0; + } + + result = ConvertToI420(decoded_, frame); + + av_frame_unref(decoded_); + + return result; + } + + int VideoDecoder::ConvertToI420(const AVFrame * source, AVFrame ** result) + { + sws_context_ = sws_getCachedContext(sws_context_, + source->width, source->height, + static_cast(source->format), + source->width, source->height, AV_PIX_FMT_YUV420P, + SWS_BILINEAR, nullptr, nullptr, nullptr); + + if (sws_context_ == nullptr) { + return AVERROR(EINVAL); + } + + AVFrame * converted = av_frame_alloc(); + + if (converted == nullptr) { + return AVERROR(ENOMEM); + } + + converted->format = AV_PIX_FMT_YUV420P; + converted->width = source->width; + converted->height = source->height; + + int error = av_frame_get_buffer(converted, 0); + + if (error < 0) { + av_frame_free(&converted); + + return error; + } + + error = sws_scale(sws_context_, source->data, source->linesize, 0, + source->height, converted->data, converted->linesize); + + if (error < 0) { + av_frame_free(&converted); + + return error; + } + + *result = converted; + + return 0; + } + + int VideoDecoder::GetWidth() const + { + return codec_context_ != nullptr ? codec_context_->width : 0; + } + + int VideoDecoder::GetHeight() const + { + return codec_context_ != nullptr ? codec_context_->height : 0; + } +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/FFmpeg.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/FFmpeg.java new file mode 100644 index 00000000..1151addb --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/FFmpeg.java @@ -0,0 +1,117 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +import dev.onvoid.webrtc.internal.NativeLoader; + +/** + * Loads this module's native library and reports what it was built against. + *

+ * The FFmpeg libraries are loaded first, in the order they depend on each + * other, and under the names they were built with. A dynamic linker matches an + * already loaded module by file name, so by the time the media library itself + * is loaded every symbol it imports is resolved without changing any search + * path. + * + * @author Alex Andres + */ +public final class FFmpeg { + + /** The name of this module's native library, without platform decoration. */ + private static final String LIBRARY = "webrtc-java-media"; + + + static { + try { + NativeLoader.loadLibrary(LIBRARY, dependencies()); + } + catch (Exception e) { + throw new RuntimeException("Load library '" + LIBRARY + "' failed", e); + } + } + + + private FFmpeg() { + // Static access only. + } + + /** + * Returns the FFmpeg version the loaded native library was built against, + * for example {@code 7.1.1}. Useful to confirm which FFmpeg an application + * actually ended up with, since the libraries may be replaced. + * + * @return The FFmpeg version string. + */ + public static native String version(); + + /** + * Returns the license of the loaded FFmpeg libraries, which must be an + * LGPL variant. A build that reports a GPL license is one this project did + * not produce, and redistributing it carries obligations this project does + * not meet. + * + * @return The FFmpeg license string, for example {@code LGPL version 2.1 or later}. + */ + public static native String license(); + + /** + * Makes sure the native library is loaded. Loading happens when this class + * is first used, so this is only needed to bring the failure forward to a + * point where it can be reported. + */ + public static void load() { + // Touching the class runs the static initializer. + } + + /** + * Returns the shared libraries to load before this module's own, in the + * order they depend on each other, named as they are in the platform jar. + * + * @return The file names of the FFmpeg libraries for this platform. + */ + private static String[] dependencies() { + String osName = System.getProperty("os.name").toLowerCase(); + + if (osName.startsWith("windows")) { + return new String[] { + "avutil-59.dll", + "swresample-5.dll", + "swscale-8.dll", + "avcodec-61.dll", + "avformat-61.dll" + }; + } + if (osName.startsWith("mac os")) { + return new String[] { + "libavutil.59.dylib", + "libswresample.5.dylib", + "libswscale.8.dylib", + "libavcodec.61.dylib", + "libavformat.61.dylib" + }; + } + + return new String[] { + "libavutil.so.59", + "libswresample.so.5", + "libswscale.so.8", + "libavcodec.so.61", + "libavformat.so.61" + }; + } + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaFileSource.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaFileSource.java new file mode 100644 index 00000000..1fced442 --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaFileSource.java @@ -0,0 +1,234 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +import java.io.IOException; +import java.nio.file.Path; + +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; + +/** + * A media file or stream as a pair of media sources, ready to make tracks + * from. + *

+ * This is the short way to do what the module is for. It opens the source, + * makes a {@link CustomVideoSource} and a {@link CustomAudioSource} for + * whichever streams it actually has, and wires a {@link MediaPlayer} to feed + * them: + * 

{@code
+ * MediaFileSource source = new MediaFileSource(Path.of("movie.mp4"));
+ *
+ * VideoTrack videoTrack = factory.createVideoTrack("video", source.getVideoSource());
+ * AudioTrack audioTrack = factory.createAudioTrack("audio", source.getAudioSource());
+ *
+ * peerConnection.addTrack(videoTrack, List.of("stream"));
+ * peerConnection.addTrack(audioTrack, List.of("stream"));
+ *
+ * source.play();
+ * }
+ *

+ * A source with no video has no video source, and likewise for audio, so check + * {@link #getVideoSource()} and {@link #getAudioSource()} for {@code null} + * before making a track, or ask {@link #getInfo()} first. + *

+ * A factory fed from here is sending pushed audio, so it cannot also send + * audio captured by its {@code AudioDeviceModule}. Use a separate factory if + * an application needs both. + *

+ * Closing this releases the player and both media sources. Dispose of any + * tracks made from them first. + * + * @author Alex Andres + */ +public class MediaFileSource implements AutoCloseable { + + private final MediaInfo info; + + private final CustomVideoSource videoSource; + + private final CustomAudioSource audioSource; + + private final MediaPlayer player; + + private boolean closed; + + + /** + * Opens the media file at the given path. + * + * @param path The path of the file to play. + * + * @throws IOException if the source cannot be opened or decoded. + */ + public MediaFileSource(Path path) throws IOException { + this(path.toAbsolutePath().toString()); + } + + /** + * Opens the given media source. + * + * @param source The path or URL of the source to play. + * + * @throws IOException if the source cannot be opened or decoded. + */ + public MediaFileSource(String source) throws IOException { + MediaReader reader = new MediaReader(source); + + // Read while the reader is still ours: the player takes it over. + info = reader.getInfo(); + + videoSource = info.hasVideo() ? new CustomVideoSource() : null; + audioSource = info.hasAudio() ? new CustomAudioSource() : null; + + try { + player = new MediaPlayer(reader, videoSource, audioSource); + } + catch (IOException | RuntimeException e) { + // The player did not take charge, so what was made here has to be + // unwound rather than left to leak. + disposeSources(); + + reader.close(); + + throw e; + } + } + + /** + * Returns what this source contains. + * + * @return The media information. + */ + public MediaInfo getInfo() { + return info; + } + + /** + * Returns the video source fed by this file, to make a video track from. + * + * @return The video source, or {@code null} if the source has no video. + */ + public CustomVideoSource getVideoSource() { + return videoSource; + } + + /** + * Returns the audio source fed by this file, to make an audio track from. + * + * @return The audio source, or {@code null} if the source has no audio. + */ + public CustomAudioSource getAudioSource() { + return audioSource; + } + + /** + * Returns the player driving this source, for anything beyond the methods + * below. + * + * @return The player. + */ + public MediaPlayer getPlayer() { + return player; + } + + /** + * Sets what to report playback events to. + * + * @param listener The listener, or {@code null} to stop reporting. + */ + public void setListener(MediaPlayerListener listener) { + player.setListener(listener); + } + + /** + * Starts or resumes playback. + */ + public void play() { + player.play(); + } + + /** + * Holds playback where it is. + */ + public void pause() { + player.pause(); + } + + /** + * Moves playback to the given position. + * + * @param positionUs The position in microseconds from the start. + */ + public void seek(long positionUs) { + player.seek(positionUs); + } + + /** + * Sets whether the source starts again when it runs out. + * + * @param looping True to play the source over and over. + */ + public void setLooping(boolean looping) { + player.setLooping(looping); + } + + /** + * Returns where playback has got to, in microseconds. + * + * @return The position in microseconds. + */ + public long getPositionUs() { + return player.getPositionUs(); + } + + /** + * Returns what the player is currently doing. + * + * @return The player state. + */ + public MediaPlayerState getState() { + return player.getState(); + } + + /** + * Stops playback and releases the player and both media sources. Closing + * a source that is already closed does nothing. + */ + @Override + public synchronized void close() { + if (closed) { + return; + } + + closed = true; + + player.close(); + + disposeSources(); + } + + private void disposeSources() { + if (videoSource != null) { + videoSource.dispose(); + } + if (audioSource != null) { + audioSource.dispose(); + } + } + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaInfo.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaInfo.java new file mode 100644 index 00000000..eb5d5ca6 --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaInfo.java @@ -0,0 +1,197 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +/** + * What a media source contains: how long it runs, and the format of the video + * and audio streams that will be played from it. + *

+ * A stream that is absent is reported as a zero size, frame rate, sample rate + * or channel count, so {@link #hasVideo()} and {@link #hasAudio()} are the + * questions to ask before reading the rest. + * + * @author Alex Andres + */ +public class MediaInfo { + + /** How long the source runs in microseconds, or 0 if it is not known. */ + private final long durationUs; + + /** The width of the video in pixels, or 0 if there is no video. */ + private final int videoWidth; + + /** The height of the video in pixels, or 0 if there is no video. */ + private final int videoHeight; + + /** The frame rate of the video, or 0 if there is no video. */ + private final double frameRate; + + /** The name of the video codec, or {@code null} if there is no video. */ + private final String videoCodec; + + /** The sample rate of the audio in Hz, or 0 if there is no audio. */ + private final int sampleRate; + + /** The channel count of the audio, or 0 if there is no audio. */ + private final int channels; + + /** The name of the audio codec, or {@code null} if there is no audio. */ + private final String audioCodec; + + + /** + * Creates media information. Called by native code once a source has been + * opened and its streams have been examined. + * + * @param durationUs How long the source runs in microseconds. + * @param videoWidth The width of the video in pixels. + * @param videoHeight The height of the video in pixels. + * @param frameRate The frame rate of the video. + * @param videoCodec The name of the video codec. + * @param sampleRate The sample rate of the audio in Hz. + * @param channels The channel count of the audio. + * @param audioCodec The name of the audio codec. + */ + public MediaInfo(long durationUs, int videoWidth, int videoHeight, + double frameRate, String videoCodec, int sampleRate, int channels, + String audioCodec) { + this.durationUs = durationUs; + this.videoWidth = videoWidth; + this.videoHeight = videoHeight; + this.frameRate = frameRate; + this.videoCodec = videoCodec; + this.sampleRate = sampleRate; + this.channels = channels; + this.audioCodec = audioCodec; + } + + /** + * Returns how long the source runs, in microseconds. Live streams and some + * containers do not say, in which case this is {@code 0}. + * + * @return The duration in microseconds, or {@code 0} if it is not known. + */ + public long getDurationUs() { + return durationUs; + } + + /** + * Returns whether the source has a video stream that can be played. + * + * @return True if there is video. + */ + public boolean hasVideo() { + return videoWidth > 0 && videoHeight > 0; + } + + /** + * Returns the width of the video in pixels. + * + * @return The width, or {@code 0} if there is no video. + */ + public int getVideoWidth() { + return videoWidth; + } + + /** + * Returns the height of the video in pixels. + * + * @return The height, or {@code 0} if there is no video. + */ + public int getVideoHeight() { + return videoHeight; + } + + /** + * Returns the frame rate of the video. For a variable frame rate source + * this is the average the container reports. + * + * @return The frame rate, or {@code 0} if there is no video. + */ + public double getFrameRate() { + return frameRate; + } + + /** + * Returns the name of the video codec, as FFmpeg names it, for example + * {@code h264} or {@code vp9}. + * + * @return The codec name, or {@code null} if there is no video. + */ + public String getVideoCodec() { + return videoCodec; + } + + /** + * Returns whether the source has an audio stream that can be played. + * + * @return True if there is audio. + */ + public boolean hasAudio() { + return sampleRate > 0 && channels > 0; + } + + /** + * Returns the sample rate of the audio in Hz, as it is in the source. The + * audio is resampled before it reaches WebRTC, so this is not necessarily + * the rate that is sent. + * + * @return The sample rate, or {@code 0} if there is no audio. + */ + public int getSampleRate() { + return sampleRate; + } + + /** + * Returns the channel count of the audio, as it is in the source. + * + * @return The channel count, or {@code 0} if there is no audio. + */ + public int getChannels() { + return channels; + } + + /** + * Returns the name of the audio codec, as FFmpeg names it, for example + * {@code aac} or {@code opus}. + * + * @return The codec name, or {@code null} if there is no audio. + */ + public String getAudioCodec() { + return audioCodec; + } + + @Override + public String toString() { + StringBuilder builder = new StringBuilder("MediaInfo["); + + builder.append(durationUs / 1_000_000.0).append("s"); + + if (hasVideo()) { + builder.append(", video ").append(videoWidth).append("x") + .append(videoHeight).append(" @ ").append(frameRate) + .append(" ").append(videoCodec); + } + if (hasAudio()) { + builder.append(", audio ").append(sampleRate).append(" Hz ") + .append(channels).append(" ch ").append(audioCodec); + } + + return builder.append("]").toString(); + } + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayer.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayer.java new file mode 100644 index 00000000..cd09f8cc --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayer.java @@ -0,0 +1,249 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +import java.io.IOException; +import java.util.Objects; + +import dev.onvoid.webrtc.internal.NativeApi; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; + +/** + * Plays a media source into webrtc-java's custom media sources, so that a + * peer connection sends a file the way it would send a camera and microphone. + *

+ * Nothing is carried through Java: the player decodes in native code and hands + * the result straight to the native side of the given sources. Frames are + * delivered in real time and carry the presentation times of the source, so + * what reaches the receiver keeps the timing of the file rather than the + * timing of a thread. + *

+ * The player takes over the reader it is given. That reader must not be used + * or closed afterwards; closing the player releases it. + *

+ * Example: + * 

{@code
+ * CustomVideoSource videoSource = new CustomVideoSource();
+ * CustomAudioSource audioSource = new CustomAudioSource();
+ *
+ * MediaPlayer player = new MediaPlayer(new MediaReader(path), videoSource, audioSource);
+ *
+ * VideoTrack videoTrack = factory.createVideoTrack("video", videoSource);
+ * AudioTrack audioTrack = factory.createAudioTrack("audio", audioSource);
+ *
+ * player.play();
+ * }
+ *

+ * A factory fed by this player is sending pushed audio, so it cannot also send + * audio captured by its {@code AudioDeviceModule}. Use a separate factory if + * an application needs both. + * + * @author Alex Andres + */ +public class MediaPlayer implements AutoCloseable { + + static { + FFmpeg.load(); + } + + /** Guards the handle against a close racing a command. */ + private final Object lock = new Object(); + + /** The native player, or 0 once this player has been closed. */ + private long handle; + + /** Read on the native player's thread, so never a stale value. */ + private volatile MediaPlayerListener listener; + + + /** + * Creates a player for the given source, delivering into the given custom + * media sources. At least one of them has to be present; media of a kind + * with no source is decoded and dropped. + * + * @param reader The source to play, which this player takes over. + * @param videoSource Where video goes, or {@code null} for none. + * @param audioSource Where audio goes, or {@code null} for none. + * + * @throws IOException if the source cannot be decoded. + * @throws IllegalArgumentException if both sources are {@code null}. + */ + public MediaPlayer(MediaReader reader, CustomVideoSource videoSource, + CustomAudioSource audioSource) throws IOException { + Objects.requireNonNull(reader, "MediaReader is null"); + + if (videoSource == null && audioSource == null) { + throw new IllegalArgumentException("A player needs at least one media source"); + } + + // Taking the reader over rather than sharing it: the native player + // owns it from here, and a Java reader that still held the same + // pointer would release it a second time. + long readerHandle = reader.detach(); + + if (readerHandle == 0) { + throw new IOException("The reader is closed"); + } + + handle = create(readerHandle, NativeApi.tableAddress(), + videoSource != null ? NativeApi.handleOf(videoSource) : 0, + audioSource != null ? NativeApi.handleOf(audioSource) : 0); + } + + /** + * Sets what to report playback events to, replacing whatever was set + * before. A listener of {@code null} stops reporting. + * + * @param listener The listener, or {@code null}. + */ + public void setListener(MediaPlayerListener listener) { + this.listener = listener; + } + + /** + * Starts or resumes playback. A player that has reached the end starts + * over. Playing a player that is already playing does nothing. + */ + public void play() { + synchronized (lock) { + start(handle); + } + } + + /** + * Holds playback where it is. Resuming carries on from there rather than + * delivering everything that fell due in the meantime. + */ + public void pause() { + synchronized (lock) { + suspend(handle); + } + } + + /** + * Moves playback to the given position. Whatever has been decoded but not + * yet delivered is dropped. + * + * @param positionUs The position in microseconds from the start. + */ + public void seek(long positionUs) { + synchronized (lock) { + seek(handle, positionUs); + } + } + + /** + * Sets whether the source starts again when it runs out. A looping player + * keeps its timing across the seam, and never reports an end of stream. + * + * @param looping True to play the source over and over. + */ + public void setLooping(boolean looping) { + synchronized (lock) { + setLooping(handle, looping); + } + } + + /** + * Returns where playback has got to within the current pass, in + * microseconds. A looping source starts again from zero. + * + * @return The position in microseconds. + */ + public long getPositionUs() { + synchronized (lock) { + return position(handle); + } + } + + /** + * Returns what the player is currently doing. + * + * @return The player state. + */ + public MediaPlayerState getState() { + synchronized (lock) { + return MediaPlayerState.of(state(handle)); + } + } + + /** + * Stops playback, releases the native player and the reader it took over, + * and waits for the player's thread to finish. Closing a player that is + * already closed does nothing. + */ + @Override + public void close() { + long closing; + + synchronized (lock) { + closing = handle; + + // Cleared first, so that a command arriving from another thread + // finds nothing to act on rather than a handle being freed. + handle = 0; + } + + dispose(closing); + } + + /** Called by native code on the player's thread. */ + private void onNativeStateChanged(int state) { + MediaPlayerListener current = listener; + + if (current != null) { + current.onStateChanged(MediaPlayerState.of(state)); + } + } + + /** Called by native code on the player's thread. */ + private void onNativeEndOfStream() { + MediaPlayerListener current = listener; + + if (current != null) { + current.onEndOfStream(); + } + } + + /** Called by native code on the player's thread. */ + private void onNativeError(String message) { + MediaPlayerListener current = listener; + + if (current != null) { + current.onError(message); + } + } + + private native long create(long readerHandle, long tableAddress, + long videoSourceHandle, long audioSourceHandle) throws IOException; + + private static native void start(long handle); + + private static native void suspend(long handle); + + private static native void seek(long handle, long positionUs); + + private static native void setLooping(long handle, boolean looping); + + private static native long position(long handle); + + private static native int state(long handle); + + private static native void dispose(long handle); + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerListener.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerListener.java new file mode 100644 index 00000000..7e7ab7fe --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerListener.java @@ -0,0 +1,54 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +/** + * What a {@link MediaPlayer} reports while it runs. + *

+ * Calls arrive on the player's own thread, not on the thread that asked for + * playback, and that thread is the one decoding the media. An implementation + * must therefore return promptly and must not call back into the player in a + * way that waits for it. + * + * @author Alex Andres + */ +public interface MediaPlayerListener { + + /** + * The player moved to another state. + * + * @param state The state it moved to. + */ + default void onStateChanged(MediaPlayerState state) { + } + + /** + * The source ran out and everything decoded from it has been delivered. A + * looping player never reports this, since it starts over instead. + */ + default void onEndOfStream() { + } + + /** + * Playback stopped because something went wrong. + * + * @param message What went wrong, as FFmpeg described it. + */ + default void onError(String message) { + } + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerState.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerState.java new file mode 100644 index 00000000..682b7344 --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerState.java @@ -0,0 +1,66 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +/** + * What a {@link MediaPlayer} is currently doing. + * + * @author Alex Andres + */ +public enum MediaPlayerState { + + /** Ready to play, but not playing. */ + IDLE, + + /** Media is being decoded and delivered in real time. */ + PLAYING, + + /** + * Playback is held where it is. Resuming carries on from there rather + * than delivering everything that fell due in the meantime. + */ + PAUSED, + + /** + * The source ran out and everything decoded from it has been delivered. + * Playing again starts it over. A looping player never reaches this. + */ + ENDED, + + /** The player has been closed and cannot be used again. */ + CLOSED; + + + /** Cached, because values() hands out a fresh array on every call. */ + private static final MediaPlayerState[] VALUES = values(); + + + /** + * Returns the state native code reported. The order of the constants above + * is the order the native {@code MediaPlayerState} enum uses, so the two + * have to be changed together. + * + * @param value The native state value. + * + * @return The matching state, or {@link #CLOSED} for a value this version + * does not know. + */ + static MediaPlayerState of(int value) { + return value >= 0 && value < VALUES.length ? VALUES[value] : CLOSED; + } + +} diff --git a/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaReader.java b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaReader.java new file mode 100644 index 00000000..da50e429 --- /dev/null +++ b/webrtc-java-media/src/main/java/dev/onvoid/webrtc/media/ffmpeg/MediaReader.java @@ -0,0 +1,120 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +import java.io.IOException; +import java.nio.file.Path; + +/** + * An opened media source, and what it contains. + *

+ * A source is anything FFmpeg can demux: a media file today, and an http, rtsp + * or rtmp URL once those protocols are enabled in the build. Opening reads the + * container and picks the video and audio stream that are meant to be played; + * a source with neither fails to open. + *

+ * A reader holds a native resource and has to be closed. It is not safe to use + * from several threads at once, other than {@link #close()}, which may be + * called more than once and from any thread. + * + * @author Alex Andres + */ +public class MediaReader implements AutoCloseable { + + static { + FFmpeg.load(); + } + + /** The native reader, or 0 once this reader has been closed. */ + private long handle; + + + /** + * Opens the media file at the given path. + * + * @param path The path of the file to open. + * + * @throws IOException if the source cannot be opened, or holds nothing + * that can be played. + */ + public MediaReader(Path path) throws IOException { + this(path.toAbsolutePath().toString()); + } + + /** + * Opens the given media source. + * + * @param source The path or URL of the source to open. + * + * @throws IOException if the source cannot be opened, or holds nothing + * that can be played. + */ + public MediaReader(String source) throws IOException { + handle = open(source); + } + + /** + * Returns what this source contains: how long it runs, and the format of + * its video and audio streams. + * + * @return The media information. + * + * @throws IllegalStateException if this reader has been closed. + */ + public MediaInfo getInfo() { + return info(handle); + } + + /** + * Releases the native reader. Closing a reader that is already closed does + * nothing. + */ + @Override + public synchronized void close() { + long closing = handle; + + // Cleared first, so that a second close finds nothing to release even + // if the first one is still running. + handle = 0; + + dispose(closing); + } + + /** + * Hands the native reader over to a caller that takes responsibility for + * releasing it, and leaves this reader closed. This is how a + * {@link MediaPlayer} adopts a reader: two owners of the same pointer + * would release it twice. + * + * @return The native reader, or {@code 0} if it was already given away or + * closed. + */ + synchronized long detach() { + long detaching = handle; + + handle = 0; + + return detaching; + } + + private static native long open(String source) throws IOException; + + private static native MediaInfo info(long handle); + + private static native void dispose(long handle); + +} diff --git a/webrtc-java-media/src/main/java/module-info.java b/webrtc-java-media/src/main/java/module-info.java new file mode 100644 index 00000000..e48c075b --- /dev/null +++ b/webrtc-java-media/src/main/java/module-info.java @@ -0,0 +1,11 @@ +/** + * Media extension for webrtc-java: reads media files and network streams with + * FFmpeg and feeds them into a peer connection. + */ +module webrtc.java.media { + + requires webrtc.java; + + exports dev.onvoid.webrtc.media.ffmpeg; + +} diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaFileSourceTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaFileSourceTest.java new file mode 100644 index 00000000..45fc37ff --- /dev/null +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaFileSourceTest.java @@ -0,0 +1,150 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.media.audio.AudioDeviceModule; +import dev.onvoid.webrtc.media.audio.AudioLayer; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSink; +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests the turnkey way to send a media file, which is the thing the issue + * behind this module actually asked for. + * + * @author Alex Andres + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Execution(ExecutionMode.SAME_THREAD) +class MediaFileSourceTest { + + private static final String ASSET = "/media-test.webm"; + + private AudioDeviceModule audioModule; + private PeerConnectionFactory factory; + + + @BeforeAll + void initFactory() { + audioModule = new AudioDeviceModule(AudioLayer.kDummyAudio); + factory = new PeerConnectionFactory(audioModule); + } + + @AfterAll + void disposeFactory() { + factory.dispose(); + audioModule.dispose(); + } + + @Test + void exposesWhatTheFileHas() throws Exception { + try (MediaFileSource source = new MediaFileSource(asset())) { + assertTrue(source.getInfo().hasVideo()); + assertTrue(source.getInfo().hasAudio()); + + assertNotNull(source.getVideoSource()); + assertNotNull(source.getAudioSource()); + + assertEquals(MediaPlayerState.IDLE, source.getState()); + } + } + + @Test + void sendsTheFileThroughTracks() throws Exception { + AtomicInteger frames = new AtomicInteger(); + AtomicInteger chunks = new AtomicInteger(); + CountDownLatch ended = new CountDownLatch(1); + + try (MediaFileSource source = new MediaFileSource(asset())) { + VideoTrack videoTrack = factory.createVideoTrack("video", source.getVideoSource()); + AudioTrack audioTrack = factory.createAudioTrack("audio", source.getAudioSource()); + + VideoTrackSink videoSink = frame -> frames.incrementAndGet(); + AudioTrackSink audioSink = (data, bits, rate, ch, count) -> chunks.incrementAndGet(); + + videoTrack.addSink(videoSink); + audioTrack.addSink(audioSink); + + source.setListener(new MediaPlayerListener() { + + @Override + public void onEndOfStream() { + ended.countDown(); + } + }); + + source.play(); + + assertTrue(ended.await(15, TimeUnit.SECONDS), "no end of stream"); + + assertTrue(frames.get() > 40, "video frames: " + frames.get()); + assertTrue(chunks.get() > 290, "audio chunks: " + chunks.get()); + + videoTrack.removeSink(videoSink); + audioTrack.removeSink(audioSink); + + // The tracks go before the source, which disposes of what they + // were made from. + videoTrack.dispose(); + audioTrack.dispose(); + } + } + + @Test + void missingFileFails() { + assertThrows(IOException.class, () -> new MediaFileSource(Paths.get("no-such-file.webm"))); + } + + @Test + void closesTwice() throws Exception { + MediaFileSource source = new MediaFileSource(asset()); + + source.close(); + source.close(); + } + + private static Path asset() throws Exception { + URL url = MediaFileSourceTest.class.getResource(ASSET); + + assertNotNull(url, "Test asset " + ASSET + " is missing"); + + return Paths.get(url.toURI()); + } + +} diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerTest.java new file mode 100644 index 00000000..e0786de0 --- /dev/null +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaPlayerTest.java @@ -0,0 +1,338 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.concurrent.atomic.AtomicReference; + +import dev.onvoid.webrtc.PeerConnectionFactory; +import dev.onvoid.webrtc.media.audio.AudioDeviceModule; +import dev.onvoid.webrtc.media.audio.AudioLayer; +import dev.onvoid.webrtc.media.audio.AudioTrack; +import dev.onvoid.webrtc.media.audio.AudioTrackSink; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; +import dev.onvoid.webrtc.media.video.VideoTrack; +import dev.onvoid.webrtc.media.video.VideoTrackSink; + +import org.junit.jupiter.api.AfterAll; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.TestInstance; +import org.junit.jupiter.api.parallel.Execution; +import org.junit.jupiter.api.parallel.ExecutionMode; + +/** + * Tests playing a media source into custom media sources. The media is + * observed where an application would see it, on the sinks of the tracks made + * from those sources, so what these assert is what a peer connection would be + * sent. + *

+ * The asset runs three seconds and is played in real time, so these tests take + * about as long as the media they play. + * + * @author Alex Andres + */ +@TestInstance(TestInstance.Lifecycle.PER_CLASS) +@Execution(ExecutionMode.SAME_THREAD) +class MediaPlayerTest { + + /** + * The committed asset: 320x240 VP8 at 15 fps, 48 kHz mono Opus, 3.008 s, + * with a keyframe every second so that seeking has somewhere to land. + */ + private static final String ASSET = "/media-test.webm"; + + /** 3.008 seconds at 15 frames a second. */ + private static final int EXPECTED_FRAMES = 45; + + /** 3.008 seconds in chunks of 10 ms. */ + private static final int EXPECTED_CHUNKS = 300; + + private AudioDeviceModule audioModule; + private PeerConnectionFactory factory; + + + @BeforeAll + void initFactory() { + // A dummy audio layer, because this factory sends pushed audio and + // must not also be capturing from a device. + audioModule = new AudioDeviceModule(AudioLayer.kDummyAudio); + factory = new PeerConnectionFactory(audioModule); + } + + @AfterAll + void disposeFactory() { + factory.dispose(); + audioModule.dispose(); + } + + @Test + void playsToEnd() throws Exception { + try (Playback playback = new Playback()) { + playback.player.play(); + + assertTrue(playback.ended.await(15, TimeUnit.SECONDS), "no end of stream"); + + // Allowing one frame either way: which side of the last frame + // interval the file ends on is the container's business. + assertTrue(Math.abs(playback.frames.get() - EXPECTED_FRAMES) <= 1, + "video frames: " + playback.frames.get()); + assertTrue(Math.abs(playback.chunks.get() - EXPECTED_CHUNKS) <= 2, + "audio chunks: " + playback.chunks.get()); + + assertEquals(MediaPlayerState.ENDED, playback.player.getState()); + } + } + + @Test + void deliversWhatWebRtcTakes() throws Exception { + try (Playback playback = new Playback()) { + playback.player.play(); + + assertTrue(playback.ended.await(15, TimeUnit.SECONDS), "no end of stream"); + + assertEquals(320, playback.width.get()); + assertEquals(240, playback.height.get()); + + // The audio has to arrive as 10 ms of 48 kHz PCM whatever the + // source was, which for this asset means it was resampled. + assertEquals(48000, playback.sampleRate.get()); + assertEquals(1, playback.channels.get()); + assertEquals(480, playback.framesPerChunk.get()); + } + } + + @Test + void pacesInRealTime() throws Exception { + try (Playback playback = new Playback()) { + long started = System.nanoTime(); + + playback.player.play(); + + assertTrue(playback.ended.await(15, TimeUnit.SECONDS), "no end of stream"); + + long elapsedMs = (System.nanoTime() - started) / 1_000_000; + + // Three seconds of media takes three seconds to play. The upper + // bound is loose because a busy machine can only ever be late. + assertTrue(elapsedMs > 2500, "finished too early: " + elapsedMs + " ms"); + assertTrue(elapsedMs < 8000, "finished too late: " + elapsedMs + " ms"); + } + } + + @Test + void loopsWithoutEnding() throws Exception { + try (Playback playback = new Playback()) { + playback.player.setLooping(true); + playback.player.play(); + + // Well past the length of the asset, so a player that did not + // start over would have run out by now. + assertFalse(playback.ended.await(4500, TimeUnit.MILLISECONDS), + "a looping player reported an end of stream"); + + assertTrue(playback.frames.get() > EXPECTED_FRAMES, + "stopped at the end of the first pass: " + playback.frames.get()); + assertEquals(MediaPlayerState.PLAYING, playback.player.getState()); + } + } + + @Test + void pauseHoldsPlayback() throws Exception { + try (Playback playback = new Playback()) { + playback.player.play(); + + Thread.sleep(800); + + playback.player.pause(); + + assertEquals(MediaPlayerState.PAUSED, playback.player.getState()); + + // Whatever was in flight when the pause landed may still arrive, + // so the count is read once things have settled. + Thread.sleep(200); + + int atPause = playback.frames.get(); + + Thread.sleep(700); + + assertEquals(atPause, playback.frames.get(), "frames kept arriving while paused"); + } + } + + @Test + void seekMovesPlayback() throws Exception { + try (Playback playback = new Playback()) { + // A seek lands on the keyframe at or before the target, so asking + // for 2.5 s starts playback at the 2 s keyframe, leaving about a + // second to play out. + playback.player.seek(2_500_000); + playback.player.play(); + + assertTrue(playback.ended.await(15, TimeUnit.SECONDS), "no end of stream"); + + // A second of the asset is fifteen frames, well short of a pass. + assertTrue(playback.frames.get() < EXPECTED_FRAMES / 2, + "played more than the tail: " + playback.frames.get()); + assertTrue(playback.frames.get() > 0, "played nothing after the seek"); + } + } + + @Test + void needsAtLeastOneSource() throws Exception { + MediaReader reader = new MediaReader(asset()); + + assertThrows(IllegalArgumentException.class, + () -> new MediaPlayer(reader, null, null)); + + reader.close(); + } + + @Test + void closesTwice() throws Exception { + Playback playback = new Playback(); + + playback.close(); + playback.close(); + } + + @Test + void adoptedReaderIsClosed() throws Exception { + MediaReader reader = new MediaReader(asset()); + + try (Playback playback = new Playback(reader)) { + // The player took the reader over, so the Java one is spent and + // must not be usable any more. + assertThrows(IllegalStateException.class, reader::getInfo); + } + } + + private static Path asset() throws Exception { + URL url = MediaPlayerTest.class.getResource(ASSET); + + assertNotNull(url, "Test asset " + ASSET + " is missing"); + + return Paths.get(url.toURI()); + } + + /** + * One playback: the sources, the tracks made from them, the sinks that + * count what arrives, and the player feeding it all. + */ + private final class Playback implements AutoCloseable { + + final CustomVideoSource videoSource = new CustomVideoSource(); + final CustomAudioSource audioSource = new CustomAudioSource(); + + final VideoTrack videoTrack; + final AudioTrack audioTrack; + final MediaPlayer player; + + final CountDownLatch ended = new CountDownLatch(1); + final AtomicInteger frames = new AtomicInteger(); + final AtomicInteger chunks = new AtomicInteger(); + final AtomicInteger width = new AtomicInteger(); + final AtomicInteger height = new AtomicInteger(); + final AtomicInteger sampleRate = new AtomicInteger(); + final AtomicInteger channels = new AtomicInteger(); + final AtomicInteger framesPerChunk = new AtomicInteger(); + final AtomicReference error = new AtomicReference<>(); + + private final VideoTrackSink videoSink = frame -> { + frames.incrementAndGet(); + width.set(frame.buffer.getWidth()); + height.set(frame.buffer.getHeight()); + }; + + private final AudioTrackSink audioSink = (data, bits, rate, ch, count) -> { + chunks.incrementAndGet(); + sampleRate.set(rate); + channels.set(ch); + framesPerChunk.set(count); + }; + + private boolean closed; + + + Playback() throws Exception { + this(new MediaReader(asset())); + } + + Playback(MediaReader reader) throws Exception { + videoTrack = factory.createVideoTrack("video", videoSource); + audioTrack = factory.createAudioTrack("audio", audioSource); + + videoTrack.addSink(videoSink); + audioTrack.addSink(audioSink); + + player = new MediaPlayer(reader, videoSource, audioSource); + player.setListener(new MediaPlayerListener() { + + @Override + public void onEndOfStream() { + ended.countDown(); + } + + @Override + public void onError(String message) { + error.set(message); + ended.countDown(); + } + }); + } + + @Override + public void close() { + if (closed) { + return; + } + + closed = true; + + player.close(); + + videoTrack.removeSink(videoSink); + audioTrack.removeSink(audioSink); + + videoTrack.dispose(); + audioTrack.dispose(); + videoSource.dispose(); + audioSource.dispose(); + + assertNull(error.get()); + } + + private void assertNull(String message) { + if (message != null) { + throw new AssertionError("playback reported an error: " + message); + } + } + } + +} diff --git a/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaReaderTest.java b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaReaderTest.java new file mode 100644 index 00000000..b154b073 --- /dev/null +++ b/webrtc-java-media/src/test/java/dev/onvoid/webrtc/media/ffmpeg/MediaReaderTest.java @@ -0,0 +1,112 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.media.ffmpeg; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import java.io.IOException; +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.junit.jupiter.api.Test; + +/** + * Tests reading a media source. The asset is three seconds of VP8 and Opus in + * WebM, which is what a peer connection would be fed with. + * + * @author Alex Andres + */ +class MediaReaderTest { + + /** The committed test asset: 320x240 VP8 at 15 fps, 48 kHz mono Opus. */ + private static final String ASSET = "/media-test.webm"; + + + @Test + void readsVideoInfo() throws Exception { + try (MediaReader reader = new MediaReader(asset())) { + MediaInfo info = reader.getInfo(); + + assertTrue(info.hasVideo()); + assertEquals(320, info.getVideoWidth()); + assertEquals(240, info.getVideoHeight()); + assertEquals(15.0, info.getFrameRate(), 0.01); + assertEquals("vp8", info.getVideoCodec()); + } + } + + @Test + void readsAudioInfo() throws Exception { + try (MediaReader reader = new MediaReader(asset())) { + MediaInfo info = reader.getInfo(); + + assertTrue(info.hasAudio()); + assertEquals(48000, info.getSampleRate()); + assertEquals(1, info.getChannels()); + assertEquals("opus", info.getAudioCodec()); + } + } + + @Test + void readsDuration() throws Exception { + try (MediaReader reader = new MediaReader(asset())) { + // The asset runs 3.008 seconds. The tolerance is there because the + // duration comes from the container, and FFmpeg may be replaced. + assertEquals(3_008_000, reader.getInfo().getDurationUs(), 50_000); + } + } + + @Test + void missingSourceFails() { + IOException e = assertThrows(IOException.class, + () -> new MediaReader(Paths.get("no-such-file.webm"))); + + // The message has to name the source and what FFmpeg objected to, + // otherwise a wrong path is a guessing game. + assertTrue(e.getMessage().contains("no-such-file.webm"), e.getMessage()); + } + + @Test + void closesTwice() throws Exception { + MediaReader reader = new MediaReader(asset()); + + reader.close(); + reader.close(); + } + + @Test + void infoAfterCloseFails() throws Exception { + MediaReader reader = new MediaReader(asset()); + + reader.close(); + + assertThrows(IllegalStateException.class, reader::getInfo); + } + + private static Path asset() throws Exception { + URL url = MediaReaderTest.class.getResource(ASSET); + + assertNotNull(url, "Test asset " + ASSET + " is missing"); + + return Paths.get(url.toURI()); + } + +} diff --git a/webrtc-java-media/src/test/resources/media-test.webm b/webrtc-java-media/src/test/resources/media-test.webm new file mode 100644 index 00000000..fb16fd6a Binary files /dev/null and b/webrtc-java-media/src/test/resources/media-test.webm differ diff --git a/webrtc-java-media/third-party/ffmpeg b/webrtc-java-media/third-party/ffmpeg new file mode 160000 index 00000000..db69d06e --- /dev/null +++ b/webrtc-java-media/third-party/ffmpeg @@ -0,0 +1 @@ +Subproject commit db69d06eeeab4f46da15030a80d539efb4503ca8 diff --git a/webrtc-jni/src/main/cpp/include/JNI_CustomAudioSource.h b/webrtc-jni/src/main/cpp/include/JNI_CustomAudioSource.h index d68c37a4..1b430a03 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_CustomAudioSource.h +++ b/webrtc-jni/src/main/cpp/include/JNI_CustomAudioSource.h @@ -39,6 +39,14 @@ extern "C" { JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_pushAudioInternal (JNIEnv *, jobject, jbyteArray, jint, jint, jint, jint); + /* + * Class: dev_onvoid_webrtc_media_audio_CustomAudioSource + * Method: pushAudioTimestamped + * Signature: ([BIIIIJ)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_pushAudioTimestamped + (JNIEnv *, jobject, jbyteArray, jint, jint, jint, jint, jlong); + #ifdef __cplusplus } #endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_CustomVideoSource.h b/webrtc-jni/src/main/cpp/include/JNI_CustomVideoSource.h index 0b6de8a2..588982fb 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_CustomVideoSource.h +++ b/webrtc-jni/src/main/cpp/include/JNI_CustomVideoSource.h @@ -39,6 +39,14 @@ extern "C" { JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_pushFrame (JNIEnv *, jobject, jobject); + /* + * Class: dev_onvoid_webrtc_media_video_CustomVideoSource + * Method: pushFrameTimestamped + * Signature: (Ldev/onvoid/webrtc/media/video/VideoFrame;J)V + */ + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_pushFrameTimestamped + (JNIEnv *, jobject, jobject, jlong); + #ifdef __cplusplus } #endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_NativeApi.h b/webrtc-jni/src/main/cpp/include/JNI_NativeApi.h new file mode 100644 index 00000000..320b8851 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/JNI_NativeApi.h @@ -0,0 +1,44 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include +/* Header for class dev_onvoid_webrtc_internal_NativeApi */ + +#ifndef _Included_dev_onvoid_webrtc_internal_NativeApi +#define _Included_dev_onvoid_webrtc_internal_NativeApi +#ifdef __cplusplus +extern "C" { +#endif + /* + * Class: dev_onvoid_webrtc_internal_NativeApi + * Method: tableAddress + * Signature: ()J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_tableAddress + (JNIEnv *, jclass); + + /* + * Class: dev_onvoid_webrtc_internal_NativeApi + * Method: version + * Signature: ()I + */ + JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_version + (JNIEnv *, jclass); + +#ifdef __cplusplus +} +#endif +#endif diff --git a/webrtc-jni/src/main/cpp/include/JNI_SyncClock.h b/webrtc-jni/src/main/cpp/include/JNI_SyncClock.h index 04a62b4c..328ff3ed 100644 --- a/webrtc-jni/src/main/cpp/include/JNI_SyncClock.h +++ b/webrtc-jni/src/main/cpp/include/JNI_SyncClock.h @@ -47,6 +47,14 @@ extern "C" { JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_SyncClock_initialize (JNIEnv *, jobject); + /* + * Class: dev_onvoid_webrtc_media_SyncClock + * Method: currentTimeUs + * Signature: ()J + */ + JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_SyncClock_currentTimeUs + (JNIEnv *, jclass); + #ifdef __cplusplus } #endif diff --git a/webrtc-jni/src/main/cpp/include/api/ExtensionApi.h b/webrtc-jni/src/main/cpp/include/api/ExtensionApi.h new file mode 100644 index 00000000..f0843f22 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/api/ExtensionApi.h @@ -0,0 +1,33 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef JNI_WEBRTC_API_EXTENSION_API_H_ +#define JNI_WEBRTC_API_EXTENSION_API_H_ + +#include "webrtc_java_api.h" + +namespace jni +{ + // Returns the function table native extension libraries use to feed media + // into the custom media sources of this library. See webrtc_java_api.h for + // the interface itself and for how an extension gets hold of this address. + // + // The table is a singleton with static storage duration, so the returned + // pointer stays valid for the lifetime of the process. + const webrtc_java_api * GetExtensionApi(); +} + +#endif diff --git a/webrtc-jni/src/main/cpp/include/media/audio/CustomAudioSource.h b/webrtc-jni/src/main/cpp/include/media/audio/CustomAudioSource.h index 3a81c5de..89f7557f 100644 --- a/webrtc-jni/src/main/cpp/include/media/audio/CustomAudioSource.h +++ b/webrtc-jni/src/main/cpp/include/media/audio/CustomAudioSource.h @@ -42,15 +42,34 @@ namespace jni SourceState state() const override; bool remote() const override; - // Push audio data with synchronization + // Push audio data with synchronization. The chunk is captured now, + // so it is stamped with this source's clock less the configured + // capture delay. This is what the Java pushAudio() calls. void PushAudioData(const void * audio_data, int bits_per_sample, int sample_rate, size_t number_of_channels, size_t number_of_frames); + // Push audio data captured at the given time, in microseconds on + // the same clock as webrtc::TimeMicros(). The capture delay is not + // applied: a caller that knows when the audio was captured has + // already accounted for it. + void PushAudioData(const void * audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames, int64_t timestamp_us); + // Set audio capture delay for synchronization adjustment void SetAudioCaptureDelay(int64_t delay_us); private: + // Hands one chunk to every sink. Both push paths hold mutex_ for + // the whole delivery, as they always have, so a sink cannot be + // removed while it is being called. + void DeliverAudioDataLocked(const void * audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames, + int64_t absolute_capture_time_ms) + RTC_EXCLUSIVE_LOCKS_REQUIRED(mutex_); + // Guards sinks_ and audio_capture_delay_us_ against concurrent // AddSink()/RemoveSink() (called by WebRTC's own signaling/worker // thread as tracks attach/detach) racing with PushAudioData() and diff --git a/webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h b/webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h index 7b8c1055..7b7037fe 100644 --- a/webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h +++ b/webrtc-jni/src/main/cpp/include/media/video/CustomVideoSource.h @@ -40,9 +40,27 @@ namespace jni SourceState state() const override; bool remote() const override; + // Delivers a frame captured now, stamping it with this source's + // clock. This is what the Java pushFrame() calls, so a caller that + // has no capture time of its own gets the timing of the moment it + // pushes. void PushFrame(const webrtc::VideoFrame & frame); + // Delivers a frame captured at the given time, in microseconds on + // the same clock as webrtc::TimeMicros(). + // + // A caller that knows when a frame was meant to be shown, such as + // one playing a media file, should use this: the encoder derives + // the outgoing RTP timestamp from the frame's NTP capture time + // (VideoStreamEncoder::OnFrame), so passing the presentation time + // keeps the sent timeline free of the jitter of the pushing + // thread, and keeps video lined up with audio pushed alongside it. + void PushFrame(const webrtc::VideoFrame & frame, int64_t timestamp_us); + private: + void DeliverFrame(const webrtc::VideoFrame & frame, int64_t timestamp_us, + int64_t ntp_time_ms); + std::shared_ptr clock_; std::atomic frame_id_; }; diff --git a/webrtc-jni/src/main/cpp/include/webrtc_java_api.h b/webrtc-jni/src/main/cpp/include/webrtc_java_api.h new file mode 100644 index 00000000..ddb7ece4 --- /dev/null +++ b/webrtc-jni/src/main/cpp/include/webrtc_java_api.h @@ -0,0 +1,195 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#ifndef WEBRTC_JAVA_API_H_ +#define WEBRTC_JAVA_API_H_ + +/* + * The C interface a native extension library uses to feed media into + * webrtc-java's custom media sources, without linking against this library or + * against WebRTC, and without routing the media through Java. + * + * Deliberately plain C: no C++ types, no JNI types, no WebRTC types. An + * extension compiled against this header keeps working against any build of + * webrtc-java that reports the same interface version, whatever compiler or + * standard library either side was built with. + * + * How an extension obtains the table: + * + * 1. Java side: dev.onvoid.webrtc.internal.NativeApi.tableAddress() returns + * the address of a "struct webrtc_java_api" owned by this library, and + * NativeApi.handleOf(source) returns the native handle of a + * CustomVideoSource or CustomAudioSource. + * 2. The extension receives both as jlong values and casts them back. + * + * There is no link-time dependency between the two native libraries, so the + * extension does not care where this library was loaded from or under which + * file name, which matters because NativeLoader extracts it to a temporary + * file with a generated name. + * + * Versioning: the "version" field is the first member of the table and never + * moves. An extension must refuse to run when it reads a version it does not + * know. Within one version, members are only ever appended, never reordered + * or removed, and "size" tells an extension how much of the table the loaded + * library actually provides. + * + * Threading: every function may be called from any thread, including threads + * that are not attached to the JVM. The push functions deliver to WebRTC + * synchronously on the calling thread, so the caller must not hold locks that + * a WebRTC callback could need, and must pace its calls in real time: a frame + * is encoded and sent when it arrives, not when its timestamp says. + */ + +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif + +/** The interface version described by this header. */ +#define WEBRTC_JAVA_API_VERSION 1u + +/** Success, returned by every function that reports a status. */ +#define WEBRTC_JAVA_OK 0 +/** A handle was null, or a required pointer inside a struct was null. */ +#define WEBRTC_JAVA_ERR_HANDLE (-1) +/** A field of the pushed frame or chunk is out of range. */ +#define WEBRTC_JAVA_ERR_FORMAT (-2) + +/** + * Called when WebRTC has released its last reference to the pixel data of a + * pushed video frame, and the extension may reuse or free it. It runs on + * whichever thread drops that reference, which is usually an encoder thread + * and not the thread that pushed the frame, and it may run after the source + * itself was disposed. It must not block. + */ +typedef void (*wj_release_fn)(void * opaque); + +/** + * One decoded video frame in I420 (planar YUV 4:2:0, 8 bit). + * + * The planes are not copied. They must stay valid and unmodified until + * "release" is called. Passing a null "release" means the caller guarantees + * the planes outlive every use of the frame, which is rarely true; prefer + * handing over a reference and dropping it in the callback. + */ +struct wj_i420_frame { + /** Frame width in pixels, greater than zero. */ + int width; + /** Frame height in pixels, greater than zero. */ + int height; + + /** Luma plane, "height" rows of "stride_y" bytes. */ + const uint8_t * y; + /** Blue-difference chroma plane, (height + 1) / 2 rows of "stride_u". */ + const uint8_t * u; + /** Red-difference chroma plane, (height + 1) / 2 rows of "stride_v". */ + const uint8_t * v; + + /** Luma row stride in bytes, at least "width". */ + int stride_y; + /** Chroma row stride in bytes, at least (width + 1) / 2. */ + int stride_u; + /** Chroma row stride in bytes, at least (width + 1) / 2. */ + int stride_v; + + /** Clockwise rotation to apply on render: 0, 90, 180 or 270. */ + int rotation; + + /** + * Capture time in the clock of now_us(). Frames must be pushed in + * increasing timestamp order. A value of zero means "stamp it with the + * source's own clock", which is what the Java pushFrame() does. + */ + int64_t timestamp_us; + + /** Called once WebRTC is done with the planes; may be null. */ + wj_release_fn release; + /** Passed to "release" unchanged. */ + void * opaque; +}; + +/** + * One chunk of decoded audio: interleaved signed 16-bit PCM in host byte + * order, exactly 10 ms long, which is the only chunk length WebRTC accepts. + * + * The samples are copied before the call returns, so the caller may reuse the + * buffer immediately. + */ +struct wj_audio_chunk { + /** "frames" * "channels" samples, interleaved. */ + const int16_t * samples; + /** Sample rate in Hz, at most 48000. */ + int sample_rate; + /** Channel count, 1 or 2. */ + int channels; + /** Frames per channel, which must equal sample_rate / 100. */ + int frames; + + /** + * Capture time in the clock of now_us(), or zero to stamp the chunk with + * the source's own clock, which is what the Java pushAudio() does. + */ + int64_t timestamp_us; +}; + +/** + * The function table this library exposes to native extensions. It is a + * singleton with static storage duration, so its address stays valid for the + * lifetime of the process and needs no release. + */ +struct webrtc_java_api { + /** WEBRTC_JAVA_API_VERSION of the library providing this table. */ + uint32_t version; + /** sizeof(struct webrtc_java_api) as the library compiled it. */ + uint32_t size; + + /** + * The monotonic clock WebRTC itself uses, in microseconds. Timestamps in + * pushed frames and chunks are interpreted in this clock, so an extension + * that wants audio and video to line up on the receiver maps its own + * presentation times onto it once and keeps that mapping. + */ + int64_t (*now_us)(void); + + /** + * Delivers one video frame to a CustomVideoSource. + * + * @param source The handle from NativeApi.handleOf(CustomVideoSource). + * @param frame The frame to deliver; borrowed for the call only. + * + * @return WEBRTC_JAVA_OK, or a negative error code, in which case + * "release" is still called before returning. + */ + int (*video_source_push)(void * source, const struct wj_i420_frame * frame); + + /** + * Delivers one 10 ms audio chunk to a CustomAudioSource. + * + * @param source The handle from NativeApi.handleOf(CustomAudioSource). + * @param chunk The chunk to deliver; copied during the call. + * + * @return WEBRTC_JAVA_OK or a negative error code. + */ + int (*audio_source_push)(void * source, const struct wj_audio_chunk * chunk); +}; + +#ifdef __cplusplus +} +#endif + +#endif diff --git a/webrtc-jni/src/main/cpp/src/JNI_CustomAudioSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_CustomAudioSource.cpp index b9af2797..99ce8e1f 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_CustomAudioSource.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_CustomAudioSource.cpp @@ -73,4 +73,20 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_push env->ReleaseByteArrayElements(audioData, data, JNI_ABORT); } -} \ No newline at end of file +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_audio_CustomAudioSource_pushAudioTimestamped +(JNIEnv * env, jobject caller, jbyteArray audioData, jint bits_per_sample, jint sampleRate, jint channels, jint frameCount, jlong timestampUs) +{ + jni::CustomAudioSource * source = GetHandle(env, caller); + CHECK_HANDLE(source); + + // The caller validated the format, the frame count and the array length. + jbyte * data = env->GetByteArrayElements(audioData, nullptr); + + if (data != nullptr) { + source->PushAudioData(data, bits_per_sample, sampleRate, channels, frameCount, timestampUs); + + env->ReleaseByteArrayElements(audioData, data, JNI_ABORT); + } +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_CustomVideoSource.cpp b/webrtc-jni/src/main/cpp/src/JNI_CustomVideoSource.cpp index 78f87c2a..a00e7f2f 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_CustomVideoSource.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_CustomVideoSource.cpp @@ -72,4 +72,18 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_push source->PushFrame(nativeFrame); } -} \ No newline at end of file +} + +JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_video_CustomVideoSource_pushFrameTimestamped +(JNIEnv * env, jobject caller, jobject javaFrame, jlong timestampUs) +{ + jni::CustomVideoSource * source = GetHandle(env, caller); + CHECK_HANDLE(source); + + if (javaFrame != nullptr) { + auto frame = jni::JavaLocalRef(env, javaFrame); + webrtc::VideoFrame nativeFrame = jni::VideoFrame::toNative(env, frame); + + source->PushFrame(nativeFrame, timestampUs); + } +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp b/webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp new file mode 100644 index 00000000..6f0ad9e4 --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/JNI_NativeApi.cpp @@ -0,0 +1,31 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "JNI_NativeApi.h" + +#include "api/ExtensionApi.h" + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_tableAddress +(JNIEnv * env, jclass caller) +{ + return reinterpret_cast(jni::GetExtensionApi()); +} + +JNIEXPORT jint JNICALL Java_dev_onvoid_webrtc_internal_NativeApi_version +(JNIEnv * env, jclass caller) +{ + return static_cast(jni::GetExtensionApi()->version); +} diff --git a/webrtc-jni/src/main/cpp/src/JNI_SyncClock.cpp b/webrtc-jni/src/main/cpp/src/JNI_SyncClock.cpp index 284e76e5..faafdc51 100644 --- a/webrtc-jni/src/main/cpp/src/JNI_SyncClock.cpp +++ b/webrtc-jni/src/main/cpp/src/JNI_SyncClock.cpp @@ -18,6 +18,8 @@ #include "JavaUtils.h" #include "media/SyncClock.h" +#include "rtc_base/time_utils.h" + JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_SyncClock_initialize (JNIEnv * env, jobject caller) { @@ -63,4 +65,12 @@ JNIEXPORT void JNICALL Java_dev_onvoid_webrtc_media_SyncClock_dispose delete clock; SetHandle(env, caller, nullptr); -} \ No newline at end of file +} + +JNIEXPORT jlong JNICALL Java_dev_onvoid_webrtc_media_SyncClock_currentTimeUs +(JNIEnv * env, jclass caller) +{ + // The clock capture timestamps handed to the custom media sources are + // interpreted in, which is the one WebRTC itself runs on. + return webrtc::TimeMicros(); +} diff --git a/webrtc-jni/src/main/cpp/src/api/ExtensionApi.cpp b/webrtc-jni/src/main/cpp/src/api/ExtensionApi.cpp new file mode 100644 index 00000000..a8d0f72e --- /dev/null +++ b/webrtc-jni/src/main/cpp/src/api/ExtensionApi.cpp @@ -0,0 +1,169 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +#include "api/ExtensionApi.h" + +#include "media/audio/CustomAudioSource.h" +#include "media/video/CustomVideoSource.h" + +#include "api/video/video_frame.h" +#include "api/video/video_rotation.h" +#include "common_video/include/video_frame_buffer.h" +#include "rtc_base/time_utils.h" + +#include + +namespace jni +{ + namespace + { + int64_t ApiNowUs() + { + return webrtc::TimeMicros(); + } + + bool ToVideoRotation(int degrees, webrtc::VideoRotation & rotation) + { + switch (degrees) { + case 0: + rotation = webrtc::kVideoRotation_0; + return true; + case 90: + rotation = webrtc::kVideoRotation_90; + return true; + case 180: + rotation = webrtc::kVideoRotation_180; + return true; + case 270: + rotation = webrtc::kVideoRotation_270; + return true; + default: + return false; + } + } + + // Reports the frame as consumed without delivering it. The contract in + // webrtc_java_api.h is that the release callback runs whatever the + // outcome, so a rejected frame must not leak the caller's buffer. + int ReleaseAndFail(const wj_i420_frame * frame, int error) + { + if (frame != nullptr && frame->release != nullptr) { + frame->release(frame->opaque); + } + + return error; + } + + int ApiVideoSourcePush(void * source, const wj_i420_frame * frame) + { + if (frame == nullptr) { + return WEBRTC_JAVA_ERR_HANDLE; + } + if (source == nullptr || frame->y == nullptr || frame->u == nullptr || frame->v == nullptr) { + return ReleaseAndFail(frame, WEBRTC_JAVA_ERR_HANDLE); + } + + const int chromaWidth = (frame->width + 1) / 2; + + if (frame->width <= 0 || frame->height <= 0 + || frame->stride_y < frame->width + || frame->stride_u < chromaWidth + || frame->stride_v < chromaWidth) { + return ReleaseAndFail(frame, WEBRTC_JAVA_ERR_FORMAT); + } + + webrtc::VideoRotation rotation; + + if (!ToVideoRotation(frame->rotation, rotation)) { + return ReleaseAndFail(frame, WEBRTC_JAVA_ERR_FORMAT); + } + + // The planes are wrapped, not copied. WebRTC keeps the buffer alive + // for as long as it needs the pixels, which outlives this call, and + // runs the callback when it drops the last reference. + wj_release_fn release = frame->release; + void * opaque = frame->opaque; + + webrtc::scoped_refptr buffer = webrtc::WrapI420Buffer( + frame->width, frame->height, + frame->y, frame->stride_y, + frame->u, frame->stride_u, + frame->v, frame->stride_v, + [release, opaque]() { + if (release != nullptr) { + release(opaque); + } + }); + + webrtc::VideoFrame videoFrame = webrtc::VideoFrame::Builder() + .set_video_frame_buffer(buffer) + .set_rotation(rotation) + .build(); + + CustomVideoSource * videoSource = static_cast(source); + + if (frame->timestamp_us != 0) { + videoSource->PushFrame(videoFrame, frame->timestamp_us); + } + else { + videoSource->PushFrame(videoFrame); + } + + return WEBRTC_JAVA_OK; + } + + int ApiAudioSourcePush(void * source, const wj_audio_chunk * chunk) + { + if (source == nullptr || chunk == nullptr || chunk->samples == nullptr) { + return WEBRTC_JAVA_ERR_HANDLE; + } + if (chunk->sample_rate <= 0 || chunk->sample_rate > 48000 + || chunk->channels < 1 || chunk->channels > 2 + || chunk->frames <= 0 || chunk->frames * 100 != chunk->sample_rate) { + return WEBRTC_JAVA_ERR_FORMAT; + } + + CustomAudioSource * audioSource = static_cast(source); + + if (chunk->timestamp_us != 0) { + audioSource->PushAudioData(chunk->samples, 16, chunk->sample_rate, + static_cast(chunk->channels), + static_cast(chunk->frames), + chunk->timestamp_us); + } + else { + audioSource->PushAudioData(chunk->samples, 16, chunk->sample_rate, + static_cast(chunk->channels), + static_cast(chunk->frames)); + } + + return WEBRTC_JAVA_OK; + } + + const webrtc_java_api kExtensionApi = { + WEBRTC_JAVA_API_VERSION, + static_cast(sizeof(webrtc_java_api)), + &ApiNowUs, + &ApiVideoSourcePush, + &ApiAudioSourcePush + }; + } + + const webrtc_java_api * GetExtensionApi() + { + return &kExtensionApi; + } +} diff --git a/webrtc-jni/src/main/cpp/src/media/audio/CustomAudioSource.cpp b/webrtc-jni/src/main/cpp/src/media/audio/CustomAudioSource.cpp index b0b67aea..7c6f5f18 100644 --- a/webrtc-jni/src/main/cpp/src/media/audio/CustomAudioSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/audio/CustomAudioSource.cpp @@ -1,5 +1,7 @@ #include "media/audio/CustomAudioSource.h" +#include "rtc_base/time_utils.h" + #include #include @@ -58,17 +60,37 @@ namespace jni // Apply delay if audio capture has inherent latency timestamp_us -= audio_capture_delay_us_; - // Calculate NTP time for this audio frame - int64_t ntp_time_ms = clock_->GetNtpTime().ToMs(); + DeliverAudioDataLocked(audio_data, bits_per_sample, sample_rate, + number_of_channels, number_of_frames, + timestamp_us / 1000); + } + + void CustomAudioSource::PushAudioData(const void * audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames, int64_t timestamp_us) + { + webrtc::MutexLock lock(&mutex_); + + // The caller's capture time is on the same clock as TimeMicros(), so + // in milliseconds it is already the clock AudioTrackSinkInterface + // requires of an absolute capture timestamp. + DeliverAudioDataLocked(audio_data, bits_per_sample, sample_rate, + number_of_channels, number_of_frames, + timestamp_us / webrtc::kNumMicrosecsPerMillisec); + } - // Create absolute capture time - absl::optional absolute_capture_time_ms = timestamp_us / 1000; + void CustomAudioSource::DeliverAudioDataLocked(const void * audio_data, int bits_per_sample, + int sample_rate, size_t number_of_channels, + size_t number_of_frames, + int64_t absolute_capture_time_ms) + { + absl::optional capture_time_ms = absolute_capture_time_ms; // Send to all sinks with timing information for (auto * sink : sinks_) { sink->OnData(audio_data, bits_per_sample, sample_rate, number_of_channels, number_of_frames, - absolute_capture_time_ms); + capture_time_ms); } // Update total samples for tracking diff --git a/webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp b/webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp index db1f979a..20890223 100644 --- a/webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp +++ b/webrtc-jni/src/main/cpp/src/media/video/CustomVideoSource.cpp @@ -16,6 +16,8 @@ #include "media/video/CustomVideoSource.h" +#include "rtc_base/time_utils.h" + namespace jni { CustomVideoSource::CustomVideoSource(std::shared_ptr clock) : @@ -45,20 +47,38 @@ namespace jni } void CustomVideoSource::PushFrame(const webrtc::VideoFrame& frame) + { + // No capture time from the caller, so the frame is captured now. + DeliverFrame(frame, clock_->GetTimestampUs(), clock_->GetNtpTime().ToMs()); + } + + void CustomVideoSource::PushFrame(const webrtc::VideoFrame& frame, int64_t timestamp_us) + { + // The caller's capture time is on the same clock as TimeMicros(), so + // the frame's age gives its NTP capture time. That is the field the + // encoder turns into the outgoing RTP timestamp, which is why a + // caller's timing reaches the wire at all. + int64_t age_ms = (webrtc::TimeMicros() - timestamp_us) / webrtc::kNumMicrosecsPerMillisec; + + DeliverFrame(frame, timestamp_us, clock_->GetNtpTime().ToMs() - age_ms); + } + + void CustomVideoSource::DeliverFrame(const webrtc::VideoFrame& frame, int64_t timestamp_us, + int64_t ntp_time_ms) { // Create frame with proper timestamp webrtc::VideoFrame timestamped_frame = frame; - // Use synchronized clock for timestamp - int64_t timestamp_us = clock_->GetTimestampUs(); timestamped_frame.set_timestamp_us(timestamp_us); - // Set RTP timestamp (90kHz clock) + // Set RTP timestamp (90kHz clock). The encoder recomputes this from + // the NTP capture time below before sending, so it only matters to a + // sink that reads the frame before it is encoded. uint32_t rtp_timestamp = static_cast((timestamp_us * 90) / 1000); timestamped_frame.set_rtp_timestamp(rtp_timestamp); // Set NTP time for synchronization - timestamped_frame.set_ntp_time_ms(clock_->GetNtpTime().ToMs()); + timestamped_frame.set_ntp_time_ms(ntp_time_ms); // Increment frame ID timestamped_frame.set_id(frame_id_++); diff --git a/webrtc/pom.xml b/webrtc/pom.xml index da3251ed..4c64f822 100644 --- a/webrtc/pom.xml +++ b/webrtc/pom.xml @@ -18,7 +18,11 @@ org.apache.maven.plugins maven-surefire-plugin + + --add-exports webrtc.java/dev.onvoid.webrtc.internal=ALL-UNNAMED --add-opens webrtc.java/dev.onvoid.webrtc=ALL-UNNAMED --add-opens webrtc.java/dev.onvoid.webrtc.logging=ALL-UNNAMED --add-opens webrtc.java/dev.onvoid.webrtc.media=ALL-UNNAMED @@ -66,6 +70,7 @@ -Xcheck:jni + --add-exports webrtc.java/dev.onvoid.webrtc.internal=ALL-UNNAMED --add-opens webrtc.java/dev.onvoid.webrtc=ALL-UNNAMED --add-opens webrtc.java/dev.onvoid.webrtc.logging=ALL-UNNAMED --add-opens webrtc.java/dev.onvoid.webrtc.media=ALL-UNNAMED diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeApi.java b/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeApi.java new file mode 100644 index 00000000..b6cb7dd7 --- /dev/null +++ b/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeApi.java @@ -0,0 +1,91 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.internal; + +import java.util.Objects; + +/** + * The entry point a native extension library uses to reach this library's + * native side directly, without going through Java for every frame. + *

+ * An extension gets two addresses from here and passes both to its own native + * code: {@link #tableAddress()}, which is a {@code struct webrtc_java_api} of + * function pointers declared in {@code webrtc_java_api.h}, and + * {@link #handleOf(NativeObject)}, which is the native object behind a + * {@code CustomVideoSource} or {@code CustomAudioSource}. The extension then + * pushes decoded media straight into that source. + *

+ * Neither native library links against the other, which is what makes this + * work at all: {@link NativeLoader} extracts the library to a temporary file + * with a generated name, so there is nothing an extension could link to. + *

+ * This is not API for applications. It has no use outside a native extension, + * and an address used after the source it belongs to was disposed crashes the + * process. + * + * @author Alex Andres + */ +public final class NativeApi { + + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + + private NativeApi() { + // Static access only. + } + + /** + * Returns the address of this library's {@code struct webrtc_java_api}. + * The table has static storage duration, so the address stays valid for + * the lifetime of the process and needs no release. + * + * @return The address of the function table. + */ + public static native long tableAddress(); + + /** + * Returns the interface version of the function table. An extension built + * against a different version must not use the table. + * + * @return The value of {@code WEBRTC_JAVA_API_VERSION} in this library. + */ + public static native int version(); + + /** + * Returns the address of the native object bound to the given Java object, + * so that a native extension can act on it directly. + * + * @param object The object whose native counterpart is wanted. + * + * @return The native handle, or {@code 0} if the object was disposed. + * + * @throws NullPointerException If the object is {@code null}. + */ + public static long handleOf(NativeObject object) { + Objects.requireNonNull(object, "NativeObject is null"); + + return object.getNativeHandle(); + } + +} diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeLoader.java b/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeLoader.java index a46dffdc..c7077c6e 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeLoader.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/internal/NativeLoader.java @@ -89,6 +89,85 @@ public static void loadLibrary(final String libName) throws Exception { } } + /** + * Loads the specified native library together with the shared libraries it + * needs, which a native extension module packages next to it. + *

+ * The dependencies are loaded first, in the given order, so that by the + * time the library itself is loaded every symbol it imports is already in + * the process. They are extracted into one temporary directory under the + * exact names given, because a dynamic linker matches an already-loaded + * module by file name: a randomly named copy would be loaded a second + * time, or not found at all. + * + * @param libName The name of the library to load, without any + * platform specific prefix, file extension or path. + * @param dependencies The file names of the libraries to load first, + * exactly as they are named in the JAR, in the order + * they depend on each other. + * + * @throws Exception if one of the libraries could not be loaded. + */ + public static void loadLibrary(final String libName, final String... dependencies) + throws Exception { + if (LOADED_LIB_SET.contains(libName)) { + return; + } + + String libFileName = System.mapLibraryName( + libName + "-" + getOSFamily() + "-" + getOSArch()); + Path tempDir = Files.createTempDirectory(libName); + + tempDir.toFile().deleteOnExit(); + + for (String dependency : dependencies) { + loadFromDirectory(tempDir, dependency); + } + + loadFromDirectory(tempDir, libFileName); + + LOADED_LIB_SET.add(libName); + } + + /** + * Extracts one library from the JAR into the given directory, keeping its + * file name, and loads it. + * + * @param directory The directory to extract into. + * @param fileName The resource name of the library, which is also the + * name it is written under. + * + * @throws Exception if the library is not in the JAR or could not be + * loaded. + */ + private static void loadFromDirectory(Path directory, String fileName) + throws Exception { + Path libPath = directory.resolve(fileName); + + try (InputStream is = NativeLoader.class.getClassLoader() + .getResourceAsStream(fileName)) { + if (is == null) { + throw new UnsatisfiedLinkError( + "Native library '" + fileName + "' is not on the classpath"); + } + + Files.copy(is, libPath, StandardCopyOption.REPLACE_EXISTING); + } + + File libFile = libPath.toFile(); + + libFile.deleteOnExit(); + + try { + System.load(libPath.toAbsolutePath().toString()); + } + catch (Throwable e) { + libFile.delete(); + + throw e; + } + } + private static String getExtension(String fileName) { final int index = getExtensionIndex(fileName); diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/SyncClock.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/SyncClock.java index 790c1d47..e7ac0fb0 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/SyncClock.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/SyncClock.java @@ -16,6 +16,7 @@ package dev.onvoid.webrtc.media; +import dev.onvoid.webrtc.internal.NativeLoader; import dev.onvoid.webrtc.internal.NativeObject; /** @@ -26,6 +27,16 @@ */ public class SyncClock extends NativeObject { + static { + try { + NativeLoader.loadLibrary("webrtc-java"); + } + catch (Exception e) { + throw new RuntimeException("Load library 'webrtc-java' failed", e); + } + } + + /** * Constructs a new SyncClock instance. */ @@ -35,6 +46,26 @@ public SyncClock() { initialize(); } + /** + * Returns the current time of the media clock, in microseconds. + *

+ * This is the clock in which capture timestamps are interpreted, for + * example by {@link dev.onvoid.webrtc.media.video.CustomVideoSource#pushFrame( + * dev.onvoid.webrtc.media.video.VideoFrame, long) pushFrame} and + * {@link dev.onvoid.webrtc.media.audio.CustomAudioSource#pushAudio(byte[], + * int, int, int, int, long) pushAudio}. It is the monotonic clock WebRTC + * itself runs on, which is what makes those timestamps comparable with + * WebRTC's own notion of now. + *

+ * It is unrelated to the timestamps an instance of this class + * reports: {@link #getTimestampUs()} counts from the moment that instance + * was created, while this clock counts from an arbitrary but process-wide + * fixed point. + * + * @return The current time of the media clock in microseconds. + */ + public static native long currentTimeUs(); + /** * Get the current timestamp in microseconds. * diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/CustomAudioSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/CustomAudioSource.java index b4ab39df..cefc6f1c 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/CustomAudioSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/audio/CustomAudioSource.java @@ -94,6 +94,63 @@ public CustomAudioSource(SyncClock clock) { */ public void pushAudio(byte[] audioData, int bits_per_sample, int sampleRate, int channels, int frameCount) { + validate(audioData, bits_per_sample, sampleRate, channels, frameCount); + + pushAudioInternal(audioData, bits_per_sample, sampleRate, channels, + frameCount); + } + + /** + * Pushes audio data that was captured at the given time. Apart from the + * timestamp this behaves exactly like + * {@link #pushAudio(byte[], int, int, int, int)}. + *

+ * A source that knows its own timing, such as one playing a media file, + * should use this so that the audio lines up with video pushed with + * matching timestamps. Chunks must still be pushed in real time. + * + * @param audioData The raw audio data bytes to process. Must hold at + * least {@code frameCount * channels * 2} bytes. + * @param bits_per_sample The number of bits per sample, which must be 16. + * @param sampleRate The sample rate of the audio in Hz (e.g., 44100, 48000). + * @param channels The number of audio channels (1 for mono, 2 for stereo). + * @param frameCount The number of frames in the provided audio data. + * @param timestampUs The capture time of the chunk, on the clock of + * {@link dev.onvoid.webrtc.media.SyncClock#currentTimeUs()}. + * + * @throws NullPointerException If the audio data is {@code null}. + * @throws IllegalArgumentException If the audio format is not 16-bit PCM, + * if a value is not positive, if the chunk + * is larger than WebRTC can take, or if the + * array is too short for the frames it is + * said to hold. + * + * @see dev.onvoid.webrtc.media.SyncClock#currentTimeUs() + */ + public void pushAudio(byte[] audioData, int bits_per_sample, int sampleRate, + int channels, int frameCount, long timestampUs) { + validate(audioData, bits_per_sample, sampleRate, channels, frameCount); + + pushAudioTimestamped(audioData, bits_per_sample, sampleRate, channels, + frameCount, timestampUs); + } + + /** + * Checks that a chunk of audio is something WebRTC can take, so that a bad + * chunk is rejected here instead of reading past the end of the array or + * aborting the process in native code. + * + * @param audioData The raw audio data bytes to process. + * @param bits_per_sample The number of bits per sample, which must be 16. + * @param sampleRate The sample rate of the audio in Hz. + * @param channels The number of audio channels. + * @param frameCount The number of frames in the provided audio data. + * + * @throws NullPointerException If the audio data is {@code null}. + * @throws IllegalArgumentException If any value is out of range. + */ + private static void validate(byte[] audioData, int bits_per_sample, + int sampleRate, int channels, int frameCount) { requireNonNull(audioData, "audioData must not be null"); if (bits_per_sample != BITS_PER_SAMPLE) { @@ -134,9 +191,6 @@ public void pushAudio(byte[] audioData, int bits_per_sample, int sampleRate, "Audio data holds %d bytes, but %d frames of %d channels need %d", audioData.length, frameCount, channels, required)); } - - pushAudioInternal(audioData, bits_per_sample, sampleRate, channels, - frameCount); } /** @@ -159,6 +213,21 @@ private native void pushAudioInternal(byte[] audioData, int bits_per_sample, int sampleRate, int channels, int frameCount); + /** + * Hands the validated audio data to the native source, stamped with the + * given capture time. + * + * @param audioData The raw audio data bytes to process. + * @param bits_per_sample The number of bits per sample. + * @param sampleRate The sample rate of the audio in Hz. + * @param channels The number of audio channels. + * @param frameCount The number of frames in the provided audio data. + * @param timestampUs The capture time of the chunk in microseconds. + */ + private native void pushAudioTimestamped(byte[] audioData, int bits_per_sample, + int sampleRate, int channels, + int frameCount, long timestampUs); + /** * Initializes the native resources required by this audio source. */ diff --git a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/CustomVideoSource.java b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/CustomVideoSource.java index 083115cf..38ac57fc 100644 --- a/webrtc/src/main/java/dev/onvoid/webrtc/media/video/CustomVideoSource.java +++ b/webrtc/src/main/java/dev/onvoid/webrtc/media/video/CustomVideoSource.java @@ -47,12 +47,41 @@ public CustomVideoSource(SyncClock clock) { } /** - * Pushes audio data to be processed by this audio source. + * Pushes a video frame to be processed by this video source. The frame is + * treated as captured at the moment of the call. * * @param frame The video frame to be pushed to the source. */ public native void pushFrame(VideoFrame frame); + /** + * Pushes a video frame that was captured, or is meant to be shown, at the + * given time. + *

+ * The sent frame rate follows the capture timestamps, not the moments the + * frames are pushed, so a source that knows its own timing, such as one + * playing a video file, should use this. The receiver then sees the + * intended timing even when the pushing thread is late, and audio pushed + * with matching timestamps stays in sync with the video. + *

+ * Frames must still be pushed in real time, because a frame is encoded and + * sent when it arrives. Timestamps must increase by at least one + * millisecond from frame to frame; WebRTC drops a frame whose capture time + * does not advance. + * + * @param frame The video frame to be pushed to the source. + * @param timestampUs The capture time of the frame, on the clock of + * {@link dev.onvoid.webrtc.media.SyncClock#currentTimeUs()}. + * + * @see dev.onvoid.webrtc.media.SyncClock#currentTimeUs() + */ + public void pushFrame(VideoFrame frame, long timestampUs) { + // Not a native method itself: two native methods of the same name + // would have to carry JNI's mangled long names, which would rename the + // existing entry point for no gain. + pushFrameTimestamped(frame, timestampUs); + } + /** * Disposes of any native resources held by this video source. * This method should be called when the video source is no longer needed @@ -60,6 +89,15 @@ public CustomVideoSource(SyncClock clock) { */ public native void dispose(); + /** + * Hands the frame to the native source, stamped with the given capture + * time. + * + * @param frame The video frame to be pushed to the source. + * @param timestampUs The capture time of the frame in microseconds. + */ + private native void pushFrameTimestamped(VideoFrame frame, long timestampUs); + /** * Initializes the native resources required by this video source. */ diff --git a/webrtc/src/main/java/module-info.java b/webrtc/src/main/java/module-info.java index 6af21845..999476b6 100644 --- a/webrtc/src/main/java/module-info.java +++ b/webrtc/src/main/java/module-info.java @@ -9,4 +9,9 @@ exports dev.onvoid.webrtc.media.video; exports dev.onvoid.webrtc.media.video.desktop; + // Not API for applications. A native extension module, such as the FFmpeg + // based media module, needs NativeApi to reach the native side of a custom + // media source directly instead of carrying every frame through Java. + exports dev.onvoid.webrtc.internal to webrtc.java.media; + } \ No newline at end of file diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/internal/NativeApiTests.java b/webrtc/src/test/java/dev/onvoid/webrtc/internal/NativeApiTests.java new file mode 100644 index 00000000..e6a81d0e --- /dev/null +++ b/webrtc/src/test/java/dev/onvoid/webrtc/internal/NativeApiTests.java @@ -0,0 +1,100 @@ +/* + * Copyright 2026 Alex Andres + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package dev.onvoid.webrtc.internal; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertNotEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import dev.onvoid.webrtc.TestBase; +import dev.onvoid.webrtc.media.SyncClock; +import dev.onvoid.webrtc.media.audio.CustomAudioSource; +import dev.onvoid.webrtc.media.video.CustomVideoSource; + +import org.junit.jupiter.api.Test; + +/** + * Covers what a native extension library gets handed: the address of the + * function table it calls, and the addresses of the media sources it feeds. + * + * @author Alex Andres + */ +class NativeApiTests extends TestBase { + + @Test + void tableAddressIsStable() { + long first = NativeApi.tableAddress(); + + assertNotEquals(0L, first); + // The table has static storage duration, so an extension may hold the + // address for the lifetime of the process. + assertEquals(first, NativeApi.tableAddress()); + } + + @Test + void versionMatchesHeader() { + // An extension refuses to run against a version it does not know, so + // this must only ever change together with webrtc_java_api.h. + assertEquals(1, NativeApi.version()); + } + + @Test + void handleOfVideoSource() { + CustomVideoSource source = new CustomVideoSource(); + + assertNotEquals(0L, NativeApi.handleOf(source)); + + source.dispose(); + + // Disposal clears the handle, which is how an extension can tell that + // the source it was given is gone. + assertEquals(0L, NativeApi.handleOf(source)); + } + + @Test + void handleOfAudioSource() { + CustomAudioSource source = new CustomAudioSource(); + + assertNotEquals(0L, NativeApi.handleOf(source)); + + source.dispose(); + + assertEquals(0L, NativeApi.handleOf(source)); + } + + @Test + void handleOfNull() { + assertThrows(NullPointerException.class, () -> NativeApi.handleOf(null)); + } + + @Test + void mediaClockAdvances() throws Exception { + // The clock the table's now_us() reports and in which the push + // timestamps are interpreted. + long first = SyncClock.currentTimeUs(); + + Thread.sleep(20); + + long second = SyncClock.currentTimeUs(); + + assertTrue(second > first, "Media clock did not advance"); + assertTrue(second - first >= 10_000, + "Media clock advanced by " + (second - first) + " us over 20 ms"); + } + +} diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CustomAudioSourceTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CustomAudioSourceTest.java index 04497e9c..0aca70b6 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CustomAudioSourceTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/audio/CustomAudioSourceTest.java @@ -103,6 +103,26 @@ void pushAudioRejectsChunkLargerThanWebRtcTakes() { customAudioSource.pushAudio(new byte[maxFrames * 2 * 2], 16, 48000, 2, maxFrames); } + @Test + void pushAudioWithTimestamp() { + // The timestamped overload takes the same chunks and rejects the same + // bad arguments; only the capture time it reports differs. + byte[] data = new byte[480 * 2 * 2]; + long timestampUs = SyncClock.currentTimeUs(); + + customAudioSource.pushAudio(data, 16, 48000, 2, 480, timestampUs); + customAudioSource.pushAudio(data, 16, 48000, 2, 480, timestampUs + 10_000); + + assertThrows(NullPointerException.class, + () -> customAudioSource.pushAudio(null, 16, 48000, 2, 480, timestampUs)); + assertThrows(IllegalArgumentException.class, + () -> customAudioSource.pushAudio(data, 8, 48000, 2, 480, timestampUs)); + assertThrows(IllegalArgumentException.class, + () -> customAudioSource.pushAudio(data, 16, 48000, 2, 481, timestampUs)); + assertThrows(IllegalArgumentException.class, + () -> customAudioSource.pushAudio(data, 16, 48000, 0, 480, timestampUs)); + } + @Test void stateAfterCreation() { assertEquals(MediaSource.State.LIVE, customAudioSource.getState()); diff --git a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/CustomVideoSourceTest.java b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/CustomVideoSourceTest.java index 04e36cc3..42a80626 100644 --- a/webrtc/src/test/java/dev/onvoid/webrtc/media/video/CustomVideoSourceTest.java +++ b/webrtc/src/test/java/dev/onvoid/webrtc/media/video/CustomVideoSourceTest.java @@ -18,6 +18,8 @@ import static org.junit.jupiter.api.Assertions.*; +import java.util.ArrayList; +import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; import java.util.concurrent.atomic.AtomicInteger; @@ -99,6 +101,68 @@ void pushVideoFrameWithDifferentResolutions() { testVideoFrame(1920, 1080); // Full HD } + @Test + void pushFrameWithTimestamp() { + // A caller that knows its own timing, such as one playing a file, + // supplies the capture time and the frame carries it through unchanged + // instead of being stamped with the moment it was pushed. + VideoTrack videoTrack = factory.createVideoTrack("videoTrack", customVideoSource); + + final List timestamps = new ArrayList<>(); + + VideoTrackSink sink = frame -> timestamps.add(frame.timestampNs); + + videoTrack.addSink(sink); + + long baseUs = SyncClock.currentTimeUs(); + long frameUs = 1_000_000L / 25; + + for (int i = 0; i < 3; i++) { + NativeI420Buffer buffer = NativeI420Buffer.allocate(320, 240); + VideoFrame frame = new VideoFrame(buffer, 0); + + customVideoSource.pushFrame(frame, baseUs + i * frameUs); + + frame.release(); + } + + assertEquals(3, timestamps.size(), "Not every frame reached the sink"); + + for (int i = 0; i < 3; i++) { + assertEquals((baseUs + i * frameUs) * 1000L, timestamps.get(i), + "Frame " + i + " did not keep its capture time"); + } + + videoTrack.removeSink(sink); + videoTrack.dispose(); + } + + @Test + void pushFrameWithoutTimestampUsesSourceClock() { + // Without a capture time the source stamps the frame itself, which is + // the behaviour of pushFrame(VideoFrame) and must stay that way. + VideoTrack videoTrack = factory.createVideoTrack("videoTrack", customVideoSource); + + final List timestamps = new ArrayList<>(); + + VideoTrackSink sink = frame -> timestamps.add(frame.timestampNs); + + videoTrack.addSink(sink); + + NativeI420Buffer buffer = NativeI420Buffer.allocate(320, 240); + VideoFrame frame = new VideoFrame(buffer, 424242L); + + customVideoSource.pushFrame(frame); + frame.release(); + + assertEquals(1, timestamps.size()); + assertNotEquals(424242L, timestamps.get(0), + "Frame kept the timestamp it was constructed with"); + + videoTrack.removeSink(sink); + videoTrack.dispose(); + } + @Test void constructWithSyncClock() { // Create a SyncClock.