LibMedia: Add separate classes managing decoding and displaying video

These are unused in this commit, but will later be used to output video
via PlaybackManager, or to decode video directly to some consumer.

Libraries/LibMedia/CMakeLists.txt

@@ -11,6 +11,8 @@ set(SOURCES
     Containers/Matroska/MatroskaDemuxer.cpp
     Containers/Matroska/Reader.cpp
     PlaybackManager.cpp
+    Providers/VideoDataProvider.cpp
+    Sinks/DisplayingVideoSink.cpp
     VideoFrame.cpp
 )
 

Libraries/LibMedia/Forward.h

@@ -1,5 +1,6 @@
 /*
  * Copyright (c) 2023, Tim Flynn <trflynn89@serenityos.org>
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
  *
  * SPDX-License-Identifier: BSD-2-Clause
  */
@@ -10,10 +11,16 @@ namespace Media {
 
 class CodedFrame;
 class DecoderError;
+class Demuxer;
+class DisplayingVideoSink;
 class FrameQueueItem;
+class MediaTimeProvider;
+class MutexedDemuxer;
 class PlaybackManager;
 class Track;
+class VideoDataProvider;
 class VideoDecoder;
 class VideoFrame;
+class VideoSink;
 
 }

Libraries/LibMedia/MutexedDemuxer.h

@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2022, Gregory Bertilson <zaggy1024@gmail.com>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Forward.h>
+#include <LibMedia/DecoderError.h>
+#include <LibThreading/MutexProtected.h>
+
+#include "Demuxer.h"
+
+namespace Media {
+
+class MutexedDemuxer final : public Demuxer {
+public:
+    MutexedDemuxer(NonnullRefPtr<Demuxer> demuxer)
+        : m_demuxer(move(demuxer))
+    {
+    }
+
+    virtual ~MutexedDemuxer() override
+    {
+        m_demuxer.with_locked([](auto& demuxer) {
+            auto to_destroy = move(demuxer);
+        });
+    }
+
+    virtual DecoderErrorOr<Vector<Track>> get_tracks_for_type(TrackType type) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_tracks_for_type(type);
+        });
+    }
+    virtual DecoderErrorOr<Optional<Track>> get_preferred_track_for_type(TrackType type) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_preferred_track_for_type(type);
+        });
+    }
+
+    virtual DecoderErrorOr<CodedFrame> get_next_sample_for_track(Track track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_next_sample_for_track(track);
+        });
+    }
+
+    virtual DecoderErrorOr<CodecID> get_codec_id_for_track(Track track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_codec_id_for_track(track);
+        });
+    }
+
+    virtual DecoderErrorOr<ReadonlyBytes> get_codec_initialization_data_for_track(Track track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->get_codec_initialization_data_for_track(track);
+        });
+    }
+
+    virtual DecoderErrorOr<Optional<AK::Duration>> seek_to_most_recent_keyframe(Track track, AK::Duration timestamp, Optional<AK::Duration> earliest_available_sample = {}) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->seek_to_most_recent_keyframe(track, timestamp, earliest_available_sample);
+        });
+    }
+
+    virtual DecoderErrorOr<AK::Duration> duration_of_track(Track const& track) override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->duration_of_track(track);
+        });
+    }
+    virtual DecoderErrorOr<AK::Duration> total_duration() override
+    {
+        return m_demuxer.with_locked([&](auto& demuxer) {
+            return demuxer->total_duration();
+        });
+    }
+
+private:
+    Threading::MutexProtected<NonnullRefPtr<Demuxer>> m_demuxer;
+};
+
+}

Libraries/LibMedia/Providers/MediaTimeProvider.h

@@ -0,0 +1,21 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/AtomicRefCounted.h>
+#include <AK/Time.h>
+
+namespace Media {
+
+class MediaTimeProvider : public AtomicRefCounted<MediaTimeProvider> {
+public:
+    virtual ~MediaTimeProvider() = default;
+
+    virtual AK::Duration current_time() const = 0;
+};
+
+}

Libraries/LibMedia/Providers/VideoDataProvider.cpp

@@ -0,0 +1,208 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibCore/EventLoop.h>
+#include <LibMedia/FFmpeg/FFmpegVideoDecoder.h>
+#include <LibMedia/MutexedDemuxer.h>
+#include <LibMedia/Providers/MediaTimeProvider.h>
+#include <LibMedia/Sinks/VideoSink.h>
+#include <LibMedia/VideoDecoder.h>
+#include <LibMedia/VideoFrame.h>
+#include <LibThreading/Thread.h>
+
+#include "VideoDataProvider.h"
+
+namespace Media {
+
+DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> VideoDataProvider::try_create(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, RefPtr<MediaTimeProvider> const& time_provider)
+{
+    auto codec_id = TRY(demuxer->get_codec_id_for_track(track));
+    auto codec_initialization_data = TRY(demuxer->get_codec_initialization_data_for_track(track));
+    auto decoder = DECODER_TRY_ALLOC(FFmpeg::FFmpegVideoDecoder::try_create(codec_id, codec_initialization_data));
+
+    auto thread_data = DECODER_TRY_ALLOC(try_make_ref_counted<VideoDataProvider::ThreadData>(demuxer, track, move(decoder), time_provider));
+    auto provider = DECODER_TRY_ALLOC(try_make_ref_counted<VideoDataProvider>(thread_data));
+
+    auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int {
+        while (!thread_data->should_thread_exit())
+            thread_data->push_data_and_decode_some_frames();
+        return 0;
+    }));
+    thread->start();
+    thread->detach();
+
+    return provider;
+}
+
+DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> VideoDataProvider::try_create(NonnullRefPtr<Demuxer> const& demuxer, Track const& track, RefPtr<MediaTimeProvider> const& time_provider)
+{
+    auto mutexed_demuxer = DECODER_TRY_ALLOC(try_make_ref_counted<MutexedDemuxer>(demuxer));
+    return try_create(mutexed_demuxer, track, time_provider);
+}
+
+VideoDataProvider::VideoDataProvider(NonnullRefPtr<ThreadData> const& thread_state)
+    : m_thread_data(thread_state)
+{
+}
+
+VideoDataProvider::~VideoDataProvider()
+{
+    m_thread_data->exit();
+}
+
+void VideoDataProvider::set_error_handler(ErrorHandler&& handler)
+{
+    m_thread_data->set_error_handler(move(handler));
+}
+
+TimedImage VideoDataProvider::retrieve_frame()
+{
+    auto locker = m_thread_data->take_lock();
+    if (m_thread_data->queue().is_empty())
+        return TimedImage();
+    auto result = m_thread_data->queue().dequeue();
+    m_thread_data->wake();
+    return result;
+}
+
+void VideoDataProvider::seek(AK::Duration timestamp)
+{
+    m_thread_data->seek(timestamp);
+}
+
+VideoDataProvider::ThreadData::ThreadData(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, NonnullOwnPtr<VideoDecoder>&& decoder, RefPtr<MediaTimeProvider> const& time_provider)
+    : m_main_thread_event_loop(Core::EventLoop::current())
+    , m_demuxer(demuxer)
+    , m_track(track)
+    , m_decoder(move(decoder))
+    , m_time_provider(time_provider)
+{
+}
+
+VideoDataProvider::ThreadData::~ThreadData() = default;
+
+void VideoDataProvider::ThreadData::set_error_handler(ErrorHandler&& handler)
+{
+    auto locker = take_lock();
+    m_error_handler = move(handler);
+    m_wait_condition.broadcast();
+}
+
+void VideoDataProvider::ThreadData::exit()
+{
+    m_exit = true;
+    m_wait_condition.broadcast();
+}
+
+VideoDataProvider::ImageQueue& VideoDataProvider::ThreadData::queue()
+{
+    return m_queue;
+}
+
+void VideoDataProvider::ThreadData::seek(AK::Duration timestamp)
+{
+    auto seek_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
+    if (seek_result.is_error()) {
+        m_error_handler(seek_result.release_error());
+    } else {
+        auto locker = take_lock();
+        m_is_in_error_state = false;
+        m_wait_condition.broadcast();
+    }
+}
+
+bool VideoDataProvider::ThreadData::should_thread_exit() const
+{
+    return m_exit;
+}
+
+void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
+{
+    // FIXME: Check if the PlaybackManager's current time is ahead of the next keyframe, and seek to it if so.
+    //        Demuxers currently can't report the next keyframe in a convenient way, so that will need implementing
+    //        before this functionality can exist.
+
+    auto set_error_and_wait_for_seek = [this](DecoderError&& error) {
+        auto locker = take_lock();
+        m_is_in_error_state = true;
+        while (!m_error_handler)
+            m_wait_condition.wait();
+        m_main_thread_event_loop.deferred_invoke([this, error = move(error)] mutable {
+            m_error_handler(move(error));
+        });
+        dbgln_if(PLAYBACK_MANAGER_DEBUG, "Video Data Provider: Encountered an error, waiting for a seek to start decoding again...");
+        while (m_is_in_error_state)
+            m_wait_condition.wait();
+    };
+
+    auto sample_result = m_demuxer->get_next_sample_for_track(m_track);
+    if (sample_result.is_error()) {
+        // FIXME: Handle the end of the stream.
+        set_error_and_wait_for_seek(sample_result.release_error());
+        return;
+    }
+
+    auto coded_frame = sample_result.release_value();
+    auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data());
+    if (decode_result.is_error()) {
+        set_error_and_wait_for_seek(decode_result.release_error());
+        return;
+    }
+
+    while (true) {
+        auto frame_result = m_decoder->get_decoded_frame();
+        if (frame_result.is_error()) {
+            if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
+                break;
+            set_error_and_wait_for_seek(frame_result.release_error());
+            break;
+        }
+
+        auto frame = frame_result.release_value();
+
+        // Convert the frame for display.
+        auto& cicp = frame->cicp();
+        auto container_cicp = coded_frame.auxiliary_data().get<CodedVideoFrameData>().container_cicp();
+        cicp.adopt_specified_values(container_cicp);
+        cicp.default_code_points_if_unspecified({ ColorPrimaries::BT709, TransferCharacteristics::BT709, MatrixCoefficients::BT709, VideoFullRangeFlag::Studio });
+
+        // BT.470 M, B/G, BT.601, BT.709 and BT.2020 have a similar transfer function to sRGB, so other applications
+        // (Chromium, VLC) forgo transfer characteristics conversion. We will emulate that behavior by
+        // handling those as sRGB instead, which causes no transfer function change in the output,
+        // unless display color management is later implemented.
+        switch (cicp.transfer_characteristics()) {
+        case TransferCharacteristics::BT470BG:
+        case TransferCharacteristics::BT470M:
+        case TransferCharacteristics::BT601:
+        case TransferCharacteristics::BT709:
+        case TransferCharacteristics::BT2020BitDepth10:
+        case TransferCharacteristics::BT2020BitDepth12:
+            cicp.set_transfer_characteristics(TransferCharacteristics::SRGB);
+            break;
+        default:
+            break;
+        }
+
+        auto bitmap_result = frame->to_bitmap();
+
+        if (bitmap_result.is_error()) {
+            set_error_and_wait_for_seek(bitmap_result.release_error());
+            return;
+        }
+
+        {
+            auto locker = take_lock();
+            while (m_queue.size() >= m_queue_max_size) {
+                m_wait_condition.wait();
+                if (should_thread_exit())
+                    return;
+            }
+            m_queue.enqueue(TimedImage(frame->timestamp(), bitmap_result.release_value()));
+        }
+    }
+}
+
+}

Libraries/LibMedia/Providers/VideoDataProvider.h

@@ -0,0 +1,91 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/Atomic.h>
+#include <AK/AtomicRefCounted.h>
+#include <AK/Forward.h>
+#include <AK/NonnullRefPtr.h>
+#include <AK/Queue.h>
+#include <AK/Time.h>
+#include <LibCore/Forward.h>
+#include <LibGfx/Bitmap.h>
+#include <LibMedia/DecoderError.h>
+#include <LibMedia/Export.h>
+#include <LibMedia/Forward.h>
+#include <LibMedia/TimedImage.h>
+#include <LibMedia/Track.h>
+#include <LibThreading/ConditionVariable.h>
+#include <LibThreading/Mutex.h>
+
+namespace Media {
+
+// Retrieves coded data from a demuxer and decodes it asynchronously into video frames ready for display.
+class MEDIA_API VideoDataProvider final : public AtomicRefCounted<VideoDataProvider> {
+    class ThreadData;
+
+public:
+    static constexpr size_t QUEUE_CAPACITY = 8;
+    using ImageQueue = Queue<TimedImage, QUEUE_CAPACITY>;
+
+    using ErrorHandler = Function<void(DecoderError&&)>;
+
+    static DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> try_create(NonnullRefPtr<MutexedDemuxer> const&, Track const&, RefPtr<MediaTimeProvider> const& = nullptr);
+    static DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> try_create(NonnullRefPtr<Demuxer> const&, Track const&, RefPtr<MediaTimeProvider> const& = nullptr);
+
+    VideoDataProvider(NonnullRefPtr<ThreadData> const&);
+    ~VideoDataProvider();
+
+    void set_error_handler(ErrorHandler&&);
+
+    TimedImage retrieve_frame();
+
+    void seek(AK::Duration timestamp);
+
+private:
+    class ThreadData final : public AtomicRefCounted<ThreadData> {
+    public:
+        ThreadData(NonnullRefPtr<MutexedDemuxer> const&, Track const&, NonnullOwnPtr<VideoDecoder>&&, RefPtr<MediaTimeProvider> const&);
+        ~ThreadData();
+
+        void set_error_handler(ErrorHandler&&);
+
+        void exit();
+
+        ImageQueue& queue();
+
+        void seek(AK::Duration timestamp);
+
+        bool should_thread_exit() const;
+        void push_data_and_decode_some_frames();
+
+        [[nodiscard]] Threading::MutexLocker take_lock() { return Threading::MutexLocker(m_mutex); }
+        void wake() { m_wait_condition.broadcast(); }
+
+    private:
+        Core::EventLoop& m_main_thread_event_loop;
+
+        Threading::Mutex m_mutex;
+        Threading::ConditionVariable m_wait_condition { m_mutex };
+        Atomic<bool> m_exit { false };
+
+        NonnullRefPtr<MutexedDemuxer> m_demuxer;
+        Track m_track;
+        NonnullOwnPtr<VideoDecoder> m_decoder;
+
+        RefPtr<MediaTimeProvider> m_time_provider;
+
+        size_t m_queue_max_size { 4 };
+        ImageQueue m_queue;
+        ErrorHandler m_error_handler;
+        bool m_is_in_error_state { false };
+    };
+
+    NonnullRefPtr<ThreadData> m_thread_data;
+};
+
+}

Libraries/LibMedia/Sinks/DisplayingVideoSink.cpp

@@ -0,0 +1,71 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#include <LibMedia/Demuxer.h>
+#include <LibMedia/Providers/MediaTimeProvider.h>
+#include <LibMedia/Providers/VideoDataProvider.h>
+
+#include "DisplayingVideoSink.h"
+
+namespace Media {
+
+ErrorOr<NonnullRefPtr<DisplayingVideoSink>> DisplayingVideoSink::try_create(NonnullRefPtr<MediaTimeProvider> const& time_provider)
+{
+    return TRY(try_make_ref_counted<DisplayingVideoSink>(time_provider));
+}
+
+DisplayingVideoSink::DisplayingVideoSink(NonnullRefPtr<MediaTimeProvider> const& time_provider)
+    : m_time_provider(time_provider)
+{
+}
+
+DisplayingVideoSink::~DisplayingVideoSink() = default;
+
+void DisplayingVideoSink::verify_track(Track const& track) const
+{
+    if (m_provider == nullptr)
+        return;
+    VERIFY(m_track.has_value());
+    VERIFY(m_track.value() == track);
+}
+
+void DisplayingVideoSink::set_provider(Track const& track, RefPtr<VideoDataProvider> const& provider)
+{
+    verify_track(track);
+    m_track = track;
+    m_provider = provider;
+}
+
+RefPtr<VideoDataProvider> DisplayingVideoSink::provider(Track const& track) const
+{
+    verify_track(track);
+    return m_provider;
+}
+
+DisplayingVideoSinkUpdateResult DisplayingVideoSink::update()
+{
+    auto current_time = m_time_provider->current_time();
+    auto result = DisplayingVideoSinkUpdateResult::NoChange;
+    while (true) {
+        if (!m_next_frame.is_valid()) {
+            m_next_frame = m_provider->retrieve_frame();
+            if (!m_next_frame.is_valid())
+                break;
+        }
+        if (m_next_frame.timestamp() > current_time)
+            break;
+        m_current_frame = m_next_frame.release_image();
+        result = DisplayingVideoSinkUpdateResult::NewFrameAvailable;
+    }
+    return result;
+}
+
+RefPtr<Gfx::Bitmap> DisplayingVideoSink::current_frame()
+{
+    return m_current_frame;
+}
+
+}

Libraries/LibMedia/Sinks/DisplayingVideoSink.h

@@ -0,0 +1,54 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/NonnullRefPtr.h>
+#include <AK/Time.h>
+#include <LibGfx/Bitmap.h>
+#include <LibMedia/Export.h>
+#include <LibMedia/Forward.h>
+#include <LibMedia/Sinks/VideoSink.h>
+#include <LibMedia/TimedImage.h>
+#include <LibMedia/Track.h>
+
+namespace Media {
+
+enum class DisplayingVideoSinkUpdateResult : u8 {
+    NewFrameAvailable,
+    NoChange,
+};
+
+class MEDIA_API DisplayingVideoSink final : public VideoSink {
+public:
+    static ErrorOr<NonnullRefPtr<DisplayingVideoSink>> try_create(NonnullRefPtr<MediaTimeProvider> const&);
+
+    DisplayingVideoSink(NonnullRefPtr<MediaTimeProvider> const&);
+    virtual ~DisplayingVideoSink() override;
+
+    virtual void set_provider(Track const&, RefPtr<VideoDataProvider> const&) override;
+    RefPtr<VideoDataProvider> provider(Track const&) const override;
+
+    // Updates the frame returned by current_frame() based on the time provider's current timestamp.
+    //
+    // Note that push_frame may block until update() is called, so do not call them from the same thread.
+    DisplayingVideoSinkUpdateResult update();
+    RefPtr<Gfx::Bitmap> current_frame();
+
+private:
+    static constexpr size_t DEFAULT_QUEUE_SIZE = 8;
+
+    void verify_track(Track const&) const;
+
+    NonnullRefPtr<MediaTimeProvider> m_time_provider;
+    RefPtr<VideoDataProvider> m_provider;
+    Optional<Track> m_track;
+
+    TimedImage m_next_frame;
+    RefPtr<Gfx::Bitmap> m_current_frame;
+};
+
+}

Libraries/LibMedia/Sinks/VideoSink.h

@@ -0,0 +1,24 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/AtomicRefCounted.h>
+#include <AK/RefPtr.h>
+#include <LibMedia/Forward.h>
+
+namespace Media {
+
+// A consumer to be attached to a VideoDataProvider in order to receive video frames from a decoding thread.
+class VideoSink : public AtomicRefCounted<VideoSink> {
+public:
+    virtual ~VideoSink() = default;
+
+    virtual void set_provider(Track const&, RefPtr<VideoDataProvider> const&) = 0;
+    virtual RefPtr<VideoDataProvider> provider(Track const&) const = 0;
+};
+
+}

Libraries/LibMedia/TimedImage.h

@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
+ *
+ * SPDX-License-Identifier: BSD-2-Clause
+ */
+
+#pragma once
+
+#include <AK/NonnullRefPtr.h>
+#include <AK/RefPtr.h>
+#include <AK/Time.h>
+#include <LibGfx/Forward.h>
+
+namespace Media {
+
+class TimedImage final {
+public:
+    TimedImage(AK::Duration timestamp, NonnullRefPtr<Gfx::Bitmap>&& image)
+        : m_timestamp(timestamp)
+        , m_image(move(image))
+    {
+    }
+    TimedImage() = default;
+
+    bool is_valid() const
+    {
+        return m_image != nullptr;
+    }
+    AK::Duration const& timestamp() const
+    {
+        VERIFY(is_valid());
+        return m_timestamp;
+    }
+    NonnullRefPtr<Gfx::Bitmap> image() const
+    {
+        VERIFY(is_valid());
+        return *m_image;
+    }
+    NonnullRefPtr<Gfx::Bitmap> release_image()
+    {
+        VERIFY(is_valid());
+        m_timestamp = AK::Duration::zero();
+        return m_image.release_nonnull();
+    }
+
+private:
+    AK::Duration m_timestamp { AK::Duration::zero() };
+    RefPtr<Gfx::Bitmap> m_image { nullptr };
+};
+
+}

Meta/gn/secondary/Userland/Libraries/LibMedia/BUILD.gn

@@ -13,6 +13,8 @@ shared_library("LibMedia") {
     "Containers/Matroska/MatroskaDemuxer.cpp",
     "Containers/Matroska/Reader.cpp",
     "PlaybackManager.cpp",
+    "Providers/VideoDataProvider.cpp",
+    "Sinks/DisplayingVideoSink.cpp",
     "VideoFrame.cpp",
   ]
   if (enable_pulseaudio) {