LibMedia: Implement media seeking

This implementation allows:
- Accurate seeking to an exact timestamp
- Seeking to the keyframe before a timestamp
- Seeking to the keyframe after a timestamp
These three options will be used to satisfy the playback position
selection in the media element's seeking steps.
This commit is contained in:
Zaggy1024
2025-10-02 18:58:11 -05:00
committed by Jelle Raaijmakers
parent b1c9a872bc
commit ccf4b3f6e9
Notes: github-actions[bot] 2025-10-28 00:33:01 +00:00
18 changed files with 682 additions and 32 deletions

View File

@@ -11,6 +11,7 @@ set(SOURCES
Containers/Matroska/MatroskaDemuxer.cpp
Containers/Matroska/Reader.cpp
PlaybackManager.cpp
PlaybackStates/PlaybackStateHandler.cpp
Providers/AudioDataProvider.cpp
Providers/GenericTimeProvider.cpp
Providers/VideoDataProvider.cpp

View File

@@ -173,7 +173,7 @@ NonnullRefPtr<DisplayingVideoSink> PlaybackManager::get_or_create_the_displaying
if (track_data.display == nullptr) {
track_data.display = MUST(Media::DisplayingVideoSink::try_create(m_time_provider));
track_data.display->set_provider(track, track_data.provider);
track_data.provider->seek(m_time_provider->current_time());
track_data.provider->seek(m_time_provider->current_time(), SeekMode::Accurate);
}
VERIFY(track_data.display->provider(track) == track_data.provider);
@@ -223,6 +223,12 @@ void PlaybackManager::pause()
m_handler->pause();
}
void PlaybackManager::seek(AK::Duration timestamp, SeekMode mode)
{
m_handler->seek(timestamp, mode);
m_is_in_error_state = false;
}
bool PlaybackManager::is_playing()
{
return m_handler->is_playing();

View File

@@ -70,6 +70,7 @@ public:
void play();
void pause();
void seek(AK::Duration timestamp, SeekMode);
bool is_playing();
PlaybackState state();

View File

@@ -11,7 +11,9 @@
#define ENUMERATE_PLAYBACK_STATE_HANDLERS(X) \
X(PlaybackStateHandler) \
X(PlayingStateHandler) \
X(PausedStateHandler)
X(PausedStateHandler) \
X(ResumingStateHandler) \
X(SeekingStateHandler)
namespace Media {

View File

@@ -13,6 +13,7 @@ namespace Media {
enum class PlaybackState : u8 {
Playing,
Paused,
Seeking,
};
}

View File

@@ -0,0 +1,19 @@
/*
* Copyright (c) 2025, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#include <LibMedia/PlaybackManager.h>
#include <LibMedia/PlaybackStates/SeekingStateHandler.h>
#include "PlaybackStateHandler.h"
namespace Media {
void PlaybackStateHandler::seek(AK::Duration timestamp, SeekMode mode)
{
manager().replace_state_handler<SeekingStateHandler>(manager().is_playing(), timestamp, mode);
}
}

View File

@@ -6,8 +6,10 @@
#pragma once
#include <AK/Time.h>
#include <LibMedia/Forward.h>
#include <LibMedia/PlaybackStates/PlaybackState.h>
#include <LibMedia/SeekMode.h>
namespace Media {
@@ -24,6 +26,7 @@ public:
virtual void play() = 0;
virtual void pause() = 0;
virtual void seek(AK::Duration timestamp, SeekMode);
virtual bool is_playing() = 0;
virtual PlaybackState state() = 0;

View File

@@ -0,0 +1,53 @@
/*
* Copyright (c) 2025, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <LibMedia/PlaybackManager.h>
#include <LibMedia/PlaybackStates/Forward.h>
#include <LibMedia/PlaybackStates/PausedStateHandler.h>
namespace Media {
class ResumingStateHandler : public PlaybackStateHandler {
public:
ResumingStateHandler(PlaybackManager& manager, bool playing)
: PlaybackStateHandler(manager)
, m_playing(playing)
{
}
virtual ~ResumingStateHandler() override = default;
void resume()
{
if (m_playing)
manager().replace_state_handler<PlayingStateHandler>();
else
manager().replace_state_handler<PausedStateHandler>();
}
virtual void on_enter() override { }
virtual void on_exit() override { }
virtual void play() override
{
m_playing = true;
}
virtual void pause() override
{
m_playing = false;
}
virtual bool is_playing() override
{
return m_playing;
}
private:
bool m_playing { false };
};
}

View File

@@ -0,0 +1,168 @@
/*
* Copyright (c) 2025, Gregory Bertilson <zaggy1024@gmail.com>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Time.h>
#include <LibMedia/PlaybackManager.h>
#include <LibMedia/PlaybackStates/Forward.h>
#include <LibMedia/PlaybackStates/ResumingStateHandler.h>
#include <LibMedia/Providers/AudioDataProvider.h>
#include <LibMedia/Providers/VideoDataProvider.h>
#include <LibMedia/SeekMode.h>
#include <LibMedia/Sinks/AudioMixingSink.h>
#include <LibMedia/Sinks/DisplayingVideoSink.h>
namespace Media {
class SeekingStateHandler final : public ResumingStateHandler {
public:
SeekingStateHandler(PlaybackManager& manager, bool playing, AK::Duration timestamp, SeekMode mode)
: ResumingStateHandler(manager, playing)
, m_target_timestamp(timestamp)
, m_mode(mode)
{
}
virtual ~SeekingStateHandler() override = default;
virtual void on_enter() override
{
ResumingStateHandler::on_enter();
begin_seek();
}
virtual void seek(AK::Duration timestamp, SeekMode mode) override
{
m_target_timestamp = timestamp;
m_mode = mode;
begin_seek();
}
virtual PlaybackState state() override
{
return PlaybackState::Seeking;
}
private:
struct SeekData : public RefCounted<SeekData> {
SeekData(PlaybackManager& manager)
: manager(manager)
{
}
NonnullRefPtr<PlaybackManager> manager;
size_t id { 0 };
AK::Duration chosen_timestamp { AK::Duration::zero() };
size_t video_seeks_in_flight { 0 };
size_t video_seeks_completed { 0 };
size_t audio_seeks_in_flight { 0 };
size_t audio_seeks_completed { 0 };
};
static void possibly_complete_seek(SeekData& seek_data)
{
if (seek_data.video_seeks_completed != seek_data.video_seeks_in_flight)
return;
if (seek_data.audio_seeks_completed != seek_data.audio_seeks_in_flight)
return;
auto& seek_handler = as<SeekingStateHandler>(*seek_data.manager->m_handler);
// Providers guarantee that their callbacks don't get called if a new seek is started, but we
// can end up with video seeks in flight while an audio seek is completing. Ensure that the
// old audio seek doesn't cause us to exit the seeking state before the current seek completes.
if (seek_handler.m_current_seek_id != seek_data.id)
return;
seek_data.manager->m_time_provider->set_time(seek_data.chosen_timestamp);
for (auto& video_track_data : seek_data.manager->m_video_track_datas) {
if (video_track_data.display == nullptr)
continue;
video_track_data.display->resume_updates();
}
seek_handler.resume();
}
static size_t count_audio_tracks(PlaybackManager& manager)
{
size_t count = 0;
for (auto const& audio_track_data : manager.m_audio_track_datas) {
if (manager.m_audio_sink->provider(audio_track_data.track) == nullptr)
continue;
count++;
}
return count;
}
void begin_seek()
{
auto seek_data = make_ref_counted<SeekData>(manager());
seek_data->id = ++m_current_seek_id;
for (auto const& video_track_data : manager().m_video_track_datas) {
if (video_track_data.display == nullptr)
continue;
seek_data->video_seeks_in_flight++;
video_track_data.display->pause_updates();
}
seek_data->audio_seeks_in_flight = count_audio_tracks(manager());
if (seek_data->video_seeks_in_flight == 0) {
seek_data->chosen_timestamp = m_target_timestamp;
for (auto const& audio_track_data : seek_data->manager->m_audio_track_datas) {
if (seek_data->manager->m_audio_sink->provider(audio_track_data.track) == nullptr)
continue;
audio_track_data.provider->seek(seek_data->chosen_timestamp, [seek_data]() {
seek_data->audio_seeks_completed++;
possibly_complete_seek(*seek_data);
});
}
return;
}
for (auto const& video_track_data : manager().m_video_track_datas) {
if (video_track_data.display == nullptr)
continue;
video_track_data.provider->seek(m_target_timestamp, m_mode, [seek_data](AK::Duration provider_timestamp) {
seek_data->chosen_timestamp = max(seek_data->chosen_timestamp, provider_timestamp);
seek_data->video_seeks_completed++;
if (seek_data->video_seeks_completed == seek_data->video_seeks_in_flight) {
// Since we're running this in a callback that can run any time after begin_seek() was called,
// we need to ensure that our audio track count is up-to-date.
seek_data->audio_seeks_in_flight = count_audio_tracks(*seek_data->manager);
if (seek_data->audio_seeks_in_flight == 0) {
possibly_complete_seek(*seek_data);
return;
}
for (auto const& audio_track_data : seek_data->manager->m_audio_track_datas) {
if (seek_data->manager->m_audio_sink->provider(audio_track_data.track) == nullptr)
continue;
audio_track_data.provider->seek(seek_data->chosen_timestamp, [seek_data]() {
seek_data->audio_seeks_completed++;
possibly_complete_seek(*seek_data);
});
}
}
});
}
}
AK::Duration m_target_timestamp;
SeekMode m_mode { SeekMode::Accurate };
size_t m_current_seek_id { 0 };
};
}

View File

@@ -26,8 +26,10 @@ DecoderErrorOr<NonnullRefPtr<AudioDataProvider>> AudioDataProvider::try_create(N
auto provider = DECODER_TRY_ALLOC(try_make_ref_counted<AudioDataProvider>(thread_data));
auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int {
while (!thread_data->should_thread_exit())
while (!thread_data->should_thread_exit()) {
thread_data->handle_seek();
thread_data->push_data_and_decode_a_block();
}
return 0;
}));
thread->start();
@@ -51,9 +53,9 @@ void AudioDataProvider::set_error_handler(ErrorHandler&& handler)
m_thread_data->set_error_handler(move(handler));
}
void AudioDataProvider::seek(AK::Duration timestamp)
void AudioDataProvider::seek(AK::Duration timestamp, SeekCompletionHandler&& completion_handler)
{
m_thread_data->seek(timestamp);
m_thread_data->seek(timestamp, move(completion_handler));
}
AudioDataProvider::ThreadData::ThreadData(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, NonnullOwnPtr<AudioDecoder>&& decoder)
@@ -89,16 +91,13 @@ void AudioDataProvider::ThreadData::exit()
m_wait_condition.broadcast();
}
void AudioDataProvider::ThreadData::seek(AK::Duration timestamp)
void AudioDataProvider::ThreadData::seek(AK::Duration timestamp, SeekCompletionHandler&& completion_handler)
{
auto demuxer_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
if (demuxer_result.is_error()) {
m_error_handler(demuxer_result.release_error());
} else {
auto locker = take_lock();
m_is_in_error_state = false;
m_wait_condition.broadcast();
}
auto locker = take_lock();
m_seek_completion_handler = move(completion_handler);
m_seek_id++;
m_seek_timestamp = timestamp;
m_wait_condition.broadcast();
}
bool AudioDataProvider::ThreadData::should_thread_exit() const
@@ -106,6 +105,121 @@ bool AudioDataProvider::ThreadData::should_thread_exit() const
return m_exit;
}
template<typename T>
void AudioDataProvider::ThreadData::process_seek_on_main_thread(u32 seek_id, T&& function)
{
m_last_processed_seek_id = seek_id;
m_main_thread_event_loop.deferred_invoke([this, seek_id, function] mutable {
if (m_seek_id != seek_id)
return;
function();
});
}
void AudioDataProvider::ThreadData::resolve_seek(u32 seek_id)
{
process_seek_on_main_thread(seek_id, [this] {
{
auto locker = take_lock();
m_is_in_error_state = false;
m_wait_condition.broadcast();
}
auto handler = move(m_seek_completion_handler);
if (handler)
handler();
});
}
bool AudioDataProvider::ThreadData::handle_seek()
{
auto seek_id = m_seek_id.load();
if (m_last_processed_seek_id == seek_id)
return false;
auto handle_error = [&](DecoderError&& error) {
{
auto locker = take_lock();
m_queue.clear();
}
process_seek_on_main_thread(seek_id,
[this, error = move(error)] mutable {
m_error_handler(move(error));
m_seek_completion_handler = nullptr;
});
};
AK::Duration timestamp;
while (true) {
{
auto locker = take_lock();
seek_id = m_seek_id;
timestamp = m_seek_timestamp;
}
auto demuxer_seek_result_or_error = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
if (demuxer_seek_result_or_error.is_error() && demuxer_seek_result_or_error.error().category() != DecoderErrorCategory::EndOfStream) {
handle_error(demuxer_seek_result_or_error.release_error());
return true;
}
auto demuxer_seek_result = demuxer_seek_result_or_error.value_or(DemuxerSeekResult::MovedPosition);
if (demuxer_seek_result == DemuxerSeekResult::MovedPosition)
m_decoder->flush();
auto new_seek_id = seek_id;
AudioBlock last_block;
while (new_seek_id == seek_id) {
auto coded_frame_result = m_demuxer->get_next_sample_for_track(m_track);
if (coded_frame_result.is_error()) {
if (coded_frame_result.error().category() == DecoderErrorCategory::EndOfStream) {
resolve_seek(seek_id);
return true;
}
handle_error(coded_frame_result.release_error());
return true;
}
auto coded_frame = coded_frame_result.release_value();
auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data());
if (decode_result.is_error()) {
handle_error(decode_result.release_error());
return true;
}
while (new_seek_id == seek_id) {
AudioBlock current_block;
auto block_result = m_decoder->write_next_block(current_block);
if (block_result.is_error()) {
if (block_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
break;
handle_error(block_result.release_error());
return true;
}
if (current_block.start_timestamp() > timestamp) {
auto locker = take_lock();
m_queue.clear();
if (!last_block.is_empty())
m_queue.enqueue(move(last_block));
m_queue.enqueue(move(current_block));
resolve_seek(seek_id);
return true;
}
last_block = move(current_block);
new_seek_id = m_seek_id;
}
}
}
}
void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
{
#if PLAYBACK_MANAGER_DEBUG
@@ -121,8 +235,11 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
m_error_handler(move(error));
});
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Audio Data Provider: Encountered an error, waiting for a seek to start decoding again...");
while (m_is_in_error_state)
while (m_is_in_error_state) {
if (handle_seek())
break;
m_wait_condition.wait();
}
};
auto sample_result = m_demuxer->get_next_sample_for_track(m_track);
@@ -146,6 +263,8 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
auto locker = take_lock();
while (m_queue.size() >= m_queue_max_size) {
if (handle_seek())
return;
m_wait_condition.wait();
if (should_thread_exit())
return;

View File

@@ -32,6 +32,7 @@ public:
using AudioQueue = Queue<AudioBlock, QUEUE_CAPACITY>;
using ErrorHandler = Function<void(DecoderError&&)>;
using SeekCompletionHandler = Function<void()>;
static DecoderErrorOr<NonnullRefPtr<AudioDataProvider>> try_create(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track);
AudioDataProvider(NonnullRefPtr<ThreadData> const&);
@@ -41,7 +42,7 @@ public:
AudioBlock retrieve_block();
void seek(AK::Duration timestamp);
void seek(AK::Duration timestamp, SeekCompletionHandler&& = nullptr);
private:
class ThreadData final : public AtomicRefCounted<ThreadData> {
@@ -52,12 +53,16 @@ private:
void set_error_handler(ErrorHandler&&);
bool should_thread_exit() const;
bool handle_seek();
template<typename T>
void process_seek_on_main_thread(u32 seek_id, T&&);
void resolve_seek(u32 seek_id);
void push_data_and_decode_a_block();
void exit();
void set_stopped(bool);
bool is_stopped() const;
void seek(AK::Duration timestamp);
void seek(AK::Duration timestamp, SeekCompletionHandler&&);
[[nodiscard]] Threading::MutexLocker take_lock() { return Threading::MutexLocker(m_mutex); }
void wake() { m_wait_condition.broadcast(); }
@@ -80,6 +85,11 @@ private:
AudioQueue m_queue;
ErrorHandler m_error_handler;
bool m_is_in_error_state { false };
u32 m_last_processed_seek_id { 0 };
Atomic<u32> m_seek_id { 0 };
SeekCompletionHandler m_seek_completion_handler;
AK::Duration m_seek_timestamp;
};
NonnullRefPtr<ThreadData> m_thread_data;

View File

@@ -27,8 +27,10 @@ DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> VideoDataProvider::try_create(N
auto provider = DECODER_TRY_ALLOC(try_make_ref_counted<VideoDataProvider>(thread_data));
auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int {
while (!thread_data->should_thread_exit())
while (!thread_data->should_thread_exit()) {
thread_data->handle_seek();
thread_data->push_data_and_decode_some_frames();
}
return 0;
}));
thread->start();
@@ -68,9 +70,9 @@ TimedImage VideoDataProvider::retrieve_frame()
return result;
}
void VideoDataProvider::seek(AK::Duration timestamp)
void VideoDataProvider::seek(AK::Duration timestamp, SeekMode seek_mode, SeekCompletionHandler&& completion_handler)
{
m_thread_data->seek(timestamp);
m_thread_data->seek(timestamp, seek_mode, move(completion_handler));
}
VideoDataProvider::ThreadData::ThreadData(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, NonnullOwnPtr<VideoDecoder>&& decoder, RefPtr<MediaTimeProvider> const& time_provider)
@@ -107,16 +109,14 @@ TimedImage VideoDataProvider::ThreadData::take_frame()
return m_queue.dequeue();
}
void VideoDataProvider::ThreadData::seek(AK::Duration timestamp)
void VideoDataProvider::ThreadData::seek(AK::Duration timestamp, SeekMode seek_mode, SeekCompletionHandler&& completion_handler)
{
auto seek_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
if (seek_result.is_error()) {
m_error_handler(seek_result.release_error());
} else {
auto locker = take_lock();
m_is_in_error_state = false;
m_wait_condition.broadcast();
}
auto locker = take_lock();
m_seek_id++;
m_seek_completion_handler = move(completion_handler);
m_seek_timestamp = timestamp;
m_seek_mode = seek_mode;
m_wait_condition.broadcast();
}
bool VideoDataProvider::ThreadData::should_thread_exit() const
@@ -155,6 +155,178 @@ void VideoDataProvider::ThreadData::queue_frame(TimedImage&& frame)
m_queue.enqueue(move(frame));
}
template<typename T>
void VideoDataProvider::ThreadData::process_seek_on_main_thread(u32 seek_id, T&& function)
{
m_last_processed_seek_id = seek_id;
m_main_thread_event_loop.deferred_invoke([this, seek_id, function] mutable {
if (m_seek_id != seek_id)
return;
function();
});
}
void VideoDataProvider::ThreadData::resolve_seek(u32 seek_id, AK::Duration const& timestamp)
{
m_is_in_error_state = false;
process_seek_on_main_thread(seek_id, [this, timestamp] {
auto handler = move(m_seek_completion_handler);
if (handler)
handler(timestamp);
});
}
bool VideoDataProvider::ThreadData::handle_seek()
{
#define CONVERT_AND_QUEUE_A_FRAME(frame) \
do { \
auto __bitmap_result = frame->to_bitmap(); \
if (__bitmap_result.is_error()) { \
handle_error(__bitmap_result.release_error()); \
return true; \
} \
queue_frame(TimedImage(frame->timestamp(), __bitmap_result.release_value())); \
} while (false)
auto seek_id = m_seek_id.load();
if (m_last_processed_seek_id == seek_id)
return false;
auto handle_error = [&](DecoderError&& error) {
m_is_in_error_state = true;
{
auto locker = take_lock();
m_queue.clear();
}
process_seek_on_main_thread(seek_id,
[this, error = move(error)] mutable {
m_error_handler(move(error));
m_seek_completion_handler = nullptr;
});
};
AK::Duration timestamp;
SeekMode mode { SeekMode::Accurate };
while (true) {
{
auto locker = take_lock();
seek_id = m_seek_id;
timestamp = m_seek_timestamp;
mode = m_seek_mode;
}
auto seek_options = mode == SeekMode::Accurate ? DemuxerSeekOptions::None : DemuxerSeekOptions::Force;
auto demuxer_seek_result_or_error = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp, seek_options);
if (demuxer_seek_result_or_error.is_error() && demuxer_seek_result_or_error.error().category() != DecoderErrorCategory::EndOfStream) {
handle_error(demuxer_seek_result_or_error.release_error());
return true;
}
auto demuxer_seek_result = demuxer_seek_result_or_error.value_or(DemuxerSeekResult::MovedPosition);
if (demuxer_seek_result == DemuxerSeekResult::MovedPosition)
m_decoder->flush();
auto is_desired_coded_frame = [mode, timestamp](CodedFrame const& frame) {
if (mode == SeekMode::Accurate)
return true;
if (mode == SeekMode::FastBefore)
return frame.is_keyframe();
if (mode == SeekMode::FastAfter)
return frame.is_keyframe() && frame.timestamp() > timestamp;
VERIFY_NOT_REACHED();
};
auto is_desired_decoded_frame = [mode, timestamp](VideoFrame const& frame) {
if (mode == SeekMode::Accurate)
return frame.timestamp() > timestamp;
return true;
};
auto resolved_time = [mode, timestamp](VideoFrame const& frame) {
if (mode == SeekMode::Accurate)
return timestamp;
if (mode == SeekMode::FastBefore)
return min(timestamp, frame.timestamp());
if (mode == SeekMode::FastAfter)
return max(timestamp, frame.timestamp());
VERIFY_NOT_REACHED();
};
auto new_seek_id = m_seek_id.load();
auto found_desired_keyframe = false;
OwnPtr<VideoFrame> last_frame;
while (new_seek_id == seek_id) {
auto coded_frame_result = m_demuxer->get_next_sample_for_track(m_track);
if (coded_frame_result.is_error()) {
if (coded_frame_result.error().category() == DecoderErrorCategory::EndOfStream) {
if (mode == SeekMode::FastAfter) {
// If we're fast seeking after the provided timestamp and reach the end of the stream, that means we
// nothing to display. Restart the seek as an accurate seek.
auto locker = take_lock();
seek_id = ++m_seek_id;
m_seek_mode = SeekMode::Accurate;
continue;
}
if (last_frame != nullptr)
CONVERT_AND_QUEUE_A_FRAME(last_frame);
resolve_seek(seek_id, timestamp);
return true;
}
handle_error(coded_frame_result.release_error());
return true;
}
auto coded_frame = coded_frame_result.release_value();
if (!found_desired_keyframe)
found_desired_keyframe = is_desired_coded_frame(coded_frame);
if (!found_desired_keyframe)
continue;
auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data());
if (decode_result.is_error()) {
handle_error(decode_result.release_error());
return true;
}
while (new_seek_id == seek_id) {
auto frame_result = m_decoder->get_decoded_frame();
if (frame_result.is_error()) {
if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
break;
handle_error(frame_result.release_error());
return true;
}
auto current_frame = frame_result.release_value();
set_cicp_values(*current_frame, coded_frame);
if (is_desired_decoded_frame(*current_frame)) {
auto locker = take_lock();
m_queue.clear();
if (last_frame != nullptr)
CONVERT_AND_QUEUE_A_FRAME(last_frame);
CONVERT_AND_QUEUE_A_FRAME(current_frame);
VERIFY(!m_queue.is_empty());
resolve_seek(seek_id, resolved_time(*current_frame));
return true;
}
last_frame = move(current_frame);
new_seek_id = m_seek_id;
}
}
}
}
void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
{
// FIXME: Check if the PlaybackManager's current time is ahead of the next keyframe, and seek to it if so.
@@ -170,8 +342,11 @@ void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
m_error_handler(move(error));
});
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Video Data Provider: Encountered an error, waiting for a seek to start decoding again...");
while (m_is_in_error_state)
while (m_is_in_error_state) {
if (handle_seek())
break;
m_wait_condition.wait();
}
};
auto sample_result = m_demuxer->get_next_sample_for_track(m_track);
@@ -209,6 +384,8 @@ void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
{
auto locker = take_lock();
while (m_queue.size() >= m_queue_max_size) {
if (handle_seek())
return;
m_wait_condition.wait();
if (should_thread_exit())
return;

View File

@@ -17,6 +17,7 @@
#include <LibMedia/DecoderError.h>
#include <LibMedia/Export.h>
#include <LibMedia/Forward.h>
#include <LibMedia/SeekMode.h>
#include <LibMedia/TimedImage.h>
#include <LibMedia/Track.h>
#include <LibThreading/ConditionVariable.h>
@@ -33,6 +34,7 @@ public:
using ImageQueue = Queue<TimedImage, QUEUE_CAPACITY>;
using ErrorHandler = Function<void(DecoderError&&)>;
using SeekCompletionHandler = Function<void(AK::Duration)>;
static DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> try_create(NonnullRefPtr<MutexedDemuxer> const&, Track const&, RefPtr<MediaTimeProvider> const& = nullptr);
static DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> try_create(NonnullRefPtr<Demuxer> const&, Track const&, RefPtr<MediaTimeProvider> const& = nullptr);
@@ -44,7 +46,7 @@ public:
TimedImage retrieve_frame();
void seek(AK::Duration timestamp);
void seek(AK::Duration timestamp, SeekMode, SeekCompletionHandler&& = nullptr);
private:
class ThreadData final : public AtomicRefCounted<ThreadData> {
@@ -59,11 +61,15 @@ private:
ImageQueue& queue();
TimedImage take_frame();
void seek(AK::Duration timestamp);
void seek(AK::Duration timestamp, SeekMode, SeekCompletionHandler&&);
bool should_thread_exit() const;
static void set_cicp_values(VideoFrame&, CodedFrame const&);
void queue_frame(TimedImage&&);
bool handle_seek();
template<typename T>
void process_seek_on_main_thread(u32 seek_id, T&&);
void resolve_seek(u32 seek_id, AK::Duration const& timestamp);
void push_data_and_decode_some_frames();
[[nodiscard]] Threading::MutexLocker take_lock() { return Threading::MutexLocker(m_mutex); }
@@ -86,6 +92,12 @@ private:
ImageQueue m_queue;
ErrorHandler m_error_handler;
bool m_is_in_error_state { false };
u32 m_last_processed_seek_id { 0 };
Atomic<u32> m_seek_id { 0 };
SeekCompletionHandler m_seek_completion_handler;
AK::Duration m_seek_timestamp;
SeekMode m_seek_mode { SeekMode::Accurate };
};
NonnullRefPtr<ThreadData> m_thread_data;

View File

@@ -0,0 +1,46 @@
/*
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
*
* SPDX-License-Identifier: BSD-2-Clause
*/
#pragma once
#include <AK/Format.h>
#include <AK/StringView.h>
#include <AK/Types.h>
namespace Media {
enum class SeekMode : u8 {
Accurate,
FastBefore,
FastAfter,
};
constexpr StringView seek_mode_to_string(SeekMode seek_mode)
{
switch (seek_mode) {
case SeekMode::Accurate:
return "Accurate"sv;
case SeekMode::FastBefore:
return "FastBefore"sv;
case SeekMode::FastAfter:
return "FastAfter"sv;
}
VERIFY_NOT_REACHED();
}
}
namespace AK {
template<>
struct Formatter<Media::SeekMode> final : Formatter<StringView> {
ErrorOr<void> format(FormatBuilder& builder, Media::SeekMode color_primaries)
{
return Formatter<StringView>::format(builder, Media::seek_mode_to_string(color_primaries));
}
};
}

View File

@@ -47,8 +47,16 @@ RefPtr<VideoDataProvider> DisplayingVideoSink::provider(Track const& track) cons
DisplayingVideoSinkUpdateResult DisplayingVideoSink::update()
{
if (m_pause_updates)
return DisplayingVideoSinkUpdateResult::NoChange;
auto current_time = m_time_provider->current_time();
auto result = DisplayingVideoSinkUpdateResult::NoChange;
if (m_cleared_current_frame) {
result = DisplayingVideoSinkUpdateResult::NewFrameAvailable;
m_cleared_current_frame = false;
}
while (true) {
if (!m_next_frame.is_valid()) {
m_next_frame = m_provider->retrieve_frame();
@@ -68,4 +76,17 @@ RefPtr<Gfx::Bitmap> DisplayingVideoSink::current_frame()
return m_current_frame;
}
void DisplayingVideoSink::pause_updates()
{
m_pause_updates = true;
}
void DisplayingVideoSink::resume_updates()
{
m_next_frame.clear();
m_current_frame = nullptr;
m_pause_updates = false;
m_cleared_current_frame = true;
}
}

View File

@@ -38,6 +38,9 @@ public:
DisplayingVideoSinkUpdateResult update();
RefPtr<Gfx::Bitmap> current_frame();
void pause_updates();
void resume_updates();
private:
static constexpr size_t DEFAULT_QUEUE_SIZE = 8;
@@ -49,6 +52,8 @@ private:
TimedImage m_next_frame;
RefPtr<Gfx::Bitmap> m_current_frame;
bool m_pause_updates { false };
bool m_cleared_current_frame { false };
};
}

View File

@@ -42,6 +42,11 @@ public:
m_timestamp = AK::Duration::zero();
return m_image.release_nonnull();
}
void clear()
{
m_timestamp = AK::Duration::zero();
m_image = nullptr;
}
private:
AK::Duration m_timestamp { AK::Duration::zero() };

View File

@@ -13,6 +13,7 @@ shared_library("LibMedia") {
"Containers/Matroska/MatroskaDemuxer.cpp",
"Containers/Matroska/Reader.cpp",
"PlaybackManager.cpp",
"PlaybackStates/PlaybackStateHandler.cpp",
"Providers/AudioDataProvider.cpp",
"Providers/GenericTimeProvider.cpp",
"Providers/VideoDataProvider.cpp",