mirror of
https://github.com/LadybirdBrowser/ladybird.git
synced 2025-12-22 10:57:18 +00:00
LibMedia: Implement media seeking
This implementation allows: - Accurate seeking to an exact timestamp - Seeking to the keyframe before a timestamp - Seeking to the keyframe after a timestamp These three options will be used to satisfy the playback position selection in the media element's seeking steps.
This commit is contained in:
committed by
Jelle Raaijmakers
parent
b1c9a872bc
commit
ccf4b3f6e9
Notes:
github-actions[bot]
2025-10-28 00:33:01 +00:00
Author: https://github.com/Zaggy1024 Commit: https://github.com/LadybirdBrowser/ladybird/commit/ccf4b3f6e94 Pull-request: https://github.com/LadybirdBrowser/ladybird/pull/6410 Reviewed-by: https://github.com/R-Goc Reviewed-by: https://github.com/gmta ✅
@@ -11,6 +11,7 @@ set(SOURCES
|
||||
Containers/Matroska/MatroskaDemuxer.cpp
|
||||
Containers/Matroska/Reader.cpp
|
||||
PlaybackManager.cpp
|
||||
PlaybackStates/PlaybackStateHandler.cpp
|
||||
Providers/AudioDataProvider.cpp
|
||||
Providers/GenericTimeProvider.cpp
|
||||
Providers/VideoDataProvider.cpp
|
||||
|
||||
@@ -173,7 +173,7 @@ NonnullRefPtr<DisplayingVideoSink> PlaybackManager::get_or_create_the_displaying
|
||||
if (track_data.display == nullptr) {
|
||||
track_data.display = MUST(Media::DisplayingVideoSink::try_create(m_time_provider));
|
||||
track_data.display->set_provider(track, track_data.provider);
|
||||
track_data.provider->seek(m_time_provider->current_time());
|
||||
track_data.provider->seek(m_time_provider->current_time(), SeekMode::Accurate);
|
||||
}
|
||||
|
||||
VERIFY(track_data.display->provider(track) == track_data.provider);
|
||||
@@ -223,6 +223,12 @@ void PlaybackManager::pause()
|
||||
m_handler->pause();
|
||||
}
|
||||
|
||||
void PlaybackManager::seek(AK::Duration timestamp, SeekMode mode)
|
||||
{
|
||||
m_handler->seek(timestamp, mode);
|
||||
m_is_in_error_state = false;
|
||||
}
|
||||
|
||||
bool PlaybackManager::is_playing()
|
||||
{
|
||||
return m_handler->is_playing();
|
||||
|
||||
@@ -70,6 +70,7 @@ public:
|
||||
|
||||
void play();
|
||||
void pause();
|
||||
void seek(AK::Duration timestamp, SeekMode);
|
||||
|
||||
bool is_playing();
|
||||
PlaybackState state();
|
||||
|
||||
@@ -11,7 +11,9 @@
|
||||
#define ENUMERATE_PLAYBACK_STATE_HANDLERS(X) \
|
||||
X(PlaybackStateHandler) \
|
||||
X(PlayingStateHandler) \
|
||||
X(PausedStateHandler)
|
||||
X(PausedStateHandler) \
|
||||
X(ResumingStateHandler) \
|
||||
X(SeekingStateHandler)
|
||||
|
||||
namespace Media {
|
||||
|
||||
|
||||
@@ -13,6 +13,7 @@ namespace Media {
|
||||
enum class PlaybackState : u8 {
|
||||
Playing,
|
||||
Paused,
|
||||
Seeking,
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
19
Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.cpp
Normal file
19
Libraries/LibMedia/PlaybackStates/PlaybackStateHandler.cpp
Normal file
@@ -0,0 +1,19 @@
|
||||
/*
|
||||
* Copyright (c) 2025, Gregory Bertilson <zaggy1024@gmail.com>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#include <LibMedia/PlaybackManager.h>
|
||||
#include <LibMedia/PlaybackStates/SeekingStateHandler.h>
|
||||
|
||||
#include "PlaybackStateHandler.h"
|
||||
|
||||
namespace Media {
|
||||
|
||||
void PlaybackStateHandler::seek(AK::Duration timestamp, SeekMode mode)
|
||||
{
|
||||
manager().replace_state_handler<SeekingStateHandler>(manager().is_playing(), timestamp, mode);
|
||||
}
|
||||
|
||||
}
|
||||
@@ -6,8 +6,10 @@
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Time.h>
|
||||
#include <LibMedia/Forward.h>
|
||||
#include <LibMedia/PlaybackStates/PlaybackState.h>
|
||||
#include <LibMedia/SeekMode.h>
|
||||
|
||||
namespace Media {
|
||||
|
||||
@@ -24,6 +26,7 @@ public:
|
||||
|
||||
virtual void play() = 0;
|
||||
virtual void pause() = 0;
|
||||
virtual void seek(AK::Duration timestamp, SeekMode);
|
||||
|
||||
virtual bool is_playing() = 0;
|
||||
virtual PlaybackState state() = 0;
|
||||
|
||||
53
Libraries/LibMedia/PlaybackStates/ResumingStateHandler.h
Normal file
53
Libraries/LibMedia/PlaybackStates/ResumingStateHandler.h
Normal file
@@ -0,0 +1,53 @@
|
||||
/*
|
||||
* Copyright (c) 2025, Gregory Bertilson <zaggy1024@gmail.com>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <LibMedia/PlaybackManager.h>
|
||||
#include <LibMedia/PlaybackStates/Forward.h>
|
||||
#include <LibMedia/PlaybackStates/PausedStateHandler.h>
|
||||
|
||||
namespace Media {
|
||||
|
||||
class ResumingStateHandler : public PlaybackStateHandler {
|
||||
public:
|
||||
ResumingStateHandler(PlaybackManager& manager, bool playing)
|
||||
: PlaybackStateHandler(manager)
|
||||
, m_playing(playing)
|
||||
{
|
||||
}
|
||||
virtual ~ResumingStateHandler() override = default;
|
||||
|
||||
void resume()
|
||||
{
|
||||
if (m_playing)
|
||||
manager().replace_state_handler<PlayingStateHandler>();
|
||||
else
|
||||
manager().replace_state_handler<PausedStateHandler>();
|
||||
}
|
||||
|
||||
virtual void on_enter() override { }
|
||||
virtual void on_exit() override { }
|
||||
|
||||
virtual void play() override
|
||||
{
|
||||
m_playing = true;
|
||||
}
|
||||
virtual void pause() override
|
||||
{
|
||||
m_playing = false;
|
||||
}
|
||||
|
||||
virtual bool is_playing() override
|
||||
{
|
||||
return m_playing;
|
||||
}
|
||||
|
||||
private:
|
||||
bool m_playing { false };
|
||||
};
|
||||
|
||||
}
|
||||
168
Libraries/LibMedia/PlaybackStates/SeekingStateHandler.h
Normal file
168
Libraries/LibMedia/PlaybackStates/SeekingStateHandler.h
Normal file
@@ -0,0 +1,168 @@
|
||||
/*
|
||||
* Copyright (c) 2025, Gregory Bertilson <zaggy1024@gmail.com>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Time.h>
|
||||
#include <LibMedia/PlaybackManager.h>
|
||||
#include <LibMedia/PlaybackStates/Forward.h>
|
||||
#include <LibMedia/PlaybackStates/ResumingStateHandler.h>
|
||||
#include <LibMedia/Providers/AudioDataProvider.h>
|
||||
#include <LibMedia/Providers/VideoDataProvider.h>
|
||||
#include <LibMedia/SeekMode.h>
|
||||
#include <LibMedia/Sinks/AudioMixingSink.h>
|
||||
#include <LibMedia/Sinks/DisplayingVideoSink.h>
|
||||
|
||||
namespace Media {
|
||||
|
||||
class SeekingStateHandler final : public ResumingStateHandler {
|
||||
public:
|
||||
SeekingStateHandler(PlaybackManager& manager, bool playing, AK::Duration timestamp, SeekMode mode)
|
||||
: ResumingStateHandler(manager, playing)
|
||||
, m_target_timestamp(timestamp)
|
||||
, m_mode(mode)
|
||||
{
|
||||
}
|
||||
virtual ~SeekingStateHandler() override = default;
|
||||
|
||||
virtual void on_enter() override
|
||||
{
|
||||
ResumingStateHandler::on_enter();
|
||||
begin_seek();
|
||||
}
|
||||
|
||||
virtual void seek(AK::Duration timestamp, SeekMode mode) override
|
||||
{
|
||||
m_target_timestamp = timestamp;
|
||||
m_mode = mode;
|
||||
begin_seek();
|
||||
}
|
||||
|
||||
virtual PlaybackState state() override
|
||||
{
|
||||
return PlaybackState::Seeking;
|
||||
}
|
||||
|
||||
private:
|
||||
struct SeekData : public RefCounted<SeekData> {
|
||||
SeekData(PlaybackManager& manager)
|
||||
: manager(manager)
|
||||
{
|
||||
}
|
||||
|
||||
NonnullRefPtr<PlaybackManager> manager;
|
||||
|
||||
size_t id { 0 };
|
||||
|
||||
AK::Duration chosen_timestamp { AK::Duration::zero() };
|
||||
|
||||
size_t video_seeks_in_flight { 0 };
|
||||
size_t video_seeks_completed { 0 };
|
||||
|
||||
size_t audio_seeks_in_flight { 0 };
|
||||
size_t audio_seeks_completed { 0 };
|
||||
};
|
||||
|
||||
static void possibly_complete_seek(SeekData& seek_data)
|
||||
{
|
||||
if (seek_data.video_seeks_completed != seek_data.video_seeks_in_flight)
|
||||
return;
|
||||
if (seek_data.audio_seeks_completed != seek_data.audio_seeks_in_flight)
|
||||
return;
|
||||
|
||||
auto& seek_handler = as<SeekingStateHandler>(*seek_data.manager->m_handler);
|
||||
|
||||
// Providers guarantee that their callbacks don't get called if a new seek is started, but we
|
||||
// can end up with video seeks in flight while an audio seek is completing. Ensure that the
|
||||
// old audio seek doesn't cause us to exit the seeking state before the current seek completes.
|
||||
if (seek_handler.m_current_seek_id != seek_data.id)
|
||||
return;
|
||||
|
||||
seek_data.manager->m_time_provider->set_time(seek_data.chosen_timestamp);
|
||||
|
||||
for (auto& video_track_data : seek_data.manager->m_video_track_datas) {
|
||||
if (video_track_data.display == nullptr)
|
||||
continue;
|
||||
video_track_data.display->resume_updates();
|
||||
}
|
||||
|
||||
seek_handler.resume();
|
||||
}
|
||||
|
||||
static size_t count_audio_tracks(PlaybackManager& manager)
|
||||
{
|
||||
size_t count = 0;
|
||||
for (auto const& audio_track_data : manager.m_audio_track_datas) {
|
||||
if (manager.m_audio_sink->provider(audio_track_data.track) == nullptr)
|
||||
continue;
|
||||
count++;
|
||||
}
|
||||
return count;
|
||||
}
|
||||
|
||||
void begin_seek()
|
||||
{
|
||||
auto seek_data = make_ref_counted<SeekData>(manager());
|
||||
seek_data->id = ++m_current_seek_id;
|
||||
|
||||
for (auto const& video_track_data : manager().m_video_track_datas) {
|
||||
if (video_track_data.display == nullptr)
|
||||
continue;
|
||||
seek_data->video_seeks_in_flight++;
|
||||
video_track_data.display->pause_updates();
|
||||
}
|
||||
|
||||
seek_data->audio_seeks_in_flight = count_audio_tracks(manager());
|
||||
|
||||
if (seek_data->video_seeks_in_flight == 0) {
|
||||
seek_data->chosen_timestamp = m_target_timestamp;
|
||||
for (auto const& audio_track_data : seek_data->manager->m_audio_track_datas) {
|
||||
if (seek_data->manager->m_audio_sink->provider(audio_track_data.track) == nullptr)
|
||||
continue;
|
||||
audio_track_data.provider->seek(seek_data->chosen_timestamp, [seek_data]() {
|
||||
seek_data->audio_seeks_completed++;
|
||||
possibly_complete_seek(*seek_data);
|
||||
});
|
||||
}
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto const& video_track_data : manager().m_video_track_datas) {
|
||||
if (video_track_data.display == nullptr)
|
||||
continue;
|
||||
video_track_data.provider->seek(m_target_timestamp, m_mode, [seek_data](AK::Duration provider_timestamp) {
|
||||
seek_data->chosen_timestamp = max(seek_data->chosen_timestamp, provider_timestamp);
|
||||
seek_data->video_seeks_completed++;
|
||||
|
||||
if (seek_data->video_seeks_completed == seek_data->video_seeks_in_flight) {
|
||||
// Since we're running this in a callback that can run any time after begin_seek() was called,
|
||||
// we need to ensure that our audio track count is up-to-date.
|
||||
seek_data->audio_seeks_in_flight = count_audio_tracks(*seek_data->manager);
|
||||
|
||||
if (seek_data->audio_seeks_in_flight == 0) {
|
||||
possibly_complete_seek(*seek_data);
|
||||
return;
|
||||
}
|
||||
|
||||
for (auto const& audio_track_data : seek_data->manager->m_audio_track_datas) {
|
||||
if (seek_data->manager->m_audio_sink->provider(audio_track_data.track) == nullptr)
|
||||
continue;
|
||||
audio_track_data.provider->seek(seek_data->chosen_timestamp, [seek_data]() {
|
||||
seek_data->audio_seeks_completed++;
|
||||
possibly_complete_seek(*seek_data);
|
||||
});
|
||||
}
|
||||
}
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
AK::Duration m_target_timestamp;
|
||||
SeekMode m_mode { SeekMode::Accurate };
|
||||
size_t m_current_seek_id { 0 };
|
||||
};
|
||||
|
||||
}
|
||||
@@ -26,8 +26,10 @@ DecoderErrorOr<NonnullRefPtr<AudioDataProvider>> AudioDataProvider::try_create(N
|
||||
auto provider = DECODER_TRY_ALLOC(try_make_ref_counted<AudioDataProvider>(thread_data));
|
||||
|
||||
auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int {
|
||||
while (!thread_data->should_thread_exit())
|
||||
while (!thread_data->should_thread_exit()) {
|
||||
thread_data->handle_seek();
|
||||
thread_data->push_data_and_decode_a_block();
|
||||
}
|
||||
return 0;
|
||||
}));
|
||||
thread->start();
|
||||
@@ -51,9 +53,9 @@ void AudioDataProvider::set_error_handler(ErrorHandler&& handler)
|
||||
m_thread_data->set_error_handler(move(handler));
|
||||
}
|
||||
|
||||
void AudioDataProvider::seek(AK::Duration timestamp)
|
||||
void AudioDataProvider::seek(AK::Duration timestamp, SeekCompletionHandler&& completion_handler)
|
||||
{
|
||||
m_thread_data->seek(timestamp);
|
||||
m_thread_data->seek(timestamp, move(completion_handler));
|
||||
}
|
||||
|
||||
AudioDataProvider::ThreadData::ThreadData(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, NonnullOwnPtr<AudioDecoder>&& decoder)
|
||||
@@ -89,16 +91,13 @@ void AudioDataProvider::ThreadData::exit()
|
||||
m_wait_condition.broadcast();
|
||||
}
|
||||
|
||||
void AudioDataProvider::ThreadData::seek(AK::Duration timestamp)
|
||||
void AudioDataProvider::ThreadData::seek(AK::Duration timestamp, SeekCompletionHandler&& completion_handler)
|
||||
{
|
||||
auto demuxer_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
|
||||
if (demuxer_result.is_error()) {
|
||||
m_error_handler(demuxer_result.release_error());
|
||||
} else {
|
||||
auto locker = take_lock();
|
||||
m_is_in_error_state = false;
|
||||
m_wait_condition.broadcast();
|
||||
}
|
||||
auto locker = take_lock();
|
||||
m_seek_completion_handler = move(completion_handler);
|
||||
m_seek_id++;
|
||||
m_seek_timestamp = timestamp;
|
||||
m_wait_condition.broadcast();
|
||||
}
|
||||
|
||||
bool AudioDataProvider::ThreadData::should_thread_exit() const
|
||||
@@ -106,6 +105,121 @@ bool AudioDataProvider::ThreadData::should_thread_exit() const
|
||||
return m_exit;
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void AudioDataProvider::ThreadData::process_seek_on_main_thread(u32 seek_id, T&& function)
|
||||
{
|
||||
m_last_processed_seek_id = seek_id;
|
||||
m_main_thread_event_loop.deferred_invoke([this, seek_id, function] mutable {
|
||||
if (m_seek_id != seek_id)
|
||||
return;
|
||||
function();
|
||||
});
|
||||
}
|
||||
|
||||
void AudioDataProvider::ThreadData::resolve_seek(u32 seek_id)
|
||||
{
|
||||
process_seek_on_main_thread(seek_id, [this] {
|
||||
{
|
||||
auto locker = take_lock();
|
||||
m_is_in_error_state = false;
|
||||
m_wait_condition.broadcast();
|
||||
}
|
||||
auto handler = move(m_seek_completion_handler);
|
||||
if (handler)
|
||||
handler();
|
||||
});
|
||||
}
|
||||
|
||||
bool AudioDataProvider::ThreadData::handle_seek()
|
||||
{
|
||||
auto seek_id = m_seek_id.load();
|
||||
if (m_last_processed_seek_id == seek_id)
|
||||
return false;
|
||||
|
||||
auto handle_error = [&](DecoderError&& error) {
|
||||
{
|
||||
auto locker = take_lock();
|
||||
m_queue.clear();
|
||||
}
|
||||
|
||||
process_seek_on_main_thread(seek_id,
|
||||
[this, error = move(error)] mutable {
|
||||
m_error_handler(move(error));
|
||||
m_seek_completion_handler = nullptr;
|
||||
});
|
||||
};
|
||||
|
||||
AK::Duration timestamp;
|
||||
|
||||
while (true) {
|
||||
{
|
||||
auto locker = take_lock();
|
||||
seek_id = m_seek_id;
|
||||
timestamp = m_seek_timestamp;
|
||||
}
|
||||
|
||||
auto demuxer_seek_result_or_error = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
|
||||
if (demuxer_seek_result_or_error.is_error() && demuxer_seek_result_or_error.error().category() != DecoderErrorCategory::EndOfStream) {
|
||||
handle_error(demuxer_seek_result_or_error.release_error());
|
||||
return true;
|
||||
}
|
||||
auto demuxer_seek_result = demuxer_seek_result_or_error.value_or(DemuxerSeekResult::MovedPosition);
|
||||
|
||||
if (demuxer_seek_result == DemuxerSeekResult::MovedPosition)
|
||||
m_decoder->flush();
|
||||
|
||||
auto new_seek_id = seek_id;
|
||||
AudioBlock last_block;
|
||||
|
||||
while (new_seek_id == seek_id) {
|
||||
auto coded_frame_result = m_demuxer->get_next_sample_for_track(m_track);
|
||||
if (coded_frame_result.is_error()) {
|
||||
if (coded_frame_result.error().category() == DecoderErrorCategory::EndOfStream) {
|
||||
resolve_seek(seek_id);
|
||||
return true;
|
||||
}
|
||||
handle_error(coded_frame_result.release_error());
|
||||
return true;
|
||||
}
|
||||
|
||||
auto coded_frame = coded_frame_result.release_value();
|
||||
auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data());
|
||||
if (decode_result.is_error()) {
|
||||
handle_error(decode_result.release_error());
|
||||
return true;
|
||||
}
|
||||
|
||||
while (new_seek_id == seek_id) {
|
||||
AudioBlock current_block;
|
||||
auto block_result = m_decoder->write_next_block(current_block);
|
||||
if (block_result.is_error()) {
|
||||
if (block_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
|
||||
break;
|
||||
handle_error(block_result.release_error());
|
||||
return true;
|
||||
}
|
||||
|
||||
if (current_block.start_timestamp() > timestamp) {
|
||||
auto locker = take_lock();
|
||||
m_queue.clear();
|
||||
|
||||
if (!last_block.is_empty())
|
||||
m_queue.enqueue(move(last_block));
|
||||
|
||||
m_queue.enqueue(move(current_block));
|
||||
|
||||
resolve_seek(seek_id);
|
||||
return true;
|
||||
}
|
||||
|
||||
last_block = move(current_block);
|
||||
|
||||
new_seek_id = m_seek_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
|
||||
{
|
||||
#if PLAYBACK_MANAGER_DEBUG
|
||||
@@ -121,8 +235,11 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
|
||||
m_error_handler(move(error));
|
||||
});
|
||||
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Audio Data Provider: Encountered an error, waiting for a seek to start decoding again...");
|
||||
while (m_is_in_error_state)
|
||||
while (m_is_in_error_state) {
|
||||
if (handle_seek())
|
||||
break;
|
||||
m_wait_condition.wait();
|
||||
}
|
||||
};
|
||||
|
||||
auto sample_result = m_demuxer->get_next_sample_for_track(m_track);
|
||||
@@ -146,6 +263,8 @@ void AudioDataProvider::ThreadData::push_data_and_decode_a_block()
|
||||
auto locker = take_lock();
|
||||
|
||||
while (m_queue.size() >= m_queue_max_size) {
|
||||
if (handle_seek())
|
||||
return;
|
||||
m_wait_condition.wait();
|
||||
if (should_thread_exit())
|
||||
return;
|
||||
|
||||
@@ -32,6 +32,7 @@ public:
|
||||
using AudioQueue = Queue<AudioBlock, QUEUE_CAPACITY>;
|
||||
|
||||
using ErrorHandler = Function<void(DecoderError&&)>;
|
||||
using SeekCompletionHandler = Function<void()>;
|
||||
|
||||
static DecoderErrorOr<NonnullRefPtr<AudioDataProvider>> try_create(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track);
|
||||
AudioDataProvider(NonnullRefPtr<ThreadData> const&);
|
||||
@@ -41,7 +42,7 @@ public:
|
||||
|
||||
AudioBlock retrieve_block();
|
||||
|
||||
void seek(AK::Duration timestamp);
|
||||
void seek(AK::Duration timestamp, SeekCompletionHandler&& = nullptr);
|
||||
|
||||
private:
|
||||
class ThreadData final : public AtomicRefCounted<ThreadData> {
|
||||
@@ -52,12 +53,16 @@ private:
|
||||
void set_error_handler(ErrorHandler&&);
|
||||
|
||||
bool should_thread_exit() const;
|
||||
bool handle_seek();
|
||||
template<typename T>
|
||||
void process_seek_on_main_thread(u32 seek_id, T&&);
|
||||
void resolve_seek(u32 seek_id);
|
||||
void push_data_and_decode_a_block();
|
||||
|
||||
void exit();
|
||||
void set_stopped(bool);
|
||||
bool is_stopped() const;
|
||||
void seek(AK::Duration timestamp);
|
||||
void seek(AK::Duration timestamp, SeekCompletionHandler&&);
|
||||
|
||||
[[nodiscard]] Threading::MutexLocker take_lock() { return Threading::MutexLocker(m_mutex); }
|
||||
void wake() { m_wait_condition.broadcast(); }
|
||||
@@ -80,6 +85,11 @@ private:
|
||||
AudioQueue m_queue;
|
||||
ErrorHandler m_error_handler;
|
||||
bool m_is_in_error_state { false };
|
||||
|
||||
u32 m_last_processed_seek_id { 0 };
|
||||
Atomic<u32> m_seek_id { 0 };
|
||||
SeekCompletionHandler m_seek_completion_handler;
|
||||
AK::Duration m_seek_timestamp;
|
||||
};
|
||||
|
||||
NonnullRefPtr<ThreadData> m_thread_data;
|
||||
|
||||
@@ -27,8 +27,10 @@ DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> VideoDataProvider::try_create(N
|
||||
auto provider = DECODER_TRY_ALLOC(try_make_ref_counted<VideoDataProvider>(thread_data));
|
||||
|
||||
auto thread = DECODER_TRY_ALLOC(Threading::Thread::try_create([thread_data]() -> int {
|
||||
while (!thread_data->should_thread_exit())
|
||||
while (!thread_data->should_thread_exit()) {
|
||||
thread_data->handle_seek();
|
||||
thread_data->push_data_and_decode_some_frames();
|
||||
}
|
||||
return 0;
|
||||
}));
|
||||
thread->start();
|
||||
@@ -68,9 +70,9 @@ TimedImage VideoDataProvider::retrieve_frame()
|
||||
return result;
|
||||
}
|
||||
|
||||
void VideoDataProvider::seek(AK::Duration timestamp)
|
||||
void VideoDataProvider::seek(AK::Duration timestamp, SeekMode seek_mode, SeekCompletionHandler&& completion_handler)
|
||||
{
|
||||
m_thread_data->seek(timestamp);
|
||||
m_thread_data->seek(timestamp, seek_mode, move(completion_handler));
|
||||
}
|
||||
|
||||
VideoDataProvider::ThreadData::ThreadData(NonnullRefPtr<MutexedDemuxer> const& demuxer, Track const& track, NonnullOwnPtr<VideoDecoder>&& decoder, RefPtr<MediaTimeProvider> const& time_provider)
|
||||
@@ -107,16 +109,14 @@ TimedImage VideoDataProvider::ThreadData::take_frame()
|
||||
return m_queue.dequeue();
|
||||
}
|
||||
|
||||
void VideoDataProvider::ThreadData::seek(AK::Duration timestamp)
|
||||
void VideoDataProvider::ThreadData::seek(AK::Duration timestamp, SeekMode seek_mode, SeekCompletionHandler&& completion_handler)
|
||||
{
|
||||
auto seek_result = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp);
|
||||
if (seek_result.is_error()) {
|
||||
m_error_handler(seek_result.release_error());
|
||||
} else {
|
||||
auto locker = take_lock();
|
||||
m_is_in_error_state = false;
|
||||
m_wait_condition.broadcast();
|
||||
}
|
||||
auto locker = take_lock();
|
||||
m_seek_id++;
|
||||
m_seek_completion_handler = move(completion_handler);
|
||||
m_seek_timestamp = timestamp;
|
||||
m_seek_mode = seek_mode;
|
||||
m_wait_condition.broadcast();
|
||||
}
|
||||
|
||||
bool VideoDataProvider::ThreadData::should_thread_exit() const
|
||||
@@ -155,6 +155,178 @@ void VideoDataProvider::ThreadData::queue_frame(TimedImage&& frame)
|
||||
m_queue.enqueue(move(frame));
|
||||
}
|
||||
|
||||
template<typename T>
|
||||
void VideoDataProvider::ThreadData::process_seek_on_main_thread(u32 seek_id, T&& function)
|
||||
{
|
||||
m_last_processed_seek_id = seek_id;
|
||||
m_main_thread_event_loop.deferred_invoke([this, seek_id, function] mutable {
|
||||
if (m_seek_id != seek_id)
|
||||
return;
|
||||
function();
|
||||
});
|
||||
}
|
||||
|
||||
void VideoDataProvider::ThreadData::resolve_seek(u32 seek_id, AK::Duration const& timestamp)
|
||||
{
|
||||
m_is_in_error_state = false;
|
||||
process_seek_on_main_thread(seek_id, [this, timestamp] {
|
||||
auto handler = move(m_seek_completion_handler);
|
||||
if (handler)
|
||||
handler(timestamp);
|
||||
});
|
||||
}
|
||||
|
||||
bool VideoDataProvider::ThreadData::handle_seek()
|
||||
{
|
||||
#define CONVERT_AND_QUEUE_A_FRAME(frame) \
|
||||
do { \
|
||||
auto __bitmap_result = frame->to_bitmap(); \
|
||||
if (__bitmap_result.is_error()) { \
|
||||
handle_error(__bitmap_result.release_error()); \
|
||||
return true; \
|
||||
} \
|
||||
queue_frame(TimedImage(frame->timestamp(), __bitmap_result.release_value())); \
|
||||
} while (false)
|
||||
|
||||
auto seek_id = m_seek_id.load();
|
||||
if (m_last_processed_seek_id == seek_id)
|
||||
return false;
|
||||
|
||||
auto handle_error = [&](DecoderError&& error) {
|
||||
m_is_in_error_state = true;
|
||||
{
|
||||
auto locker = take_lock();
|
||||
m_queue.clear();
|
||||
}
|
||||
process_seek_on_main_thread(seek_id,
|
||||
[this, error = move(error)] mutable {
|
||||
m_error_handler(move(error));
|
||||
m_seek_completion_handler = nullptr;
|
||||
});
|
||||
};
|
||||
|
||||
AK::Duration timestamp;
|
||||
SeekMode mode { SeekMode::Accurate };
|
||||
|
||||
while (true) {
|
||||
{
|
||||
auto locker = take_lock();
|
||||
seek_id = m_seek_id;
|
||||
timestamp = m_seek_timestamp;
|
||||
mode = m_seek_mode;
|
||||
}
|
||||
|
||||
auto seek_options = mode == SeekMode::Accurate ? DemuxerSeekOptions::None : DemuxerSeekOptions::Force;
|
||||
auto demuxer_seek_result_or_error = m_demuxer->seek_to_most_recent_keyframe(m_track, timestamp, seek_options);
|
||||
if (demuxer_seek_result_or_error.is_error() && demuxer_seek_result_or_error.error().category() != DecoderErrorCategory::EndOfStream) {
|
||||
handle_error(demuxer_seek_result_or_error.release_error());
|
||||
return true;
|
||||
}
|
||||
auto demuxer_seek_result = demuxer_seek_result_or_error.value_or(DemuxerSeekResult::MovedPosition);
|
||||
|
||||
if (demuxer_seek_result == DemuxerSeekResult::MovedPosition)
|
||||
m_decoder->flush();
|
||||
|
||||
auto is_desired_coded_frame = [mode, timestamp](CodedFrame const& frame) {
|
||||
if (mode == SeekMode::Accurate)
|
||||
return true;
|
||||
if (mode == SeekMode::FastBefore)
|
||||
return frame.is_keyframe();
|
||||
if (mode == SeekMode::FastAfter)
|
||||
return frame.is_keyframe() && frame.timestamp() > timestamp;
|
||||
VERIFY_NOT_REACHED();
|
||||
};
|
||||
|
||||
auto is_desired_decoded_frame = [mode, timestamp](VideoFrame const& frame) {
|
||||
if (mode == SeekMode::Accurate)
|
||||
return frame.timestamp() > timestamp;
|
||||
return true;
|
||||
};
|
||||
|
||||
auto resolved_time = [mode, timestamp](VideoFrame const& frame) {
|
||||
if (mode == SeekMode::Accurate)
|
||||
return timestamp;
|
||||
if (mode == SeekMode::FastBefore)
|
||||
return min(timestamp, frame.timestamp());
|
||||
if (mode == SeekMode::FastAfter)
|
||||
return max(timestamp, frame.timestamp());
|
||||
VERIFY_NOT_REACHED();
|
||||
};
|
||||
|
||||
auto new_seek_id = m_seek_id.load();
|
||||
auto found_desired_keyframe = false;
|
||||
OwnPtr<VideoFrame> last_frame;
|
||||
|
||||
while (new_seek_id == seek_id) {
|
||||
auto coded_frame_result = m_demuxer->get_next_sample_for_track(m_track);
|
||||
if (coded_frame_result.is_error()) {
|
||||
if (coded_frame_result.error().category() == DecoderErrorCategory::EndOfStream) {
|
||||
if (mode == SeekMode::FastAfter) {
|
||||
// If we're fast seeking after the provided timestamp and reach the end of the stream, that means we
|
||||
// nothing to display. Restart the seek as an accurate seek.
|
||||
auto locker = take_lock();
|
||||
seek_id = ++m_seek_id;
|
||||
m_seek_mode = SeekMode::Accurate;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (last_frame != nullptr)
|
||||
CONVERT_AND_QUEUE_A_FRAME(last_frame);
|
||||
|
||||
resolve_seek(seek_id, timestamp);
|
||||
return true;
|
||||
}
|
||||
handle_error(coded_frame_result.release_error());
|
||||
return true;
|
||||
}
|
||||
|
||||
auto coded_frame = coded_frame_result.release_value();
|
||||
|
||||
if (!found_desired_keyframe)
|
||||
found_desired_keyframe = is_desired_coded_frame(coded_frame);
|
||||
|
||||
if (!found_desired_keyframe)
|
||||
continue;
|
||||
|
||||
auto decode_result = m_decoder->receive_coded_data(coded_frame.timestamp(), coded_frame.data());
|
||||
if (decode_result.is_error()) {
|
||||
handle_error(decode_result.release_error());
|
||||
return true;
|
||||
}
|
||||
|
||||
while (new_seek_id == seek_id) {
|
||||
auto frame_result = m_decoder->get_decoded_frame();
|
||||
if (frame_result.is_error()) {
|
||||
if (frame_result.error().category() == DecoderErrorCategory::NeedsMoreInput)
|
||||
break;
|
||||
handle_error(frame_result.release_error());
|
||||
return true;
|
||||
}
|
||||
|
||||
auto current_frame = frame_result.release_value();
|
||||
set_cicp_values(*current_frame, coded_frame);
|
||||
if (is_desired_decoded_frame(*current_frame)) {
|
||||
auto locker = take_lock();
|
||||
m_queue.clear();
|
||||
|
||||
if (last_frame != nullptr)
|
||||
CONVERT_AND_QUEUE_A_FRAME(last_frame);
|
||||
|
||||
CONVERT_AND_QUEUE_A_FRAME(current_frame);
|
||||
|
||||
VERIFY(!m_queue.is_empty());
|
||||
resolve_seek(seek_id, resolved_time(*current_frame));
|
||||
return true;
|
||||
}
|
||||
|
||||
last_frame = move(current_frame);
|
||||
|
||||
new_seek_id = m_seek_id;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
|
||||
{
|
||||
// FIXME: Check if the PlaybackManager's current time is ahead of the next keyframe, and seek to it if so.
|
||||
@@ -170,8 +342,11 @@ void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
|
||||
m_error_handler(move(error));
|
||||
});
|
||||
dbgln_if(PLAYBACK_MANAGER_DEBUG, "Video Data Provider: Encountered an error, waiting for a seek to start decoding again...");
|
||||
while (m_is_in_error_state)
|
||||
while (m_is_in_error_state) {
|
||||
if (handle_seek())
|
||||
break;
|
||||
m_wait_condition.wait();
|
||||
}
|
||||
};
|
||||
|
||||
auto sample_result = m_demuxer->get_next_sample_for_track(m_track);
|
||||
@@ -209,6 +384,8 @@ void VideoDataProvider::ThreadData::push_data_and_decode_some_frames()
|
||||
{
|
||||
auto locker = take_lock();
|
||||
while (m_queue.size() >= m_queue_max_size) {
|
||||
if (handle_seek())
|
||||
return;
|
||||
m_wait_condition.wait();
|
||||
if (should_thread_exit())
|
||||
return;
|
||||
|
||||
@@ -17,6 +17,7 @@
|
||||
#include <LibMedia/DecoderError.h>
|
||||
#include <LibMedia/Export.h>
|
||||
#include <LibMedia/Forward.h>
|
||||
#include <LibMedia/SeekMode.h>
|
||||
#include <LibMedia/TimedImage.h>
|
||||
#include <LibMedia/Track.h>
|
||||
#include <LibThreading/ConditionVariable.h>
|
||||
@@ -33,6 +34,7 @@ public:
|
||||
using ImageQueue = Queue<TimedImage, QUEUE_CAPACITY>;
|
||||
|
||||
using ErrorHandler = Function<void(DecoderError&&)>;
|
||||
using SeekCompletionHandler = Function<void(AK::Duration)>;
|
||||
|
||||
static DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> try_create(NonnullRefPtr<MutexedDemuxer> const&, Track const&, RefPtr<MediaTimeProvider> const& = nullptr);
|
||||
static DecoderErrorOr<NonnullRefPtr<VideoDataProvider>> try_create(NonnullRefPtr<Demuxer> const&, Track const&, RefPtr<MediaTimeProvider> const& = nullptr);
|
||||
@@ -44,7 +46,7 @@ public:
|
||||
|
||||
TimedImage retrieve_frame();
|
||||
|
||||
void seek(AK::Duration timestamp);
|
||||
void seek(AK::Duration timestamp, SeekMode, SeekCompletionHandler&& = nullptr);
|
||||
|
||||
private:
|
||||
class ThreadData final : public AtomicRefCounted<ThreadData> {
|
||||
@@ -59,11 +61,15 @@ private:
|
||||
ImageQueue& queue();
|
||||
TimedImage take_frame();
|
||||
|
||||
void seek(AK::Duration timestamp);
|
||||
void seek(AK::Duration timestamp, SeekMode, SeekCompletionHandler&&);
|
||||
|
||||
bool should_thread_exit() const;
|
||||
static void set_cicp_values(VideoFrame&, CodedFrame const&);
|
||||
void queue_frame(TimedImage&&);
|
||||
bool handle_seek();
|
||||
template<typename T>
|
||||
void process_seek_on_main_thread(u32 seek_id, T&&);
|
||||
void resolve_seek(u32 seek_id, AK::Duration const& timestamp);
|
||||
void push_data_and_decode_some_frames();
|
||||
|
||||
[[nodiscard]] Threading::MutexLocker take_lock() { return Threading::MutexLocker(m_mutex); }
|
||||
@@ -86,6 +92,12 @@ private:
|
||||
ImageQueue m_queue;
|
||||
ErrorHandler m_error_handler;
|
||||
bool m_is_in_error_state { false };
|
||||
|
||||
u32 m_last_processed_seek_id { 0 };
|
||||
Atomic<u32> m_seek_id { 0 };
|
||||
SeekCompletionHandler m_seek_completion_handler;
|
||||
AK::Duration m_seek_timestamp;
|
||||
SeekMode m_seek_mode { SeekMode::Accurate };
|
||||
};
|
||||
|
||||
NonnullRefPtr<ThreadData> m_thread_data;
|
||||
|
||||
46
Libraries/LibMedia/SeekMode.h
Normal file
46
Libraries/LibMedia/SeekMode.h
Normal file
@@ -0,0 +1,46 @@
|
||||
/*
|
||||
* Copyright (c) 2025, Gregory Bertilson <gregory@ladybird.org>
|
||||
*
|
||||
* SPDX-License-Identifier: BSD-2-Clause
|
||||
*/
|
||||
|
||||
#pragma once
|
||||
|
||||
#include <AK/Format.h>
|
||||
#include <AK/StringView.h>
|
||||
#include <AK/Types.h>
|
||||
|
||||
namespace Media {
|
||||
|
||||
enum class SeekMode : u8 {
|
||||
Accurate,
|
||||
FastBefore,
|
||||
FastAfter,
|
||||
};
|
||||
|
||||
constexpr StringView seek_mode_to_string(SeekMode seek_mode)
|
||||
{
|
||||
switch (seek_mode) {
|
||||
case SeekMode::Accurate:
|
||||
return "Accurate"sv;
|
||||
case SeekMode::FastBefore:
|
||||
return "FastBefore"sv;
|
||||
case SeekMode::FastAfter:
|
||||
return "FastAfter"sv;
|
||||
}
|
||||
VERIFY_NOT_REACHED();
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
namespace AK {
|
||||
|
||||
template<>
|
||||
struct Formatter<Media::SeekMode> final : Formatter<StringView> {
|
||||
ErrorOr<void> format(FormatBuilder& builder, Media::SeekMode color_primaries)
|
||||
{
|
||||
return Formatter<StringView>::format(builder, Media::seek_mode_to_string(color_primaries));
|
||||
}
|
||||
};
|
||||
|
||||
}
|
||||
@@ -47,8 +47,16 @@ RefPtr<VideoDataProvider> DisplayingVideoSink::provider(Track const& track) cons
|
||||
|
||||
DisplayingVideoSinkUpdateResult DisplayingVideoSink::update()
|
||||
{
|
||||
if (m_pause_updates)
|
||||
return DisplayingVideoSinkUpdateResult::NoChange;
|
||||
|
||||
auto current_time = m_time_provider->current_time();
|
||||
auto result = DisplayingVideoSinkUpdateResult::NoChange;
|
||||
if (m_cleared_current_frame) {
|
||||
result = DisplayingVideoSinkUpdateResult::NewFrameAvailable;
|
||||
m_cleared_current_frame = false;
|
||||
}
|
||||
|
||||
while (true) {
|
||||
if (!m_next_frame.is_valid()) {
|
||||
m_next_frame = m_provider->retrieve_frame();
|
||||
@@ -68,4 +76,17 @@ RefPtr<Gfx::Bitmap> DisplayingVideoSink::current_frame()
|
||||
return m_current_frame;
|
||||
}
|
||||
|
||||
void DisplayingVideoSink::pause_updates()
|
||||
{
|
||||
m_pause_updates = true;
|
||||
}
|
||||
|
||||
void DisplayingVideoSink::resume_updates()
|
||||
{
|
||||
m_next_frame.clear();
|
||||
m_current_frame = nullptr;
|
||||
m_pause_updates = false;
|
||||
m_cleared_current_frame = true;
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
@@ -38,6 +38,9 @@ public:
|
||||
DisplayingVideoSinkUpdateResult update();
|
||||
RefPtr<Gfx::Bitmap> current_frame();
|
||||
|
||||
void pause_updates();
|
||||
void resume_updates();
|
||||
|
||||
private:
|
||||
static constexpr size_t DEFAULT_QUEUE_SIZE = 8;
|
||||
|
||||
@@ -49,6 +52,8 @@ private:
|
||||
|
||||
TimedImage m_next_frame;
|
||||
RefPtr<Gfx::Bitmap> m_current_frame;
|
||||
bool m_pause_updates { false };
|
||||
bool m_cleared_current_frame { false };
|
||||
};
|
||||
|
||||
}
|
||||
|
||||
@@ -42,6 +42,11 @@ public:
|
||||
m_timestamp = AK::Duration::zero();
|
||||
return m_image.release_nonnull();
|
||||
}
|
||||
void clear()
|
||||
{
|
||||
m_timestamp = AK::Duration::zero();
|
||||
m_image = nullptr;
|
||||
}
|
||||
|
||||
private:
|
||||
AK::Duration m_timestamp { AK::Duration::zero() };
|
||||
|
||||
@@ -13,6 +13,7 @@ shared_library("LibMedia") {
|
||||
"Containers/Matroska/MatroskaDemuxer.cpp",
|
||||
"Containers/Matroska/Reader.cpp",
|
||||
"PlaybackManager.cpp",
|
||||
"PlaybackStates/PlaybackStateHandler.cpp",
|
||||
"Providers/AudioDataProvider.cpp",
|
||||
"Providers/GenericTimeProvider.cpp",
|
||||
"Providers/VideoDataProvider.cpp",
|
||||
|
||||
Reference in New Issue
Block a user