player: add video cache for fast exact seeking

DOCS/interface-changes/video-history.txt

@@ -0,0 +1,2 @@
+add `--video-history` and `--video-history-max-bytes` options
+add `video-history-state` property

DOCS/man/input.rst

@@ -2652,6 +2652,31 @@ Property list
     Whether the demuxer is idle, which means that the demuxer cache is filled
     to the requested amount, and is currently not reading more data.
 
+``video-history-state``
+    State of ``--video-history``. When querying the property with the client
+    API using ``MPV_FORMAT_NODE``, or with Lua ``mp.get_property_native``, this
+    will return a mpv_node with the following contents:
+
+    ::
+
+        MPV_FORMAT_NODE_MAP
+            "status"  MPV_FORMAT_STRING
+            "ranges"  MPV_FORMAT_NODE_ARRAY
+                MPV_FORMAT_NODE_MAP
+                    "start"  MPV_FORMAT_DOUBLE
+                    "end"    MPV_FORMAT_DOUBLE
+            "bytes"   MPV_FORMAT_INT64
+
+    ``status`` is ``ready`` if there is a range that can be seeked to, and
+    ``filling`` if there is none yet. Otherwise, it is the reason why nothing
+    is cached: ``disabled``, ``no-video``, ``reverse-playback``,
+    ``unsupported-pipeline``, ``hardware-decoding`` or ``audio-passthrough``.
+
+    ``ranges`` are the time ranges that can be seeked to without decoding, in
+    no particular order.
+
+    ``bytes`` is the approximate memory used by the cached video and audio frames.
+
 ``demuxer-cache-state``
     Each entry in ``seekable-ranges`` represents a region in the demuxer cache
     that can be seeked to, with a ``start`` and ``end`` fields containing the

DOCS/man/options.rst

@@ -392,6 +392,28 @@ Playback Control
     chapter instead. A negative value means always go back to the previous
     chapter.
 
+``--video-history=<seconds>``
+    Cache the last ``<seconds>`` of played video and audio in memory after
+    decoding and filtering (default: 0, disabled). Exact seeks and frame steps
+    to cached positions are instant. Fixes short backward seeks being slow in
+    files with large keyframe interval.
+
+    Needs a lot of memory for uncompressed video frames. 10 seconds of
+    1920x1080 8-bit 4:2:0 video at 60 FPS take about 2 GB.
+    See ``--video-history-max-bytes``.
+
+    Does not work with hardware decoding (except copy-back modes), reverse
+    playback, ``--lavfi-complex``, encoding, audio passthrough. See property
+    ``video-history-state``.
+
+    Disables direct rendering (``--vd-lavc-dr``) and makes decoder not drop
+    frames (``--framedrop=decoder``).
+
+``--video-history-max-bytes=<bytesize>``
+    Maximum amount of memory for ``--video-history`` (default: 2GiB).
+
+    Same suffixes as ``--demuxer-max-bytes``.
+
 ``--hr-seek=<no|absolute|yes|default>``
     Select when to use precise seeks that are not limited to keyframes. Such
     seeks require decoding video from the previous keyframe up to the target

filters/f_decoder_wrapper.c

@@ -1425,6 +1425,7 @@ struct mp_decoder_wrapper *mp_decoder_wrapper_create(struct mp_filter *parent,
             p->dec_root_filter->stream_info = &p->stream_info;
             p->stream_info = (struct mp_stream_info){
                 .dr_vo = sinfo->dr_vo,
+                .disable_dr = sinfo->disable_dr,
                 .hwdec_devs = sinfo->hwdec_devs,
             };
         }

filters/f_output_chain.c

@@ -389,7 +389,8 @@ static void get_display_res(struct mp_stream_info *i, int *res)
         vo_control(p->vo, VOCTRL_GET_DISPLAY_RES, res);
 }
 
-void mp_output_chain_set_vo(struct mp_output_chain *c, struct vo *vo)
+void mp_output_chain_set_vo(struct mp_output_chain *c, struct vo *vo,
+                             bool allow_dr)
 {
     struct chain *p = c->f->priv;
 
@@ -398,6 +399,7 @@ void mp_output_chain_set_vo(struct mp_output_chain *c, struct vo *vo)
     p->stream_info.vflip = vo ? vo->driver->caps & VO_CAP_VFLIP : false;
     p->stream_info.rotate90 = vo ? vo->driver->caps & VO_CAP_ROTATE90 : false;
     p->stream_info.dr_vo = vo;
+    p->stream_info.disable_dr = !allow_dr;
     p->vo = vo;
     update_output_caps(p);
 }

filters/f_output_chain.h

@@ -49,10 +49,12 @@ struct mp_output_chain *mp_output_chain_create(struct mp_filter *parent,
                                                enum mp_output_chain_type type);
 
 // Set the VO, which will be used to determine basic capabilities like format
-// and rotation support, and to init hardware filtering things.
+// and rotation support, and to init hardware filtering things. If allow_dr is
+// false, decoders will not use the VO for direct rendering.
 // For type==MP_OUTPUT_CHAIN_VIDEO only.
 struct vo;
-void mp_output_chain_set_vo(struct mp_output_chain *p, struct vo *vo);
+void mp_output_chain_set_vo(struct mp_output_chain *p, struct vo *vo,
+                             bool allow_dr);
 
 // Set the AO. The AO format will be used to determine the filter chain output.
 // The API user may be asked to update the AO midstream if ao_needs_update is

filters/filter.h

@@ -409,6 +409,7 @@ struct mp_stream_info {
     bool rotate90;
     bool force_swdec;
     struct vo *dr_vo; // for calling vo_get_image()
+    bool disable_dr;
 };
 
 // Search for a parent filter (including f) that has this set, and return it.

meson.build

@@ -182,6 +182,7 @@ sources = files(
     'player/scripting.c',
     'player/sub.c',
     'player/video.c',
+    'player/history.c',
 
     ## clipboard
     'player/clipboard/clipboard.c',

options/options.c

@@ -882,6 +882,8 @@ static const m_option_t mp_opts[] = {
     {"video-sync-max-audio-change", OPT_DOUBLE(sync_max_audio_change),
         M_RANGE(0, 1)},
     {"video-sync-max-factor", OPT_INT(sync_max_factor), M_RANGE(1, 10)},
+    {"video-history", OPT_DOUBLE(video_history), M_RANGE(0, 3600), .flags = UPDATE_VD},
+    {"video-history-max-bytes", OPT_BYTE_SIZE(video_history_max_bytes), M_RANGE(0, INT64_MAX)},
     {"hr-seek", OPT_CHOICE(hr_seek,
         {"no", -1}, {"absolute", 0}, {"yes", 1}, {"always", 1}, {"default", 2})},
     {"hr-seek-demuxer-offset", OPT_FLOAT(hr_seek_demuxer_offset)},
@@ -1045,6 +1047,7 @@ static const struct MPOpts mp_default_opts = {
     .ordered_chapters = true,
     .chapter_merge_threshold = 100,
     .chapter_seek_threshold = 5.0,
+    .video_history_max_bytes = 2LL * 1024 * 1024 * 1024,
     .hr_seek = 2,
     .hr_seek_framedrop = true,
     .sync_max_video_change = 1,

options/options.h

@@ -270,6 +270,8 @@ typedef struct MPOpts {
     double sync_max_video_change;
     double sync_max_audio_change;
     int sync_max_factor;
+    double video_history;
+    int64_t video_history_max_bytes;
     int hr_seek;
     float hr_seek_demuxer_offset;
     bool hr_seek_framedrop;

player/audio.c

@@ -39,6 +39,7 @@
 
 #include "core.h"
 #include "command.h"
+#include "history.h"
 
 enum {
     AD_OK = 0,
@@ -110,6 +111,7 @@ int reinit_audio_filters(struct MPContext *mpctx)
     if (!ao_c)
         return 0;
 
+    history_invalidate(mpctx);
     double delay = mp_output_get_measured_total_delay(ao_c->filter);
 
     if (recreate_audio_filters(mpctx) < 0)
@@ -229,12 +231,19 @@ static void ao_chain_reset_state(struct ao_chain *ao_c)
 
 void reset_audio_state(struct MPContext *mpctx)
 {
+    history_reset(mpctx);
     if (mpctx->ao_chain) {
-        ao_chain_reset_state(mpctx->ao_chain);
         struct track *t = mpctx->ao_chain->track;
         if (t && t->dec)
             mp_decoder_wrapper_set_play_dir(t->dec, mpctx->play_dir);
     }
+    reset_audio_output_state(mpctx);
+}
+
+void reset_audio_output_state(struct MPContext *mpctx)
+{
+    if (mpctx->ao_chain)
+        ao_chain_reset_state(mpctx->ao_chain);
     mpctx->audio_status = mpctx->ao_chain ? STATUS_SYNCING : STATUS_EOF;
     mpctx->delay = 0;
     mpctx->logged_async_diff = -1;
@@ -708,6 +717,8 @@ static void ao_process(struct mp_filter *f)
         return;
     }
 
+    history_skip(mpctx, MP_FRAME_AUDIO, f->ppins[0]);
+
     // Due to mp_async_queue_set_notifier() this function is called when the
     // queue becomes full. This affects state changes in the normal playloop,
     // so wake it up. But avoid redundant wakeups during normal playback.
@@ -721,10 +732,12 @@ static void ao_process(struct mp_filter *f)
     if (ao_c->untimed_throttle)
         return;
 
-    if (!mp_pin_can_transfer_data(ao_c->queue_filter->pins[0], f->ppins[0]))
+    bool replay = history_replaying(mpctx, MP_FRAME_AUDIO);
+    if (replay ? !mp_pin_in_needs_data(ao_c->queue_filter->pins[0]) :
+        !mp_pin_can_transfer_data(ao_c->queue_filter->pins[0], f->ppins[0]))
         return;
 
-    struct mp_frame frame = mp_pin_out_read(f->ppins[0]);
+    struct mp_frame frame = history_read(mpctx, MP_FRAME_AUDIO, f->ppins[0], &replay);
     if (frame.type == MP_FRAME_AUDIO) {
         struct mp_aframe *af = frame.data;
 
@@ -733,14 +746,16 @@ static void ao_process(struct mp_filter *f)
             endpts *= mpctx->play_dir;
             // Avoid decoding and discarding the entire rest of the file.
             if (mp_aframe_get_pts(af) >= endpts) {
-                mp_pin_out_unread(f->ppins[0], frame);
+                history_unread(mpctx, frame, f->ppins[0], replay);
                 if (!ao_c->out_eof) {
                     ao_c->out_eof = true;
                     mp_pin_in_write(ao_c->queue_filter->pins[0], MP_EOF_FRAME);
                 }
                 return;
             }
         }
+        if (!replay)
+            history_add(mpctx, frame);
         double startpts = mpctx->audio_status == STATUS_SYNCING ?
                                             ao_c->start_pts : MP_NOPTS_VALUE;
         mp_aframe_clip_timestamps(af, startpts, endpts);

player/command.c

@@ -85,6 +85,7 @@
 #include "osdep/terminal.h"
 
 #include "core.h"
+#include "history.h"
 
 #ifdef _WIN32
 #include <windows.h>
@@ -1614,6 +1615,19 @@ static int mp_property_demuxer_cache_idle(void *ctx, struct m_property *prop,
     return m_property_bool_ro(action, arg, s.idle);
 }
 
+static int mp_property_video_history(void *ctx, struct m_property *prop,
+                                     int action, void *arg)
+{
+    if (action == M_PROPERTY_GET_TYPE) {
+        *(struct m_option *)arg = (struct m_option){.type = CONF_TYPE_NODE};
+        return M_PROPERTY_OK;
+    }
+    if (action != M_PROPERTY_GET)
+        return M_PROPERTY_NOT_IMPLEMENTED;
+    history_get_state(ctx, arg);
+    return M_PROPERTY_OK;
+}
+
 static int mp_property_demuxer_cache_state(void *ctx, struct m_property *prop,
                                            int action, void *arg)
 {
@@ -4743,6 +4757,7 @@ static const struct m_property mp_properties_base[] = {
     {"demuxer-cache-time", mp_property_demuxer_cache_time},
     {"demuxer-cache-idle", mp_property_demuxer_cache_idle},
     {"demuxer-start-time", mp_property_demuxer_start_time},
+    {"video-history-state", mp_property_video_history},
     {"demuxer-cache-state", mp_property_demuxer_cache_state},
     {"cache-buffering-state", mp_property_cache_buffering},
     {"paused-for-cache", mp_property_paused_for_cache},
@@ -4942,7 +4957,8 @@ static const char *const *const mp_event_property_change[] = {
       "decoder-frame-drop-count", "frame-drop-count", "video-frame-info",
       "vf-metadata", "af-metadata", "sub-start", "sub-end", "secondary-sub-start",
       "secondary-sub-end", "sub-lines", "secondary-sub-lines", "video-out-params",
-      "video-dec-params", "video-params", "deinterlace-active", "video-target-params"),
+      "video-dec-params", "video-params", "deinterlace-active", "video-target-params",
+      "video-history-state"),
     E(MP_EVENT_DURATION_UPDATE, "duration"),
     E(MPV_EVENT_VIDEO_RECONFIG, "video-out-params", "video-params",
       "video-format", "video-codec", "video-bitrate", "dwidth", "dheight",
@@ -8453,6 +8469,12 @@ void mp_option_run_callback(struct MPContext *mpctx, struct mp_option_callback *
     void *opt_ptr = co ? co->data : NULL;
     uint64_t flags = callback->flags;
 
+    // These change what the decoders or filters output.
+    if (flags & (UPDATE_IMGPAR | UPDATE_HWDEC | UPDATE_AD) ||
+        opt_ptr == &opts->playback_speed || opt_ptr == &opts->playback_pitch ||
+        opt_ptr == &opts->pitch_correction)
+        history_invalidate(mpctx);
+
     if (flags & UPDATE_TERM)
         mp_update_logging(mpctx, false);
 

player/core.h

@@ -336,6 +336,7 @@ typedef struct MPContext {
     struct ao_chain *ao_chain;
 
     struct vo_chain *vo_chain;
+    struct playback_history *history;
 
     struct vo *video_out;
     // next_frame[0] is the next frame, next_frame[1] the one after that.
@@ -510,6 +511,7 @@ struct mp_abort_entry {
 
 // audio.c
 void reset_audio_state(struct MPContext *mpctx);
+void reset_audio_output_state(struct MPContext *mpctx);
 void reinit_audio_chain(struct MPContext *mpctx);
 int init_audio_decoder(struct MPContext *mpctx, struct track *track);
 void uninit_audio_decoder(struct MPContext *mpctx);
@@ -691,6 +693,7 @@ bool disc_nav_mouse_pos_to_src(struct MPContext *mpctx, int src_w, int src_h,
 
 // video.c
 void reset_video_state(struct MPContext *mpctx);
+void reset_video_output_state(struct MPContext *mpctx);
 int init_video_decoder(struct MPContext *mpctx, struct track *track);
 void uninit_video_decoder(struct MPContext *mpctx);
 void reinit_video_decoder(struct MPContext *mpctx);