demux_lavf: fix unconstrained cache size with wrapped AVFrame packets

Fixes excessive memory usage with avdevice input. After [1] packets
returned from avdevice contains wrapped AVFrame. mpv estimates packet
size to limit how many of them are cached. The size were not correctly
calculated, because packed contained only AVFrame* and not the actual
data in buffer. Fix this by calculating wrapped size and add it to our
size estimation.

[1] 6ca43a9675

demux/cache.c

@@ -216,6 +216,7 @@ int64_t demux_cache_write(struct demux_cache *cache, struct demux_packet *dp)
     }
 
     mp_assert(!dp->is_cached);
+    mp_assert(!dp->is_wrapped_avframe);
     mp_assert(dp->len <= INT32_MAX);
     mp_assert(dp->avpacket->flags >= 0 && dp->avpacket->flags <= INT32_MAX);
     mp_assert(dp->avpacket->side_data_elems >= 0 &&

demux/demux.c

@@ -2047,7 +2047,7 @@ static void add_packet_locked(struct sh_stream *stream, demux_packet_t *dp)
 
     record_packet(in, dp);
 
-    if (in->cache && in->d_user->opts->disk_cache) {
+    if (in->cache && in->d_user->opts->disk_cache && !dp->is_wrapped_avframe) {
         int64_t pos = demux_cache_write(in->cache, dp);
         if (pos >= 0) {
             demux_packet_unref_contents(dp);

demux/demux_lavf.c

@@ -1264,6 +1264,7 @@ static bool demux_lavf_read_packet(struct demuxer *demux,
     dp->duration = pkt->duration * av_q2d(st->time_base);
     dp->pos = pkt->pos;
     dp->keyframe = pkt->flags & AV_PKT_FLAG_KEY;
+    dp->is_wrapped_avframe = st->codecpar->codec_id == AV_CODEC_ID_WRAPPED_AVFRAME;
     av_packet_unref(pkt);
 
     if (priv->format_hack.clear_filepos)

demux/packet.c

@@ -22,6 +22,7 @@
 
 #include <libavcodec/avcodec.h>
 #include <libavutil/hdr_dynamic_metadata.h>
+#include <libavutil/imgutils.h>
 #include <libavutil/intreadwrite.h>
 
 #include "common/av_common.h"
@@ -43,6 +44,7 @@ void demux_packet_unref_contents(struct demux_packet *dp)
         av_packet_free(&dp->avpacket);
         dp->buffer = NULL;
         dp->len = 0;
+        dp->is_wrapped_avframe = false;
     }
 }
 
@@ -166,6 +168,7 @@ void demux_packet_copy_attribs(struct demux_packet *dst, struct demux_packet *sr
     dst->dts = src->dts;
     dst->duration = src->duration;
     dst->pos = src->pos;
+    dst->is_wrapped_avframe = src->is_wrapped_avframe;
     dst->segmented = src->segmented;
     dst->start = src->start;
     dst->end = src->end;
@@ -209,6 +212,31 @@ size_t demux_packet_estimate_total_size(struct demux_packet *dp)
     if (dp->avpacket) {
         mp_assert(!dp->is_cached);
         size += ROUND_ALLOC(dp->len);
+        if (dp->is_wrapped_avframe) {
+            mp_require(dp->buffer);
+
+            const AVFrame *frame = (AVFrame *)dp->buffer;
+            if (frame->hw_frames_ctx) {
+                const AVHWFramesContext *hwctx = (AVHWFramesContext *)frame->hw_frames_ctx->data;
+                int r = av_image_get_buffer_size(hwctx->sw_format, hwctx->width, hwctx->height, 16);
+                mp_require(r >= 0);
+                size += ROUND_ALLOC(r);
+            }
+            for (int i = 0; i < MP_ARRAY_SIZE(frame->buf) && frame->buf[i]; ++i)
+                size += ROUND_ALLOC(frame->buf[i]->size);
+
+            size += ROUND_ALLOC(frame->nb_extended_buf * sizeof(frame->extended_buf[0]));
+            for (int i = 0; i < frame->nb_extended_buf; ++i) {
+                size += ROUND_ALLOC(sizeof(*frame->extended_buf[i]));
+                size += ROUND_ALLOC(frame->extended_buf[i]->size);
+            }
+
+            size += ROUND_ALLOC(frame->nb_side_data * sizeof(frame->side_data[0]));
+            for (int i = 0; i < frame->nb_side_data; ++i) {
+                size += ROUND_ALLOC(sizeof(*frame->side_data[i]));
+                size += ROUND_ALLOC(frame->side_data[i]->size);
+            }
+        }
         size += ROUND_ALLOC(sizeof(AVPacket));
         size += 8 * sizeof(void *); // ta  overhead
         size += ROUND_ALLOC(sizeof(AVBufferRef));

demux/packet.h

@@ -53,6 +53,9 @@ typedef struct demux_packet {
     // If true, cached_data is valid, while buffer/len are not.
     bool is_cached : 1;
 
+    // If true, this is a wrapped AVFrame
+    bool is_wrapped_avframe : 1;
+
     // segmentation (ordered chapters, EDL)
     bool segmented;
     struct mp_codec_params *codec;  // set to non-NULL iff segmented is set