encode: rewrite half of it

The main change is that we wait with opening the muxer ("writing
headers") until we have data from all streams. This fixes race
conditions at init due to broken assumptions in the old code.

This also changes a lot of other stuff. I found and fixed a few API
violations (often things for which better mechanisms were invented, and
the old ones are not valid anymore). I try to get away from the public
mutex and shared fields in encode_lavc_context. For now it's still
needed for some timestamp-related fields, but most are gone. It also
removes some bad code duplication between audio and video paths.

common/common.h

@@ -106,4 +106,15 @@ char *mp_tprintf_buf(char *buf, size_t buf_size, const char *format, ...)
 
 char **mp_dup_str_array(void *tctx, char **s);
 
+// We generally do not handle allocation failure of small malloc()s. This would
+// create a large number of rarely tested code paths, which would probably
+// regress and cause security issues. We prefer to fail fast.
+// This macro generally behaves like an assert(), except it will make sure to
+// kill the process even with NDEBUG.
+#define MP_HANDLE_OOM(x) do {   \
+        assert(x);              \
+        if (!(x))               \
+            abort();            \
+    } while (0)
+
 #endif /* MPLAYER_MPCOMMON_H */

common/encode.h

@@ -54,17 +54,16 @@ struct encode_opts {
     char **remove_metadata;
 };
 
-// interface for mplayer.c
-struct encode_lavc_context *encode_lavc_init(struct encode_opts *options,
-                                             struct mpv_global *global);
-void encode_lavc_free(struct encode_lavc_context *ctx);
+// interface for player core
+struct encode_lavc_context *encode_lavc_init(struct mpv_global *global);
+bool encode_lavc_free(struct encode_lavc_context *ctx);
 void encode_lavc_discontinuity(struct encode_lavc_context *ctx);
 bool encode_lavc_showhelp(struct mp_log *log, struct encode_opts *options);
 int encode_lavc_getstatus(struct encode_lavc_context *ctx, char *buf, int bufsize, float relative_position);
-void encode_lavc_expect_stream(struct encode_lavc_context *ctx, int mt);
+void encode_lavc_expect_stream(struct encode_lavc_context *ctx,
+                               enum stream_type type);
 void encode_lavc_set_metadata(struct encode_lavc_context *ctx,
                               struct mp_tags *metadata);
-void encode_lavc_set_video_fps(struct encode_lavc_context *ctx, float fps);
 void encode_lavc_set_audio_pts(struct encode_lavc_context *ctx, double pts);
 bool encode_lavc_didfail(struct encode_lavc_context *ctx); // check if encoding failed
 

common/encode_lavc.h

@@ -31,43 +31,26 @@
 #include <libavutil/opt.h>
 #include <libavutil/mathematics.h>
 
+#include "common/common.h"
 #include "encode.h"
 #include "video/csputils.h"
 
 struct encode_lavc_context {
+    // --- Immutable after init
     struct mpv_global *global;
     struct encode_opts *options;
     struct mp_log *log;
-    struct mp_tags *metadata;
+    struct encode_priv *priv;
+    AVOutputFormat *oformat;
+    const char *filename;
 
     // All entry points must be guarded with the lock. Functions called by
     // the playback core lock this automatically, but ao_lavc.c and vo_lavc.c
     // must lock manually before accessing state.
     pthread_mutex_t lock;
 
-    float vo_fps;
-
-    // FFmpeg contexts.
-    AVFormatContext *avc;
-    AVStream *vst;
-    AVStream *ast;
-    AVCodecContext *vcc;
-    AVCodecContext *acc;
-
-    // these are processed from the options
-    AVRational timebase;
-    AVCodec *vc;
-    AVCodec *ac;
-    AVDictionary *foptions;
-    AVDictionary *aoptions;
-    AVDictionary *voptions;
-
-    // values created during encoding
-    int header_written; // -1 means currently writing
-
     // sync to audio mode
     double audio_pts_offset;
-    double last_video_in_pts;
 
     double last_audio_in_pts;
     int64_t samples_since_last_pts;
@@ -75,40 +58,54 @@ struct encode_lavc_context {
     // anti discontinuity mode
     double next_in_pts;
     double discontinuity_pts_offset;
-
-    long long abytes;
-    long long vbytes;
-    struct stream *twopass_bytebuffer_a;
-    struct stream *twopass_bytebuffer_v;
-    double t0;
-    unsigned int frames;
-    double audioseconds;
-
-    bool expect_video;
-    bool expect_audio;
-    bool video_first;
-    bool audio_first;
-
-    // has encoding failed?
-    bool failed;
 };
 
-// interface for vo/ao drivers
-int encode_lavc_alloc_stream(struct encode_lavc_context *ctx,
-                             enum AVMediaType mt, AVStream **stream_out,
-                             AVCodecContext **codec_out);
-void encode_lavc_write_stats(struct encode_lavc_context *ctx,
-                             AVCodecContext *stream);
-int encode_lavc_write_frame(struct encode_lavc_context *ctx, AVStream *stream,
-                            AVPacket *packet);
-int encode_lavc_supports_pixfmt(struct encode_lavc_context *ctx, enum AVPixelFormat format);
-int encode_lavc_open_codec(struct encode_lavc_context *ctx,
-                           AVCodecContext *codec);
-int encode_lavc_available(struct encode_lavc_context *ctx);
-int encode_lavc_timesyncfailed(struct encode_lavc_context *ctx);
-int encode_lavc_start(struct encode_lavc_context *ctx); // returns 1 on success
-double encode_lavc_getoffset(struct encode_lavc_context *ctx,
-                             AVCodecContext *codec);
-void encode_lavc_fail(struct encode_lavc_context *ctx, const char *format, ...); // report failure of encoding
+// --- interface for vo/ao drivers
+
+// Static information after encoder init. This never changes (even if there are
+// dynamic runtime changes, they have to work over AVPacket side data).
+// For use in encoder_context, most fields are copied from encoder_context.encoder
+// by encoder_init_codec_and_muxer().
+struct encoder_stream_info {
+    AVRational timebase; // timebase used by the encoder (in frames/out packets)
+    AVCodecParameters *codecpar;
+};
+
+// The encoder parts for each stream (no muxing parts included).
+// This is private to each stream.
+struct encoder_context {
+    struct mpv_global *global;
+    struct encode_opts *options;
+    struct mp_log *log;
+    AVOutputFormat *oformat;
+
+    // (avoid using this)
+    struct encode_lavc_context *encode_lavc_ctx;
+
+    enum stream_type type;
+
+    // (different access restrictions before/after encoder init)
+    struct encoder_stream_info info;
+    AVCodecContext *encoder;
+    struct mux_stream *mux_stream;
+
+    struct stream *twopass_bytebuffer;
+};
+
+// Free with talloc_free(). (Keep in mind actual deinitialization requires
+// sending a flush packet.)
+// This can fail and return NULL.
+struct encoder_context *encoder_context_alloc(struct encode_lavc_context *ctx,
+                                              enum stream_type type,
+                                              struct mp_log *log);
+
+// After setting your codec parameters on p->encoder, you call this to "open"
+// the encoder. This also initializes p->mux_stream. Returns false on failure.
+bool encoder_init_codec_and_muxer(struct encoder_context *p);
+
+// Encode the frame and write the packet. frame is ref'ed as need.
+bool encoder_encode(struct encoder_context *p, AVFrame *frame);
+
+double encoder_get_offset(struct encoder_context *p);
 
 #endif