video: make IMGFMT_RGB0 etc. exist even if libavutil doesn't support it

These formats are helpful for distinguishing surfaces with and without
alpha. Unfortunately, Libav and older version of FFmpeg don't support
them, so code will break. Fix this by treating these formats specially
on the mpv side, mapping them to RGBA on Libav, and unseting the alpha
bit in the mp_imgfmt_desc struct.

video/fmt-conversion.c

@@ -159,16 +159,22 @@ static const struct {
     {IMGFMT_GBRP14_BE, PIX_FMT_GBRP14BE},
     {IMGFMT_GBRP14_LE, PIX_FMT_GBRP14LE},
 
+    {IMGFMT_RGBA64_BE,  PIX_FMT_RGBA64BE},
+    {IMGFMT_RGBA64_LE,  PIX_FMT_RGBA64LE},
+    {IMGFMT_BGRA64_BE,  PIX_FMT_BGRA64BE},
+    {IMGFMT_BGRA64_LE,  PIX_FMT_BGRA64LE},
+
     {IMGFMT_BGR0,  PIX_FMT_BGR0},
     {IMGFMT_0RGB,  PIX_FMT_0RGB},
     {IMGFMT_RGB0,  PIX_FMT_RGB0},
     {IMGFMT_0BGR,  PIX_FMT_0BGR},
     {IMGFMT_BGR0,  PIX_FMT_BGR0},
-
+#else
-    {IMGFMT_RGBA64_BE,  PIX_FMT_RGBA64BE},
+    {IMGFMT_BGR0,  PIX_FMT_BGRA},
-    {IMGFMT_RGBA64_LE,  PIX_FMT_RGBA64LE},
+    {IMGFMT_0RGB,  PIX_FMT_ARGB},
-    {IMGFMT_BGRA64_BE,  PIX_FMT_BGRA64BE},
+    {IMGFMT_RGB0,  PIX_FMT_RGBA},
-    {IMGFMT_BGRA64_LE,  PIX_FMT_BGRA64LE},
+    {IMGFMT_0BGR,  PIX_FMT_ABGR},
+    {IMGFMT_BGR0,  PIX_FMT_BGRA},
 #endif
 
 #if HAVE_AVCODEC_NEW_VDPAU_API

video/img_format.c

@@ -157,12 +157,20 @@ const char *mp_imgfmt_to_name(unsigned int fmt)
     return NULL;
 }
 
-static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
+struct mp_imgfmt_desc mp_imgfmt_get_desc(int mpfmt)
 {
-    const AVPixFmtDescriptor *pd = &av_pix_fmt_descriptors[fmt];
+    enum PixelFormat fmt = imgfmt2pixfmt(mpfmt);
-    int mpfmt = pixfmt2imgfmt(fmt);
+    if (fmt == PIX_FMT_NONE) {
-    if (!pd || !mpfmt)
+        const char *name = mp_imgfmt_to_name(mpfmt);
+        if (name) {
+            mp_msg(MSGT_DECVIDEO, MSGL_V,
+                   "libavutil does not know image format '%s'\n", name);
+        }
         return (struct mp_imgfmt_desc) {0};
+    }
+
+    const AVPixFmtDescriptor *pd = &av_pix_fmt_descriptors[fmt];
+    assert(pd);
 
     struct mp_imgfmt_desc desc = {
         .id = mpfmt,
@@ -217,6 +225,9 @@ static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
         desc.flags |= MP_IMGFLAG_ALPHA;
 #endif
 
+    if (mpfmt >= IMGFMT_RGB0_START && mpfmt <= IMGFMT_RGB0_END)
+        desc.flags &= ~MP_IMGFLAG_ALPHA;
+
     if (desc.num_planes == pd->nb_components)
         desc.flags |= MP_IMGFLAG_PLANAR;
 
@@ -251,19 +262,6 @@ static struct mp_imgfmt_desc get_avutil_fmt(enum PixelFormat fmt)
     return desc;
 }
 
-struct mp_imgfmt_desc mp_imgfmt_get_desc(unsigned int out_fmt)
-{
-    struct mp_imgfmt_desc fmt = {0};
-    enum PixelFormat avfmt = imgfmt2pixfmt(out_fmt);
-    if (avfmt != PIX_FMT_NONE)
-        fmt = get_avutil_fmt(avfmt);
-    if (!fmt.id) {
-        mp_msg(MSGT_DECVIDEO, MSGL_V, "mp_image: unknown out_fmt: 0x%X\n",
-               out_fmt);
-    }
-    return fmt;
-}
-
 // Find a format that is MP_IMGFLAG_YUV_P with the following configuration.
 int mp_imgfmt_find_yuv_planar(int xs, int ys, int planes, int component_bits)
 {

video/img_format.h

@@ -78,7 +78,7 @@ struct mp_imgfmt_desc {
     int8_t ys[MP_MAX_PLANES];
 };
 
-struct mp_imgfmt_desc mp_imgfmt_get_desc(unsigned int out_fmt);
+struct mp_imgfmt_desc mp_imgfmt_get_desc(int imgfmt);
 
 enum mp_imgfmt {
     IMGFMT_NONE = 0,
@@ -176,13 +176,9 @@ enum mp_imgfmt {
 
     // Byte accessed (low address to high address)
     IMGFMT_ARGB,
-    IMGFMT_0RGB,                // "0" is a padding byte (as opposed to alpha)
     IMGFMT_BGRA,
-    IMGFMT_BGR0,
     IMGFMT_ABGR,
-    IMGFMT_0BGR,
     IMGFMT_RGBA,
-    IMGFMT_RGB0,
     IMGFMT_BGR24,               // 3 bytes per pixel
     IMGFMT_RGB24,
     IMGFMT_RGB48_LE,            // 6 bytes per pixel, uint16_t channels
@@ -192,6 +188,15 @@ enum mp_imgfmt {
     IMGFMT_BGRA64_LE,
     IMGFMT_BGRA64_BE,
 
+    // Like e.g. IMGFMT_ARGB, but has a padding byte instead of alpha
+    IMGFMT_0RGB,
+    IMGFMT_BGR0,
+    IMGFMT_0BGR,
+    IMGFMT_RGB0,
+
+    IMGFMT_RGB0_START = IMGFMT_0RGB,
+    IMGFMT_RGB0_END = IMGFMT_RGB0,
+
     // Accessed with bit-shifts (components ordered from LSB to MSB)
     IMGFMT_RGB8,                // r3 g3 b2
     IMGFMT_BGR8,