vo_dmabuf_wayland: reject formats not supported by the GPU

The linux-dmabuf protocol gives us the main device being used. Using
that along with some drm code, we can get what drm formats are supported
by the GPU. This is pretty crude as we just check against the first
primary plane we find and hope for the best. Also the protocol can
technically update formats at any time (e.g. if the device changes). We
do update what gets stored in  wl->gpu_formats but the next commit won't
make any attempt to address this hypothetical scenario. It's complicated
and this is better than nothing. In practice, this will only work
against one constant device during VO runtime. Additionally, we can add
this check to ra_compatible_format to filter out additional unsupported
formats.

video/out/hwdec/dmabuf_interop_wl.c

@@ -44,10 +44,10 @@ static bool map(struct ra_hwdec_mapper *mapper,
     if (mapper_p->desc.nb_layers != 1) {
         MP_VERBOSE(mapper, "Mapped surface has separate layers - expected composed layers.\n");
         return false;
-    } else if (!ra_compatible_format(mapper->ra, drm_format,
-        mapper_p->desc.objects[0].format_modifier)) {
+    } else if (!ra_compatible_format(mapper->ra, mapper->src->params.hw_subfmt,
+               drm_format, mapper_p->desc.objects[0].format_modifier)) {
         MP_VERBOSE(mapper, "Mapped surface with format %s; drm format '%s(%016" PRIx64 ")' "
-                   "is not supported by compositor.\n",
+                   "is not supported by compositor and GPU combination.\n",
                    mp_imgfmt_to_name(mapper->src->params.hw_subfmt),
                    mp_tag_str(drm_format),
                    mapper_p->desc.objects[0].format_modifier);

video/out/vo_dmabuf_wayland.c

@@ -199,7 +199,7 @@ static void vaapi_dmabuf_importer(struct buffer *buf, struct mp_image *src,
         goto done;
     }
     buf->drm_format = desc.layers[layer_no].drm_format;
-    if (!ra_compatible_format(p->ctx->ra, buf->drm_format, desc.objects[0].drm_format_modifier)) {
+    if (!ra_compatible_format(p->ctx->ra, src->params.hw_subfmt, buf->drm_format, desc.objects[0].drm_format_modifier)) {
         MP_VERBOSE(vo, "%s(%016" PRIx64 ") is not supported.\n",
                    mp_tag_str(buf->drm_format), desc.objects[0].drm_format_modifier);
         buf->drm_format = 0;
@@ -681,7 +681,7 @@ static int reconfig(struct vo *vo, struct mp_image *img)
         return VO_ERROR;
     }
 
-    if (!ra_compatible_format(p->ctx->ra, p->drm_format, p->drm_modifier)) {
+    if (!ra_compatible_format(p->ctx->ra, img->params.hw_subfmt, p->drm_format, p->drm_modifier)) {
         MP_ERR(vo, "Format '%s' with modifier '(%016" PRIx64 ")' is not supported by"
                " the compositor.\n", mp_tag_str(p->drm_format), p->drm_modifier);
         return VO_ERROR;

video/out/wayland_common.c

@@ -46,6 +46,11 @@
 #include "single-pixel-buffer-v1.h"
 #include "fractional-scale-v1.h"
 
+#if HAVE_DRM
+#include <xf86drm.h>
+#include <xf86drmMode.h>
+#endif
+
 #if HAVE_WAYLAND_PROTOCOLS_1_32
 #include "cursor-shape-v1.h"
 #endif
@@ -207,6 +212,7 @@ static int spawn_cursor(struct vo_wayland_state *wl);
 static void add_feedback(struct vo_wayland_feedback_pool *fback_pool,
                          struct wp_presentation_feedback *fback);
 static void apply_keepaspect(struct vo_wayland_state *wl, int *width, int *height);
+static void get_gpu_drm_formats(struct vo_wayland_state *wl);
 static void get_shape_device(struct vo_wayland_state *wl, struct vo_wayland_seat *s);
 static void guess_focus(struct vo_wayland_state *wl);
 static void handle_key_input(struct vo_wayland_seat *s, uint32_t key, uint32_t state, bool no_emit);
@@ -1370,6 +1376,16 @@ static void main_device(void *data,
                         struct zwp_linux_dmabuf_feedback_v1 *zwp_linux_dmabuf_feedback_v1,
                         struct wl_array *device)
 {
+    struct vo_wayland_state *wl = data;
+
+    // Despite being an array, the protocol specifically states there can only be
+    // one main device so break as soon as we get one.
+    dev_t *id;
+    wl_array_for_each(id, device) {
+        memcpy(&wl->main_device_id, id, sizeof(dev_t));
+        break;
+    }
+    get_gpu_drm_formats(wl);
 }
 
 static void tranche_done(void *data,
@@ -1776,6 +1792,109 @@ static char **get_displays_spanned(struct vo_wayland_state *wl)
     return names;
 }
 
+static void get_gpu_drm_formats(struct vo_wayland_state *wl)
+{
+#if HAVE_DRM
+    drmDevice *device = NULL;
+    drmModePlaneRes *res = NULL;
+    drmModePlane *plane = NULL;
+
+    if (drmGetDeviceFromDevId(wl->main_device_id, 0, &device) != 0) {
+        MP_WARN(wl, "Unable to get drm device from device id: %s\n", mp_strerror(errno));
+        goto done;
+    }
+
+    // Pick the first path we get and hope for the best.
+    char *path = NULL;
+    for (int i = 0; i < device->available_nodes; ++i) {
+        if (device->nodes[0]) {
+            path = device->nodes[0];
+            break;
+        }
+    }
+
+    if (!path || !path[0]) {
+        MP_WARN(wl, "Unable to find a valid drm device node.\n");
+        goto done;
+    }
+
+    int fd = open(path, O_RDWR | O_CLOEXEC);
+    if (fd < 0) {
+        MP_WARN(wl, "Unable to open DRM node path '%s': %s\n", path, mp_strerror(errno));
+        goto done;
+    }
+
+    // Need to set this in order to access plane information.
+    if (drmSetClientCap(fd, DRM_CLIENT_CAP_ATOMIC, 1)) {
+        MP_WARN(wl, "Unable to set DRM atomic cap: %s\n", mp_strerror(errno));
+        goto done;
+    }
+
+    res = drmModeGetPlaneResources(fd);
+    if (!res) {
+        MP_WARN(wl, "Unable to get DRM plane resources: %s\n", mp_strerror(errno));
+        goto done;
+    }
+
+    if (!res->count_planes) {
+        MP_WARN(wl, "No DRM planes were found.\n");
+        goto done;
+    }
+
+    // Only check the formats on the first primary plane we find as a crude guess.
+    int index = -1;
+    for (int i = 0; i < res->count_planes; ++i) {
+	    drmModeObjectProperties *props = NULL;
+		props = drmModeObjectGetProperties(fd, res->planes[i], DRM_MODE_OBJECT_PLANE);
+        if (!props) {
+            MP_VERBOSE(wl, "Unable to get DRM plane properties: %s\n", mp_strerror(errno));
+            continue;
+        }
+        for (int j = 0; j < props->count_props; ++j) {
+		    drmModePropertyRes *prop = drmModeGetProperty(fd, props->props[j]);
+            if (!prop) {
+                MP_VERBOSE(wl, "Unable to get DRM plane property: %s\n", mp_strerror(errno));
+                continue;
+            }
+            if (strcmp(prop->name, "type") == 0) {
+                for (int k = 0; k < prop->count_values; ++k) {
+                    if (prop->values[k] == DRM_PLANE_TYPE_PRIMARY)
+                        index = i;
+                }
+            }
+		    drmModeFreeProperty(prop);
+            if (index > -1)
+                break;
+        }
+        drmModeFreeObjectProperties(props);
+        if (index > -1)
+            break;
+    }
+
+    if (index == -1) {
+        MP_WARN(wl, "Unable to get DRM plane: %s\n", mp_strerror(errno));
+        goto done;
+    }
+
+    plane = drmModeGetPlane(fd, res->planes[index]);
+    wl->num_gpu_formats = plane->count_formats;
+
+    if (wl->gpu_formats)
+        talloc_free(wl->gpu_formats);
+
+    wl->gpu_formats = talloc_zero_array(wl, int, wl->num_gpu_formats);
+    for (int i = 0; i < wl->num_gpu_formats; ++i) {
+        MP_DBG(wl, "DRM primary plane supports drm format: %s\n", mp_tag_str(plane->formats[i]));
+        wl->gpu_formats[i] = plane->formats[i];
+    }
+
+done:
+    drmModeFreePlane(plane);
+    drmModeFreePlaneResources(res);
+    drmFreeDevice(&device);
+#endif
+}
+
 static int get_mods(struct vo_wayland_seat *s)
 {
     static char* const mod_names[] = {

video/out/wayland_common.h

@@ -102,10 +102,13 @@ struct vo_wayland_state {
     struct zwp_idle_inhibitor_v1 *idle_inhibitor;
 
     /* linux-dmabuf */
+    dev_t main_device_id;
     struct zwp_linux_dmabuf_v1 *dmabuf;
     struct zwp_linux_dmabuf_feedback_v1 *dmabuf_feedback;
     compositor_format *compositor_format_map;
     uint32_t compositor_format_size;
+    uint32_t *gpu_formats;
+    int num_gpu_formats;
 
     /* presentation-time */
     struct wp_presentation  *presentation;

video/out/wldmabuf/ra_wldmabuf.c

@@ -28,12 +28,28 @@ static void destroy(struct ra *ra)
     talloc_free(ra->priv);
 }
 
-bool ra_compatible_format(struct ra* ra, uint32_t drm_format, uint64_t modifier)
+bool ra_compatible_format(struct ra *ra, int imgfmt, uint32_t drm_format, uint64_t modifier)
 {
-    struct priv* p = ra->priv;
+    struct priv *p = ra->priv;
     struct vo_wayland_state *wl = p->vo->wl;
     const compositor_format *formats = wl->compositor_format_map;
 
+
+    // If we were able to make the DRM query, filter out the GPU formats.
+    // If not, just assume they all work and hope for the best.
+    if (wl->gpu_formats) {
+        bool supported_gpu_format = false;
+        for (int i = 0; i < wl->num_gpu_formats; i++) {
+            if (drm_format == wl->gpu_formats[i]) {
+                supported_gpu_format = true;
+                break;
+            }
+        }
+        if (!supported_gpu_format)
+            return false;
+    }
+
+    // Always check if the compositor supports the format.
     for (int i = 0; i < wl->compositor_format_size / sizeof(compositor_format); i++) {
         if (drm_format == formats[i].format && modifier == formats[i].modifier)
             return true;

video/out/wldmabuf/ra_wldmabuf.h

@@ -19,5 +19,5 @@
 #include "video/out/wayland_common.h"
 
 struct ra *ra_create_wayland(struct mp_log *log, struct vo *vo);
-bool ra_compatible_format(struct ra* ra, uint32_t drm_format, uint64_t modifier);
+bool ra_compatible_format(struct ra *ra, int imgfmt, uint32_t drm_format, uint64_t modifier);
 bool ra_is_wldmabuf(struct ra *ra);