player: make force-window in auto-profiles actually work

The previous commit was incomplete (and I didn't notice due to a broken
test procedure).

The annoying part is that actually creating the VO was separate; redo
this and merge the code for this into handle_force_window() as well.
This will also make implementing proper reaction to runtime option
changes easier. (Only the part for actually listening to option changes
is missing.)

player/core.h

@@ -488,7 +488,7 @@ void execute_queued_seek(struct MPContext *mpctx);
 void run_playloop(struct MPContext *mpctx);
 void mp_idle(struct MPContext *mpctx);
 void idle_loop(struct MPContext *mpctx);
-void handle_force_window(struct MPContext *mpctx, bool reconfig);
+int handle_force_window(struct MPContext *mpctx, bool reconfig);
 void add_frame_pts(struct MPContext *mpctx, double pts);
 int get_past_frame_durations(struct MPContext *mpctx, double *fd, int num);
 void seek_to_last_frame(struct MPContext *mpctx);

player/loadfile.c

@@ -1072,8 +1072,7 @@ static void play_current_file(struct MPContext *mpctx)
 
     mpctx->max_frames = opts->play_frames;
 
-    if (opts->force_vo == 2)
+    handle_force_window(mpctx, false);
-        handle_force_window(mpctx, false);
 
     MP_INFO(mpctx, "Playing: %s\n", mpctx->filename);
 

player/main.c

@@ -466,22 +466,8 @@ int mp_initialize(struct MPContext *mpctx, char **options)
     if (opts->consolecontrols && cas_terminal_owner(mpctx, mpctx))
         terminal_setup_getch(mpctx->input);
 
-    if (opts->force_vo) {
+    if (handle_force_window(mpctx, false) < 0)
-        struct vo_extra ex = {
+        return -1;
-            .input_ctx = mpctx->input,
-            .osd = mpctx->osd,
-            .encode_lavc_ctx = mpctx->encode_lavc_ctx,
-        };
-        mpctx->video_out = init_best_video_out(mpctx->global, &ex);
-        if (!mpctx->video_out) {
-            MP_FATAL(mpctx, "Error opening/initializing "
-                    "the selected video_out (-vo) device.\n");
-            return -1;
-        }
-        if (opts->force_vo == 2)
-            handle_force_window(mpctx, false);
-        mpctx->mouse_cursor_visible = true;
-    }
 
 #if !defined(__MINGW32__)
     mpctx->ipc_ctx = mp_init_ipc(mpctx->clients, mpctx->global);

player/playloop.c

@@ -863,18 +863,35 @@ static void handle_chapter_change(struct MPContext *mpctx)
 
 // Execute a forceful refresh of the VO window, if it hasn't had a valid frame
 // for a while. The problem is that a VO with no valid frame (vo->hasframe==0)
-// doesn't redraw video and doesn't OSD interaction. So screw it, hard.
+// doesn't redraw video and doesn't do OSD interaction. So screw it, hard.
 // It also closes the VO if force_window or video display is not active.
-void handle_force_window(struct MPContext *mpctx, bool reconfig)
+int handle_force_window(struct MPContext *mpctx, bool reconfig)
 {
     // Don't interfere with real video playback
     if (mpctx->d_video)
-        return;
+        return 0;
 
-    if (!mpctx->opts->force_vo && mpctx->video_out)
+    if (!mpctx->opts->force_vo) {
         uninit_video_out(mpctx);
+        return 0;
+    }
 
-    if (mpctx->video_out && (!mpctx->video_out->config_ok || reconfig)) {
+    if (!mpctx->video_out) {
+        struct vo_extra ex = {
+            .input_ctx = mpctx->input,
+            .osd = mpctx->osd,
+            .encode_lavc_ctx = mpctx->encode_lavc_ctx,
+        };
+        mpctx->video_out = init_best_video_out(mpctx->global, &ex);
+        if (!mpctx->video_out)
+            goto err;
+        mpctx->mouse_cursor_visible = true;
+    }
+
+    if (mpctx->opts->force_vo != 2 && !mpctx->playback_initialized)
+        return 0;
+
+    if (!mpctx->video_out->config_ok || reconfig) {
         struct vo *vo = mpctx->video_out;
         MP_INFO(mpctx, "Creating non-video VO window.\n");
         // Pick whatever works
@@ -894,16 +911,21 @@ void handle_force_window(struct MPContext *mpctx, bool reconfig)
             .w = w,   .h = h,
             .d_w = w, .d_h = h,
         };
-        if (vo_reconfig(vo, &p, 0) < 0) {
+        if (vo_reconfig(vo, &p, 0) < 0)
-            mpctx->opts->force_vo = 0;
+            goto err;
-            uninit_video_out(mpctx);
-            return;
-        }
         vo_control(vo, VOCTRL_RESTORE_SCREENSAVER, NULL);
         vo_set_paused(vo, true);
         vo_redraw(vo);
         mp_notify(mpctx, MPV_EVENT_VIDEO_RECONFIG, NULL);
     }
+
+    return 0;
+
+err:
+    mpctx->opts->force_vo = 0;
+    uninit_video_out(mpctx);
+    MP_FATAL(mpctx, "Error opening/initializing the VO window.\n");
+    return -1;
 }
 
 // Potentially needed by some Lua scripts, which assume TICK always comes.