sub: destroy/recreate ASS_Renderer when disabling/enablings subs

Keeping ASS_Renderers around for a potentially large number of subtitle
tracks could lead to excessive memory usage, especially since the libass
cache is broken (caches even unneeded data), and might consume up to
~500MB of memory for no reason.

player/sub.c

@@ -55,6 +55,7 @@ void uninit_sub(struct MPContext *mpctx, int order)
 {
     if (mpctx->d_sub[order]) {
         reset_subtitles(mpctx, order);
+        sub_select(mpctx->d_sub[order], false);
         mpctx->d_sub[order] = NULL; // not destroyed
         osd_set_sub(mpctx->osd, OSDTYPE_SUB + order, NULL);
         reselect_demux_streams(mpctx);
@@ -183,6 +184,7 @@ void reinit_subs(struct MPContext *mpctx, int order)
         track->dec_sub = sub_create(mpctx->global);
     mpctx->d_sub[order] = track->dec_sub;
 
+    sub_select(track->dec_sub, true);
     reinit_subdec(mpctx, track);
     osd_set_sub(mpctx->osd, OSDTYPE_SUB + order, track->dec_sub);
     sub_control(track->dec_sub, SD_CTRL_SET_TOP, &(bool){!!order});

sub/dec_sub.c

@@ -301,6 +301,14 @@ void sub_reset(struct dec_sub *sub)
     pthread_mutex_unlock(&sub->lock);
 }
 
+void sub_select(struct dec_sub *sub, bool selected)
+{
+    pthread_mutex_lock(&sub->lock);
+    if (sub->sd && sub->sd->driver->select)
+        sub->sd->driver->select(sub->sd, selected);
+    pthread_mutex_unlock(&sub->lock);
+}
+
 int sub_control(struct dec_sub *sub, enum sd_ctrl cmd, void *arg)
 {
     int r = CONTROL_UNKNOWN;

sub/dec_sub.h

@@ -41,6 +41,7 @@ void sub_get_bitmaps(struct dec_sub *sub, struct mp_osd_res dim, double pts,
                      struct sub_bitmaps *res);
 char *sub_get_text(struct dec_sub *sub, double pts);
 void sub_reset(struct dec_sub *sub);
+void sub_select(struct dec_sub *sub, bool selected);
 
 int sub_control(struct dec_sub *sub, enum sd_ctrl cmd, void *arg);
 

sub/sd.h

@@ -29,6 +29,7 @@ struct sd_functions {
     int  (*init)(struct sd *sd);
     void (*decode)(struct sd *sd, struct demux_packet *packet);
     void (*reset)(struct sd *sd);
+    void (*select)(struct sd *sd, bool selected);
     void (*uninit)(struct sd *sd);
 
     bool (*accepts_packet)(struct sd *sd); // implicit default if NULL: true

sub/sd_ass.c

@@ -131,6 +131,22 @@ static bool supports_format(const char *format)
            lavc_conv_supports_format(format);
 }
 
+static void enable_output(struct sd *sd, bool enable)
+{
+    struct sd_ass_priv *ctx = sd->priv;
+    if (enable == !!ctx->ass_renderer)
+        return;
+    if (ctx->ass_renderer) {
+        ass_renderer_done(ctx->ass_renderer);
+        ctx->ass_renderer = NULL;
+    } else {
+        ctx->ass_renderer = ass_renderer_init(ctx->ass_library);
+
+        mp_ass_configure_fonts(ctx->ass_renderer, sd->opts->sub_text_style,
+                               sd->global, sd->log);
+    }
+}
+
 static int init(struct sd *sd)
 {
     struct MPOpts *opts = sd->opts;
@@ -157,11 +173,6 @@ static int init(struct sd *sd)
     if (opts->ass_style_override)
         ass_set_style_overrides(ctx->ass_library, opts->ass_force_style_list);
 
-    ctx->ass_renderer = ass_renderer_init(ctx->ass_library);
-
-    mp_ass_configure_fonts(ctx->ass_renderer, opts->sub_text_style,
-                           sd->global, sd->log);
-
     ctx->ass_track = ass_new_track(ctx->ass_library);
     if (!ctx->is_converted)
         ctx->ass_track->track_type = TRACK_TYPE_ASS;
@@ -189,6 +200,8 @@ static int init(struct sd *sd)
 
     ctx->sub_speed *= opts->sub_speed;
 
+    enable_output(sd, true);
+
     return 0;
 }
 
@@ -384,11 +397,11 @@ static void get_bitmaps(struct sd *sd, struct mp_osd_res dim, double pts,
     bool no_ass = !opts->ass_enabled || ctx->on_top;
     bool converted = ctx->is_converted || no_ass;
     ASS_Track *track = no_ass ? ctx->shadow_track : ctx->ass_track;
+    ASS_Renderer *renderer = ctx->ass_renderer;
 
-    if (pts == MP_NOPTS_VALUE)
+    if (pts == MP_NOPTS_VALUE || !renderer)
         return;
 
-    ASS_Renderer *renderer = ctx->ass_renderer;
     double scale = dim.display_par;
     if (!converted && (!opts->ass_style_override ||
                        opts->ass_vsfilter_aspect_compat))
@@ -577,7 +590,7 @@ static void uninit(struct sd *sd)
     if (ctx->converter)
         lavc_conv_uninit(ctx->converter);
     ass_free_track(ctx->ass_track);
-    ass_renderer_done(ctx->ass_renderer);
+    enable_output(sd, false);
     ass_library_done(ctx->ass_library);
 }
 
@@ -615,6 +628,7 @@ const struct sd_functions sd_ass = {
     .get_text = get_text,
     .control = control,
     .reset = reset,
+    .select = enable_output,
     .uninit = uninit,
 };