mzte-mpv: add live chat to subtitle transcoder

plugins/README.md

@@ -4,5 +4,5 @@ This directory contains a set of plugins for various software.
 
 ## list
 
-- `mpv-sbskip`: a MPV plugin written in Zig to skip SponsorBlock segments marked by yt-dlp.
+- `mzte-mpv`: a MPV plugin written in Zig to skip SponsorBlock segments marked by yt-dlp and convert downloaded YouTube live_chat.json files into subtitles, passed into mpv.
 - `tampermonkey-mzte-css`: a Tampermonkey userscript, which injects some minimal CSS into every page, which removes rounded corners (fuck rounded corners!) and improves fonts.

plugins/mzte-mpv/build.zig

@@ -5,7 +5,7 @@ pub fn build(b: *std.Build) void {
     const optimize = b.standardOptimizeOption(.{});
 
     const lib = b.addSharedLibrary(.{
-        .name = "mpv-sbskip",
+        .name = "mzte-mpv",
         .root_source_file = .{ .path = "src/main.zig" },
         .target = target,
         .optimize = optimize,
@@ -17,7 +17,7 @@ pub fn build(b: *std.Build) void {
         // this is not a standard MPV installation path, but instead one that makes sense.
         // this requires a symlink ../../../.local/share/mpv/scripts => ~/.config/mpv/scripts
         .dest_dir = .{ .override = .{ .custom = "share/mpv/scripts" } },
-        .dest_sub_path = "sbskip.so",
+        .dest_sub_path = "mzte-mpv.so",
     });
     b.getInstallStep().dependOn(&install_step.step);
 }

plugins/mzte-mpv/src/main.zig

@@ -0,0 +1,53 @@
+const std = @import("std");
+const c = ffi.c;
+
+const ffi = @import("ffi.zig");
+const util = @import("util.zig");
+
+pub const std_options = struct {
+    pub const log_level = .debug;
+    pub fn logFn(
+        comptime message_level: std.log.Level,
+        comptime scope: @TypeOf(.enum_literal),
+        comptime format: []const u8,
+        args: anytype,
+    ) void {
+        _ = scope;
+
+        const stderr = std.io.getStdErr().writer();
+
+        stderr.print("[mzte-mpv {s}] " ++ format ++ "\n", .{@tagName(message_level)} ++ args) catch return;
+    }
+};
+
+export fn mpv_open_cplugin(handle: *c.mpv_handle) callconv(.C) c_int {
+    tryMain(handle) catch |e| {
+        if (@errorReturnTrace()) |ert|
+            std.debug.dumpStackTrace(ert.*);
+        std.log.err("FATAL: {}\n", .{e});
+        return -1;
+    };
+    return 0;
+}
+
+fn tryMain(mpv: *c.mpv_handle) !void {
+    var modules = .{
+        @import("modules/LiveChat.zig").create(),
+        @import("modules/SBSkip.zig").create(),
+    };
+    // need this weird loop here for pointer access for fields to work
+    inline for (comptime std.meta.fieldNames(@TypeOf(modules))) |f|
+        try @field(modules, f).setup(mpv);
+    defer inline for (comptime std.meta.fieldNames(@TypeOf(modules))) |f|
+        @field(modules, f).deinit();
+
+    std.log.info("loaded with client name '{s}'", .{c.mpv_client_name(mpv)});
+
+    while (true) {
+        const ev = @as(*c.mpv_event, c.mpv_wait_event(mpv, -1));
+        try ffi.checkMpvError(ev.@"error");
+        inline for (comptime std.meta.fieldNames(@TypeOf(modules))) |f|
+            try @field(modules, f).onEvent(mpv, ev);
+        if (ev.event_id == c.MPV_EVENT_SHUTDOWN) break;
+    }
+}

plugins/mzte-mpv/src/modules/LiveChat.zig

@@ -0,0 +1,178 @@
+//! This module will find .live_chat.json files from yt-dlp and transcode them to WEBVTT, which
+//! is then transferred to mpv via a pipe.
+//! The live_chat file must be next to the video being viewed.
+const std = @import("std");
+const c = ffi.c;
+
+const ffi = @import("../ffi.zig");
+const util = @import("../util.zig");
+
+// Zig segfaults when this is a ZST
+padding: u1 = 0,
+
+const LiveChat = @This();
+
+pub fn onEvent(self: *LiveChat, mpv: *c.mpv_handle, ev: *c.mpv_event) !void {
+    _ = self;
+    switch (ev.event_id) {
+        c.MPV_EVENT_PROPERTY_CHANGE => {
+            const evprop: *c.mpv_event_property = @ptrCast(@alignCast(ev.data));
+            if (std.mem.eql(u8, std.mem.span(evprop.name), "stream-open-filename")) {
+                var buf: [std.fs.MAX_PATH_BYTES]u8 = undefined;
+
+                const str = std.mem.span((@as(?*[*:0]const u8, @ptrCast(@alignCast(evprop.data))) orelse return).*);
+
+                // Don't check live_chat for non-file streams
+                if (std.mem.containsAtLeast(u8, str, 1, "://")) return;
+                const fname = fname: {
+                    const dot_idx = std.mem.lastIndexOfScalar(u8, str, '.') orelse return;
+                    break :fname try std.fmt.bufPrintZ(&buf, "{s}.live_chat.json", .{str[0..dot_idx]});
+                };
+                const file = std.fs.cwd().openFileZ(fname, .{}) catch |e| switch (e) {
+                    error.FileNotFound => return,
+                    else => return e,
+                };
+                errdefer file.close();
+                std.log.info("initializing subtitle transcoder: {s}", .{fname});
+
+                const pipe = try std.os.pipe2(0);
+
+                // This needs to be done here instead of the separate thread. MPV will instantly
+                // give up if there's nothing to be read from the pipe when the command is called.
+                try (std.fs.File{ .handle = pipe[1] }).writer().writeAll(
+                    \\WEBVTT - MZTE-MPV transcoded live stream chat
+                    \\
+                    \\00:00.000 --> 00:05.000
+                    \\[MZTE-MPV] Live chat subtitle transcoder initialized
+                    \\
+                    \\
+                );
+
+                const sub_addr = try std.fmt.bufPrintZ(&buf, "fdclose://{}", .{pipe[0]});
+                try ffi.checkMpvError(c.mpv_command_async(
+                    mpv,
+                    0,
+                    @constCast(&[_:null]?[*:0]const u8{ "sub-add", sub_addr.ptr, "select", "MZTE-MPV live chat" }),
+                ));
+
+                // Quite stupidly, MPV will wait until the WHOLE subtitle stream is received before
+                // adding the track. We still do this in a separate thread so we don't have to
+                // buffer the WEBVTT data and MPV can concurrently decode it.
+                (try std.Thread.spawn(.{}, transcoderThread, .{ file, pipe[1] })).detach();
+            }
+        },
+        else => {},
+    }
+}
+
+fn transcoderThread(jsonf: std.fs.File, pipefd: std.c.fd_t) !void {
+    defer jsonf.close();
+    var pipe = std.fs.File{ .handle = pipefd };
+    defer pipe.close();
+
+    var writer = std.io.bufferedWriter(pipe.writer());
+
+    try writer.flush();
+
+    var reader = std.io.bufferedReader(jsonf.reader());
+    var line_buf = std.ArrayList(u8).init(std.heap.c_allocator);
+    defer line_buf.deinit();
+
+    while (true) {
+        line_buf.clearRetainingCapacity();
+        reader.reader().streamUntilDelimiter(line_buf.writer(), '\n', null) catch |e| switch (e) {
+            error.EndOfStream => break,
+            else => return e,
+        };
+        processLine(line_buf.items, pipe.writer()) catch |e| {
+            std.log.warn("failed to parse chat entry: {}", .{e});
+        };
+    }
+
+    try writer.flush();
+}
+
+/// I have yet to find who is responsible for this but oh boy...
+const ChatEntry = struct {
+    replayChatItemAction: struct {
+        actions: []struct {
+            addChatItemAction: struct {
+                item: struct {
+                    liveChatTextMessageRenderer: struct {
+                        message: struct {
+                            runs: []struct {
+                                text: ?[]u8 = null,
+                            },
+                        },
+                        authorName: struct {
+                            simpleText: []u8,
+                        },
+                    },
+                },
+            },
+        },
+        videoOffsetTimeMsec: usize,
+    },
+};
+
+const WebVttTime = struct {
+    ms: usize,
+
+    pub fn format(
+        self: WebVttTime,
+        comptime _: []const u8,
+        _: std.fmt.FormatOptions,
+        writer: anytype,
+    ) !void {
+        const time: @Vector(4, usize) = @splat(self.ms);
+        const div: @Vector(4, usize) = .{ std.time.ms_per_hour, std.time.ms_per_min, std.time.ms_per_s, 1 };
+        const mod: @Vector(4, usize) = .{ 1, 60, 60, 1000 };
+        const times = @divTrunc(time, div) % mod;
+
+        try writer.print("{d:0>2}:{d:0>2}:{d:0>2}.{d:0>3}", .{ times[0], times[1], times[2], times[3] });
+    }
+};
+
+fn processLine(line: []const u8, pipe: anytype) !void {
+    const parsed = try std.json.parseFromSlice(
+        ChatEntry,
+        std.heap.c_allocator,
+        line,
+        .{ .ignore_unknown_fields = true },
+    );
+    defer parsed.deinit();
+
+    // Show chat messages for 5 seconds
+    const ms = parsed.value.replayChatItemAction.videoOffsetTimeMsec;
+    try pipe.print("{} --> {}\n", .{ WebVttTime{ .ms = ms }, WebVttTime{ .ms = ms + 5000 } });
+
+    for (parsed.value.replayChatItemAction.actions) |act| {
+        try pipe.print("<b>&lt;{s}&gt;:</b> ", .{
+            act.addChatItemAction.item.liveChatTextMessageRenderer.authorName.simpleText,
+        });
+        for (act.addChatItemAction.item.liveChatTextMessageRenderer.message.runs) |seg| {
+            if (seg.text) |txt| {
+                std.mem.replaceScalar(u8, txt, '\n', '\\');
+                try pipe.writeAll(txt);
+            } else {
+                // Emojis and such
+                try pipe.writeAll("&lt;?&gt;");
+            }
+        }
+        try pipe.writeByte('\n');
+    }
+    try pipe.writeByte('\n');
+}
+
+pub fn create() LiveChat {
+    return .{};
+}
+
+pub fn setup(self: *LiveChat, mpv: *c.mpv_handle) !void {
+    _ = self;
+    try ffi.checkMpvError(c.mpv_observe_property(mpv, 0, "stream-open-filename", c.MPV_FORMAT_STRING));
+}
+
+pub fn deinit(self: *LiveChat) void {
+    _ = self;
+}

plugins/mzte-mpv/src/modules/SBSkip.zig

@@ -1,7 +1,15 @@
 const std = @import("std");
-const ffi = @import("ffi.zig");
 const c = ffi.c;
 
+const ffi = @import("../ffi.zig");
+const util = @import("../util.zig");
+
+const ChapterSet = std.AutoHashMap(isize, void);
+
+skipped_chapters: ChapterSet,
+
+const SBSkip = @This();
+
 const blacklist = std.ComptimeStringMap(void, .{
     .{ "Endcards/Credits", {} },
     .{ "Interaction Reminder", {} },
@@ -12,77 +20,27 @@ const blacklist = std.ComptimeStringMap(void, .{
     .{ "Unpaid/Self Promotion", {} },
 });
 
-pub const std_options = struct {
-    pub const log_level = .debug;
-    pub fn logFn(
-        comptime message_level: std.log.Level,
-        comptime scope: @TypeOf(.enum_literal),
-        comptime format: []const u8,
-        args: anytype,
-    ) void {
-        _ = scope;
-
-        const stderr = std.io.getStdErr().writer();
-
-        stderr.print("[sbskip {s}] " ++ format ++ "\n", .{@tagName(message_level)} ++ args) catch return;
+pub fn onEvent(self: *SBSkip, mpv: *c.mpv_handle, ev: *c.mpv_event) !void {
+    switch (ev.event_id) {
+        c.MPV_EVENT_PROPERTY_CHANGE => {
+            const evprop: *c.mpv_event_property = @ptrCast(@alignCast(ev.data));
+            if (std.mem.eql(u8, std.mem.span(evprop.name), "chapter")) {
+                const chapter_id_ptr = @as(?*i64, @ptrCast(@alignCast(evprop.data)));
+                if (chapter_id_ptr) |chptr|
+                    try self.onChapterChange(mpv, @intCast(chptr.*));
+            }
+        },
+        c.MPV_EVENT_FILE_LOADED => self.skipped_chapters.clearRetainingCapacity(),
+        else => {},
     }
-};
-
-export fn mpv_open_cplugin(handle: *c.mpv_handle) callconv(.C) c_int {
-    tryMain(handle) catch |e| {
-        if (@errorReturnTrace()) |ert|
-            std.debug.dumpStackTrace(ert.*);
-        std.log.err("{}", .{e});
-        return -1;
-    };
-    return 0;
-}
-
-fn tryMain(mpv: *c.mpv_handle) !void {
-    var skipped_chapter_ids = std.AutoHashMap(isize, void).init(std.heap.c_allocator);
-    defer skipped_chapter_ids.deinit();
-
-    try ffi.checkMpvError(c.mpv_observe_property(mpv, 0, "chapter", c.MPV_FORMAT_INT64));
-
-    std.log.info("loaded with client name '{s}'", .{c.mpv_client_name(mpv)});
-
-    while (true) {
-        const ev = @as(*c.mpv_event, c.mpv_wait_event(mpv, -1));
-        try ffi.checkMpvError(ev.@"error");
-        switch (ev.event_id) {
-            c.MPV_EVENT_PROPERTY_CHANGE => {
-                const evprop: *c.mpv_event_property = @ptrCast(@alignCast(ev.data));
-                if (std.mem.eql(u8, "chapter", std.mem.span(evprop.name))) {
-                    std.debug.assert(evprop.format == c.MPV_FORMAT_INT64);
-                    const chapter_id_ptr = @as(?*i64, @ptrCast(@alignCast(evprop.data)));
-                    if (chapter_id_ptr) |chptr|
-                        try onChapterChange(mpv, @intCast(chptr.*), &skipped_chapter_ids);
-                }
-            },
-            c.MPV_EVENT_FILE_LOADED => skipped_chapter_ids.clearRetainingCapacity(),
-            c.MPV_EVENT_SHUTDOWN => break,
-            else => {},
-        }
-    }
-}
-
-fn msg(mpv: *c.mpv_handle, comptime fmt: []const u8, args: anytype) !void {
-    std.log.info(fmt, args);
-
-    var buf: [1024 * 4]u8 = undefined;
-    const osd_msg = try std.fmt.bufPrintZ(&buf, "[sbskip] " ++ fmt, args);
-    try ffi.checkMpvError(c.mpv_command(
-        mpv,
-        @constCast(&[_:null]?[*:0]const u8{ "show-text", osd_msg, "4000" }),
-    ));
 }
 
 fn onChapterChange(
+    self: *SBSkip,
     mpv: *c.mpv_handle,
     chapter_id: isize,
-    skipped: *std.AutoHashMap(isize, void),
 ) !void {
-    if (chapter_id < 0 or skipped.contains(chapter_id))
+    if (chapter_id < 0 or self.skipped_chapters.contains(chapter_id))
         return;
 
     // fuck these ubiquitous duck typing implementations everywhere! we have structs, for fuck's sake!
@@ -125,8 +83,8 @@ fn onChapterChange(
             c.MPV_FORMAT_DOUBLE,
             @constCast(&end_time),
         ));
-        try skipped.put(chapter_id, {});
-        try msg(mpv, "skipped: {s}", .{reason});
+        try self.skipped_chapters.put(chapter_id, {});
+        try util.msg(mpv, "skipped: {s}", .{reason});
     }
 }
 
@@ -168,3 +126,18 @@ const Chapter = struct {
         return null;
     }
 };
+
+pub fn setup(self: *SBSkip, mpv: *c.mpv_handle) !void {
+    _ = self;
+    try ffi.checkMpvError(c.mpv_observe_property(mpv, 0, "chapter", c.MPV_FORMAT_INT64));
+}
+
+pub fn create() SBSkip {
+    return .{
+        .skipped_chapters = ChapterSet.init(std.heap.c_allocator),
+    };
+}
+
+pub fn deinit(self: *SBSkip) void {
+    self.skipped_chapters.deinit();
+}

plugins/mzte-mpv/src/util.zig

@@ -0,0 +1,16 @@
+const std = @import("std");
+const c = ffi.c;
+
+const ffi = @import("ffi.zig");
+
+pub fn msg(mpv: *c.mpv_handle, comptime fmt: []const u8, args: anytype) !void {
+    std.log.info(fmt, args);
+
+    var buf: [1024 * 4]u8 = undefined;
+    const osd_msg = try std.fmt.bufPrintZ(&buf, "[sbskip] " ++ fmt, args);
+    try ffi.checkMpvError(c.mpv_command(
+        mpv,
+        @constCast(&[_:null]?[*:0]const u8{ "show-text", osd_msg, "4000" }),
+    ));
+}
+

setup/commands/install-plugins.rkt

@@ -3,5 +3,5 @@
 (provide run)
 
 (define (run)
-  (install-zig "plugins/mpv-sbskip")
+  (install-zig "plugins/mzte-mpv")
   (build-haxe "plugins/tampermonkey-mzte-css"))