refactor(thumbnail): move code into react component

src/utils/thumbnailCreator.ts

@@ -1,51 +0,0 @@
-export interface Thumbnail {
-  from: number;
-  to: number;
-  imgUrl: string;
-}
-export const SCALE_FACTOR = 1;
-export default async function* extractThumbnails(
-  videoUrl: string,
-  numThumbnails: number
-): AsyncGenerator<Thumbnail, Thumbnail> {
-  const canvas = document.createElement("canvas");
-  const ctx = canvas.getContext("2d");
-  if (!ctx) return { from: -1, to: -1, imgUrl: "" };
-  const video = document.createElement("video");
-  video.src = videoUrl;
-  video.crossOrigin = "anonymous";
-
-  // Wait for the video metadata to load
-  await new Promise((resolve, reject) => {
-    video.addEventListener("loadedmetadata", resolve);
-    video.addEventListener("error", reject);
-  });
-
-  canvas.height = video.videoHeight * SCALE_FACTOR;
-  canvas.width = video.videoWidth * SCALE_FACTOR;
-
-  for (let i = 0; i <= numThumbnails; i += 1) {
-    const from = (i / (numThumbnails + 1)) * video.duration;
-    const to = ((i + 1) / (numThumbnails + 1)) * video.duration;
-
-    // Seek to the specified time
-    video.currentTime = from;
-    await new Promise((resolve) => {
-      video.addEventListener("seeked", resolve);
-    });
-
-    // Draw the video frame on the canvas
-    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
-
-    // Convert the canvas to a data URL and add it to the list of thumbnails
-    const imgUrl = canvas.toDataURL();
-
-    yield {
-      from,
-      to,
-      imgUrl,
-    };
-  }
-
-  return { from: -1, to: -1, imgUrl: "" };
-}

src/video/components/VideoPlayerBase.tsx

@@ -7,6 +7,7 @@ import { useInterface } from "@/video/state/logic/interface";
 import { useMeta } from "@/video/state/logic/meta";
 
 import { MetaAction } from "./actions/MetaAction";
+import ThumbnailGeneratorInternal from "./internal/ThumbnailGeneratorInternal";
 import { VideoElementInternal } from "./internal/VideoElementInternal";
 import {
   VideoPlayerContextProvider,
@@ -48,6 +49,7 @@ function VideoPlayerBaseWithState(props: VideoPlayerBaseProps) {
         ].join(" ")}
       >
         <MetaAction />
+        <ThumbnailGeneratorInternal />
         <VideoElementInternal autoPlay={props.autoPlay} />
         <CastingInternal />
         <WrapperRegisterInternal wrapper={ref.current} />

src/video/components/actions/ThumbnailAction.tsx

@@ -1,4 +1,4 @@
-import { RefObject } from "react";
+import { RefObject, useMemo } from "react";
 
 import { Icon, Icons } from "@/components/Icon";
 import { formatSeconds } from "@/utils/formatSeconds";
@@ -7,6 +7,77 @@ import { VideoProgressEvent } from "@/video/state/logic/progress";
 import { useSource } from "@/video/state/logic/source";
 
 const THUMBNAIL_HEIGHT = 100;
+function position(
+  rectLeft: number,
+  rectWidth: number,
+  thumbnailWidth: number,
+  hoverPos: number
+): number {
+  const relativePosition = hoverPos - rectLeft;
+  if (relativePosition <= thumbnailWidth / 2) {
+    return rectLeft;
+  }
+  if (relativePosition >= rectWidth - thumbnailWidth / 2) {
+    return rectWidth + rectLeft - thumbnailWidth;
+  }
+  return relativePosition + rectLeft - thumbnailWidth / 2;
+}
+function useThumbnailWidth() {
+  const videoEl = useMemo(() => document.getElementsByTagName("video")[0], []);
+  const aspectRatio = videoEl.videoWidth / videoEl.videoHeight;
+  return THUMBNAIL_HEIGHT * aspectRatio;
+}
+
+function LoadingThumbnail({ pos }: { pos: number }) {
+  const videoEl = useMemo(() => document.getElementsByTagName("video")[0], []);
+  const aspectRatio = videoEl.videoWidth / videoEl.videoHeight;
+  const thumbnailWidth = THUMBNAIL_HEIGHT * aspectRatio;
+  return (
+    <div
+      className="absolute bottom-32 flex items-center justify-center rounded bg-black"
+      style={{
+        left: `${pos}px`,
+        width: `${thumbnailWidth}px`,
+        height: `${THUMBNAIL_HEIGHT}px`,
+      }}
+    >
+      <Icon
+        className="roll-infinite text-6xl text-bink-600"
+        icon={Icons.MOVIE_WEB}
+      />
+    </div>
+  );
+}
+
+function ThumbnailTime({ hoverTime, pos }: { hoverTime: number; pos: number }) {
+  const videoEl = useMemo(() => document.getElementsByTagName("video")[0], []);
+  const thumbnailWidth = useThumbnailWidth();
+  return (
+    <div
+      className="absolute bottom-24 text-white"
+      style={{
+        left: `${pos + thumbnailWidth / 2 - 18}px`,
+      }}
+    >
+      {formatSeconds(hoverTime, videoEl.duration > 60 * 60)}
+    </div>
+  );
+}
+
+function ThumbnailImage({ src, pos }: { src: string; pos: number }) {
+  const thumbnailWidth = useThumbnailWidth();
+  return (
+    <img
+      height={THUMBNAIL_HEIGHT}
+      width={thumbnailWidth}
+      className="absolute bottom-32 rounded"
+      src={src}
+      style={{
+        left: `${pos}px`,
+      }}
+    />
+  );
+}
 export default function ThumbnailAction({
   parentRef,
   hoverPosition,
@@ -18,63 +89,32 @@ export default function ThumbnailAction({
 }) {
   const descriptor = useVideoPlayerDescriptor();
   const source = useSource(descriptor);
+  const thumbnailWidth = useThumbnailWidth();
   if (!parentRef.current) return null;
-  const videoEl = document.getElementsByTagName("video")[0];
-  const aspectRatio = videoEl.videoWidth / videoEl.videoHeight;
   const rect = parentRef.current.getBoundingClientRect();
   if (!rect.width) return null;
+
   const hoverPercent = (hoverPosition - rect.left) / rect.width;
   const hoverTime = videoTime.duration * hoverPercent;
-
-  const thumbnailWidth = THUMBNAIL_HEIGHT * aspectRatio;
-  const pos = () => {
-    const relativePosition = hoverPosition - rect.left;
-    if (relativePosition <= thumbnailWidth / 2) {
-      return rect.left;
-    }
-    if (relativePosition >= rect.width - thumbnailWidth / 2) {
-      return rect.width + rect.left - thumbnailWidth;
-    }
-    return relativePosition + rect.left - thumbnailWidth / 2;
-  };
   const src = source.source?.thumbnails.find(
     (x) => x.from < hoverTime && x.to > hoverTime
   )?.imgUrl;
   return (
-    <div className="text-center">
+    <div className="pointer-events-none">
       {!src ? (
-        <div
+        <LoadingThumbnail
-          style={{
+          pos={position(rect.left, rect.width, thumbnailWidth, hoverPosition)}
-            left: `${pos()}px`,
+        />
-            width: `${thumbnailWidth}px`,
-            height: `${THUMBNAIL_HEIGHT}px`,
-          }}
-          className="absolute bottom-32 flex items-center justify-center rounded bg-black"
-        >
-          <Icon
-            className="roll-infinite text-6xl text-bink-600"
-            icon={Icons.MOVIE_WEB}
-          />
-        </div>
       ) : (
-        <img
+        <ThumbnailImage
-          height={THUMBNAIL_HEIGHT}
+          pos={position(rect.left, rect.width, thumbnailWidth, hoverPosition)}
-          width={thumbnailWidth}
-          style={{
-            left: `${pos()}px`,
-          }}
-          className="absolute bottom-32 rounded"
           src={src}
         />
       )}
-      <div
+      <ThumbnailTime
-        style={{
+        hoverTime={hoverTime}
-          left: `${pos() + thumbnailWidth / 2 - 18}px`,
+        pos={position(rect.left, rect.width, thumbnailWidth, hoverPosition)}
-        }}
+      />
-        className="absolute bottom-24 text-white"
-      >
-        {formatSeconds(hoverTime, videoEl.duration > 60 * 60)}
-      </div>
     </div>
   );
 }

src/video/components/internal/ThumbnailGeneratorInternal.tsx

@@ -0,0 +1,114 @@
+import Hls from "hls.js";
+import { RefObject, useCallback, useEffect, useRef, useState } from "react";
+
+import { MWStreamType } from "@/backend/helpers/streams";
+import { getPlayerState } from "@/video/state/cache";
+import { useVideoPlayerDescriptor } from "@/video/state/hooks";
+import { updateSource, useSource } from "@/video/state/logic/source";
+import { Thumbnail } from "@/video/state/types";
+
+async function* generate(
+  videoUrl: string,
+  streamType: MWStreamType,
+  videoRef: RefObject<HTMLVideoElement>,
+  canvasRef: RefObject<HTMLCanvasElement>,
+  numThumbnails = 20
+): AsyncGenerator<Thumbnail, Thumbnail> {
+  const video = videoRef.current;
+  const canvas = canvasRef.current;
+  if (!video) return { from: -1, to: -1, imgUrl: "" };
+  if (!canvas) return { from: -1, to: -1, imgUrl: "" };
+  console.log("extracting started", streamType.toString());
+  if (streamType === MWStreamType.HLS) {
+    const hls = new Hls();
+    console.log("new hls instance");
+
+    hls.attachMedia(video);
+    hls.loadSource(videoUrl);
+  }
+  await new Promise((resolve, reject) => {
+    video.addEventListener("loadedmetadata", resolve);
+    video.addEventListener("error", reject);
+  });
+
+  canvas.height = video.videoHeight * 1;
+  canvas.width = video.videoWidth * 1;
+  let i = 0;
+  while (i < numThumbnails) {
+    const from = i * video.duration;
+    const to = (i + 1) * video.duration;
+
+    // Seek to the specified time
+    video.currentTime = from;
+    console.log(from, to);
+    console.time("seek loaded");
+    await new Promise((resolve) => {
+      video.addEventListener("seeked", resolve);
+    });
+    console.timeEnd("seek loaded");
+    console.log("loaded", video.currentTime, streamType.toString());
+
+    const ctx = canvas.getContext("2d");
+    if (!ctx) return { from: -1, to: -1, imgUrl: "" };
+    // Draw the video frame on the canvas
+    ctx.drawImage(video, 0, 0, canvas.width, canvas.height);
+
+    // Convert the canvas to a data URL and add it to the list of thumbnails
+    const imgUrl = canvas.toDataURL();
+    i += 1;
+    yield {
+      from,
+      to,
+      imgUrl,
+    };
+  }
+
+  return { from: -1, to: -1, imgUrl: "" };
+}
+
+export default function ThumbnailGeneratorInternal() {
+  const videoRef = useRef<HTMLVideoElement>(document.createElement("video"));
+  const canvasRef = useRef<HTMLCanvasElement>(document.createElement("canvas"));
+  const descriptor = useVideoPlayerDescriptor();
+  const source = useSource(descriptor);
+  const thumbnails = useRef<Thumbnail[]>([]);
+  const abortController = useRef<AbortController>(new AbortController());
+  const generator = useCallback(
+    async (url: string, type: MWStreamType) => {
+      for await (const thumbnail of generate(url, type, videoRef, canvasRef)) {
+        if (abortController.current.signal.aborted) {
+          console.log("broke out of loop", type.toString());
+          break;
+        }
+
+        thumbnails.current = [...thumbnails.current, thumbnail];
+        const state = getPlayerState(descriptor);
+        if (!state.source) return;
+        console.log("ran");
+        state.source.thumbnails = thumbnails.current;
+        console.log(thumbnails.current);
+
+        updateSource(descriptor, state);
+        console.log("ran 2");
+      }
+    },
+    [descriptor]
+  );
+
+  useEffect(() => {
+    const state = getPlayerState(descriptor);
+    if (!state.source) return;
+    const { url, type } = state.source;
+    generator(url, type);
+  }, [descriptor, generator, source.source?.url]);
+
+  useEffect(() => {
+    const controller = abortController.current;
+    return () => {
+      console.log("abort");
+      controller.abort();
+    };
+  }, []);
+
+  return null;
+}

src/video/components/popouts/CaptionSettingsPopout.tsx

@@ -71,7 +71,7 @@ export function CaptionSettingsPopout(props: {
           </label>
           <div className="flex flex-row gap-2">
             {colors.map((color) => (
-              <CaptionColorSelector color={color} />
+              <CaptionColorSelector key={color} color={color} />
             ))}
           </div>
         </div>

src/video/state/logic/source.ts

@@ -1,11 +1,10 @@
 import { useEffect, useState } from "react";
 
 import { MWStreamQuality, MWStreamType } from "@/backend/helpers/streams";
-import { Thumbnail } from "@/utils/thumbnailCreator";
 
 import { getPlayerState } from "../cache";
 import { listenEvent, sendEvent, unlistenEvent } from "../events";
-import { VideoPlayerState } from "../types";
+import { Thumbnail, VideoPlayerState } from "../types";
 
 export type VideoSourceEvent = {
   source: null | {

src/video/state/providers/videoStateProvider.ts

@@ -11,7 +11,6 @@ import {
   canWebkitFullscreen,
   canWebkitPictureInPicture,
 } from "@/utils/detectFeatures";
-import extractThumbnails from "@/utils/thumbnailCreator";
 import {
   getStoredVolume,
   setStoredVolume,
@@ -64,7 +63,6 @@ export function createVideoStateProvider(
 ): VideoPlayerStateProvider {
   const player = playerEl;
   const state = getPlayerState(descriptor);
-
   return {
     getId() {
       return "video";
@@ -148,6 +146,16 @@ export function createVideoStateProvider(
 
       // reset before assign new one so the old HLS instance gets destroyed
       resetStateForSource(descriptor, state);
+      // update state
+      state.source = {
+        quality: source.quality,
+        type: source.type,
+        url: source.source,
+        caption: null,
+        embedId: source.embedId,
+        providerId: source.providerId,
+        thumbnails: [],
+      };
 
       if (source?.type === MWStreamType.HLS) {
         if (player.canPlayType("application/vnd.apple.mpegurl")) {
@@ -186,25 +194,6 @@ export function createVideoStateProvider(
         player.src = source.source;
       }
 
-      // update state
-      state.source = {
-        quality: source.quality,
-        type: source.type,
-        url: source.source,
-        caption: null,
-        embedId: source.embedId,
-        providerId: source.providerId,
-        thumbnails: [],
-      };
-
-      (async () => {
-        for await (const thumbnail of extractThumbnails(source.source, 20)) {
-          if (!state.source) return;
-          state.source.thumbnails = [...state.source.thumbnails, thumbnail];
-          updateSource(descriptor, state);
-        }
-      })();
-
       updateSource(descriptor, state);
     },
     setCaption(id, url) {

src/video/state/types.ts

@@ -6,10 +6,14 @@ import {
   MWStreamType,
 } from "@/backend/helpers/streams";
 import { DetailedMeta } from "@/backend/metadata/getmeta";
-import { Thumbnail } from "@/utils/thumbnailCreator";
 
 import { VideoPlayerStateProvider } from "./providers/providerTypes";
 
+export interface Thumbnail {
+  from: number;
+  to: number;
+  imgUrl: string;
+}
 export type VideoPlayerMeta = {
   meta: DetailedMeta;
   captions: MWCaption[];

src/views/SettingsModal.tsx

@@ -122,7 +122,7 @@ export default function SettingsModal(props: {
                   </label>
                   <div className="flex flex-row gap-2">
                     {colors.map((color) => (
-                      <CaptionColorSelector color={color} />
+                      <CaptionColorSelector key={color} color={color} />
                     ))}
                   </div>
                 </div>