new backend interfaces

src/backend/embeds/testEmbedScraper.ts

@@ -0,0 +1,20 @@
+import { MWEmbedType } from "../helpers/embed";
+import { registerEmbedScraper } from "../helpers/register";
+import { MWStreamType } from "../helpers/streams";
+
+registerEmbedScraper({
+  id: "testembed",
+  rank: 23,
+  for: MWEmbedType.OPENLOAD,
+
+  async getStream({ progress, url }) {
+    console.log("scraping url: ", url);
+    progress(25);
+    progress(50);
+    progress(75);
+    return {
+      streamUrl: "hello-world",
+      type: MWStreamType.MP4,
+    };
+  },
+});

src/backend/helpers/embed.ts

@@ -0,0 +1,24 @@
+import { MWStream } from "./streams";
+
+export enum MWEmbedType {
+  OPENLOAD = "openload",
+}
+
+export type MWEmbed = {
+  type: MWEmbedType | null;
+  url: string;
+};
+
+export type MWEmbedContext = {
+  progress(percentage: number): void;
+  url: string;
+};
+
+export type MWEmbedScraper = {
+  id: string;
+  for: MWEmbedType;
+  rank: number;
+  disabled?: boolean;
+
+  getStream(ctx: MWEmbedContext): Promise<MWStream>;
+};

src/backend/helpers/provider.ts

@@ -0,0 +1,23 @@
+import { MWMediaType } from "../metadata/types";
+import { MWEmbed } from "./embed";
+import { MWStream } from "./streams";
+
+export type MWProviderScrapeResult = {
+  stream?: MWStream;
+  embeds: MWEmbed[];
+};
+
+export type MWProviderContext = {
+  progress(percentage: number): void;
+  imdbId: string;
+  tmdbId: string;
+};
+
+export type MWProvider = {
+  id: string;
+  rank: number;
+  disabled?: boolean;
+  type: MWMediaType[];
+
+  scrape(ctx: MWProviderContext): Promise<MWProviderScrapeResult>;
+};

src/backend/helpers/register.ts

@@ -0,0 +1,61 @@
+import { MWEmbedScraper } from "./embed";
+import { MWProvider } from "./provider";
+
+let providers: MWProvider[] = [];
+let embeds: MWEmbedScraper[] = [];
+
+export function registerProvider(provider: MWProvider) {
+  if (provider.disabled) return;
+  providers.push(provider);
+}
+export function registerEmbedScraper(embed: MWEmbedScraper) {
+  if (embed.disabled) return;
+  embeds.push(embed);
+}
+
+export function initializeScraperStore() {
+  // sort by ranking
+  providers = providers.sort((a, b) => a.rank - b.rank);
+  embeds = embeds.sort((a, b) => a.rank - b.rank);
+
+  // check for invalid ranks
+  let lastRank: null | number = null;
+  providers.forEach((v) => {
+    if (lastRank === null) {
+      lastRank = v.rank;
+      return;
+    }
+    if (lastRank === v.rank)
+      throw new Error(`Duplicate rank number for provider ${v.id}`);
+    lastRank = v.rank;
+  });
+  lastRank = null;
+  providers.forEach((v) => {
+    if (lastRank === null) {
+      lastRank = v.rank;
+      return;
+    }
+    if (lastRank === v.rank)
+      throw new Error(`Duplicate rank number for embed scraper ${v.id}`);
+    lastRank = v.rank;
+  });
+
+  // check for duplicate ids
+  const providerIds = providers.map((v) => v.id);
+  if (
+    providerIds.length > 0 &&
+    new Set(providerIds).size !== providerIds.length
+  )
+    throw new Error("Duplicate IDS in providers");
+  const embedIds = embeds.map((v) => v.id);
+  if (embedIds.length > 0 && new Set(embedIds).size !== embedIds.length)
+    throw new Error("Duplicate IDS in embed scrapers");
+}
+
+export function getProviders(): MWProvider[] {
+  return providers;
+}
+
+export function getEmbeds(): MWEmbedScraper[] {
+  return embeds;
+}

src/backend/helpers/streams.ts

@@ -0,0 +1,9 @@
+export enum MWStreamType {
+  MP4 = "mp4",
+  HLS = "hls",
+}
+
+export type MWStream = {
+  streamUrl: string;
+  type: MWStreamType;
+};

src/backend/index.ts

@@ -0,0 +1,17 @@
+import { initializeScraperStore } from "./helpers/register";
+
+// TODO backend system:
+//  - run providers/embedscrapers in webworkers for multithreading and isolation
+//  - caption support
+//  - hooks to run all providers one by one
+//  - move over old providers to new system
+//  - implement jons providers/embedscrapers
+
+// providers
+// -- nothing here yet
+import "./providers/testProvider";
+
+// embeds
+// -- nothing here yet
+
+initializeScraperStore();

src/backend/metadata/search.ts

@@ -1,4 +1,5 @@
 import { MWMediaType, MWQuery } from "@/providers";
+import { MWMediaMeta } from "./types";
 
 const JW_API_BASE = "https://apis.justwatch.com";
 
@@ -27,18 +28,10 @@ type JWPage<T> = {
   total_results: number;
 };
 
-export type MWSearchResult = {
-  title: string;
-  id: string;
-  year: string;
-  poster?: string;
-  type: MWMediaType;
-};
-
 export async function searchForMedia({
   searchQuery,
   type,
-}: MWQuery): Promise<MWSearchResult[]> {
+}: MWQuery): Promise<MWMediaMeta[]> {
   const body: JWSearchQuery = {
     content_types: [],
     page: 1,
@@ -56,7 +49,7 @@ export async function searchForMedia({
     )}`
   ).then((res) => res.json() as Promise<JWPage<JWSearchResults>>);
 
-  return data.items.map<MWSearchResult>((v) => ({
+  return data.items.map<MWMediaMeta>((v) => ({
     title: v.title,
     id: v.id.toString(),
     year: v.original_release_year.toString(),

src/backend/metadata/types.ts

@@ -0,0 +1,13 @@
+export enum MWMediaType {
+  MOVIE = "movie",
+  SERIES = "series",
+  ANIME = "anime",
+}
+
+export type MWMediaMeta = {
+  title: string;
+  id: string;
+  year: string;
+  poster?: string;
+  type: MWMediaType;
+};

src/backend/providermeta/.gitkeep

@@ -1 +0,0 @@
-this folder will be used for provider helper methods and the like

src/backend/providers/.gitkeep

@@ -1 +0,0 @@
-the new list of all providers, the old ones will go and be rewritten

src/backend/providers/testProvider.ts

@@ -0,0 +1,32 @@
+import { MWEmbedType } from "../helpers/embed";
+import { registerProvider } from "../helpers/register";
+import { MWStreamType } from "../helpers/streams";
+import { MWMediaType } from "../metadata/types";
+
+registerProvider({
+  id: "testprov",
+  rank: 42,
+  type: [MWMediaType.MOVIE],
+
+  async scrape({ progress, imdbId, tmdbId }) {
+    console.log("scraping provider for: ", imdbId, tmdbId);
+    progress(25);
+    progress(50);
+    progress(75);
+
+    // providers can optionally provide a stream themselves,
+    // incase they host their own streams instead of using embeds
+    return {
+      stream: {
+        streamUrl: "hello-world",
+        type: MWStreamType.HLS,
+      },
+      embeds: [
+        {
+          type: MWEmbedType.OPENLOAD,
+          url: "https://google.com",
+        },
+      ],
+    };
+  },
+});

src/index.tsx

@@ -7,6 +7,7 @@ import { conf } from "@/setup/config";
 import App from "@/setup/App";
 import "@/setup/i18n";
 import "@/setup/index.css";
+import "@/backend";
 
 // initialize
 const key =