Merge pull request #333 from Jordaar/dev

feat(providers): add gomovies, kissasian providers and upcloud, streamsb, mp4upload embed scrapers

src/backend/embeds/mp4upload.ts

@@ -0,0 +1,32 @@
+import { MWEmbedType } from "@/backend/helpers/embed";
+import { registerEmbedScraper } from "@/backend/helpers/register";
+import { MWStreamQuality, MWStreamType } from "@/backend/helpers/streams";
+
+import { proxiedFetch } from "../helpers/fetch";
+
+registerEmbedScraper({
+  id: "mp4upload",
+  displayName: "mp4upload",
+  for: MWEmbedType.MP4UPLOAD,
+  rank: 170,
+  async getStream({ url }) {
+    const embed = await proxiedFetch<any>(url);
+
+    const playerSrcRegex =
+      /(?<=player\.src\()\s*{\s*type:\s*"[^"]+",\s*src:\s*"([^"]+)"\s*}\s*(?=\);)/s;
+
+    const playerSrc = embed.match(playerSrcRegex);
+
+    const streamUrl = playerSrc[1];
+
+    if (!streamUrl) throw new Error("Stream url not found");
+
+    return {
+      embedId: MWEmbedType.MP4UPLOAD,
+      streamUrl,
+      quality: MWStreamQuality.Q1080P,
+      captions: [],
+      type: MWStreamType.MP4,
+    };
+  },
+});

src/backend/embeds/streamsb.ts

@@ -0,0 +1,211 @@
+import Base64 from "crypto-js/enc-base64";
+import Utf8 from "crypto-js/enc-utf8";
+
+import { MWEmbedType } from "@/backend/helpers/embed";
+import { proxiedFetch } from "@/backend/helpers/fetch";
+import { registerEmbedScraper } from "@/backend/helpers/register";
+import {
+  MWCaptionType,
+  MWStreamQuality,
+  MWStreamType,
+} from "@/backend/helpers/streams";
+
+const qualityOrder = [
+  MWStreamQuality.Q1080P,
+  MWStreamQuality.Q720P,
+  MWStreamQuality.Q480P,
+  MWStreamQuality.Q360P,
+];
+
+async function fetchCaptchaToken(domain: string, recaptchaKey: string) {
+  const domainHash = Base64.stringify(Utf8.parse(domain)).replace(/=/g, ".");
+
+  const recaptchaRender = await proxiedFetch<any>(
+    `https://www.google.com/recaptcha/api.js?render=${recaptchaKey}`
+  );
+
+  const vToken = recaptchaRender.substring(
+    recaptchaRender.indexOf("/releases/") + 10,
+    recaptchaRender.indexOf("/recaptcha__en.js")
+  );
+
+  const recaptchaAnchor = await proxiedFetch<any>(
+    `https://www.google.com/recaptcha/api2/anchor?ar=1&hl=en&size=invisible&cb=flicklax&k=${recaptchaKey}&co=${domainHash}&v=${vToken}`
+  );
+
+  const cToken = new DOMParser()
+    .parseFromString(recaptchaAnchor, "text/html")
+    .getElementById("recaptcha-token")
+    ?.getAttribute("value");
+
+  if (!cToken) throw new Error("Unable to find cToken");
+
+  const payload = {
+    v: vToken,
+    reason: "q",
+    k: recaptchaKey,
+    c: cToken,
+    sa: "",
+    co: domain,
+  };
+
+  const tokenData = await proxiedFetch<string>(
+    `https://www.google.com/recaptcha/api2/reload?${new URLSearchParams(
+      payload
+    ).toString()}`,
+    {
+      headers: { referer: "https://www.google.com/recaptcha/api2/" },
+      method: "POST",
+    }
+  );
+
+  const token = tokenData.match('rresp","(.+?)"');
+  return token ? token[1] : null;
+}
+
+registerEmbedScraper({
+  id: "streamsb",
+  displayName: "StreamSB",
+  for: MWEmbedType.STREAMSB,
+  rank: 150,
+  async getStream({ url, progress }) {
+    /* Url variations
+    - domain.com/{id}?.html
+    - domain.com/{id}
+    - domain.com/embed-{id}
+    - domain.com/d/{id}
+    - domain.com/e/{id}
+    - domain.com/e/{id}-embed
+    */
+    const streamsbUrl = url
+      .replace(".html", "")
+      .replace("embed-", "")
+      .replace("e/", "")
+      .replace("d/", "");
+
+    const parsedUrl = new URL(streamsbUrl);
+    const base = await proxiedFetch<any>(
+      `${parsedUrl.origin}/d${parsedUrl.pathname}`
+    );
+
+    progress(20);
+
+    // Parse captions from url
+    const captionUrl = parsedUrl.searchParams.get("caption_1");
+    const captionLang = parsedUrl.searchParams.get("sub_1");
+
+    const basePage = new DOMParser().parseFromString(base, "text/html");
+
+    const downloadVideoFunctions = basePage.querySelectorAll(
+      "[onclick^=download_video]"
+    );
+
+    let dlDetails = [];
+    for (const func of downloadVideoFunctions) {
+      const funcContents = func.getAttribute("onclick");
+      const regExpFunc = /download_video\('(.+?)','(.+?)','(.+?)'\)/;
+      const matchesFunc = regExpFunc.exec(funcContents ?? "");
+      if (matchesFunc !== null) {
+        const quality = func.querySelector("span")?.textContent;
+        const regExpQuality = /(.+?) \((.+?)\)/;
+        const matchesQuality = regExpQuality.exec(quality ?? "");
+        if (matchesQuality !== null) {
+          dlDetails.push({
+            parameters: [matchesFunc[1], matchesFunc[2], matchesFunc[3]],
+            quality: {
+              label: matchesQuality[1].trim(),
+              size: matchesQuality[2],
+            },
+          });
+        }
+      }
+    }
+
+    dlDetails = dlDetails.sort((a, b) => {
+      const aQuality = qualityOrder.indexOf(a.quality.label as MWStreamQuality);
+      const bQuality = qualityOrder.indexOf(b.quality.label as MWStreamQuality);
+      return aQuality - bQuality;
+    });
+
+    progress(40);
+
+    let dls = await Promise.all(
+      dlDetails.map(async (dl) => {
+        const getDownload = await proxiedFetch<any>(
+          `/dl?op=download_orig&id=${dl.parameters[0]}&mode=${dl.parameters[1]}&hash=${dl.parameters[2]}`,
+          {
+            baseURL: parsedUrl.origin,
+          }
+        );
+
+        const downloadPage = new DOMParser().parseFromString(
+          getDownload,
+          "text/html"
+        );
+
+        const recaptchaKey = downloadPage
+          .querySelector(".g-recaptcha")
+          ?.getAttribute("data-sitekey");
+        if (!recaptchaKey) throw new Error("Unable to get captcha key");
+
+        const captchaToken = await fetchCaptchaToken(
+          parsedUrl.origin,
+          recaptchaKey
+        );
+        if (!captchaToken) throw new Error("Unable to get captcha token");
+
+        const dlForm = new FormData();
+        dlForm.append("op", "download_orig");
+        dlForm.append("id", dl.parameters[0]);
+        dlForm.append("mode", dl.parameters[1]);
+        dlForm.append("hash", dl.parameters[2]);
+        dlForm.append("g-recaptcha-response", captchaToken);
+
+        const download = await proxiedFetch<any>(
+          `/dl?op=download_orig&id=${dl.parameters[0]}&mode=${dl.parameters[1]}&hash=${dl.parameters[2]}`,
+          {
+            baseURL: parsedUrl.origin,
+            method: "POST",
+            body: dlForm,
+          }
+        );
+
+        const dlLink = new DOMParser()
+          .parseFromString(download, "text/html")
+          .querySelector(".btn.btn-light.btn-lg")
+          ?.getAttribute("href");
+
+        return {
+          quality: dl.quality.label as MWStreamQuality,
+          url: dlLink,
+          size: dl.quality.size,
+          captions:
+            captionUrl && captionLang
+              ? [
+                  {
+                    url: captionUrl,
+                    langIso: captionLang,
+                    type: MWCaptionType.VTT,
+                  },
+                ]
+              : [],
+        };
+      })
+    );
+    dls = dls.filter((d) => !!d.url);
+
+    progress(60);
+
+    // TODO: Quality selection for embed scrapers
+    const dl = dls[0];
+    if (!dl.url) throw new Error("No stream url found");
+
+    return {
+      embedId: MWEmbedType.STREAMSB,
+      streamUrl: dl.url,
+      quality: dl.quality,
+      captions: dl.captions,
+      type: MWStreamType.MP4,
+    };
+  },
+});

src/backend/embeds/upcloud.ts

@@ -0,0 +1,93 @@
+import { AES, enc } from "crypto-js";
+
+import { MWEmbedType } from "@/backend/helpers/embed";
+import { registerEmbedScraper } from "@/backend/helpers/register";
+import {
+  MWCaptionType,
+  MWStreamQuality,
+  MWStreamType,
+} from "@/backend/helpers/streams";
+
+import { proxiedFetch } from "../helpers/fetch";
+
+interface StreamRes {
+  server: number;
+  sources: string;
+  tracks: {
+    file: string;
+    kind: "captions" | "thumbnails";
+    label: string;
+  }[];
+}
+
+function isJSON(json: string) {
+  try {
+    JSON.parse(json);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+registerEmbedScraper({
+  id: "upcloud",
+  displayName: "UpCloud",
+  for: MWEmbedType.UPCLOUD,
+  rank: 200,
+  async getStream({ url }) {
+    // Example url: https://dokicloud.one/embed-4/{id}?z=
+    const parsedUrl = new URL(url.replace("embed-5", "embed-4"));
+
+    const dataPath = parsedUrl.pathname.split("/");
+    const dataId = dataPath[dataPath.length - 1];
+
+    const streamRes = await proxiedFetch<StreamRes>(
+      `${parsedUrl.origin}/ajax/embed-4/getSources?id=${dataId}`,
+      {
+        headers: {
+          Referer: parsedUrl.origin,
+          "X-Requested-With": "XMLHttpRequest",
+        },
+      }
+    );
+
+    let sources:
+      | {
+          file: string;
+          type: string;
+        }
+      | string = streamRes.sources;
+
+    if (!isJSON(sources) || typeof sources === "string") {
+      const decryptionKey = await proxiedFetch<string>(
+        `https://raw.githubusercontent.com/enimax-anime/key/e4/key.txt`
+      );
+
+      const decryptedStream = AES.decrypt(sources, decryptionKey).toString(
+        enc.Utf8
+      );
+
+      const parsedStream = JSON.parse(decryptedStream)[0];
+      if (!parsedStream) throw new Error("No stream found");
+      sources = parsedStream as { file: string; type: string };
+    }
+
+    return {
+      embedId: MWEmbedType.UPCLOUD,
+      streamUrl: sources.file,
+      quality: MWStreamQuality.Q1080P,
+      type: MWStreamType.HLS,
+      captions: streamRes.tracks
+        .filter((sub) => sub.kind === "captions")
+        .map((sub) => {
+          return {
+            langIso: sub.label,
+            url: sub.file,
+            type: sub.file.endsWith("vtt")
+              ? MWCaptionType.VTT
+              : MWCaptionType.UNKNOWN,
+          };
+        }),
+    };
+  },
+});

src/backend/helpers/embed.ts

@@ -4,6 +4,9 @@ export enum MWEmbedType {
   M4UFREE = "m4ufree",
   STREAMM4U = "streamm4u",
   PLAYM4U = "playm4u",
+  UPCLOUD = "upcloud",
+  STREAMSB = "streamsb",
+  MP4UPLOAD = "mp4upload",
 }
 
 export type MWEmbed = {

src/backend/index.ts

@@ -9,11 +9,16 @@ import "./providers/m4ufree";
 import "./providers/hdwatched";
 import "./providers/2embed";
 import "./providers/sflix";
+import "./providers/gomovies";
+import "./providers/kissasian";
 import "./providers/streamflix";
 import "./providers/remotestream";
 
 // embeds
 import "./embeds/streamm4u";
 import "./embeds/playm4u";
+import "./embeds/upcloud";
+import "./embeds/streamsb";
+import "./embeds/mp4upload";
 
 initializeScraperStore();

src/backend/providers/2embed.ts

@@ -191,6 +191,7 @@ registerProvider({
   displayName: "2Embed",
   rank: 125,
   type: [MWMediaType.MOVIE, MWMediaType.SERIES],
+  disabled: true, // Disabled, not working
   async scrape({ media, episode, progress }) {
     let embedUrl = `${twoEmbedBase}/embed/tmdb/movie?id=${media.tmdbId}`;
 

src/backend/providers/gomovies.ts

@@ -0,0 +1,162 @@
+import { MWEmbedType } from "../helpers/embed";
+import { proxiedFetch } from "../helpers/fetch";
+import { registerProvider } from "../helpers/register";
+import { MWMediaType } from "../metadata/types";
+
+const gomoviesBase = "https://gomovies.sx";
+
+registerProvider({
+  id: "gomovies",
+  displayName: "GOmovies",
+  rank: 300,
+  type: [MWMediaType.MOVIE, MWMediaType.SERIES],
+
+  async scrape({ media, episode }) {
+    const search = await proxiedFetch<any>("/ajax/search", {
+      baseURL: gomoviesBase,
+      method: "POST",
+      body: JSON.stringify({
+        keyword: media.meta.title,
+      }),
+      headers: {
+        "X-Requested-With": "XMLHttpRequest",
+      },
+    });
+
+    const searchPage = new DOMParser().parseFromString(search, "text/html");
+    const mediaElements = searchPage.querySelectorAll("a.nav-item");
+
+    const mediaData = Array.from(mediaElements).map((movieEl) => {
+      const name = movieEl?.querySelector("h3.film-name")?.textContent;
+      const year = movieEl?.querySelector(
+        "div.film-infor span:first-of-type"
+      )?.textContent;
+      const path = movieEl.getAttribute("href");
+      return { name, year, path };
+    });
+
+    const targetMedia = mediaData.find(
+      (m) =>
+        m.name === media.meta.title &&
+        (media.meta.type === MWMediaType.MOVIE
+          ? m.year === media.meta.year
+          : true)
+    );
+    if (!targetMedia?.path) throw new Error("Media not found");
+
+    // Example movie path: /movie/watch-{slug}-{id}
+    // Example series path: /tv/watch-{slug}-{id}
+    let mediaId = targetMedia.path.split("-").pop()?.replace("/", "");
+
+    let sources = null;
+    if (media.meta.type === MWMediaType.SERIES) {
+      const seasons = await proxiedFetch<any>(
+        `/ajax/v2/tv/seasons/${mediaId}`,
+        {
+          baseURL: gomoviesBase,
+          headers: {
+            "X-Requested-With": "XMLHttpRequest",
+          },
+        }
+      );
+
+      const seasonsEl = new DOMParser()
+        .parseFromString(seasons, "text/html")
+        .querySelectorAll(".ss-item");
+
+      const seasonsData = [...seasonsEl].map((season) => ({
+        number: season.innerHTML.replace("Season ", ""),
+        dataId: season.getAttribute("data-id"),
+      }));
+
+      const seasonNumber = media.meta.seasonData.number;
+      const targetSeason = seasonsData.find(
+        (season) => +season.number === seasonNumber
+      );
+      if (!targetSeason) throw new Error("Season not found");
+
+      const episodes = await proxiedFetch<any>(
+        `/ajax/v2/season/episodes/${targetSeason.dataId}`,
+        {
+          baseURL: gomoviesBase,
+          headers: {
+            "X-Requested-With": "XMLHttpRequest",
+          },
+        }
+      );
+
+      const episodesEl = new DOMParser()
+        .parseFromString(episodes, "text/html")
+        .querySelectorAll(".eps-item");
+
+      const episodesData = Array.from(episodesEl).map((ep) => ({
+        dataId: ep.getAttribute("data-id"),
+        number: ep
+          .querySelector("strong")
+          ?.textContent?.replace("Eps", "")
+          .replace(":", "")
+          .trim(),
+      }));
+
+      const episodeNumber = media.meta.seasonData.episodes.find(
+        (e) => e.id === episode
+      )?.number;
+
+      const targetEpisode = episodesData.find((ep) =>
+        ep.number ? +ep.number : ep.number === episodeNumber
+      );
+
+      if (!targetEpisode?.dataId) throw new Error("Episode not found");
+
+      mediaId = targetEpisode.dataId;
+
+      sources = await proxiedFetch<any>(`/ajax/v2/episode/servers/${mediaId}`, {
+        baseURL: gomoviesBase,
+        headers: {
+          "X-Requested-With": "XMLHttpRequest",
+        },
+      });
+    } else {
+      sources = await proxiedFetch<any>(`/ajax/movie/episodes/${mediaId}`, {
+        baseURL: gomoviesBase,
+        headers: {
+          "X-Requested-With": "XMLHttpRequest",
+        },
+      });
+    }
+
+    const upcloud = new DOMParser()
+      .parseFromString(sources, "text/html")
+      .querySelector('a[title*="upcloud" i]');
+
+    const upcloudDataId =
+      upcloud?.getAttribute("data-id") ?? upcloud?.getAttribute("data-linkid");
+
+    if (!upcloudDataId) throw new Error("Upcloud source not available");
+
+    const upcloudSource = await proxiedFetch<{
+      type: "iframe" | string;
+      link: string;
+      sources: [];
+      title: string;
+      tracks: [];
+    }>(`/ajax/sources/${upcloudDataId}`, {
+      baseURL: gomoviesBase,
+      headers: {
+        "X-Requested-With": "XMLHttpRequest",
+      },
+    });
+
+    if (!upcloudSource.link || upcloudSource.type !== "iframe")
+      throw new Error("No upcloud stream found");
+
+    return {
+      embeds: [
+        {
+          type: MWEmbedType.UPCLOUD,
+          url: upcloudSource.link,
+        },
+      ],
+    };
+  },
+});

src/backend/providers/kissasian.ts

@@ -0,0 +1,119 @@
+import { MWEmbedType } from "../helpers/embed";
+import { proxiedFetch } from "../helpers/fetch";
+import { registerProvider } from "../helpers/register";
+import { MWMediaType } from "../metadata/types";
+
+const kissasianBase = "https://kissasian.li";
+
+const embedProviders = [
+  {
+    type: MWEmbedType.MP4UPLOAD,
+    id: "mp",
+  },
+  {
+    type: MWEmbedType.STREAMSB,
+    id: "sb",
+  },
+];
+
+registerProvider({
+  id: "kissasian",
+  displayName: "KissAsian",
+  rank: 130,
+  type: [MWMediaType.MOVIE, MWMediaType.SERIES],
+
+  async scrape({ media, episode, progress }) {
+    let seasonNumber = "";
+    let episodeNumber = "";
+
+    if (media.meta.type === MWMediaType.SERIES) {
+      seasonNumber =
+        media.meta.seasonData.number === 1
+          ? ""
+          : `${media.meta.seasonData.number}`;
+      episodeNumber = `${
+        media.meta.seasonData.episodes.find((e) => e.id === episode)?.number ??
+        ""
+      }`;
+    }
+
+    const searchForm = new FormData();
+    searchForm.append("keyword", `${media.meta.title} ${seasonNumber}`.trim());
+    searchForm.append("type", "Drama");
+
+    const search = await proxiedFetch<any>("/Search/SearchSuggest", {
+      baseURL: kissasianBase,
+      method: "POST",
+      body: searchForm,
+    });
+
+    const searchPage = new DOMParser().parseFromString(search, "text/html");
+
+    const dramas = Array.from(searchPage.querySelectorAll("a")).map((drama) => {
+      return {
+        name: drama.textContent,
+        url: drama.href,
+      };
+    });
+
+    const targetDrama =
+      dramas.find(
+        (d) => d.name?.toLowerCase() === media.meta.title.toLowerCase()
+      ) ?? dramas[0];
+    if (!targetDrama) throw new Error("Drama not found");
+
+    progress(30);
+
+    const drama = await proxiedFetch<any>(targetDrama.url);
+
+    const dramaPage = new DOMParser().parseFromString(drama, "text/html");
+
+    const episodesEl = dramaPage.querySelectorAll("tbody tr:not(:first-child)");
+
+    const episodes = Array.from(episodesEl)
+      .map((ep) => {
+        const number = ep
+          ?.querySelector("td.episodeSub a")
+          ?.textContent?.split("Episode")[1]
+          ?.trim();
+        const url = ep?.querySelector("td.episodeSub a")?.getAttribute("href");
+        return { number, url };
+      })
+      .filter((e) => !!e.url);
+
+    const targetEpisode =
+      media.meta.type === MWMediaType.MOVIE
+        ? episodes[0]
+        : episodes.find((e) => e.number === `${episodeNumber}`);
+    if (!targetEpisode?.url) throw new Error("Episode not found");
+
+    progress(70);
+
+    let embeds = await Promise.all(
+      embedProviders.map(async (provider) => {
+        const watch = await proxiedFetch<any>(
+          `${targetEpisode.url}&s=${provider.id}`,
+          {
+            baseURL: kissasianBase,
+          }
+        );
+
+        const watchPage = new DOMParser().parseFromString(watch, "text/html");
+
+        const embedUrl = watchPage
+          .querySelector("iframe[id=my_video_1]")
+          ?.getAttribute("src");
+
+        return {
+          type: provider.type,
+          url: embedUrl ?? "",
+        };
+      })
+    );
+    embeds = embeds.filter((e) => e.url !== "");
+
+    return {
+      embeds,
+    };
+  },
+});