Add servers (#1640)

src/it/animeworld/build.gradle

@@ -1,11 +1,12 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'ANIMEWORLD.tv'
     pkgNameSuffix = 'it.animeworld'
     extClass = '.ANIMEWORLD'
-    extVersionCode = 24
+    extVersionCode = 25
     libVersion = '13'
 }
 
@@ -13,6 +14,7 @@ dependencies {
     implementation(project(':lib-streamtape-extractor'))
     implementation(project(':lib-streamsb-extractor'))
     implementation(project(':lib-dood-extractor'))
+    implementation "dev.datlag.jsunpacker:jsunpacker:1.0.1"
 }
 
 apply from: "$rootDir/common.gradle"

src/it/animeworld/src/eu/kanade/tachiyomi/animeextension/it/animeworld/ANIMEWORLD.kt

@@ -4,6 +4,8 @@ import android.app.Application
 import android.content.SharedPreferences
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
+import eu.kanade.tachiyomi.animeextension.it.animeworld.extractors.FilemoonExtractor
+import eu.kanade.tachiyomi.animeextension.it.animeworld.extractors.StreamHideExtractor
 import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
 import eu.kanade.tachiyomi.animesource.model.AnimeFilter
 import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
@@ -16,7 +18,13 @@ import eu.kanade.tachiyomi.lib.streamsbextractor.StreamSBExtractor
 import eu.kanade.tachiyomi.lib.streamtapeextractor.StreamTapeExtractor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.Headers
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
@@ -24,6 +32,7 @@ import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import uy.kohesive.injekt.Injekt
 import uy.kohesive.injekt.api.get
+import uy.kohesive.injekt.injectLazy
 import java.lang.Exception
 
 class ANIMEWORLD : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
@@ -38,6 +47,8 @@ class ANIMEWORLD : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override val client: OkHttpClient = network.cloudflareClient
 
+    private val json: Json by injectLazy()
+
     private val preferences: SharedPreferences by lazy {
         Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
     }
@@ -85,7 +96,10 @@ class ANIMEWORLD : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         return videosFromElement(document)
     }
 
-    override fun videoListSelector() = "center a[href*=dood], center a[href*=streamtape], center a[href*=animeworld.biz], center a[href*=streamingaw.online][id=alternativeDownloadLink]"
+    override fun videoListSelector() = "center a[href*=https://doo]," +
+        "center a[href*=streamtape]," +
+        "center a[href*=animeworld.biz]," +
+        "center a[href*=streamingaw.online][id=alternativeDownloadLink]"
 
     private fun videosFromElement(document: Document): List<Video> {
         val videoList = mutableListOf<Video>()
@@ -93,42 +107,84 @@ class ANIMEWORLD : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         // displaying Videolist empty show the element's text
         val copyrightError = document.select("div.alert.alert-primary:contains(Copyright)")
         if (copyrightError.hasText()) throw Exception(copyrightError.text())
+
+        val serverList = mutableListOf<Pair<String, String>>()
+
         val elements = document.select(videoListSelector())
-        for (element in elements) {
+        val epId = document.selectFirst("div#player[data-episode-id]")?.attr("data-episode-id")
-            val url = element.attr("href")
+
-            val location = element.ownerDocument()!!.location()
+        val altServers = mutableListOf<Pair<String, String>>()
-            val videoHeaders = Headers.headersOf("Referer", location)
+        val altList = listOf("StreamHide", "FileMoon", "StreamSB")
-            when {
+        document.select("div.servers > div.widget-title span.server-tab").forEach {
-                url.contains("animeworld.biz") || url.contains("sbembed.com") || url.contains("sbembed1.com") || url.contains("sbplay.org") ||
+            val name = it.text()
-                    url.contains("sbvideo.net") || url.contains("streamsb.net") || url.contains("sbplay.one") ||
+            if (altList.any { t -> t.contains(name, true) }) {
-                    url.contains("cloudemb.com") || url.contains("playersb.com") || url.contains("tubesb.com") ||
+                altServers.add(Pair(name, it.attr("data-name")))
-                    url.contains("sbplay1.com") || url.contains("embedsb.com") || url.contains("watchsb.com") ||
-                    url.contains("sbplay2.com") || url.contains("japopav.tv") || url.contains("viewsb.com") ||
-                    url.contains("sbfast") || url.contains("sbfull.com") || url.contains("javplaya.com") ||
-                    url.contains("ssbstream.net") || url.contains("p1ayerjavseen.com") || url.contains("sbthe.com")
-                -> {
-                    val videos = StreamSBExtractor(client).videosFromUrl(url.replace("/d/", "/e/"), headers)
-                    videoList.addAll(videos)
-                }
-                url.contains("streamingaw") -> {
-                    videoList.add(
-                        Video(url, "AnimeWorld Server", url),
-                    )
-                }
-                url.contains("dood") -> {
-                    val video = DoodExtractor(client).videoFromUrl(url.replace("/d/", "/e/"))
-                    if (video != null) {
-                        videoList.add(video)
-                    }
-                }
-                url.contains("streamtape") -> {
-                    val video = StreamTapeExtractor(client).videoFromUrl(url.replace("/v/", "/e/"))
-                    if (video != null) {
-                        videoList.add(video)
-                    }
-                }
             }
         }
+
+        altServers.forEach { serverPair ->
+            val dataId = document.selectFirst("div.server[data-name=${serverPair.second}] li.episode a[data-episode-id=$epId]")?.attr("data-id")
+            dataId?.let {
+                val apiUrl = "$baseUrl/api/episode/info?id=$it&alt=0"
+                val apiHeaders = headers.newBuilder()
+                    .add("Accept", "application/json, text/javascript, */*; q=0.01")
+                    .add("Content-Type", "application/json")
+                    .add("Host", baseUrl.toHttpUrl().host)
+                    .add("Referer", document.location())
+                    .add("X-Requested-With", "XMLHttpRequest")
+                    .build()
+                val target = json.decodeFromString<ServerResponse>(
+                    client.newCall(GET(apiUrl, headers = apiHeaders)).execute().body.string(),
+                ).target
+                serverList.add(Pair(serverPair.first, target))
+            }
+        }
+
+        for (element in elements) {
+            val url = element.attr("href")
+            val name = element.text().substringAfter("ownload ").substringBefore(" ")
+            serverList.add(Pair(name, url))
+        }
+
+        videoList.addAll(
+            serverList.parallelMap { server ->
+                runCatching {
+                    val url = server.second
+                    when {
+                        url.contains("animeworld.biz") || url.contains("sbembed.com") || url.contains("sbembed1.com") || url.contains("sbplay.org") ||
+                            url.contains("sbvideo.net") || url.contains("streamsb.net") || url.contains("sbplay.one") ||
+                            url.contains("cloudemb.com") || url.contains("playersb.com") || url.contains("tubesb.com") ||
+                            url.contains("sbplay1.com") || url.contains("embedsb.com") || url.contains("watchsb.com") ||
+                            url.contains("sbplay2.com") || url.contains("japopav.tv") || url.contains("viewsb.com") ||
+                            url.contains("sbfast") || url.contains("sbfull.com") || url.contains("javplaya.com") ||
+                            url.contains("ssbstream.net") || url.contains("p1ayerjavseen.com") || url.contains("sbthe.com") ||
+                            url.contains("animeworld.biz")
+                        -> {
+                            StreamSBExtractor(client).videosFromUrl(url.replace("/d/", "/e/"), headers)
+                        }
+                        url.contains("streamingaw") -> {
+                            listOf(Video(url, "AnimeWorld Server", url))
+                        }
+                        url.contains("https://doo") -> {
+                            val video = DoodExtractor(client).videoFromUrl(url, redirect = true)
+                            video?.let { listOf(it) }
+                        }
+                        url.contains("streamtape") -> {
+                            val video = StreamTapeExtractor(client).videoFromUrl(url.replace("/v/", "/e/"))
+                            video?.let { listOf(it) }
+                        }
+                        url.contains("filemoon") -> {
+                            FilemoonExtractor(client, headers).videosFromUrl(url, prefix = "${server.first} - ")
+                        }
+                        server.first.contains("streamhide", true) -> {
+                            StreamHideExtractor(client).videosFromUrl(url, headers)
+                        }
+                        else -> null
+                    }
+                }.getOrNull()
+            }.filterNotNull().flatten(),
+        )
+
         return videoList
     }
 
@@ -488,4 +544,17 @@ class ANIMEWORLD : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         }
         screen.addPreference(videoQualityPref)
     }
+
+    // Utilities
+
+    @Serializable
+    data class ServerResponse(
+        val target: String,
+    )
+
+    // From Dopebox
+    private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
+        runBlocking {
+            map { async(Dispatchers.Default) { f(it) } }.awaitAll()
+        }
 }

src/it/animeworld/src/eu/kanade/tachiyomi/animeextension/it/animeworld/extractors/FilemoonExtractor.kt

@@ -0,0 +1,34 @@
+package eu.kanade.tachiyomi.animeextension.it.animeworld.extractors
+
+import dev.datlag.jsunpacker.JsUnpacker
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.OkHttpClient
+
+class FilemoonExtractor(private val client: OkHttpClient, private val headers: Headers) {
+    fun videosFromUrl(url: String, prefix: String = ""): List<Video> {
+        val jsE = client.newCall(GET(url)).execute().asJsoup().selectFirst("script:containsData(eval)")!!.data()
+        val masterUrl = JsUnpacker.unpackAndCombine(jsE)?.substringAfter("{file:\"")
+            ?.substringBefore("\"}") ?: return emptyList()
+        val masterPlaylist = client.newCall(GET(masterUrl)).execute().body.string()
+        val videoList = mutableListOf<Video>()
+        masterPlaylist.substringAfter("#EXT-X-STREAM-INF:").split("#EXT-X-STREAM-INF:")
+            .forEach {
+                val quality = "Filemoon:" + it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",") + "p "
+                val videoUrl = it.substringAfter("\n").substringBefore("\n")
+
+                val videoHeaders = headers.newBuilder()
+                    .add("Accept", "*/*")
+                    .add("Host", videoUrl.toHttpUrl().host)
+                    .add("Origin", "https://${url.toHttpUrl().host}")
+                    .add("Referer", "https://${url.toHttpUrl().host}/")
+                    .build()
+
+                videoList.add(Video(videoUrl, prefix + quality, videoUrl, headers = videoHeaders))
+            }
+        return videoList
+    }
+}

src/it/animeworld/src/eu/kanade/tachiyomi/animeextension/it/animeworld/extractors/StreamHideExtractor.kt

@@ -0,0 +1,52 @@
+package eu.kanade.tachiyomi.animeextension.it.animeworld.extractors
+
+import dev.datlag.jsunpacker.JsUnpacker
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.OkHttpClient
+
+class StreamHideExtractor(private val client: OkHttpClient) {
+    fun videosFromUrl(url: String, headers: Headers): List<Video> {
+        val videoList = mutableListOf<Video>()
+
+        val url = OkHttpClient().newBuilder().followRedirects(false).build()
+            .newCall(GET(url, headers)).execute().request.url.toString()
+
+        val packed = client.newCall(GET(url)).execute()
+            .asJsoup().selectFirst("script:containsData(eval)")?.data() ?: return emptyList()
+        val unpacked = JsUnpacker.unpackAndCombine(packed) ?: return emptyList()
+        val masterUrl = Regex("""file: ?"(.*?)"""").find(unpacked)?.groupValues?.get(1) ?: return emptyList()
+
+        val masterHeaders = headers.newBuilder()
+            .add("Accept", "*/*")
+            .add("Host", masterUrl.toHttpUrl().host)
+            .add("Origin", "https://${url.toHttpUrl().host}")
+            .add("Referer", "https://${url.toHttpUrl().host}/")
+            .build()
+        val masterPlaylist = client.newCall(
+            GET(masterUrl, headers = masterHeaders),
+        ).execute().body.string()
+
+        val masterBase = "https://${masterUrl.toHttpUrl().host}${masterUrl.toHttpUrl().encodedPath}"
+            .substringBeforeLast("/") + "/"
+
+        masterPlaylist.substringAfter("#EXT-X-STREAM-INF:").split("#EXT-X-STREAM-INF:")
+            .forEach {
+                val quality = "StreamHide - " + it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",") + "p "
+                val videoUrl = masterBase + it.substringAfter("\n").substringBefore("\n")
+
+                val videoHeaders = headers.newBuilder()
+                    .add("Accept", "*/*")
+                    .add("Host", videoUrl.toHttpUrl().host)
+                    .add("Origin", "https://${url.toHttpUrl().host}")
+                    .add("Referer", "https://${url.toHttpUrl().host}/")
+                    .build()
+
+                videoList.add(Video(videoUrl, quality, videoUrl, headers = videoHeaders))
+            }
+        return videoList
+    }
+}