fix(pt/pobreflix): Fix video extractor + add more extractors + parallelize (#2610)

multisrc/overrides/dooplay/pobreflix/additional.gradle

@@ -1,4 +1,6 @@
 dependencies {
     implementation(project(":lib-filemoon-extractor"))
+    implementation(project(":lib-streamwish-extractor"))
+    implementation(project(":lib-streamtape-extractor"))
     implementation(project(":lib-playlist-utils"))
 }

multisrc/overrides/dooplay/pobreflix/src/Pobreflix.kt

@@ -3,9 +3,11 @@ package eu.kanade.tachiyomi.animeextension.pt.pobreflix
 import android.util.Base64
 import eu.kanade.tachiyomi.animeextension.pt.pobreflix.extractors.EplayerExtractor
 import eu.kanade.tachiyomi.animeextension.pt.pobreflix.extractors.MyStreamExtractor
-import eu.kanade.tachiyomi.animeextension.pt.pobreflix.extractors.PainelfxExtractor
+import eu.kanade.tachiyomi.animeextension.pt.pobreflix.extractors.SuperFlixExtractor
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.lib.filemoonextractor.FilemoonExtractor
+import eu.kanade.tachiyomi.lib.streamtapeextractor.StreamTapeExtractor
+import eu.kanade.tachiyomi.lib.streamwishextractor.StreamWishExtractor
 import eu.kanade.tachiyomi.multisrc.dooplay.DooPlay
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.util.asJsoup
@@ -24,10 +26,12 @@ class Pobreflix : DooPlay(
     override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/series/page/$page/", headers)
 
     // ============================ Video Links =============================
-    private val painelfxExtractor by lazy { PainelfxExtractor(client, headers, ::genericExtractor) }
     private val eplayerExtractor by lazy { EplayerExtractor(client) }
     private val filemoonExtractor by lazy { FilemoonExtractor(client) }
     private val mystreamExtractor by lazy { MyStreamExtractor(client, headers) }
+    private val streamtapeExtractor by lazy { StreamTapeExtractor(client) }
+    private val streamwishExtractor by lazy { StreamWishExtractor(client, headers) }
+    private val superflixExtractor by lazy { SuperFlixExtractor(client, headers, ::genericExtractor) }
 
     override fun videoListParse(response: Response): List<Video> {
         val doc = response.use { it.asJsoup() }
@@ -39,8 +43,8 @@ class Pobreflix : DooPlay(
                     ?: return@flatMap emptyList()
                 val url = data.replace("\\", "").substringAfter("url\":\"").substringBefore('"')
                 when {
-                    url.contains("painelfx") ->
+                    url.contains("superflix") ->
-                        painelfxExtractor.videosFromUrl(url)
+                        superflixExtractor.videosFromUrl(url)
                     else -> genericExtractor(url)
                 }
             }.getOrElse { emptyList() }
@@ -48,11 +52,20 @@ class Pobreflix : DooPlay(
     }
 
     private fun genericExtractor(url: String, language: String = ""): List<Video> {
+        val langSubstr = "[$language]"
         return when {
-            url.contains("filemoon") -> filemoonExtractor.videosFromUrl(url, headers = headers)
+            url.contains("filemoon") ->
+                filemoonExtractor.videosFromUrl(url, "$langSubstr Filemoon - ", headers = headers)
             url.contains("watch.brplayer") || url.contains("/watch?v=") ->
                 mystreamExtractor.videosFromUrl(url, language)
-            url.contains("embedplayer") -> eplayerExtractor.videosFromUrl(url, language)
+            url.contains("embedplayer") ->
+                eplayerExtractor.videosFromUrl(url, language)
+            url.contains("streamtape") ->
+                streamtapeExtractor.videosFromUrl(url, "$langSubstr Streamtape")
+            url.contains("filelions") ->
+                streamwishExtractor.videosFromUrl(url, videoNameGen = { "$langSubstr FileLions - $it" })
+            url.contains("streamwish") ->
+                streamwishExtractor.videosFromUrl(url, videoNameGen = { "$langSubstr Streamwish - $it" })
             else -> emptyList()
         }
     }

multisrc/overrides/dooplay/pobreflix/src/extractors/PainelfxExtractor.kt

@@ -1,80 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.pt.pobreflix.extractors
-
-import eu.kanade.tachiyomi.animesource.model.Video
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.util.asJsoup
-import okhttp3.FormBody
-import okhttp3.Headers
-import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.OkHttpClient
-
-class PainelfxExtractor(
-    private val client: OkHttpClient,
-    private val headers: Headers,
-    private val genericExtractor: (String, String) -> List<Video>,
-) {
-    fun videosFromUrl(url: String): List<Video> {
-        val docHeaders = headers.newBuilder().set("Referer", "https://gastronomiabrasileira.net/").build()
-        val doc = client.newCall(GET(url, docHeaders)).execute().use { it.asJsoup() }
-        val lang = when (url.substringAfterLast("/")) {
-            "leg" -> "Legendado"
-            else -> "Dublado"
-        }
-
-        val host = url.toHttpUrl().host
-        val videoHeaders = headers.newBuilder().set("Referer", "https://$host/").build()
-
-        val buttons = doc.select("div.panel-body > button")
-
-        val encodedHexList = when {
-            buttons.isNotEmpty() -> {
-                buttons.map { elem ->
-                    val form = FormBody.Builder()
-                        .add("idS", elem.attr("idS"))
-                        .build()
-                    client.newCall(POST("https://$host/CallEpi", body = form)).execute()
-                        .use { it.body.string() }
-                }
-            }
-            else -> {
-                val script = doc.selectFirst("script:containsData(idS:)")?.data() ?: return emptyList()
-                val idList = script.split("idS:").drop(1).map { it.substringAfter('"').substringBefore('"') }
-
-                idList.map { idS ->
-                    val form = FormBody.Builder()
-                        .add("id", idS)
-                        .build()
-                    client.newCall(POST("https://$host/CallPlayer", body = form)).execute()
-                        .use { it.body.string() }
-                }
-            }
-        }
-
-        return encodedHexList.flatMap {
-            videosFromHex(it, lang, videoHeaders)
-        }
-    }
-
-    private fun videosFromHex(hex: String, lang: String, videoHeaders: Headers): List<Video> {
-        val decoded = hex.decodeHex().let(::String).replace("\\", "")
-        return if (decoded.contains("video\"")) {
-            decoded.substringAfter("video").split("{").drop(1).map {
-                val videoUrl = it.substringAfter("file\":\"").substringBefore('"')
-                val quality = it.substringAfter("label\":\"").substringBefore('"')
-                Video(videoUrl, "$lang - $quality", videoUrl, videoHeaders)
-            }
-        } else {
-            val url = decoded.substringAfter("\"url\":\"").substringBefore('"')
-            genericExtractor(url, lang)
-        }
-    }
-
-    // Stolen from AnimixPlay(EN) / GogoCdnExtractor
-    private fun String.decodeHex(): ByteArray {
-        check(length % 2 == 0) { "Must have an even length" }
-        return chunked(2)
-            .map { it.toInt(16).toByte() }
-            .toByteArray()
-    }
-}

multisrc/overrides/dooplay/pobreflix/src/extractors/SuperFlixExtractor.kt

@@ -0,0 +1,139 @@
+package eu.kanade.tachiyomi.animeextension.pt.pobreflix.extractors
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.json.Json
+import okhttp3.FormBody
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.OkHttpClient
+import uy.kohesive.injekt.injectLazy
+
+class SuperFlixExtractor(
+    private val client: OkHttpClient,
+    private val defaultHeaders: Headers,
+    private val genericExtractor: (String, String) -> List<Video>,
+) {
+    private val json: Json by injectLazy()
+
+    fun videosFromUrl(url: String): List<Video> {
+        val links = linksFromUrl(url)
+
+        val fixedLinks = links.parallelCatchingFlatMap {
+            val (language, linkUrl) = it
+            when {
+                linkUrl.contains("?vid=") -> linksFromPlayer(linkUrl, language)
+                else -> listOf(it)
+            }
+        }
+
+        return fixedLinks.parallelCatchingFlatMap { genericExtractor(it.second, it.first) }
+    }
+
+    private fun linksFromPlayer(url: String, language: String): List<Pair<String, String>> {
+        val httpUrl = url.toHttpUrl()
+        val id = httpUrl.queryParameter("vid")!!
+        val headers = defaultHeaders.newBuilder()
+            .set("referer", "$API_DOMAIN/")
+            .set("origin", API_DOMAIN)
+            .build()
+
+        val doc = client.newCall(GET(url, headers)).execute().use { it.asJsoup() }
+
+        val baseUrl = "https://" + httpUrl.host
+        val apiUrl = "$baseUrl/ajax_sources.php"
+
+        val apiHeaders = headers.newBuilder()
+            .set("referer", url)
+            .set("origin", baseUrl)
+            .set("X-Requested-With", "XMLHttpRequest")
+            .build()
+
+        return doc.select("ul > li[data-order-value]").mapNotNull {
+            val name = it.attr("data-dropdown-value")
+            val order = it.attr("data-order-value")
+
+            val formBody = FormBody.Builder()
+                .add("vid", id)
+                .add("alternative", name)
+                .add("ord", order)
+                .build()
+
+            val req = client.newCall(POST(apiUrl, apiHeaders, formBody)).execute()
+                .use { it.body.string() }
+
+            runCatching {
+                val iframeUrl = json.decodeFromString<PlayerLinkDto>(req).iframe!!
+                val iframeServer = iframeUrl.toHttpUrl().queryParameter("sv")!!
+                language to when (name) {
+                    "1xbet" -> "https://watch.brplayer.site/watch?v=${iframeServer.trim('/')}"
+                    else -> iframeServer
+                }
+            }.getOrNull()
+        }
+    }
+
+    @Serializable
+    data class PlayerLinkDto(val iframe: String? = null)
+
+    private fun linksFromUrl(url: String): List<Pair<String, String>> {
+        val doc = client.newCall(GET(url, defaultHeaders)).execute().use { it.asJsoup() }
+
+        val items = doc.select("div.select_language").mapNotNull {
+            val target = it.attr("data-target")
+            val id = doc.selectFirst("div.players_select div[data-target=$target] div[data-id]")
+                ?.attr("data-id")
+                ?: return@mapNotNull null
+
+            it.text() to id // (Language, videoId)
+        }
+
+        val headers = defaultHeaders.newBuilder()
+            .set("Origin", API_DOMAIN)
+            .set("Referer", url)
+            .set("X-Requested-With", "XMLHttpRequest")
+            .build()
+
+        return items.mapNotNull {
+            runCatching {
+                it.first to getLink(it.second, headers)!!
+            }.getOrNull()
+        }
+    }
+
+    private fun getLink(id: String, headers: Headers): String? {
+        val body = FormBody.Builder()
+            .add("action", "getPlayer")
+            .add("video_id", id)
+            .build()
+
+        val res = client.newCall(POST("$API_DOMAIN/api", headers, body)).execute()
+            .use { it.body.string() }
+
+        return json.decodeFromString<ApiResponseDto>(res).data?.video_url
+    }
+
+    @Serializable
+    data class ApiResponseDto(val data: DataDto? = null)
+
+    @Serializable
+    data class DataDto(val video_url: String? = null)
+
+    private inline fun <A, B> Iterable<A>.parallelCatchingFlatMap(crossinline f: suspend (A) -> Iterable<B>): List<B> =
+        runBlocking {
+            map {
+                async(Dispatchers.Default) {
+                    runCatching { f(it) }.getOrElse { emptyList() }
+                }
+            }.awaitAll().flatten()
+        }
+}
+
+private const val API_DOMAIN = "https://superflixapi.top"

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/dooplay/DooPlayGenerator.kt

@@ -24,7 +24,7 @@ class DooPlayGenerator : ThemeSourceGenerator {
         SingleLang("Kinoking", "https://kinoking.cc", "de", isNsfw = false, overrideVersionCode = 19),
         SingleLang("Multimovies", "https://multimovies.live", "en", isNsfw = false, overrideVersionCode = 13),
         SingleLang("Pi Fansubs", "https://pifansubs.org", "pt-BR", isNsfw = true, overrideVersionCode = 17),
-        SingleLang("Pobreflix", "https://pobreflix.biz", "pt-BR", isNsfw = true, overrideVersionCode = 2),
+        SingleLang("Pobreflix", "https://pobreflix.biz", "pt-BR", isNsfw = true, overrideVersionCode = 3),
         SingleLang("UniqueStream", "https://uniquestream.net", "en", isNsfw = false, overrideVersionCode = 2),
     )