AllAnime: Fix gogo, streamlare, add parallel map for external hosts and update streamsb (again) (#1291)

* Fix gogo stuff

* Update streamsb (again), make external hosts parallel, update streamlare extractor

lib/streamsb-extractor/src/main/java/eu/kanade/tachiyomi/lib/streamsbextractor/StreamSBExtractor.kt

@@ -33,9 +33,9 @@ class StreamSBExtractor(private val client: OkHttpClient) {
             .substringBefore(".html")
         return if (common) {
             val hexBytes = bytesToHex(id.toByteArray())
-            "$sbUrl/sources50/625a364258615242766475327c7c${hexBytes}7c7c4761574550654f7461566d347c7c73747265616d7362"
+            "$sbUrl/sources51/625a364258615242766475327c7c${hexBytes}7c7c4761574550654f7461566d347c7c73747265616d7362"
         } else {
-            "$sbUrl/sources50/${bytesToHex("||$id||||streamsb".toByteArray())}/"
+            "$sbUrl/sources51/${bytesToHex("||$id||||streamsb".toByteArray())}/"
         }
     }
 

src/en/allanime/build.gradle

@@ -6,11 +6,12 @@ ext {
     extName = 'AllAnime'
     pkgNameSuffix = 'en.allanime'
     extClass = '.AllAnime'
-    extVersionCode = 7
+    extVersionCode = 8
     libVersion = '13'
 }
 
 dependencies {
+    compileOnly libs.bundles.coroutines
     implementation(project(':lib-streamsb-extractor'))
     implementation(project(':lib-dood-extractor'))
     implementation(project(':lib-okru-extractor'))

src/en/allanime/src/eu/kanade/tachiyomi/animeextension/en/allanime/AllAnime.kt

@@ -21,6 +21,10 @@ import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
 import eu.kanade.tachiyomi.lib.streamsbextractor.StreamSBExtractor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.asObservableSuccess
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
 import okhttp3.OkHttpClient
@@ -246,6 +250,7 @@ class AllAnime : ConfigurableAnimeSource, AnimeHttpSource() {
 
         val videoJson = json.decodeFromString<EpisodeResult>(body)
         val videoList = mutableListOf<Pair<Video, Float>>()
+        val serverList = mutableListOf<Server>()
 
         val altHosterSelection = preferences.getStringSet(
             "alt_hoster_selection",
@@ -269,84 +274,120 @@ class AllAnime : ConfigurableAnimeSource, AnimeHttpSource() {
                         (hosterSelection.contains("uv-mp4") && video.sourceName.lowercase().contains("uv-mp4")) ||
                         (hosterSelection.contains("pn-hls") && video.sourceName.lowercase().contains("pn-hls"))
                     ) -> {
-                    val extractor = AllAnimeExtractor(client)
+                    serverList.add(Server(video.sourceUrl, "internal", video.priority))
-                    val videos = runCatching {
-                        extractor.videoFromUrl(video.sourceUrl, video.sourceName)
-                    }.getOrNull() ?: emptyList()
-                    for (v in videos) {
-                        videoList.add(Pair(v, video.priority))
-                    }
                 }
                 altHosterSelection.contains("player") && video.type == "player" -> {
-                    videoList.add(
+                    serverList.add(Server(video.sourceUrl, "player", video.priority))
-                        Pair(
-                            Video(
-                                video.sourceUrl,
-                                "Original (player ${video.sourceName})",
-                                video.sourceUrl
-                            ),
-                            video.priority
-                        )
-
-                    )
                 }
                 altHosterSelection.contains("streamsb") && video.sourceUrl.contains("streamsb") -> {
-                    val extractor = StreamSBExtractor(client)
+                    serverList.add(Server(video.sourceUrl, "streamsb", video.priority))
-                    val videos = runCatching {
-                        extractor.videosFromUrl(video.sourceUrl, headers)
-                    }.getOrNull() ?: emptyList()
-                    for (v in videos) {
-                        videoList.add(Pair(v, video.priority))
-                    }
                 }
-                altHosterSelection.contains("vidstreaming") && (video.sourceUrl.contains("vidstreaming") || video.sourceUrl.contains("https://gogo")) -> {
+                altHosterSelection.contains("vidstreaming") && (
-                    val extractor = VidstreamingExtractor(client, json)
+                    video.sourceUrl.contains("vidstreaming") || video.sourceUrl.contains("https://gogo") ||
-                    val videos = runCatching {
+                        video.sourceUrl.contains("playgo1.cc")
-                        extractor.videosFromUrl(video.sourceUrl.replace(Regex("^//"), "https://"))
+                    ) -> {
-                    }.getOrNull() ?: emptyList()
+                    serverList.add(Server(video.sourceUrl, "gogo", video.priority))
-                    for (v in videos) {
-                        videoList.add(Pair(v, video.priority))
-                    }
                 }
                 altHosterSelection.contains("doodstream") && video.sourceUrl.contains("dood") -> {
-                    val extractor = DoodExtractor(client)
+                    serverList.add(Server(video.sourceUrl, "dood", video.priority))
-                    val videos = runCatching {
-                        extractor.videosFromUrl(video.sourceUrl)
-                    }.getOrNull() ?: emptyList()
-                    for (v in videos) {
-                        videoList.add(Pair(v, video.priority))
-                    }
                 }
                 altHosterSelection.contains("okru") && video.sourceUrl.contains("ok.ru") -> {
-                    val extractor = OkruExtractor(client)
+                    serverList.add(Server(video.sourceUrl, "okru", video.priority))
-                    val videos = runCatching {
-                        extractor.videosFromUrl(video.sourceUrl)
-                    }.getOrNull() ?: emptyList()
-                    for (v in videos) {
-                        videoList.add(Pair(v, video.priority))
-                    }
                 }
                 altHosterSelection.contains("mp4upload") && video.sourceUrl.contains("mp4upload.com") -> {
-                    val headers = headers.newBuilder().set("referer", "https://mp4upload.com/").build()
+                    serverList.add(Server(video.sourceUrl, "mp4upload", video.priority))
-                    val videos = runCatching {
-                        Mp4uploadExtractor(client).getVideoFromUrl(video.sourceUrl, headers)
-                    }.getOrNull() ?: emptyList()
-                    for (v in videos) {
-                        videoList.add(Pair(v, video.priority))
-                    }
                 }
                 altHosterSelection.contains("streamlare") && video.sourceUrl.contains("streamlare.com") -> {
-                    val extractor = StreamlareExtractor(client)
+                    serverList.add(Server(video.sourceUrl, "streamlare", video.priority))
-                    val videos = runCatching {
-                        extractor.videosFromUrl(video.sourceUrl)
-                    }.getOrNull() ?: emptyList()
-                    for (v in videos) {
-                        videoList.add(Pair(v, video.priority))
-                    }
                 }
             }
         }
 
+        videoList.addAll(
+            serverList.parallelMap { server ->
+                runCatching {
+                    when (server.sourceName) {
+                        "internal" -> {
+                            val extractor = AllAnimeExtractor(client)
+                            val videos = runCatching {
+                                extractor.videoFromUrl(server.sourceUrl, server.sourceName)
+                            }.getOrNull() ?: emptyList()
+                            videos.map {
+                                Pair(it, server.priority)
+                            }
+                        }
+                        "player" -> {
+                            listOf(
+                                Pair(
+                                    Video(
+                                        server.sourceUrl,
+                                        "Original (player ${server.sourceName})",
+                                        server.sourceUrl
+                                    ),
+                                    server.priority
+                                )
+                            )
+                        }
+                        "streamsb" -> {
+                            val extractor = StreamSBExtractor(client)
+                            val videos = runCatching {
+                                extractor.videosFromUrl(server.sourceUrl, headers)
+                            }.getOrNull() ?: emptyList()
+                            videos.map {
+                                Pair(it, server.priority)
+                            }
+                        }
+                        "gogo" -> {
+                            val extractor = VidstreamingExtractor(client, json)
+                            val videos = runCatching {
+                                extractor.videosFromUrl(server.sourceUrl.replace(Regex("^//"), "https://"))
+                            }.getOrNull() ?: emptyList()
+                            videos.map {
+                                Pair(it, server.priority)
+                            }
+                        }
+                        "dood" -> {
+                            val extractor = DoodExtractor(client)
+                            val videos = runCatching {
+                                extractor.videosFromUrl(server.sourceUrl)
+                            }.getOrNull() ?: emptyList()
+                            videos.map {
+                                Pair(it, server.priority)
+                            }
+                        }
+                        "okru" -> {
+                            val extractor = OkruExtractor(client)
+                            val videos = runCatching {
+                                extractor.videosFromUrl(server.sourceUrl)
+                            }.getOrNull() ?: emptyList()
+                            videos.map {
+                                Pair(it, server.priority)
+                            }
+                        }
+                        "mp4upload" -> {
+                            val headers = headers.newBuilder().set("referer", "https://mp4upload.com/").build()
+                            val videos = runCatching {
+                                Mp4uploadExtractor(client).getVideoFromUrl(server.sourceUrl, headers)
+                            }.getOrNull() ?: emptyList()
+                            videos.map {
+                                Pair(it, server.priority)
+                            }
+                        }
+                        "streamlare" -> {
+                            val extractor = StreamlareExtractor(client)
+                            val videos = runCatching {
+                                extractor.videosFromUrl(server.sourceUrl)
+                            }.getOrNull() ?: emptyList()
+                            videos.map {
+                                Pair(it, server.priority)
+                            }
+                        }
+                        else -> null
+                    }
+                }.getOrNull()
+            }.filterNotNull().flatten()
+        )
+
         return prioritySort(videoList)
     }
 
@@ -366,6 +407,12 @@ class AllAnime : ConfigurableAnimeSource, AnimeHttpSource() {
         ).reversed().map { t -> t.first }
     }
 
+    data class Server(
+        val sourceUrl: String,
+        val sourceName: String,
+        val priority: Float,
+    )
+
     private fun parseStatus(string: String?): Int {
         return when (string) {
             "Releasing" -> SAnime.ONGOING
@@ -490,4 +537,10 @@ class AllAnime : ConfigurableAnimeSource, AnimeHttpSource() {
         screen.addPreference(titleStylePref)
         screen.addPreference(subPref)
     }
+
+    // From Dopebox
+    private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
+        runBlocking {
+            map { async(Dispatchers.Default) { f(it) } }.awaitAll()
+        }
 }

src/en/allanime/src/eu/kanade/tachiyomi/animeextension/en/allanime/extractors/AllAnimeExtractor.kt

@@ -7,6 +7,7 @@ import kotlinx.serialization.Serializable
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
 import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
 import okhttp3.OkHttpClient
 import uy.kohesive.injekt.injectLazy
 import java.util.Locale
@@ -141,6 +142,24 @@ class AllAnimeExtractor(private val client: OkHttpClient) {
                         )
                     }
 
+                    if (!masterPlaylist.contains("#EXT-X-STREAM-INF:")) {
+                        val headers = Headers.headersOf(
+                            "Accept", "*/*",
+                            "Host", link.link.toHttpUrl().host,
+                            "Origin", "https://allanimenews.com",
+                            "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0"
+                        )
+                        return try {
+                            if (audioList.isEmpty()) {
+                                listOf(Video(link.link, "$name - ${link.resolutionStr}", link.link, subtitleTracks = subtitles, headers = headers))
+                            } else {
+                                listOf(Video(link.link, "$name - ${link.resolutionStr}", link.link, subtitleTracks = subtitles, audioTracks = audioList, headers = headers))
+                            }
+                        } catch (_: Error) {
+                            listOf(Video(link.link, "$name - ${link.resolutionStr}", link.link, headers = headers))
+                        }
+                    }
+
                     masterPlaylist.substringAfter("#EXT-X-STREAM-INF:").split("#EXT-X-STREAM-INF:")
                         .forEach {
                             val bandwidth = if (it.contains("AVERAGE-BANDWIDTH")) {

src/en/allanime/src/eu/kanade/tachiyomi/animeextension/en/allanime/extractors/StreamlareExtractor.kt

@@ -1,6 +1,7 @@
 package eu.kanade.tachiyomi.animeextension.en.allanime.extractors
 
 import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
@@ -18,13 +19,27 @@ class StreamlareExtractor(private val client: OkHttpClient) {
             )
         ).execute().body!!.string()
 
-        playlist.substringAfter("\"label\":\"").split("\"label\":\"").forEach {
+        val type = playlist.substringAfter("\"type\":\"").substringBefore("\"")
-            val quality = it.substringAfter("\"label\":\"").substringBefore("\",") + " (Sl-mp4)"
+        if (type == "hls") {
-            val token = it.substringAfter("\"file\":\"https:\\/\\/larecontent.com\\/video?token=")
+            val masterPlaylistUrl = playlist.substringAfter("\"file\":\"").substringBefore("\"").replace("\\/", "/")
-                .substringBefore("\",")
+            val masterPlaylist = client.newCall(GET(masterPlaylistUrl)).execute().body!!.string()
-            val response = client.newCall(POST("https://larecontent.com/video?token=$token")).execute()
+
-            val videoUrl = response.request.url.toString()
+            val separator = "#EXT-X-STREAM-INF"
-            videoList.add(Video(videoUrl, quality, videoUrl))
+            masterPlaylist.substringAfter(separator).split(separator).map {
+                val quality = it.substringAfter("RESOLUTION=").substringAfter("x").substringBefore(",") + "p"
+                var videoUrl = it.substringAfter("\n").substringBefore("\n")
+                if (!videoUrl.startsWith("http")) videoUrl = "${masterPlaylistUrl.substringBefore("master.m3u8")}$videoUrl"
+                videoList.add(Video(videoUrl, "$quality (Streamlare)", videoUrl))
+            }
+        } else {
+            playlist.substringAfter("\"label\":\"").split("\"label\":\"").forEach {
+                val quality = it.substringAfter("\"label\":\"").substringBefore("\",") + " (Sl-mp4)"
+                val token = it.substringAfter("\"file\":\"https:\\/\\/larecontent.com\\/video?token=")
+                    .substringBefore("\",")
+                val response = client.newCall(POST("https://larecontent.com/video?token=$token")).execute()
+                val videoUrl = response.request.url.toString()
+                videoList.add(Video(videoUrl, "$quality (Streamlare)", videoUrl))
+            }
         }
         return videoList
     }