feat(gogoanime): add mp4upload + parallelize + hoster toggles, refactoring (#1682)

src/en/gogoanime/build.gradle

@@ -5,13 +5,14 @@ ext {
     extName = 'Gogoanime'
     pkgNameSuffix = 'en.gogoanime'
     extClass = '.GogoAnime'
-    extVersionCode = 65
+    extVersionCode = 66
     libVersion = '13'
 }
 
 dependencies {
     implementation(project(':lib-streamsb-extractor'))
     implementation(project(':lib-dood-extractor'))
+    implementation "dev.datlag.jsunpacker:jsunpacker:1.0.1"
 }
 
 apply from: "$rootDir/common.gradle"

src/en/gogoanime/src/eu/kanade/tachiyomi/animeextension/en/gogoanime/GogoAnime.kt

@@ -3,8 +3,10 @@ package eu.kanade.tachiyomi.animeextension.en.gogoanime
 import android.app.Application
 import android.content.SharedPreferences
 import androidx.preference.ListPreference
+import androidx.preference.MultiSelectListPreference
 import androidx.preference.PreferenceScreen
 import eu.kanade.tachiyomi.animeextension.en.gogoanime.extractors.GogoCdnExtractor
+import eu.kanade.tachiyomi.animeextension.en.gogoanime.extractors.Mp4uploadExtractor
 import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
 import eu.kanade.tachiyomi.animesource.model.AnimeFilter
 import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
@@ -16,6 +18,10 @@ import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
 import eu.kanade.tachiyomi.lib.streamsbextractor.StreamSBExtractor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.json.Json
 import okhttp3.OkHttpClient
@@ -33,7 +39,7 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     override val name = "Gogoanime"
 
-    override val baseUrl by lazy { preferences.getString("preferred_domain", "https://gogoanime.cl")!! }
+    override val baseUrl by lazy { preferences.getString(PREF_DOMAIN_KEY, PREF_DOMAIN_DEFAULT)!! }
 
     override val lang = "en"
 
@@ -47,98 +53,37 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
     }
 
-    override fun popularAnimeSelector(): String = "div.img a"
+    // ============================== Popular ===============================
 
     override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/popular.html?page=$page")
 
-    override fun popularAnimeFromElement(element: Element): SAnime {
+    override fun popularAnimeSelector(): String = "div.img a"
-        val anime = SAnime.create()
+
-        anime.setUrlWithoutDomain(element.attr("href"))
+    override fun popularAnimeFromElement(element: Element): SAnime = SAnime.create().apply {
-        anime.thumbnail_url = element.selectFirst("img")!!.attr("src")
+        setUrlWithoutDomain(element.attr("href"))
-        anime.title = element.attr("title")
+        thumbnail_url = element.selectFirst("img")!!.attr("src")
-        return anime
+        title = element.attr("title")
     }
 
     override fun popularAnimeNextPageSelector(): String = "ul.pagination-list li:last-child:not(.selected)"
 
-    override fun episodeListSelector() = "ul#episode_page li a"
+    // =============================== Latest ===============================
 
-    override fun episodeListParse(response: Response): List<SEpisode> {
+    override fun latestUpdatesRequest(page: Int): Request =
-        val document = response.asJsoup()
+        GET("https://ajax.gogo-load.com/ajax/page-recent-release-ongoing.html?page=$page&type=1", headers)
-        val totalEpisodes = document.select(episodeListSelector()).last()!!.attr("ep_end")
+
-        val id = document.select("input#movie_id").attr("value")
+    override fun latestUpdatesSelector(): String = "div.added_series_body.popular li a:has(div)"
-        return episodesRequest(totalEpisodes, id)
+
+    override fun latestUpdatesFromElement(element: Element): SAnime = SAnime.create().apply {
+        setUrlWithoutDomain(element.attr("abs:href"))
+        thumbnail_url = element.select("div.thumbnail-popular").attr("style")
+            .substringAfter("background: url('").substringBefore("');")
+        title = element.attr("title")
     }
 
-    private fun episodesRequest(totalEpisodes: String, id: String): List<SEpisode> {
+    override fun latestUpdatesNextPageSelector(): String = popularAnimeNextPageSelector()
-        val request = GET("https://ajax.gogo-load.com/ajax/load-list-episode?ep_start=0&ep_end=$totalEpisodes&id=$id", headers)
-        val epResponse = client.newCall(request).execute()
-        val document = epResponse.asJsoup()
-        return document.select("a").map { episodeFromElement(it) }
-    }
 
-    override fun episodeFromElement(element: Element): SEpisode {
+    // =============================== Search ===============================
-        val episode = SEpisode.create()
-        episode.setUrlWithoutDomain(baseUrl + element.attr("href").substringAfter(" "))
-        val ep = element.selectFirst("div.name")!!.ownText().substringAfter(" ")
-        episode.episode_number = ep.toFloat()
-        episode.name = "Episode $ep"
-        return episode
-    }
-
-    override fun videoListParse(response: Response): List<Video> {
-        val document = response.asJsoup()
-        val extractor = GogoCdnExtractor(network.client, json)
-        val videoList = mutableListOf<Video>()
-
-        // GogoCdn:
-        document.select("div.anime_muti_link > ul > li.vidcdn > a")
-            .firstOrNull()?.attr("data-video")
-            ?.let { videoList.addAll(extractor.videosFromUrl(it.replace(Regex("^//"), "https://"))) }
-        // Vidstreaming:
-        document.select("div.anime_muti_link > ul > li.anime > a")
-            .firstOrNull()?.attr("data-video")
-            ?.let { videoList.addAll(extractor.videosFromUrl(it.replace(Regex("^//"), "https://"))) }
-        // Doodstream mirror:
-        document.select("div.anime_muti_link > ul > li.doodstream > a")
-            .firstOrNull()?.attr("data-video")
-            ?.let { videoList.addAll(DoodExtractor(client).videosFromUrl(it.replace(Regex("^//"), "https://"))) }
-        // StreamSB mirror:
-        document.select("div.anime_muti_link > ul > li.streamsb > a")
-            .firstOrNull()?.attr("data-video")
-            ?.let { videoList.addAll(StreamSBExtractor(client).videosFromUrl(it.replace(Regex("^//"), "https://"), headers)) }
-        return videoList.sort()
-    }
-
-    override fun videoListSelector() = throw Exception("not used")
-
-    override fun videoFromElement(element: Element) = throw Exception("not used")
-
-    override fun videoUrlParse(document: Document) = throw Exception("not used")
-
-    override fun List<Video>.sort(): List<Video> {
-        val quality = preferences.getString("preferred_quality", "1080")!!
-        val server = preferences.getString("preferred_server", "Gogostream")!!
-
-        return this.sortedWith(
-            compareBy(
-                { it.quality.contains(quality) },
-                { it.quality.contains(server) },
-            ),
-        ).reversed()
-    }
-
-    override fun searchAnimeFromElement(element: Element): SAnime {
-        val anime = SAnime.create()
-        anime.setUrlWithoutDomain(element.attr("href"))
-        anime.thumbnail_url = element.selectFirst("img")!!.attr("src")
-        anime.title = element.attr("title")
-        return anime
-    }
-
-    override fun searchAnimeNextPageSelector(): String = "ul.pagination-list li:last-child:not(.selected)"
-
-    override fun searchAnimeSelector(): String = "div.img a"
 
     override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
         val filterList = if (filters.isEmpty()) getFilterList() else filters
@@ -155,24 +100,124 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         }
     }
 
-    override fun animeDetailsParse(document: Document): SAnime {
+    override fun searchAnimeSelector(): String = popularAnimeSelector()
-        val anime = SAnime.create()
-        anime.title = document.select("div.anime_info_body_bg h1").text()
-        anime.genre = document.select("p.type:eq(5) a").joinToString("") { it.text() }
-        anime.description = document.selectFirst("p.type:eq(4)")!!.ownText()
-        anime.status = parseStatus(document.select("p.type:eq(7) a").text())
 
-        // add alternative name to anime description
+    override fun searchAnimeFromElement(element: Element): SAnime = popularAnimeFromElement(element)
-        val altName = "Other name(s): "
+
-        document.selectFirst("p.type:eq(8)")?.ownText()?.let {
+    override fun searchAnimeNextPageSelector(): String = popularAnimeNextPageSelector()
-            if (it.isBlank().not()) {
+
-                anime.description = when {
+    // =========================== Anime Details ============================
-                    anime.description.isNullOrBlank() -> altName + it
+
-                    else -> anime.description + "\n\n$altName" + it
+    override fun animeDetailsParse(document: Document): SAnime {
+        return SAnime.create().apply {
+            title = document.select("div.anime_info_body_bg h1").text()
+            genre = document.select("p.type:eq(5) a").joinToString("") { it.text() }
+            description = document.selectFirst("p.type:eq(4)")!!.ownText()
+            status = parseStatus(document.select("p.type:eq(7) a").text())
+
+            // add alternative name to anime description
+            val altName = "Other name(s): "
+            document.selectFirst("p.type:eq(8)")?.ownText()?.let {
+                if (it.isBlank().not()) {
+                    description = when {
+                        description.isNullOrBlank() -> altName + it
+                        else -> description + "\n\n$altName" + it
+                    }
                 }
             }
         }
-        return anime
+    }
+
+    // ============================== Episodes ==============================
+
+    private fun episodesRequest(totalEpisodes: String, id: String): List<SEpisode> {
+        val request = GET("https://ajax.gogo-load.com/ajax/load-list-episode?ep_start=0&ep_end=$totalEpisodes&id=$id", headers)
+        val epResponse = client.newCall(request).execute()
+        val document = epResponse.asJsoup()
+        return document.select("a").map { episodeFromElement(it) }
+    }
+
+    override fun episodeListSelector() = "ul#episode_page li a"
+
+    override fun episodeListParse(response: Response): List<SEpisode> {
+        val document = response.asJsoup()
+        val totalEpisodes = document.select(episodeListSelector()).last()!!.attr("ep_end")
+        val id = document.select("input#movie_id").attr("value")
+        return episodesRequest(totalEpisodes, id)
+    }
+
+    override fun episodeFromElement(element: Element): SEpisode {
+        val ep = element.selectFirst("div.name")!!.ownText().substringAfter(" ")
+        return SEpisode.create().apply {
+            setUrlWithoutDomain(element.attr("abs:href"))
+            episode_number = ep.toFloat()
+            name = "Episode $ep"
+        }
+    }
+
+    // ============================ Video Links =============================
+
+    override fun videoListParse(response: Response): List<Video> {
+        val document = response.asJsoup()
+        val gogoExtractor = GogoCdnExtractor(client, json)
+        val hosterSelection = preferences.getStringSet(PREF_HOSTER_KEY, PREF_HOSTER_DEFAULT)!!
+
+        val videoList = mutableListOf<Video>()
+
+        videoList.addAll(
+            document.select("div.anime_muti_link > ul > li").parallelMap { server ->
+                runCatching {
+                    val className = server.className()
+                    if (!hosterSelection.contains(className)) return@runCatching null
+                    val serverUrl = server.selectFirst("a")
+                        ?.attr("data-video")
+                        ?.replace(Regex("^//"), "https://")
+                        ?: return@runCatching null
+                    when (className) {
+                        "anime" -> {
+                            gogoExtractor.videosFromUrl(serverUrl)
+                        }
+                        "vidcdn" -> {
+                            gogoExtractor.videosFromUrl(serverUrl)
+                        }
+                        "streamsb" -> {
+                            StreamSBExtractor(client).videosFromUrl(serverUrl, headers)
+                        }
+                        "doodstream" -> {
+                            DoodExtractor(client).videosFromUrl(serverUrl)
+                        }
+                        "mp4upload" -> {
+                            val headers = headers.newBuilder().set("Referer", "https://mp4upload.com/").build()
+                            Mp4uploadExtractor(client).getVideoFromUrl(serverUrl, headers)
+                        }
+                        else -> null
+                    }
+                }.getOrNull()
+            }.filterNotNull().flatten(),
+        )
+
+        return videoList.sort()
+    }
+
+    override fun videoListSelector() = throw Exception("not used")
+
+    override fun videoFromElement(element: Element) = throw Exception("not used")
+
+    override fun videoUrlParse(document: Document) = throw Exception("not used")
+
+    // ============================= Utilities ==============================
+
+    override fun List<Video>.sort(): List<Video> {
+        val quality = preferences.getString(PREF_QUALITY_KEY, PREF_QUALITY_DEFAULT)!!
+        val server = preferences.getString(PREF_SERVER_KEY, PREF_SERVER_TITLE)!!
+
+        return this.sortedWith(
+            compareBy(
+                { it.quality.contains(quality) },
+                { Regex("""(\d+)p""").find(it.quality)?.groupValues?.get(1)?.toIntOrNull() ?: 0 },
+                { it.quality.contains(server) },
+            ),
+        ).reversed()
     }
 
     private fun parseStatus(statusString: String): Int {
@@ -183,29 +228,58 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         }
     }
 
-    override fun latestUpdatesNextPageSelector(): String = "ul.pagination-list li:last-child:not(.selected)"
+    // From Dopebox
+    private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
+        runBlocking {
+            map { async(Dispatchers.Default) { f(it) } }.awaitAll()
+        }
 
-    override fun latestUpdatesFromElement(element: Element): SAnime {
+    companion object {
-        val anime = SAnime.create()
+        private val HOSTERS = arrayOf(
-        anime.setUrlWithoutDomain(baseUrl + element.attr("href"))
+            "Gogostream",
-        val style = element.select("div.thumbnail-popular").attr("style")
+            "Vidstreaming",
-        anime.thumbnail_url = style.substringAfter("background: url('").substringBefore("');")
+            "Doodstream",
-        anime.title = element.attr("title")
+            "StreamSB",
-        return anime
+            "Mp4upload",
+        )
+        private val HOSTERS_NAMES = arrayOf( // Names that appears in the gogo html
+            "vidcdn",
+            "anime",
+            "doodstream",
+            "streamsb",
+            "mp4upload",
+        )
+
+        private const val PREF_DOMAIN_KEY = "preferred_domain"
+        private const val PREF_DOMAIN_TITLE = "Preferred domain (requires app restart)"
+        private val PREF_DOMAIN_ENTRIES = arrayOf("gogoanime.cl", "gogoanime.llc")
+        private val PREF_DOMAIN_ENTRY_VALUES = PREF_DOMAIN_ENTRIES.map { "https://$it" }.toTypedArray()
+        private val PREF_DOMAIN_DEFAULT = "https://gogoanime.cl"
+
+        private const val PREF_QUALITY_KEY = "preferred_quality"
+        private const val PREF_QUALITY_TITLE = "Preferred quality"
+        private val PREF_QUALITY_ENTRY_VALUES = arrayOf("1080", "720", "480", "360")
+        private val PREF_QUALITY_ENTRIES = PREF_QUALITY_ENTRY_VALUES.map { "${it}p" }.toTypedArray()
+        private const val PREF_QUALITY_DEFAULT = "1080"
+
+        private const val PREF_SERVER_KEY = "preferred_server"
+        private const val PREF_SERVER_TITLE = "Preferred server"
+        private const val PREF_SERVER_DEFAULT = "Gogostream"
+
+        private const val PREF_HOSTER_KEY = "hoster_selection"
+        private const val PREF_HOSTER_TITLE = "Enable/Disable Hosts"
+        private val PREF_HOSTER_DEFAULT = HOSTERS_NAMES.toSet()
     }
 
-    override fun latestUpdatesRequest(page: Int): Request =
+    // ============================== Settings ==============================
-        GET("https://ajax.gogo-load.com/ajax/page-recent-release-ongoing.html?page=$page&type=1", headers)
-
-    override fun latestUpdatesSelector(): String = "div.added_series_body.popular li a:has(div)"
 
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
-        val domainPref = ListPreference(screen.context).apply {
+        ListPreference(screen.context).apply {
-            key = "preferred_domain"
+            key = PREF_DOMAIN_KEY
-            title = "Preferred domain (requires app restart)"
+            title = PREF_DOMAIN_TITLE
-            entries = arrayOf("gogoanime.cl", "gogoanime.llc")
+            entries = PREF_DOMAIN_ENTRIES
-            entryValues = arrayOf("https://gogoanime.cl", "https://gogoanime.llc")
+            entryValues = PREF_DOMAIN_ENTRY_VALUES
-            setDefaultValue("https://gogoanime.cl")
+            setDefaultValue(PREF_DOMAIN_DEFAULT)
             summary = "%s"
 
             setOnPreferenceChangeListener { _, newValue ->
@@ -214,13 +288,14 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
                 val entry = entryValues[index] as String
                 preferences.edit().putString(key, entry).commit()
             }
-        }
+        }.let { screen.addPreference(it) }
-        val videoQualityPref = ListPreference(screen.context).apply {
+
-            key = "preferred_quality"
+        ListPreference(screen.context).apply {
-            title = "Preferred quality"
+            key = PREF_QUALITY_KEY
-            entries = arrayOf("1080p", "720p", "480p", "360p")
+            title = PREF_QUALITY_TITLE
-            entryValues = arrayOf("1080", "720", "480", "360")
+            entries = PREF_QUALITY_ENTRIES
-            setDefaultValue("1080")
+            entryValues = PREF_QUALITY_ENTRY_VALUES
+            setDefaultValue(PREF_QUALITY_DEFAULT)
             summary = "%s"
 
             setOnPreferenceChangeListener { _, newValue ->
@@ -229,13 +304,14 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
                 val entry = entryValues[index] as String
                 preferences.edit().putString(key, entry).commit()
             }
-        }
+        }.let { screen.addPreference(it) }
-        val videoServerPref = ListPreference(screen.context).apply {
+
-            key = "preferred_server"
+        ListPreference(screen.context).apply {
-            title = "Preferred server"
+            key = PREF_SERVER_KEY
-            entries = arrayOf("Gogostream", "Vidstreaming", "Doodstream", "StreamSB")
+            title = PREF_SERVER_TITLE
-            entryValues = arrayOf("Gogostream", "Vidstreaming", "Doodstream", "StreamSB")
+            entries = HOSTERS
-            setDefaultValue("Gogostream")
+            entryValues = HOSTERS
+            setDefaultValue(PREF_SERVER_DEFAULT)
             summary = "%s"
 
             setOnPreferenceChangeListener { _, newValue ->
@@ -244,13 +320,24 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
                 val entry = entryValues[index] as String
                 preferences.edit().putString(key, entry).commit()
             }
-        }
+        }.let { screen.addPreference(it) }
-        screen.addPreference(domainPref)
+
-        screen.addPreference(videoQualityPref)
+        MultiSelectListPreference(screen.context).apply {
-        screen.addPreference(videoServerPref)
+            key = PREF_HOSTER_KEY
+            title = PREF_HOSTER_TITLE
+            entries = HOSTERS
+            entryValues = HOSTERS_NAMES
+            setDefaultValue(PREF_HOSTER_DEFAULT)
+
+            setOnPreferenceChangeListener { _, newValue ->
+                @Suppress("UNCHECKED_CAST")
+                preferences.edit().putStringSet(key, newValue as Set<String>).commit()
+            }
+        }.let { screen.addPreference(it) }
     }
 
-    // Filters
+    // ============================== Filters ===============================
+
     override fun getFilterList(): AnimeFilterList = AnimeFilterList(
         AnimeFilter.Header("Text search ignores filters"),
         GenreFilter(),

src/en/gogoanime/src/eu/kanade/tachiyomi/animeextension/en/gogoanime/extractors/Mp4uploadExtractor.kt

@@ -0,0 +1,30 @@
+package eu.kanade.tachiyomi.animeextension.en.gogoanime.extractors
+
+import dev.datlag.jsunpacker.JsUnpacker
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+import org.jsoup.Jsoup
+
+class Mp4uploadExtractor(private val client: OkHttpClient) {
+    fun getVideoFromUrl(url: String, headers: Headers, prefix: String = ""): List<Video> {
+        val body = client.newCall(GET(url, headers = headers)).execute().body.string()
+
+        val videoUrl = if (body.contains("eval(function(p,a,c,k,e,d)")) {
+            val packed = body.substringAfter("<script type='text/javascript'>eval(function(p,a,c,k,e,d)")
+                .substringBefore("</script>")
+            body.substringAfter("<script type='text/javascript'>eval(function(p,a,c,k,e,d)")
+                .substringBefore("</script>")
+            val unpacked = JsUnpacker.unpackAndCombine("eval(function(p,a,c,k,e,d)" + packed) ?: return emptyList()
+            unpacked.substringAfter("player.src(\"").substringBefore("\");")
+        } else {
+            val script = Jsoup.parse(body).selectFirst("script:containsData(player.src)")?.data() ?: return emptyList()
+            script.substringAfter("src: \"").substringBefore("\"")
+        }
+
+        return listOf(
+            Video(videoUrl, "${prefix}Mp4upload", videoUrl, headers = Headers.headersOf("Referer", "https://www.mp4upload.com/")),
+        )
+    }
+}