fix(id/OtakuDesu): Fix video extractor + refactor (#1669)

src/id/otakudesu/build.gradle

@@ -1,12 +1,18 @@
-apply plugin: 'com.android.application'
+plugins {
-apply plugin: 'kotlin-android'
+    alias(libs.plugins.android.application)
+    alias(libs.plugins.kotlin.android)
+}
 
 ext {
     extName = 'OtakuDesu'
     pkgNameSuffix = 'id.otakudesu'
     extClass = '.OtakuDesu'
-    extVersionCode = 18
+    extVersionCode = 19
     libVersion = '13'
 }
 
+dependencies {
+    implementation(project(":lib-yourupload-extractor"))
+}
+
 apply from: "$rootDir/common.gradle"

src/id/otakudesu/src/eu/kanade/tachiyomi/animeextension/id/otakudesu/OtakuDesu.kt

@@ -2,6 +2,7 @@ package eu.kanade.tachiyomi.animeextension.id.otakudesu
 
 import android.app.Application
 import android.content.SharedPreferences
+import android.util.Base64
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
 import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
@@ -12,11 +13,19 @@ import eu.kanade.tachiyomi.animesource.model.SAnime
 import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
+import eu.kanade.tachiyomi.lib.youruploadextractor.YourUploadExtractor
 import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.async
+import kotlinx.coroutines.awaitAll
+import kotlinx.coroutines.runBlocking
+import okhttp3.FormBody
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
+import org.jsoup.Jsoup
 import org.jsoup.nodes.Document
 import org.jsoup.nodes.Element
 import uy.kohesive.injekt.Injekt
@@ -41,30 +50,27 @@ class OtakuDesu : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
     }
 
+    // =========================== Anime Details ============================
     override fun animeDetailsParse(document: Document): SAnime {
-        val anime = SAnime.create()
+        return SAnime.create().apply {
-        val zing = document.select("div.infozingle")
+            val info = document.selectFirst("div.infozingle")!!
-        val status = parseStatus(zing.select("p:nth-child(6) > span").text().replace("Status: ", ""))
+            title = info.getInfo("Judul") ?: ""
-        anime.title = zing.select("p:nth-child(1) > span").text().replace("Judul: ", "")
+            genre = info.getInfo("Genre")
-        anime.genre = zing.select("p:nth-child(11) > span").text().replace("Genre: ", "")
+            status = parseStatus(info.getInfo("Status"))
-        anime.status = status
+            artist = info.getInfo("Studio")
-        anime.artist = zing.select("p:nth-child(10) > span").text()
+            author = info.getInfo("Produser")
-        anime.author = zing.select("p:nth-child(4) > span").text()
 
-        // Others
+            description = buildString {
-        // Jap title
+                info.getInfo("Japanese", false)?.let { append("$it\n") }
-        anime.description = document.select("p:nth-child(2) > span").text()
+                info.getInfo("Skor", false)?.let { append("$it\n") }
-        // Score
+                info.getInfo("Total Episode", false)?.let { append("$it\n") }
-        anime.description = anime.description + "\n" + document.select("p:nth-child(3) > span").text()
+                append("\n\nSynopsis:\n")
-        // Total Episode
+                document.select("div.sinopc > p").eachText().forEach { append("$it\n\n") }
-        anime.description = anime.description + "\n" + document.select("p:nth-child(7) > span").text()
+            }
-        // Synopsis
+        }
-        anime.description = anime.description + "\n\n\nSynopsis: \n" + document.select("div.sinopc > p").joinToString("\n\n") { it.text() }
-
-        return anime
     }
 
-    private fun parseStatus(statusString: String): Int {
+    private fun parseStatus(statusString: String?): Int {
         return when (statusString) {
             "Ongoing" -> SAnime.ONGOING
             "Completed" -> SAnime.COMPLETED
@@ -72,87 +78,67 @@ class OtakuDesu : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         }
     }
 
+    // ============================== Episodes ==============================
+    private val nameRegex by lazy { ".+?(?=Episode)|\\sSubtitle.+".toRegex() }
     override fun episodeFromElement(element: Element): SEpisode {
-        val episode = SEpisode.create()
+        return SEpisode.create().apply {
-        val epsNum = getNumberFromEpsString(element.select("span > a").text())
+            val link = element.selectFirst("span > a")!!
-        episode.setUrlWithoutDomain(element.select("span > a").attr("href"))
+            val text = link.text()
-        episode.episode_number = when {
+            episode_number = text.substringAfter("Episode ")
-            (epsNum.isNotEmpty()) -> epsNum.toFloat()
+                .substringBefore(" ")
-            else -> 1F
+                .toFloatOrNull() ?: 1F
+            setUrlWithoutDomain(link.attr("href"))
+            name = text.replace(nameRegex, "")
+            date_upload = element.selectFirst("span.zeebr")?.text().toDate()
         }
-        episode.name = element.select("span > a").text().replace(".+?(?=Episode)|\\sSubtitle.+".toRegex(), "")
-        episode.date_upload = reconstructDate(element.select("span.zeebr").text())
-
-        return episode
     }
 
-    private fun getNumberFromEpsString(epsStr: String): String {
+    override fun episodeListSelector() = "#venkonten > div.venser > div:nth-child(8) > ul > li"
-        return epsStr.filter { it.isDigit() }
-    }
-
-    private fun reconstructDate(Str: String): Long {
-        val newStr = Str.replace("Januari", "Jan").replace("Februari", "Feb").replace("Maret", "Mar").replace("April", "Apr").replace("Mei", "May").replace("Juni", "Jun").replace("Juli", "Jul").replace("Agustus", "Aug").replace("September", "Sep").replace("Oktober", "Oct").replace("November", "Nov").replace("Desember", "Dec")
-
-        val pattern = SimpleDateFormat("d MMM yyyy", Locale.US)
-        return pattern.parse(newStr.replace(",", " "))!!.time
-    }
-
-    override fun episodeListSelector(): String = "#venkonten > div.venser > div:nth-child(8) > ul > li"
 
+    // =============================== Latest ===============================
     override fun latestUpdatesFromElement(element: Element): SAnime {
-        val anime = SAnime.create()
+        return SAnime.create().apply {
-        anime.setUrlWithoutDomain(element.selectFirst("div.thumb > a")!!.attr("href"))
+            setUrlWithoutDomain(element.attr("href"))
-        anime.thumbnail_url = element.selectFirst("div.thumb > a > div.thumbz > img")!!.attr("src")
+            thumbnail_url = element.selectFirst("img")!!.attr("src")
-        anime.title = element.select("div.thumb > a > div.thumbz > h2").text()
+            title = element.selectFirst("h2")!!.text()
-        return anime
+        }
     }
 
-    override fun latestUpdatesNextPageSelector(): String = "a.next.page-numbers"
+    override fun latestUpdatesNextPageSelector() = "a.next.page-numbers"
 
-    override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/ongoing-anime/page/$page")
+    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/ongoing-anime/page/$page")
 
-    override fun latestUpdatesSelector(): String = "div.detpost"
+    override fun latestUpdatesSelector() = "div.detpost div.thumb > a"
 
-    override fun popularAnimeFromElement(element: Element): SAnime {
+    // ============================== Popular ===============================
-        val anime = SAnime.create()
+    override fun popularAnimeFromElement(element: Element) = latestUpdatesFromElement(element)
-        anime.setUrlWithoutDomain(element.selectFirst("div.thumb > a")!!.attr("href"))
+    override fun popularAnimeNextPageSelector() = latestUpdatesNextPageSelector()
-        anime.thumbnail_url = element.selectFirst("div.thumb > a > div.thumbz > img")!!.attr("src")
+    override fun popularAnimeRequest(page: Int) = GET("$baseUrl/complete-anime/page/$page")
-        anime.title = element.select("div.thumb > a > div.thumbz > h2").text()
+    override fun popularAnimeSelector() = latestUpdatesSelector()
-        return anime
-    }
-
-    override fun popularAnimeNextPageSelector(): String = "a.next.page-numbers"
-
-    override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/complete-anime/page/$page")
-
-    override fun popularAnimeSelector(): String = "div.detpost"
 
+    // =============================== Search ===============================
     override fun searchAnimeFromElement(element: Element): SAnime = throw Exception("not used")
 
     private fun searchAnimeFromElement(element: Element, ui: String): SAnime {
-        val anime = SAnime.create()
+        return SAnime.create().apply {
-        anime.setUrlWithoutDomain(
             when (ui) {
-                "search" -> element.selectFirst("h2 > a")!!.attr("href")
+                "search" -> {
-                "genres" -> element.select(".col-anime-title > a").attr("href")
+                    val link = element.selectFirst("h2 > a")!!
-                else -> element.selectFirst("div.thumb > a")!!.attr("href")
+                    setUrlWithoutDomain(link.attr("href"))
-            },
+                    title = link.text().replace(" Subtitle Indonesia", "")
-        )
+                    thumbnail_url = element.selectFirst("img")!!.attr("src")
-
+                }
-        anime.thumbnail_url = when (ui) {
+                else -> {
-            "search" -> element.selectFirst("img")!!.attr("src")
+                    val link = element.selectFirst(".col-anime-title > a")!!
-            "genres" -> element.select(".col-anime-cover > img").attr("src")
+                    setUrlWithoutDomain(link.attr("href"))
-            else -> element.selectFirst("div.thumb > a > div.thumbz > img")!!.attr("src")
+                    title = link.text()
+                    thumbnail_url = element.selectFirst(".col-anime-cover > img")!!.attr("src")
+                }
+            }
         }
-        anime.title = when (ui) {
-            "search" -> element.select("h2 > a").text().replace(" Subtitle Indonesia", "")
-            "genres" -> element.select(".col-anime-title > a").text()
-            else -> element.select("div.thumb > a > div.thumbz > h2").text()
-        }
-        return anime
     }
 
-    override fun searchAnimeNextPageSelector(): String = "a.next.page-numbers"
+    override fun searchAnimeNextPageSelector() = latestUpdatesNextPageSelector()
 
     override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
         val filterList = if (filters.isEmpty()) getFilterList() else filters
@@ -165,94 +151,125 @@ class OtakuDesu : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         }
     }
 
-    override fun searchAnimeSelector(): String = "#venkonten > div > div.venser > div > div > ul > li"
+    override fun searchAnimeSelector() = "#venkonten > div > div.venser > div > div > ul > li"
+    private val genreSelector = ".col-anime"
 
     override fun searchAnimeParse(response: Response): AnimesPage {
         val document = response.asJsoup()
 
         val ui = when {
-            document.select(".col-anime").isNullOrEmpty() -> "search"
+            document.selectFirst(genreSelector) == null -> "search"
-            document.select("#venkonten > div > div.venser > div > div > ul > li").isNullOrEmpty() -> "genres"
+            document.selectFirst(searchAnimeSelector()) == null -> "genres"
             else -> "unknown"
         }
 
         val animes = when (ui) {
-            "genres" -> document.select(".col-anime").map { element -> searchAnimeFromElement(element, ui) }
+            "genres" -> document.select(genreSelector).map { searchAnimeFromElement(it, ui) }
-            "search" -> document.select("#venkonten > div > div.venser > div > div > ul > li").map { element -> searchAnimeFromElement(element, ui) }
+            "search" -> document.select(searchAnimeSelector()).map { searchAnimeFromElement(it, ui) }
-            else -> document.select("div.detpost").map { element -> popularAnimeFromElement(element) }
+            else -> document.select(latestUpdatesSelector()).map(::latestUpdatesFromElement)
         }
 
-        val hasNextPage = searchAnimeNextPageSelector().let { selector ->
+        val hasNextPage = document.selectFirst(searchAnimeNextPageSelector()) != null
-            document.select(selector).first()
-        } != null
 
         return AnimesPage(animes, hasNextPage)
     }
 
-    override fun videoListSelector() = "div.download > ul > li > a:nth-child(2)"
+    // ============================ Video Links =============================
+    override fun videoListSelector() = "div.mirrorstream ul li > a"
 
-    override fun List<Video>.sort(): List<Video> {
+    override fun videoListParse(response: Response): List<Video> {
-        val quality = preferences.getString("preferred_quality", null)
+        val doc = response.use { it.asJsoup() }
-        if (quality != null) {
+        val script = doc.selectFirst("script:containsData({action:)")!!
-            val newList = mutableListOf<Video>()
+            .data()
-            var preferred = 0
+
-            for (video in this) {
+        val nonceAction = script.substringAfter("{action:\"").substringBefore('"')
-                if (video.quality.contains(quality)) {
+        val action = script.substringAfter("action:\"").substringBefore('"')
-                    newList.add(preferred, video)
+
-                    preferred++
+        val nonce = getNonce(nonceAction)
-                } else {
+
-                    newList.add(video)
+        return doc.select(videoListSelector())
+            .parallelMapNotNull {
+                runCatching { getEmbedLinks(it, action, nonce) }.getOrNull()
+            }
+            .parallelMapNotNull {
+                runCatching {
+                    getVideosFromEmbed(it.first, it.second)
+                }.getOrNull()
+            }.flatten()
+    }
+
+    private fun getEmbedLinks(element: Element, action: String, nonce: String): Pair<String, String> {
+        val decodedData = element.attr("data-content").b64Decode()
+            .drop(1)
+            .dropLast(1)
+
+        val (id, mirror, quality) = decodedData.split(",").map {
+            it.substringAfter(":").replace("\"", "")
+        }
+
+        val form = FormBody.Builder().apply {
+            add("id", id)
+            add("i", mirror)
+            add("q", quality)
+            add("nonce", nonce)
+            add("action", action)
+        }.build()
+
+        val doc = client.newCall(POST("$baseUrl/wp-admin/admin-ajax.php", body = form))
+            .execute()
+            .body.string()
+            .substringAfter(":\"")
+            .substringBefore('"')
+            .b64Decode()
+            .let(Jsoup::parse)
+
+        val url = doc.selectFirst("iframe")!!.attr("src")
+
+        return Pair(quality, url)
+    }
+
+    private fun getVideosFromEmbed(quality: String, link: String): List<Video> {
+        return when {
+            "yourupload" in link -> {
+                val id = link.substringAfter("id=").substringBefore("&")
+                val url = "https://yourupload.com/embed/$id"
+                YourUploadExtractor(client).videoFromUrl(url, headers, "YourUpload - $quality")
+            }
+            "desustream" in link -> {
+                client.newCall(GET(link, headers)).execute().use {
+                    val doc = it.asJsoup()
+                    val script = doc.selectFirst("script:containsData(sources)")!!.data()
+                    val videoUrl = script.substringAfter("sources:[{")
+                        .substringAfter("file':'")
+                        .substringBefore("'")
+                    listOf(Video(videoUrl, "DesuStream - $quality", videoUrl, headers))
                 }
             }
-            return newList
+            "mp4upload" in link -> {
-        }
+                client.newCall(GET(link, headers)).execute().use {
-        return this
+                    val doc = it.asJsoup()
-    }
+                    val script = doc.selectFirst("script:containsData(player.src)")!!.data()
-
+                    val videoUrl = script.substringAfter("src: \"").substringBefore('"')
-    override fun videoFromElement(element: Element): Video {
+                    listOf(Video(videoUrl, "Mp4upload - $quality", videoUrl, headers))
-        val res = client.newCall(GET(element.attr("href"))).execute().asJsoup()
+                }
-        val scr = res.select("script:containsData(dlbutton)").html()
-        var url = element.attr("href").substringBefore("/v/")
-        val numbs = scr.substringAfter("\" + (").substringBefore(") + \"")
-        val firstString = scr.substringAfter(" = \"").substringBefore("\" + (")
-        val num = numbs.substringBefore(" % ").toInt()
-        val lastString = scr.substringAfter("913) + \"").substringBefore("\";")
-        val nums = num % 51245 + num % 913
-        url += firstString + nums.toString() + lastString
-        val quality = with(lastString) {
-            when {
-                contains("1080p") -> "1080p"
-                contains("720p") -> "720p"
-                contains("480p") -> "480p"
-                contains("360p") -> "360p"
-                else -> "Default"
             }
+            else -> emptyList()
         }
-        return Video(url, quality, url)
     }
 
+    private fun getNonce(action: String): String {
+        val form = FormBody.Builder().add("action", action).build()
+        return client.newCall(POST("$baseUrl/wp-admin/admin-ajax.php", body = form))
+            .execute()
+            .use {
+                it.body.string().substringAfter(":\"").substringBefore('"')
+            }
+    }
+
+    override fun videoFromElement(element: Element) = throw Exception("not used")
     override fun videoUrlParse(document: Document) = throw Exception("not used")
 
-    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+    // ============================== Filters ===============================
-        val videoQualityPref = ListPreference(screen.context).apply {
-            key = "preferred_quality"
-            title = "Preferred quality"
-            entries = arrayOf("1080p", "720p", "480p", "360p")
-            entryValues = arrayOf("1080", "720", "480", "360")
-            setDefaultValue("1080")
-            summary = "%s"
-
-            setOnPreferenceChangeListener { _, newValue ->
-                val selected = newValue as String
-                val index = findIndexOfValue(selected)
-                val entry = entryValues[index] as String
-                preferences.edit().putString(key, entry).commit()
-            }
-        }
-        screen.addPreference(videoQualityPref)
-    }
-
-    // filter
     override fun getFilterList(): AnimeFilterList = AnimeFilterList(
         AnimeFilter.Header("Text search ignores filters"),
         GenreFilter(),
@@ -305,4 +322,66 @@ class OtakuDesu : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
         AnimeFilter.Select<String>(displayName, vals.map { it.first }.toTypedArray()) {
         fun toUriPart() = vals[state].second
     }
+
+    // ============================== Settings ==============================
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        val videoQualityPref = ListPreference(screen.context).apply {
+            key = PREF_QUALITY_KEY
+            title = PREF_QUALITY_TITLE
+            entries = PREF_QUALITY_ENTRIES
+            entryValues = PREF_QUALITY_ENTRIES
+            setDefaultValue(PREF_QUALITY_DEFAULT)
+            summary = "%s"
+            setOnPreferenceChangeListener { _, newValue ->
+                val selected = newValue as String
+                val index = findIndexOfValue(selected)
+                val entry = entryValues[index] as String
+                preferences.edit().putString(key, entry).commit()
+            }
+        }
+        screen.addPreference(videoQualityPref)
+    }
+
+    // ============================= Utilities ==============================
+    private fun Element.getInfo(info: String, cut: Boolean = true): String? {
+        return selectFirst("p > span:has(b:contains($info))")?.text()
+            ?.let {
+                when {
+                    cut -> it.substringAfter(":")
+                    else -> it
+                }.trim()
+            }
+    }
+
+    private fun String?.toDate(): Long {
+        return runCatching { DATE_FORMATTER.parse(this?.trim() ?: "")?.time }
+            .getOrNull() ?: 0L
+    }
+
+    override fun List<Video>.sort(): List<Video> {
+        val quality = preferences.getString(PREF_QUALITY_KEY, PREF_QUALITY_DEFAULT)!!
+        return sortedWith(
+            compareByDescending { it.quality.equals(quality) },
+        )
+    }
+
+    private inline fun <A, B> Iterable<A>.parallelMapNotNull(crossinline f: suspend (A) -> B?): List<B> =
+        runBlocking {
+            map { async(Dispatchers.Default) { f(it) } }.awaitAll().filterNotNull()
+        }
+
+    private fun String.b64Decode(): String {
+        return String(Base64.decode(this, Base64.DEFAULT))
+    }
+
+    companion object {
+        private val DATE_FORMATTER by lazy {
+            SimpleDateFormat("d MMM,yyyy", Locale("id", "ID"))
+        }
+
+        private const val PREF_QUALITY_KEY = "preferred_quality"
+        private const val PREF_QUALITY_TITLE = "Preferred quality"
+        private const val PREF_QUALITY_DEFAULT = "1080p"
+        private val PREF_QUALITY_ENTRIES = arrayOf("1080p", "720p", "480p", "360p")
+    }
 }