feat(pl/desuonline): Convert to multisrc theme and fix video extractors (#2039)

multisrc/overrides/animestream/desuonline/additional.gradle

@@ -0,0 +1,4 @@
+dependencies {
+    implementation(project(':lib-okru-extractor'))
+    implementation(project(':lib-sibnet-extractor'))
+}

multisrc/overrides/animestream/desuonline/src/DesuOnline.kt

@@ -0,0 +1,86 @@
+package eu.kanade.tachiyomi.animeextension.pl.desuonline
+
+import androidx.preference.ListPreference
+import androidx.preference.PreferenceScreen
+import eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors.CDAExtractor
+import eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors.GoogleDriveExtractor
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
+import eu.kanade.tachiyomi.lib.sibnetextractor.SibnetExtractor
+import eu.kanade.tachiyomi.multisrc.animestream.AnimeStream
+import okhttp3.Response
+import java.text.SimpleDateFormat
+import java.util.Locale
+
+class DesuOnline : AnimeStream(
+    "pl",
+    "desu-online",
+    "https://desu-online.pl",
+) {
+    override val dateFormatter by lazy {
+        SimpleDateFormat("d MMMM, yyyy", Locale("pl", "PL"))
+    }
+
+    private val prefServerKey = "preferred_server"
+    private val prefServerDefault = "CDA"
+
+    // ============================ Video Links =============================
+
+    override fun videoListParse(response: Response): List<Video> =
+        super.videoListParse(response).ifEmpty { throw Exception("Failed to fetch videos") }
+
+    private val okruExtractor by lazy { OkruExtractor(client) }
+    private val cdaExtractor by lazy { CDAExtractor(client, headers, "$baseUrl/") }
+    private val sibnetExtractor by lazy { SibnetExtractor(client) }
+    private val gdriveExtractor by lazy { GoogleDriveExtractor(client, headers) }
+
+    override fun getVideoList(url: String, name: String): List<Video> {
+        return when {
+            url.contains("ok.ru") -> okruExtractor.videosFromUrl(url, name)
+            url.contains("cda.pl") -> cdaExtractor.videosFromUrl(url, name)
+            url.contains("sibnet") -> sibnetExtractor.videosFromUrl(url, prefix = "$name - ")
+            url.contains("drive.google.com") -> {
+                val id = Regex("[\\w-]{28,}").find(url)?.groupValues?.get(0) ?: return emptyList()
+                gdriveExtractor.videosFromUrl("https://drive.google.com/uc?id=$id", videoName = name)
+            }
+            else -> emptyList()
+        }
+    }
+
+    // ============================= Utilities ==============================
+
+    override fun List<Video>.sort(): List<Video> {
+        val quality = preferences.getString(videoSortPrefKey, videoSortPrefDefault)!!
+        val server = preferences.getString(prefServerKey, prefServerDefault)!!
+
+        return sortedWith(
+            compareBy(
+                { it.quality.contains(server, true) },
+                { it.quality.contains(quality) },
+                { Regex("""(\d+)p""").find(it.quality)?.groupValues?.get(1)?.toIntOrNull() ?: 0 },
+            ),
+        ).reversed()
+    }
+
+    // ============================== Settings ==============================
+
+    override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        super.setupPreferenceScreen(screen) // Quality preferences
+
+        ListPreference(screen.context).apply {
+            key = prefServerKey
+            title = "Preferred server"
+            entries = arrayOf("CDA", "Sibnet", "Google Drive", "ok.ru")
+            entryValues = arrayOf("CDA", "sibnet", "gd", "okru")
+            setDefaultValue(prefServerDefault)
+            summary = "%s"
+
+            setOnPreferenceChangeListener { _, newValue ->
+                val selected = newValue as String
+                val index = findIndexOfValue(selected)
+                val entry = entryValues[index] as String
+                preferences.edit().putString(key, entry).commit()
+            }
+        }.also(screen::addPreference)
+    }
+}

multisrc/overrides/animestream/desuonline/src/extractors/CDAExtractor.kt

@@ -0,0 +1,110 @@
+package eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.encodeToString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.add
+import kotlinx.serialization.json.buildJsonObject
+import kotlinx.serialization.json.put
+import kotlinx.serialization.json.putJsonArray
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.RequestBody.Companion.toRequestBody
+import okhttp3.Response
+import uy.kohesive.injekt.injectLazy
+
+class CDAExtractor(private val client: OkHttpClient, private val headers: Headers, private val referer: String) {
+
+    private val json: Json by injectLazy()
+
+    fun videosFromUrl(url: String, name: String): List<Video> {
+        val urlHost = url.toHttpUrl().host
+
+        val docHeaders = headers.newBuilder().apply {
+            add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+            add("Host", urlHost)
+            add("Referer", referer)
+        }.build()
+        val doc = client.newCall(
+            GET(url, headers = docHeaders),
+        ).execute().asJsoup()
+
+        val playerData = doc.selectFirst("div[id~=mediaplayer][player_data]")
+            ?.attr("player_data")
+            ?.let { json.decodeFromString<PlayerData>(it) }
+            ?: return emptyList()
+
+        val timestamp = playerData.api.ts.substringBefore("_")
+        val videoData = playerData.video
+
+        var idCounter = 1
+        return videoData.qualities.map { (quality, qualityId) ->
+            val postBody = json.encodeToString(
+                buildJsonObject {
+                    put("id", idCounter)
+                    put("jsonrpc", "2.0")
+                    put("method", "videoGetLink")
+                    putJsonArray("params") {
+                        add(url.toHttpUrl().pathSegments.last())
+                        add(qualityId)
+                        add(timestamp.toInt())
+                        add(videoData.hash2)
+                    }
+                },
+            ).toRequestBody("application/json; charset=utf-8".toMediaType())
+
+            val postHeaders = headers.newBuilder().apply {
+                add("Accept", "application/json, text/javascript, */*; q=0.01")
+                add("Host", "www.cda.pl")
+                add("Origin", "https://$urlHost")
+                add("Referer", url)
+            }.build()
+
+            val videoUrl = client.newCall(
+                POST("https://www.cda.pl/", headers = postHeaders, body = postBody),
+            ).execute().parseAs<PostResponse>().result.resp
+
+            idCounter++
+
+            Video(videoUrl, "$name - $quality", videoUrl)
+        }
+    }
+
+    @Serializable
+    data class PlayerData(
+        val api: PlayerApi,
+        val video: PlayerVideoData,
+    ) {
+        @Serializable
+        data class PlayerApi(
+            val ts: String,
+        )
+
+        @Serializable
+        data class PlayerVideoData(
+            val hash2: String,
+            val qualities: Map<String, String>,
+        )
+    }
+
+    @Serializable
+    data class PostResponse(
+        val result: PostResult,
+    ) {
+        @Serializable
+        data class PostResult(
+            val resp: String,
+        )
+    }
+
+    private inline fun <reified T> Response.parseAs(transform: (String) -> String = { it }): T {
+        val responseBody = use { transform(it.body.string()) }
+        return json.decodeFromString(responseBody)
+    }
+}

multisrc/overrides/animestream/desuonline/src/extractors/GoogleDriveExtractor.kt

@@ -0,0 +1,98 @@
+package eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.network.POST
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
+import okhttp3.HttpUrl.Companion.toHttpUrl
+import okhttp3.MediaType.Companion.toMediaType
+import okhttp3.OkHttpClient
+import okhttp3.RequestBody.Companion.toRequestBody
+
+class GoogleDriveExtractor(private val client: OkHttpClient, private val headers: Headers) {
+    // Needs to be the form of `https://drive.google.com/uc?id=GOOGLEDRIVEITEMID`
+    fun videosFromUrl(itemUrl: String, videoName: String = "Video"): List<Video> {
+        val itemHeaders = headers.newBuilder()
+            .add("Accept", "*/*")
+            .add("Accept-Language", "en-US,en;q=0.5")
+            .add("Connection", "keep-alive")
+            .add("Cookie", getCookie(itemUrl))
+            .add("Host", "drive.google.com")
+            .build()
+
+        val itemResponse = client.newCall(
+            GET(itemUrl, headers = itemHeaders),
+        ).execute()
+
+        val noRedirectClient = OkHttpClient().newBuilder().followRedirects(false).build()
+        val document = itemResponse.asJsoup()
+
+        val itemSize = document.selectFirst("span.uc-name-size")?.let {
+            " ${it.ownText().trim()} "
+        } ?: ""
+        val url = document.selectFirst("form#download-form")?.attr("action") ?: return emptyList()
+        val redirectHeaders = headers.newBuilder()
+            .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+            .add("Connection", "keep-alive")
+            .add("Content-Length", "0")
+            .add("Content-Type", "application/x-www-form-urlencoded")
+            .add("Cookie", getCookie(url))
+            .add("Host", "drive.google.com")
+            .add("Origin", "https://drive.google.com")
+            .add("Referer", url.substringBeforeLast("&at="))
+            .build()
+
+        val response = noRedirectClient.newCall(
+            POST(url, headers = redirectHeaders, body = "".toRequestBody("application/x-www-form-urlencoded".toMediaType())),
+        ).execute()
+        val redirected = response.headers["location"] ?: return listOf(Video(url, videoName + itemSize, url))
+
+        val redirectedHeaders = headers.newBuilder()
+            .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+            .add("Connection", "keep-alive")
+            .add("Host", redirected.toHttpUrl().host)
+            .add("Referer", "https://drive.google.com/")
+            .build()
+
+        val redirectedResponseHeaders = noRedirectClient.newCall(
+            GET(redirected, headers = redirectedHeaders),
+        ).execute().headers
+        val authCookie = redirectedResponseHeaders.firstOrNull {
+            it.first == "set-cookie" && it.second.startsWith("AUTH_")
+        }?.second?.substringBefore(";") ?: return listOf(Video(url, videoName + itemSize, url))
+        val newRedirected = redirectedResponseHeaders["location"] ?: return listOf(Video(redirected, videoName + itemSize, redirected))
+
+        val googleDriveRedirectHeaders = headers.newBuilder()
+            .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+            .add("Connection", "keep-alive")
+            .add("Cookie", getCookie(newRedirected))
+            .add("Host", "drive.google.com")
+            .add("Referer", "https://drive.google.com/")
+            .build()
+        val googleDriveRedirectUrl = noRedirectClient.newCall(
+            GET(newRedirected, headers = googleDriveRedirectHeaders),
+        ).execute().headers["location"]!!
+
+        val videoHeaders = headers.newBuilder()
+            .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
+            .add("Connection", "keep-alive")
+            .add("Cookie", authCookie)
+            .add("Host", googleDriveRedirectUrl.toHttpUrl().host)
+            .add("Referer", "https://drive.google.com/")
+            .build()
+
+        return listOf(
+            Video(googleDriveRedirectUrl, videoName + itemSize, googleDriveRedirectUrl, headers = videoHeaders),
+        )
+    }
+
+    private fun getCookie(url: String): String {
+        val cookieList = client.cookieJar.loadForRequest(url.toHttpUrl())
+        return if (cookieList.isNotEmpty()) {
+            cookieList.joinToString("; ") { "${it.name}=${it.value}" }
+        } else {
+            ""
+        }
+    }
+}

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/animestream/AnimeStreamGenerator.kt

@@ -16,6 +16,7 @@ class AnimeStreamGenerator : ThemeSourceGenerator {
         SingleLang("Animenosub", "https://animenosub.com", "en", isNsfw = true),
         SingleLang("AnimeTitans", "https://animetitans.com", "ar", isNsfw = false, overrideVersionCode = 11),
         SingleLang("AnimeXin", "https://animexin.vip", "all", isNsfw = false, overrideVersionCode = 4),
+        SingleLang("desu-online", "https://desu-online.pl", "pl", className = "DesuOnline", isNsfw = false),
         SingleLang("Hstream", "https://hstream.moe", "en", isNsfw = true, overrideVersionCode = 3),
         SingleLang("LMAnime", "https://lmanime.com", "all", isNsfw = false, overrideVersionCode = 2),
         SingleLang("MiniOppai", "https://minioppai.org", "id", isNsfw = true, overrideVersionCode = 2),

src/pl/desuonline/AndroidManifest.xml

@@ -1,2 +0,0 @@
-<?xml version="1.0" encoding="utf-8"?>
-<manifest />

src/pl/desuonline/build.gradle

@@ -1,12 +0,0 @@
-apply plugin: 'com.android.application'
-apply plugin: 'kotlin-android'
-
-ext {
-    extName = 'desu-online'
-    pkgNameSuffix = 'pl.desuonline'
-    extClass = '.DesuOnline'
-    extVersionCode = 1
-    libVersion = '13'
-}
-
-apply from: "$rootDir/common.gradle"

src/pl/desuonline/src/eu/kanade/tachiyomi/animeextension/pl/desuonline/DesuOnline.kt

@@ -1,238 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.pl.desuonline
-
-import android.app.Application
-import android.content.SharedPreferences
-import android.util.Base64
-import androidx.preference.ListPreference
-import androidx.preference.PreferenceScreen
-import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
-import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
-import eu.kanade.tachiyomi.animesource.model.SAnime
-import eu.kanade.tachiyomi.animesource.model.SEpisode
-import eu.kanade.tachiyomi.animesource.model.Video
-import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
-import eu.kanade.tachiyomi.network.GET
-import eu.kanade.tachiyomi.network.POST
-import eu.kanade.tachiyomi.util.asJsoup
-import kotlinx.serialization.decodeFromString
-import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
-import okhttp3.HttpUrl.Companion.toHttpUrl
-import okhttp3.MediaType.Companion.toMediaType
-import okhttp3.RequestBody
-import okhttp3.RequestBody.Companion.toRequestBody
-import okhttp3.Response
-import org.jsoup.nodes.Document
-import org.jsoup.nodes.Element
-import uy.kohesive.injekt.Injekt
-import uy.kohesive.injekt.api.get
-import java.text.SimpleDateFormat
-import java.util.Locale
-
-class DesuOnline : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
-
-    override val name = "desu-online"
-
-    override val baseUrl = "https://desu-online.pl"
-
-    override val lang = "pl"
-
-    override val supportsLatest = true
-
-    override val client = network.cloudflareClient
-
-    private val json = Json {
-        ignoreUnknownKeys = true
-    }
-
-    private val preferences: SharedPreferences by lazy {
-        Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
-    }
-
-    // ============================== Popular ===============================
-
-    override fun popularAnimeRequest(page: Int) = GET("$baseUrl/anime/?page=$page&order=popular")
-
-    override fun popularAnimeSelector() = "div.listupd div.bsx > a"
-
-    override fun popularAnimeNextPageSelector() = "div.pagination > a.next, div.hpage > a.r"
-
-    override fun popularAnimeFromElement(element: Element): SAnime {
-        val animeTitle = element.select("div.tt > h2").text().trim()
-        val img = element.select("div.limit > img")
-        return SAnime.create().apply {
-            setUrlWithoutDomain(element.attr("href"))
-            title = animeTitle
-            thumbnail_url = img.attr("data-src")
-        }
-    }
-
-    // =========================== Anime Details ============================
-
-    override fun animeDetailsParse(document: Document): SAnime {
-        val img = document.select("div.thumb > img").ifEmpty { null }
-        val studio = document.select("div.info-content > div.spe > span:contains(Studio:)").ifEmpty { null }
-        val statusSpan = document.select("div.info-content > div.spe > span:contains(Status:)").ifEmpty { null }
-        val desc = document.select("div[itemprop=description] > p:last-child").ifEmpty { null }
-        val director = document.select("div.info-content > div.spe > span:contains(Reżyser:)").ifEmpty { null }
-        val genres = document.select("div.genxed > a")
-        return SAnime.create().apply {
-            title = document.select("h1.entry-title").text()
-            thumbnail_url = img?.attr("data-src")
-            author = studio?.text()?.substringAfter("Studio: ")
-            status = parseStatus(statusSpan?.text()?.substringAfter("Status: "))
-            description = desc?.text()?.trim()
-            artist = director?.text()?.substringAfter("Reżyser: ")
-            genre = genres.joinToString { it.text() }
-        }
-    }
-
-    // ============================== Episodes ==============================
-
-    override fun episodeListSelector() = "div.eplister > ul > li"
-
-    override fun episodeFromElement(element: Element): SEpisode {
-        val a = element.select("a")
-        val epNum = a.select("div.epl-num").text()
-        val epTitle = a.select("div.epl-title").text()
-        val date = a.select("div.epl-date").text()
-        return SEpisode.create().apply {
-            setUrlWithoutDomain(a.attr("href"))
-            name = "Odcinek $epNum: $epTitle"
-            date_upload = parseDate(date)
-            episode_number = epNum.substringBefore(" ").toFloatOrNull() ?: 0F
-        }
-    }
-
-    // ============================ Video Links =============================
-
-    override fun videoListParse(response: Response): List<Video> {
-        val videoList = mutableListOf<Video>()
-        val document = response.asJsoup()
-        document.select("select.mirror > option").filter { it ->
-            it.text().contains("CDA")
-        }.map {
-            val mirror = it.text().trim()
-
-            val iframe = String(Base64.decode(it.attr("value"), Base64.DEFAULT))
-            val src = iframe.substringAfter("src=\"").substringBefore("\"")
-
-            val videoId = src.toHttpUrl().encodedPathSegments.last()
-
-            val playerDataHtml = client.newCall(GET(src)).execute()
-                .asJsoup().select("div[id=mediaplayer$videoId]")
-                .attr("player_data")
-            val playerData = json.decodeFromString<JsonObject>(playerDataHtml)
-
-            val timeStamp = playerData["api"]!!.jsonObject["ts"]!!.jsonPrimitive.content
-                .substringBefore("_")
-            val videoData = playerData["video"]!!.jsonObject
-            val hash = videoData["hash2"]!!.jsonPrimitive.content
-
-            val qualities = videoData["qualities"]!!.jsonObject
-
-            qualities.keys.reversed().map { quality ->
-                val qualityId = qualities[quality]!!.jsonPrimitive.content
-                val body = cdaBody(videoId, qualityId, timeStamp, hash)
-                val videoResponse = json.decodeFromString<JsonObject>(
-                    client.newCall(POST("https://www.cda.pl/", body = body))
-                        .execute().body.string(),
-                )
-                val videoUrl = videoResponse["result"]!!.jsonObject["resp"]!!.jsonPrimitive.content
-                videoList.add(Video(videoUrl, "$mirror: $quality", videoUrl))
-            }
-        }
-        return videoList
-    }
-
-    override fun videoFromElement(element: Element): Video = throw Exception("not used")
-    override fun videoListSelector(): String = throw Exception("not used")
-    override fun videoUrlParse(document: Document): String = throw Exception("not used")
-
-    // =============================== Search ===============================
-
-    override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList) =
-        GET("$baseUrl/anime/?page=$page&s=$query")
-
-    override fun searchAnimeSelector() = popularAnimeSelector()
-    override fun searchAnimeNextPageSelector() = popularAnimeNextPageSelector()
-    override fun searchAnimeFromElement(element: Element) = popularAnimeFromElement(element)
-
-    // =============================== Latest ===============================
-
-    override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/anime/?page=$page&order=latest")
-
-    override fun latestUpdatesSelector() = popularAnimeSelector()
-    override fun latestUpdatesNextPageSelector() = popularAnimeNextPageSelector()
-    override fun latestUpdatesFromElement(element: Element) = popularAnimeFromElement(element)
-
-    // =============================== Preferences ===============================
-
-    override fun setupPreferenceScreen(screen: PreferenceScreen) {
-        val videoQualityPref = ListPreference(screen.context).apply {
-            key = "preferred_quality"
-            title = "Preferred quality"
-            entries = arrayOf("1080p", "720p", "480p", "360p")
-            entryValues = arrayOf("1080", "720", "480", "360")
-            setDefaultValue("1080")
-            summary = "%s"
-
-            setOnPreferenceChangeListener { _, newValue ->
-                val selected = newValue as String
-                val index = findIndexOfValue(selected)
-                val entry = entryValues[index] as String
-                preferences.edit().putString(key, entry).commit()
-            }
-        }
-        screen.addPreference(videoQualityPref)
-    }
-
-    override fun List<Video>.sort(): List<Video> {
-        val quality = preferences.getString("preferred_quality", "1080")
-        if (quality != null) {
-            val newList = mutableListOf<Video>()
-            var preferred = 0
-            for (video in this) {
-                if (video.quality.contains(quality)) {
-                    newList.add(preferred, video)
-                    preferred++
-                } else {
-                    newList.add(video)
-                }
-            }
-            return newList
-        }
-        return this
-    }
-
-    // ============================= Utilities ==============================
-
-    private fun parseStatus(statusString: String?): Int {
-        return when (statusString?.trim()) {
-            "Zakończony" -> SAnime.COMPLETED
-            "Emitowany" -> SAnime.ONGOING
-            else -> SAnime.UNKNOWN
-        }
-    }
-
-    private fun parseDate(dateStr: String): Long {
-        return runCatching { DATE_FORMATTER.parse(dateStr)?.time }
-            .getOrNull() ?: 0L
-    }
-
-    private fun cdaBody(
-        videoId: String,
-        qualityId: String,
-        timeStamp: String,
-        hash: String,
-    ): RequestBody {
-        return "{\"jsonrpc\":\"2.0\",\"method\":\"videoGetLink\",\"params\":[\"$videoId\",\"$qualityId\",$timeStamp,\"$hash\",{}],\"id\":4}"
-            .toRequestBody("application/json".toMediaType())
-    }
-}
-
-private val DATE_FORMATTER by lazy {
-    SimpleDateFormat("d MMM, yyyy", Locale("pl"))
-}