fix(all/animexin): Fix YoutubeExtractor (#2360)

Co-authored-by: folke <folke.steen85@gmail.com>

multisrc/overrides/animestream/animexin/additional.gradle

@@ -2,5 +2,6 @@ dependencies {
     implementation(project(':lib-dailymotion-extractor'))
     implementation(project(':lib-okru-extractor'))
     implementation(project(':lib-gdriveplayer-extractor'))
+    implementation(project(':lib-dood-extractor'))
     implementation "dev.datlag.jsunpacker:jsunpacker:1.0.1"
 }

multisrc/overrides/animestream/animexin/src/AnimeXin.kt

@@ -2,11 +2,11 @@ package eu.kanade.tachiyomi.animeextension.all.animexin
 
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
-import eu.kanade.tachiyomi.animeextension.all.animexin.extractors.DoodExtractor
 import eu.kanade.tachiyomi.animeextension.all.animexin.extractors.VidstreamingExtractor
 import eu.kanade.tachiyomi.animeextension.all.animexin.extractors.YouTubeExtractor
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.lib.dailymotionextractor.DailymotionExtractor
+import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
 import eu.kanade.tachiyomi.lib.gdriveplayerextractor.GdrivePlayerExtractor
 import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
 import eu.kanade.tachiyomi.multisrc.animestream.AnimeStream
@@ -19,33 +19,28 @@ class AnimeXin : AnimeStream(
     override val id = 4620219025406449669
 
     // ============================ Video Links =============================
+    private val dailymotionExtractor by lazy { DailymotionExtractor(client, headers) }
+    private val doodExtractor by lazy { DoodExtractor(client) }
+    private val gdrivePlayerExtractor by lazy { GdrivePlayerExtractor(client) }
+    private val okruExtractor by lazy { OkruExtractor(client) }
+    private val vidstreamingExtractor by lazy { VidstreamingExtractor(client) }
+    private val youTubeExtractor by lazy { YouTubeExtractor(client) }
+
     override fun getVideoList(url: String, name: String): List<Video> {
         val prefix = "$name - "
         return when {
-            url.contains("ok.ru") -> {
+            url.contains("ok.ru") -> okruExtractor.videosFromUrl(url, prefix)
-                OkruExtractor(client).videosFromUrl(url, prefix = prefix)
+            url.contains("dailymotion") -> dailymotionExtractor.videosFromUrl(url, prefix)
-            }
+            url.contains("https://dood") -> doodExtractor.videosFromUrl(url, name)
-
-            url.contains("dailymotion") -> {
-                DailymotionExtractor(client, headers).videosFromUrl(url, prefix)
-            }
-            url.contains("https://dood") -> {
-                DoodExtractor(client).videosFromUrl(url, quality = name)
-            }
             url.contains("gdriveplayer") -> {
                 val gdriveHeaders = headersBuilder()
                     .add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
-                    .add("Host", "gdriveplayer.to")
                     .add("Referer", "$baseUrl/")
                     .build()
-                GdrivePlayerExtractor(client).videosFromUrl(url, name = name, headers = gdriveHeaders)
+                gdrivePlayerExtractor.videosFromUrl(url, name, gdriveHeaders)
-            }
-            url.contains("youtube.com") -> {
-                YouTubeExtractor(client).videosFromUrl(url, prefix = prefix)
-            }
-            url.contains("vidstreaming") -> {
-                VidstreamingExtractor(client).videosFromUrl(url, prefix = prefix)
             }
+            url.contains("youtube.com") -> youTubeExtractor.videosFromUrl(url, prefix)
+            url.contains("vidstreaming") -> vidstreamingExtractor.videosFromUrl(url, prefix)
             else -> emptyList()
         }
     }
@@ -53,7 +48,8 @@ class AnimeXin : AnimeStream(
     // ============================== Settings ==============================
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
         super.setupPreferenceScreen(screen) // Quality preferences
-        val videoLangPref = ListPreference(screen.context).apply {
+
+        ListPreference(screen.context).apply {
             key = PREF_LANG_KEY
             title = PREF_LANG_TITLE
             entries = PREF_LANG_VALUES
@@ -67,9 +63,7 @@ class AnimeXin : AnimeStream(
                 val entry = entryValues[index] as String
                 preferences.edit().putString(key, entry).commit()
             }
-        }
+        }.also(screen::addPreference)
-
-        screen.addPreference(videoLangPref)
     }
 
     // ============================= Utilities ==============================

multisrc/overrides/animestream/animexin/src/extractors/DoodExtractor.kt

@@ -1,80 +0,0 @@
-package eu.kanade.tachiyomi.animeextension.all.animexin.extractors
-
-import eu.kanade.tachiyomi.animesource.model.Track
-import eu.kanade.tachiyomi.animesource.model.Video
-import eu.kanade.tachiyomi.network.GET
-import okhttp3.Headers
-import okhttp3.OkHttpClient
-
-class DoodExtractor(private val client: OkHttpClient) {
-
-    fun videoFromUrl(
-        url: String,
-        quality: String? = null,
-        redirect: Boolean = true,
-    ): Video? {
-        val newQuality = quality ?: "Doodstream" + if (redirect) " mirror" else ""
-
-        return try {
-            val response = client.newCall(GET(url)).execute()
-            val newUrl = if (redirect) response.request.url.toString() else url
-
-            val doodTld = newUrl.substringAfter("https://dood.").substringBefore("/")
-            val content = response.body.string()
-
-            val subtitleList = mutableListOf<Track>()
-            val subtitleRegex = """src:'//(srt[^']*?)',\s*label:'([^']*?)'""".toRegex()
-            try {
-                subtitleList.addAll(
-                    subtitleRegex.findAll(content).map {
-                        Track(
-                            "https://" + it.groupValues[1],
-                            it.groupValues[2],
-                        )
-                    },
-                )
-            } catch (a: Exception) { }
-
-            if (!content.contains("'/pass_md5/")) return null
-            val md5 = content.substringAfter("'/pass_md5/").substringBefore("',")
-            val token = md5.substringAfterLast("/")
-            val randomString = getRandomString()
-            val expiry = System.currentTimeMillis()
-            val videoUrlStart = client.newCall(
-                GET(
-                    "https://dood.$doodTld/pass_md5/$md5",
-                    Headers.headersOf("referer", newUrl),
-                ),
-            ).execute().body.string()
-            val videoUrl = "$videoUrlStart$randomString?token=$token&expiry=$expiry"
-            try {
-                Video(newUrl, newQuality, videoUrl, headers = doodHeaders(doodTld), subtitleTracks = subtitleList)
-            } catch (a: Exception) {
-                Video(newUrl, newQuality, videoUrl, headers = doodHeaders(doodTld))
-            }
-        } catch (e: Exception) {
-            null
-        }
-    }
-
-    fun videosFromUrl(
-        url: String,
-        quality: String? = null,
-        redirect: Boolean = true,
-    ): List<Video> {
-        val video = videoFromUrl(url, quality, redirect)
-        return video?.let { listOf(it) } ?: emptyList<Video>()
-    }
-
-    private fun getRandomString(length: Int = 10): String {
-        val allowedChars = ('A'..'Z') + ('a'..'z') + ('0'..'9')
-        return (1..length)
-            .map { allowedChars.random() }
-            .joinToString("")
-    }
-
-    private fun doodHeaders(tld: String) = Headers.Builder().apply {
-        add("User-Agent", "Aniyomi")
-        add("Referer", "https://dood.$tld/")
-    }.build()
-}

multisrc/overrides/animestream/animexin/src/extractors/YouTubeExtractor.kt

@@ -1,18 +1,16 @@
 package eu.kanade.tachiyomi.animeextension.all.animexin.extractors
 
 import android.annotation.SuppressLint
+import android.util.Log
 import eu.kanade.tachiyomi.animesource.model.Track
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.POST
 import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.SerialName
+import kotlinx.serialization.Serializable
 import kotlinx.serialization.decodeFromString
 import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.JsonObject
-import kotlinx.serialization.json.jsonArray
-import kotlinx.serialization.json.jsonObject
-import kotlinx.serialization.json.jsonPrimitive
-import kotlinx.serialization.json.long
 import okhttp3.Headers
 import okhttp3.MediaType.Companion.toMediaType
 import okhttp3.OkHttpClient
@@ -29,39 +27,41 @@ class YouTubeExtractor(private val client: OkHttpClient) {
         // TODO: Make code prettier
         // GET KEY
 
-        var ytcfgString = ""
         val videoId = url.substringAfter("/embed/")
 
-        val document = client.newCall(
+        val document = client.newCall(GET(url.replace("/embed/", "/watch?v=")))
-            GET(url.replace("/embed/", "/watch?v=")),
+            .execute()
-        ).execute().asJsoup()
+            .use { it.asJsoup() }
 
-        for (element in document.select("script")) {
+        val ytcfg = document.selectFirst("script:containsData(window.ytcfg=window.ytcfg)")
-            val scriptData = element.data()
+            ?.data() ?: run {
-            if (scriptData.startsWith("(function() {window.ytplayer={};")) {
+            Log.e("YouTubeExtractor", "Failed while trying to fetch the api key >:(")
-                ytcfgString = scriptData
+            return emptyList()
-            }
         }
 
-        val apiKey = getKey(ytcfgString, "innertubeApiKey")
+        val clientName = ytcfg.substringAfter("INNERTUBE_CONTEXT_CLIENT_NAME\":", "")
+            .substringBefore(",", "").ifEmpty { "5" }
 
-        val playerUrl = "https://www.youtube.com/youtubei/v1/player?key=$apiKey&prettyPrint=false"
+        val apiKey = ytcfg
+            .substringAfter("innertubeApiKey\":\"", "")
+            .substringBefore('"')
+
+        val playerUrl = "$YOUTUBE_URL/youtubei/v1/player?key=$apiKey&prettyPrint=false"
 
         val body = """
             {
                "context":{
                   "client":{
-                     "clientName":"ANDROID",
+                     "clientName":"IOS",
-                     "clientVersion":"17.31.35",
+                     "clientVersion":"17.33.2",
-                     "androidSdkVersion":30,
+                     "deviceModel": "iPhone14,3",
-                     "userAgent":"com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip",
+                     "userAgent": "com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)",
-                     "hl":"en",
+                     "hl": "en",
-                     "timeZone":"UTC",
+                     "timeZone": "UTC",
-                     "utcOffsetMinutes":0
+                     "utcOffsetMinutes": 0
                   }
                },
                "videoId":"$videoId",
-               "params":"8AEB",
                "playbackContext":{
                   "contentPlaybackContext":{
                      "html5Preference":"HTML5_PREF_WANTS"
@@ -72,97 +72,39 @@ class YouTubeExtractor(private val client: OkHttpClient) {
             }
         """.trimIndent().toRequestBody("application/json".toMediaType())
 
-        val headers = Headers.headersOf(
+        val headers = Headers.Builder().apply {
-            "X-YouTube-Client-Name", "3",
+            add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8")
-            "X-YouTube-Client-Version", "17.31.35",
+            add("Origin", YOUTUBE_URL)
-            "Origin", "https://www.youtube.com",
+            add("User-Agent", "com.google.ios.youtube/17.33.2 (iPhone14,3; U; CPU iOS 15_6 like Mac OS X)")
-            "User-Agent", "com.google.android.youtube/17.31.35 (Linux; U; Android 11) gzip",
+            add("X-Youtube-Client-Name", clientName)
-            "content-type", "application/json",
+            add("X-Youtube-Client-Version", "17.33.2")
-        )
+        }.build()
 
-        val postResponse = client.newCall(
+        val ytResponse = client.newCall(POST(playerUrl, headers, body)).execute()
-            POST(playerUrl, headers = headers, body = body),
+            .use { json.decodeFromString<YoutubeResponse>(it.body.string()) }
-        ).execute()
 
-        val responseObject = json.decodeFromString<JsonObject>(postResponse.body.string())
+        val formats = ytResponse.streamingData.adaptiveFormats
-        val videoList = mutableListOf<Video>()
-
-        val formats = responseObject["streamingData"]!!
-            .jsonObject["adaptiveFormats"]!!
-            .jsonArray
-
-        val audioTracks = mutableListOf<Track>()
-        val subtitleTracks = mutableListOf<Track>()
 
         // Get Audio
-        for (format in formats) {
+        val audioTracks = formats.filter { it.mimeType.startsWith("audio/webm") }
-            if (format.jsonObject["mimeType"]!!.jsonPrimitive.content.startsWith("audio/webm")) {
+            .map { Track(it.url, it.audioQuality!! + " (${formatBits(it.averageBitrate!!)}ps)") }
-                try {
-                    audioTracks.add(
-                        Track(
-                            format.jsonObject["url"]!!.jsonPrimitive.content,
-                            format.jsonObject["audioQuality"]!!.jsonPrimitive.content +
-                                " (${formatBits(format.jsonObject["averageBitrate"]!!.jsonPrimitive.long)}ps)",
-                        ),
-                    )
-                } catch (a: Exception) { }
-            }
-        }
 
         // Get Subtitles
-        if (responseObject.containsKey("captions")) {
+        val subs = ytResponse.captions?.renderer?.captionTracks?.map {
-            val captionTracks = responseObject["captions"]!!
+            Track(it.baseUrl, it.label)
-                .jsonObject["playerCaptionsTracklistRenderer"]!!
+        } ?: emptyList()
-                .jsonObject["captionTracks"]!!
-                .jsonArray
 
-            for (caption in captionTracks) {
+        // Get videos, finally
-                val captionJson = caption.jsonObject
+        return formats.filter { it.mimeType.startsWith("video/mp4") }.map {
-                try {
+            val codecs = it.mimeType.substringAfter("codecs=\"").substringBefore("\"")
-                    subtitleTracks.add(
+            Video(
-                        Track(
+                it.url,
-                            // TODO: Would replacing srv3 with vtt work for every video?
+                prefix + it.qualityLabel.orEmpty() + " ($codecs)",
-                            captionJson["baseUrl"]!!.jsonPrimitive.content.replace("srv3", "vtt"),
+                it.url,
-                            captionJson["name"]!!.jsonObject["runs"]!!.jsonArray[0].jsonObject["text"]!!.jsonPrimitive.content,
+                subtitleTracks = subs,
-                        ),
+                audioTracks = audioTracks,
-                    )
+            )
-                } catch (a: Exception) { }
-            }
         }
-
-        // List formats
-        for (format in formats) {
-            val mimeType = format.jsonObject["mimeType"]!!.jsonPrimitive.content
-            if (mimeType.startsWith("video/mp4")) {
-                videoList.add(
-                    try {
-                        Video(
-                            format.jsonObject["url"]!!.jsonPrimitive.content,
-                            prefix + format.jsonObject["qualityLabel"]!!.jsonPrimitive.content +
-                                " (${mimeType.substringAfter("codecs=\"").substringBefore("\"")})",
-                            format.jsonObject["url"]!!.jsonPrimitive.content,
-                            audioTracks = audioTracks,
-                            subtitleTracks = subtitleTracks,
-                        )
-                    } catch (a: Exception) {
-                        Video(
-                            format.jsonObject["url"]!!.jsonPrimitive.content,
-                            prefix + format.jsonObject["qualityLabel"]!!.jsonPrimitive.content +
-                                " (${mimeType.substringAfter("codecs=\"").substringBefore("\"")})",
-                            format.jsonObject["url"]!!.jsonPrimitive.content,
-                        )
-                    },
-
-                )
-            }
-        }
-
-        return videoList
-    }
-
-    fun getKey(string: String, key: String): String {
-        var pattern = Regex("\"$key\":\"(.*?)\"")
-        return pattern.find(string)?.groupValues?.get(1) ?: ""
     }
 
     @SuppressLint("DefaultLocale")
@@ -179,4 +121,44 @@ class YouTubeExtractor(private val client: OkHttpClient) {
         }
         return "%.0f%cb".format(bits / 1000.0, currentChar)
     }
+
+    @Serializable
+    data class YoutubeResponse(
+        val streamingData: AdaptiveDto,
+        val captions: CaptionsDto? = null,
+    )
+
+    @Serializable
+    data class AdaptiveDto(val adaptiveFormats: List<TrackDto>)
+
+    @Serializable
+    data class TrackDto(
+        val mimeType: String,
+        val url: String,
+        val averageBitrate: Long? = null,
+        val qualityLabel: String? = null,
+        val audioQuality: String? = null,
+    )
+
+    @Serializable
+    data class CaptionsDto(
+        @SerialName("playerCaptionsTracklistRenderer")
+        val renderer: CaptionsRendererDto,
+    ) {
+        @Serializable
+        data class CaptionsRendererDto(val captionTracks: List<CaptionItem>)
+
+        @Serializable
+        data class CaptionItem(val baseUrl: String, val name: NameDto) {
+            @Serializable
+            data class NameDto(val runs: List<GodDamnitYoutube>)
+
+            @Serializable
+            data class GodDamnitYoutube(val text: String)
+
+            val label by lazy { name.runs.first().text }
+        }
+    }
 }
+
+private const val YOUTUBE_URL = "https://www.youtube.com"

multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/animestream/AnimeStreamGenerator.kt

@@ -15,7 +15,7 @@ class AnimeStreamGenerator : ThemeSourceGenerator {
         SingleLang("AnimeKhor", "https://animekhor.xyz", "en", isNsfw = false, overrideVersionCode = 2),
         SingleLang("Animenosub", "https://animenosub.com", "en", isNsfw = true, overrideVersionCode = 3),
         SingleLang("AnimeTitans", "https://animetitans.com", "ar", isNsfw = false, overrideVersionCode = 13),
-        SingleLang("AnimeXin", "https://animexin.vip", "all", isNsfw = false, overrideVersionCode = 6),
+        SingleLang("AnimeXin", "https://animexin.vip", "all", isNsfw = false, overrideVersionCode = 7),
         SingleLang("AsyaAnimeleri", "https://asyaanimeleri.com", "tr", isNsfw = false, overrideVersionCode = 1),
         SingleLang("ChineseAnime", "https://chineseanime.top", "all", isNsfw = false, overrideVersionCode = 2),
         SingleLang("desu-online", "https://desu-online.pl", "pl", className = "DesuOnline", isNsfw = false, overrideVersionCode = 3),