Yomiroll: give option to unlock local region locked subs (#1350)

for old anime like one piece

src/all/kamyroll/src/eu/kanade/tachiyomi/animeextension/all/kamyroll/AccessTokenInterceptor.kt

@@ -18,6 +18,8 @@ import java.net.HttpURLConnection
 import java.net.InetSocketAddress
 import java.net.PasswordAuthentication
 import java.net.Proxy
+import java.text.SimpleDateFormat
+import java.util.Locale
 
 class AccessTokenInterceptor(
     private val crUrl: String,
@@ -39,7 +41,7 @@ class AccessTokenInterceptor(
                     if (accessTokenN != newAccessToken) {
                         return chain.proceed(newRequestWithAccessToken(request, newAccessToken))
                     }
-                    val refreshedToken = refreshAccessToken()
+                    val refreshedToken = refreshAccessToken(TOKEN_PREF_KEY)
                     // Retry the request
                     return chain.proceed(
                         newRequestWithAccessToken(chain.request(), refreshedToken),
@@ -58,21 +60,28 @@ class AccessTokenInterceptor(
 
     fun getAccessToken(): AccessToken {
         return preferences.getString(TOKEN_PREF_KEY, null)?.toAccessToken()
-            ?: refreshAccessToken()
+            ?: refreshAccessToken(TOKEN_PREF_KEY)
     }
 
-    private fun refreshAccessToken(): AccessToken {
+    fun getLocalToken(force: Boolean = false): AccessToken? {
-        val client = OkHttpClient()
+        if (!preferences.getBoolean(PREF_FETCH_LOCAL_SUBS, false) && !force) return null
-            .newBuilder().build()
+        synchronized(this) {
-        val proxy = client.newBuilder()
+            val now = System.currentTimeMillis() + 1800000 // add 30 minutes for safety
-            .proxy(
+            val localToken = preferences.getString(LOCAL_TOKEN_PREF_KEY, null)?.toAccessToken()
-                Proxy(
+            return if (force || localToken == null || localToken.policyExpire!! < now) {
-                    Proxy.Type.SOCKS,
+                refreshAccessToken(LOCAL_TOKEN_PREF_KEY, false)
-                    InetSocketAddress("cr-unblocker.us.to", 1080),
+            } else {
-                ),
+                localToken
-            )
+            }
-            .build()
+        }
+    }
 
+    fun removeLocalToken() {
+        preferences.edit().putString(LOCAL_TOKEN_PREF_KEY, null).apply()
+    }
+
+    private fun refreshAccessToken(PREF_KEY: String, useProxy: Boolean = true): AccessToken {
+        val client = OkHttpClient().newBuilder().build()
         Authenticator.setDefault(
             object : Authenticator() {
                 override fun getPasswordAuthentication(): PasswordAuthentication {
@@ -80,21 +89,22 @@ class AccessTokenInterceptor(
                 }
             },
         )
-
+        val usedClient = if (useProxy) {
-        // Thanks Stormzy
+            client.newBuilder()
-        val refreshTokenResp = client.newCall(GET("https://raw.githubusercontent.com/Samfun75/File-host/main/aniyomi/refreshToken.txt")).execute()
+                .proxy(
-        val refreshToken = refreshTokenResp.body.string().replace("[\n\r]".toRegex(), "")
+                    Proxy(
-        val headers = Headers.headersOf(
+                        Proxy.Type.SOCKS,
-            "Content-Type",
+                        InetSocketAddress("cr-unblocker.us.to", 1080),
-            "application/x-www-form-urlencoded",
+                    ),
-            "Authorization",
+                )
-            "Basic a3ZvcGlzdXZ6Yy0teG96Y21kMXk6R21JSTExenVPVnRnTjdlSWZrSlpibzVuLTRHTlZ0cU8=",
+                .build()
-        )
+        } else {
-        val postBody = "grant_type=refresh_token&refresh_token=$refreshToken&scope=offline_access".toRequestBody("application/x-www-form-urlencoded".toMediaType())
+            client
-        val response = proxy.newCall(POST("$crUrl/auth/v1/token", headers, postBody)).execute()
+        }
+        val response = usedClient.newCall(getRequest(client)).execute()
         val parsedJson = json.decodeFromString<AccessToken>(response.body.string())
 
-        val policy = proxy.newCall(newRequestWithAccessToken(GET("$crUrl/index/v2"), parsedJson)).execute()
+        val policy = usedClient.newCall(newRequestWithAccessToken(GET("$crUrl/index/v2"), parsedJson)).execute()
         val policyJson = json.decodeFromString<Policy>(policy.body.string())
         val allTokens = AccessToken(
             parsedJson.access_token,
@@ -103,11 +113,27 @@ class AccessTokenInterceptor(
             policyJson.cms.signature,
             policyJson.cms.key_pair_id,
             policyJson.cms.bucket,
+            DateFormatter.parse(policyJson.cms.expires)?.time,
         )
-        preferences.edit().putString(TOKEN_PREF_KEY, allTokens.toJsonString()).apply()
+        preferences.edit().putString(PREF_KEY, allTokens.toJsonString()).apply()
         return allTokens
     }
 
+    private fun getRequest(client: OkHttpClient): Request {
+        val refreshTokenResp = client.newCall(
+            GET("https://raw.githubusercontent.com/Samfun75/File-host/main/aniyomi/refreshToken.txt"),
+        ).execute()
+        val refreshToken = refreshTokenResp.body.string().replace("[\n\r]".toRegex(), "")
+        val headers = Headers.headersOf(
+            "Content-Type", "application/x-www-form-urlencoded",
+            "Authorization", "Basic a3ZvcGlzdXZ6Yy0teG96Y21kMXk6R21JSTExenVPVnRnTjdlSWZrSlpibzVuLTRHTlZ0cU8="
+        )
+        val postBody = "grant_type=refresh_token&refresh_token=$refreshToken&scope=offline_access".toRequestBody(
+            "application/x-www-form-urlencoded".toMediaType(),
+        )
+        return POST("$crUrl/auth/v1/token", headers, postBody)
+    }
+
     private fun AccessToken.toJsonString(): String {
         return json.encodeToString(this)
     }
@@ -117,6 +143,12 @@ class AccessTokenInterceptor(
     }
 
     companion object {
-        val TOKEN_PREF_KEY = "access_token_data"
+        private const val TOKEN_PREF_KEY = "access_token_data"
+        private const val LOCAL_TOKEN_PREF_KEY = "local_access_token_data_test_adwa"
+        private const val PREF_FETCH_LOCAL_SUBS = "preferred_local_subs"
+
+        private val DateFormatter by lazy {
+            SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH)
+        }
     }
 }

src/all/kamyroll/src/eu/kanade/tachiyomi/animeextension/all/kamyroll/DataModel.kt

@@ -12,6 +12,7 @@ data class AccessToken(
     val signature: String? = null,
     val key_pair_id: String? = null,
     val bucket: String? = null,
+    val policyExpire: Long? = null,
 )
 
 @Serializable
@@ -24,6 +25,7 @@ data class Policy(
         val signature: String,
         val key_pair_id: String,
         val bucket: String,
+        val expires: String,
     )
 }
 
@@ -156,9 +158,9 @@ data class EpisodeData(
 
 @Serializable
 data class VideoStreams(
-    val streams: Stream,
+    val streams: Stream? = null,
-    val subtitles: JsonObject,
+    val subtitles: JsonObject? = null,
-    val audio_locale: String,
+    val audio_locale: String? = null,
 ) {
     @Serializable
     data class Stream(

src/all/kamyroll/src/eu/kanade/tachiyomi/animeextension/all/kamyroll/Yomiroll.kt

@@ -1,9 +1,11 @@
 package eu.kanade.tachiyomi.animeextension.all.kamyroll
 
 import android.app.Application
+import android.content.Context
 import android.content.SharedPreferences
 import androidx.preference.ListPreference
 import androidx.preference.PreferenceScreen
+import androidx.preference.SwitchPreferenceCompat
 import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
 import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
 import eu.kanade.tachiyomi.animesource.model.AnimesPage
@@ -64,6 +66,13 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
         private val DateFormatter by lazy {
             SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss", Locale.ENGLISH)
         }
+
+        private const val PREF_QLT = "preferred_quality"
+        private const val PREF_AUD = "preferred_audio"
+        private const val PREF_SUB = "preferred_sub"
+        private const val PREF_SUB_TYPE = "preferred_sub_type"
+        // there is one in AccessTokenInterceptor too for below
+        private const val PREF_FETCH_LOCAL_SUBS = "preferred_local_subs"
     }
 
     // ============================== Popular ===============================
@@ -243,12 +252,13 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
     override fun fetchVideoList(episode: SEpisode): Observable<List<Video>> {
         val urlJson = json.decodeFromString<EpisodeData>(episode.url)
         val dubLocale = preferences.getString("preferred_audio", "en-US")!!
-        val policyJson = tokenInterceptor.getAccessToken()
+        val proxyToken = tokenInterceptor.getAccessToken()
+        val localToken = tokenInterceptor.getLocalToken()
         val videoList = urlJson.ids.filter {
             it.second == dubLocale || it.second == "ja-JP" || it.second == "en-US" || it.second == ""
         }.parallelMap { media ->
             runCatching {
-                extractVideo(media, policyJson)
+                extractVideo(media, proxyToken, localToken)
             }.getOrNull()
         }.filterNotNull().flatten()
 
@@ -257,18 +267,31 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
 
     // ============================= Utilities ==============================
 
-    private fun extractVideo(media: Pair<String, String>, policyJson: AccessToken): List<Video> {
+    private fun extractVideo(media: Pair<String, String>, proxyToken: AccessToken, localToken: AccessToken?): List<Video> {
         val (mediaId, aud) = media
-        val response = client.newCall(GET("$crUrl/cms/v2${policyJson.bucket}/videos/$mediaId/streams?Policy=${policyJson.policy}&Signature=${policyJson.signature}&Key-Pair-Id=${policyJson.key_pair_id}")).execute()
+        val response = client.newCall(getVideoRequest(mediaId, proxyToken)).execute()
         val streams = json.decodeFromString<VideoStreams>(response.body.string())
 
+        val localStreams = if (aud == "ja-JP" && preferences.getBoolean(PREF_FETCH_LOCAL_SUBS, false)) {
+            val localResponse = client.newCall(getVideoRequest(mediaId, localToken!!)).execute()
+            json.decodeFromString(localResponse.body.string())
+        } else {
+            VideoStreams()
+        }
+
         var subsList = emptyList<Track>()
         val subLocale = preferences.getString("preferred_sub", "en-US")!!.getLocale()
         try {
-            subsList = streams.subtitles.entries.map { (_, value) ->
+            val tempSubs = mutableListOf<Track>()
+            streams.subtitles?.entries?.map { (_, value) ->
                 val sub = json.decodeFromString<Subtitle>(value.jsonObject.toString())
-                Track(sub.url, sub.locale.getLocale())
+                tempSubs.add(Track(sub.url, sub.locale.getLocale()))
-            }.sortedWith(
+            }
+            localStreams.subtitles?.entries?.map { (_, value) ->
+                val sub = json.decodeFromString<Subtitle>(value.jsonObject.toString())
+                tempSubs.add(Track(sub.url, sub.locale.getLocale()))
+            }
+            subsList = tempSubs.sortedWith(
                 compareBy(
                     { it.lang },
                     { it.lang.contains(subLocale) },
@@ -276,12 +299,25 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
             )
         } catch (_: Error) {}
 
-        val audLang = aud.ifBlank { streams.audio_locale }
+        val audLang = aud.ifBlank { streams.audio_locale } ?: localStreams.audio_locale ?: "ja-JP"
-        return streams.streams.adaptive_hls.entries.parallelMap { (_, value) ->
+        val videoList = mutableListOf<Video>()
+        videoList.addAll(
+            getStreams(streams, audLang, subsList),
+        )
+        videoList.addAll(
+            getStreams(localStreams, audLang, subsList),
+        )
+
+        return videoList.distinctBy { it.quality }
+    }
+
+    private fun getStreams(streams: VideoStreams, audLang: String, subsList: List<Track>): List<Video> {
+        return streams.streams?.adaptive_hls?.entries?.parallelMap { (_, value) ->
             val stream = json.decodeFromString<HlsLinks>(value.jsonObject.toString())
             runCatching {
-                val playlist = client.newCall(GET(stream.url)).execute().body.string()
+                val playlist = client.newCall(GET(stream.url)).execute()
-                playlist.substringAfter("#EXT-X-STREAM-INF:")
+                if (playlist.code != 200) return@parallelMap null
+                playlist.body.string().substringAfter("#EXT-X-STREAM-INF:")
                     .split("#EXT-X-STREAM-INF:").map {
                         val hardsub = stream.hardsub_locale.let { hs ->
                             if (hs.isNotBlank()) " - HardSub: $hs" else ""
@@ -304,9 +340,11 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
                         }
                     }
             }.getOrNull()
-        }
+        }?.filterNotNull()?.flatten() ?: emptyList()
-            .filterNotNull()
+    }
-            .flatten()
+
+    private fun getVideoRequest(mediaId: String, token: AccessToken): Request {
+        return GET("$crUrl/cms/v2${token.bucket}/videos/$mediaId/streams?Policy=${token.policy}&Signature=${token.signature}&Key-Pair-Id=${token.key_pair_id}")
     }
 
     private val df = DecimalFormat("0.#")
@@ -428,7 +466,7 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
 
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
         val videoQualityPref = ListPreference(screen.context).apply {
-            key = "preferred_quality"
+            key = PREF_QLT
             title = "Preferred quality"
             entries = arrayOf("1080p", "720p", "480p", "360p", "240p", "80p")
             entryValues = arrayOf("1080", "720", "480", "360", "240", "80")
@@ -444,7 +482,7 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
         }
 
         val audLocalePref = ListPreference(screen.context).apply {
-            key = "preferred_audio"
+            key = PREF_AUD
             title = "Preferred Audio Language"
             entries = locale.map { it.second }.toTypedArray()
             entryValues = locale.map { it.first }.toTypedArray()
@@ -460,7 +498,7 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
         }
 
         val subLocalePref = ListPreference(screen.context).apply {
-            key = "preferred_sub"
+            key = PREF_SUB
             title = "Preferred Sub Language"
             entries = locale.map { it.second }.toTypedArray()
             entryValues = locale.map { it.first }.toTypedArray()
@@ -476,7 +514,7 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
         }
 
         val subTypePref = ListPreference(screen.context).apply {
-            key = "preferred_sub_type"
+            key = PREF_SUB_TYPE
             title = "Preferred Sub Type"
             entries = arrayOf("Softsub", "Hardsub")
             entryValues = arrayOf("soft", "hard")
@@ -490,12 +528,65 @@ class Yomiroll : ConfigurableAnimeSource, AnimeHttpSource() {
                 preferences.edit().putString(key, entry).commit()
             }
         }
+
         screen.addPreference(videoQualityPref)
         screen.addPreference(audLocalePref)
         screen.addPreference(subLocalePref)
         screen.addPreference(subTypePref)
+        screen.addPreference(localSubsPreference(screen))
     }
 
+    // From Jellyfin
+    private abstract class LocalSubsPreference(context: Context) : SwitchPreferenceCompat(context) {
+        abstract fun reload()
+    }
+
+    private fun localSubsPreference(screen: PreferenceScreen) =
+        (
+            object : LocalSubsPreference(screen.context) {
+                override fun reload() {
+                    this.apply {
+                        key = PREF_FETCH_LOCAL_SUBS
+                        title = "Fetch Local Subs (Don't Spam this please!)"
+                        Thread {
+                            summary = try {
+                                val storedToken = tokenInterceptor.getLocalToken()
+                                """Token location: ${storedToken?.bucket?.substringAfter("/")?.substringBefore("/")}
+                                |Expires: ${storedToken?.policyExpire?.let { DateFormatter.format(it) } ?: "---"}
+                                """.trimMargin()
+                            } catch (e: Exception) {
+                                ""
+                            }
+                        }.start()
+                        setDefaultValue(false)
+
+                        setOnPreferenceChangeListener { _, newValue ->
+                            val new = newValue as Boolean
+                            if (new) {
+                                Thread {
+                                    summary = try {
+                                        val storedToken = tokenInterceptor.getLocalToken(true)!!
+                                        """Token location: ${storedToken.bucket?.substringAfter("/")?.substringBefore("/") ?: ""}
+                                            |Expires: ${storedToken.policyExpire?.let { DateFormatter.format(it) } ?: ""}
+                                        """.trimMargin()
+                                    } catch (e: Exception) {
+                                        ""
+                                    }
+                                }.start()
+                            } else {
+                                Thread {
+                                    tokenInterceptor.removeLocalToken()
+                                    summary = """Token location:
+                                        |Expires:""".trimMargin()
+                                }.start()
+                            }
+                            preferences.edit().putBoolean(key, new).commit()
+                        }
+                    }
+                }
+            }
+            ).apply { reload() }
+
     // From Dopebox
     private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
         runBlocking {