Fix video extraction (#1239)

src/id/neonime/build.gradle

@@ -1,12 +1,19 @@
 apply plugin: 'com.android.application'
 apply plugin: 'kotlin-android'
+apply plugin: 'kotlinx-serialization'
 
 ext {
     extName = 'NeoNime'
     pkgNameSuffix = 'id.neonime'
     extClass = '.NeoNime'
-    extVersionCode = 8
+    extVersionCode = 9
     libVersion = '13'
 }
 
+dependencies {
+    implementation(project(':lib-fembed-extractor'))
+    implementation(project(':lib-okru-extractor'))
+    implementation "dev.datlag.jsunpacker:jsunpacker:1.0.1"
+}
+
 apply from: "$rootDir/common.gradle"

src/id/neonime/src/eu/kanade/tachiyomi/animeextension/id/neonime/NeoNime.kt

@@ -3,7 +3,12 @@ package eu.kanade.tachiyomi.animeextension.id.neonime
 import android.app.Application
 import android.content.SharedPreferences
 import androidx.preference.ListPreference
+import androidx.preference.MultiSelectListPreference
 import androidx.preference.PreferenceScreen
+import eu.kanade.tachiyomi.animeextension.id.neonime.extractors.BloggerExtractor
+import eu.kanade.tachiyomi.animeextension.id.neonime.extractors.GdrivePlayerExtractor
+import eu.kanade.tachiyomi.animeextension.id.neonime.extractors.LinkBoxExtractor
+import eu.kanade.tachiyomi.animeextension.id.neonime.extractors.YourUploadExtractor
 import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
 import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
 import eu.kanade.tachiyomi.animesource.model.AnimesPage
@@ -11,9 +16,12 @@ import eu.kanade.tachiyomi.animesource.model.SAnime
 import eu.kanade.tachiyomi.animesource.model.SEpisode
 import eu.kanade.tachiyomi.animesource.model.Video
 import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
+import eu.kanade.tachiyomi.lib.fembedextractor.FembedExtractor
+import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
 import eu.kanade.tachiyomi.network.GET
 import eu.kanade.tachiyomi.network.asObservableSuccess
 import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
 import okhttp3.OkHttpClient
 import okhttp3.Request
 import okhttp3.Response
@@ -191,9 +199,72 @@ class NeoNime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
     }
 
     // Video
+
     override fun videoListSelector() = "div > ul >ul > li >a:nth-child(6)"
 
-    override fun videoUrlParse(document: Document) = throw Exception("not used")
+    override fun videoUrlParse(document: Document): String = throw Exception("Not Used")
+
+    override fun videoListParse(response: Response): List<Video> {
+        val document = response.asJsoup()
+        val videoList = mutableListOf<Video>()
+
+        val hosterSelection = preferences.getStringSet(
+            "hoster_selection",
+            setOf("blogger", "linkbox", "fembed", "okru", "yourupload", "gdriveplayer")
+        )!!
+
+        document.select("div.player2 > div.embed2 > div").forEach {
+            val iframe = it.selectFirst("iframe") ?: return@forEach
+
+            var link = iframe.attr("data-src")
+            if (!link.startsWith("http")) {
+                link = "https:$link"
+            }
+
+            when {
+                hosterSelection.contains("linkbox") && link.contains("linkbox.to") -> {
+                    videoList.addAll(LinkBoxExtractor(client).videosFromUrl(link, it.text()))
+                }
+                hosterSelection.contains("fembed") && link.contains("fembed.com") -> {
+                    videoList.addAll(FembedExtractor(client).videosFromUrl(link))
+                }
+                hosterSelection.contains("okru") && link.contains("ok.ru") -> {
+                    videoList.addAll(OkruExtractor(client).videosFromUrl(link))
+                }
+                hosterSelection.contains("yourupload") && link.contains("blogger.com") -> {
+                    videoList.addAll(BloggerExtractor(client).videosFromUrl(link, it.text()))
+                }
+                hosterSelection.contains("linkbox") && link.contains("yourupload.com") -> {
+                    val yuHeaders = headers.newBuilder().add("referer", "https://www.yourupload.com/").build()
+                    videoList.addAll(YourUploadExtractor(client).videoFromUrl(link, yuHeaders, it.text()))
+                }
+                hosterSelection.contains("gdriveplayer") && link.contains("neonime.fun") -> {
+                    val headers = Headers.headersOf(
+                        "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+                        "Referer", response.request.url.toString(),
+                        "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0"
+                    )
+                    var iframe = client.newCall(
+                        GET(link, headers = headers)
+                    ).execute().asJsoup()
+
+                    var iframeUrl = iframe.selectFirst("iframe").attr("src")
+
+                    if (!iframeUrl.startsWith("http")) {
+                        iframeUrl = "https:$iframeUrl"
+                    }
+
+                    when {
+                        iframeUrl.contains("gdriveplayer.to") -> {
+                            videoList.addAll(GdrivePlayerExtractor(client).videosFromUrl(iframeUrl, it.text()))
+                        }
+                    }
+                }
+            }
+        }
+
+        return videoList.sort()
+    }
 
     override fun List<Video>.sort(): List<Video> {
         val quality = preferences.getString("preferred_quality", null)
@@ -237,6 +308,17 @@ class NeoNime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
 
     // screen
     override fun setupPreferenceScreen(screen: PreferenceScreen) {
+        val hostSelection = MultiSelectListPreference(screen.context).apply {
+            key = "hoster_selection"
+            title = "Enable/Disable Hosts"
+            entries = arrayOf("Blogger", "Linkbox", "Fembed", "Ok.ru", "YourUpload", "GdrivePlayer")
+            entryValues = arrayOf("blogger", "linkbox", "fembed", "okru", "yourupload", "gdriveplayer")
+            setDefaultValue(setOf("blogger", "linkbox", "fembed", "okru", "yourupload", "gdriveplayer"))
+
+            setOnPreferenceChangeListener { _, newValue ->
+                preferences.edit().putStringSet(key, newValue as Set<String>).commit()
+            }
+        }
         val videoQualityPref = ListPreference(screen.context).apply {
             key = "preferred_quality"
             title = "Preferred quality"
@@ -252,6 +334,7 @@ class NeoNime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
                 preferences.edit().putString(key, entry).commit()
             }
         }
+        screen.addPreference(hostSelection)
         screen.addPreference(videoQualityPref)
     }
 }

src/id/neonime/src/eu/kanade/tachiyomi/animeextension/id/neonime/extractors/BloggerExtractor.kt

@@ -0,0 +1,38 @@
+package eu.kanade.tachiyomi.animeextension.id.neonime.extractors
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.Serializable
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import okhttp3.OkHttpClient
+import uy.kohesive.injekt.injectLazy
+
+@Serializable
+data class VideoConfig(
+    val streams: List<Stream>
+) {
+    @Serializable
+    data class Stream(
+        val play_url: String,
+        val format_id: Int
+    )
+}
+
+class BloggerExtractor(private val client: OkHttpClient) {
+
+    private val json: Json by injectLazy()
+
+    fun videosFromUrl(url: String, name: String): List<Video> {
+        val document = client.newCall(GET(url)).execute().asJsoup()
+
+        val jsElement = document.selectFirst("script:containsData(VIDEO_CONFIG)") ?: return emptyList()
+        val js = jsElement.data()
+        val json = json.decodeFromString<VideoConfig>(js.substringAfter("var VIDEO_CONFIG = "))
+
+        return json.streams.map {
+            Video(it.play_url, "${it.format_id} - $name", it.play_url)
+        }
+    }
+}

src/id/neonime/src/eu/kanade/tachiyomi/animeextension/id/neonime/extractors/GdrivePlayerExtractor.kt

@@ -0,0 +1,135 @@
+package eu.kanade.tachiyomi.animeextension.id.neonime.extractors
+
+import android.util.Base64
+import dev.datlag.jsunpacker.JsUnpacker
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.jsonPrimitive
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+import java.security.DigestException
+import java.security.MessageDigest
+import javax.crypto.Cipher
+import javax.crypto.spec.IvParameterSpec
+import javax.crypto.spec.SecretKeySpec
+
+class GdrivePlayerExtractor(private val client: OkHttpClient) {
+
+    fun videosFromUrl(url: String, name: String): List<Video> {
+        val headers = Headers.headersOf(
+            "Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+            "Host", "gdriveplayer.to",
+            "Referer", "https://neonime.fun/",
+            "User-Agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:101.0) Gecko/20100101 Firefox/101.0"
+        )
+
+        val body = client.newCall(GET(url.replace(".me", ".to"), headers = headers)).execute()
+            .body!!.string()
+
+        val eval = JsUnpacker.unpackAndCombine(body)!!.replace("\\", "")
+        val json = Json.decodeFromString<JsonObject>(REGEX_DATAJSON.getFirst(eval))
+        val sojson = REGEX_SOJSON.getFirst(eval)
+            .split(Regex("\\D+"))
+            .joinToString("") {
+                Char(it.toInt()).toString()
+            }
+        val password = REGEX_PASSWORD.getFirst(sojson).toByteArray()
+        val decrypted = decryptAES(password, json)!!
+        val secondEval = JsUnpacker.unpackAndCombine(decrypted)!!.replace("\\", "")
+        return REGEX_VIDEOURL.findAll(secondEval)
+            .distinctBy { it.groupValues[2] } // remove duplicates by quality
+            .map {
+                val qualityStr = it.groupValues[2]
+                val quality = "$PLAYER_NAME ${qualityStr}p - $name"
+                val videoUrl = "https:" + it.groupValues[1] + "&res=$qualityStr"
+                Video(videoUrl, quality, videoUrl)
+            }.toList()
+    }
+
+    private fun decryptAES(password: ByteArray, json: JsonObject): String? {
+        val salt = json["s"]!!.jsonPrimitive.content
+        val encodedCiphetext = json["ct"]!!.jsonPrimitive.content
+        val ciphertext = Base64.decode(encodedCiphetext, Base64.DEFAULT)
+        val (key, iv) = GenerateKeyAndIv(password, salt.decodeHex())
+            ?: return null
+        val keySpec = SecretKeySpec(key, "AES")
+        val ivSpec = IvParameterSpec(iv)
+        val cipher = Cipher.getInstance("AES/CBC/NoPadding")
+        cipher.init(Cipher.DECRYPT_MODE, keySpec, ivSpec)
+        val decryptedData = String(cipher.doFinal(ciphertext))
+        return decryptedData
+    }
+
+    // https://stackoverflow.com/a/41434590/8166854
+    private fun GenerateKeyAndIv(
+        password: ByteArray,
+        salt: ByteArray,
+        hashAlgorithm: String = "MD5",
+        keyLength: Int = 32,
+        ivLength: Int = 16,
+        iterations: Int = 1
+    ): List<ByteArray>? {
+
+        val md = MessageDigest.getInstance(hashAlgorithm)
+        val digestLength = md.getDigestLength()
+        val targetKeySize = keyLength + ivLength
+        val requiredLength = (targetKeySize + digestLength - 1) / digestLength * digestLength
+        var generatedData = ByteArray(requiredLength)
+        var generatedLength = 0
+
+        try {
+            md.reset()
+
+            while (generatedLength < targetKeySize) {
+                if (generatedLength > 0)
+                    md.update(
+                        generatedData,
+                        generatedLength - digestLength,
+                        digestLength
+                    )
+
+                md.update(password)
+                md.update(salt, 0, 8)
+                md.digest(generatedData, generatedLength, digestLength)
+
+                for (i in 1 until iterations) {
+                    md.update(generatedData, generatedLength, digestLength)
+                    md.digest(generatedData, generatedLength, digestLength)
+                }
+
+                generatedLength += digestLength
+            }
+            val result = listOf(
+                generatedData.copyOfRange(0, keyLength),
+                generatedData.copyOfRange(keyLength, targetKeySize)
+            )
+            return result
+        } catch (e: DigestException) {
+            return null
+        }
+    }
+
+    private fun Regex.getFirst(item: String): String {
+        return find(item)?.groups?.elementAt(1)?.value!!
+    }
+
+    // Stolen from AnimixPlay(EN) / GogoCdnExtractor
+    private fun String.decodeHex(): ByteArray {
+        check(length % 2 == 0) { "Must have an even length" }
+        return chunked(2)
+            .map { it.toInt(16).toByte() }
+            .toByteArray()
+    }
+
+    companion object {
+        private const val PLAYER_NAME = "GDRIVE"
+
+        private val REGEX_DATAJSON = Regex("data='(\\S+?)'")
+        private val REGEX_PASSWORD = Regex("var pass = \"(\\S+?)\"")
+        private val REGEX_SOJSON = Regex("null,['|\"](\\w+)['|\"]")
+        private val REGEX_VIDEOURL = Regex("file\":\"(\\S+?)\".*?res=(\\d+)")
+    }
+}

src/id/neonime/src/eu/kanade/tachiyomi/animeextension/id/neonime/extractors/LinkBoxExtractor.kt

@@ -0,0 +1,36 @@
+package eu.kanade.tachiyomi.animeextension.id.neonime.extractors
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import kotlinx.serialization.decodeFromString
+import kotlinx.serialization.json.Json
+import kotlinx.serialization.json.JsonObject
+import kotlinx.serialization.json.jsonArray
+import kotlinx.serialization.json.jsonObject
+import okhttp3.OkHttpClient
+
+class LinkBoxExtractor(private val client: OkHttpClient) {
+    fun videosFromUrl(url: String, name: String): List<Video> {
+        val videoList = mutableListOf<Video>()
+        val id = if (url.contains("/file/")) {
+            url.substringAfter("/file/")
+        } else {
+            url.substringAfter("?id=")
+        }
+        val request = client.newCall(GET("https://www.linkbox.to/api/open/get_url?itemId=$id")).execute().asJsoup()
+        val responseJson = Json.decodeFromString<JsonObject>(request.select("body").text())
+        val data = responseJson["data"]?.jsonObject
+        val resolutions = data!!.jsonObject["rList"]!!.jsonArray
+        resolutions.map {
+            videoList.add(
+                Video(
+                    it.jsonObject["url"].toString().replace("\"", ""),
+                    "${it.jsonObject["resolution"].toString().replace("\"", "")} - $name",
+                    it.jsonObject["url"].toString().replace("\"", "")
+                )
+            )
+        }
+        return videoList
+    }
+}

src/id/neonime/src/eu/kanade/tachiyomi/animeextension/id/neonime/extractors/YourUploadExtractor.kt

@@ -0,0 +1,28 @@
+package eu.kanade.tachiyomi.animeextension.id.neonime.extractors
+
+import eu.kanade.tachiyomi.animesource.model.Video
+import eu.kanade.tachiyomi.network.GET
+import eu.kanade.tachiyomi.util.asJsoup
+import okhttp3.Headers
+import okhttp3.OkHttpClient
+
+class YourUploadExtractor(private val client: OkHttpClient) {
+    fun videoFromUrl(url: String, headers: Headers, name: String): List<Video> {
+        val videoList = mutableListOf<Video>()
+        return try {
+            val document = client.newCall(GET(url, headers = headers)).execute()
+            if (document.isSuccessful) {
+                val content = document.asJsoup()
+                val baseData =
+                    content!!.selectFirst("script:containsData(jwplayerOptions)")!!.data()
+                if (!baseData.isNullOrEmpty()) {
+                    val basicUrl = baseData.substringAfter("file: '").substringBefore("',")
+                    videoList.add(Video(basicUrl, "Original - $name", basicUrl, headers = headers))
+                }
+            }
+            videoList
+        } catch (e: Exception) {
+            videoList
+        }
+    }
+}