fix(es/mundodonghua): Video extraction fixes (#2258)

This commit is contained in:
imper1aldev
2023-09-25 05:35:32 -06:00
committed by GitHub
parent 4ca64844cc
commit 59deee5353
6 changed files with 385 additions and 75 deletions

View File

@ -0,0 +1,17 @@
plugins {
id("com.android.library")
kotlin("android")
}
android {
compileSdk = AndroidConfig.compileSdk
namespace = "eu.kanade.tachiyomi.lib.fastreamextractor"
defaultConfig {
minSdk = AndroidConfig.minSdk
}
}
dependencies {
compileOnly(libs.bundles.common)
}

View File

@ -0,0 +1,72 @@
package eu.kanade.tachiyomi.lib.fastreamextractor
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.jsonObject
import okhttp3.OkHttpClient
import uy.kohesive.injekt.injectLazy
class FastreamExtractor(private val client: OkHttpClient) {
private val json: Json by injectLazy()
private fun fetchUrls(text: String?): List<String> {
if (text.isNullOrEmpty()) return listOf()
val linkRegex = "(http|ftp|https):\\/\\/([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:\\/~+#-]*[\\w@?^=%&\\/~+#-])".toRegex()
return linkRegex.findAll(text).map { it.value.trim().removeSurrounding("\"") }.toList()
}
fun videoFromUrl(url: String, server: String = "Fastream"): List<Video> {
val videoList = mutableListOf<Video>()
try {
val document = client.newCall(GET(url)).execute()
if (document.isSuccessful) {
val content = document.asJsoup()
content.select("script").forEach {
if (it!!.data().contains("jwplayer(jwplayer(\"vplayer\").setup({")) {
val basicUrl = it.data().substringAfter("file: '").substringBefore("',")
videoList.add(Video(basicUrl, server, basicUrl, headers = null))
} else {
val packedRegex = Regex("eval\\(function\\(p,a,c,k,e,.*\\)\\)")
val qualities = listOf(
Pair("Low", "360p"),
Pair("Normal", "480p"),
Pair("HD", "720p"),
Pair("Full", "1080p"),
)
packedRegex.findAll(it.data()).map { packed -> packed.value }.toList().map { eval ->
val fastreamRegex = "fastream.*?\\.m3u8([^&\">]?)".toRegex()
val unpack = JsUnpacker.unpack(eval)
fetchUrls(unpack.first()).map { url ->
if (fastreamRegex.containsMatchIn(url)) {
val urlQualities = url.split(",").filter { p -> !p.contains("m3u8") }
val baseUrl = urlQualities.first()
val jsonQualities = "{ \"qualityLabels\": { ${unpack.first().substringAfter("\\'qualityLabels\\':{").substringBefore("},")} }}"
val jObject = json.decodeFromString<JsonObject>(jsonQualities)
val jQualities = jObject["qualityLabels"]!!.jsonObject.map { jsonElement ->
val jQuality = jsonElement.value.toString().replace("\"", "")
qualities.find { q -> q.first.contains(jQuality) }?.second
}.toTypedArray()
var qualityIdx = 0
urlQualities.map { _url ->
if (!_url.contains("http")) {
val quality = "$server:${jQualities[qualityIdx]}"
val videoUrl = "$baseUrl$_url/master.m3u8"
qualityIdx++
videoList.add(Video(videoUrl, quality, videoUrl, headers = null))
}
}
}
}
}
}
}
}
} catch (_: Exception) {
}
return videoList
}
}

View File

@ -0,0 +1,193 @@
package eu.kanade.tachiyomi.lib.fastreamextractor
import kotlin.math.pow
object JsUnpacker {
/**
* Regex to detect packed functions.
*/
private val PACKED_REGEX = Regex("eval[(]function[(]p,a,c,k,e,[r|d]?", setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE))
/**
* Regex to get and group the packed javascript.
* Needed to get information and unpack the code.
*/
private val PACKED_EXTRACT_REGEX = Regex("[}][(]'(.*)', *(\\d+), *(\\d+), *'(.*?)'[.]split[(]'[|]'[)]", setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE))
/**
* Matches function names and variables to de-obfuscate the code.
*/
private val UNPACK_REPLACE_REGEX = Regex("\\b\\w+\\b", setOf(RegexOption.IGNORE_CASE, RegexOption.MULTILINE))
/**
* Check if script is packed.
*
* @param scriptBlock the String to check if it is packed.
*
* @return whether the [scriptBlock] contains packed code or not.
*/
fun detect(scriptBlock: String): Boolean {
return scriptBlock.contains(PACKED_REGEX)
}
/**
* Check if scripts are packed.
*
* @param scriptBlock (multiple) String(s) to check if it is packed.
*
* @return the packed scripts passed in [scriptBlock].
*/
fun detect(vararg scriptBlock: String): List<String> {
return scriptBlock.mapNotNull {
if (it.contains(PACKED_REGEX)) {
it
} else {
null
}
}
}
/**
* Check if scripts are packed.
*
* @param scriptBlocks multiple Strings to check if it is packed.
*
* @return the packed scripts passed in [scriptBlocks].
*/
fun detect(scriptBlocks: Collection<String>): List<String> {
return detect(*scriptBlocks.toTypedArray())
}
/**
* Unpack the passed [scriptBlock].
* It matches all found occurrences and returns them as separate Strings in a list.
*
* @param scriptBlock the String to unpack.
*
* @return unpacked code in a list or an empty list if non is packed.
*/
fun unpack(scriptBlock: String): Sequence<String> {
return if (!detect(scriptBlock)) {
emptySequence()
} else {
unpacking(scriptBlock)
}
}
/**
* Unpack the passed [scriptBlock].
* It matches all found occurrences and combines them into a single String.
*
* @param scriptBlock the String to unpack.
*
* @return unpacked code in a list combined by a whitespace to a single String.
*/
fun unpackAndCombine(scriptBlock: String): String? {
val unpacked = unpack(scriptBlock)
return if (unpacked.toList().isEmpty()) {
null
} else {
unpacked.joinToString(" ")
}
}
/**
* Unpack the passed [scriptBlock].
* It matches all found occurrences and returns them as separate Strings in a list.
*
* @param scriptBlock (multiple) String(s) to unpack.
*
* @return unpacked code in a flat list or an empty list if non is packed.
*/
fun unpack(vararg scriptBlock: String): List<String> {
val packedScripts = detect(*scriptBlock)
return packedScripts.flatMap {
unpacking(it)
}
}
/**
* Unpack the passed [scriptBlocks].
* It matches all found occurrences and returns them as separate Strings in a list.
*
* @param scriptBlocks multiple Strings to unpack.
*
* @return unpacked code in a flat list or an empty list if non is packed.
*/
fun unpack(scriptBlocks: Collection<String>): List<String> {
return unpack(*scriptBlocks.toTypedArray())
}
/**
* Unpacking functionality.
* Match all found occurrences, get the information group and unbase it.
* If found symtabs are more or less than the count provided in code, the occurrence will be ignored
* because it cannot be unpacked correctly.
*
* @param scriptBlock the String to unpack.
*
* @return a list of all unpacked code from all found packed and unpackable occurrences found.
*/
private fun unpacking(scriptBlock: String): Sequence<String> {
val unpacked = PACKED_EXTRACT_REGEX.findAll(scriptBlock).mapNotNull { result ->
val payload = result.groups[1]?.value
val symtab = result.groups[4]?.value?.split('|')
val radix = result.groups[2]?.value?.toIntOrNull() ?: 10
val count = result.groups[3]?.value?.toIntOrNull()
val unbaser = Unbaser(radix)
if (symtab == null || count == null || symtab.size != count) {
null
} else {
payload?.replace(UNPACK_REPLACE_REGEX) { match ->
val word = match.value
val unbased = symtab[unbaser.unbase(word)]
unbased.ifEmpty {
word
}
}
}
}
return unpacked
}
internal data class Unbaser(
private val base: Int,
) {
private val selector: Int = when {
base > 62 -> 95
base > 54 -> 62
base > 52 -> 54
else -> 52
}
fun unbase(value: String): Int {
return if (base in 2..36) {
value.toIntOrNull(base) ?: 0
} else {
val dict = ALPHABET[selector]?.toCharArray()?.mapIndexed { index, c ->
c to index
}?.toMap()
var returnVal = 0
val valArray = value.toCharArray().reversed()
for (i in valArray.indices) {
val cipher = valArray[i]
returnVal += (base.toFloat().pow(i) * (dict?.get(cipher) ?: 0)).toInt()
}
returnVal
}
}
companion object {
private val ALPHABET = mapOf<Int, String>(
52 to "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOP",
54 to "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQR",
62 to "0123456789abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ",
95 to " !\"#\$%&\\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~",
)
}
}
}

View File

@ -15,9 +15,9 @@ import uy.kohesive.injekt.injectLazy
class FilemoonExtractor(private val client: OkHttpClient) {
private val json: Json by injectLazy()
fun videosFromUrl(url: String, prefix: String = "Filemoon - ", headers: Headers? = null): List<Video> {
fun videosFromUrl(url: String, prefix: String = "Filemoon - ", headers: Headers? = null, useHeadersForHtml: Boolean = false): List<Video> {
return runCatching {
val doc = client.newCall(GET(url)).execute().asJsoup()
val doc = if (useHeadersForHtml) client.newCall(GET(url, headers = (headers?.newBuilder() ?: Headers.Builder()).build())).execute().asJsoup() else client.newCall(GET(url)).execute().asJsoup()
val jsEval = doc.selectFirst("script:containsData(eval):containsData(m3u8)")!!.data()
val unpacked = JsUnpacker.unpackAndCombine(jsEval).orEmpty()
val masterUrl = unpacked.takeIf(String::isNotBlank)

View File

@ -5,8 +5,15 @@ ext {
extName = 'MundoDonghua'
pkgNameSuffix = 'es.mundodonghua'
extClass = '.MundoDonghua'
extVersionCode = 9
extVersionCode = 10
libVersion = '13'
}
dependencies {
implementation(project(':lib-filemoon-extractor'))
implementation(project(':lib-voe-extractor'))
implementation(project(':lib-playlist-utils'))
implementation(project(':lib-dailymotion-extractor'))
}
apply from: "$rootDir/common.gradle"

View File

@ -5,7 +5,6 @@ import android.content.SharedPreferences
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.animeextension.es.mundodonghua.extractors.JsUnpacker
import eu.kanade.tachiyomi.animeextension.es.mundodonghua.extractors.ProteaExtractor
import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
import eu.kanade.tachiyomi.animesource.model.AnimeFilter
import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
@ -13,8 +12,13 @@ import eu.kanade.tachiyomi.animesource.model.SAnime
import eu.kanade.tachiyomi.animesource.model.SEpisode
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
import eu.kanade.tachiyomi.lib.dailymotionextractor.DailymotionExtractor
import eu.kanade.tachiyomi.lib.filemoonextractor.FilemoonExtractor
import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
import eu.kanade.tachiyomi.lib.voeextractor.VoeExtractor
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
@ -22,8 +26,6 @@ import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.lang.Exception
class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override val name = "MundoDonghua"
@ -46,9 +48,9 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun popularAnimeFromElement(element: Element): SAnime {
val anime = SAnime.create()
anime.setUrlWithoutDomain(getExternalOrInternalUrl(element.select("a.angled-img").attr("href")))
anime.setUrlWithoutDomain(element.select("a.angled-img").attr("href"))
anime.title = element.select("a.angled-img div.bottom-info.white h5").text().removeSurrounding("\"")
anime.thumbnail_url = baseUrl + element.select("a.angled-img div.img img").attr("src")
anime.thumbnail_url = element.select("a.angled-img div.img img").attr("abs:src")
return anime
}
@ -56,10 +58,8 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun animeDetailsParse(document: Document): SAnime {
val anime = SAnime.create()
anime.thumbnail_url = getExternalOrInternalUrl(
document.selectFirst("div.col-md-4.col-xs-12.mb-10 div.row.sm-row > div.side-banner > div.banner-side-serie")!!
.attr("style").substringAfter("background-image: url(").substringBefore(")"),
)
anime.thumbnail_url = baseUrl + document.selectFirst("div.col-md-4.col-xs-12.mb-10 div.row.sm-row > div.side-banner > div.banner-side-serie")!!
.attr("style").substringAfter("background-image: url(").substringBefore(")")
anime.title = document.selectFirst("div.col-md-4.col-xs-12.mb-10 div.row.sm-row div div.sf.fc-dark.ls-title-serie")!!.html()
anime.description = document.selectFirst("section div.row div.col-md-8 div.sm-row p.text-justify")!!.text().removeSurrounding("\"")
anime.genre = document.select("div.col-md-8.col-xs-12 div.sm-row a.generos span.label").joinToString { it.text() }
@ -72,7 +72,7 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun episodeFromElement(element: Element): SEpisode {
val episode = SEpisode.create()
val epNum = element.attr("href").split("/").last().toFloat()
episode.setUrlWithoutDomain(getExternalOrInternalUrl(element.attr("href")))
episode.setUrlWithoutDomain(element.attr("href"))
episode.episode_number = epNum
episode.name = "Episodio $epNum"
return episode
@ -84,22 +84,10 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
private fun fetchUrls(text: String?): List<String> {
if (text.isNullOrEmpty()) return listOf()
val linkRegex = Regex("""(https?://(www\.)?[-a-zA-Z0-9@:%._+~#=]{1,256}\.[a-zA-Z0-9()]{1,6}\b([-a-zA-Z0-9()@:%_+.~#?&/=]*))""")
val linkRegex = "(http|ftp|https):\\/\\/([\\w_-]+(?:(?:\\.[\\w_-]+)+))([\\w.,@?^=%&:\\/~+#-]*[\\w@?^=%&\\/~+#-])".toRegex()
return linkRegex.findAll(text).map { it.value.trim().removeSurrounding("\"") }.toList()
}
private fun fixUrl(url: String): String {
if (url.startsWith("http")) return url
if (url.isEmpty()) return ""
val startsWithNoHttp = url.startsWith("//")
if (startsWithNoHttp) {
return "https:$url"
} else {
if (url.startsWith('/')) return baseUrl + url
return "$baseUrl/$url"
}
}
override fun videoListParse(response: Response): List<Video> {
val document = response.asJsoup()
val videoList = mutableListOf<Video>()
@ -109,29 +97,64 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
packedRegex.findAll(script.data()).map {
it.value
}.toList().map {
val unpack = getAndUnpack(it)
if (unpack!!.first()!!.contains("protea_tab")) {
val protearegex = Regex("(protea_tab.*slug.*,type)")
val slug = protearegex.findAll(unpack!!.first()).map {
it.value.replace(Regex("(protea_tab.*slug\":\")"), "").replace("\"},type", "")
}.first()
val requestlink = "$baseUrl/api_donghua.php?slug=$slug"
val headers = headers.newBuilder()
.set("authority", "www.mundodonghua.com")
.set("accept", "*/*")
.set("accept-language", "es-MX,es-419;q=0.9,es;q=0.8,en;q=0.7")
.set("dnt", "1")
.set("Connection", "keep-alive")
.set("Sec-Fetch-Dest", "empty")
.set("Sec-Fetch-Mode", "no-cors")
.set("Sec-Fetch-Site", "same-origin")
.set("TE", "trailers")
.set("Pragma", "no-cache")
.set("Cache-Control", "no-cache")
.set("referer", response!!.request!!.url!!.toString())
.set("user-agent", "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/104.0.0.0 Safari/537.36")
val unpack = getAndUnpack(it).first()
if (unpack.contains("amagi_tab")) {
fetchUrls(unpack).map { url ->
try {
VoeExtractor(client).videoFromUrl(url, "VoeCDN")?.let { videoList.add(it) }
} catch (_: Exception) {}
}
}
if (unpack.contains("fmoon_tab")) {
fetchUrls(unpack).map { url ->
try {
val newHeaders = headers.newBuilder()
.add("authority", url.toHttpUrl().host)
.add("referer", "$baseUrl/")
.add("Origin", "https://${url.toHttpUrl().host}")
.build()
ProteaExtractor().videosFromUrl(requestlink, "Protea", headers = headers).map { vid -> videoList.add(vid) }
FilemoonExtractor(client).videosFromUrl(url, prefix = "Filemoon:", headers = newHeaders, useHeadersForHtml = true).let { videoList.addAll(it) }
} catch (_: Exception) {}
}
}
if (unpack.contains("protea_tab")) {
try {
val slug = unpack.substringAfter("\"slug\":\"").substringBefore("\"")
val newHeaders = headers.newBuilder()
.add("referer", "${response.request.url}")
.add("authority", baseUrl.substringAfter("//"))
.add("accept", "*/*")
.build()
val slugPlayer = client.newCall(GET("$baseUrl/api_donghua.php?slug=$slug", headers = newHeaders)).execute().asJsoup().body().toString().substringAfter("\"url\":\"").substringBefore("\"")
val videoHeaders = headers.newBuilder()
.add("authority", "www.mdplayer.xyz")
.add("referer", "$baseUrl/")
.build()
val videoId = client.newCall(GET("https://www.mdplayer.xyz/nemonicplayer/dmplayer.php?key=$slugPlayer", headers = videoHeaders))
.execute().asJsoup().body().toString().substringAfter("video-id=\"").substringBefore("\"")
DailymotionExtractor(client, headers).videosFromUrl("https://www.dailymotion.com/embed/video/$videoId", prefix = "Dailymotion:").let { videoList.addAll(it) }
} catch (_: Exception) {}
}
if (unpack.contains("asura_tab")) {
fetchUrls(unpack).map { url ->
try {
if (url.contains("redirector")) {
val newHeaders = headers.newBuilder()
.add("authority", "www.mdnemonicplayer.xyz")
.add("accept", "*/*")
.add("origin", baseUrl)
.add("referer", "$baseUrl/")
.build()
PlaylistUtils(client, newHeaders).extractFromHls(url, videoNameGen = { "Asura:$it" }).let { videoList.addAll(it) }
}
} catch (_: Exception) {}
}
}
}
}
@ -146,24 +169,21 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun videoFromElement(element: Element) = throw Exception("not used")
override fun List<Video>.sort(): List<Video> {
return try {
val videoSorted = this.sortedWith(
compareBy<Video> { it.quality.replace("[0-9]".toRegex(), "") }.thenByDescending { getNumberFromString(it.quality) },
).toTypedArray()
val userPreferredQuality = preferences.getString("preferred_quality", "Protea:720p")
val preferredIdx = videoSorted.indexOfFirst { x -> x.quality == userPreferredQuality }
if (preferredIdx != -1) {
videoSorted.drop(preferredIdx + 1)
videoSorted[0] = videoSorted[preferredIdx]
}
videoSorted.toList()
} catch (e: Exception) {
this
val quality = preferences.getString("preferred_quality", "VoeCDN")
if (quality != null) {
val newList = mutableListOf<Video>()
var preferred = 0
for (video in this) {
if (video.quality == quality) {
newList.add(preferred, video)
preferred++
} else {
newList.add(video)
}
}
private fun getNumberFromString(epsStr: String): String {
return epsStr.filter { it.isDigit() }.ifEmpty { "0" }
return newList
}
return this
}
override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
@ -242,10 +262,6 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun searchAnimeSelector(): String = popularAnimeSelector()
private fun getExternalOrInternalUrl(url: String): String {
return if (url.contains("https")) url else "$baseUrl/$url"
}
private fun parseStatus(statusString: String): Int {
return when {
statusString.contains("En Emisión") -> SAnime.ONGOING
@ -264,18 +280,23 @@ class MundoDonghua : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun setupPreferenceScreen(screen: PreferenceScreen) {
val qualities = arrayOf(
"Protea:1080p",
"Protea:720p",
"Protea:480p",
"Protea:380p",
"Protea:360p",
"VoeCDN",
"Dailymotion:1080p",
"Dailymotion:720p",
"Dailymotion:480p",
"Filemoon:1080p",
"Filemoon:720p",
"Filemoon:480p",
"Asura:1080p",
"Asura:720p",
"Asura:480p",
)
val videoQualityPref = ListPreference(screen.context).apply {
key = "preferred_quality"
title = "Preferred quality"
entries = qualities
entryValues = qualities
setDefaultValue("Protea:720p")
setDefaultValue("VoeCDN")
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->