feat(gogoanime): add mp4upload + parallelize + hoster toggles, refactoring (#1682)

This commit is contained in:
Secozzi
2023-06-06 23:45:43 +02:00
committed by GitHub
parent cf9eff65e5
commit b563ca5974
3 changed files with 252 additions and 134 deletions

View File

@ -5,13 +5,14 @@ ext {
extName = 'Gogoanime'
pkgNameSuffix = 'en.gogoanime'
extClass = '.GogoAnime'
extVersionCode = 65
extVersionCode = 66
libVersion = '13'
}
dependencies {
implementation(project(':lib-streamsb-extractor'))
implementation(project(':lib-dood-extractor'))
implementation "dev.datlag.jsunpacker:jsunpacker:1.0.1"
}
apply from: "$rootDir/common.gradle"

View File

@ -3,8 +3,10 @@ package eu.kanade.tachiyomi.animeextension.en.gogoanime
import android.app.Application
import android.content.SharedPreferences
import androidx.preference.ListPreference
import androidx.preference.MultiSelectListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.animeextension.en.gogoanime.extractors.GogoCdnExtractor
import eu.kanade.tachiyomi.animeextension.en.gogoanime.extractors.Mp4uploadExtractor
import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
import eu.kanade.tachiyomi.animesource.model.AnimeFilter
import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
@ -16,6 +18,10 @@ import eu.kanade.tachiyomi.lib.doodextractor.DoodExtractor
import eu.kanade.tachiyomi.lib.streamsbextractor.StreamSBExtractor
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.coroutines.Dispatchers
import kotlinx.coroutines.async
import kotlinx.coroutines.awaitAll
import kotlinx.coroutines.runBlocking
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.json.Json
import okhttp3.OkHttpClient
@ -33,7 +39,7 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override val name = "Gogoanime"
override val baseUrl by lazy { preferences.getString("preferred_domain", "https://gogoanime.cl")!! }
override val baseUrl by lazy { preferences.getString(PREF_DOMAIN_KEY, PREF_DOMAIN_DEFAULT)!! }
override val lang = "en"
@ -47,98 +53,37 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
}
override fun popularAnimeSelector(): String = "div.img a"
// ============================== Popular ===============================
override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/popular.html?page=$page")
override fun popularAnimeFromElement(element: Element): SAnime {
val anime = SAnime.create()
anime.setUrlWithoutDomain(element.attr("href"))
anime.thumbnail_url = element.selectFirst("img")!!.attr("src")
anime.title = element.attr("title")
return anime
override fun popularAnimeSelector(): String = "div.img a"
override fun popularAnimeFromElement(element: Element): SAnime = SAnime.create().apply {
setUrlWithoutDomain(element.attr("href"))
thumbnail_url = element.selectFirst("img")!!.attr("src")
title = element.attr("title")
}
override fun popularAnimeNextPageSelector(): String = "ul.pagination-list li:last-child:not(.selected)"
override fun episodeListSelector() = "ul#episode_page li a"
// =============================== Latest ===============================
override fun episodeListParse(response: Response): List<SEpisode> {
val document = response.asJsoup()
val totalEpisodes = document.select(episodeListSelector()).last()!!.attr("ep_end")
val id = document.select("input#movie_id").attr("value")
return episodesRequest(totalEpisodes, id)
override fun latestUpdatesRequest(page: Int): Request =
GET("https://ajax.gogo-load.com/ajax/page-recent-release-ongoing.html?page=$page&type=1", headers)
override fun latestUpdatesSelector(): String = "div.added_series_body.popular li a:has(div)"
override fun latestUpdatesFromElement(element: Element): SAnime = SAnime.create().apply {
setUrlWithoutDomain(element.attr("abs:href"))
thumbnail_url = element.select("div.thumbnail-popular").attr("style")
.substringAfter("background: url('").substringBefore("');")
title = element.attr("title")
}
private fun episodesRequest(totalEpisodes: String, id: String): List<SEpisode> {
val request = GET("https://ajax.gogo-load.com/ajax/load-list-episode?ep_start=0&ep_end=$totalEpisodes&id=$id", headers)
val epResponse = client.newCall(request).execute()
val document = epResponse.asJsoup()
return document.select("a").map { episodeFromElement(it) }
}
override fun latestUpdatesNextPageSelector(): String = popularAnimeNextPageSelector()
override fun episodeFromElement(element: Element): SEpisode {
val episode = SEpisode.create()
episode.setUrlWithoutDomain(baseUrl + element.attr("href").substringAfter(" "))
val ep = element.selectFirst("div.name")!!.ownText().substringAfter(" ")
episode.episode_number = ep.toFloat()
episode.name = "Episode $ep"
return episode
}
override fun videoListParse(response: Response): List<Video> {
val document = response.asJsoup()
val extractor = GogoCdnExtractor(network.client, json)
val videoList = mutableListOf<Video>()
// GogoCdn:
document.select("div.anime_muti_link > ul > li.vidcdn > a")
.firstOrNull()?.attr("data-video")
?.let { videoList.addAll(extractor.videosFromUrl(it.replace(Regex("^//"), "https://"))) }
// Vidstreaming:
document.select("div.anime_muti_link > ul > li.anime > a")
.firstOrNull()?.attr("data-video")
?.let { videoList.addAll(extractor.videosFromUrl(it.replace(Regex("^//"), "https://"))) }
// Doodstream mirror:
document.select("div.anime_muti_link > ul > li.doodstream > a")
.firstOrNull()?.attr("data-video")
?.let { videoList.addAll(DoodExtractor(client).videosFromUrl(it.replace(Regex("^//"), "https://"))) }
// StreamSB mirror:
document.select("div.anime_muti_link > ul > li.streamsb > a")
.firstOrNull()?.attr("data-video")
?.let { videoList.addAll(StreamSBExtractor(client).videosFromUrl(it.replace(Regex("^//"), "https://"), headers)) }
return videoList.sort()
}
override fun videoListSelector() = throw Exception("not used")
override fun videoFromElement(element: Element) = throw Exception("not used")
override fun videoUrlParse(document: Document) = throw Exception("not used")
override fun List<Video>.sort(): List<Video> {
val quality = preferences.getString("preferred_quality", "1080")!!
val server = preferences.getString("preferred_server", "Gogostream")!!
return this.sortedWith(
compareBy(
{ it.quality.contains(quality) },
{ it.quality.contains(server) },
),
).reversed()
}
override fun searchAnimeFromElement(element: Element): SAnime {
val anime = SAnime.create()
anime.setUrlWithoutDomain(element.attr("href"))
anime.thumbnail_url = element.selectFirst("img")!!.attr("src")
anime.title = element.attr("title")
return anime
}
override fun searchAnimeNextPageSelector(): String = "ul.pagination-list li:last-child:not(.selected)"
override fun searchAnimeSelector(): String = "div.img a"
// =============================== Search ===============================
override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
val filterList = if (filters.isEmpty()) getFilterList() else filters
@ -155,24 +100,124 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
}
}
override fun animeDetailsParse(document: Document): SAnime {
val anime = SAnime.create()
anime.title = document.select("div.anime_info_body_bg h1").text()
anime.genre = document.select("p.type:eq(5) a").joinToString("") { it.text() }
anime.description = document.selectFirst("p.type:eq(4)")!!.ownText()
anime.status = parseStatus(document.select("p.type:eq(7) a").text())
override fun searchAnimeSelector(): String = popularAnimeSelector()
// add alternative name to anime description
val altName = "Other name(s): "
document.selectFirst("p.type:eq(8)")?.ownText()?.let {
if (it.isBlank().not()) {
anime.description = when {
anime.description.isNullOrBlank() -> altName + it
else -> anime.description + "\n\n$altName" + it
override fun searchAnimeFromElement(element: Element): SAnime = popularAnimeFromElement(element)
override fun searchAnimeNextPageSelector(): String = popularAnimeNextPageSelector()
// =========================== Anime Details ============================
override fun animeDetailsParse(document: Document): SAnime {
return SAnime.create().apply {
title = document.select("div.anime_info_body_bg h1").text()
genre = document.select("p.type:eq(5) a").joinToString("") { it.text() }
description = document.selectFirst("p.type:eq(4)")!!.ownText()
status = parseStatus(document.select("p.type:eq(7) a").text())
// add alternative name to anime description
val altName = "Other name(s): "
document.selectFirst("p.type:eq(8)")?.ownText()?.let {
if (it.isBlank().not()) {
description = when {
description.isNullOrBlank() -> altName + it
else -> description + "\n\n$altName" + it
}
}
}
}
return anime
}
// ============================== Episodes ==============================
private fun episodesRequest(totalEpisodes: String, id: String): List<SEpisode> {
val request = GET("https://ajax.gogo-load.com/ajax/load-list-episode?ep_start=0&ep_end=$totalEpisodes&id=$id", headers)
val epResponse = client.newCall(request).execute()
val document = epResponse.asJsoup()
return document.select("a").map { episodeFromElement(it) }
}
override fun episodeListSelector() = "ul#episode_page li a"
override fun episodeListParse(response: Response): List<SEpisode> {
val document = response.asJsoup()
val totalEpisodes = document.select(episodeListSelector()).last()!!.attr("ep_end")
val id = document.select("input#movie_id").attr("value")
return episodesRequest(totalEpisodes, id)
}
override fun episodeFromElement(element: Element): SEpisode {
val ep = element.selectFirst("div.name")!!.ownText().substringAfter(" ")
return SEpisode.create().apply {
setUrlWithoutDomain(element.attr("abs:href"))
episode_number = ep.toFloat()
name = "Episode $ep"
}
}
// ============================ Video Links =============================
override fun videoListParse(response: Response): List<Video> {
val document = response.asJsoup()
val gogoExtractor = GogoCdnExtractor(client, json)
val hosterSelection = preferences.getStringSet(PREF_HOSTER_KEY, PREF_HOSTER_DEFAULT)!!
val videoList = mutableListOf<Video>()
videoList.addAll(
document.select("div.anime_muti_link > ul > li").parallelMap { server ->
runCatching {
val className = server.className()
if (!hosterSelection.contains(className)) return@runCatching null
val serverUrl = server.selectFirst("a")
?.attr("data-video")
?.replace(Regex("^//"), "https://")
?: return@runCatching null
when (className) {
"anime" -> {
gogoExtractor.videosFromUrl(serverUrl)
}
"vidcdn" -> {
gogoExtractor.videosFromUrl(serverUrl)
}
"streamsb" -> {
StreamSBExtractor(client).videosFromUrl(serverUrl, headers)
}
"doodstream" -> {
DoodExtractor(client).videosFromUrl(serverUrl)
}
"mp4upload" -> {
val headers = headers.newBuilder().set("Referer", "https://mp4upload.com/").build()
Mp4uploadExtractor(client).getVideoFromUrl(serverUrl, headers)
}
else -> null
}
}.getOrNull()
}.filterNotNull().flatten(),
)
return videoList.sort()
}
override fun videoListSelector() = throw Exception("not used")
override fun videoFromElement(element: Element) = throw Exception("not used")
override fun videoUrlParse(document: Document) = throw Exception("not used")
// ============================= Utilities ==============================
override fun List<Video>.sort(): List<Video> {
val quality = preferences.getString(PREF_QUALITY_KEY, PREF_QUALITY_DEFAULT)!!
val server = preferences.getString(PREF_SERVER_KEY, PREF_SERVER_TITLE)!!
return this.sortedWith(
compareBy(
{ it.quality.contains(quality) },
{ Regex("""(\d+)p""").find(it.quality)?.groupValues?.get(1)?.toIntOrNull() ?: 0 },
{ it.quality.contains(server) },
),
).reversed()
}
private fun parseStatus(statusString: String): Int {
@ -183,29 +228,58 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
}
}
override fun latestUpdatesNextPageSelector(): String = "ul.pagination-list li:last-child:not(.selected)"
// From Dopebox
private fun <A, B> Iterable<A>.parallelMap(f: suspend (A) -> B): List<B> =
runBlocking {
map { async(Dispatchers.Default) { f(it) } }.awaitAll()
}
override fun latestUpdatesFromElement(element: Element): SAnime {
val anime = SAnime.create()
anime.setUrlWithoutDomain(baseUrl + element.attr("href"))
val style = element.select("div.thumbnail-popular").attr("style")
anime.thumbnail_url = style.substringAfter("background: url('").substringBefore("');")
anime.title = element.attr("title")
return anime
companion object {
private val HOSTERS = arrayOf(
"Gogostream",
"Vidstreaming",
"Doodstream",
"StreamSB",
"Mp4upload",
)
private val HOSTERS_NAMES = arrayOf( // Names that appears in the gogo html
"vidcdn",
"anime",
"doodstream",
"streamsb",
"mp4upload",
)
private const val PREF_DOMAIN_KEY = "preferred_domain"
private const val PREF_DOMAIN_TITLE = "Preferred domain (requires app restart)"
private val PREF_DOMAIN_ENTRIES = arrayOf("gogoanime.cl", "gogoanime.llc")
private val PREF_DOMAIN_ENTRY_VALUES = PREF_DOMAIN_ENTRIES.map { "https://$it" }.toTypedArray()
private val PREF_DOMAIN_DEFAULT = "https://gogoanime.cl"
private const val PREF_QUALITY_KEY = "preferred_quality"
private const val PREF_QUALITY_TITLE = "Preferred quality"
private val PREF_QUALITY_ENTRY_VALUES = arrayOf("1080", "720", "480", "360")
private val PREF_QUALITY_ENTRIES = PREF_QUALITY_ENTRY_VALUES.map { "${it}p" }.toTypedArray()
private const val PREF_QUALITY_DEFAULT = "1080"
private const val PREF_SERVER_KEY = "preferred_server"
private const val PREF_SERVER_TITLE = "Preferred server"
private const val PREF_SERVER_DEFAULT = "Gogostream"
private const val PREF_HOSTER_KEY = "hoster_selection"
private const val PREF_HOSTER_TITLE = "Enable/Disable Hosts"
private val PREF_HOSTER_DEFAULT = HOSTERS_NAMES.toSet()
}
override fun latestUpdatesRequest(page: Int): Request =
GET("https://ajax.gogo-load.com/ajax/page-recent-release-ongoing.html?page=$page&type=1", headers)
override fun latestUpdatesSelector(): String = "div.added_series_body.popular li a:has(div)"
// ============================== Settings ==============================
override fun setupPreferenceScreen(screen: PreferenceScreen) {
val domainPref = ListPreference(screen.context).apply {
key = "preferred_domain"
title = "Preferred domain (requires app restart)"
entries = arrayOf("gogoanime.cl", "gogoanime.llc")
entryValues = arrayOf("https://gogoanime.cl", "https://gogoanime.llc")
setDefaultValue("https://gogoanime.cl")
ListPreference(screen.context).apply {
key = PREF_DOMAIN_KEY
title = PREF_DOMAIN_TITLE
entries = PREF_DOMAIN_ENTRIES
entryValues = PREF_DOMAIN_ENTRY_VALUES
setDefaultValue(PREF_DOMAIN_DEFAULT)
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
@ -214,13 +288,14 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
val videoQualityPref = ListPreference(screen.context).apply {
key = "preferred_quality"
title = "Preferred quality"
entries = arrayOf("1080p", "720p", "480p", "360p")
entryValues = arrayOf("1080", "720", "480", "360")
setDefaultValue("1080")
}.let { screen.addPreference(it) }
ListPreference(screen.context).apply {
key = PREF_QUALITY_KEY
title = PREF_QUALITY_TITLE
entries = PREF_QUALITY_ENTRIES
entryValues = PREF_QUALITY_ENTRY_VALUES
setDefaultValue(PREF_QUALITY_DEFAULT)
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
@ -229,13 +304,14 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
val videoServerPref = ListPreference(screen.context).apply {
key = "preferred_server"
title = "Preferred server"
entries = arrayOf("Gogostream", "Vidstreaming", "Doodstream", "StreamSB")
entryValues = arrayOf("Gogostream", "Vidstreaming", "Doodstream", "StreamSB")
setDefaultValue("Gogostream")
}.let { screen.addPreference(it) }
ListPreference(screen.context).apply {
key = PREF_SERVER_KEY
title = PREF_SERVER_TITLE
entries = HOSTERS
entryValues = HOSTERS
setDefaultValue(PREF_SERVER_DEFAULT)
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
@ -244,13 +320,24 @@ class GogoAnime : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
screen.addPreference(domainPref)
screen.addPreference(videoQualityPref)
screen.addPreference(videoServerPref)
}.let { screen.addPreference(it) }
MultiSelectListPreference(screen.context).apply {
key = PREF_HOSTER_KEY
title = PREF_HOSTER_TITLE
entries = HOSTERS
entryValues = HOSTERS_NAMES
setDefaultValue(PREF_HOSTER_DEFAULT)
setOnPreferenceChangeListener { _, newValue ->
@Suppress("UNCHECKED_CAST")
preferences.edit().putStringSet(key, newValue as Set<String>).commit()
}
}.let { screen.addPreference(it) }
}
// Filters
// ============================== Filters ===============================
override fun getFilterList(): AnimeFilterList = AnimeFilterList(
AnimeFilter.Header("Text search ignores filters"),
GenreFilter(),

View File

@ -0,0 +1,30 @@
package eu.kanade.tachiyomi.animeextension.en.gogoanime.extractors
import dev.datlag.jsunpacker.JsUnpacker
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.network.GET
import okhttp3.Headers
import okhttp3.OkHttpClient
import org.jsoup.Jsoup
class Mp4uploadExtractor(private val client: OkHttpClient) {
fun getVideoFromUrl(url: String, headers: Headers, prefix: String = ""): List<Video> {
val body = client.newCall(GET(url, headers = headers)).execute().body.string()
val videoUrl = if (body.contains("eval(function(p,a,c,k,e,d)")) {
val packed = body.substringAfter("<script type='text/javascript'>eval(function(p,a,c,k,e,d)")
.substringBefore("</script>")
body.substringAfter("<script type='text/javascript'>eval(function(p,a,c,k,e,d)")
.substringBefore("</script>")
val unpacked = JsUnpacker.unpackAndCombine("eval(function(p,a,c,k,e,d)" + packed) ?: return emptyList()
unpacked.substringAfter("player.src(\"").substringBefore("\");")
} else {
val script = Jsoup.parse(body).selectFirst("script:containsData(player.src)")?.data() ?: return emptyList()
script.substringAfter("src: \"").substringBefore("\"")
}
return listOf(
Video(videoUrl, "${prefix}Mp4upload", videoUrl, headers = Headers.headersOf("Referer", "https://www.mp4upload.com/")),
)
}
}