fix(it/toonitalia): Update baseUrl + add extractors (#2104)

This commit is contained in:
Claudemirovsky
2023-08-30 09:33:45 -03:00
committed by GitHub
parent 5b777bdc35
commit 3e800631e5
3 changed files with 225 additions and 280 deletions

View File

@ -1,16 +1,21 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
plugins {
alias(libs.plugins.android.application)
alias(libs.plugins.kotlin.android)
}
ext {
extName = 'Toonitalia'
pkgNameSuffix = 'it.toonitalia'
extClass = '.Toonitalia'
extVersionCode = 9
extVersionCode = 10
libVersion = '13'
}
dependencies {
implementation(project(':lib-voe-extractor'))
implementation(project(':lib-streamtape-extractor'))
implementation(project(':lib-playlist-utils'))
implementation("dev.datlag.jsunpacker:jsunpacker:1.0.1")
}
apply from: "$rootDir/common.gradle"

View File

@ -1,9 +1,9 @@
package eu.kanade.tachiyomi.animeextension.it.toonitalia
import android.app.Application
import android.content.SharedPreferences
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.animeextension.it.toonitalia.extractors.MaxStreamExtractor
import eu.kanade.tachiyomi.animeextension.it.toonitalia.extractors.StreamZExtractor
import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
import eu.kanade.tachiyomi.animesource.model.AnimeFilter
@ -13,11 +13,11 @@ import eu.kanade.tachiyomi.animesource.model.SAnime
import eu.kanade.tachiyomi.animesource.model.SEpisode
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
import eu.kanade.tachiyomi.lib.streamtapeextractor.StreamTapeExtractor
import eu.kanade.tachiyomi.lib.voeextractor.VoeExtractor
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.HttpUrl.Companion.toHttpUrlOrNull
import okhttp3.OkHttpClient
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
@ -30,38 +30,34 @@ class Toonitalia : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override val name = "Toonitalia"
override val baseUrl = "https://toonitalia.co"
override val baseUrl = "https://toonitalia.green"
override val lang = "it"
override val supportsLatest = false
override val client: OkHttpClient = network.cloudflareClient
override val client = network.cloudflareClient
private val preferences: SharedPreferences by lazy {
private val preferences by lazy {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
}
// ============================== Popular ===============================
override fun popularAnimeRequest(page: Int) = GET("$baseUrl/page/$page", headers)
override fun popularAnimeRequest(page: Int): Request {
return GET("$baseUrl/page/$page", headers = headers)
override fun popularAnimeSelector() = "#primary > main#main > article"
override fun popularAnimeFromElement(element: Element) = SAnime.create().apply {
element.selectFirst("h2 > a")!!.run {
title = text()
setUrlWithoutDomain(attr("href"))
}
thumbnail_url = element.selectFirst("img")!!.attr("src")
}
override fun popularAnimeSelector(): String = "div#primary > main#main > article"
override fun popularAnimeFromElement(element: Element): SAnime {
val anime = SAnime.create()
anime.title = element.select("h2 > a").text()
anime.thumbnail_url = element.selectFirst("img")!!.attr("src")
anime.setUrlWithoutDomain(element.selectFirst("a")!!.attr("href").substringAfter(baseUrl))
return anime
}
override fun popularAnimeNextPageSelector(): String = "div.nav-links > span.current ~ a"
override fun popularAnimeNextPageSelector() = "nav.pagination a.next"
// =============================== Latest ===============================
override fun latestUpdatesRequest(page: Int): Request = throw Exception("Not used")
override fun latestUpdatesSelector(): String = throw Exception("Not used")
@ -71,292 +67,153 @@ class Toonitalia : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override fun latestUpdatesNextPageSelector(): String = throw Exception("Not used")
// =============================== Search ===============================
override fun searchAnimeParse(response: Response): AnimesPage {
val document = response.asJsoup()
val document = response.use { it.asJsoup() }
val animes = if (response.request.url.toString().substringAfter(baseUrl).startsWith("/?s=")) {
document.select(searchAnimeSelector()).map { element ->
searchAnimeFromElement(element)
}
val isNormalSearch = document.location().contains("/?s=")
val animes = if (isNormalSearch) {
document.select(searchAnimeSelector()).map(::searchAnimeFromElement)
} else {
document.select(searchIndexAnimeSelector()).map { element ->
searchIndexAnimeFromElement(element)
}
document.select(searchIndexAnimeSelector()).map(::searchIndexAnimeFromElement)
}
val hasNextPage = searchAnimeNextPageSelector()?.let { selector ->
document.select(selector).first()
} != null
val hasNextPage = document.selectFirst(searchAnimeNextPageSelector()) != null
return AnimesPage(animes, hasNextPage)
}
override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
return if (query.isNotBlank()) {
GET("$baseUrl/?s=$query", headers = headers)
GET("$baseUrl/page/$page/?s=$query", headers = headers)
} else {
val url = "$baseUrl".toHttpUrlOrNull()!!.newBuilder()
filters.forEach { filter ->
when (filter) {
is IndexFilter -> url.addPathSegment(filter.toUriPart())
else -> {}
}
val url = "$baseUrl".toHttpUrl().newBuilder().apply {
filters.filterIsInstance<IndexFilter>()
.firstOrNull()
?.also { addPathSegment(it.toUriPart()) }
}
var newUrl = url.toString()
if (page > 1) {
newUrl += "/?lcp_page0=$page#lcp_instance_0"
}
GET(newUrl, headers = headers)
val newUrl = url.toString() + "/?lcp_page0=$page#lcp_instance_0"
GET(newUrl, headers)
}
}
override fun searchAnimeSelector(): String = "section#primary > main#main > article"
override fun searchAnimeSelector() = popularAnimeSelector()
private fun searchIndexAnimeSelector(): String = "div.entry-content > ul.lcp_catlist > li"
private fun searchIndexAnimeSelector() = "div.entry-content > ul.lcp_catlist > li"
override fun searchAnimeFromElement(element: Element): SAnime {
val anime = SAnime.create()
anime.title = element.selectFirst("h2")!!.text()
anime.setUrlWithoutDomain(element.selectFirst("a")!!.attr("href").substringAfter(baseUrl))
return anime
override fun searchAnimeFromElement(element: Element) = popularAnimeFromElement(element)
private fun searchIndexAnimeFromElement(element: Element) = SAnime.create().apply {
element.selectFirst("a")!!.run {
title = text()
setUrlWithoutDomain(attr("href"))
}
}
private fun searchIndexAnimeFromElement(element: Element): SAnime {
val anime = SAnime.create()
anime.title = element.select("a").text()
anime.setUrlWithoutDomain(element.selectFirst("a")!!.attr("href").substringAfter(baseUrl))
return anime
}
override fun searchAnimeNextPageSelector(): String = "ul.lcp_paginator > li.lcp_currentpage ~ li"
override fun searchAnimeNextPageSelector() =
"nav.navigation div.nav-previous, " + // Normal search
"ul.lcp_paginator > li > a.lcp_nextlink" // Index search
// =========================== Anime Details ============================
override fun animeDetailsParse(document: Document) = SAnime.create().apply {
title = document.selectFirst("h1.entry-title")!!.text()
thumbnail_url = document.selectFirst("header.entry-header img")!!.attr("abs:src")
override fun animeDetailsParse(document: Document): SAnime {
val anime = SAnime.create()
anime.thumbnail_url = document.select("div.entry-content > h2 > img").attr("src")
anime.title = document.select("header.entry-header > h1.entry-title").text()
var descInfo = ""
document.selectFirst("div.entry-content > h2 + p + p")!!.childNodes().filter {
s ->
s.nodeName() != "br"
}.forEach {
if (it.nodeName() == "span") {
if (it.nextSibling() != null) {
descInfo += "\n"
}
descInfo += "${it.childNode(0)} "
} else if (it.nodeName() == "#text") {
val infoStr = it.toString().trim()
if (infoStr.isNotBlank()) descInfo += infoStr
// Cursed sources should have cursed code!
description = document.selectFirst("article > div.entry-content")!!
.also { it.select("center").remove() } // Remove unnecessary data
.wholeText()
.replace(",", ", ").replace(" ", " ") // Fix text
.lines()
.map(String::trim)
.filterNot { it.startsWith("Titolo:") }
.also { lines ->
genre = lines.firstOrNull { it.startsWith("Genere:") }
?.substringAfter("Genere: ")
}
}
var descElement = document.selectFirst("div.entry-content > h3:contains(Trama:) + p")
if (descElement == null) {
descElement = document.selectFirst("div.entry-content > p:has(span:contains(Trama:))")
}
val description = if (descElement == null) {
"Nessuna descrizione disponibile\n\n$descInfo"
} else {
descElement.childNodes().filter {
s ->
s.nodeName() == "#text"
}.joinToString(separator = "\n\n") { it.toString() }.trim() + "\n\n" + descInfo
}
anime.description = description
anime.genre = document.select("footer.entry-footer > span.cat-links > a").joinToString(separator = ", ") { it.text() }
return anime
.joinToString("\n")
.substringAfter("Trama: ")
}
// ============================== Episodes ==============================
private val episodeNumRegex by lazy { Regex("\\s(\\d+x\\d+)\\s?") }
override fun episodeListParse(response: Response): List<SEpisode> {
val document = response.asJsoup()
val episodeList = mutableListOf<SEpisode>()
val doc = response.use { it.asJsoup() }
val url = doc.location()
// Select single seasons episodes
val singleEpisode = document.select("div.entry-content > h3:contains(Episodi) + p")
if (singleEpisode.isNotEmpty() && singleEpisode.text().isNotEmpty()) {
var episode = SEpisode.create()
var isValid = false
var counter = 1
for (child in singleEpisode.first()!!.childNodes()) {
if (child.nodeName() == "br" || (child.nextSibling() == null && child.nodeName() == "a")) {
episode.url = response.request.url.toString() + "#$counter"
if (isValid) {
episodeList.add(episode)
isValid = false
}
episode = SEpisode.create()
counter++
} else if (child.nodeName() == "a") {
isValid = true
} else {
val name = child.toString().trim().substringBeforeLast("")
if (name.isNotEmpty()) {
episode.name = "Episode ${name.trim()}"
episode.episode_number = counter.toFloat()
}
}
}
if ("/film-anime/" in url) {
return listOf(
SEpisode.create().apply {
setUrlWithoutDomain("$url#0")
episode_number = 1F
name = doc.selectFirst("h1.entry-title")!!.text()
},
)
}
// Select multiple seasons
val seasons = document.select("div.entry-content > h3:contains(Stagione) + p")
if (seasons.isNotEmpty()) {
var counter = 1
seasons.forEach {
var episode = SEpisode.create()
var isValid = false
for (child in it.childNodes()) {
if (child.nodeName() == "br" || (child.nextSibling() == null && child.nodeName() == "a")) {
episode.url = response.request.url.toString() + "#$counter"
if (isValid) {
episodeList.add(episode)
isValid = false
}
episode = SEpisode.create()
counter++
} else if (child.nodeName() == "a") {
isValid = true
} else {
val name = child.toString().trim().substringBeforeLast("")
if (name.isNotEmpty()) {
episode.name = "Episode ${name.trim()}"
episode.episode_number = counter.toFloat()
}
}
}
val epNames = doc.select(episodeListSelector() + ">td:not(:has(a))").eachText()
return epNames.mapIndexed { index, item ->
SEpisode.create().apply {
setUrlWithoutDomain("$url#$index")
val (season, episode) = episodeNumRegex.find(item)
?.groupValues
?.last()
?.split("x")
?: listOf("01", "01")
name = "Stagione $season - Episodi $episode"
episode_number = "$season.${episode.padStart(3, '0')}".toFloatOrNull() ?: 1F
}
}
// Select movie
val movie = document.select("div.entry-content > p:contains(Link Streaming)")
if (movie.isNotEmpty()) {
val episode = SEpisode.create()
for (child in movie.first()!!.childNodes()) {
if (child.nodeName() == "br" || (child.nextSibling() == null && child.nodeName() == "a")) {
// episode.url = links.joinToString(separator = "///")
episode.url = response.request.url.toString() + "#1"
} else if (child.nodeName() == "a") {
} else {
val name = child.toString().trim().substringBeforeLast("")
if (name.isNotEmpty()) {
episode.name = "Movie"
episode.episode_number = 1F
}
}
}
episodeList.add(episode)
}
return episodeList.reversed()
}.reversed()
}
override fun episodeFromElement(element: Element): SEpisode = throw Exception("Not used")
override fun episodeFromElement(element: Element) = throw Exception("Not used")
override fun episodeListSelector(): String = throw Exception("Not used")
override fun episodeListSelector() = "article > div.entry-content table tr:has(a)"
// ============================ Video Links =============================
override fun videoListRequest(episode: SEpisode): Request {
return GET(episode.url, headers = headers)
}
override fun videoListParse(response: Response): List<Video> {
val document = response.asJsoup()
val videoList = mutableListOf<Video>()
val document = response.use { it.asJsoup() }
val episodeNumber = response.request.url.fragment!!.toInt()
// Select single seasons episodes
val singleEpisode = document.select("div.entry-content > h3:contains(Episodi) + p")
if (singleEpisode.isNotEmpty() && singleEpisode.text().isNotEmpty()) {
var counter = 1
for (child in singleEpisode.first()!!.childNodes()) {
if (child.nodeName() == "a" && counter == episodeNumber) {
videoList.addAll(extractVideos(child.attr("href"), child.childNode(0).toString()))
}
val episode = document.select(episodeListSelector())
.getOrNull(episodeNumber)
?: return emptyList()
if (child.nodeName() == "br" || child.nextSibling() == null) {
counter++
return episode.select("a").flatMap {
runCatching {
val url = it.attr("href")
val hosterUrl = when {
url.contains("uprot.net") -> bypassUprot(url)
else -> url
}
}
hosterUrl?.let(::extractVideos)
}.getOrNull() ?: emptyList()
}
// Select multiple seasons
val seasons = document.select("div.entry-content > h3:contains(Stagione) + p")
if (seasons.isNotEmpty()) {
var counter = 1
seasons.forEach {
for (child in it.childNodes()) {
if (child.nodeName() == "a" && counter == episodeNumber) {
videoList.addAll(extractVideos(child.attr("href"), child.childNode(0).toString()))
}
if (child.nodeName() == "br" || child.nextSibling() == null) {
counter++
}
}
}
}
// Select movie
val movie = document.select("div.entry-content > p:contains(Link Streaming)")
if (movie.isNotEmpty()) {
for (child in movie.first()!!.childNodes()) {
if (child.nodeName() == "a") {
videoList.addAll(extractVideos(child.attr("href"), child.childNode(0).toString()))
}
}
}
return videoList.sort()
}
private val voeExtractor by lazy { VoeExtractor(client) }
private val streamZExtractor by lazy { StreamZExtractor(client) }
private val streamTapeExtractor by lazy { StreamTapeExtractor(client) }
private val maxStreamExtractor by lazy { MaxStreamExtractor(client, headers) }
private fun extractVideos(url: String): List<Video> =
when {
"https://voe.sx" in url -> voeExtractor.videoFromUrl(url)?.let(::listOf)
"https://streamtape" in url -> streamTapeExtractor.videoFromUrl(url)?.let(::listOf)
"https://maxstream" in url -> maxStreamExtractor.videosFromUrl(url)
"https://streamz" in url || "streamz.cc" in url -> {
streamZExtractor.videoFromUrl(url, "StreamZ")?.let(::listOf)
}
else -> null
} ?: emptyList()
override fun videoFromElement(element: Element): Video = throw Exception("Not used")
override fun videoListSelector(): String = throw Exception("Not used")
override fun videoUrlParse(document: Document): String = throw Exception("Not used")
// ============================= Utilities ==============================
private fun extractVideos(url: String, name: String): List<Video> {
return when {
url.contains("https://voe.sx") || url.contains("https://20demidistance9elongations.com") ||
url.contains("https://telyn610zoanthropy.com")
-> {
val video = VoeExtractor(client).videoFromUrl(url, name)
if (video == null) {
emptyList()
} else {
listOf(video)
}
}
url.contains("https://streamz") || url.contains("streamz.cc") -> {
val video = StreamZExtractor(client).videoFromUrl(url, name)
if (video == null) {
emptyList()
} else {
listOf(video)
}
}
else -> { emptyList() }
}
}
// ============================== Filters ===============================
override fun getFilterList() = AnimeFilterList(
AnimeFilter.Header("NOTA: ignorato se si utilizza la ricerca di testo!"),
AnimeFilter.Separator(),
@ -367,10 +224,8 @@ class Toonitalia : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
private fun getIndexList() = arrayOf(
Pair("<selezionare>", ""),
Pair("Anime", "anime"),
Pair("Anime Sub-ita", "anime-sub-ita"),
Pair("Serie Tv", "serie-tv"),
Pair("Film Animazione", "film-animazione"),
Pair("Lista Anime e Cartoni", "lista-anime-e-cartoni"),
Pair("Lista Film Anime", "lista-film-anime"),
)
open class UriPartFilter(displayName: String, private val vals: Array<Pair<String, String>>) :
@ -379,10 +234,10 @@ class Toonitalia : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
}
override fun List<Video>.sort(): List<Video> {
val quality = preferences.getString("preferred_quality", "1080")!!
val server = preferences.getString("preferred_server", "VOE")!!
val quality = preferences.getString(PREF_QUALITY_KEY, PREF_QUALITY_DEFAULT)!!
val server = preferences.getString(PREF_SERVER_KEY, PREF_SERVER_DEFAULT)!!
return this.sortedWith(
return sortedWith(
compareBy(
{ it.quality.contains(server) },
{ it.quality.contains(quality) },
@ -391,12 +246,12 @@ class Toonitalia : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
}
override fun setupPreferenceScreen(screen: PreferenceScreen) {
val videoQualityPref = ListPreference(screen.context).apply {
key = "preferred_quality"
title = "Preferred quality"
entries = arrayOf("1080p", "720p", "480p", "360p", "240p", "80p")
entryValues = arrayOf("1080", "720", "480", "360", "240", "80")
setDefaultValue("1080")
ListPreference(screen.context).apply {
key = PREF_QUALITY_KEY
title = PREF_QUALITY_TITLE
entries = PREF_QUALITY_ENTRIES
entryValues = PREF_QUALITY_VALUES
setDefaultValue(PREF_QUALITY_DEFAULT)
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
@ -405,14 +260,14 @@ class Toonitalia : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
}.also(screen::addPreference)
val serverPref = ListPreference(screen.context).apply {
key = "preferred_server"
title = "Preferred server"
entries = arrayOf("StreamZ", "VOE", "StreamZ Sub-Ita", "VOE Sub-Ita")
entryValues = arrayOf("StreamZ", "VOE", "StreamZ Sub-Ita", "VOE Sub-Ita")
setDefaultValue("StreamZ")
ListPreference(screen.context).apply {
key = PREF_SERVER_KEY
title = PREF_SERVER_TITLE
entries = PREF_SERVER_ENTRIES
entryValues = PREF_SERVER_VALUES
setDefaultValue(PREF_SERVER_DEFAULT)
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
@ -421,9 +276,27 @@ class Toonitalia : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
}.also(screen::addPreference)
}
screen.addPreference(videoQualityPref)
screen.addPreference(serverPref)
// ============================= Utilities ==============================
private fun bypassUprot(url: String): String? =
client.newCall(GET(url, headers)).execute()
.use { it.asJsoup() }
.selectFirst("a:has(button.button.is-info)")
?.attr("href")
companion object {
private const val PREF_QUALITY_KEY = "preferred_quality"
private const val PREF_QUALITY_TITLE = "Preferred quality"
private const val PREF_QUALITY_DEFAULT = "1080"
private val PREF_QUALITY_ENTRIES = arrayOf("1080p", "720p", "480p", "360p", "240p", "80p")
private val PREF_QUALITY_VALUES = arrayOf("1080", "720", "480", "360", "240", "80")
private const val PREF_SERVER_KEY = "preferred_server"
private const val PREF_SERVER_TITLE = "Preferred server"
private const val PREF_SERVER_DEFAULT = "StreamZ"
private val PREF_SERVER_ENTRIES = arrayOf("StreamZ", "VOE", "StreamZ Sub-Ita", "VOE Sub-Ita", "MaxStream", "StreamTape")
private val PREF_SERVER_VALUES = PREF_SERVER_ENTRIES
}
}

View File

@ -0,0 +1,67 @@
package eu.kanade.tachiyomi.animeextension.it.toonitalia.extractors
import dev.datlag.jsunpacker.JsUnpacker
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.Headers
import okhttp3.OkHttpClient
class MaxStreamExtractor(private val client: OkHttpClient, private val headers: Headers) {
private val playlistUtils by lazy { PlaylistUtils(client, headers) }
fun videosFromUrl(url: String): List<Video> {
val doc = client.newCall(GET(url, headers)).execute()
.use { it.asJsoup() }
val location = doc.location()
if (location.contains("/dd/")) return videosFromCast(location.replace("/dd/", "/cast3/"))
val scripts = doc.select(SCRIPT_SELECTOR).ifEmpty {
return emptyList()
}
val playlists = scripts.mapNotNull {
JsUnpacker.unpackAndCombine(it.data())
?.substringAfter("src:\"", "")
?.substringBefore('"', "")
?.takeIf(String::isNotBlank)
}
return playlists.flatMap { link ->
playlistUtils.extractFromHls(link, location, videoNameGen = { "MaxStream - $it" })
}
}
private fun videosFromCast(url: String): List<Video> {
val script = client.newCall(GET(url, headers)).execute()
.use { it.asJsoup() }
.selectFirst("script:containsData(document.write)")
?.data()
?: return emptyList()
val numberList = NUMBER_LIST_REGEX.find(script)?.groupValues?.last()
?.split(", ")
?.mapNotNull(String::toIntOrNull)
?: return emptyList()
val offset = numberList.first() - 32
val decodedData = numberList.joinToString("") {
Char(it - offset).toString()
}.trim()
val newHeaders = headers.newBuilder().set("Referer", url).build()
val newUrl = decodedData.substringAfter("get('").substringBefore("'")
val docBody = client.newCall(GET(newUrl, newHeaders)).execute()
.use { it.body.string() }
val videoUrl = docBody.substringAfter(".cast('").substringBefore("'")
return listOf(Video(videoUrl, "MaxStream CAST Scarica", videoUrl, newHeaders))
}
companion object {
private const val SCRIPT_SELECTOR = "script:containsData(eval):containsData(m3u8)"
private val NUMBER_LIST_REGEX by lazy { Regex("\\[(.*)\\]") }
}
}