feat(pl/desuonline): Convert to multisrc theme and fix video extractors (#2039)

This commit is contained in:
Secozzi
2023-08-13 22:11:16 +02:00
committed by GitHub
parent dc0c125749
commit 5ab164e537
14 changed files with 299 additions and 252 deletions

View File

@ -0,0 +1,4 @@
dependencies {
implementation(project(':lib-okru-extractor'))
implementation(project(':lib-sibnet-extractor'))
}

View File

Before

Width:  |  Height:  |  Size: 2.8 KiB

After

Width:  |  Height:  |  Size: 2.8 KiB

View File

Before

Width:  |  Height:  |  Size: 1.5 KiB

After

Width:  |  Height:  |  Size: 1.5 KiB

View File

Before

Width:  |  Height:  |  Size: 4.6 KiB

After

Width:  |  Height:  |  Size: 4.6 KiB

View File

Before

Width:  |  Height:  |  Size: 8.9 KiB

After

Width:  |  Height:  |  Size: 8.9 KiB

View File

Before

Width:  |  Height:  |  Size: 15 KiB

After

Width:  |  Height:  |  Size: 15 KiB

View File

Before

Width:  |  Height:  |  Size: 126 KiB

After

Width:  |  Height:  |  Size: 126 KiB

View File

@ -0,0 +1,86 @@
package eu.kanade.tachiyomi.animeextension.pl.desuonline
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors.CDAExtractor
import eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors.GoogleDriveExtractor
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.lib.okruextractor.OkruExtractor
import eu.kanade.tachiyomi.lib.sibnetextractor.SibnetExtractor
import eu.kanade.tachiyomi.multisrc.animestream.AnimeStream
import okhttp3.Response
import java.text.SimpleDateFormat
import java.util.Locale
class DesuOnline : AnimeStream(
"pl",
"desu-online",
"https://desu-online.pl",
) {
override val dateFormatter by lazy {
SimpleDateFormat("d MMMM, yyyy", Locale("pl", "PL"))
}
private val prefServerKey = "preferred_server"
private val prefServerDefault = "CDA"
// ============================ Video Links =============================
override fun videoListParse(response: Response): List<Video> =
super.videoListParse(response).ifEmpty { throw Exception("Failed to fetch videos") }
private val okruExtractor by lazy { OkruExtractor(client) }
private val cdaExtractor by lazy { CDAExtractor(client, headers, "$baseUrl/") }
private val sibnetExtractor by lazy { SibnetExtractor(client) }
private val gdriveExtractor by lazy { GoogleDriveExtractor(client, headers) }
override fun getVideoList(url: String, name: String): List<Video> {
return when {
url.contains("ok.ru") -> okruExtractor.videosFromUrl(url, name)
url.contains("cda.pl") -> cdaExtractor.videosFromUrl(url, name)
url.contains("sibnet") -> sibnetExtractor.videosFromUrl(url, prefix = "$name - ")
url.contains("drive.google.com") -> {
val id = Regex("[\\w-]{28,}").find(url)?.groupValues?.get(0) ?: return emptyList()
gdriveExtractor.videosFromUrl("https://drive.google.com/uc?id=$id", videoName = name)
}
else -> emptyList()
}
}
// ============================= Utilities ==============================
override fun List<Video>.sort(): List<Video> {
val quality = preferences.getString(videoSortPrefKey, videoSortPrefDefault)!!
val server = preferences.getString(prefServerKey, prefServerDefault)!!
return sortedWith(
compareBy(
{ it.quality.contains(server, true) },
{ it.quality.contains(quality) },
{ Regex("""(\d+)p""").find(it.quality)?.groupValues?.get(1)?.toIntOrNull() ?: 0 },
),
).reversed()
}
// ============================== Settings ==============================
override fun setupPreferenceScreen(screen: PreferenceScreen) {
super.setupPreferenceScreen(screen) // Quality preferences
ListPreference(screen.context).apply {
key = prefServerKey
title = "Preferred server"
entries = arrayOf("CDA", "Sibnet", "Google Drive", "ok.ru")
entryValues = arrayOf("CDA", "sibnet", "gd", "okru")
setDefaultValue(prefServerDefault)
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
val selected = newValue as String
val index = findIndexOfValue(selected)
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}.also(screen::addPreference)
}
}

View File

@ -0,0 +1,110 @@
package eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.Serializable
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.add
import kotlinx.serialization.json.buildJsonObject
import kotlinx.serialization.json.put
import kotlinx.serialization.json.putJsonArray
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import uy.kohesive.injekt.injectLazy
class CDAExtractor(private val client: OkHttpClient, private val headers: Headers, private val referer: String) {
private val json: Json by injectLazy()
fun videosFromUrl(url: String, name: String): List<Video> {
val urlHost = url.toHttpUrl().host
val docHeaders = headers.newBuilder().apply {
add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
add("Host", urlHost)
add("Referer", referer)
}.build()
val doc = client.newCall(
GET(url, headers = docHeaders),
).execute().asJsoup()
val playerData = doc.selectFirst("div[id~=mediaplayer][player_data]")
?.attr("player_data")
?.let { json.decodeFromString<PlayerData>(it) }
?: return emptyList()
val timestamp = playerData.api.ts.substringBefore("_")
val videoData = playerData.video
var idCounter = 1
return videoData.qualities.map { (quality, qualityId) ->
val postBody = json.encodeToString(
buildJsonObject {
put("id", idCounter)
put("jsonrpc", "2.0")
put("method", "videoGetLink")
putJsonArray("params") {
add(url.toHttpUrl().pathSegments.last())
add(qualityId)
add(timestamp.toInt())
add(videoData.hash2)
}
},
).toRequestBody("application/json; charset=utf-8".toMediaType())
val postHeaders = headers.newBuilder().apply {
add("Accept", "application/json, text/javascript, */*; q=0.01")
add("Host", "www.cda.pl")
add("Origin", "https://$urlHost")
add("Referer", url)
}.build()
val videoUrl = client.newCall(
POST("https://www.cda.pl/", headers = postHeaders, body = postBody),
).execute().parseAs<PostResponse>().result.resp
idCounter++
Video(videoUrl, "$name - $quality", videoUrl)
}
}
@Serializable
data class PlayerData(
val api: PlayerApi,
val video: PlayerVideoData,
) {
@Serializable
data class PlayerApi(
val ts: String,
)
@Serializable
data class PlayerVideoData(
val hash2: String,
val qualities: Map<String, String>,
)
}
@Serializable
data class PostResponse(
val result: PostResult,
) {
@Serializable
data class PostResult(
val resp: String,
)
}
private inline fun <reified T> Response.parseAs(transform: (String) -> String = { it }): T {
val responseBody = use { transform(it.body.string()) }
return json.decodeFromString(responseBody)
}
}

View File

@ -0,0 +1,98 @@
package eu.kanade.tachiyomi.animeextension.pl.desuonline.extractors
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.OkHttpClient
import okhttp3.RequestBody.Companion.toRequestBody
class GoogleDriveExtractor(private val client: OkHttpClient, private val headers: Headers) {
// Needs to be the form of `https://drive.google.com/uc?id=GOOGLEDRIVEITEMID`
fun videosFromUrl(itemUrl: String, videoName: String = "Video"): List<Video> {
val itemHeaders = headers.newBuilder()
.add("Accept", "*/*")
.add("Accept-Language", "en-US,en;q=0.5")
.add("Connection", "keep-alive")
.add("Cookie", getCookie(itemUrl))
.add("Host", "drive.google.com")
.build()
val itemResponse = client.newCall(
GET(itemUrl, headers = itemHeaders),
).execute()
val noRedirectClient = OkHttpClient().newBuilder().followRedirects(false).build()
val document = itemResponse.asJsoup()
val itemSize = document.selectFirst("span.uc-name-size")?.let {
" ${it.ownText().trim()} "
} ?: ""
val url = document.selectFirst("form#download-form")?.attr("action") ?: return emptyList()
val redirectHeaders = headers.newBuilder()
.add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
.add("Connection", "keep-alive")
.add("Content-Length", "0")
.add("Content-Type", "application/x-www-form-urlencoded")
.add("Cookie", getCookie(url))
.add("Host", "drive.google.com")
.add("Origin", "https://drive.google.com")
.add("Referer", url.substringBeforeLast("&at="))
.build()
val response = noRedirectClient.newCall(
POST(url, headers = redirectHeaders, body = "".toRequestBody("application/x-www-form-urlencoded".toMediaType())),
).execute()
val redirected = response.headers["location"] ?: return listOf(Video(url, videoName + itemSize, url))
val redirectedHeaders = headers.newBuilder()
.add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
.add("Connection", "keep-alive")
.add("Host", redirected.toHttpUrl().host)
.add("Referer", "https://drive.google.com/")
.build()
val redirectedResponseHeaders = noRedirectClient.newCall(
GET(redirected, headers = redirectedHeaders),
).execute().headers
val authCookie = redirectedResponseHeaders.firstOrNull {
it.first == "set-cookie" && it.second.startsWith("AUTH_")
}?.second?.substringBefore(";") ?: return listOf(Video(url, videoName + itemSize, url))
val newRedirected = redirectedResponseHeaders["location"] ?: return listOf(Video(redirected, videoName + itemSize, redirected))
val googleDriveRedirectHeaders = headers.newBuilder()
.add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
.add("Connection", "keep-alive")
.add("Cookie", getCookie(newRedirected))
.add("Host", "drive.google.com")
.add("Referer", "https://drive.google.com/")
.build()
val googleDriveRedirectUrl = noRedirectClient.newCall(
GET(newRedirected, headers = googleDriveRedirectHeaders),
).execute().headers["location"]!!
val videoHeaders = headers.newBuilder()
.add("Accept", "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8")
.add("Connection", "keep-alive")
.add("Cookie", authCookie)
.add("Host", googleDriveRedirectUrl.toHttpUrl().host)
.add("Referer", "https://drive.google.com/")
.build()
return listOf(
Video(googleDriveRedirectUrl, videoName + itemSize, googleDriveRedirectUrl, headers = videoHeaders),
)
}
private fun getCookie(url: String): String {
val cookieList = client.cookieJar.loadForRequest(url.toHttpUrl())
return if (cookieList.isNotEmpty()) {
cookieList.joinToString("; ") { "${it.name}=${it.value}" }
} else {
""
}
}
}

View File

@ -16,6 +16,7 @@ class AnimeStreamGenerator : ThemeSourceGenerator {
SingleLang("Animenosub", "https://animenosub.com", "en", isNsfw = true),
SingleLang("AnimeTitans", "https://animetitans.com", "ar", isNsfw = false, overrideVersionCode = 11),
SingleLang("AnimeXin", "https://animexin.vip", "all", isNsfw = false, overrideVersionCode = 4),
SingleLang("desu-online", "https://desu-online.pl", "pl", className = "DesuOnline", isNsfw = false),
SingleLang("Hstream", "https://hstream.moe", "en", isNsfw = true, overrideVersionCode = 3),
SingleLang("LMAnime", "https://lmanime.com", "all", isNsfw = false, overrideVersionCode = 2),
SingleLang("MiniOppai", "https://minioppai.org", "id", isNsfw = true, overrideVersionCode = 2),

View File

@ -1,2 +0,0 @@
<?xml version="1.0" encoding="utf-8"?>
<manifest />

View File

@ -1,12 +0,0 @@
apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
ext {
extName = 'desu-online'
pkgNameSuffix = 'pl.desuonline'
extClass = '.DesuOnline'
extVersionCode = 1
libVersion = '13'
}
apply from: "$rootDir/common.gradle"

View File

@ -1,238 +0,0 @@
package eu.kanade.tachiyomi.animeextension.pl.desuonline
import android.app.Application
import android.content.SharedPreferences
import android.util.Base64
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
import eu.kanade.tachiyomi.animesource.model.SAnime
import eu.kanade.tachiyomi.animesource.model.SEpisode
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.POST
import eu.kanade.tachiyomi.util.asJsoup
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.MediaType.Companion.toMediaType
import okhttp3.RequestBody
import okhttp3.RequestBody.Companion.toRequestBody
import okhttp3.Response
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import uy.kohesive.injekt.Injekt
import uy.kohesive.injekt.api.get
import java.text.SimpleDateFormat
import java.util.Locale
class DesuOnline : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override val name = "desu-online"
override val baseUrl = "https://desu-online.pl"
override val lang = "pl"
override val supportsLatest = true
override val client = network.cloudflareClient
private val json = Json {
ignoreUnknownKeys = true
}
private val preferences: SharedPreferences by lazy {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
}
// ============================== Popular ===============================
override fun popularAnimeRequest(page: Int) = GET("$baseUrl/anime/?page=$page&order=popular")
override fun popularAnimeSelector() = "div.listupd div.bsx > a"
override fun popularAnimeNextPageSelector() = "div.pagination > a.next, div.hpage > a.r"
override fun popularAnimeFromElement(element: Element): SAnime {
val animeTitle = element.select("div.tt > h2").text().trim()
val img = element.select("div.limit > img")
return SAnime.create().apply {
setUrlWithoutDomain(element.attr("href"))
title = animeTitle
thumbnail_url = img.attr("data-src")
}
}
// =========================== Anime Details ============================
override fun animeDetailsParse(document: Document): SAnime {
val img = document.select("div.thumb > img").ifEmpty { null }
val studio = document.select("div.info-content > div.spe > span:contains(Studio:)").ifEmpty { null }
val statusSpan = document.select("div.info-content > div.spe > span:contains(Status:)").ifEmpty { null }
val desc = document.select("div[itemprop=description] > p:last-child").ifEmpty { null }
val director = document.select("div.info-content > div.spe > span:contains(Reżyser:)").ifEmpty { null }
val genres = document.select("div.genxed > a")
return SAnime.create().apply {
title = document.select("h1.entry-title").text()
thumbnail_url = img?.attr("data-src")
author = studio?.text()?.substringAfter("Studio: ")
status = parseStatus(statusSpan?.text()?.substringAfter("Status: "))
description = desc?.text()?.trim()
artist = director?.text()?.substringAfter("Reżyser: ")
genre = genres.joinToString { it.text() }
}
}
// ============================== Episodes ==============================
override fun episodeListSelector() = "div.eplister > ul > li"
override fun episodeFromElement(element: Element): SEpisode {
val a = element.select("a")
val epNum = a.select("div.epl-num").text()
val epTitle = a.select("div.epl-title").text()
val date = a.select("div.epl-date").text()
return SEpisode.create().apply {
setUrlWithoutDomain(a.attr("href"))
name = "Odcinek $epNum: $epTitle"
date_upload = parseDate(date)
episode_number = epNum.substringBefore(" ").toFloatOrNull() ?: 0F
}
}
// ============================ Video Links =============================
override fun videoListParse(response: Response): List<Video> {
val videoList = mutableListOf<Video>()
val document = response.asJsoup()
document.select("select.mirror > option").filter { it ->
it.text().contains("CDA")
}.map {
val mirror = it.text().trim()
val iframe = String(Base64.decode(it.attr("value"), Base64.DEFAULT))
val src = iframe.substringAfter("src=\"").substringBefore("\"")
val videoId = src.toHttpUrl().encodedPathSegments.last()
val playerDataHtml = client.newCall(GET(src)).execute()
.asJsoup().select("div[id=mediaplayer$videoId]")
.attr("player_data")
val playerData = json.decodeFromString<JsonObject>(playerDataHtml)
val timeStamp = playerData["api"]!!.jsonObject["ts"]!!.jsonPrimitive.content
.substringBefore("_")
val videoData = playerData["video"]!!.jsonObject
val hash = videoData["hash2"]!!.jsonPrimitive.content
val qualities = videoData["qualities"]!!.jsonObject
qualities.keys.reversed().map { quality ->
val qualityId = qualities[quality]!!.jsonPrimitive.content
val body = cdaBody(videoId, qualityId, timeStamp, hash)
val videoResponse = json.decodeFromString<JsonObject>(
client.newCall(POST("https://www.cda.pl/", body = body))
.execute().body.string(),
)
val videoUrl = videoResponse["result"]!!.jsonObject["resp"]!!.jsonPrimitive.content
videoList.add(Video(videoUrl, "$mirror: $quality", videoUrl))
}
}
return videoList
}
override fun videoFromElement(element: Element): Video = throw Exception("not used")
override fun videoListSelector(): String = throw Exception("not used")
override fun videoUrlParse(document: Document): String = throw Exception("not used")
// =============================== Search ===============================
override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList) =
GET("$baseUrl/anime/?page=$page&s=$query")
override fun searchAnimeSelector() = popularAnimeSelector()
override fun searchAnimeNextPageSelector() = popularAnimeNextPageSelector()
override fun searchAnimeFromElement(element: Element) = popularAnimeFromElement(element)
// =============================== Latest ===============================
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/anime/?page=$page&order=latest")
override fun latestUpdatesSelector() = popularAnimeSelector()
override fun latestUpdatesNextPageSelector() = popularAnimeNextPageSelector()
override fun latestUpdatesFromElement(element: Element) = popularAnimeFromElement(element)
// =============================== Preferences ===============================
override fun setupPreferenceScreen(screen: PreferenceScreen) {
val videoQualityPref = ListPreference(screen.context).apply {
key = "preferred_quality"
title = "Preferred quality"
entries = arrayOf("1080p", "720p", "480p", "360p")
entryValues = arrayOf("1080", "720", "480", "360")
setDefaultValue("1080")
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
val selected = newValue as String
val index = findIndexOfValue(selected)
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
screen.addPreference(videoQualityPref)
}
override fun List<Video>.sort(): List<Video> {
val quality = preferences.getString("preferred_quality", "1080")
if (quality != null) {
val newList = mutableListOf<Video>()
var preferred = 0
for (video in this) {
if (video.quality.contains(quality)) {
newList.add(preferred, video)
preferred++
} else {
newList.add(video)
}
}
return newList
}
return this
}
// ============================= Utilities ==============================
private fun parseStatus(statusString: String?): Int {
return when (statusString?.trim()) {
"Zakończony" -> SAnime.COMPLETED
"Emitowany" -> SAnime.ONGOING
else -> SAnime.UNKNOWN
}
}
private fun parseDate(dateStr: String): Long {
return runCatching { DATE_FORMATTER.parse(dateStr)?.time }
.getOrNull() ?: 0L
}
private fun cdaBody(
videoId: String,
qualityId: String,
timeStamp: String,
hash: String,
): RequestBody {
return "{\"jsonrpc\":\"2.0\",\"method\":\"videoGetLink\",\"params\":[\"$videoId\",\"$qualityId\",$timeStamp,\"$hash\",{}],\"id\":4}"
.toRequestBody("application/json".toMediaType())
}
}
private val DATE_FORMATTER by lazy {
SimpleDateFormat("d MMM, yyyy", Locale("pl"))
}