fix(en/putlocker): Update episode list extraction and refactor (#2798)

This commit is contained in:
Samfun75 2024-01-20 20:28:36 +03:00 committed by GitHub
parent 2392762c40
commit 255bf2d6a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 280 additions and 232 deletions

View File

@ -1,7 +1,11 @@
ext {
extName = 'PutLocker'
extClass = '.PutLocker'
extVersionCode = 3
extVersionCode = 4
}
apply from: "$rootDir/common.gradle"
dependencies {
implementation project(':lib-playlist-utils')
}

View File

@ -4,25 +4,22 @@ import android.app.Application
import android.content.SharedPreferences
import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.animeextension.en.putlocker.extractors.PutServerExtractor
import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
import eu.kanade.tachiyomi.animesource.model.SAnime
import eu.kanade.tachiyomi.animesource.model.SEpisode
import eu.kanade.tachiyomi.animesource.model.Track
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.parallelCatchingFlatMap
import eu.kanade.tachiyomi.util.parallelCatchingFlatMapBlocking
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.jsonPrimitive
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Request
import okhttp3.Response
import org.jsoup.Jsoup
@ -37,7 +34,7 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override val name = "PutLocker"
override val baseUrl by lazy { preferences.getString("preferred_domain", "https://putlocker.vip")!! }
override val baseUrl = "https://ww7.putlocker.vip"
override val lang = "en"
@ -49,38 +46,50 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
}
private val putServerExtractor by lazy { PutServerExtractor(client) }
// ============================== Popular ===============================
override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/putlocker/")
override fun popularAnimeSelector(): String = "div#movie-featured > div.ml-item"
override fun popularAnimeSelector(): String =
"div#tab-movie > div.ml-item, div#tab-tv-show > div.ml-item"
override fun popularAnimeNextPageSelector(): String = "Nothing"
override fun popularAnimeNextPageSelector(): String? = null
override fun popularAnimeFromElement(element: Element): SAnime {
return SAnime.create().apply {
setUrlWithoutDomain(element.select("a").attr("abs:href"))
thumbnail_url = element.select("a > img").attr("abs:data-original")
title = element.select("a > div.mli-info > h2").text()
setUrlWithoutDomain(
element.select("div.mli-poster > a")
.attr("abs:href"),
)
title = element.select("div.mli-info h3").text()
thumbnail_url = element.select("div.mli-poster > a > img")
.attr("abs:data-original")
}
}
// =============================== Latest ===============================
override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/filter/$page?genre=all&country=all&types=all&year=all&sort=updated")
override fun latestUpdatesRequest(page: Int): Request =
GET("$baseUrl/filter/$page?genre=all&country=all&types=all&year=all&sort=updated")
override fun latestUpdatesSelector(): String = "div.movies-list > div.ml-item"
override fun latestUpdatesSelector(): String = "div.movies-list > div.ml-item > div.mli-poster"
override fun latestUpdatesNextPageSelector(): String = "div#pagination li a[title=Last]"
override fun latestUpdatesNextPageSelector(): String = "div#pagination li.active ~ li"
override fun latestUpdatesFromElement(element: Element): SAnime = popularAnimeFromElement(element)
// =============================== Search ===============================
override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
val re = Regex("[^A-Za-z0-9 ]")
val cleanQuery = re.replace(query, "").replace(" ", "+").lowercase()
return GET("$baseUrl/movie/search/$cleanQuery/$page/")
return "[^A-Za-z0-9 ]".toRegex()
.replace(query, "")
.replace(" ", "+")
.lowercase()
.let {
GET("$baseUrl/movie/search/$it/$page/")
}
}
override fun searchAnimeSelector(): String = latestUpdatesSelector()
@ -91,68 +100,96 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
// =========================== Anime Details ============================
override fun animeDetailsParse(document: Document): SAnime {
val anime = SAnime.create()
val descElement = document.select("div.mvic-desc")
anime.title = descElement.select("h3").text()
anime.genre = descElement.select("div.mvic-info > div.mvici-left p:contains(Genre) a").joinToString { it.text() }
anime.author = document.select("div.mvic-info > div.mvici-left p:contains(Director) a").joinToString { it.text() }
anime.status = document.select("div.mvic-info > div.mvici-right p:contains(Episode)")?.let {
if (it.text().isNullOrBlank()) SAnime.COMPLETED else SAnime.UNKNOWN
} ?: SAnime.COMPLETED
override fun animeDetailsParse(document: Document): SAnime = SAnime.create().apply {
document.select("div.mvic-desc").let { descElement ->
val mLeft = descElement
.select("div.mvic-info > div.mvici-left")
val mRight = descElement
.select("div.mvic-info > div.mvici-right")
var description = descElement.select("div.desc").text()?.let { it + "\n" }
val extraDescription = document.select("div.mvic-info > div.mvici-right")
extraDescription.select("p:contains(Quality)").text().let {
description += if (it.isNotBlank()) "\n$it" else ""
status = SAnime.COMPLETED
genre = mLeft.select("p:contains(Genre) a").joinToString { it.text() }
author = mLeft.select("p:contains(Production) a").first()?.text()
description = buildString {
appendLine(descElement.select("div.desc").text())
appendLine()
appendLine(mLeft.select("p:contains(Production) a").joinToString { it.text() })
appendLine(mLeft.select("p:contains(Country)").text())
mRight.select("p").mapNotNull { appendLine(it.text()) }
}
}
extraDescription.select("p:contains(Release)").text().let {
description += if (it.isNotBlank()) "\n$it" else ""
}
extraDescription.select("p:contains(IMDb)").text().let {
description += if (it.isNotBlank()) "\n$it" else ""
}
anime.description = description
return anime
}
// ============================== Episodes ==============================
override fun episodeListRequest(anime: SAnime): Request {
val id = anime.url.split("-").last().replace("/", "")
return GET("$baseUrl/ajax/movie_episodes/$id")
}
override fun episodeListRequest(anime: SAnime): Request =
GET("$baseUrl${anime.url}/watching.html")
override fun episodeListParse(response: Response): List<SEpisode> {
val html = json.decodeFromString<JsonObject>(response.body.string())["html"]!!.jsonPrimitive.content
val parsedHtml = Jsoup.parse(JSONUtil.unescape(html))
val rawEpisodes = parsedHtml.select("div[id^=sv]").mapNotNull { server ->
val linkElement = server.select("div.les-content > a")
linkElement.map { epLinkElement ->
val dataId = epLinkElement.attr("data-id")!!
val ep = dataId.substringAfter("_").substringBefore("_").toInt()
val title = if (ep == 0) {
"Movie"
} else {
"Episode $ep: " + epLinkElement.attr("title").substringAfter("Episode $ep")
.replace("-", "").trim()
}
Pair(title, dataId)
}
}.flatten()
val doc = response.use { it.asJsoup() }
val (type, mediaId) = doc.selectFirst("script:containsData(total_episode)")
?.data()
?.let {
val t = it
.substringBefore("name:")
.substringAfter("type:")
.replace(",", "")
.trim()
val mId = it
.substringAfter("id:")
.substringBefore(",")
.replace("\"", "")
.trim()
Pair(t, mId)
} ?: return emptyList()
return rawEpisodes.groupBy { it.second.substringAfter("_").substringBefore("_").toInt() }
.mapNotNull { group ->
SEpisode.create().apply {
url = EpLinks(
ep_num = group.key,
ids = group.value.map { it.second },
).toJson()
name = group.value.first().first
episode_number = group.key.toFloat()
}
return when (type) {
"1" -> {
listOf(
SEpisode.create().apply {
url = EpLinks(
dataId = "1_full",
mediaId = mediaId,
).toJson()
name = "Movie"
episode_number = 1F
},
)
}
else -> {
client.newCall(
GET("$baseUrl/ajax/movie/seasons/$mediaId"),
).execute()
.use { it.body.string() }
.parseHtml()
.select("div.dropdown-menu > a")
.mapNotNull { it.attr("data-id") }
.sortedDescending()
.parallelCatchingFlatMapBlocking { season ->
client.newCall(
GET("$baseUrl/ajax/movie/season/episodes/${mediaId}_$season"),
).execute()
.use { it.body.string() }
.parseHtml()
.select("a")
.mapNotNull { elem ->
val dataId = elem.attr("data-id")
val epFloat = dataId
.substringAfter("_")
.toFloatOrNull()
?: 0F
SEpisode.create().apply {
url = EpLinks(
dataId = dataId,
mediaId = mediaId,
).toJson()
name = "Season $season ${elem.text()}"
episode_number = epFloat
}
}.sortedByDescending { it.episode_number }
}
}
}
}
override fun episodeListSelector(): String = throw UnsupportedOperationException()
@ -162,9 +199,22 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
// ============================ Video Links =============================
override suspend fun getVideoList(episode: SEpisode): List<Video> {
val urlJson = json.decodeFromString<EpLinks>(episode.url)
val videoList = urlJson.ids.parallelCatchingFlatMap(::extractVideo)
return videoList.sort()
val media = json.decodeFromString<EpLinks>(episode.url)
return client.newCall(
GET("$baseUrl/ajax/movie/episode/servers/${media.mediaId}_${media.dataId}"),
).execute()
.use { it.body.string() }
.parseHtml()
.select("a")
.mapNotNull { elem ->
Triple(
elem.attr("data-name"),
elem.attr("data-id"),
elem.text(),
)
}
.parallelCatchingFlatMap { putServerExtractor.extractVideo(it, baseUrl) }
.sort()
}
override fun videoFromElement(element: Element): Video = throw UnsupportedOperationException()
@ -175,118 +225,13 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
// ============================= Utilities ==============================
private fun extractVideo(dataId: String): List<Video> {
val url = "$baseUrl/ajax/movie_embed/$dataId"
val embedResp = client.newCall(GET(url)).execute().body.string()
val embedUrl = json.decodeFromString<JsonObject>(embedResp)["src"]!!.jsonPrimitive.content
val vidReferer = Headers.headersOf("Referer", embedUrl)
val vidResponse = extractVideoEmbed(embedUrl, vidReferer)
if (!vidResponse.startsWith("{\"sources\"")) return emptyList()
val vidJson = json.decodeFromString<Sources>(vidResponse)
val subsList = extractSubs(vidJson.tracks)
private fun EpLinks.toJson(): String = json.encodeToString(this)
val videoList = mutableListOf<Video>()
val serverId = dataId.substringAfterLast("_")
vidJson.sources.map { source ->
videoList.addAll(extractVideoLinks(source, vidReferer, subsList, serverId))
}
if (!vidJson.backupLink.isNullOrBlank()) {
vidJson.backupLink.let { bakUrl ->
val bakReferer = Headers.headersOf("Referer", bakUrl)
val bakResponse = extractVideoEmbed(bakUrl, bakReferer)
if (bakResponse.startsWith("{\"sources\"")) {
val bakJson = json.decodeFromString<Sources>(bakResponse)
val bakSubsList = extractSubs(bakJson.tracks)
val bakserverId = "$serverId - Backup"
bakJson.sources.map { bakSource ->
videoList.addAll(
extractVideoLinks(
bakSource,
bakReferer,
bakSubsList,
bakserverId,
),
)
}
}
private fun String.parseHtml(): Document =
json.decodeFromString<JsonObject>(this@parseHtml)["html"]!!
.jsonPrimitive.content.run {
Jsoup.parse(JSONUtil.unescape(this@run))
}
}
return videoList
}
private fun extractSubs(tracks: List<SubTrack>?): List<Track> {
val subsList = mutableListOf<Track>()
try {
tracks?.map { sub ->
sub.file.let {
subsList.add(
Track(
sub.file,
sub.label,
),
)
}
}
} catch (_: Error) {}
return subsList
}
private fun extractVideoEmbed(embedUrl: String, vidReferer: Headers): String {
val embedHost = embedUrl.substringBefore("/embed-player")
val referer = Headers.headersOf("Referer", baseUrl)
val playerResp = client.newCall(GET(embedUrl, referer)).execute().asJsoup()
val player = playerResp.select("div#player")
val vidId = "\"" + player.attr("data-id") + "\""
val vidHash = player.attr("data-hash")
val cipher = CryptoAES.encrypt(vidHash, vidId)
val vidUrl = "$embedHost/ajax/getSources/".toHttpUrl().newBuilder()
.addQueryParameter("id", cipher.cipherText)
.addQueryParameter("h", cipher.password)
.addQueryParameter("a", cipher.iv)
.addQueryParameter("t", cipher.salt)
.build().toString()
val resp = client.newCall(GET(vidUrl, vidReferer)).execute()
return resp.body.string()
}
private fun extractVideoLinks(source: VidSource, vidReferer: Headers, subsList: List<Track>, serverId: String): List<Video> {
val videoList = mutableListOf<Video>()
if (source.file.endsWith(".m3u8")) {
val videoLink = source.file
val resp = client.newCall(GET(videoLink, vidReferer)).execute()
val masterPlaylist = resp.body.string()
if (resp.code == 200) {
masterPlaylist.substringAfter("#EXT-X-STREAM-INF:")
.split("#EXT-X-STREAM-INF:").map {
val quality = if (serverId == "1") {
it.substringAfter("NAME=\"").substringBefore("\"") + " - Server $serverId"
} else {
it.substringAfter("RESOLUTION=").split(",")[0].split("\n")[0].substringAfter("x") + "p - Server $serverId"
}
var videoUrl = it.substringAfter("\n").substringBefore("\n")
if (videoUrl.startsWith("https").not()) {
val host = videoLink.substringBefore("/m3u8")
videoUrl = host + videoUrl
}
try {
videoList.add(Video(videoUrl, quality, videoUrl, subtitleTracks = subsList, headers = vidReferer))
} catch (e: Error) {
videoList.add(Video(videoUrl, quality, videoUrl, headers = vidReferer))
}
}
}
} else {
val quality = "${source.label} - Server $serverId (${source.type})"
try {
videoList.add(Video(source.file, quality, source.file, subtitleTracks = subsList, headers = vidReferer))
} catch (e: Error) {
videoList.add(Video(source.file, quality, source.file, headers = vidReferer))
}
}
return videoList
}
override fun List<Video>.sort(): List<Video> {
val quality = preferences.getString("preferred_quality", null)
@ -309,21 +254,6 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
}
override fun setupPreferenceScreen(screen: PreferenceScreen) {
val domainPref = ListPreference(screen.context).apply {
key = "preferred_domain"
title = "Preferred domain (requires app restart)"
entries = arrayOf("putlocker.vip")
entryValues = arrayOf("https://putlocker.vip")
setDefaultValue("https://putlocker.vip")
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
val selected = newValue as String
val index = findIndexOfValue(selected)
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
val videoQualityPref = ListPreference(screen.context).apply {
key = "preferred_quality"
title = "Preferred quality"
@ -339,40 +269,6 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
preferences.edit().putString(key, entry).commit()
}
}
screen.addPreference(domainPref)
screen.addPreference(videoQualityPref)
}
private fun EpLinks.toJson(): String {
return json.encodeToString(this)
}
@Serializable
data class EpLinks(
val ep_num: Int,
val ids: List<String>,
)
@Serializable
data class VidSource(
val file: String,
val label: String?,
val type: String?,
)
@Serializable
data class SubTrack(
val default: Boolean,
val file: String,
val label: String,
val kind: String,
)
@Serializable
data class Sources(
val sources: List<VidSource>,
val tracks: List<SubTrack>?,
val backupLink: String?,
)
}

View File

@ -0,0 +1,34 @@
package eu.kanade.tachiyomi.animeextension.en.putlocker
import kotlinx.serialization.Serializable
@Serializable
data class EpLinks(
val dataId: String,
val mediaId: String,
)
@Serializable
data class EpResp(
val status: Boolean,
val src: String,
)
@Serializable
data class VidSource(
val file: String,
val type: String?,
)
@Serializable
data class SubTrack(
val file: String,
val label: String?,
)
@Serializable
data class Sources(
val sources: List<VidSource>,
val tracks: List<SubTrack>?,
val backupLink: String?,
)

View File

@ -0,0 +1,114 @@
package eu.kanade.tachiyomi.animeextension.en.putlocker.extractors
import eu.kanade.tachiyomi.animeextension.en.putlocker.CryptoAES
import eu.kanade.tachiyomi.animeextension.en.putlocker.EpResp
import eu.kanade.tachiyomi.animeextension.en.putlocker.Sources
import eu.kanade.tachiyomi.animeextension.en.putlocker.SubTrack
import eu.kanade.tachiyomi.animeextension.en.putlocker.VidSource
import eu.kanade.tachiyomi.animesource.model.Track
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.parseAs
import kotlinx.serialization.json.Json
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import uy.kohesive.injekt.injectLazy
class PutServerExtractor(private val client: OkHttpClient) {
private val json: Json by injectLazy()
private val playlistUtils by lazy { PlaylistUtils(client) }
fun extractVideo(videoData: Triple<String, String, String>, baseUrl: String): List<Video> {
val embedUrl = client.newCall(
GET("$baseUrl/ajax/movie/episode/server/sources/${videoData.second}_${videoData.first}"),
).execute()
.parseAs<EpResp>()
.src
val vidReferer = Headers.headersOf("Referer", "$baseUrl/")
val vidResponse = extractVideoEmbed(embedUrl, vidReferer)
if (!vidResponse.contains("sources")) return emptyList()
val vidJson = json.decodeFromString<Sources>(vidResponse)
return vidJson.sources.flatMap { source ->
extractVideoLinks(
source,
"$baseUrl/",
extractSubs(vidJson.tracks),
videoData.third,
)
}.ifEmpty {
if (!vidJson.backupLink.isNullOrBlank()) {
vidJson.backupLink.let { bakUrl ->
val bakResponse = extractVideoEmbed(bakUrl, vidReferer)
if (bakResponse.contains("sources")) {
val bakJson = json.decodeFromString<Sources>(bakResponse)
bakJson.sources.flatMap { bakSource ->
extractVideoLinks(
bakSource,
"$baseUrl/",
extractSubs(bakJson.tracks),
"${videoData.third} - Backup",
)
}
} else {
emptyList()
}
}
} else {
emptyList()
}
}
}
private fun extractSubs(tracks: List<SubTrack>?): List<Track> {
return tracks?.mapNotNull { sub ->
Track(
sub.file,
sub.label ?: return@mapNotNull null,
)
} ?: emptyList()
}
private fun extractVideoEmbed(embedUrl: String, referer: Headers): String {
val embedHost = embedUrl.substringBefore("/embed-player")
val playerResp = client.newCall(GET(embedUrl, referer)).execute().asJsoup()
val player = playerResp.select("div#player")
val vidId = "\"" + player.attr("data-id") + "\""
val vidHash = player.attr("data-hash")
val cipher = CryptoAES.encrypt(vidHash, vidId)
val vidUrl = "$embedHost/ajax/getSources/".toHttpUrl().newBuilder()
.addQueryParameter("id", cipher.cipherText)
.addQueryParameter("h", cipher.password)
.addQueryParameter("a", cipher.iv)
.addQueryParameter("t", cipher.salt)
.build().toString()
return client.newCall(GET(vidUrl, referer)).execute().use { it.body.string() }
}
private fun extractVideoLinks(source: VidSource, vidReferer: String, subsList: List<Track>, serverId: String): List<Video> {
return if (source.file.endsWith(".m3u8")) {
playlistUtils.extractFromHls(
source.file,
vidReferer,
videoNameGen = { q -> "$serverId - $q" },
subtitleList = subsList,
)
} else {
listOf(
Video(
source.file,
"$serverId (${source.type})",
source.file,
subtitleTracks = subsList,
),
)
}
}
}