fix(en/putlocker): Update episode list extraction and refactor (#2798)

This commit is contained in:
Samfun75 2024-01-20 20:28:36 +03:00 committed by GitHub
parent 2392762c40
commit 255bf2d6a6
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
4 changed files with 280 additions and 232 deletions

View File

@ -1,7 +1,11 @@
ext { ext {
extName = 'PutLocker' extName = 'PutLocker'
extClass = '.PutLocker' extClass = '.PutLocker'
extVersionCode = 3 extVersionCode = 4
} }
apply from: "$rootDir/common.gradle" apply from: "$rootDir/common.gradle"
dependencies {
implementation project(':lib-playlist-utils')
}

View File

@ -4,25 +4,22 @@ import android.app.Application
import android.content.SharedPreferences import android.content.SharedPreferences
import androidx.preference.ListPreference import androidx.preference.ListPreference
import androidx.preference.PreferenceScreen import androidx.preference.PreferenceScreen
import eu.kanade.tachiyomi.animeextension.en.putlocker.extractors.PutServerExtractor
import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource import eu.kanade.tachiyomi.animesource.ConfigurableAnimeSource
import eu.kanade.tachiyomi.animesource.model.AnimeFilterList import eu.kanade.tachiyomi.animesource.model.AnimeFilterList
import eu.kanade.tachiyomi.animesource.model.SAnime import eu.kanade.tachiyomi.animesource.model.SAnime
import eu.kanade.tachiyomi.animesource.model.SEpisode import eu.kanade.tachiyomi.animesource.model.SEpisode
import eu.kanade.tachiyomi.animesource.model.Track
import eu.kanade.tachiyomi.animesource.model.Video import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource import eu.kanade.tachiyomi.animesource.online.ParsedAnimeHttpSource
import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.parallelCatchingFlatMap import eu.kanade.tachiyomi.util.parallelCatchingFlatMap
import eu.kanade.tachiyomi.util.parallelCatchingFlatMapBlocking
import kotlinx.serialization.ExperimentalSerializationApi import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.Serializable
import kotlinx.serialization.decodeFromString
import kotlinx.serialization.encodeToString import kotlinx.serialization.encodeToString
import kotlinx.serialization.json.Json import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.jsonPrimitive import kotlinx.serialization.json.jsonPrimitive
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.Request import okhttp3.Request
import okhttp3.Response import okhttp3.Response
import org.jsoup.Jsoup import org.jsoup.Jsoup
@ -37,7 +34,7 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
override val name = "PutLocker" override val name = "PutLocker"
override val baseUrl by lazy { preferences.getString("preferred_domain", "https://putlocker.vip")!! } override val baseUrl = "https://ww7.putlocker.vip"
override val lang = "en" override val lang = "en"
@ -49,38 +46,50 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000) Injekt.get<Application>().getSharedPreferences("source_$id", 0x0000)
} }
private val putServerExtractor by lazy { PutServerExtractor(client) }
// ============================== Popular =============================== // ============================== Popular ===============================
override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/putlocker/") override fun popularAnimeRequest(page: Int): Request = GET("$baseUrl/putlocker/")
override fun popularAnimeSelector(): String = "div#movie-featured > div.ml-item" override fun popularAnimeSelector(): String =
"div#tab-movie > div.ml-item, div#tab-tv-show > div.ml-item"
override fun popularAnimeNextPageSelector(): String = "Nothing" override fun popularAnimeNextPageSelector(): String? = null
override fun popularAnimeFromElement(element: Element): SAnime { override fun popularAnimeFromElement(element: Element): SAnime {
return SAnime.create().apply { return SAnime.create().apply {
setUrlWithoutDomain(element.select("a").attr("abs:href")) setUrlWithoutDomain(
thumbnail_url = element.select("a > img").attr("abs:data-original") element.select("div.mli-poster > a")
title = element.select("a > div.mli-info > h2").text() .attr("abs:href"),
)
title = element.select("div.mli-info h3").text()
thumbnail_url = element.select("div.mli-poster > a > img")
.attr("abs:data-original")
} }
} }
// =============================== Latest =============================== // =============================== Latest ===============================
override fun latestUpdatesRequest(page: Int): Request = GET("$baseUrl/filter/$page?genre=all&country=all&types=all&year=all&sort=updated") override fun latestUpdatesRequest(page: Int): Request =
GET("$baseUrl/filter/$page?genre=all&country=all&types=all&year=all&sort=updated")
override fun latestUpdatesSelector(): String = "div.movies-list > div.ml-item" override fun latestUpdatesSelector(): String = "div.movies-list > div.ml-item > div.mli-poster"
override fun latestUpdatesNextPageSelector(): String = "div#pagination li a[title=Last]" override fun latestUpdatesNextPageSelector(): String = "div#pagination li.active ~ li"
override fun latestUpdatesFromElement(element: Element): SAnime = popularAnimeFromElement(element) override fun latestUpdatesFromElement(element: Element): SAnime = popularAnimeFromElement(element)
// =============================== Search =============================== // =============================== Search ===============================
override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request { override fun searchAnimeRequest(page: Int, query: String, filters: AnimeFilterList): Request {
val re = Regex("[^A-Za-z0-9 ]") return "[^A-Za-z0-9 ]".toRegex()
val cleanQuery = re.replace(query, "").replace(" ", "+").lowercase() .replace(query, "")
return GET("$baseUrl/movie/search/$cleanQuery/$page/") .replace(" ", "+")
.lowercase()
.let {
GET("$baseUrl/movie/search/$it/$page/")
}
} }
override fun searchAnimeSelector(): String = latestUpdatesSelector() override fun searchAnimeSelector(): String = latestUpdatesSelector()
@ -91,68 +100,96 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
// =========================== Anime Details ============================ // =========================== Anime Details ============================
override fun animeDetailsParse(document: Document): SAnime { override fun animeDetailsParse(document: Document): SAnime = SAnime.create().apply {
val anime = SAnime.create() document.select("div.mvic-desc").let { descElement ->
val descElement = document.select("div.mvic-desc") val mLeft = descElement
anime.title = descElement.select("h3").text() .select("div.mvic-info > div.mvici-left")
anime.genre = descElement.select("div.mvic-info > div.mvici-left p:contains(Genre) a").joinToString { it.text() } val mRight = descElement
anime.author = document.select("div.mvic-info > div.mvici-left p:contains(Director) a").joinToString { it.text() } .select("div.mvic-info > div.mvici-right")
anime.status = document.select("div.mvic-info > div.mvici-right p:contains(Episode)")?.let {
if (it.text().isNullOrBlank()) SAnime.COMPLETED else SAnime.UNKNOWN
} ?: SAnime.COMPLETED
var description = descElement.select("div.desc").text()?.let { it + "\n" } status = SAnime.COMPLETED
val extraDescription = document.select("div.mvic-info > div.mvici-right") genre = mLeft.select("p:contains(Genre) a").joinToString { it.text() }
extraDescription.select("p:contains(Quality)").text().let { author = mLeft.select("p:contains(Production) a").first()?.text()
description += if (it.isNotBlank()) "\n$it" else "" description = buildString {
appendLine(descElement.select("div.desc").text())
appendLine()
appendLine(mLeft.select("p:contains(Production) a").joinToString { it.text() })
appendLine(mLeft.select("p:contains(Country)").text())
mRight.select("p").mapNotNull { appendLine(it.text()) }
}
} }
extraDescription.select("p:contains(Release)").text().let {
description += if (it.isNotBlank()) "\n$it" else ""
}
extraDescription.select("p:contains(IMDb)").text().let {
description += if (it.isNotBlank()) "\n$it" else ""
}
anime.description = description
return anime
} }
// ============================== Episodes ============================== // ============================== Episodes ==============================
override fun episodeListRequest(anime: SAnime): Request { override fun episodeListRequest(anime: SAnime): Request =
val id = anime.url.split("-").last().replace("/", "") GET("$baseUrl${anime.url}/watching.html")
return GET("$baseUrl/ajax/movie_episodes/$id")
}
override fun episodeListParse(response: Response): List<SEpisode> { override fun episodeListParse(response: Response): List<SEpisode> {
val html = json.decodeFromString<JsonObject>(response.body.string())["html"]!!.jsonPrimitive.content val doc = response.use { it.asJsoup() }
val parsedHtml = Jsoup.parse(JSONUtil.unescape(html)) val (type, mediaId) = doc.selectFirst("script:containsData(total_episode)")
val rawEpisodes = parsedHtml.select("div[id^=sv]").mapNotNull { server -> ?.data()
val linkElement = server.select("div.les-content > a") ?.let {
linkElement.map { epLinkElement -> val t = it
val dataId = epLinkElement.attr("data-id")!! .substringBefore("name:")
val ep = dataId.substringAfter("_").substringBefore("_").toInt() .substringAfter("type:")
val title = if (ep == 0) { .replace(",", "")
"Movie" .trim()
} else { val mId = it
"Episode $ep: " + epLinkElement.attr("title").substringAfter("Episode $ep") .substringAfter("id:")
.replace("-", "").trim() .substringBefore(",")
} .replace("\"", "")
Pair(title, dataId) .trim()
} Pair(t, mId)
}.flatten() } ?: return emptyList()
return rawEpisodes.groupBy { it.second.substringAfter("_").substringBefore("_").toInt() } return when (type) {
.mapNotNull { group -> "1" -> {
SEpisode.create().apply { listOf(
url = EpLinks( SEpisode.create().apply {
ep_num = group.key, url = EpLinks(
ids = group.value.map { it.second }, dataId = "1_full",
).toJson() mediaId = mediaId,
name = group.value.first().first ).toJson()
episode_number = group.key.toFloat() name = "Movie"
} episode_number = 1F
},
)
} }
else -> {
client.newCall(
GET("$baseUrl/ajax/movie/seasons/$mediaId"),
).execute()
.use { it.body.string() }
.parseHtml()
.select("div.dropdown-menu > a")
.mapNotNull { it.attr("data-id") }
.sortedDescending()
.parallelCatchingFlatMapBlocking { season ->
client.newCall(
GET("$baseUrl/ajax/movie/season/episodes/${mediaId}_$season"),
).execute()
.use { it.body.string() }
.parseHtml()
.select("a")
.mapNotNull { elem ->
val dataId = elem.attr("data-id")
val epFloat = dataId
.substringAfter("_")
.toFloatOrNull()
?: 0F
SEpisode.create().apply {
url = EpLinks(
dataId = dataId,
mediaId = mediaId,
).toJson()
name = "Season $season ${elem.text()}"
episode_number = epFloat
}
}.sortedByDescending { it.episode_number }
}
}
}
} }
override fun episodeListSelector(): String = throw UnsupportedOperationException() override fun episodeListSelector(): String = throw UnsupportedOperationException()
@ -162,9 +199,22 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
// ============================ Video Links ============================= // ============================ Video Links =============================
override suspend fun getVideoList(episode: SEpisode): List<Video> { override suspend fun getVideoList(episode: SEpisode): List<Video> {
val urlJson = json.decodeFromString<EpLinks>(episode.url) val media = json.decodeFromString<EpLinks>(episode.url)
val videoList = urlJson.ids.parallelCatchingFlatMap(::extractVideo) return client.newCall(
return videoList.sort() GET("$baseUrl/ajax/movie/episode/servers/${media.mediaId}_${media.dataId}"),
).execute()
.use { it.body.string() }
.parseHtml()
.select("a")
.mapNotNull { elem ->
Triple(
elem.attr("data-name"),
elem.attr("data-id"),
elem.text(),
)
}
.parallelCatchingFlatMap { putServerExtractor.extractVideo(it, baseUrl) }
.sort()
} }
override fun videoFromElement(element: Element): Video = throw UnsupportedOperationException() override fun videoFromElement(element: Element): Video = throw UnsupportedOperationException()
@ -175,118 +225,13 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
// ============================= Utilities ============================== // ============================= Utilities ==============================
private fun extractVideo(dataId: String): List<Video> { private fun EpLinks.toJson(): String = json.encodeToString(this)
val url = "$baseUrl/ajax/movie_embed/$dataId"
val embedResp = client.newCall(GET(url)).execute().body.string()
val embedUrl = json.decodeFromString<JsonObject>(embedResp)["src"]!!.jsonPrimitive.content
val vidReferer = Headers.headersOf("Referer", embedUrl)
val vidResponse = extractVideoEmbed(embedUrl, vidReferer)
if (!vidResponse.startsWith("{\"sources\"")) return emptyList()
val vidJson = json.decodeFromString<Sources>(vidResponse)
val subsList = extractSubs(vidJson.tracks)
val videoList = mutableListOf<Video>() private fun String.parseHtml(): Document =
val serverId = dataId.substringAfterLast("_") json.decodeFromString<JsonObject>(this@parseHtml)["html"]!!
vidJson.sources.map { source -> .jsonPrimitive.content.run {
videoList.addAll(extractVideoLinks(source, vidReferer, subsList, serverId)) Jsoup.parse(JSONUtil.unescape(this@run))
}
if (!vidJson.backupLink.isNullOrBlank()) {
vidJson.backupLink.let { bakUrl ->
val bakReferer = Headers.headersOf("Referer", bakUrl)
val bakResponse = extractVideoEmbed(bakUrl, bakReferer)
if (bakResponse.startsWith("{\"sources\"")) {
val bakJson = json.decodeFromString<Sources>(bakResponse)
val bakSubsList = extractSubs(bakJson.tracks)
val bakserverId = "$serverId - Backup"
bakJson.sources.map { bakSource ->
videoList.addAll(
extractVideoLinks(
bakSource,
bakReferer,
bakSubsList,
bakserverId,
),
)
}
}
} }
}
return videoList
}
private fun extractSubs(tracks: List<SubTrack>?): List<Track> {
val subsList = mutableListOf<Track>()
try {
tracks?.map { sub ->
sub.file.let {
subsList.add(
Track(
sub.file,
sub.label,
),
)
}
}
} catch (_: Error) {}
return subsList
}
private fun extractVideoEmbed(embedUrl: String, vidReferer: Headers): String {
val embedHost = embedUrl.substringBefore("/embed-player")
val referer = Headers.headersOf("Referer", baseUrl)
val playerResp = client.newCall(GET(embedUrl, referer)).execute().asJsoup()
val player = playerResp.select("div#player")
val vidId = "\"" + player.attr("data-id") + "\""
val vidHash = player.attr("data-hash")
val cipher = CryptoAES.encrypt(vidHash, vidId)
val vidUrl = "$embedHost/ajax/getSources/".toHttpUrl().newBuilder()
.addQueryParameter("id", cipher.cipherText)
.addQueryParameter("h", cipher.password)
.addQueryParameter("a", cipher.iv)
.addQueryParameter("t", cipher.salt)
.build().toString()
val resp = client.newCall(GET(vidUrl, vidReferer)).execute()
return resp.body.string()
}
private fun extractVideoLinks(source: VidSource, vidReferer: Headers, subsList: List<Track>, serverId: String): List<Video> {
val videoList = mutableListOf<Video>()
if (source.file.endsWith(".m3u8")) {
val videoLink = source.file
val resp = client.newCall(GET(videoLink, vidReferer)).execute()
val masterPlaylist = resp.body.string()
if (resp.code == 200) {
masterPlaylist.substringAfter("#EXT-X-STREAM-INF:")
.split("#EXT-X-STREAM-INF:").map {
val quality = if (serverId == "1") {
it.substringAfter("NAME=\"").substringBefore("\"") + " - Server $serverId"
} else {
it.substringAfter("RESOLUTION=").split(",")[0].split("\n")[0].substringAfter("x") + "p - Server $serverId"
}
var videoUrl = it.substringAfter("\n").substringBefore("\n")
if (videoUrl.startsWith("https").not()) {
val host = videoLink.substringBefore("/m3u8")
videoUrl = host + videoUrl
}
try {
videoList.add(Video(videoUrl, quality, videoUrl, subtitleTracks = subsList, headers = vidReferer))
} catch (e: Error) {
videoList.add(Video(videoUrl, quality, videoUrl, headers = vidReferer))
}
}
}
} else {
val quality = "${source.label} - Server $serverId (${source.type})"
try {
videoList.add(Video(source.file, quality, source.file, subtitleTracks = subsList, headers = vidReferer))
} catch (e: Error) {
videoList.add(Video(source.file, quality, source.file, headers = vidReferer))
}
}
return videoList
}
override fun List<Video>.sort(): List<Video> { override fun List<Video>.sort(): List<Video> {
val quality = preferences.getString("preferred_quality", null) val quality = preferences.getString("preferred_quality", null)
@ -309,21 +254,6 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
} }
override fun setupPreferenceScreen(screen: PreferenceScreen) { override fun setupPreferenceScreen(screen: PreferenceScreen) {
val domainPref = ListPreference(screen.context).apply {
key = "preferred_domain"
title = "Preferred domain (requires app restart)"
entries = arrayOf("putlocker.vip")
entryValues = arrayOf("https://putlocker.vip")
setDefaultValue("https://putlocker.vip")
summary = "%s"
setOnPreferenceChangeListener { _, newValue ->
val selected = newValue as String
val index = findIndexOfValue(selected)
val entry = entryValues[index] as String
preferences.edit().putString(key, entry).commit()
}
}
val videoQualityPref = ListPreference(screen.context).apply { val videoQualityPref = ListPreference(screen.context).apply {
key = "preferred_quality" key = "preferred_quality"
title = "Preferred quality" title = "Preferred quality"
@ -339,40 +269,6 @@ class PutLocker : ConfigurableAnimeSource, ParsedAnimeHttpSource() {
preferences.edit().putString(key, entry).commit() preferences.edit().putString(key, entry).commit()
} }
} }
screen.addPreference(domainPref)
screen.addPreference(videoQualityPref) screen.addPreference(videoQualityPref)
} }
private fun EpLinks.toJson(): String {
return json.encodeToString(this)
}
@Serializable
data class EpLinks(
val ep_num: Int,
val ids: List<String>,
)
@Serializable
data class VidSource(
val file: String,
val label: String?,
val type: String?,
)
@Serializable
data class SubTrack(
val default: Boolean,
val file: String,
val label: String,
val kind: String,
)
@Serializable
data class Sources(
val sources: List<VidSource>,
val tracks: List<SubTrack>?,
val backupLink: String?,
)
} }

View File

@ -0,0 +1,34 @@
package eu.kanade.tachiyomi.animeextension.en.putlocker
import kotlinx.serialization.Serializable
@Serializable
data class EpLinks(
val dataId: String,
val mediaId: String,
)
@Serializable
data class EpResp(
val status: Boolean,
val src: String,
)
@Serializable
data class VidSource(
val file: String,
val type: String?,
)
@Serializable
data class SubTrack(
val file: String,
val label: String?,
)
@Serializable
data class Sources(
val sources: List<VidSource>,
val tracks: List<SubTrack>?,
val backupLink: String?,
)

View File

@ -0,0 +1,114 @@
package eu.kanade.tachiyomi.animeextension.en.putlocker.extractors
import eu.kanade.tachiyomi.animeextension.en.putlocker.CryptoAES
import eu.kanade.tachiyomi.animeextension.en.putlocker.EpResp
import eu.kanade.tachiyomi.animeextension.en.putlocker.Sources
import eu.kanade.tachiyomi.animeextension.en.putlocker.SubTrack
import eu.kanade.tachiyomi.animeextension.en.putlocker.VidSource
import eu.kanade.tachiyomi.animesource.model.Track
import eu.kanade.tachiyomi.animesource.model.Video
import eu.kanade.tachiyomi.lib.playlistutils.PlaylistUtils
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.util.asJsoup
import eu.kanade.tachiyomi.util.parseAs
import kotlinx.serialization.json.Json
import okhttp3.Headers
import okhttp3.HttpUrl.Companion.toHttpUrl
import okhttp3.OkHttpClient
import uy.kohesive.injekt.injectLazy
class PutServerExtractor(private val client: OkHttpClient) {
private val json: Json by injectLazy()
private val playlistUtils by lazy { PlaylistUtils(client) }
fun extractVideo(videoData: Triple<String, String, String>, baseUrl: String): List<Video> {
val embedUrl = client.newCall(
GET("$baseUrl/ajax/movie/episode/server/sources/${videoData.second}_${videoData.first}"),
).execute()
.parseAs<EpResp>()
.src
val vidReferer = Headers.headersOf("Referer", "$baseUrl/")
val vidResponse = extractVideoEmbed(embedUrl, vidReferer)
if (!vidResponse.contains("sources")) return emptyList()
val vidJson = json.decodeFromString<Sources>(vidResponse)
return vidJson.sources.flatMap { source ->
extractVideoLinks(
source,
"$baseUrl/",
extractSubs(vidJson.tracks),
videoData.third,
)
}.ifEmpty {
if (!vidJson.backupLink.isNullOrBlank()) {
vidJson.backupLink.let { bakUrl ->
val bakResponse = extractVideoEmbed(bakUrl, vidReferer)
if (bakResponse.contains("sources")) {
val bakJson = json.decodeFromString<Sources>(bakResponse)
bakJson.sources.flatMap { bakSource ->
extractVideoLinks(
bakSource,
"$baseUrl/",
extractSubs(bakJson.tracks),
"${videoData.third} - Backup",
)
}
} else {
emptyList()
}
}
} else {
emptyList()
}
}
}
private fun extractSubs(tracks: List<SubTrack>?): List<Track> {
return tracks?.mapNotNull { sub ->
Track(
sub.file,
sub.label ?: return@mapNotNull null,
)
} ?: emptyList()
}
private fun extractVideoEmbed(embedUrl: String, referer: Headers): String {
val embedHost = embedUrl.substringBefore("/embed-player")
val playerResp = client.newCall(GET(embedUrl, referer)).execute().asJsoup()
val player = playerResp.select("div#player")
val vidId = "\"" + player.attr("data-id") + "\""
val vidHash = player.attr("data-hash")
val cipher = CryptoAES.encrypt(vidHash, vidId)
val vidUrl = "$embedHost/ajax/getSources/".toHttpUrl().newBuilder()
.addQueryParameter("id", cipher.cipherText)
.addQueryParameter("h", cipher.password)
.addQueryParameter("a", cipher.iv)
.addQueryParameter("t", cipher.salt)
.build().toString()
return client.newCall(GET(vidUrl, referer)).execute().use { it.body.string() }
}
private fun extractVideoLinks(source: VidSource, vidReferer: String, subsList: List<Track>, serverId: String): List<Video> {
return if (source.file.endsWith(".m3u8")) {
playlistUtils.extractFromHls(
source.file,
vidReferer,
videoNameGen = { q -> "$serverId - $q" },
subtitleList = subsList,
)
} else {
listOf(
Video(
source.file,
"$serverId (${source.type})",
source.file,
subtitleTracks = subsList,
),
)
}
}
}