Webtoons split (#6444)

* Create Webtoons.kt

* Added Webtoons Generator and translate

* Updated Vesion numbers

* Update WebtoonsTranslateGenerator.kt

* Added Icons and ovverides

* Removed non split files

* Update WebtoonsGenerator.kt

* Added id overrides for a few languages

* Added ID Override for  Indonesian

* Fixed backwards compability

* Fix backward compability
This commit is contained in:
Johannes Joens
2021-04-07 23:52:58 +12:00
committed by GitHub
parent 5cf8547ca9
commit a3b9c284de
23 changed files with 260 additions and 190 deletions

View File

@ -0,0 +1,243 @@
package eu.kanade.tachiyomi.multisrc.webtoons
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.Filter.Header
import eu.kanade.tachiyomi.source.model.Filter.Select
import eu.kanade.tachiyomi.source.model.Filter.Separator
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.Cookie
import okhttp3.CookieJar
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import org.json.JSONObject
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import java.text.SimpleDateFormat
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.model.MangasPage
import java.util.Locale
import java.util.Calendar
open class Webtoons(
override val name: String,
override val baseUrl: String,
override val lang: String,
open val langCode: String = lang,
open val localeForCookie: String = lang,
private val dateFormat: SimpleDateFormat = SimpleDateFormat("MMM d, yyyy", Locale.ENGLISH)
) : ParsedHttpSource() {
override val supportsLatest = true
override val client: OkHttpClient = super.client.newBuilder()
.cookieJar(
object : CookieJar {
override fun saveFromResponse(url: HttpUrl, cookies: List<Cookie>) {}
override fun loadForRequest(url: HttpUrl): List<Cookie> {
return listOf<Cookie>(
Cookie.Builder()
.domain("www.webtoons.com")
.path("/")
.name("ageGatePass")
.value("true")
.name("locale")
.value(localeForCookie)
.name("needGDPR")
.value("false")
.build()
)
}
}
)
.build()
private val day: String
get() {
return when (Calendar.getInstance().get(Calendar.DAY_OF_WEEK)) {
Calendar.SUNDAY -> "div._list_SUNDAY"
Calendar.MONDAY -> "div._list_MONDAY"
Calendar.TUESDAY -> "div._list_TUESDAY"
Calendar.WEDNESDAY -> "div._list_WEDNESDAY"
Calendar.THURSDAY -> "div._list_THURSDAY"
Calendar.FRIDAY -> "div._list_FRIDAY"
Calendar.SATURDAY -> "div._list_SATURDAY"
else -> {
"div"
}
}
}
override fun popularMangaSelector() = "not using"
override fun latestUpdatesSelector() = "div#dailyList > $day li > a"
override fun headersBuilder(): Headers.Builder = super.headersBuilder()
.add("Referer", "https://www.webtoons.com/$langCode/")
protected val mobileHeaders: Headers = super.headersBuilder()
.add("Referer", "https://m.webtoons.com")
.build()
override fun popularMangaRequest(page: Int) = GET("$baseUrl/$langCode/dailySchedule", headers)
override fun popularMangaParse(response: Response): MangasPage {
val mangas = mutableListOf<SManga>()
val document = response.asJsoup()
var maxChild = 0
// For ongoing webtoons rows are ordered by descending popularity, count how many rows there are
document.select("div#dailyList > div").forEach { day ->
day.select("li").count().let { rowCount ->
if (rowCount > maxChild) maxChild = rowCount
}
}
// Process each row
for (i in 1..maxChild) {
document.select("div#dailyList > div li:nth-child($i) a").map { mangas.add(popularMangaFromElement(it)) }
}
// Add completed webtoons, no sorting needed
document.select("div.daily_lst.comp li a").map { mangas.add(popularMangaFromElement(it)) }
return MangasPage(mangas.distinctBy { it.url }, false)
}
override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/$langCode/dailySchedule?sortOrder=UPDATE&webtoonCompleteType=ONGOING", headers)
override fun popularMangaFromElement(element: Element): SManga {
val manga = SManga.create()
manga.setUrlWithoutDomain(element.attr("href"))
manga.title = element.select("p.subj").text()
manga.thumbnail_url = element.select("img").attr("abs:src")
return manga
}
override fun latestUpdatesFromElement(element: Element): SManga = popularMangaFromElement(element)
override fun popularMangaNextPageSelector(): String? = null
override fun latestUpdatesNextPageSelector(): String? = null
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
val url = HttpUrl.parse("$baseUrl/$langCode/search?keyword=$query")?.newBuilder()!!
val uriPart = (filters.find { it is SearchType } as? SearchType)?.toUriPart() ?: ""
url.addQueryParameter("searchType", uriPart)
if (uriPart != "WEBTOON" && page > 1) url.addQueryParameter("page", page.toString())
return GET(url.toString(), headers)
}
override fun searchMangaSelector() = "#content > div.card_wrap.search li a"
override fun searchMangaFromElement(element: Element): SManga = popularMangaFromElement(element)
override fun searchMangaNextPageSelector() = "div.more_area, div.paginate a[onclick] + a"
open fun parseDetailsThumbnail(document: Document): String? {
val picElement = document.select("#content > div.cont_box > div.detail_body")
val discoverPic = document.select("#content > div.cont_box > div.detail_header > span.thmb")
return discoverPic.select("img").not("[alt='Representative image']").first()?.attr("src") ?: picElement.attr("style")?.substringAfter("url(")?.substringBeforeLast(")")
}
override fun mangaDetailsParse(document: Document): SManga {
val detailElement = document.select("#content > div.cont_box > div.detail_header > div.info")
val infoElement = document.select("#_asideDetail")
val manga = SManga.create()
manga.author = detailElement.select(".author:nth-of-type(1)").first()?.ownText()
manga.artist = detailElement.select(".author:nth-of-type(2)").first()?.ownText() ?: manga.author
manga.genre = detailElement.select(".genre").joinToString(", ") { it.text() }
manga.description = infoElement.select("p.summary").text()
manga.status = infoElement.select("p.day_info").text().orEmpty().let { parseStatus(it) }
manga.thumbnail_url = parseDetailsThumbnail(document)
return manga
}
private fun parseStatus(status: String) = when {
status.contains("UP") -> SManga.ONGOING
status.contains("COMPLETED") -> SManga.COMPLETED
else -> SManga.UNKNOWN
}
override fun imageUrlParse(document: Document): String = document.select("img").first().attr("src")
// Filters
override fun getFilterList(): FilterList {
return FilterList(
Header("Query can not be blank"),
Separator(),
SearchType(getOfficialList())
)
}
override fun chapterListSelector() = "ul#_episodeList li[id*=episode]"
private class SearchType(vals: Array<Pair<String, String>>) : UriPartFilter("Official or Challenge", vals)
private fun getOfficialList() = arrayOf(
Pair("Any", ""),
Pair("Official only", "WEBTOON"),
Pair("Challenge only", "CHALLENGE")
)
open class UriPartFilter(displayName: String, private val vals: Array<Pair<String, String>>) :
Select<String>(displayName, vals.map { it.first }.toTypedArray()) {
fun toUriPart() = vals[state].second
}
override fun chapterFromElement(element: Element): SChapter {
val urlElement = element.select("a")
val chapter = SChapter.create()
chapter.setUrlWithoutDomain(urlElement.attr("href"))
chapter.name = element.select("a > div.row > div.info > p.sub_title > span.ellipsis").text()
val select = element.select("a > div.row > div.num")
if (select.isNotEmpty()) {
chapter.name += " Ch. " + select.text().substringAfter("#")
}
if (element.select(".ico_bgm").isNotEmpty()) {
chapter.name += ""
}
chapter.date_upload = element.select("a > div.row > div.info > p.date").text()?.let { chapterParseDate(it) } ?: 0
return chapter
}
open fun chapterParseDate(date: String): Long {
return dateFormat.parse(date)?.time ?: 0
}
override fun chapterListRequest(manga: SManga) = GET("https://m.webtoons.com" + manga.url, mobileHeaders)
override fun pageListParse(document: Document): List<Page> {
val pages = document.select("div#_imageList > img").mapIndexed { i, element -> Page(i, "", element.attr("data-url")) }
if (pages.isNotEmpty()) { return pages }
val docString = document.toString()
val docUrlRegex = Regex("documentURL:.*?'(.*?)'")
val motiontoonPathRegex = Regex("jpg:.*?'(.*?)\\{")
val docUrl = docUrlRegex.find(docString)!!.destructured.toList()[0]
val motiontoonPath = motiontoonPathRegex.find(docString)!!.destructured.toList()[0]
val motiontoonJson = JSONObject(client.newCall(GET(docUrl, headers)).execute().body()!!.string()).getJSONObject("assets").getJSONObject("image")
val keys = motiontoonJson.keys().asSequence().toList().filter { it.contains("layer") }
return keys.mapIndexed { i, key ->
Page(i, "", motiontoonPath + motiontoonJson.getString(key))
}
}
}

View File

@ -0,0 +1,26 @@
package eu.kanade.tachiyomi.multisrc.webtoons
import generator.ThemeSourceData.SingleLang
import generator.ThemeSourceData.MultiLang
import generator.ThemeSourceGenerator
class WebtoonsGenerator : ThemeSourceGenerator {
override val themePkg = "webtoons"
override val themeClass = "Webtoons"
override val baseVersionCode: Int = 1
override val sources = listOf(
MultiLang("Webtoons.com", "https://www.webtoons.com", listOf("en", "fr", "es", "id", "th", "zh"), className = "WebtoonsFactory", pkgName = "webtoons", overrideVersionCode = 26),
SingleLang("Dongman Manhua", "https://www.dongmanmanhua.cn", "zh")
)
companion object {
@JvmStatic
fun main(args: Array<String>) {
WebtoonsGenerator().createAll()
}
}
}

View File

@ -0,0 +1,231 @@
package eu.kanade.tachiyomi.multisrc.webtoons
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.network.asObservableSuccess
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import okhttp3.Headers
import okhttp3.HttpUrl
import okhttp3.Request
import okhttp3.Response
import org.json.JSONObject
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import rx.Observable
import java.util.ArrayList
open class WebtoonsTranslate (
override val name: String,
override val baseUrl: String,
override val lang: String,
private val translateLangCode: String
) : Webtoons(name, baseUrl, lang) {
// popularMangaRequest already returns manga sorted by latest update
override val supportsLatest = false
private val apiBaseUrl = HttpUrl.parse("https://global.apis.naver.com")!!
private val mobileBaseUrl = HttpUrl.parse("https://m.webtoons.com")!!
private val thumbnailBaseUrl = "https://mwebtoon-phinf.pstatic.net"
private val pageListUrlPattern = "/lineWebtoon/ctrans/translatedEpisodeDetail_jsonp.json?titleNo=%s&episodeNo=%d&languageCode=%s&teamVersion=%d"
private val pageSize = 24
override fun headersBuilder(): Headers.Builder = super.headersBuilder()
.removeAll("Referer")
.add("Referer", mobileBaseUrl.toString())
private fun mangaRequest(page: Int, requeztSize: Int): Request {
val url = apiBaseUrl
.resolve("/lineWebtoon/ctrans/translatedWebtoons_jsonp.json")!!
.newBuilder()
.addQueryParameter("orderType", "UPDATE")
.addQueryParameter("offset", "${(page - 1) * requeztSize}")
.addQueryParameter("size", "$requeztSize")
.addQueryParameter("languageCode", translateLangCode)
.build()
return GET(url.toString(), headers)
}
// Webtoons translations doesn't really have a "popular" sort; just "UPDATE", "TITLE_ASC",
// and "TITLE_DESC". Pick UPDATE as the most useful sort.
override fun popularMangaRequest(page: Int): Request = mangaRequest(page, pageSize)
override fun popularMangaParse(response: Response): MangasPage {
val offset = response.request().url().queryParameter("offset")!!.toInt()
var totalCount: Int
val mangas = mutableListOf<SManga>()
JSONObject(response.body()!!.string()).let { json ->
json.getString("code").let { code ->
if (code != "000") throw Exception("Error getting popular manga: error code $code")
}
json.getJSONObject("result").let { results ->
totalCount = results.getInt("totalCount")
results.getJSONArray("titleList").let { array ->
for (i in 0 until array.length()) {
mangas.add(mangaFromJson(array[i] as JSONObject))
}
}
}
}
return MangasPage(mangas, totalCount > pageSize + offset)
}
private fun mangaFromJson(json: JSONObject): SManga {
val relativeThumnailURL = json.getString("thumbnailIPadUrl")
?: json.getString("thumbnailMobileUrl")
return SManga.create().apply {
title = json.getString("representTitle")
author = json.getString("writeAuthorName")
artist = json.getString("pictureAuthorName") ?: author
thumbnail_url = if (relativeThumnailURL != null) "$thumbnailBaseUrl$relativeThumnailURL" else null
status = SManga.UNKNOWN
url = mobileBaseUrl
.resolve("/translate/episodeList")!!
.newBuilder()
.addQueryParameter("titleNo", json.getInt("titleNo").toString())
.addQueryParameter("languageCode", translateLangCode)
.addQueryParameter("teamVersion", json.optInt("teamVersion", 0).toString())
.build()
.toString()
}
}
override fun fetchSearchManga(page: Int, query: String, filters: FilterList): Observable<MangasPage> {
return client.newCall(searchMangaRequest(page, query, filters))
.asObservableSuccess()
.map { response ->
searchMangaParse(response, query)
}
}
/**
* Don't see a search function for Fan Translations, so let's do it client side.
* There's 75 webtoons as of 2019/11/21, a hardcoded request of 200 should be a sufficient request
* to get all titles, in 1 request, for quite a while
*/
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request = mangaRequest(page, 200)
private fun searchMangaParse(response: Response, query: String): MangasPage {
val mangas = mutableListOf<SManga>()
JSONObject(response.body()!!.string()).let { json ->
json.getString("code").let { code ->
if (code != "000") throw Exception("Error getting manga: error code $code")
}
json.getJSONObject("result").getJSONArray("titleList").let { array ->
for (i in 0 until array.length()) {
(array[i] as JSONObject).let { jsonManga ->
if (jsonManga.getString("representTitle").contains(query, ignoreCase = true))
mangas.add(mangaFromJson(jsonManga))
}
}
}
}
return MangasPage(mangas, false)
}
override fun mangaDetailsRequest(manga: SManga): Request {
return GET(manga.url, headers)
}
override fun mangaDetailsParse(document: Document): SManga {
val getMetaProp = fun(property: String): String =
document.head().select("meta[property=\"$property\"]").attr("content")
var parsedAuthor = getMetaProp("com-linewebtoon:webtoon:author")
var parsedArtist = parsedAuthor
val authorSplit = parsedAuthor.split(" / ", limit = 2)
if (authorSplit.count() > 1) {
parsedAuthor = authorSplit[0]
parsedArtist = authorSplit[1]
}
return SManga.create().apply {
title = getMetaProp("og:title")
artist = parsedArtist
author = parsedAuthor
description = getMetaProp("og:description")
status = SManga.UNKNOWN
thumbnail_url = getMetaProp("og:image")
}
}
override fun chapterListSelector(): String = throw Exception("Not used")
override fun chapterFromElement(element: Element): SChapter = throw Exception("Not used")
override fun pageListParse(document: Document): List<Page> = throw Exception("Not used")
override fun chapterListRequest(manga: SManga): Request {
val titleNo = HttpUrl.parse(manga.url)!!
.queryParameter("titleNo")
val chapterUrl = apiBaseUrl
.resolve("/lineWebtoon/ctrans/translatedEpisodes_jsonp.json")!!
.newBuilder()
.addQueryParameter("titleNo", titleNo)
.addQueryParameter("languageCode", translateLangCode)
.addQueryParameter("offset", "0")
.addQueryParameter("limit", "10000")
.toString()
return GET(chapterUrl, mobileHeaders)
}
override fun chapterListParse(response: Response): List<SChapter> {
val chapterData = response.body()!!.string()
val chapterJson = JSONObject(chapterData)
val responseCode = chapterJson.getString("code")
if (responseCode != "000") {
val message = chapterJson.optString("message", "error code $responseCode")
throw Exception("Error getting chapter list: $message")
}
val results = chapterJson.getJSONObject("result").getJSONArray("episodes")
val ret = ArrayList<SChapter>()
for (i in 0 until results.length()) {
val result = results.getJSONObject(i)
if (result.getBoolean("translateCompleted")) {
ret.add(parseChapterJson(result))
}
}
ret.reverse()
return ret
}
private fun parseChapterJson(obj: JSONObject) = SChapter.create().apply {
name = obj.getString("title") + " #" + obj.getString("episodeSeq")
chapter_number = obj.getInt("episodeSeq").toFloat()
date_upload = obj.getLong("updateYmdt")
scanlator = obj.getString("teamVersion")
if (scanlator == "0") {
scanlator = "(wiki)"
}
url = String.format(pageListUrlPattern, obj.getInt("titleNo"), obj.getInt("episodeNo"), obj.getString("languageCode"), obj.getInt("teamVersion"))
}
override fun pageListRequest(chapter: SChapter): Request {
return GET(apiBaseUrl.resolve(chapter.url).toString(), headers)
}
override fun pageListParse(response: Response): List<Page> {
val pageJson = JSONObject(response.body()!!.string())
val results = pageJson.getJSONObject("result").getJSONArray("imageInfo")
val ret = ArrayList<Page>()
for (i in 0 until results.length()) {
val result = results.getJSONObject(i)
ret.add(Page(i, "", result.getString("imageUrl")))
}
return ret
}
override fun getFilterList(): FilterList = FilterList()
}

View File

@ -0,0 +1,24 @@
package eu.kanade.tachiyomi.multisrc.webtoons
import generator.ThemeSourceData.SingleLang
import generator.ThemeSourceData.MultiLang
import generator.ThemeSourceGenerator
class WebtoonsTranslateGenerator : ThemeSourceGenerator {
override val themePkg = "webtoons"
override val themeClass = "WebtoonsTranslation"
override val baseVersionCode: Int = 1
override val sources = listOf(
MultiLang("Webtoons.com Translations", "https://translate.webtoons.com", listOf("en", "zh-hans", "zh-hant", "th", "id", "fr", "vi", "ru", "ar", "fil", "de", "hi", "it", "ja", "pt-br", "tr", "ms", "pl", "pt", "bg", "da", "nl", "ro", "mn", "el", "lt", "cs", "sv", "bn", "fa", "uk", "es"), className = "WebtoonsTranslateFactory", pkgName = "webtoonstranslate"),
)
companion object {
@JvmStatic
fun main(args: Array<String>) {
WebtoonsTranslateGenerator().createAll()
}
}
}