From 493c8d56c8b921401e5d34406179b9d853954294 Mon Sep 17 00:00:00 2001 From: DitFranXX <45893338+DitFranXX@users.noreply.github.com> Date: Sat, 26 Dec 2020 08:14:06 +0900 Subject: [PATCH] Update NewToki Extension to v1.2.19 (#5258) * Mitigation about IP Ban by bunch of request on latest on manatoki. RateLimit, And make default to not make bunch of request. Split ManaToki into separate file as it became bigger. * More rate limit NewToki and ManaToki Shares IP bans. Also when refresh the batch of mangas, It could be banned too. So rate limit on ChapterList and MangaDetail request too. (And apply both of them) * Fix lint about `java.util` and some format --- src/ko/newtoki/build.gradle | 8 +- .../extension/ko/newtoki/ManaToki.kt | 224 ++++++++++++++++++ .../tachiyomi/extension/ko/newtoki/NewToki.kt | 89 ++++++- .../extension/ko/newtoki/NewTokiFactory.kt | 159 +------------ 4 files changed, 311 insertions(+), 169 deletions(-) create mode 100644 src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt diff --git a/src/ko/newtoki/build.gradle b/src/ko/newtoki/build.gradle index 14f32704b..e0bbe5d22 100644 --- a/src/ko/newtoki/build.gradle +++ b/src/ko/newtoki/build.gradle @@ -2,11 +2,15 @@ apply plugin: 'com.android.application' apply plugin: 'kotlin-android' ext { - extName = 'NewToki / ManaToki(ManaMoa)' + extName = 'NewToki / ManaToki' pkgNameSuffix = 'ko.newtoki' extClass = '.NewTokiFactory' - extVersionCode = 18 + extVersionCode = 19 libVersion = '1.2' } +dependencies { + implementation project(':lib-ratelimit') +} + apply from: "$rootDir/common.gradle" diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt new file mode 100644 index 000000000..63c0d162d --- /dev/null +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/ManaToki.kt @@ -0,0 +1,224 @@ +package eu.kanade.tachiyomi.extension.ko.newtoki + +import eu.kanade.tachiyomi.network.GET +import eu.kanade.tachiyomi.network.asObservableSuccess +import eu.kanade.tachiyomi.source.model.Filter +import eu.kanade.tachiyomi.source.model.FilterList +import eu.kanade.tachiyomi.source.model.MangasPage +import eu.kanade.tachiyomi.source.model.SManga +import eu.kanade.tachiyomi.util.asJsoup +import okhttp3.CacheControl +import okhttp3.HttpUrl +import okhttp3.Request +import okhttp3.Response +import org.jsoup.nodes.Element +import rx.Observable +import java.util.concurrent.TimeUnit + +/* + * ManaToki Is too big to support in a Factory File., So split into separate file. + */ + +class ManaToki(domainNumber: Long) : NewToki("ManaToki", "https://manatoki$domainNumber.net", "comic") { + // / ! DO NOT CHANGE THIS ! Only the site name changed from newtoki. + override val id by lazy { generateSourceId("NewToki", lang, versionId) } + override val supportsLatest by lazy { getExperimentLatest() } + + override fun latestUpdatesSelector() = ".media.post-list" + override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page") + override fun latestUpdatesNextPageSelector() = "nav.pg_wrap > .pg > strong" + override fun fetchLatestUpdates(page: Int): Observable { + // if this is true, Handle Only 10 mangas with accurate Details per page. (Real Latest Page has 70 mangas.) + // Else, Parse from Latest page. which is incomplete. + val isParseWithDetail = getLatestWithDetail() + val reqPage = if (isParseWithDetail) ((page - 1) / 7 + 1) else page + return rateLimitedClient.newCall(latestUpdatesRequest(reqPage)) + .asObservableSuccess() + .map { response -> + if (isParseWithDetail) latestUpdatesParseWithDetailPage(response, page) + else latestUpdatesParseWithLatestPage(response) + } + } + + private fun latestUpdatesParseWithDetailPage(response: Response, page: Int): MangasPage { + val document = response.asJsoup() + + // given cache time to prevent repeated lots of request in latest. + val cacheControl = CacheControl.Builder().maxAge(28, TimeUnit.DAYS).maxStale(28, TimeUnit.DAYS).build() + + val rm = 70 * ((page - 1) / 7) + val min = (page - 1) * 10 - rm + val max = page * 10 - rm + val elements = document.select("${latestUpdatesSelector()} p > a").slice(min until max) + val mangas = elements.map { element -> + val url = element.attr("abs:href") + val manga = mangaDetailsParse(rateLimitedClient.newCall(GET(url, cache = cacheControl)).execute()) + manga.url = getUrlPath(url) + manga + } + + val hasNextPage = try { + !document.select(popularMangaNextPageSelector()).text().contains("10") + } catch (_: Exception) { + false + } + + return MangasPage(mangas, hasNextPage) + } + + private fun latestUpdatesParseWithLatestPage(response: Response): MangasPage { + val document = response.asJsoup() + + val mangas = document.select(latestUpdatesSelector()).map { element -> + latestUpdatesElementParse(element) + } + + val hasNextPage = try { + !document.select(popularMangaNextPageSelector()).text().contains("10") + } catch (_: Exception) { + false + } + + return MangasPage(mangas, hasNextPage) + } + + private fun latestUpdatesElementParse(element: Element): SManga { + val linkElement = element.select("a.btn-primary") + val rawTitle = element.select(".post-subject > a").first().ownText().trim() + + // TODO: Make Clear Regex. + val chapterRegex = Regex("""((?:\s+)(?:(?:(?:[0-9]+권)?(?:[0-9]+부)?(?:[0-9]*?시즌[0-9]*?)?)?(?:\s*)(?:(?:[0-9]+)(?:[-.](?:[0-9]+))?)?(?:\s*[~,]\s*)?(?:[0-9]+)(?:[-.](?:[0-9]+))?)(?:화))""") + val title = rawTitle.trim().replace(chapterRegex, "") + // val regexSpecialChapter = Regex("(부록|단편|외전|.+편)") + // val lastTitleWord = excludeChapterTitle.split(" ").last() + // val title = excludeChapterTitle.replace(lastTitleWord, lastTitleWord.replace(regexSpecialChapter, "")) + + val manga = SManga.create() + manga.url = getUrlPath(linkElement.attr("href")) + manga.title = title + manga.thumbnail_url = element.select(".img-item > img").attr("src") + manga.initialized = false + return manga + } + + override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { + val url = HttpUrl.parse("$baseUrl/comic" + (if (page > 1) "/p$page" else ""))!!.newBuilder() + + if (!query.isBlank()) { + url.addQueryParameter("stx", query) + return GET(url.toString()) + } + + filters.forEach { filter -> + when (filter) { + is SearchPublishTypeList -> { + if (filter.state > 0) { + url.addQueryParameter("publish", filter.values[filter.state]) + } + } + + is SearchJaumTypeList -> { + if (filter.state > 0) { + url.addQueryParameter("jaum", filter.values[filter.state]) + } + } + + is SearchGenreTypeList -> { + if (filter.state > 0) { + url.addQueryParameter("tag", filter.values[filter.state]) + } + } + } + } + + return GET(url.toString()) + } + + // [...document.querySelectorAll("form.form td")[2].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') + private class SearchPublishTypeList : Filter.Select( + "Publish", + arrayOf( + "전체", + "미분류", + "주간", + "격주", + "월간", + "격월/비정기", + "단편", + "단행본", + "완결" + ) + ) + + // [...document.querySelectorAll("form.form td")[3].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') + private class SearchJaumTypeList : Filter.Select( + "Jaum", + arrayOf( + "전체", + "ㄱ", + "ㄴ", + "ㄷ", + "ㄹ", + "ㅁ", + "ㅂ", + "ㅅ", + "ㅇ", + "ㅈ", + "ㅊ", + "ㅋ", + "ㅌ", + "ㅍ", + "ㅎ", + "0-9", + "a-z" + ) + ) + + // [...document.querySelectorAll("form.form td")[4].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') + private class SearchGenreTypeList : Filter.Select( + "Genre", + arrayOf( + "전체", + "17", + "BL", + "SF", + "TS", + "개그", + "게임", + "공포", + "도박", + "드라마", + "라노벨", + "러브코미디", + "로맨스", + "먹방", + "미스터리", + "백합", + "붕탁", + "성인", + "순정", + "스릴러", + "스포츠", + "시대", + "애니화", + "액션", + "역사", + "음악", + "이세계", + "일상", + "일상+치유", + "전생", + "추리", + "판타지", + "학원", + "호러" + ) + ) + + override fun getFilterList() = FilterList( + Filter.Header("Filter can't use with query"), + SearchPublishTypeList(), + SearchJaumTypeList(), + SearchGenreTypeList() + ) +} diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt index 4dce2efff..5b19de4f3 100644 --- a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewToki.kt @@ -7,7 +7,8 @@ import android.support.v7.preference.CheckBoxPreference import android.support.v7.preference.EditTextPreference import android.support.v7.preference.PreferenceScreen import android.widget.Toast -import eu.kanade.tachiyomi.extension.BuildConfig +import eu.kanade.tachiyomi.extensions.BuildConfig +import eu.kanade.tachiyomi.lib.ratelimit.RateLimitInterceptor import eu.kanade.tachiyomi.network.GET import eu.kanade.tachiyomi.network.asObservableSuccess import eu.kanade.tachiyomi.source.ConfigurableSource @@ -40,6 +41,9 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String override val lang: String = "ko" override val supportsLatest = true override val client: OkHttpClient = network.cloudflareClient + protected val rateLimitedClient: OkHttpClient = network.cloudflareClient.newBuilder() + .addNetworkInterceptor(RateLimitInterceptor(2, 5)) + .build() override fun popularMangaSelector() = "div#webtoon-list > ul > li" @@ -101,11 +105,13 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String // only exists on chapter with proper manga detail page. val fullListButton = document.select(".comic-navbar .toon-nav a").last() - val list: List = if (firstChapterButton?.text()?.contains("첫회보기") ?: false) { // Check this page is detail page + val list: List = if (firstChapterButton?.text()?.contains("첫회보기") + ?: false) { // Check this page is detail page val details = mangaDetailsParse(document) details.url = urlPath listOf(details) - } else if (fullListButton?.text()?.contains("전체목록") ?: false) { // Check this page is chapter page + } else if (fullListButton?.text()?.contains("전체목록") + ?: false) { // Check this page is chapter page val url = fullListButton.attr("abs:href") val details = mangaDetailsParse(client.newCall(GET(url)).execute()) details.url = getUrlPath(url) @@ -172,13 +178,30 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String if (name.contains("번외") || name.contains("특별편")) return -2f val regex = Regex("([0-9]+)(?:[-.]([0-9]+))?(?:화)") val (ch_primal, ch_second) = regex.find(name)!!.destructured - return (ch_primal + if (ch_second.isBlank()) "" else ".$ch_second").toFloatOrNull() ?: -1f + return (ch_primal + if (ch_second.isBlank()) "" else ".$ch_second").toFloatOrNull() + ?: -1f } catch (e: Exception) { e.printStackTrace() return -1f } } + override fun fetchMangaDetails(manga: SManga): Observable { + return rateLimitedClient.newCall(mangaDetailsRequest(manga)) + .asObservableSuccess() + .map { response -> + mangaDetailsParse(response).apply { initialized = true } + } + } + + override fun fetchChapterList(manga: SManga): Observable> { + return rateLimitedClient.newCall(chapterListRequest(manga)) + .asObservableSuccess() + .map { response -> + chapterListParse(response) + } + } + @SuppressLint("SimpleDateFormat") private fun parseChapterDate(date: String): Long { return try { @@ -209,8 +232,10 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String private val htmlDataRegex = Regex("""html_data\+='([^']+)'""") override fun pageListParse(document: Document): List { - val script = document.select("script:containsData(html_data)").firstOrNull()?.data() ?: throw Exception("data script not found") - val loadScript = document.select("script:containsData(data_attribute)").firstOrNull()?.data() ?: throw Exception("load script not found") + val script = document.select("script:containsData(html_data)").firstOrNull()?.data() + ?: throw Exception("data script not found") + val loadScript = document.select("script:containsData(data_attribute)").firstOrNull()?.data() + ?: throw Exception("load script not found") val dataAttr = "abs:data-" + loadScript.substringAfter("data_attribute: '").substringBefore("',") return htmlDataRegex.findAll(script).map { it.groupValues[1] } @@ -218,7 +243,7 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String .flatMap { it.split(".") } .joinToString("") { it.toIntOrNull(16)?.toChar()?.toString() ?: "" } .let { Jsoup.parse(it) } - .select("img[src=/img/loading-image.gif]") + .select("img[src=/img/loading-image.gif], .view-img > img[itemprop]") .mapIndexed { i, img -> Page(i, "", if (img.hasAttr(dataAttr)) img.attr(dataAttr) else img.attr("abs:content")) } } @@ -275,9 +300,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String } } + val latestWithDetailPref = androidx.preference.CheckBoxPreference(screen.context).apply { + key = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + title = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + summary = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY + + setOnPreferenceChangeListener { _, newValue -> + try { + val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, newValue as Boolean).commit() + // Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() + res + } catch (e: Exception) { + e.printStackTrace() + false + } + } + } + screen.addPreference(baseUrlPref) if (name == "ManaToki") { screen.addPreference(latestExperimentPref) + screen.addPreference(latestWithDetailPref) } } @@ -319,9 +362,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String } } + val latestWithDetailPref = CheckBoxPreference(screen.context).apply { + key = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + title = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE + summary = EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY + + setOnPreferenceChangeListener { _, newValue -> + try { + val res = preferences.edit().putBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, newValue as Boolean).commit() + // Toast.makeText(screen.context, RESTART_TACHIYOMI, Toast.LENGTH_LONG).show() + res + } catch (e: Exception) { + e.printStackTrace() + false + } + } + } + screen.addPreference(baseUrlPref) if (name == "ManaToki") { screen.addPreference(latestExperimentPref) + screen.addPreference(latestWithDetailPref) } } @@ -335,17 +396,27 @@ open class NewToki(override val name: String, private val defaultBaseUrl: String private fun getPrefBaseUrl(): String = preferences.getString(BASE_URL_PREF, defaultBaseUrl)!! protected fun getExperimentLatest(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_PREF, false) + protected fun getLatestWithDetail(): Boolean = preferences.getBoolean(EXPERIMENTAL_LATEST_WITH_DETAIL_PREF, false) companion object { + private const val RESTART_TACHIYOMI = "Restart Tachiyomi to apply new setting." + private const val BASE_URL_PREF_TITLE = "Override BaseUrl" private const val BASE_URL_PREF = "overrideBaseUrl_v${BuildConfig.VERSION_NAME}" private const val BASE_URL_PREF_SUMMARY = "For temporary uses. Update extension will erase this setting." - private const val RESTART_TACHIYOMI = "Restart Tachiyomi to apply new setting." // Setting: Experimental Latest Fetcher private const val EXPERIMENTAL_LATEST_PREF_TITLE = "Enable Latest (Experimental)" private const val EXPERIMENTAL_LATEST_PREF = "fetchLatestExperiment" - private const val EXPERIMENTAL_LATEST_PREF_SUMMARY = "Fetch Latest Manga using Latest Chapters. May has duplicates, Also requires LOTS OF requests (70 per page)" + private const val EXPERIMENTAL_LATEST_PREF_SUMMARY = "Fetch Latest Manga using Latest Chapters. May has duplicates and May DB corruption on certain Tachiyomi builds" + + // Setting: Experimental Latest Fetcher With Full Details (Optional) + private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_TITLE = "Fetch Latest with detail (Optional)" + private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF = "fetchLatestWithDetail" + private const val EXPERIMENTAL_LATEST_WITH_DETAIL_PREF_SUMMARY = + "Parse latest manga details with detail pages. This will reduce DB corruption on certain Tachiyomi builds.\n" + + "But makes chance of IP Ban, Also makes bunch of requests, For prevent IP ban, rate limit is set. so may slow,\n" + + "Still, It's experiment. Required to enable `Enable Latest (Experimental).`" const val PREFIX_ID_SEARCH = "id:" } diff --git a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt index e09c7d36a..7c8cf74a7 100644 --- a/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt +++ b/src/ko/newtoki/src/eu/kanade/tachiyomi/extension/ko/newtoki/NewTokiFactory.kt @@ -5,17 +5,12 @@ import eu.kanade.tachiyomi.source.Source import eu.kanade.tachiyomi.source.SourceFactory import eu.kanade.tachiyomi.source.model.Filter import eu.kanade.tachiyomi.source.model.FilterList -import eu.kanade.tachiyomi.source.model.MangasPage -import eu.kanade.tachiyomi.util.asJsoup -import okhttp3.CacheControl import okhttp3.HttpUrl import okhttp3.Request -import okhttp3.Response import java.security.MessageDigest import java.text.SimpleDateFormat import java.util.Date import java.util.Locale -import java.util.concurrent.TimeUnit.DAYS /** * Source changes domain names every few days (e.g. newtoki31.net to newtoki32.net) @@ -29,163 +24,11 @@ private val domainNumber = 32 + ((Date().time - SimpleDateFormat("yyyy-MM-dd", L class NewTokiFactory : SourceFactory { override fun createSources(): List = listOf( - NewTokiManga(), + ManaToki(domainNumber), NewTokiWebtoon() ) } -class NewTokiManga : NewToki("ManaToki", "https://manatoki$domainNumber.net", "comic") { - // / ! DO NOT CHANGE THIS ! Only the site name changed from newtoki. - override val id by lazy { generateSourceId("NewToki", lang, versionId) } - override val supportsLatest by lazy { getExperimentLatest() } - - // this does 70 request per page.... - override fun latestUpdatesSelector() = ".media.post-list p > a" - override fun latestUpdatesRequest(page: Int) = GET("$baseUrl/page/update?hid=update&page=$page") - override fun latestUpdatesNextPageSelector() = "nav.pg_wrap > .pg > strong" - override fun latestUpdatesParse(response: Response): MangasPage { - val document = response.asJsoup() - - // given cache time to prevent repeated lots of request in latest. - val cacheControl = CacheControl.Builder().maxAge(14, DAYS).maxStale(14, DAYS).build() - val mangas = document.select(latestUpdatesSelector()).map { element -> - val url = element.attr("abs:href") - val manga = mangaDetailsParse(client.newCall(GET(url, cache = cacheControl)).execute()) - manga.url = getUrlPath(url) - manga - } - - val hasNextPage = try { - !document.select(popularMangaNextPageSelector()).text().contains("10") - } catch (_: Exception) { - false - } - - return MangasPage(mangas, hasNextPage) - } - - override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request { - val url = HttpUrl.parse("$baseUrl/comic" + (if (page > 1) "/p$page" else ""))!!.newBuilder() - - if (!query.isBlank()) { - url.addQueryParameter("stx", query) - return GET(url.toString()) - } - - filters.forEach { filter -> - when (filter) { - is SearchPublishTypeList -> { - if (filter.state > 0) { - url.addQueryParameter("publish", filter.values[filter.state]) - } - } - - is SearchJaumTypeList -> { - if (filter.state > 0) { - url.addQueryParameter("jaum", filter.values[filter.state]) - } - } - - is SearchGenreTypeList -> { - if (filter.state > 0) { - url.addQueryParameter("tag", filter.values[filter.state]) - } - } - } - } - - return GET(url.toString()) - } - - // [...document.querySelectorAll("form.form td")[2].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') - private class SearchPublishTypeList : Filter.Select( - "Publish", - arrayOf( - "전체", - "미분류", - "주간", - "격주", - "월간", - "격월/비정기", - "단편", - "단행본", - "완결" - ) - ) - - // [...document.querySelectorAll("form.form td")[3].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') - private class SearchJaumTypeList : Filter.Select( - "Jaum", - arrayOf( - "전체", - "ㄱ", - "ㄴ", - "ㄷ", - "ㄹ", - "ㅁ", - "ㅂ", - "ㅅ", - "ㅇ", - "ㅈ", - "ㅊ", - "ㅋ", - "ㅌ", - "ㅍ", - "ㅎ", - "0-9", - "a-z" - ) - ) - - // [...document.querySelectorAll("form.form td")[4].querySelectorAll("a")].map((el, i) => `"${el.innerText.trim()}"`).join(',\n') - private class SearchGenreTypeList : Filter.Select( - "Genre", - arrayOf( - "전체", - "17", - "BL", - "SF", - "TS", - "개그", - "게임", - "공포", - "도박", - "드라마", - "라노벨", - "러브코미디", - "로맨스", - "먹방", - "미스터리", - "백합", - "붕탁", - "성인", - "순정", - "스릴러", - "스포츠", - "시대", - "애니화", - "액션", - "역사", - "음악", - "이세계", - "일상", - "일상+치유", - "전생", - "추리", - "판타지", - "학원", - "호러" - ) - ) - - override fun getFilterList() = FilterList( - Filter.Header("Filter can't use with query"), - SearchPublishTypeList(), - SearchJaumTypeList(), - SearchGenreTypeList() - ) -} - class NewTokiWebtoon : NewToki("NewToki", "https://newtoki$domainNumber.com", "webtoon") { // / ! DO NOT CHANGE THIS ! Prevent to treating as a new site override val id by lazy { generateSourceId("NewToki (Webtoon)", lang, versionId) }