lib-themesources, split Genkan into single-source extensions (#5154)

* lib themesources copied from SnakeDoc83/tachiyomi-extensions/library

* update to the newer Genkan

* update genkan generator

* GenkanOriginal

* code cleanup

* add all Genkan sources

* generate inside generated-src, res override

* src override

* move overrides out of library

* move overrides to a better place

* remove leftover generated files

* remove leftover generated files

* add generators main class

* comment the code

* Now sources are purely generated

* uncomment generators

* enhance comments

* icons by @as280093

* fix pathing issues

* nullpointerexception proof

* runAllGenerators task

* more flexibility in lib structure, fix a fiew errors

* update github workflows

* correct nonames scans directory name

* rename SK Scans to Sleeping Knight Scans

* fix typo

* update depencencies

* remove defaultRes from dependencies

* fix bug with nsfw

* fix nsfw generation

* themesourcesLibraryVersion is included in build.gradle extVersionCode

* improve javadoc

* fix formatting and language code generation

* comply with #5214

* common dependencies

* rename and move lib/themesources into /multisrc

* use not depricated form

* cleanup runAllGenerators task

* cleanup even more

* oops extra file

* remove test code

* comments

* update docs and refactor

* update docs

* requested changes

* clean up dependencies

* sealed dataClass

* refactor

* refactor string generators

* bring back writeAndroidManifest

* update overrideVersionCode javadoc

* update overrideVersionCode javadoc

* move dependency to extension source

* refactor runAllGenerators

* improve docs

* remove extra file
This commit is contained in:
Aria Moradi
2021-02-06 14:32:04 -08:00
committed by GitHub
parent 5bdd8924b9
commit 3f081f69ac
83 changed files with 512 additions and 136 deletions

View File

@ -0,0 +1,33 @@
package eu.kanade.tachiyomi.multisrc
import java.io.File
/**
* Finds and calls all `ThemeSourceGenerator`s
*/
fun main(args: Array<String>) {
val userDir = System.getProperty("user.dir")!!
val sourcesDirPath = "$userDir/multisrc/src/main/java/eu/kanade/tachiyomi/multisrc"
val sourcesDir = File(sourcesDirPath)
val directories = sourcesDir.list()!!.filter {
File(sourcesDir, it).isDirectory
}
// find all theme packages
directories.forEach { themeSource ->
// find all XxxGenerator.kt files and invoke main from them
File("$sourcesDirPath/$themeSource").list()!!
.filter {
it.endsWith("Generator.kt")
}.map {// find java class and extract method lists
Class.forName("eu/kanade/tachiyomi/multisrc/$themeSource/$it".replace("/", ".").substringBefore(".kt")).methods.asList()
}
.flatten()
.filter { it.name == "main" }
.forEach { it.invoke(null, emptyArray<String>()) }
}
}

View File

@ -0,0 +1,225 @@
package eu.kanade.tachiyomi.multisrc
import java.io.File
import java.nio.file.Files
import java.nio.file.StandardCopyOption
import java.util.Locale
/**
* This is meant to be used in place of a factory extension, specifically for what would be a multi-source extension.
* A multi-lang (but not multi-source) extension should still be made as a factory extensiion.
* Use a generator for initial setup of a theme source or when all of the inheritors need a version bump.
* Source list (val sources) should be kept up to date.
*/
interface ThemeSourceGenerator {
/**
* The class that the sources inherit from.
*/
val themeClass: String
/**
* The package that contains themeClass.
*/
val themePkg: String
/**
* Base theme version, starts with 1 and should be increased when based theme class changes
*/
val baseVersionCode: Int
/**
* The list of sources to be created or updated.
*/
val sources: List<ThemeSourceData>
fun createAll() {
val userDir = System.getProperty("user.dir")!!
sources.forEach { source ->
createGradleProject(source, themePkg, themeClass, baseVersionCode, userDir)
}
}
companion object {
private fun pkgNameSuffix(source: ThemeSourceData, separator: String): String {
return if (source is ThemeSourceData.SingleLang)
listOf(source.lang.substringBefore("-"), source.pkgName).joinToString(separator)
else
listOf("all", source.pkgName).joinToString(separator)
}
private fun themeSuffix(themePkg: String, separator: String): String {
return listOf("eu", "kanade", "tachiyomi", "multisrc", themePkg).joinToString(separator)
}
private fun writeGradle(gradle: File, source: ThemeSourceData, baseVersionCode: Int) {
gradle.writeText("""apply plugin: 'com.android.application'
apply plugin: 'kotlin-android'
ext {
extName = '${source.name}'
pkgNameSuffix = '${pkgNameSuffix(source, ".")}'
extClass = '.${source.className}'
extVersionCode = ${baseVersionCode + source.overrideVersionCode + multisrcLibraryVersion}
libVersion = '1.2'
${if (source.isNsfw) " containsNsfw = true\n" else ""}}
apply from: "${'$'}rootDir/common.gradle"
"""
)
}
private fun writeAndroidManifest(androidManifestFile: File) {
androidManifestFile.writeText(
"<?xml version=\"1.0\" encoding=\"utf-8\"?>\n" +
"<manifest package=\"eu.kanade.tachiyomi.extension\" />\n"
)
}
/**
* Clears directory recursively
*/
private fun purgeDirectory(dir: File) {
for (file in dir.listFiles()!!) {
if (file.isDirectory) purgeDirectory(file)
file.delete()
}
}
fun createGradleProject(source: ThemeSourceData, themePkg: String, themeClass: String, baseVersionCode: Int, userDir: String) {
val projectRootPath = "$userDir/generated-src/${pkgNameSuffix(source, "/")}"
val projectSrcPath = "$projectRootPath/src/eu/kanade/tachiyomi/extension/${pkgNameSuffix(source, "/")}"
val overridesPath = "$userDir/multisrc/overrides" // userDir = tachiyomi-extensions project root path
val resOverridesPath = "$overridesPath/res/$themePkg"
val srcOverridesPath = "$overridesPath/src/$themePkg"
val projectGradleFile = File("$projectRootPath/build.gradle")
val projectAndroidManifestFile = File("$projectRootPath/AndroidManifest.xml")
File(projectRootPath).let { projectRootFile ->
println("Working on $source")
projectRootFile.mkdirs()
// remove everything from past runs
purgeDirectory(projectRootFile)
writeGradle(projectGradleFile, source, baseVersionCode)
writeAndroidManifest(projectAndroidManifestFile)
writeSourceFiles(projectSrcPath, srcOverridesPath, source, themePkg, themeClass)
copyThemeClasses(userDir, themePkg, projectRootPath)
copyResFiles(resOverridesPath, source, projectRootPath)
}
}
private fun copyThemeClasses(userDir: String, themePkg: String, projectRootPath: String) {
val themeSrcPath = "$userDir/multisrc/src/main/java/${themeSuffix(themePkg, "/")}"
val themeSrcFile = File(themeSrcPath)
val themeDestPath = "$projectRootPath/src/${themeSuffix(themePkg, "/")}"
val themeDestFile = File(themeDestPath)
themeDestFile.mkdirs()
themeSrcFile.list()!!
.filter { it.endsWith(".kt") && !it.endsWith("Generator.kt") }
.forEach { Files.copy(File("$themeSrcPath/$it").toPath(), File("$themeDestPath/$it").toPath(), StandardCopyOption.REPLACE_EXISTING) }
}
private fun copyResFiles(resOverridesPath: String, source: ThemeSourceData, projectRootPath: String): Any {
// check if res override exists if not copy default res
val resOverride = File("$resOverridesPath/${source.pkgName}")
return if (resOverride.exists())
resOverride.copyRecursively(File("$projectRootPath/res"))
else
File("$resOverridesPath/default").let { res ->
if (res.exists()) res.copyRecursively(File("$projectRootPath/res"))
}
}
private fun writeSourceFiles(projectSrcPath: String, srcOverridePath: String, source: ThemeSourceData, themePkg: String, themeClass: String) {
val projectSrcFile = File(projectSrcPath)
projectSrcFile.mkdirs()
val srcOverride = File("$srcOverridePath/${source.pkgName}")
if (srcOverride.exists())
srcOverride.copyRecursively(projectSrcFile)
else
writeSourceClass(projectSrcFile, source, themePkg, themeClass)
}
private fun writeSourceClass(classPath: File, source: ThemeSourceData, themePkg: String, themeClass: String) {
fun factoryClassText(): String {
val sourceListString =
(source as ThemeSourceData.MultiLang).lang.map {
" $themeClass(\"${source.name}\", \"${source.baseUrl}\", \"$it\"),"
}.joinToString("\n")
return """class ${source.className} : SourceFactory {
override fun createSources(): List<Source> = listOf(
$sourceListString
)
}"""
}
File("$classPath/${source.className}.kt").writeText(
"""package eu.kanade.tachiyomi.extension.${pkgNameSuffix(source, ".")}
${if (source.isNsfw) "\nimport eu.kanade.tachiyomi.annotations.Nsfw" else ""}
import eu.kanade.tachiyomi.multisrc.$themePkg.$themeClass
${if (source is ThemeSourceData.MultiLang)
"""import eu.kanade.tachiyomi.source.Source
import eu.kanade.tachiyomi.source.SourceFactory
"""
else ""}${if (source.isNsfw) "\n@Nsfw" else ""}
${if (source is ThemeSourceData.SingleLang) {
"class ${source.className} : $themeClass(\"${source.name}\", \"${source.baseUrl}\", \"${source.lang}\")\n"
} else
factoryClassText()
}
""")
}
sealed class ThemeSourceData {
abstract val name: String
abstract val baseUrl: String
abstract val isNsfw: Boolean
abstract val className: String
abstract val pkgName: String
/**
* overrideVersionCode defaults to 0, if a source changes their source override code or
* a previous existing source suddenly needs source code overrides, overrideVersionCode
* should be increased.
* When a new source is added with overrides, overrideVersionCode should still be set to 0
*
* Note: source code overrides are located in "multisrc/overrides/src/<themeName>/<sourceName>"
*/
abstract val overrideVersionCode: Int
data class SingleLang(
override val name: String,
override val baseUrl: String,
val lang: String,
override val isNsfw: Boolean = false,
override val className: String = name.replace(" ", ""),
override val pkgName: String = className.toLowerCase(Locale.ENGLISH),
override val overrideVersionCode: Int = 0,
) : ThemeSourceData()
data class MultiLang(
override val name: String,
override val baseUrl: String,
val lang: List<String>,
override val isNsfw: Boolean = false,
override val className: String = name.replace(" ", "") + "Factory",
override val pkgName: String = className.substringBefore("Factory").toLowerCase(Locale.ENGLISH),
override val overrideVersionCode: Int = 0,
) : ThemeSourceData()
}
}
}
/**
* This variable should be increased when the multisrc library changes in a way that prompts global extension upgrade
*/
const val multisrcLibraryVersion = 0

View File

@ -0,0 +1,180 @@
package eu.kanade.tachiyomi.multisrc.genkan
import eu.kanade.tachiyomi.network.GET
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.Page
import eu.kanade.tachiyomi.source.model.SChapter
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.source.online.ParsedHttpSource
import eu.kanade.tachiyomi.util.asJsoup
import java.text.SimpleDateFormat
import java.util.Calendar
import java.util.Locale
import okhttp3.OkHttpClient
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
open class Genkan(
override val name: String,
override val baseUrl: String,
override val lang: String
) : ParsedHttpSource() {
override val supportsLatest = true
override val client: OkHttpClient = network.cloudflareClient
override fun popularMangaSelector() = "div.list-item"
override fun popularMangaRequest(page: Int): Request {
return GET("$baseUrl/comics?page=$page", headers)
}
override fun latestUpdatesSelector() = popularMangaSelector()
// Track which manga titles have been added to latestUpdates's MangasPage
private val latestUpdatesTitles = mutableSetOf<String>()
override fun latestUpdatesRequest(page: Int): Request {
if (page == 1) latestUpdatesTitles.clear()
return GET("$baseUrl/latest?page=$page", headers)
}
// To prevent dupes, only add manga to MangasPage if its title is not one we've added already
override fun latestUpdatesParse(response: Response): MangasPage {
val latestManga = mutableListOf<SManga>()
val document = response.asJsoup()
document.select(latestUpdatesSelector()).forEach { element ->
latestUpdatesFromElement(element).let { manga ->
if (manga.title !in latestUpdatesTitles) {
latestManga.add(manga)
latestUpdatesTitles.add(manga.title)
}
}
}
return MangasPage(latestManga, document.select(latestUpdatesNextPageSelector()).hasText())
}
override fun popularMangaFromElement(element: Element): SManga {
val manga = SManga.create()
element.select("a.list-title").first().let {
manga.setUrlWithoutDomain(it.attr("href"))
manga.title = it.text()
}
manga.thumbnail_url = styleToUrl(element.select("a.media-content").first())
return manga
}
override fun latestUpdatesFromElement(element: Element): SManga = popularMangaFromElement(element)
override fun popularMangaNextPageSelector() = "[rel=next]"
override fun latestUpdatesNextPageSelector() = popularMangaNextPageSelector()
// Search
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
return GET("$baseUrl/comics?query=$query", headers)
}
override fun searchMangaSelector() = popularMangaSelector()
override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
// Details
private fun styleToUrl(element: Element): String {
return element.attr("style").substringAfter("(").substringBefore(")")
.let { if (it.startsWith("http")) it else baseUrl + it }
}
override fun mangaDetailsParse(document: Document): SManga {
return SManga.create().apply {
title = document.select("div#content h5").first().text()
description = document.select("div.col-lg-9").text().substringAfter("Description ").substringBefore(" Volume")
thumbnail_url = styleToUrl(document.select("div.media a").first())
}
}
override fun chapterListSelector() = "div.col-lg-9 div.flex"
override fun chapterFromElement(element: Element): SChapter {
return SChapter.create().apply {
val urlElement = element.select("a.item-author")
val chapNum = urlElement.attr("href").split("/").last()
setUrlWithoutDomain(urlElement.attr("href"))
name = if (urlElement.text().contains("Chapter $chapNum")) {
urlElement.text()
} else {
"Ch. $chapNum: ${urlElement.text()}"
}
date_upload = parseChapterDate(element.select("a.item-company").first().text()) ?: 0
}
}
companion object {
val dateFormat by lazy {
SimpleDateFormat("MMM d, yyyy", Locale.US)
}
}
// If the date string contains the word "ago" send it off for relative date parsing otherwise use dateFormat
private fun parseChapterDate(string: String): Long? {
return if ("ago" in string) {
parseRelativeDate(string) ?: 0
} else {
dateFormat.parse(string)?.time ?: 0
}
}
// Subtract relative date (e.g. posted 3 days ago)
private fun parseRelativeDate(date: String): Long? {
val trimmedDate = date.substringBefore(" ago").removeSuffix("s").split(" ")
val calendar = Calendar.getInstance()
when (trimmedDate[1]) {
"year" -> calendar.apply { add(Calendar.YEAR, -trimmedDate[0].toInt()) }
"month" -> calendar.apply { add(Calendar.MONTH, -trimmedDate[0].toInt()) }
"week" -> calendar.apply { add(Calendar.WEEK_OF_MONTH, -trimmedDate[0].toInt()) }
"day" -> calendar.apply { add(Calendar.DAY_OF_MONTH, -trimmedDate[0].toInt()) }
"hour" -> calendar.apply { add(Calendar.HOUR_OF_DAY, -trimmedDate[0].toInt()) }
"minute" -> calendar.apply { add(Calendar.MINUTE, -trimmedDate[0].toInt()) }
"second" -> calendar.apply { add(Calendar.SECOND, 0) }
}
return calendar.timeInMillis
}
override fun pageListParse(document: Document): List<Page> {
val pages = mutableListOf<Page>()
val allImages = document.select("div#pages-container + script").first().data()
.substringAfter("[").substringBefore("];")
.replace(Regex("""["\\]"""), "")
.split(",")
for (i in allImages.indices) {
pages.add(Page(i, "", allImages[i]))
}
return pages
}
override fun imageUrlParse(document: Document): String = throw UnsupportedOperationException("Not used")
override fun imageRequest(page: Page): Request {
return if (page.imageUrl!!.startsWith("http")) GET(page.imageUrl!!, headers) else GET(baseUrl + page.imageUrl!!, headers)
}
override fun getFilterList() = FilterList()
}

View File

@ -0,0 +1,31 @@
package eu.kanade.tachiyomi.multisrc.genkan
import eu.kanade.tachiyomi.multisrc.ThemeSourceGenerator
import eu.kanade.tachiyomi.multisrc.ThemeSourceGenerator.Companion.ThemeSourceData
class GenkanGenerator : ThemeSourceGenerator {
override val themePkg = "genkan"
override val themeClass = "Genkan"
override val baseVersionCode: Int = 1
override val sources = listOf(
ThemeSourceData.MultiLang("Leviatan Scans", "https://leviatanscans.com", listOf("en", "es"),
className = "LeviatanScansFactory", pkgName = "leviatanscans", overrideVersionCode = 1),
ThemeSourceGenerator.Companion.ThemeSourceData.SingleLang("Hunlight Scans", "https://hunlight-scans.info", "en"),
ThemeSourceData.SingleLang("ZeroScans", "https://zeroscans.com", "en"),
ThemeSourceData.SingleLang("The Nonames Scans", "https://the-nonames.com", "en"),
ThemeSourceData.SingleLang("Edelgarde Scans", "https://edelgardescans.com", "en"),
ThemeSourceData.SingleLang("Method Scans", "https://methodscans.com", "en"),
ThemeSourceData.SingleLang("Sleeping Knight Scans", "https://skscans.com", "en")
)
companion object {
@JvmStatic
fun main(args: Array<String>) {
GenkanGenerator().createAll()
}
}
}

View File

@ -0,0 +1,75 @@
package eu.kanade.tachiyomi.multisrc.genkan
import eu.kanade.tachiyomi.source.model.FilterList
import eu.kanade.tachiyomi.source.model.MangasPage
import eu.kanade.tachiyomi.source.model.SManga
import eu.kanade.tachiyomi.util.asJsoup
import okhttp3.Request
import okhttp3.Response
import org.jsoup.nodes.Document
import org.jsoup.nodes.Element
import org.jsoup.select.Elements
/**
* For sites using the older Genkan CMS that didn't have a search function
*/
open class GenkanOriginal(
override val name: String,
override val baseUrl: String,
override val lang: String
) : Genkan(name, baseUrl, lang) {
private var searchQuery = ""
private var searchPage = 1
private var nextPageSelectorElement = Elements()
override fun searchMangaRequest(page: Int, query: String, filters: FilterList): Request {
if (page == 1) searchPage = 1
searchQuery = query
return popularMangaRequest(page)
}
override fun searchMangaParse(response: Response): MangasPage {
val searchMatches = mutableListOf<SManga>()
val document = response.asJsoup()
searchMatches.addAll(getMatchesFrom(document))
/* call another function if there's more pages to search
not doing it this way can lead to a false "no results found"
if no matches are found on the first page but there are matches
on subsequent pages */
nextPageSelectorElement = document.select(searchMangaNextPageSelector())
while (nextPageSelectorElement.hasText()) {
searchMatches.addAll(searchMorePages())
}
return MangasPage(searchMatches, false)
}
// search the given document for matches
private fun getMatchesFrom(document: Document): MutableList<SManga> {
val searchMatches = mutableListOf<SManga>()
document.select(searchMangaSelector())
.filter { it.text().contains(searchQuery, ignoreCase = true) }
.map { searchMatches.add(searchMangaFromElement(it)) }
return searchMatches
}
// search additional pages if called
private fun searchMorePages(): MutableList<SManga> {
searchPage++
val nextPage = client.newCall(popularMangaRequest(searchPage)).execute().asJsoup()
val searchMatches = mutableListOf<SManga>()
searchMatches.addAll(getMatchesFrom(nextPage))
nextPageSelectorElement = nextPage.select(searchMangaNextPageSelector())
return searchMatches
}
override fun searchMangaSelector() = popularMangaSelector()
override fun searchMangaFromElement(element: Element) = popularMangaFromElement(element)
override fun searchMangaNextPageSelector() = popularMangaNextPageSelector()
}

View File

@ -0,0 +1,26 @@
package eu.kanade.tachiyomi.multisrc.genkan
import eu.kanade.tachiyomi.multisrc.ThemeSourceGenerator
import eu.kanade.tachiyomi.multisrc.ThemeSourceGenerator.Companion.ThemeSourceData
class GenkanOriginalGenerator : ThemeSourceGenerator {
override val themePkg = "genkan"
override val themeClass = "GenkanOriginal"
override val baseVersionCode: Int = 1
override val sources = listOf(
ThemeSourceData.SingleLang("Reaper Scans", "https://reaperscans.com", "en"),
ThemeSourceData.SingleLang("Hatigarm Scans", "https://hatigarmscanz.net", "en"),
ThemeSourceData.SingleLang("SecretScans", "https://secretscans.co", "en"),
)
companion object {
@JvmStatic
fun main(args: Array<String>) {
GenkanOriginalGenerator().createAll()
}
}
}