From e6fcd0c7535aef3f2e5014060ab2e5aeb2343d23 Mon Sep 17 00:00:00 2001 From: Claudemirovsky <63046606+Claudemirovsky@users.noreply.github.com> Date: Mon, 17 Jul 2023 07:43:52 +0000 Subject: [PATCH] [skip ci] chore: Merge upstream changes (#1916) --- .../{report_issue.yml => 01_report_issue.yml} | 16 +- ...quest_source.yml => 02_request_source.yml} | 3 +- ...rl_change.yml => 03_report_url_change.yml} | 18 +- .../ISSUE_TEMPLATE/04_report_dead_source.yml | 63 +++++ ...est_feature.yml => 05_request_feature.yml} | 14 +- .../{request_meta.yml => 06_request_meta.yml} | 4 +- ...est_removal.yml => 07_request_removal.yml} | 0 .github/workflows/batch_close_issues.yml | 5 +- .github/workflows/build_pull_request.yml | 11 +- .github/workflows/build_push.yml | 11 +- .github/workflows/issue_moderator.yml | 12 +- CONTRIBUTING.md | 218 ++++++++++++------ multisrc/build.gradle.kts | 40 +--- .../java/generator/ThemeSourceGenerator.kt | 15 +- settings.gradle.kts | 15 +- .../de/animestream/AnimeStream.kt | 2 +- 16 files changed, 301 insertions(+), 146 deletions(-) rename .github/ISSUE_TEMPLATE/{report_issue.yml => 01_report_issue.yml} (89%) rename .github/ISSUE_TEMPLATE/{request_source.yml => 02_request_source.yml} (98%) rename .github/ISSUE_TEMPLATE/{report_url_change.yml => 03_report_url_change.yml} (76%) create mode 100644 .github/ISSUE_TEMPLATE/04_report_dead_source.yml rename .github/ISSUE_TEMPLATE/{request_feature.yml => 05_request_feature.yml} (88%) rename .github/ISSUE_TEMPLATE/{request_meta.yml => 06_request_meta.yml} (93%) rename .github/ISSUE_TEMPLATE/{request_removal.yml => 07_request_removal.yml} (100%) diff --git a/.github/ISSUE_TEMPLATE/report_issue.yml b/.github/ISSUE_TEMPLATE/01_report_issue.yml similarity index 89% rename from .github/ISSUE_TEMPLATE/report_issue.yml rename to .github/ISSUE_TEMPLATE/01_report_issue.yml index cbb1872f0..5582f73b0 100644 --- a/.github/ISSUE_TEMPLATE/report_issue.yml +++ b/.github/ISSUE_TEMPLATE/01_report_issue.yml @@ -2,10 +2,11 @@ name: 🐞 Issue report description: Report a source issue in Aniyomi labels: [Bug] body: + - type: input id: source attributes: - label: Source information and language + label: Source information description: | You can find the extension name and version in **Browse → Extensions**. placeholder: | @@ -13,6 +14,15 @@ body: validations: required: true + - type: input + id: language + attributes: + label: Source language + placeholder: | + Example: "English" + validations: + required: true + - type: textarea id: reproduce-steps attributes: @@ -53,7 +63,7 @@ body: description: | You can find your Aniyomi version in **More → About**. placeholder: | - Example: "0.12.3.10" or "Beta r6080" + Example: "0.12.3.10" or "Preview r6151" validations: required: true @@ -85,7 +95,7 @@ body: required: true - label: I have written a short but informative title. required: true - - label: I have updated the app to version **[0.12.3.10](https://github.com/aniyomiorg/aniyomi/releases/latest)** + - label: I have updated the app to version **[0.12.3.10](https://github.com/aniyomiorg/aniyomi/releases/latest)**. required: true - label: I have updated all installed extensions. required: true diff --git a/.github/ISSUE_TEMPLATE/request_source.yml b/.github/ISSUE_TEMPLATE/02_request_source.yml similarity index 98% rename from .github/ISSUE_TEMPLATE/request_source.yml rename to .github/ISSUE_TEMPLATE/02_request_source.yml index 2277cb592..bc4cd1315 100644 --- a/.github/ISSUE_TEMPLATE/request_source.yml +++ b/.github/ISSUE_TEMPLATE/02_request_source.yml @@ -2,6 +2,7 @@ name: 🌐 Source request description: Suggest a new source for Aniyomi labels: [Source request] body: + - type: input id: name attributes: @@ -23,7 +24,7 @@ body: - type: input id: language attributes: - label: Language + label: Source language placeholder: | Example: "English" validations: diff --git a/.github/ISSUE_TEMPLATE/report_url_change.yml b/.github/ISSUE_TEMPLATE/03_report_url_change.yml similarity index 76% rename from .github/ISSUE_TEMPLATE/report_url_change.yml rename to .github/ISSUE_TEMPLATE/03_report_url_change.yml index a73f2bb89..b01c19fdd 100644 --- a/.github/ISSUE_TEMPLATE/report_url_change.yml +++ b/.github/ISSUE_TEMPLATE/03_report_url_change.yml @@ -1,15 +1,25 @@ name: 🔗 URL change report -description: Report a URL change for an existing source +description: Report URL change of an existing source labels: [Bug,Domain changed] body: + - type: input id: source attributes: - label: Source information and language + label: Source information description: | You can find the extension name and version in **Browse → Extensions**. placeholder: | - Example: "NotRealSource 13.1 (English)" + Example: "NotRealSource 13.1" + validations: + required: true + + - type: input + id: language + attributes: + label: Source language + placeholder: | + Example: "English" validations: required: true @@ -41,7 +51,7 @@ body: required: true - label: I have updated all installed extensions. required: true - - label: I have checked if the source URL is not already updated by opening WebView. + - label: I have opened WebView and checked that the source URL is not updated yet. required: true - label: I will fill out all of the requested information in this form. required: true diff --git a/.github/ISSUE_TEMPLATE/04_report_dead_source.yml b/.github/ISSUE_TEMPLATE/04_report_dead_source.yml new file mode 100644 index 000000000..59942ab1a --- /dev/null +++ b/.github/ISSUE_TEMPLATE/04_report_dead_source.yml @@ -0,0 +1,63 @@ +name: ❌ Dead source report +description: Source is down and website is closed +labels: [Source is down] +body: + + - type: markdown + attributes: + value: | + ### Notice + If you have a lot of dead sources to report, please go back and submit a single meta request. + + - type: input + id: source + attributes: + label: Source name + description: | + You can find the extension name in **Browse → Extensions**. + placeholder: | + Example: "NotRealSource" + validations: + required: true + + - type: input + id: language + attributes: + label: Source language + placeholder: | + Example: "English" + validations: + required: true + + - type: input + id: link + attributes: + label: Source link + placeholder: | + Example: "https://notrealsource.org" + validations: + required: true + + - type: textarea + id: other-details + attributes: + label: Other details + placeholder: | + Additional details and attachments. + + - type: checkboxes + id: acknowledgements + attributes: + label: Acknowledgements + description: Your issue will be closed if you haven't done these steps. + options: + - label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open or closed issue. + required: true + - label: I have written a title with source name. + required: true + - label: I have updated all installed extensions. + required: true + - label: I have opened WebView and checked that the source website is down. + required: true + - label: I will fill out all of the requested information in this form. + required: true diff --git a/.github/ISSUE_TEMPLATE/request_feature.yml b/.github/ISSUE_TEMPLATE/05_request_feature.yml similarity index 88% rename from .github/ISSUE_TEMPLATE/request_feature.yml rename to .github/ISSUE_TEMPLATE/05_request_feature.yml index dc5735922..ae3dc0e5a 100644 --- a/.github/ISSUE_TEMPLATE/request_feature.yml +++ b/.github/ISSUE_TEMPLATE/05_request_feature.yml @@ -2,14 +2,24 @@ name: ⭐ Feature request description: Suggest a feature to improve an existing source labels: [Feature request] body: + - type: input id: source attributes: - label: Source name and language + label: Source name description: | You can find the extension name in **Browse → Extensions**. placeholder: | - Example: "DopeBox (English)" + Example: "DopeBox" + validations: + required: true + + - type: input + id: language + attributes: + label: Source language + placeholder: | + Example: "English" validations: required: true diff --git a/.github/ISSUE_TEMPLATE/request_meta.yml b/.github/ISSUE_TEMPLATE/06_request_meta.yml similarity index 93% rename from .github/ISSUE_TEMPLATE/request_meta.yml rename to .github/ISSUE_TEMPLATE/06_request_meta.yml index 3dcf72416..ec45f3de0 100644 --- a/.github/ISSUE_TEMPLATE/request_meta.yml +++ b/.github/ISSUE_TEMPLATE/06_request_meta.yml @@ -1,4 +1,4 @@ -name: 🧠 Meta feature request +name: 🧠 Meta request description: Suggest improvements to the project labels: [Meta request] body: @@ -27,7 +27,7 @@ body: label: Acknowledgements description: Your issue will be closed if you haven't done these steps. options: - - label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open issue. + - label: I have searched the existing issues and this is a new ticket, **NOT** a duplicate or related to another open (or closed) issue. required: true - label: I have written a short but informative title. required: true diff --git a/.github/ISSUE_TEMPLATE/request_removal.yml b/.github/ISSUE_TEMPLATE/07_request_removal.yml similarity index 100% rename from .github/ISSUE_TEMPLATE/request_removal.yml rename to .github/ISSUE_TEMPLATE/07_request_removal.yml diff --git a/.github/workflows/batch_close_issues.yml b/.github/workflows/batch_close_issues.yml index f41e195d3..4f987fab8 100644 --- a/.github/workflows/batch_close_issues.yml +++ b/.github/workflows/batch_close_issues.yml @@ -18,9 +18,8 @@ jobs: # Close everything older than ~6 months days-before-issue-stale: 180 days-before-issue-close: 0 - any-of-issue-labels: "Source request" - exempt-issue-labels: do-not-autoclose - close-issue-message: "In an effort to have a more manageable issue backlog, we're closing older requests that weren't addressed. If you think the source may still benefit others, please open a new request." + exempt-issue-labels: "do-not-autoclose,Meta request" + close-issue-message: "In an effort to have a more manageable issue backlog, we're closing older requests that weren't addressed since there's a low chance of it being addressed if it hasn't already. If your request is still relevant, please [open a new request](https://github.com/aniyomiorg/aniyomi-extensions/issues/new/choose)." close-issue-reason: not_planned ascending: true operations-per-run: 250 diff --git a/.github/workflows/build_pull_request.yml b/.github/workflows/build_pull_request.yml index 2daa94ea5..f285dfd61 100644 --- a/.github/workflows/build_pull_request.yml +++ b/.github/workflows/build_pull_request.yml @@ -4,6 +4,7 @@ on: pull_request: paths-ignore: - '**.md' + - '.github/workflows/issue_moderator.yml' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number }} @@ -50,6 +51,8 @@ jobs: isIndividualChanged=1 elif [[ ${changedFile} == multisrc/* ]]; then isMultisrcChanged=1 + elif [[ ${changedFile} == .github/workflows/issue_moderator.yml ]]; then + true elif [[ ${changedFile} == *.md ]]; then true else @@ -63,7 +66,7 @@ jobs: - name: Generate multisrc sources if: ${{ steps.parse-changed-files.outputs.isMultisrcChanged == '1' }} - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 with: arguments: :multisrc:generateExtensions @@ -111,7 +114,7 @@ jobs: distribution: adopt - name: Generate sources from the multi-source library - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 env: CI_MODULE_GEN: "true" with: @@ -119,7 +122,7 @@ jobs: cache-read-only: true - name: Build extensions (chunk ${{ matrix.chunk }}) - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 env: CI_MULTISRC: "true" CI_CHUNK_NUM: ${{ matrix.chunk }} @@ -145,7 +148,7 @@ jobs: distribution: adopt - name: Build extensions (chunk ${{ matrix.chunk }}) - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 env: CI_MULTISRC: "false" CI_CHUNK_NUM: ${{ matrix.chunk }} diff --git a/.github/workflows/build_push.yml b/.github/workflows/build_push.yml index f55b62130..79568a559 100644 --- a/.github/workflows/build_push.yml +++ b/.github/workflows/build_push.yml @@ -6,6 +6,7 @@ on: - master paths-ignore: - '**.md' + - '.github/workflows/issue_moderator.yml' concurrency: group: ${{ github.workflow }} @@ -62,7 +63,7 @@ jobs: distribution: adopt - name: Generate multisrc sources - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 with: arguments: :multisrc:generateExtensions @@ -115,14 +116,14 @@ jobs: echo ${{ secrets.SIGNING_KEY }} | base64 -d > signingkey.jks - name: Generate sources from the multi-source library - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 env: CI_MODULE_GEN: "true" with: arguments: :multisrc:generateExtensions - name: Build extensions (chunk ${{ matrix.chunk }}) - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 env: CI_MULTISRC: "true" CI_CHUNK_NUM: ${{ matrix.chunk }} @@ -166,7 +167,7 @@ jobs: echo ${{ secrets.SIGNING_KEY }} | base64 -d > signingkey.jks - name: Build extensions (chunk ${{ matrix.chunk }}) - uses: gradle/gradle-command-action@v2 + uses: gradle/gradle-build-action@v2 env: CI_MULTISRC: "false" CI_CHUNK_NUM: ${{ matrix.chunk }} @@ -190,8 +191,8 @@ jobs: publish_repo: name: Publish repo needs: - - build_individual - build_multisrc + - build_individual if: "github.repository == 'aniyomiorg/aniyomi-extensions'" runs-on: ubuntu-latest steps: diff --git a/.github/workflows/issue_moderator.yml b/.github/workflows/issue_moderator.yml index 315d289fb..4b12d99ed 100644 --- a/.github/workflows/issue_moderator.yml +++ b/.github/workflows/issue_moderator.yml @@ -14,10 +14,16 @@ jobs: uses: tachiyomiorg/issue-moderator-action@v2 with: repo-token: ${{ secrets.GITHUB_TOKEN }} + duplicate-label: Duplicate + duplicate-check-enabled: true - duplicate-check-label: Source request + duplicate-check-labels: | + ["Source request", "Domain changed"] + existing-check-enabled: true - existing-check-label: Source request + existing-check-labels: | + ["Source request", "Domain changed"] + auto-close-rules: | [ { @@ -37,7 +43,7 @@ jobs: }, { "type": "both", - "regex": ".*(?:fail(?:ed|ure)?|can\\s*(?:no|')?t|unable|get past|(?Steps 1. Make sure to delete "repo" branch in your fork. You may also want to disable Actions in the repo settings. + + **Also make sure you are using the latest version of Git as many commands used here are pretty new.** + 2. Do a partial clone. ```bash - git clone --filter=blob:none --no-checkout + git clone --filter=blob:none --sparse cd aniyomi-extensions/ ``` 3. Configure sparse checkout. + + There are two modes of pattern matching. The default is cone (🔺) mode. + Cone mode enables significantly faster pattern matching for big monorepos + and the sparse index feature to make Git commands more responsive. + In this mode, you can only filter by file path, which is less flexible + and might require more work when the project structure changes. + + You can skip this code block to use legacy mode if you want easier filters. + It won't be much slower as the repo doesn't have that many files. + + To enable cone mode together with sparse index, follow these steps: + + ```bash + git sparse-checkout set --cone --sparse-index + # add project folders + git sparse-checkout add .run buildSrc core gradle lib multisrc/src/main/java/generator + # add a single source + git sparse-checkout add src// + # add a multisrc theme + git sparse-checkout add multisrc/src/main/java/eu/kanade/tachiyomi/multisrc/ + git sparse-checkout add multisrc/overrides/ + ``` + + To remove a source, open `.git/info/sparse-checkout` and delete the exact + lines you typed when adding it. Don't touch the other auto-generated lines + unless you fully understand how cone mode works, or you might break it. + + To use the legacy non-cone mode, follow these steps: + ```bash # enable sparse checkout - git sparse-checkout set + git sparse-checkout set --no-cone # edit sparse checkout filter vim .git/info/sparse-checkout # alternatively, if you have VS Code installed @@ -85,10 +117,14 @@ Some alternative steps can be followed to ignore "repo" branch and skip unrelate # or type the source name directly ``` + + Explanation: the rules are like `gitignore`. We first exclude all sources + while retaining project folders, then add the needed sources back manually. + 4. Configure remotes. ```bash # add upstream - git remote add upstream + git remote add upstream # optionally disable push to upstream git remote set-url --push upstream no_pushing # ignore 'repo' branch of upstream @@ -100,8 +136,6 @@ Some alternative steps can be followed to ignore "repo" branch and skip unrelate git remote update # track master of upstream instead of fork git branch master -u upstream/master - # checkout - git switch master ``` 5. Useful configurations. (optional) ```bash @@ -109,15 +143,28 @@ Some alternative steps can be followed to ignore "repo" branch and skip unrelate git config remote.origin.prune true # fast-forward only when pulling master branch git config pull.ff only + # Add an alias to sync master branch without fetching useless blobs. + # If you run `git pull` to fast-forward in a blobless clone like this, + # all blobs (files) in the new commits are still fetched regardless of + # sparse rules, which makes the local repo accumulate unused files. + # Use `git sync-master` to avoid this. Be careful if you have changes + # on master branch, which is not a good practice. + git config alias.sync-master '!git switch master && git fetch upstream && git reset --keep FETCH_HEAD' ``` 6. Later, if you change the sparse checkout filter, run `git sparse-checkout reapply`. -Read more on [partial clone](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/), [sparse checkout](https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/) and [negative refspecs](https://github.blog/2020-10-19-git-2-29-released/#user-content-negative-refspecs). +Read more on +[Git's object model](https://github.blog/2020-12-17-commits-are-snapshots-not-diffs/), +[partial clone](https://github.blog/2020-12-21-get-up-to-speed-with-partial-clone-and-shallow-clone/), +[sparse checkout](https://github.blog/2020-01-17-bring-your-monorepo-down-to-size-with-sparse-checkout/), +[sparse index](https://github.blog/2021-11-10-make-your-monorepo-feel-small-with-gits-sparse-index/), +and [negative refspecs](https://github.blog/2020-10-19-git-2-29-released/#user-content-negative-refspecs). ## Getting help -- Join [the Discord server](https://discord.gg/F32UjdJZrR) for online help and to ask questions while developing your extension. +- Join [the Discord server](https://discord.gg/F32UjdJZrR) for online help and to ask questions while developing your extension. When doing so, please ask it in the `#dev` channel. + - There are some features and tricks that are not explored in this document. Refer to existing extension code for examples. ## Writing an extension @@ -139,18 +186,19 @@ $ tree src/// src/// ├── AndroidManifest.xml ├── build.gradle +├── build.gradle ├── res -│ ├── mipmap-hdpi -│ │ └── ic_launcher.png -│ ├── mipmap-mdpi -│ │ └── ic_launcher.png -│ ├── mipmap-xhdpi -│ │ └── ic_launcher.png -│ ├── mipmap-xxhdpi -│ │ └── ic_launcher.png -│ ├── mipmap-xxxhdpi -│ │ └── ic_launcher.png -│ └── web_hi_res_512.png +│   ├── mipmap-hdpi +│   │   └── ic_launcher.png +│   ├── mipmap-mdpi +│   │   └── ic_launcher.png +│   ├── mipmap-xhdpi +│   │   └── ic_launcher.png +│   ├── mipmap-xxhdpi +│   │   └── ic_launcher.png +│   ├── mipmap-xxxhdpi +│   │   └── ic_launcher.png +│   └── web_hi_res_512.png └── src └── eu └── kanade @@ -193,7 +241,7 @@ apply from: "$rootDir/common.gradle" | `libVersion` | (Optional, defaults to `13`) The version of the [extensions library](https://github.com/aniyomiorg/extensions-lib) used. | | `containsNsfw` | (Optional, defaults to `false`) Flag to indicate that a source contains NSFW content. | -The extension's version name is generated automatically by concatenating `libVersion` and `extVersionCode`. With the example used above, the version would be `12.1`. +The extension's version name is generated automatically by concatenating `libVersion` and `extVersionCode`. With the example used above, the version would be `13`. ### Core dependencies @@ -201,34 +249,47 @@ The extension's version name is generated automatically by concatenating `libVer Extensions rely on [extensions-lib](https://github.com/aniyomiorg/extensions-lib), which provides some interfaces and stubs from the [app](https://github.com/aniyomiorg/aniyomi) for compilation purposes. The actual implementations can be found [here](https://github.com/aniyomiorg/aniyomi/tree/master/app/src/main/java/eu/kanade/tachiyomi/animesource). Referencing the actual implementation will help with understanding extensions' call flow. -#### Rate limiting library +#### CryptoAES library -[`lib-ratelimit`](https://github.com/aniyomiorg/aniyomi-extensions/tree/master/lib/ratelimit) is a library for adding rate limiting functionality as an [OkHttp interceptor](https://square.github.io/okhttp/interceptors/). +The [`lib-cryptoaes`](https://github.com/aniyomiorg/aniyomi-extensions/tree/master/lib/cryptoaes) provides utilities for decrypting AES-encrypted data, like data encrypted with AES+EvpKDF (The key-derivation algorithm used by the [cryptojs](https://cryptojs.gitbook.io/docs/) library). It also includes some utilities to decrypt strings in the [jsfuck](https://jsfuck.com/) format. ```gradle dependencies { - implementation project(':lib-ratelimit') + implementation(project(":lib-cryptoaes")) +} +``` + +#### Unpacker library + +The [`lib-unpacker`](https://github.com/aniyomiorg/aniyomi-extensions/tree/master/lib/unpacker) library provides a deobfuscator(unpacker) for javascript code obfuscated with the [jspacker](http://dean.edwards.name/packer/) algorithm. + +```gradle +dependencies { + implementation(project(":lib-unpacker")) +} +``` + +#### Synchrony library + +[`lib-synchrony`](https://github.com/aniyomiorg/aniyomi-extensions/tree/master/lib/synchrony) is a library that bundles and runs the [synchrony](https://github.com/relative/synchrony) deobfuscator with your extension to help when deobfuscating obfuscated javascript. Useful to get data on highly obfuscated javascript code. + +```gradle +dependencies { + implementation(project(":lib-synchrony")) } ``` #### Additional dependencies -You may find yourself needing additional functionality and wanting to add more dependencies to your `build.gradle` file. Since extensions are run within the main Aniyomi app, you can make use of [its dependencies](https://github.com/aniyomiorg/aniyomi/blob/master/app/build.gradle.kts). - -For example, an extension that needs coroutines, it could add the following: - -```gradle -dependencies { - compileOnly(libs.bundles.coroutines) -} -``` +If you find yourself needing additional functionality, you can add more dependencies to your `build.gradle` file. +Many of [the dependencies](https://github.com/aniyomiorg/aniyomi/blob/master/app/build.gradle.kts) from the main Aniyomi app are exposed to extensions by default. > Note that several dependencies are already exposed to all extensions via Gradle version catalog. > To view which are available view `libs.versions.toml` under the `gradle` folder -Notice that we're using `compileOnly` instead of `implementation`, since the app already contains it. You could use `implementation` instead for a new dependency, or you prefer not to rely on whatever the main app has at the expense of app size. +Notice that we're using `compileOnly` instead of `implementation` if the app already contains it. You could use `implementation` instead for a new dependency, or you prefer not to rely on whatever the main app has at the expense of app size. -Note that using `compileOnly` restricts you to versions that must be compatible with those used in [Aniyomi v0.10.12+](https://github.com/aniyomiorg/aniyomi/blob/v0.10.12/app/build.gradle.kts) for proper backwards compatibility. +Note that using `compileOnly` restricts you to versions that must be compatible with those used in [the latest stable version of Aniyomi](https://github.com/aniyomiorg/aniyomi/releases/latest). ### Extension main class @@ -246,7 +307,7 @@ The class which is referenced and defined by `extClass` in `build.gradle`. This | ----- | ----------- | | `name` | Name displayed in the "Sources" tab in Aniyomi. | | `baseUrl` | Base URL of the source without any trailing slashes. | -| `lang` | An ISO 639-1 compliant language code (two letters in lower case). | +| `lang` | An ISO 639-1 compliant language code (two letters in lower case in most cases, but can also include the country/dialect part by using a simple dash character). | | `id` | Identifier of your source, automatically set in `AnimeHttpSource`. It should only be manually overriden if you need to copy an existing autogenerated ID. | ### Extension call flow @@ -256,9 +317,9 @@ The class which is referenced and defined by `extClass` in `build.gradle`. This a.k.a. the Browse source entry point in the app (invoked by tapping on the source name). - The app calls `fetchPopularAnime` which should return a `AnimesPage` containing the first batch of found `SAnime` entries. - - This method supports pagination. When user scrolls the manga list and more results must be fetched, the app calls it again with increasing `page` values(starting with `page=1`). This continues until `AnimesPage.hasNextPage` is passed as `true` and `AnimesPage.mangas` is not empty. -- To show the list properly, the app needs `url`, `title` and `thumbnail_url`. You must set them here. The rest of the fields could be filled later.(refer to Anime Details below) - - You should set `thumbnail_url` if is available, if not, `fetchAnimeDetails` will be **immediately** called.(this will increase network calls heavily and should be avoided) + - This method supports pagination. When user scrolls the manga list and more results must be fetched, the app calls it again with increasing `page` values(starting with `page=1`). This continues while `AnimesPage.hasNextPage` is passed as `true` and `AnimesPage.mangas` is not empty. +- To show the list properly, the app needs `url`, `title` and `thumbnail_url`. You **must** set them here. The rest of the fields could be filled later.(refer to Anime Details below). + - You should set `thumbnail_url` if is available, if not, `fetchAnimeDetails` will be **immediately** called. (this will increase network calls heavily and should be avoided). #### Latest Anime @@ -271,30 +332,30 @@ a.k.a. the Latest source entry point in the app (invoked by tapping on the "Late - When the user searches inside the app, `fetchSearchAnime` will be called and the rest of the flow is similar to what happens with `fetchPopularAnime`. - If search functionality is not available, return `Observable.just(AnimesPage(emptyList(), false))` -- `getFilterList` will be called to get all filters and filter types. **TODO: explain more about `Filter`** +- `getFilterList` will be called to get all filters and filter types. ##### Filters -The search flow have support to filters that can be added to a `FilterList` inside the `getFilterList` method. When the user changes the filters' state, they will be passed to the `searchRequest`, and they can be iterated to create the request (by getting the `filter.state` value, where the type varies depending on the `Filter` used). You can check the filter types available [here](https://github.com/aniyomiorg/aniyomi/blob/master/app/src/main/java/eu/kanade/tachiyomi/source/model/Filter.kt) and in the table below. +The search flow have support to filters that can be added to a `AnimeFilterList` inside the `getFilterList` method. When the user changes the filter's state, they will be passed to the `searchAnimeRequest`, and they can be iterated to create the request (by getting the `filter.state` value, where the type varies depending on the `AnimeFilter` used). You can check the filter types available [here](https://github.com/aniyomiorg/aniyomi/blob/master/source-api/src/main/java/eu/kanade/tachiyomi/animesource/model/AnimeFilter.kt) and in the table below. | Filter | State type | Description | | ------ | ---------- | ----------- | -| `Filter.Header` | None | A simple header. Useful for separating sections in the list or showing any note or warning to the user. | -| `Filter.Separator` | None | A line separator. Useful for visual distinction between sections. | -| `Filter.Select` | `Int` | A select control, similar to HTML's ``. | -| `Filter.CheckBox` | `Boolean` | A checkbox control, similar to HTML's ``. The state is `true` if it's checked. | -| `Filter.TriState` | `Int` | A enhanced checkbox control that supports an excluding state. The state can be compared with `STATE_IGNORE`, `STATE_INCLUDE` and `STATE_EXCLUDE` constants of the class. | -| `Filter.Group` | `List` | A group of filters (preferentially of the same type). The state will be a `List` with all the states. | -| `Filter.Sort` | `Selection` | A control for sorting, with support for the ordering. The state indicates which item index is selected and if the sorting is `ascending`. | +| `AnimeFilter.Header` | None | A simple header. Useful for separating sections in the list or showing any note or warning to the user. | +| `AnimeFilter.Separator` | None | A line separator. Useful for visual distinction between sections. | +| `AnimeFilter.Select` | `Int` | A select control, similar to HTML's ``. | +| `AnimeFilter.CheckBox` | `Boolean` | A checkbox control, similar to HTML's ``. The state is `true` if it's checked. | +| `AnimeFilter.TriState` | `Int` | A enhanced checkbox control that supports an excluding state. The state can be compared with `STATE_IGNORE`, `STATE_INCLUDE` and `STATE_EXCLUDE` constants of the class. | +| `AnimeFilter.Group` | `List` | A group of filters (preferentially of the same type). The state will be a `List` with all the states. | +| `AnimeFilter.Sort` | `Selection` | A control for sorting, with support for the ordering. The state indicates which item index is selected and if the sorting is `ascending`. | -All control filters can have a default state set. It's usually recommended if the source have filters to make the initial state match the popular manga list, so when the user open the filter sheet, the state is equal and represents the current manga showing. +All control filters can have a default state set. It's usually recommended if the source have filters to make the initial state match the popular anime list, so when the user open the filter sheet, the state is equal and represents the current anime showing. -The `Filter` classes can also be extended, so you can create new custom filters like the `UriPartFilter`: +The `AnimeFilter` classes can also be extended, so you can create new custom filters like the `UriPartFilter`: ```kotlin open class UriPartFilter(displayName: String, private val vals: Array>) : - Filter.Select(displayName, vals.map { it.first }.toTypedArray()) { + AnimeFilter.Select(displayName, vals.map { it.first }.toTypedArray()) { fun toUriPart() = vals[state].second } ``` @@ -306,11 +367,11 @@ open class UriPartFilter(displayName: String, private val vals: Array - │ ├── Generator.kt - │ └── .kt + │   └── kanade + │   └── tachiyomi + │   └── multisrc + │   └── + │   ├── Generator.kt + │   └── .kt └── generator ├── GeneratorMain.kt ├── IntelijConfigurationGeneratorMain.kt @@ -443,13 +505,18 @@ multisrc - `multisrc/src/main/java/eu/kanade/tachiyomi/multisrc//.kt` defines the the theme's default implementation. - `multisrc/src/main/java/eu/kanade/tachiyomi/multisrc//Generator.kt` defines the the theme's generator class, this is similar to a `AnimeSourceFactory` class. - `multisrc/overrides//default/res` is the theme's default icons, if a source doesn't have overrides for `res`, then default icons will be used. -- `multisrc/overrides//default/additional.gradle.kts` defines additional gradle code, this will be copied at the end of all generated sources from this theme. +- `multisrc/overrides//default/additional.gradle` defines additional gradle code, this will be copied at the end of all generated sources from this theme. - `multisrc/overrides//` contains overrides for a source that is defined inside the `Generator.kt` class. - `multisrc/overrides///src` contains source overrides. - `multisrc/overrides///res` contains override for icons. - `multisrc/overrides///additional.gradle` defines additional gradle code, this will be copied at the end of the generated gradle file below the theme's `additional.gradle`. - `multisrc/overrides///AndroidManifest.xml` is copied as an override to the default `AndroidManifest.xml` generation if it exists. +> **Note** +> +> Files ending with `Gen.kt` (i.e. `multisrc/src/main/java/eu/kanade/tachiyomi/multisrc//XxxGen.kt`) +> are considered helper files and won't be copied to generated sources. + ### Development workflow There are three steps in running and testing a theme source: @@ -553,7 +620,7 @@ Inspecting the Logcat allows you to get a good look at the call flow and it's mo If you want to take a deeper look into the network flow, such as taking a look into the request and response bodies, you can use an external tool like `mitm-proxy`. #### Setup your proxy server -We are going to use [mitm-proxy](https://mitmproxy.org/) but you can replace it with any other Web Debugger (i.e. Charles, burp, Fiddler etc). To install and execute, follow the commands bellow. +We are going to use [mitm-proxy](https://mitmproxy.org/) but you can replace it with any other Web Debugger (i.e. Charles, Burp Suite, Fiddler etc). To install and execute, follow the commands bellow. ```console Install the tool. @@ -576,16 +643,29 @@ After installing and running, open your browser and navigate to http://127.0.0.1 #### OkHttp proxy setup Since most of the manga sources are going to use HTTPS, we need to disable SSL verification in order to use the web debugger. For that, add this code to inside your source class: - ```kotlin -class AnimeSource : MadTheme( +package eu.kanade.tachiyomi.animeextension.en.animesource + +import android.annotation.SuppressLint +import eu.kanade.tachiyomi.multisrc.animetheme.AnimeTheme +import okhttp3.OkHttpClient +import java.net.InetSocketAddress +import java.net.Proxy +import java.security.SecureRandom +import java.security.cert.X509Certificate +import javax.net.ssl.SSLContext +import javax.net.ssl.TrustManager +import javax.net.ssl.X509TrustManager + +class AnimeSource : AnimeTheme( "AnimeSource", "https://example.com", "en" ) { private fun OkHttpClient.Builder.ignoreAllSSLErrors(): OkHttpClient.Builder { - val naiveTrustManager = object : X509TrustManager { - override fun getAcceptedIssuers(): Array = arrayOf() + val naiveTrustManager = @SuppressLint("CustomX509TrustManager") + object : X509TrustManager { + override fun getAcceptedIssuers(): Array = emptyArray() override fun checkClientTrusted(certs: Array, authType: String) = Unit override fun checkServerTrusted(certs: Array, authType: String) = Unit } @@ -596,15 +676,15 @@ class AnimeSource : MadTheme( }.socketFactory sslSocketFactory(insecureSocketFactory, naiveTrustManager) - hostnameVerifier(HostnameVerifier { _, _ -> true }) + hostnameVerifier { _, _ -> true } return this } override val client: OkHttpClient = network.cloudflareClient.newBuilder() .ignoreAllSSLErrors() .proxy(Proxy(Proxy.Type.HTTP, InetSocketAddress("10.0.2.2", 8080))) - .... .build() +} ``` Note: `10.0.2.2` is usually the address of your loopback interface in the android emulator. If Aniyomi tells you that it's unable to connect to 10.0.2.2:8080 you will likely need to change it (the same if you are using hardware device). diff --git a/multisrc/build.gradle.kts b/multisrc/build.gradle.kts index fe890de21..cd231c70a 100644 --- a/multisrc/build.gradle.kts +++ b/multisrc/build.gradle.kts @@ -35,40 +35,20 @@ dependencies { } tasks { - register("generateExtensions") { - doLast { - val isWindows = System.getProperty("os.name").toString().lowercase().contains("win") - var classPath = ( - configurations.compileOnly.get().asFileTree.toList() + - listOf( - configurations.androidApis.get().asFileTree.first().absolutePath, // android.jar path - "$projectDir/build/intermediates/aar_main_jar/debug/classes.jar", // jar made from this module - ) - ) - .joinToString(if (isWindows) ";" else ":") + register("generateExtensions") { + classpath = configurations.compileOnly.get() + + configurations.androidApis.get() + // android.jar path + files("$buildDir/intermediates/aar_main_jar/debug/classes.jar") // jar made from this module + mainClass.set("generator.GeneratorMainKt") - var javaPath = "${System.getProperty("java.home")}/bin/java" + workingDir = workingDir.parentFile // project root - val mainClass = "generator.GeneratorMainKt" // Main class we want to execute + errorOutput = System.out // for GitHub workflow commands - if (isWindows) { - classPath = classPath.replace("/", "\\") - javaPath = javaPath.replace("/", "\\") - } - - val javaProcess = ProcessBuilder() - .directory(null).command(javaPath, "-classpath", classPath, mainClass) - .redirectErrorStream(true).start() - - javaProcess.inputStream - .bufferedReader() - .forEachLine(logger::info) - - val exitCode = javaProcess.waitFor() - if (exitCode != 0) { - throw Exception("Java process failed with exit code: $exitCode") - } + if (!logger.isInfoEnabled) { + standardOutput = org.gradle.internal.io.NullOutputStream.INSTANCE } + dependsOn("ktFormat", "ktLint", "assembleDebug") } diff --git a/multisrc/src/main/java/generator/ThemeSourceGenerator.kt b/multisrc/src/main/java/generator/ThemeSourceGenerator.kt index 92afd47b2..2eca26b43 100644 --- a/multisrc/src/main/java/generator/ThemeSourceGenerator.kt +++ b/multisrc/src/main/java/generator/ThemeSourceGenerator.kt @@ -132,9 +132,9 @@ interface ThemeSourceGenerator { File(projectRootPath).let { projectRootFile -> println("Generating $source") - projectRootFile.mkdirs() // remove everything from past runs - cleanDirectory(projectRootFile) + projectRootFile.deleteRecursively() + projectRootFile.mkdirs() writeGradle(projectGradleFile, source, themePkg, baseVersionCode, defaultAdditionalGradlePath, additionalGradleOverridePath) writeAndroidManifest(projectAndroidManifestFile, manifestOverridePath, defaultAndroidManifestPath) @@ -173,7 +173,7 @@ interface ThemeSourceGenerator { File(themeSrcPath).walk() .map { it.toString().replace(themeSrcPath, "") } - .filter { it.endsWith(".kt") && !it.endsWith("Generator.kt") } + .filter { it.endsWith(".kt") && !it.endsWith("Generator.kt") && !it.endsWith("Gen.kt") } .forEach { File("$themeSrcPath/$it").copyTo( File("$themeDestPath/$it"), @@ -239,15 +239,6 @@ interface ThemeSourceGenerator { """.trimMargin(), ) } - - private fun cleanDirectory(dir: File) { - dir.listFiles()?.forEach { - if (it.isDirectory) { - cleanDirectory(it) - } - it.delete() - } - } } } diff --git a/settings.gradle.kts b/settings.gradle.kts index 6c4e5152e..0b42751df 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -16,6 +16,7 @@ pluginManagement { include(":core") +// all the directories under /lib instead of manually adding each to a list File(rootDir, "lib").eachDir { val libName = it.name include(":lib-$libName") @@ -36,13 +37,13 @@ if (System.getenv("CI") == null || System.getenv("CI_MODULE_GEN") == "true") { project(name).projectDir = File("src/${dir.name}/${subdir.name}") } } - // Loads generated extensions from multisrc + // Loads all generated extensions from multisrc File(rootDir, "generated-src").eachDir { dir -> - dir.eachDir { subdir -> - val name = ":extensions:multisrc:${dir.name}:${subdir.name}" - include(name) - project(name).projectDir = File("generated-src/${dir.name}/${subdir.name}") - } + dir.eachDir { subdir -> + val name = ":extensions:multisrc:${dir.name}:${subdir.name}" + include(name) + project(name).projectDir = File("generated-src/${dir.name}/${subdir.name}") + } } /** @@ -67,7 +68,7 @@ if (System.getenv("CI") == null || System.getenv("CI_MODULE_GEN") == "true") { include(":multisrc") project(":multisrc").projectDir = File("multisrc") - // Loads generated extensions from multisrc + // Loads all generated extensions from multisrc File(rootDir, "generated-src").getChunk(chunk, chunkSize)?.forEach { val name = ":extensions:multisrc:${it.parentFile.name}:${it.name}" println(name) diff --git a/src/de/animestream/src/eu/kanade/tachiyomi/animeextension/de/animestream/AnimeStream.kt b/src/de/animestream/src/eu/kanade/tachiyomi/animeextension/de/animestream/AnimeStream.kt index b8560974e..df54673ae 100644 --- a/src/de/animestream/src/eu/kanade/tachiyomi/animeextension/de/animestream/AnimeStream.kt +++ b/src/de/animestream/src/eu/kanade/tachiyomi/animeextension/de/animestream/AnimeStream.kt @@ -3,6 +3,7 @@ package eu.kanade.tachiyomi.animeextension.de.animestream import android.app.Application import android.content.SharedPreferences import android.os.Build +import android.util.Base64 import androidx.annotation.RequiresApi import androidx.preference.ListPreference import androidx.preference.MultiSelectListPreference @@ -24,7 +25,6 @@ import org.jsoup.nodes.Document import org.jsoup.nodes.Element import uy.kohesive.injekt.Injekt import uy.kohesive.injekt.api.get -import android.util.Base64 import kotlin.Exception class AnimeStream : ConfigurableAnimeSource, ParsedAnimeHttpSource() {