From cec4ddb0fd362ec5869ec04612827e934402b893 Mon Sep 17 00:00:00 2001 From: Stypox Date: Thu, 27 Nov 2025 14:39:05 +0100 Subject: [PATCH] Depend directly on NewPipeExtractor's git repository This should solve the Jitpack problems, where sometimes built artifacts disappear. This is not also done for nanojson and FilePicker because: - nanojson is not a gradle project, and thus cannot be built by gradle - FilePicker is using a very old version of gradle, and while that's not a problem when the project is built on its own, when importing is as a subproject it gets built with the same gradle version as NewPipe, which causes gradle issues However, for some reason we've been using the same Jitpack artifacts for the FilePicker and for nanojson since ages without issues, it's just NewPipeExtractor artifacts that sometimes disappear. --- .gitignore | 1 + app/build.gradle.kts | 6 +-- gradle/libs.versions.toml | 25 +++++---- settings.gradle.kts | 108 ++++++++++++++++++++++++++++++++++---- 4 files changed, 117 insertions(+), 23 deletions(-) diff --git a/.gitignore b/.gitignore index 49267a9f064..614b18c02cb 100644 --- a/.gitignore +++ b/.gitignore @@ -11,6 +11,7 @@ captures/ *.class app/debug/ app/release/ +checkouts/ # vscode / eclipse files *.classpath diff --git a/app/build.gradle.kts b/app/build.gradle.kts index 5cf357c7458..40e050196da 100644 --- a/app/build.gradle.kts +++ b/app/build.gradle.kts @@ -206,9 +206,9 @@ dependencies { coreLibraryDesugaring(libs.android.desugar) /** NewPipe libraries **/ - implementation(libs.newpipe.nanojson) - implementation(libs.newpipe.extractor) - implementation(libs.newpipe.filepicker) + implementation(libs.teamnewpipe.nanojson) + implementation(libs.teamnewpipe.extractor) + implementation(libs.teamnewpipe.filepicker) /** Checkstyle **/ checkstyle(libs.puppycrawl.checkstyle) diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 38fca9b2ee5..384cd454c67 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -19,6 +19,8 @@ documentfile = "1.1.0" exoplayer = "2.19.1" fragment = "1.8.9" groupie = "2.10.1" +# referenced in settings.gradle.kts +includegitPlugin = "0.3.1" jsoup = "1.21.2" junit = "4.13.2" junit-ext = "1.3.0" @@ -48,17 +50,18 @@ sonarqube = "7.0.1.6134" statesaver = "1.4.1" # TODO: Drop because it is deprecated and incompatible with KSP2 stetho = "1.6.0" swiperefreshlayout = "1.1.0" -# You can use a local version by uncommenting a few lines in settings.gradle -# Or you can use a commit you pushed to GitHub by just replacing TeamNewPipe with your GitHub -# name and the commit hash with the commit hash of the (pushed) commit you want to test # This works thanks to JitPack: https://jitpack.io/ teamnewpipe-filepicker = "5.0.0" teamnewpipe-nanojson = "e9d656ddb49a412a5a0a5d5ef20ca7ef09549996" -# WORKAROUND: if you get errors with the NewPipeExtractor dependency, replace `v0.XX.Y` with -# the corresponding commit hash, since JitPack sometimes deletes artifacts. -# If there’s already a git hash, just add more of it to the end (or remove a letter) -# to cause jitpack to regenerate the artifact. -teamnewpipe-newpipe-extractor = "3af73262cc60cf555fd5f1d691f6c58e2db38ef5" +# The NewPipeExtractor commit hash is used in settings.gradle.kts to depend on the NewPipeExtractor +# git repositories directly. nanojson and filepicker use incompatible build systems so we still rely +# on Jitpack for those two. +# If you want to use a local copy of NewPipeExtractor (provided that you have cloned it in +# `../NewPipeExtractor`), you can add `useLocalNewPipeExtractor=true` in `local.properties`. +# Or you use a commit you pushed to GitHub by just replacing TeamNewPipe with your GitHub +# name in settings.gradle.kts and update the commit hash here with the commit hash of the +# (pushed) commit you want to test. +teamnewpipe-newpipeextractor = "724cc46f422a647884021d156bb0242784309843" viewpager2 = "1.1.0" webkit = "1.14.0" work = "2.10.5" # Newer versions require minSdk >= 23 @@ -113,9 +116,6 @@ lisawray-groupie-core = { module = "com.github.lisawray.groupie:groupie", versio lisawray-groupie-viewbinding = { module = "com.github.lisawray.groupie:groupie-viewbinding", version.ref = "groupie" } livefront-bridge = { module = "com.github.livefront:bridge", version.ref = "bridge" } mockito-core = { module = "org.mockito:mockito-core", version.ref = "mockitoCore" } -newpipe-extractor = { module = "com.github.TeamNewPipe:NewPipeExtractor", version.ref = "teamnewpipe-newpipe-extractor" } -newpipe-filepicker = { module = "com.github.TeamNewPipe:NoNonsense-FilePicker", version.ref = "teamnewpipe-filepicker" } -newpipe-nanojson = { module = "com.github.TeamNewPipe:nanojson", version.ref = "teamnewpipe-nanojson" } noties-markwon-core = { module = "io.noties.markwon:core", version.ref = "markwon" } noties-markwon-linkify = { module = "io.noties.markwon:linkify", version.ref = "markwon" } ocpsoft-prettytime = { module = "org.ocpsoft.prettytime:prettytime", version.ref = "prettytime" } @@ -128,6 +128,9 @@ squareup-leakcanary-plumber = { module = "com.squareup.leakcanary:plumber-androi squareup-leakcanary-watcher = { module = "com.squareup.leakcanary:leakcanary-object-watcher-android", version.ref = "leakcanary" } squareup-okhttp = { module = "com.squareup.okhttp3:okhttp", version.ref = "okhttp" } squareup-picasso = { module = "com.squareup.picasso:picasso", version.ref = "picasso" } +teamnewpipe-extractor = { module = "git.included.build:NewPipeExtractor" } +teamnewpipe-filepicker = { module = "com.github.TeamNewPipe:NoNonsense-FilePicker", version.ref = "teamnewpipe-filepicker" } +teamnewpipe-nanojson = { module = "com.github.TeamNewPipe:nanojson", version.ref = "teamnewpipe-nanojson" } [plugins] android-application = { id = "com.android.application", version.ref = "agp" } diff --git a/settings.gradle.kts b/settings.gradle.kts index 60a40c985c9..66fdb716411 100644 --- a/settings.gradle.kts +++ b/settings.gradle.kts @@ -2,6 +2,10 @@ * SPDX-FileCopyrightText: 2025 NewPipe e.V. * SPDX-License-Identifier: GPL-3.0-or-later */ +import me.champeau.gradle.igp.gitRepositories +import org.eclipse.jgit.api.Git +import java.io.FileInputStream +import java.util.Properties pluginManagement { repositories { @@ -10,6 +14,20 @@ pluginManagement { mavenCentral() } } + +plugins { + // need to manually read version catalog because it is not available in settings.gradle.kts + // this code is duplicate with the below but there is no way to avoid it... + fun findInVersionCatalog(versionIdentifier: String): String { + val regex = "^.*$versionIdentifier *= *\"([^\"]+)\".*$".toRegex() + return File("gradle/libs.versions.toml") + .readLines() + .firstNotNullOf { regex.find(it)?.groupValues?.get(1) } + } + + id("me.champeau.includegit") version findInVersionCatalog("includegitPlugin") +} + dependencyResolutionManagement { repositoriesMode.set(RepositoriesMode.FAIL_ON_PROJECT_REPOS) repositories { @@ -21,13 +39,85 @@ dependencyResolutionManagement { } include (":app") -// Use a local copy of NewPipe Extractor by uncommenting the lines below. -// We assume, that NewPipe and NewPipe Extractor have the same parent directory. -// If this is not the case, please change the path in includeBuild(). -//includeBuild("../NewPipeExtractor") { -// dependencySubstitution { -// substitute(module("com.github.TeamNewPipe:NewPipeExtractor")) -// .using(project(":extractor")) -// } -//} +// All of the code below handles depending on libraries from git repos, in particular +// NewPipeExtractor. The git commit to checkout can be updated in libs.versions.toml. +// If you want to use a local copy of NewPipeExtractor (provided that you have cloned it in +// `../NewPipeExtractor`), you can add `useLocalNewPipeExtractor=true` to `local.properties`. +// Or you use a commit you pushed to GitHub by just replacing TeamNewPipe with your GitHub +// name below here and update the commit hash in libs.versions.toml with the commit hash of the +// (pushed) commit you want to test. + +data class IncludeGitRepo( + val name: String, + val uri: String, + val projectPath: String, + val commit: String, +) + +// need to manually read version catalog because it is not available in settings.gradle.kts +// this code is duplicate with the above but there is no way to avoid it... +fun findInVersionCatalog(versionIdentifier: String): String { + val regex = "^.*$versionIdentifier *= *\"([^\"]+)\".*$".toRegex() + return File("gradle/libs.versions.toml") + .readLines() + .firstNotNullOf { regex.find(it)?.groupValues?.get(1) } +} + +val newPipeExtractor = IncludeGitRepo( + name = "NewPipeExtractor", + uri = "https://github.com/TeamNewPipe/NewPipeExtractor", + projectPath = ":extractor", + commit = findInVersionCatalog("teamnewpipe-newpipeextractor"), +) + +val localProperties = Properties().apply { + try { + load(FileInputStream(File(rootDir, "local.properties"))) + } catch (e: Throwable) { + println("Warning: can't read local.properties: $e") + } +} + +if (localProperties.getOrDefault("useLocalNewPipeExtractor", "") == "true") { + includeBuild("../${newPipeExtractor.name}") { + dependencySubstitution { + substitute(module("git.included.build:${newPipeExtractor.name}")) + .using(project(newPipeExtractor.projectPath)) + } + } + +} else { + // if the repo has already been cloned, the gitRepositories plugin is buggy and doesn't + // fetch the remote repo before trying to checkout the commit (in case the commit has changed), + // and doesn't clone the repo again if the remote changed, so we need to do it manually + val repo = newPipeExtractor + val file = File("$rootDir/checkouts/${repo.name}") + if (file.isDirectory) { + val git = Git.open(file) + val sameRemote = git.remoteList().call() + .any { rem -> rem.urIs.any { uri -> uri.toString() == repo.uri } } + if (sameRemote) { + // the commit may have changed, fetch again + git.fetch().call() + } else { + // the remote changed, delete the repository and start from scratch + println("Git: remote for ${repo.name} changed, deleting the current folder") + file.deleteRecursively() + } + } + + gitRepositories { + include(repo.name) { + uri.set(repo.uri) + commit.set(repo.commit) + autoInclude.set(false) + includeBuild("") { + dependencySubstitution { + substitute(module("git.included.build:${repo.name}")) + .using(project(repo.projectPath)) + } + } + } + } +}