diff --git a/.circleci/config.yml b/.circleci/config.yml index c3cbb085ce..34aaf0bfb4 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -151,6 +151,11 @@ commands: - setup-rust-toolchain: rust-version: stable - install-android-ndk + - run: + name: Install required Python dependencies + command: | + sudo apt update + sudo apt install --yes --no-install-recommends python3-venv - run: name: Restrict to Linux builds only command: | diff --git a/CHANGELOG.md b/CHANGELOG.md index e287c44668..35eb7f0832 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,17 @@ [Full changelog](https://github.com/mozilla/glean/compare/v70.0.0...main) +* Android + * Updated to Gradle 9.5.0 ([#3612](https://github.com/mozilla/glean/pull/3612)) + * BREAKING CHANGE: The Glean Gradle plugin no longer uses JetBrains' Python envs plugin to + download a bundled Miniconda, as that plugin is unmaintained and incompatible with Gradle 9. + The plugin now always creates a virtual environment using the Python standard library `venv` + module, so a Python 3.9 or later interpreter must be available on the `PATH`, or pointed at + with the `GLEAN_PYTHON` environment variable. Consumers that set `ext.gleanPythonEnvDir` are + unaffected. Applications applying the Glean Gradle plugin no longer need to apply the + `com.jetbrains.python.envs` plugin themselves and can remove it from their build + ([#3612](https://github.com/mozilla/glean/pull/3612)) + # v70.0.0 (2026-08-20) [Full changelog](https://github.com/mozilla/glean/compare/v69.0.0...v70.0.0) diff --git a/bin/check-artifact.sh b/bin/check-artifact.sh index 0cd3e0e9b6..29d3d8e406 100755 --- a/bin/check-artifact.sh +++ b/bin/check-artifact.sh @@ -41,8 +41,8 @@ check_files() { echo "Artifact: $artifact" if [[ -z "$artifact" ]]; then - echo "No artifact found. Skipping." - return + echo "No artifact found to check. Abort." >&2 + exit 1 fi shift @@ -82,8 +82,8 @@ check_symbol() { echo "Symbol: $symbol" if [[ -z "$artifact" ]]; then - echo "No artifact found. Skipping." - return + echo "No artifact found to check. Abort." >&2 + exit 1 fi nm=$(find "$(rustc --print sysroot)" -name "llvm-nm") @@ -125,6 +125,7 @@ case "$ARTIFACT_ID" in check_symbol "$ARTIFACT" "glean_core_uniffi_contract_version" "${REQUIRED_FILES_TEST[@]}" ;; *) - echo "Unknown Artifact ID" + echo "No check defined for artifact '${ARTIFACT_ID}'. Abort." >&2 + exit 1 ;; esac diff --git a/build.gradle b/build.gradle index fa83f35057..9a2283b9cc 100644 --- a/build.gradle +++ b/build.gradle @@ -131,10 +131,6 @@ subprojects { withSourcesJar() } } - - lint { - baseline = file("lint-baseline.xml") - } } } tasks.withType(org.jetbrains.kotlin.gradle.tasks.KotlinCompile).all { @@ -159,11 +155,11 @@ subprojects { } detekt { - input = files("${projectDir}/glean-core", "${projectDir}/samples/android", "buildSrc") - config = files("${projectDir}/.detekt.yml") + source.from(files("${projectDir}/glean-core", "${projectDir}/samples/android", "buildSrc")) + config.from(files("${projectDir}/.detekt.yml")) buildUponDefaultConfig = true reports { - xml.enabled = false + xml.required = false } } diff --git a/docs/user/user/adding-glean-to-your-project/kotlin.md b/docs/user/user/adding-glean-to-your-project/kotlin.md index 4fd34b3dbf..779069d9e9 100644 --- a/docs/user/user/adding-glean-to-your-project/kotlin.md +++ b/docs/user/user/adding-glean-to-your-project/kotlin.md @@ -60,10 +60,12 @@ This package is standalone and its version will be exported from the main Glean ## Setting up metrics and pings code generation -In order for the Glean Kotlin SDK to generate an API for your metrics, two Gradle plugins must be included in your build: +In order for the Glean Kotlin SDK to generate an API for your metrics, the +[Glean Gradle plugin](https://github.com/mozilla/glean/tree/main/gradle-plugin/) must be included in your build. -- The [Glean Gradle plugin](https://github.com/mozilla/glean/tree/main/gradle-plugin/) -- JetBrains' [Python envs plugin](https://github.com/JetBrains/gradle-python-envs/) +The plugin creates its own Python virtual environment to run `glean_parser` in, so a Python 3.9 or +later interpreter needs to be available on the `PATH`. Set the `GLEAN_PYTHON` environment variable to +point at a specific interpreter instead. The Glean Gradle plugin is distributed through Mozilla's Maven, so we need to tell your build where to look for it by adding the following to the top of your `build.gradle`: @@ -87,14 +89,6 @@ buildscript { > As above, the `{latest-components-version}` placeholder in the above link should be replaced with the version number of the Glean SDK used in your project. -The JetBrains Python plugin is distributed in the Gradle plugin repository, so it can be included with: - -```Groovy -plugins { - id "com.jetbrains.python.envs" version "0.0.26" -} -``` - Right before the end of the same file, we need to apply the Glean Gradle plugin. Set any [additional parameters](../../language-bindings/android/android-build-configuration-options.md) to control the behavior of the Glean Gradle plugin before calling `apply plugin`. @@ -107,15 +101,6 @@ apply plugin: "org.mozilla.telemetry.glean-gradle-plugin" {{#include ../../../shared/blockquote-info.html}} -##### Rosetta 2 required on Apple Silicon - -> On Apple Silicon machines (M1/M2/M3 MacBooks and iMacs) Rosetta 2 is required for the bundled Python. -> See the [Apple documentation about Rosetta 2](https://support.apple.com/en-us/HT211861) -> and [Bug 1775420](https://bugzilla.mozilla.org/show_bug.cgi?id=1775420) for details. -> You can install it with `softwareupdate --install-rosetta` - -{{#include ../../../shared/blockquote-info.html}} - ##### Offline builds > The Glean Gradle plugin has limited support for [offline builds](../../language-bindings/android/android-offline-builds.md) of applications that use the Glean SDK. diff --git a/glean-core/android-native/build.gradle b/glean-core/android-native/build.gradle index 53716d5235..cf20f09d28 100644 --- a/glean-core/android-native/build.gradle +++ b/glean-core/android-native/build.gradle @@ -19,11 +19,11 @@ android { namespace = "mozilla.telemetry.glean_native" ndkVersion = rootProject.ext.build.ndkVersion - compileSdkVersion = rootProject.ext.build.compileSdkVersion + compileSdk = rootProject.ext.build.compileSdkVersion defaultConfig { - minSdkVersion rootProject.ext.build['minSdkVersion'] - targetSdkVersion rootProject.ext.build['targetSdkVersion'] + minSdk = rootProject.ext.build['minSdkVersion'] + targetSdk = rootProject.ext.build['targetSdkVersion'] } buildTypes { @@ -34,7 +34,7 @@ android { release { minifyEnabled false - proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' consumerProguardFiles "$projectDir/proguard-rules-consumer.pro" } @@ -108,22 +108,24 @@ dependencies { } } -afterEvaluate { - // The `cargoBuild` task isn't available until after evaluation. - android.libraryVariants.all { variant -> - def productFlavor = "" - variant.productFlavors.each { - productFlavor += "${it.name.capitalize()}" - } - def buildType = "${variant.buildType.name.capitalize()}" - tasks["merge${productFlavor}${buildType}JniLibFolders"].dependsOn(tasks["cargoBuild"]) +// Referenced by name so `cargoBuild` does not have to exist yet. +tasks.configureEach { task -> + if (task.name.startsWith("merge") && task.name.endsWith("JniLibFolders")) { + task.dependsOn("cargoBuild") } } // Extract JNI dispatch libraries from the JAR into a directory, so that we can then package them // into our own glean-native-desktopLibraries JAR. +def jnaJar = configurations.jna.elements.map { + if (it.size() != 1) { + throw new GradleException("Expected exactly one JNA jar, found ${it.size()}") + } + it.first().asFile +} + def extractLibJniDispatch = tasks.register("extractLibJniDispatch", Copy) { - from zipTree(configurations.jna.singleFile).matching { + from(zipTree(jnaJar)) { include "**/libjnidispatch.*" } into layout.buildDirectory.dir("libjnidispatch").get() @@ -140,6 +142,7 @@ def packageLibsForTest = tasks.register("packageLibsForTest", Jar) { def copyGleanNative = tasks.register("copyGleanNative", Copy) { from layout.buildDirectory.dir("rustJniLibs/desktop") into layout.buildDirectory.dir("gleanNative") + dependsOn("cargoBuild${rootProject.ext.nativeRustTarget.capitalize()}") } artifacts { diff --git a/glean-core/android/build.gradle b/glean-core/android/build.gradle index 5966a1b668..e880718c69 100644 --- a/glean-core/android/build.gradle +++ b/glean-core/android/build.gradle @@ -6,10 +6,8 @@ // https://github.com/mozilla/application-services/blob/c40e2ccb422cf4af9ffdf095149cec34de1d4bef/components/fxa-client/android/build.gradle import groovy.json.JsonOutput - -plugins { - alias libs.plugins.gradle.python.envs -} +import javax.inject.Inject +import org.gradle.process.ExecOperations apply plugin: 'com.android.library' apply plugin: 'kotlin-android' @@ -24,9 +22,6 @@ apply plugin: 'kotlinx-serialization' */ File GLEAN_PING_SCHEMA_PATH = file("$rootDir/glean.1.schema.json") -// This will store the uniffi-bindgen generated files for our component -def UNIFFI_OUT_DIR = layout.buildDirectory.dir("generated/uniffi/") - // Set configuration for the glean_parser ext.allowGleanInternal = true ext.gleanNamespace = "mozilla.telemetry.glean" @@ -38,18 +33,18 @@ kotlin { android { namespace = "mozilla.telemetry.glean" - ndkVersion rootProject.ext.build.ndkVersion - compileSdkVersion rootProject.ext.build.compileSdkVersion + ndkVersion = rootProject.ext.build.ndkVersion + compileSdk = rootProject.ext.build.compileSdkVersion defaultConfig { - minSdkVersion rootProject.ext.build['minSdkVersion'] - targetSdkVersion rootProject.ext.build['targetSdkVersion'] + minSdk = rootProject.ext.build['minSdkVersion'] + targetSdk = rootProject.ext.build['targetSdkVersion'] // Carefully escape the string here so it will support `\` in // Windows paths correctly. buildConfigField("String", "GLEAN_PING_SCHEMA_PATH", JsonOutput.toJson(GLEAN_PING_SCHEMA_PATH.path)) - testInstrumentationRunner "android.support.test.runner.AndroidJUnitRunner" + testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner" } buildTypes { @@ -60,7 +55,7 @@ android { release { minifyEnabled false - proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' consumerProguardFiles "$projectDir/proguard-rules-consumer.pro" } @@ -69,8 +64,6 @@ android { } } - sourceSets.main.kotlin.srcDirs += UNIFFI_OUT_DIR - // Uncomment to include debug symbols in native library builds. // packagingOptions { doNotStrip "**/*.so" } @@ -89,25 +82,8 @@ android { } } -kotlin { - jvmToolchain(rootProject.ext.build.jvmTargetCompatibility) -} afterEvaluate { - android.libraryVariants.all { variant -> - def variantName = variant.name.capitalize(); - def testTask = tasks["test${variantName}UnitTest"] - } - - def gleanNative = configurations.getByName("gleanNative") - android.libraryVariants.all { variant -> - def variantName = variant.name.capitalize(); - def compileTask = tasks["compile${variantName}Kotlin"] - - compileTask.dependsOn(generateUniffiBindings) - variant.registerJavaGeneratingTask(generateUniffiBindings, gleanNative.singleFile) - } - if (project.hasProperty("coverage")) { jacoco { toolVersion = libs.versions.jacoco @@ -194,29 +170,46 @@ dependencies { apply from: "$rootDir/publish.gradle" ext.configurePublish() -def generateUniffiBindings = tasks.register("generateUniffiBindings") { +// Gradle 9 removed Project.exec(). +interface InjectedExecOps { + @Inject ExecOperations getExecOps() +} +def execOps = objects.newInstance(InjectedExecOps).execOps + +// AGP owns the output location, so the task exposes it as a wired property. +abstract class GenerateUniffiBindingsTask extends DefaultTask { + @OutputDirectory + abstract DirectoryProperty getOutputDir() +} + +// Resolved up front: the task action must not reach back into the project. +def nativeLibPattern = "${rootProject.ext.nativeRustTarget}/libxul.*" +def cargoWorkingDir = rootDir +def uniffiConfigFile = file("${projectDir}/../uniffi.toml") + +def generateUniffiBindings = tasks.register("generateUniffiBindings", GenerateUniffiBindingsTask) { def gleanNative = configurations.getByName("gleanNative") doFirst { - def libraryPath = gleanNative.asFileTree.matching { - include "${nativeRustTarget}/libxul.*" - }.singleFile + def libraryFiles = gleanNative.asFileTree.matching { + include nativeLibPattern + } - if (libraryPath == null) { + if (libraryFiles.isEmpty()) { throw new GradleException("libglean_ffi dynamic library path not found") } + def libraryPath = libraryFiles.singleFile - exec { - workingDir project.rootDir + execOps.exec { + workingDir cargoWorkingDir commandLine 'cargo', 'uniffi-bindgen', 'generate', '--no-format', '--library', libraryPath, '--language', 'kotlin', - '--config', "${project.projectDir}/../uniffi.toml", - '--out-dir', UNIFFI_OUT_DIR.get() + '--config', uniffiConfigFile, + '--out-dir', outputDir.get() } } - outputs.dir UNIFFI_OUT_DIR // Re-generate if native changes inputs.files gleanNative // Re-generate if the interface definition changes. @@ -229,6 +222,10 @@ def generateUniffiBindings = tasks.register("generateUniffiBindings") { inputs.file "${project.rootDir}/Cargo.lock" } +androidComponents.onVariants(androidComponents.selector().all(), { variant -> + variant.sources.java.addGeneratedSourceDirectory(generateUniffiBindings) { task -> task.outputDir } +}) + // Generate markdown docs for the collected metrics. ext.gleanDocsDirectory = "$rootDir/docs/user/user/collected-metrics" ext.gleanYamlFiles = [ diff --git a/glean-core/android/src/test/java/mozilla/telemetry/glean/TestUtil.kt b/glean-core/android/src/test/java/mozilla/telemetry/glean/TestUtil.kt index 4d203e376d..8d1965b6c3 100644 --- a/glean-core/android/src/test/java/mozilla/telemetry/glean/TestUtil.kt +++ b/glean-core/android/src/test/java/mozilla/telemetry/glean/TestUtil.kt @@ -52,7 +52,7 @@ object GleanBuildInfo { * Checks ping content against the Glean ping schema. * * This uses the Python utility, glean_parser, to perform the actual checking. - * This is installed in its own Miniconda environment as part of the build + * This is installed in its own virtual environment as part of the build * configuration by glean-gradle-plugin. * * @param content The JSON content of the ping @@ -62,7 +62,7 @@ internal fun checkPingSchema(content: JSONObject) { val os = System.getProperty("os.name")?.lowercase() val pythonExecutable = if (os?.indexOf("win")?.compareTo(0) == 0) { - "${BuildConfig.GLEAN_MINICONDA_DIR}/python" + "${BuildConfig.GLEAN_MINICONDA_DIR}/Scripts/python" } else { "${BuildConfig.GLEAN_MINICONDA_DIR}/bin/python" } diff --git a/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy b/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy index 7d46b7e6f7..a904ddfaa7 100644 --- a/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy +++ b/gradle-plugin/src/main/groovy/mozilla/telemetry/glean-gradle-plugin/GleanGradlePlugin.groovy @@ -16,16 +16,17 @@ import org.gradle.api.file.FileSystemLocation import org.gradle.api.artifacts.ComponentMetadataRule import org.gradle.api.artifacts.ComponentMetadataContext import org.gradle.api.artifacts.component.ModuleComponentIdentifier -import org.gradle.api.internal.artifacts.ArtifactAttributes +import org.gradle.api.artifacts.type.ArtifactTypeDefinition import org.gradle.api.tasks.Exec import org.gradle.api.tasks.OutputDirectory +import org.gradle.api.tasks.PathSensitive +import org.gradle.api.tasks.PathSensitivity import org.gradle.api.tasks.TaskProvider import org.gradle.api.file.DirectoryProperty +import org.gradle.work.DisableCachingByDefault import groovy.transform.CompileStatic -import java.util.concurrent.Semaphore - // The suppression "GrPackage" is needed below since Android Studio wants this file to have // a package name, but adding one causes the build to fail with: // "'.../GleanGradlePlugin.groovy' should not contain a package statement" @@ -33,6 +34,7 @@ import java.util.concurrent.Semaphore // Exec task with a wired output dir, required to register it via addGeneratedSourceDirectory. @SuppressWarnings("GrPackage") +@DisableCachingByDefault(because = "Shells out to glean_parser, whose environment is not tracked as an input") abstract class GenerateGleanMetricsAPITask extends Exec { @OutputDirectory abstract DirectoryProperty getOutputDir() @@ -42,8 +44,10 @@ abstract class GenerateGleanMetricsAPITask extends Exec { * A helper class to extract metrics.yaml files from AAR files. */ @SuppressWarnings("GrPackage") +@DisableCachingByDefault(because = "Extracting a single file is cheaper than caching the result") abstract class GleanMetricsYamlTransform implements TransformAction { @InputArtifact + @PathSensitive(PathSensitivity.RELATIVE) abstract Provider getInputArtifact() @Override @@ -60,14 +64,9 @@ abstract class GleanMetricsYamlTransform implements TransformAction { // The version of glean_parser to install from PyPI. private String GLEAN_PARSER_VERSION = "20.1" - // The version of Miniconda is explicitly specified. - // Miniconda3-4.5.12 is known to not work on Windows. - private String MINICONDA_VERSION = "24.3.0-0" private String TASK_NAME_PREFIX = "gleanGenerateMetrics" - private Semaphore bootstrapMinicondaSemaphore = new Semaphore(1) - /* This script runs a given Python module as a "main" module, like * `python -m module`. However, it first checks that the installed * package is at the desired version, and if not, upgrades it using `pip`. @@ -145,14 +144,13 @@ except: // Are we doing an offline build (by passing `--offline` to `./gradle`)? private Boolean isOffline - static File getPythonCommand(File envDir, boolean isOffline) { - // Note that the command line is OS dependant: on linux/mac is Miniconda3/bin/python. + // Null when the caller supplies their own environment via gleanPythonEnvDir. + private TaskProvider installGleanParser + + static File getPythonCommand(File envDir) { + // Note that the command line is OS dependant. if (Os.isFamily(Os.FAMILY_WINDOWS)) { - if (isOffline) { - return new File(envDir, "Scripts\\python") - } else { - return new File(envDir, "python") - } + return new File(envDir, "Scripts\\python") } return new File(envDir, "bin/python") @@ -185,6 +183,9 @@ except: def namespaceProvider = variant.namespace.map({ ns -> "namespace=${ns}.GleanMetrics" }) def generateKotlinAPI = project.tasks.register("${TASK_NAME_PREFIX}SourceFor${variant.name.capitalize()}", GenerateGleanMetricsAPITask) { + if (installGleanParser != null) { + dependsOn(installGleanParser) + } description = "Generate the Kotlin code for the Metrics API" if (project.ext.has("allowMetricsFromAAR")) { @@ -192,7 +193,7 @@ except: // https://github.com/google/protobuf-gradle-plugin/blob/6d99a421c8d15710045e4e8d31a3af6cb0cc3b70/src/main/groovy/com/google/protobuf/gradle/ProtobufPlugin.groovy#L270-L277 inputs.files variant.compileConfiguration.incoming.artifactView { attributes { - it.attribute(ArtifactAttributes.ARTIFACT_FORMAT, 'glean-metrics-yaml') + it.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, 'glean-metrics-yaml') } }.files } @@ -202,7 +203,7 @@ except: inputs.files(getYamlFiles(project)).optional(true) workingDir project.rootDir - commandLine getPythonCommand(envDir, isOffline) + commandLine getPythonCommand(envDir) def gleanNamespace = "mozilla.components.service.glean" if (project.ext.has("gleanNamespace")) { @@ -273,6 +274,9 @@ except: } TaskProvider generateGleanMetricsDocs = project.tasks.register("${TASK_NAME_PREFIX}DocsFor${variant.name.capitalize()}", Exec) { + if (installGleanParser != null) { + dependsOn(installGleanParser) + } description = "Generate the Markdown docs for the collected metrics" def gleanDocsDirectory = "${project.projectDir}/docs" @@ -285,7 +289,7 @@ except: // https://github.com/google/protobuf-gradle-plugin/blob/6d99a421c8d15710045e4e8d31a3af6cb0cc3b70/src/main/groovy/com/google/protobuf/gradle/ProtobufPlugin.groovy#L270-L277 inputs.files variant.compileConfiguration.incoming.artifactView { attributes { - it.attribute(ArtifactAttributes.ARTIFACT_FORMAT, 'glean-metrics-yaml') + it.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, 'glean-metrics-yaml') } }.files } @@ -296,7 +300,7 @@ except: outputs.dir gleanDocsDirectory workingDir project.rootDir - commandLine getPythonCommand(envDir, isOffline) + commandLine getPythonCommand(envDir) args "-c" args runPythonScript @@ -356,52 +360,56 @@ except: } File setupPythonEnvironmentTasks(Project project, String parserVersion) { - // For offline mode: - // 1. We use the system Python on the PATH, for one set by GLEAN_PYTHON - // 2. We create a virtual environment in ~/.gradle/glean/pythonenv based on - // that Python. - // 3. We expect the wheels for glean_parser and all its depenencies in - // $rootDir/glean-wheels, or GLEAN_PYTHON_WHEELS_DIR. These can be - // downloaded in advance easily with `pip download glean_parser`. - // For online mode: - // 1. We install miniconda into ~/.gradle/glean/ - // 2. glean_parser is installed using pip from pypi.org - if (isOffline) { - // This installs a virtual environment in `~/.gradle/glean/pythonenv`, so it is shared - // between multiple projects using Glean. - File envDir = new File( - project.getGradle().gradleUserHomeDir, - "glean/pythonenv" - ) - - TaskProvider createGleanPythonVirtualEnv = project.tasks.register("createGleanPythonVirtualEnv", Exec) { - description = "Create a Python virtual environment for Glean" - - outputs.dir(envDir) - - String pythonBinary = System.getenv("GLEAN_PYTHON") - if (!pythonBinary) { - if (Os.isFamily(Os.FAMILY_WINDOWS)) { - pythonBinary = "python" - } else { - pythonBinary = "python3" - } + // 1. We use the system Python on the PATH, or the one set by GLEAN_PYTHON. + // 2. We create a virtual environment in ~/.gradle/glean/pythonenv based on that + // Python, so it is shared between multiple projects using Glean. + // 3. glean_parser is installed using pip from pypi.org. In offline mode we instead + // expect the wheels for glean_parser and all its dependencies in + // $rootDir/glean-wheels, or GLEAN_PYTHON_WHEELS_DIR. These can be downloaded in + // advance easily with `pip download glean_parser`. + File envDir = new File( + project.getGradle().gradleUserHomeDir, + "glean/pythonenv" + ) + + TaskProvider createGleanPythonVirtualEnv = project.tasks.register("createGleanPythonVirtualEnv", Exec) { + description = "Create a Python virtual environment for Glean" + + outputs.dir(envDir) + + String pythonBinary = System.getenv("GLEAN_PYTHON") + if (!pythonBinary) { + if (Os.isFamily(Os.FAMILY_WINDOWS)) { + pythonBinary = "python" + } else { + pythonBinary = "python3" } + } + if (isOffline) { project.logger.warn("Building in offline mode, therefore, Glean is using a supplied Python at ${pythonBinary}") project.logger.warn("The Python binary can be overridden with the GLEAN_PYTHON env var.") - - commandLine pythonBinary - args "-m" - args "venv" - args envDir.toString() + } else { + project.logger.info("Glean is using the Python at ${pythonBinary}, overridable with the GLEAN_PYTHON env var.") } - TaskProvider installGleanParser = project.tasks.register("installGleanParser", Exec) { - description = "Install glean_parser" + commandLine pythonBinary + args "-m" + args "venv" + args envDir.toString() + } - outputs.dir(envDir) + installGleanParser = project.tasks.register("installGleanParser", Exec) { + description = "Install glean_parser" + outputs.dir(envDir) + + commandLine getPythonCommand(envDir) + args "-m" + args "pip" + args "install" + + if (isOffline) { String pythonPackagesDir = System.getenv("GLEAN_PYTHON_WHEELS_DIR") if (!pythonPackagesDir) { pythonPackagesDir = "${project.rootDir}/glean-wheels" @@ -410,90 +418,22 @@ except: project.logger.warn("Installing glean_parser from cached Python packages in ${pythonPackagesDir}") project.logger.warn("This can be overridden with the GLEAN_PYTHON_WHEELS_DIR env var.") - commandLine getPythonCommand(envDir, isOffline) - args "-m" - args "pip" - args "install" args "glean_parser" args "--no-index" args "-f" args pythonPackagesDir + } else { + // A git package (a la `git+https://github.com`) is installed as given. + args(parserVersion.matches("git.+") ? parserVersion : "glean_parser~=${parserVersion}") } + } - installGleanParser.configure { - dependsOn(createGleanPythonVirtualEnv) - } - project.preBuild.finalizedBy(installGleanParser) - - return envDir - } else { - // This sets up tasks to install a Miniconda3 environment. It installs - // into the gradle user home directory so that it will be shared between - // all libraries that use Glean. This is important because it is - // approximately 300MB in installed size. - File condaBootstrapDir = new File( - project.getGradle().gradleUserHomeDir, - "glean/bootstrap-${MINICONDA_VERSION}" - ) - - // Even though we are installing the Miniconda environment to the gradle user - // home directory, the gradle-python-envs plugin is hardcoded to download the - // installer to the project's build directory. Doing so will fail if the - // project's build directory doesn't already exist. This task ensures that - // the project's build directory exists before downloading and installing the - // Miniconda environment. - // See https://github.com/JetBrains/gradle-python-envs/issues/26 - // The fix in the above is not actually sufficient -- we need to add createBuildDir - // as a dependency of Bootstrap_CONDA (where conda is installed), as the preBuild - // task alone isn't early enough. - TaskProvider createBuildDir = project.tasks.register("createBuildDir") { - description = "Make sure the build dir exists before creating the Python Environments" - - outputs.dir(project.buildDir) - - doLast { - project.logger.lifecycle("Creating build directory:" + project.buildDir.getPath()) - project.buildDir.mkdir() - } - } - - project.envs { - bootstrapDirectory = condaBootstrapDir - pipInstallOptions = "--trusted-host pypi.python.org --no-cache-dir" - - // Setup a miniconda environment. conda is used because it works - // non-interactively on Windows, unlike the standard Python installers - - // If we have a git package (a la `git+https://github.com`) we install that. - if (parserVersion.matches("git.+")) { - conda "Miniconda3", "Miniconda3-py311_${MINICONDA_VERSION}", "64", [parserVersion] - } else { - conda "Miniconda3", "Miniconda3-py311_${MINICONDA_VERSION}", "64", ["glean_parser~=${parserVersion}"] - } - } - File envDir = new File( - condaBootstrapDir, - "Miniconda3" - ) - project.tasks.configureEach { task -> - if (task.name.startsWith('Bootstrap_CONDA')) { - task.dependsOn(createBuildDir) - - // The Bootstrap_CONDA* tasks all install miniconda to the - // same place, so they can't run at the same time. This - // holds a semaphore while running the task to make sure - // only one of these classes of tasks runs at the same time. - // Solution proposed in this Gradle bug: - // https://github.com/gradle/gradle/issues/7047#issuecomment-430139316 - task.doFirst { bootstrapMinicondaSemaphore.acquire() } - task.doLast { bootstrapMinicondaSemaphore.release() } - } - } - project.preBuild.dependsOn(createBuildDir) - project.preBuild.finalizedBy("build_envs") - - return envDir + installGleanParser.configure { + dependsOn(createGleanPythonVirtualEnv) } + project.preBuild.finalizedBy(installGleanParser) + + return envDir } void setupExtractMetricsFromAARTasks(Project project) { @@ -519,8 +459,8 @@ except: // but there's no good way to access the including script's classpath from `apply from:` // scripts. See https://stackoverflow.com/a/37060550. The 'android-exploded-aar' string is // very unlikely to change, so it's just hard-coded. - from.attribute(ArtifactAttributes.ARTIFACT_FORMAT, "android-exploded-aar") - to.attribute(ArtifactAttributes.ARTIFACT_FORMAT, "glean-metrics-yaml") + from.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, "android-exploded-aar") + to.attribute(ArtifactTypeDefinition.ARTIFACT_TYPE_ATTRIBUTE, "glean-metrics-yaml") } } } diff --git a/gradle.properties b/gradle.properties index 96c34d118a..7d82b06c88 100644 --- a/gradle.properties +++ b/gradle.properties @@ -23,3 +23,22 @@ libLicense=MPL-2.0 libLicenseUrl=https://www.mozilla.org/en-US/MPL/2.0/ android.useAndroidX=true + +# Opt in to Android Gradle plugin 9 defaults ahead of the upgrade, so the +# behavior changes are absorbed here rather than all at once. +# +# android.dependency.useConstraints=false is left at the AGP 8 default: with it off, +# checkClasspath fails to fingerprint its own compileVersionMap input on Gradle 9.5. +# +# android.onlyEnableUnitTestForTheTestedBuildType is also left off, so the release and +# withoutLib unit tests keep running. Adopt it with the AGP 9 upgrade. +android.default.androidx.test.runner=true +android.enableAppCompileTimeRClass=true +android.proguard.failOnMissingFiles=true +android.r8.globalOptionsInConsumerRules.disallowed=true +android.r8.optimizedResourceShrinking=true +android.r8.proguardAndroidTxt.disallowed=true +android.r8.strictFullModeForKeepRules=true +android.sdk.defaultTargetSdkToCompileSdkIfUnset=true +android.sourceset.disallowProvider=true +android.uniquePackageNames=true diff --git a/gradle/libs.versions.toml b/gradle/libs.versions.toml index 2e825a89e2..c12fde2545 100644 --- a/gradle/libs.versions.toml +++ b/gradle/libs.versions.toml @@ -13,7 +13,7 @@ kotlinx-coroutines = "1.11.0" kotlinx-serialization = "1.11.0" # Mozilla -rust-android-gradle = "0.9.6" +rust-android-gradle = "0.10.0" # AndroidX androidx-annotation = "1.10.0" @@ -44,7 +44,6 @@ robolectric = "4.16" # Miscellaneous Gradle plugins jacoco = "0.8.14" -python-envs = "0.0.31" [libraries] # AGP @@ -90,4 +89,3 @@ robolectric = { group = "org.robolectric", name = "robolectric", version.ref = " [plugins] detekt = { id = "io.gitlab.arturbosch.detekt", version.ref = "detekt" } -gradle-python-envs = { id = "com.jetbrains.python.envs", version.ref = "python-envs" } diff --git a/gradle/wrapper/gradle-wrapper.jar b/gradle/wrapper/gradle-wrapper.jar index 1b33c55baa..b1b8ef56b4 100644 Binary files a/gradle/wrapper/gradle-wrapper.jar and b/gradle/wrapper/gradle-wrapper.jar differ diff --git a/gradle/wrapper/gradle-wrapper.properties b/gradle/wrapper/gradle-wrapper.properties index d4081da476..4b271a6469 100644 --- a/gradle/wrapper/gradle-wrapper.properties +++ b/gradle/wrapper/gradle-wrapper.properties @@ -1,7 +1,10 @@ distributionBase=GRADLE_USER_HOME distributionPath=wrapper/dists -distributionUrl=https\://services.gradle.org/distributions/gradle-8.14.3-bin.zip +distributionSha256Sum=553c78f50dafcd54d65b9a444649057857469edf836431389695608536d6b746 +distributionUrl=https\://services.gradle.org/distributions/gradle-9.5.0-bin.zip networkTimeout=10000 +retries=0 +retryBackOffMs=500 validateDistributionUrl=true zipStoreBase=GRADLE_USER_HOME zipStorePath=wrapper/dists diff --git a/gradlew b/gradlew index 23d15a9367..b9bb139f79 100755 --- a/gradlew +++ b/gradlew @@ -1,7 +1,7 @@ #!/bin/sh # -# Copyright © 2015-2021 the original authors. +# Copyright © 2015 the original authors. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -57,7 +57,7 @@ # Darwin, MinGW, and NonStop. # # (3) This script is generated from the Groovy template -# https://github.com/gradle/gradle/blob/HEAD/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt +# https://github.com/gradle/gradle/blob/3d91ce3b8caaf77ad09f381f43615b715b53f72c/platforms/jvm/plugins-application/src/main/resources/org/gradle/api/internal/plugins/unixStartScript.txt # within the Gradle project. # # You can find Gradle at https://github.com/gradle/gradle/. @@ -114,7 +114,6 @@ case "$( uname )" in #( NONSTOP* ) nonstop=true ;; esac -CLASSPATH="\\\"\\\"" # Determine the Java command to use to start the JVM. @@ -172,7 +171,6 @@ fi # For Cygwin or MSYS, switch paths to Windows format before running java if "$cygwin" || "$msys" ; then APP_HOME=$( cygpath --path --mixed "$APP_HOME" ) - CLASSPATH=$( cygpath --path --mixed "$CLASSPATH" ) JAVACMD=$( cygpath --unix "$JAVACMD" ) @@ -212,7 +210,6 @@ DEFAULT_JVM_OPTS='"-Xmx64m" "-Xms64m"' set -- \ "-Dorg.gradle.appname=$APP_BASE_NAME" \ - -classpath "$CLASSPATH" \ -jar "$APP_HOME/gradle/wrapper/gradle-wrapper.jar" \ "$@" diff --git a/gradlew.bat b/gradlew.bat index 5eed7ee845..aa5f10b069 100644 --- a/gradlew.bat +++ b/gradlew.bat @@ -23,8 +23,8 @@ @rem @rem ########################################################################## -@rem Set local scope for the variables with windows NT shell -if "%OS%"=="Windows_NT" setlocal +@rem Set local scope for the variables, and ensure extensions are enabled +setlocal EnableExtensions set DIRNAME=%~dp0 if "%DIRNAME%"=="" set DIRNAME=. @@ -51,7 +51,7 @@ echo. 1>&2 echo Please set the JAVA_HOME variable in your environment to match the 1>&2 echo location of your Java installation. 1>&2 -goto fail +"%COMSPEC%" /c exit 1 :findJavaFromJavaHome set JAVA_HOME=%JAVA_HOME:"=% @@ -65,30 +65,18 @@ echo. 1>&2 echo Please set the JAVA_HOME variable in your environment to match the 1>&2 echo location of your Java installation. 1>&2 -goto fail +"%COMSPEC%" /c exit 1 :execute @rem Setup the command line -set CLASSPATH= @rem Execute Gradle -"%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -classpath "%CLASSPATH%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* +@rem endlocal doesn't take effect until after the line is parsed and variables are expanded +@rem which allows us to clear the local environment before executing the java command +endlocal & "%JAVA_EXE%" %DEFAULT_JVM_OPTS% %JAVA_OPTS% %GRADLE_OPTS% "-Dorg.gradle.appname=%APP_BASE_NAME%" -jar "%APP_HOME%\gradle\wrapper\gradle-wrapper.jar" %* & call :exitWithErrorLevel -:end -@rem End local scope for the variables with windows NT shell -if %ERRORLEVEL% equ 0 goto mainEnd - -:fail -rem Set variable GRADLE_EXIT_CONSOLE if you need the _script_ return code instead of -rem the _cmd.exe /c_ return code! -set EXIT_CODE=%ERRORLEVEL% -if %EXIT_CODE% equ 0 set EXIT_CODE=1 -if not ""=="%GRADLE_EXIT_CONSOLE%" exit %EXIT_CODE% -exit /b %EXIT_CODE% - -:mainEnd -if "%OS%"=="Windows_NT" endlocal - -:omega +:exitWithErrorLevel +@rem Use "%COMSPEC%" /c exit to allow operators to work properly in scripts +"%COMSPEC%" /c exit %ERRORLEVEL% diff --git a/lint-baseline.xml b/lint-baseline.xml deleted file mode 100644 index cb44c83605..0000000000 --- a/lint-baseline.xml +++ /dev/null @@ -1,37 +0,0 @@ - - - - - - - - - - - - - - - - diff --git a/publish.gradle b/publish.gradle index d666d5ff8b..65678082df 100644 --- a/publish.gradle +++ b/publish.gradle @@ -74,6 +74,7 @@ ext.configurePublish = { if (isNative) { publishing.publications.withType(MavenPublication).each {publication -> def checkFileTask = task "checkFilesForMavenArtifact-${publication.artifactId}"(type: Exec) { + dependsOn("publish") commandLine "${rootProject.projectDir}/bin/check-artifact.sh", project.buildDir, publication.artifactId } checkMavenArtifacts.dependsOn(checkFileTask) diff --git a/samples/android/app/build.gradle b/samples/android/app/build.gradle index 026c9df16f..47745d0030 100644 --- a/samples/android/app/build.gradle +++ b/samples/android/app/build.gradle @@ -2,10 +2,6 @@ * License, v. 2.0. If a copy of the MPL was not distributed with this * file, You can obtain one at http://mozilla.org/MPL/2.0/. */ -plugins { - alias libs.plugins.gradle.python.envs -} - apply plugin: 'com.android.application' apply plugin: 'kotlin-android' apply plugin: 'kotlinx-serialization' @@ -17,13 +13,13 @@ kotlin { android { namespace = "org.mozilla.samples.gleancore" - ndkVersion rootProject.ext.build.ndkVersion - compileSdkVersion rootProject.ext.build.compileSdkVersion + ndkVersion = rootProject.ext.build.ndkVersion + compileSdk = rootProject.ext.build.compileSdkVersion defaultConfig { applicationId "org.mozilla.samples.gleancore" - minSdkVersion rootProject.ext.build['minSdkVersion'] - targetSdkVersion rootProject.ext.build['targetSdkVersion'] + minSdk = rootProject.ext.build['minSdkVersion'] + targetSdk = rootProject.ext.build['targetSdkVersion'] // Keep the version in sync with gleanExpireByVersion which is // set below, near the end of the file. versionCode 1 @@ -35,7 +31,7 @@ android { buildTypes { release { minifyEnabled false - proguardFiles getDefaultProguardFile('proguard-android.txt'), 'proguard-rules.pro' + proguardFiles getDefaultProguardFile('proguard-android-optimize.txt'), 'proguard-rules.pro' } }