WIP

2026-01-11 23:40:37 -05:00 · 2023-12-19 13:52:14 -08:00
parent 6da87262a4
commit 8ceeeb9611
69 changed files with 5970 additions and 13633 deletions
--- a/app/src/main/kotlin/org/nixos/gradle2nix/GradleRunner.kt
+++ b/app/src/main/kotlin/org/nixos/gradle2nix/GradleRunner.kt
@@ -2,6 +2,9 @@ package org.nixos.gradle2nix

 import org.gradle.tooling.GradleConnector
 import org.gradle.tooling.ProjectConnection
+import org.nixos.gradle2nix.model.PARAM_INCLUDE_CONFIGURATIONS
+import org.nixos.gradle2nix.model.PARAM_INCLUDE_PROJECTS
+import org.nixos.gradle2nix.model.RESOLVE_ALL_TASK

 fun connect(config: Config): ProjectConnection =
    GradleConnector.newConnector()
@@ -28,19 +31,23 @@ fun ProjectConnection.build(
            }
            addArguments(config.gradleArgs)
            addArguments(
+                "--gradle-user-home=${config.gradleHome}",
                "--init-script=${config.appHome}/init.gradle",
                "--write-verification-metadata", "sha256"
            )
            if (config.projectFilter != null) {
-                addArguments("-D${PARAM_INCLUDE_PROJECTS}")
+                addArguments("-D$PARAM_INCLUDE_PROJECTS")
            }
            if (config.configurationFilter != null) {
-                addArguments("-D${PARAM_INCLUDE_CONFIGURATIONS}")
+                addArguments("-D$PARAM_INCLUDE_CONFIGURATIONS")
            }
            if (config.logger.verbose) {
                setStandardOutput(System.err)
                setStandardError(System.err)
            }
+            if (config.logger.stacktrace) {
+                addArguments("--stacktrace")
+            }
        }
        .run()
 }
--- a/app/src/main/kotlin/org/nixos/gradle2nix/Main.kt
+++ b/app/src/main/kotlin/org/nixos/gradle2nix/Main.kt
@@ -116,7 +116,8 @@ class Gradle2Nix : CliktCommand(
        if (appHome == null) {
            System.err.println("Error: could not locate the /share directory in the gradle2nix installation")
        }
-        val gradleHome = System.getenv("GRADLE_USER_HOME")?.let(::File) ?: File("${System.getProperty("user.home")}/.gradle")
+        val gradleHome =
+            System.getenv("GRADLE_USER_HOME")?.let(::File) ?: File("${System.getProperty("user.home")}/.gradle")
        val logger = Logger(verbose = !quiet, stacktrace = debug)

        val config = Config(
@@ -152,7 +153,7 @@ class Gradle2Nix : CliktCommand(
        val env = try {
            processDependencies(config)
        } catch (e: Throwable) {
-            logger.error("Dependency parsing failed", e)
+            logger.error("dependency parsing failed", e)
        }

        val outDir = outDir ?: projectDir
--- a/app/src/main/kotlin/org/nixos/gradle2nix/Process.kt
+++ b/app/src/main/kotlin/org/nixos/gradle2nix/Process.kt
@@ -14,9 +14,8 @@ import okio.HashingSource
 import okio.blackholeSink
 import okio.buffer
 import okio.source
-import org.nixos.gradle2nix.dependencygraph.model.Repository
-import org.nixos.gradle2nix.dependencygraph.model.ResolvedConfiguration
-import org.nixos.gradle2nix.dependencygraph.model.ResolvedDependency
+import org.nixos.gradle2nix.model.Repository
+import org.nixos.gradle2nix.model.ResolvedConfiguration
 import org.nixos.gradle2nix.env.ArtifactFile
 import org.nixos.gradle2nix.env.ArtifactSet
 import org.nixos.gradle2nix.env.Env
@@ -49,6 +48,8 @@ fun processDependencies(config: Config): Env {
        ModuleVersionId(ModuleId(it.group, it.name), it.version)
    } ?: emptyMap()
    val moduleCache = mutableMapOf<ModuleVersionId, GradleModule?>()
+    val pomCache = mutableMapOf<ModuleVersionId, Pair<String, ArtifactFile>?>()
+    val ivyCache = mutableMapOf<ModuleVersionId, Pair<String, ArtifactFile>?>()
    val configurations = readDependencyGraph(config)

    val repositories = configurations
@@ -64,11 +65,11 @@ fun processDependencies(config: Config): Env {
        }
    if (repositories.isEmpty()) {
        config.logger.warn("no repositories found in any configuration")
-        return Env(emptyMap())
+        return emptyMap()
    }
    config.logger.debug("Repositories:\n  ${repositories.values.joinToString("\n  ")}")

-    val modules = configurations.asSequence()
+    return configurations.asSequence()
        .flatMap { it.allDependencies.asSequence() }
        .filterNot { it.id.startsWith("project ") || it.repository == null || it.repository !in repositories }
        .groupBy { ModuleId(it.coordinates.group, it.coordinates.module) }
@@ -83,34 +84,40 @@ fun processDependencies(config: Config): Env {
                    val component = verificationComponents[componentId]
                        ?: verifyComponentFilesInCache(config, componentId)
                        ?: verifyComponentFilesInTestRepository(config, componentId)
+                        ?: config.logger.error("$componentId: no dependency metadata found")
+
                    val gradleModule = moduleCache.getOrPut(componentId) {
-                        maybeGetGradleModule(config.logger, componentId, dep.repositories)
+                        maybeDownloadGradleModule(config.logger, component, dep.repositories)?.artifact?.second
                    }
-                    ArtifactSet(
-                        needsPomRedirect = repositories.values.any {
-                            "mavenPom" in it.metadataSources &&
-                                    "ignoreGradleMetadataRedirection" !in it.metadataSources
-                        },
-                        needsIvyRedirect = repositories.values.any {
-                            "ivyDescriptor" in it.metadataSources &&
-                                    "ignoreGradleMetadataRedirection" !in it.metadataSources
-                        },
-                        files = (component?.artifacts ?: emptyList()).associate { meta ->
-                            meta.name to ArtifactFile(
-                                urls = dep.repositories
-                                    .flatMap { repository -> artifactUrls(componentId, meta.name, repository, gradleModule) }
-                                    .distinct(),
-                                hash = meta.checksums.first().toSri()
+                    val pomArtifact = pomCache.getOrPut(componentId) {
+                        maybeDownloadMavenPom(config.logger, component, dep.repositories, gradleModule)
+                    }
+                    val ivyArtifact = ivyCache.getOrPut(componentId) {
+                        maybeDownloadIvyDescriptor(config.logger, component, dep.repositories, gradleModule)
+                    }
+
+                    val files = buildMap {
+                        if (pomArtifact != null) put(pomArtifact.first, pomArtifact.second)
+                        if (ivyArtifact != null) put(ivyArtifact.first, ivyArtifact.second)
+                        for (artifact in component.artifacts) {
+                            put(
+                                artifact.name,
+                                ArtifactFile(
+                                    urls = dep.repositories.flatMap { repo ->
+                                        artifactUrls(componentId, artifact.name, repo, gradleModule)
+                                    }.distinct(),
+                                    hash = artifact.checksums.first().toSri()
+                                )
                            )
-                        }.toSortedMap()
-                    )
+                        }
+                    }.toSortedMap()
+
+                    ArtifactSet(files)
                }
                .toSortedMap(Version.Comparator.reversed())
            Module(versions)
        }
        .toSortedMap(compareBy(ModuleId::toString))
-
-    return Env(modules)
 }

 private fun readVerificationMetadata(config: Config): VerificationMetadata? {
@@ -162,25 +169,87 @@ private fun verifyComponentFilesInTestRepository(
    return Component(id, verifications.toList())
 }

-@OptIn(ExperimentalSerializationApi::class)
-private fun maybeGetGradleModule(logger: Logger, id: ModuleVersionId, repos: List<Repository>): GradleModule? {
-    val filename = with(id) { "$name-$version.module" }
-    val reposWithGradleMetadata = repos
-        .filter { "gradleMetadata" in it.metadataSources }
-        .flatMap { artifactUrls(id, filename, it, null)}
-
-    for (url in reposWithGradleMetadata) {
+private fun maybeDownloadGradleModule(
+    logger: Logger,
+    component: Component,
+    repos: List<Repository>
+): ArtifactDownload<Pair<String, GradleModule>>? {
+    if (component.artifacts.none { it.name.endsWith(".module") }) return null
+    val filename = with(component.id) { "$name-$version.module" }
+    return maybeDownloadArtifact(logger, component.id, filename, repos)?.let { artifact ->
        try {
-            return URL(url).openStream().buffered().use { input ->
-                JsonFormat.decodeFromStream(input)
-            }
+            ArtifactDownload(
+                filename to JsonFormat.decodeFromString<GradleModule>(artifact.artifact),
+                artifact.url,
+                artifact.hash
+            )
        } catch (e: SerializationException) {
-            logger.error("$id: failed to parse Gradle module metadata ($url)", e)
+            logger.warn("${component.id}: failed to parse Gradle module metadata from ${artifact.url}")
+            null
+        }
+    }
+}
+
+private fun maybeDownloadMavenPom(
+    logger: Logger,
+    component: Component,
+    repos: List<Repository>,
+    gradleModule: GradleModule?
+): Pair<String, ArtifactFile>? {
+    if (component.artifacts.any { it.name.endsWith(".pom") }) return null
+    val pomRepos = repos.filter { "mavenPom" in it.metadataSources }
+    if (pomRepos.isEmpty()) return null
+    val filename = with(component.id) { "$name-$version.pom" }
+
+    return maybeDownloadArtifact(logger, component.id, filename, pomRepos)?.let { artifact ->
+        filename to ArtifactFile(
+            urls = pomRepos.flatMap { repo ->
+                artifactUrls(component.id, filename, repo, gradleModule)
+            }.distinct(),
+            hash = artifact.hash.toSri()
+        )
+    }
+}
+
+private fun maybeDownloadIvyDescriptor(
+    logger: Logger,
+    component: Component,
+    repos: List<Repository>,
+    gradleModule: GradleModule?
+): Pair<String, ArtifactFile>? {
+    if (component.artifacts.any { it.name == "ivy.xml" }) return null
+    val ivyRepos = repos.filter { "ivyDescriptor" in it.metadataSources }
+    if (ivyRepos.isEmpty()) return null
+    return maybeDownloadArtifact(logger, component.id, "ivy.xml", ivyRepos)?.let { artifact ->
+        "ivy.xml" to ArtifactFile(
+            urls = ivyRepos.flatMap { repo ->
+                artifactUrls(component.id, "ivy.xml", repo, gradleModule)
+            }.distinct(),
+            hash = artifact.hash.toSri()
+        )
+    }
+}
+
+private fun maybeDownloadArtifact(
+    logger: Logger,
+    id: ModuleVersionId,
+    filename: String,
+    repos: List<Repository>
+): ArtifactDownload<String>? {
+    val urls = repos.flatMap { artifactUrls(id, filename, it, null)}
+
+    for (url in urls) {
+        try {
+            val source = HashingSource.sha256(URL(url).openStream().source())
+            val text = source.buffer().readUtf8()
+            val hash = source.hash
+            return ArtifactDownload(text, url, Sha256(hash.hex()))
        } catch (e: IOException) {
            // Pass
        }
    }

+    logger.debug("artifact $filename not found in any repository")
    return null
 }

@@ -276,3 +345,9 @@ private data class MergedDependency(
    val id: ModuleVersionId,
    val repositories: List<Repository>
 )
+
+private data class ArtifactDownload<T>(
+    val artifact: T,
+    val url: String,
+    val hash: Checksum
+)
--- a/app/src/main/kotlin/org/nixos/gradle2nix/env/Env.kt
+++ b/app/src/main/kotlin/org/nixos/gradle2nix/env/Env.kt
@@ -12,11 +12,7 @@ import kotlinx.serialization.encoding.Encoder
 import org.gradle.internal.impldep.com.google.common.collect.ImmutableMap
 import org.gradle.internal.impldep.com.google.common.primitives.Longs

-@Serializable
-@JvmInline
-value class Env(
-    val modules: Map<ModuleId, Module>,
-)
+typealias Env = Map<ModuleId, Module>

@Serializable
@JvmInline
@@ -25,23 +21,30 @@ value class Module(
 )

@Serializable
-data class ArtifactSet(
-    val needsPomRedirect: Boolean,
-    val needsIvyRedirect: Boolean,
+@JvmInline
+value class ArtifactSet(
    val files: Map<String, ArtifactFile>
 )

@Serializable
-data class ArtifactFile(
+data class ArtifactFile internal constructor(
    val urls: List<String>,
    val hash: String,
-)
+) {
+
+    companion object {
+        operator fun invoke(urls: List<String>, hash: String) = ArtifactFile(urls.sorted(), hash)
+    }
+}

@Serializable(ModuleId.Serializer::class)
 data class ModuleId(
    val group: String,
    val name: String,
-) {
+) : Comparable<ModuleId> {
+
+    override fun compareTo(other: ModuleId): Int =
+        compareValuesBy(this, other, ModuleId::group, ModuleId::name)

    override fun toString(): String = "$group:$name"

@@ -52,7 +55,7 @@ data class ModuleId(
        )

        override fun serialize(encoder: Encoder, value: ModuleId) {
-            encoder.encodeString("${value.name}:${value.group}")
+            encoder.encodeString(value.toString())
        }

        override fun deserialize(decoder: Decoder): ModuleId {
@@ -66,20 +69,55 @@ data class ModuleId(
    }
 }

+@Serializable(ModuleVersionId.Serializer::class)
 data class ModuleVersionId(
    val moduleId: ModuleId,
    val version: Version
-) {
+) : Comparable<ModuleVersionId> {
+    constructor(group: String, name: String, version: Version) : this(ModuleId(group, name), version)
+
    val group: String get() = moduleId.group
    val name: String get() = moduleId.name

-    override fun toString(): String = "$moduleId:$version"
+    override fun compareTo(other: ModuleVersionId): Int =
+        compareValuesBy(
+            this,
+            other,
+            ModuleVersionId::moduleId,
+            ModuleVersionId::version
+        )
+
+
+    override fun toString(): String = "$group:$name:$version"
+
+    internal object Serializer : KSerializer<ModuleVersionId> {
+        override val descriptor: SerialDescriptor = PrimitiveSerialDescriptor(
+            Version::class.qualifiedName!!,
+            PrimitiveKind.STRING
+        )
+
+        override fun serialize(encoder: Encoder, value: ModuleVersionId) {
+            encoder.encodeString(value.toString())
+        }
+
+        override fun deserialize(decoder: Decoder): ModuleVersionId {
+            val encoded = decoder.decodeString()
+            val parts = encoded.split(":")
+            if (parts.size != 3 || parts.any(String::isBlank)) {
+                throw SerializationException("invalid module version id: $encoded")
+            }
+            return ModuleVersionId(
+                moduleId = ModuleId(parts[0], parts[1]),
+                version = Version(parts[3])
+            )
+        }
+    }
 }

@Serializable(Version.Serializer::class)
 class Version(val source: String, val parts: List<String>, base: Version?) : Comparable<Version> {

-    val base: Version
+    private val base: Version
    val numericParts: List<Long?>

    init {
--- a/app/src/main/kotlin/org/nixos/gradle2nix/metadata/VerificationMetadata.kt
+++ b/app/src/main/kotlin/org/nixos/gradle2nix/metadata/VerificationMetadata.kt
@@ -3,13 +3,12 @@ package org.nixos.gradle2nix.metadata
 import java.io.File
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable
-import nl.adaptivity.xmlutil.XmlStreaming
 import nl.adaptivity.xmlutil.serialization.XML
 import nl.adaptivity.xmlutil.serialization.XmlChildrenName
 import nl.adaptivity.xmlutil.serialization.XmlElement
 import nl.adaptivity.xmlutil.serialization.XmlSerialName
+import nl.adaptivity.xmlutil.xmlStreaming
 import org.nixos.gradle2nix.Logger
-import org.nixos.gradle2nix.DependencyCoordinates
 import org.nixos.gradle2nix.env.ModuleVersionId
 import org.nixos.gradle2nix.env.Version

@@ -108,6 +107,8 @@ data class Component(
    val version: Version,
    val artifacts: List<Artifact> = emptyList(),
 ) {
+    val id: ModuleVersionId get() = ModuleVersionId(group, name, version)
+
    constructor(id: ModuleVersionId, artifacts: List<Artifact>) : this(
        id.group,
        id.name,
@@ -134,7 +135,7 @@ val XmlFormat = XML {

 fun parseVerificationMetadata(logger: Logger, metadata: File): VerificationMetadata? {
    return try {
-        metadata.reader().buffered().let(XmlStreaming::newReader).use { input ->
+        metadata.reader().buffered().let(xmlStreaming::newReader).use { input ->
            XmlFormat.decodeFromReader(input)
        }
    } catch (e: Exception) {
--- a/app/src/test/kotlin/org/nixos/gradle2nix/GoldenTest.kt
+++ b/app/src/test/kotlin/org/nixos/gradle2nix/GoldenTest.kt
@@ -17,6 +17,9 @@ class GoldenTest : FunSpec({
        golden("dependency/snapshot-dynamic")
        golden("dependency/snapshot-redirect")
    }
+    context("included-build") {
+        golden("included-build")
+    }
    context("integration") {
        golden("integration/settings-buildscript")
    }
--- a/app/src/test/kotlin/org/nixos/gradle2nix/TestUtil.kt
+++ b/app/src/test/kotlin/org/nixos/gradle2nix/TestUtil.kt
@@ -73,7 +73,7 @@ suspend fun TestScope.fixture(
                if (!tempDir.resolve("settings.gradle").exists() && !tempDir.resolve("settings.gradle.kts").exists()) {
                    Files.createFile(tempDir.resolve("settings.gradle").toPath())
                }
-                app.main(listOf("-d", tempDir.toString()) + args.withM2())
+                app.main(listOf("-d", tempDir.toString()) + listOf("--debug") + args.withM2() + "-Dorg.gradle.internal.operations.trace=${tempDir.resolve("build").absolutePath}")
                val file = tempDir.resolve("${app.envFile}.json")
                file.shouldBeAFile()
                val env: Env = file.inputStream().buffered().use { input ->