Rewrite plugin, use filenames in lockfile

app/src/main/kotlin/org/nixos/gradle2nix/GradleRunner.kt

@@ -1,53 +1,93 @@
 package org.nixos.gradle2nix
 
+import java.io.File
+import kotlin.coroutines.resume
+import kotlin.coroutines.resumeWithException
+import kotlin.io.path.absolutePathString
+import kotlinx.coroutines.suspendCancellableCoroutine
+import org.gradle.tooling.GradleConnectionException
 import org.gradle.tooling.GradleConnector
 import org.gradle.tooling.ProjectConnection
-import org.nixos.gradle2nix.model.PARAM_INCLUDE_CONFIGURATIONS
-import org.nixos.gradle2nix.model.PARAM_INCLUDE_PROJECTS
+import org.gradle.tooling.ResultHandler
+import org.gradle.tooling.model.gradle.GradleBuild
+import org.nixos.gradle2nix.model.DependencySet
 import org.nixos.gradle2nix.model.RESOLVE_ALL_TASK
 
-fun connect(config: Config): ProjectConnection =
+fun connect(config: Config, projectDir: File = config.projectDir): ProjectConnection =
     GradleConnector.newConnector()
         .apply {
             if (config.gradleVersion != null) {
                 useGradleVersion(config.gradleVersion)
             }
         }
-        .forProjectDirectory(config.projectDir)
+        .forProjectDirectory(projectDir)
         .connect()
 
-fun ProjectConnection.build(
-    config: Config,
-) {
-    newBuild()
+suspend fun ProjectConnection.buildModel(): GradleBuild = suspendCancellableCoroutine { continuation ->
+    val cancellationTokenSource = GradleConnector.newCancellationTokenSource()
+
+    continuation.invokeOnCancellation { cancellationTokenSource.cancel() }
+
+    action { controller -> controller.buildModel }
+        .withCancellationToken(cancellationTokenSource.token())
+        .run(object : ResultHandler<GradleBuild> {
+            override fun onComplete(result: GradleBuild) {
+                continuation.resume(result)
+            }
+
+            override fun onFailure(failure: GradleConnectionException) {
+                continuation.resumeWithException(failure)
+            }
+        })
+}
+
+suspend fun ProjectConnection.build(config: Config): DependencySet = suspendCancellableCoroutine { continuation ->
+    val cancellationTokenSource = GradleConnector.newCancellationTokenSource()
+
+    continuation.invokeOnCancellation { cancellationTokenSource.cancel() }
+
+    action { controller -> controller.getModel(DependencySet::class.java) }
+        .withCancellationToken(cancellationTokenSource.token())
         .apply {
             if (config.tasks.isNotEmpty()) {
                 forTasks(*config.tasks.toTypedArray())
             } else {
                 forTasks(RESOLVE_ALL_TASK)
             }
-            if (config.gradleJdk != null) {
-                setJavaHome(config.gradleJdk)
-            }
-            addArguments(config.gradleArgs)
-            addArguments(
-                "--gradle-user-home=${config.gradleHome}",
-                "--init-script=${config.appHome}/init.gradle",
-                "--write-verification-metadata", "sha256"
-            )
-            if (config.projectFilter != null) {
-                addArguments("-D$PARAM_INCLUDE_PROJECTS")
-            }
-            if (config.configurationFilter != null) {
-                addArguments("-D$PARAM_INCLUDE_CONFIGURATIONS")
-            }
-            if (config.logger.verbose) {
-                setStandardOutput(System.err)
-                setStandardError(System.err)
-            }
+        }
+        .setJavaHome(config.gradleJdk)
+        .addArguments(config.gradleArgs)
+        .addArguments(
+            "--no-parallel",
+            "--refresh-dependencies",
+            "--gradle-user-home=${config.gradleHome}",
+            "--init-script=${config.appHome}/init.gradle",
+            "--write-verification-metadata", "sha256"
+        )
+        .apply {
             if (config.logger.stacktrace) {
                 addArguments("--stacktrace")
             }
+            if (config.logger.logLevel <= LogLevel.debug) {
+                setStandardOutput(System.err)
+                setStandardError(System.err)
+            }
+            if (config.dumpEvents) {
+                withSystemProperties(
+                    mapOf(
+                        "org.gradle.internal.operations.trace" to
+                                config.outDir.toPath().resolve("debug").absolutePathString()
+                    )
+                )
+            }
         }
-        .run()
+        .run(object : ResultHandler<DependencySet> {
+            override fun onComplete(result: DependencySet) {
+                continuation.resume(result)
+            }
+
+            override fun onFailure(failure: GradleConnectionException) {
+                continuation.resumeWithException(failure)
+            }
+        })
 }

app/src/main/kotlin/org/nixos/gradle2nix/Logger.kt

@@ -5,43 +5,44 @@ import kotlin.system.exitProcess
 
 class Logger(
     val out: PrintStream = System.err,
-    val verbose: Boolean,
-    val stacktrace: Boolean = false
+    val logLevel: LogLevel = LogLevel.warn,
+    val stacktrace: Boolean = false,
 ) {
 
     fun debug(message: String, error: Throwable? = null) {
-        if (!stacktrace) return
-        out.println(message)
-        if (error == null) return
-        error.message?.let { println("  Cause: $it") }
-        error.printStackTrace(out)
+        if (logLevel <= LogLevel.debug) {
+            out.println("[DEBUG] $message")
+            printError(error)
+        }
     }
 
-    fun log(message: String, error: Throwable? = null) {
-        if (!verbose) return
-        out.println(message)
-        if (error == null) return
-        error.message?.let { println("  Cause: $it") }
-        if (stacktrace) error.printStackTrace(out)
+    fun info(message: String, error: Throwable? = null) {
+        if (logLevel <= LogLevel.info) {
+            out.println("[INFO] $message")
+            printError(error)
+        }
     }
 
     fun warn(message: String, error: Throwable? = null) {
-        out.println("Warning: $message")
+        if (logLevel <= LogLevel.warn) {
+            out.println("[WARN] $message")
+            printError(error)
+        }
+    }
+
+    fun error(message: String, error: Throwable? = null): Nothing {
+        out.println("[ERROR] $message")
+        printError(error)
+        exitProcess(1)
+    }
+
+    private fun printError(error: Throwable?) {
         if (error == null) return
         error.message?.let { println("  Cause: $it") }
         if (stacktrace) error.printStackTrace(out)
     }
 
-    fun error(message: String, error: Throwable? = null): Nothing {
-        out.println("Error: $message")
-        if (error != null) {
-            error.message?.let { println("  Cause: $it") }
-            if (stacktrace) error.printStackTrace(out)
-        }
-        exitProcess(1)
-    }
-
-    operator fun component1() = ::log
+    operator fun component1() = ::info
     operator fun component2() = ::warn
     operator fun component3() = ::error
 }

app/src/main/kotlin/org/nixos/gradle2nix/Main.kt

@@ -10,11 +10,16 @@ import com.github.ajalt.clikt.parameters.options.flag
 import com.github.ajalt.clikt.parameters.options.multiple
 import com.github.ajalt.clikt.parameters.options.option
 import com.github.ajalt.clikt.parameters.options.validate
+import com.github.ajalt.clikt.parameters.types.choice
+import com.github.ajalt.clikt.parameters.types.enum
 import com.github.ajalt.clikt.parameters.types.file
 import java.io.File
+import kotlinx.coroutines.runBlocking
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.encodeToStream
+import org.gradle.tooling.model.gradle.GradleBuild
+import org.nixos.gradle2nix.model.DependencySet
 
 data class Config(
     val appHome: File,
@@ -22,11 +27,11 @@ data class Config(
     val gradleVersion: String?,
     val gradleJdk: File?,
     val gradleArgs: List<String>,
-    val projectFilter: String?,
-    val configurationFilter: String?,
+    val outDir: File,
     val projectDir: File,
     val tasks: List<String>,
     val logger: Logger,
+    val dumpEvents: Boolean
 )
 
 @OptIn(ExperimentalSerializationApi::class)
@@ -36,6 +41,13 @@ val JsonFormat = Json {
     prettyPrintIndent = "  "
 }
 
+enum class LogLevel {
+    debug,
+    info,
+    warn,
+    error,
+}
+
 class Gradle2Nix : CliktCommand(
     name = "gradle2nix"
 ) {
@@ -51,17 +63,6 @@ class Gradle2Nix : CliktCommand(
         help = "JDK home directory to use for launching Gradle (default: ${System.getProperty("java.home")})"
     ).file(canBeFile = false, canBeDir = true)
 
-    private val projectFilter: String? by option(
-        "--projects", "-p",
-        metavar = "REGEX",
-        help = "Regex to filter Gradle projects (default: include all projects)"
-    )
-
-    private val configurationFilter: String? by option(
-        "--configurations", "-c",
-        metavar = "REGEX",
-        help = "Regex to filter Gradle configurations (default: include all configurations)")
-
     val outDir: File? by option(
         "--out-dir", "-o",
         metavar = "DIR",
@@ -74,11 +75,12 @@ class Gradle2Nix : CliktCommand(
         help = "Prefix for environment files (.json and .nix)")
         .default("gradle-env")
 
-    private val debug: Boolean by option("--debug", help = "Enable debug logging")
-        .flag(default = false)
-
-    private val quiet: Boolean by option("--quiet", "-q", help = "Disable logging")
-        .flag(default = false)
+    private val logLevel: LogLevel by option(
+        "--log", "-l",
+        metavar = "LEVEL",
+        help = "Print messages with priority of at least LEVEL")
+        .enum<LogLevel>()
+        .default(LogLevel.error)
 
     private val projectDir: File by option(
         "--projectDir", "-d",
@@ -99,6 +101,16 @@ class Gradle2Nix : CliktCommand(
         help = "Gradle tasks to run"
     ).multiple()
 
+    private val dumpEvents: Boolean by option(
+        "--dump-events",
+        help = "Dump Gradle event logs to the output directory",
+    ).flag()
+
+    private val stacktrace: Boolean by option(
+        "--stacktrace",
+        help = "Print a stack trace on error"
+    ).flag()
+
     private val gradleArgs: List<String> by argument(
         name = "ARGS",
         help = "Extra arguments to pass to Gradle"
@@ -118,7 +130,7 @@ class Gradle2Nix : CliktCommand(
         }
         val gradleHome =
             System.getenv("GRADLE_USER_HOME")?.let(::File) ?: File("${System.getProperty("user.home")}/.gradle")
-        val logger = Logger(verbose = !quiet, stacktrace = debug)
+        val logger = Logger(logLevel = logLevel, stacktrace = stacktrace)
 
         val config = Config(
             File(appHome),
@@ -126,11 +138,11 @@ class Gradle2Nix : CliktCommand(
             gradleVersion,
             gradleJdk,
             gradleArgs,
-            projectFilter,
-            configurationFilter,
+            outDir ?: projectDir,
             projectDir,
             tasks,
-            logger
+            logger,
+            dumpEvents
         )
 
         val metadata = File("$projectDir/gradle/verification-metadata.xml")
@@ -146,19 +158,37 @@ class Gradle2Nix : CliktCommand(
             }
         }
 
+        val buildSrcs = connect(config).use { connection ->
+            val root = runBlocking { connection.buildModel() }
+            val builds: List<GradleBuild> = buildList {
+                add(root)
+                addAll(root.editableBuilds)
+            }
+            builds.mapNotNull { build ->
+                build.rootProject.projectDirectory.resolve("buildSrc").takeIf { it.exists() }
+            }
+        }
+
+        val dependencySets = mutableListOf<DependencySet>()
+
         connect(config).use { connection ->
-            connection.build(config)
+            dependencySets.add(runBlocking { connection.build(config) })
+        }
+
+        for (buildSrc in buildSrcs) {
+            connect(config, buildSrc).use { connection ->
+                dependencySets.add(runBlocking { connection.build(config) })
+            }
         }
 
         val env = try {
-            processDependencies(config)
+            processDependencies(config, dependencySets)
         } catch (e: Throwable) {
             logger.error("dependency parsing failed", e)
         }
 
-        val outDir = outDir ?: projectDir
-        val json = outDir.resolve("$envFile.json")
-        logger.log("Writing environment to $json")
+        val json = config.outDir.resolve("$envFile.json")
+        logger.info("Writing environment to $json")
         json.outputStream().buffered().use { output ->
             JsonFormat.encodeToStream(env, output)
         }

app/src/main/kotlin/org/nixos/gradle2nix/Process.kt

@@ -4,23 +4,13 @@ import org.nixos.gradle2nix.metadata.Artifact as ArtifactMetadata
 import java.io.File
 import java.io.IOException
 import java.net.URI
-import java.net.URL
-import kotlinx.serialization.ExperimentalSerializationApi
-import kotlinx.serialization.SerializationException
-import kotlinx.serialization.json.Json
-import kotlinx.serialization.json.decodeFromStream
 import okio.ByteString.Companion.decodeHex
 import okio.HashingSource
 import okio.blackholeSink
 import okio.buffer
 import okio.source
-import org.nixos.gradle2nix.model.Repository
-import org.nixos.gradle2nix.model.ResolvedConfiguration
-import org.nixos.gradle2nix.env.ArtifactFile
-import org.nixos.gradle2nix.env.ArtifactSet
+import org.nixos.gradle2nix.env.Artifact
 import org.nixos.gradle2nix.env.Env
-import org.nixos.gradle2nix.env.Module
-import org.nixos.gradle2nix.env.ModuleId
 import org.nixos.gradle2nix.metadata.Checksum
 import org.nixos.gradle2nix.metadata.Component
 import org.nixos.gradle2nix.metadata.Md5
@@ -30,239 +20,100 @@ import org.nixos.gradle2nix.metadata.Sha512
 import org.nixos.gradle2nix.metadata.VerificationMetadata
 import org.nixos.gradle2nix.metadata.parseVerificationMetadata
 import org.nixos.gradle2nix.model.DependencyCoordinates
-import org.nixos.gradle2nix.model.Version
-import org.nixos.gradle2nix.module.GradleModule
-import org.nixos.gradle2nix.module.Variant
+import org.nixos.gradle2nix.model.DependencySet
 
-// Local Maven repository for testing
-private val m2 = System.getProperty("org.nixos.gradle2nix.m2")
-
-private fun shouldSkipRepository(repository: Repository): Boolean {
-    return repository.artifactResources.all { it.startsWith("file:") && (m2 == null || !it.startsWith(m2)) } ||
-            repository.metadataResources.all { it.startsWith("file:") && (m2 == null || !it.startsWith(m2)) }
-}
-
-fun processDependencies(config: Config): Env {
+fun processDependencies(
+    config: Config,
+    dependencySets: Iterable<DependencySet>
+): Env {
     val verificationMetadata = readVerificationMetadata(config)
-    val verificationComponents = verificationMetadata?.components?.associateBy { it.id } ?: emptyMap()
-    val moduleCache = mutableMapOf<DependencyCoordinates, GradleModule?>()
-    val pomCache = mutableMapOf<DependencyCoordinates, Pair<String, ArtifactFile>?>()
-    val ivyCache = mutableMapOf<DependencyCoordinates, Pair<String, ArtifactFile>?>()
-    val configurations = readDependencyGraph(config)
+    val verificationComponents = verificationMetadata?.components?.associateBy { it.id.id } ?: emptyMap()
 
-    val repositories = configurations
-        .flatMap { it.repositories }
-        .associateBy { it.id }
-        .filterNot { (id, repo) ->
-            if (shouldSkipRepository(repo)) {
-                config.logger.warn("$id: all URLs are files; skipping")
-                true
-            } else {
-                false
+    return buildMap<DependencyCoordinates, Map<String, Artifact>> {
+        for (dependencySet in dependencySets) {
+            val env = dependencySet.toEnv(config, verificationComponents)
+
+            for ((id, artifacts) in env) {
+                merge(id, artifacts) { a, b ->
+                    buildMap {
+                        putAll(a)
+                        for ((name, artifact) in b) {
+                            merge(name, artifact) { aa, ba ->
+                                check(aa.hash == ba.hash) {
+                                    config.logger.error("""
+                                        Conflicting hashes found for $id:$name:
+                                          1: ${aa.hash}
+                                          2: ${ba.hash}
+                                    """.trimIndent())
+                                }
+
+                                Artifact(
+                                    (aa.urls + ba.urls).distinct().sorted(),
+                                    aa.hash
+                                )
+                            }
+                        }
+                    }
+                }
             }
         }
-    if (repositories.isEmpty()) {
-        config.logger.warn("no repositories found in any configuration")
-        return emptyMap()
+    }.mapValues { (_, artifacts) ->
+        artifacts.toSortedMap()
+    }.toSortedMap(coordinatesComparator)
+        .mapKeys { (coordinates, _) -> coordinates.id }
+}
+
+private fun DependencySet.toEnv(config: Config, verificationComponents: Map<String, Component>): Map<DependencyCoordinates, Map<String, Artifact>> {
+    return dependencies.associate { dep ->
+        val component = verificationComponents[dep.coordinates.id]
+            ?: verifyComponentFilesInCache(config, dep.coordinates)
+            ?: config.logger.error("${dep.coordinates}: no dependency metadata found")
+
+        dep.coordinates to dep.artifacts.mapNotNull { resolvedArtifact ->
+            val artifact = component.artifacts.find { it.name == resolvedArtifact.name }
+                ?.let { Artifact(resolvedArtifact.urls.sorted(), it.checksums.first().toSri()) }
+                ?: downloadArtifact(resolvedArtifact.urls.sorted())
+            artifact?.let { resolvedArtifact.filename to it }
+        }.sortedBy { it.first }.toMap()
     }
-    config.logger.debug("Repositories:\n  ${repositories.values.joinToString("\n  ")}")
-
-    return configurations.asSequence()
-        .flatMap { it.allDependencies.asSequence() }
-        .filterNot { it.repository == null || it.repository !in repositories }
-        .groupBy { ModuleId(it.id.group, it.id.module) }
-        .mapValues { (_, deps) ->
-            val byVersion = deps.groupBy { it.id }
-                .mapValues { (componentId, deps) ->
-                    val dep = MergedDependency(
-                        id = componentId,
-                        repositories = deps.mapNotNull { repositories[it.repository] }.distinct()
-                    )
-                    val component = verificationComponents[componentId]
-                        ?: verifyComponentFilesInCache(config, componentId)
-                        ?: verifyComponentFilesInTestRepository(config, componentId)
-                        ?: config.logger.error("$componentId: no dependency metadata found")
-
-                    val gradleModule = moduleCache.getOrPut(componentId) {
-                        maybeDownloadGradleModule(config.logger, component, dep.repositories)?.artifact?.second
-                    }
-                    val pomArtifact = pomCache.getOrPut(componentId) {
-                        maybeDownloadMavenPom(config.logger, component, dep.repositories, gradleModule)
-                    }
-                    val ivyArtifact = ivyCache.getOrPut(componentId) {
-                        maybeDownloadIvyDescriptor(config.logger, component, dep.repositories)
-                    }
-
-                    val files = buildMap {
-                        if (pomArtifact != null) put(pomArtifact.first, pomArtifact.second)
-                        if (ivyArtifact != null) put(ivyArtifact.first, ivyArtifact.second)
-                        for (artifact in component.artifacts) {
-                            put(
-                                artifact.name,
-                                ArtifactFile(
-                                    urls = dep.repositories.flatMap { repo ->
-                                        artifactUrls(config.logger, componentId, artifact.name, repo, gradleModule)
-                                    }.distinct(),
-                                    hash = artifact.checksums.first().toSri()
-                                )
-                            )
-                        }
-                    }.toSortedMap()
-
-                    ArtifactSet(files)
-                }
-                .mapKeys { Version(it.key.version) }
-                .toSortedMap(Version.Comparator.reversed())
-            Module(byVersion)
-        }
-        .toSortedMap(compareBy(ModuleId::toString))
 }
 
 private fun readVerificationMetadata(config: Config): VerificationMetadata? {
     return parseVerificationMetadata(config.logger, config.projectDir.resolve("gradle/verification-metadata.xml"))
 }
 
-@OptIn(ExperimentalSerializationApi::class)
-private fun readDependencyGraph(config: Config): List<ResolvedConfiguration> {
-    return config.projectDir.resolve("build/reports/nix-dependency-graph/dependency-graph.json")
-        .inputStream()
-        .buffered()
-        .use { input -> Json.decodeFromStream(input) }
-}
-
 private fun verifyComponentFilesInCache(
     config: Config,
     id: DependencyCoordinates,
 ): Component? {
-    val cacheDir = with(id) { config.gradleHome.resolve("caches/modules-2/files-2.1/$group/$module/$version") }
+    val cacheDir = with(id) { config.gradleHome.resolve("caches/modules-2/files-2.1/$group/$artifact/$version") }
     if (!cacheDir.exists()) {
         return null
     }
     val verifications = cacheDir.walk().filter { it.isFile }.map { f ->
-        ArtifactMetadata(f.name, sha256 = Sha256(f.sha256()))
+        ArtifactMetadata(f.name.replaceFirst(id.version, id.timestampedVersion), sha256 = Sha256(f.sha256()))
     }
-    config.logger.log("$id: obtained artifact hashes from Gradle cache.")
+    config.logger.info("${id.id}: obtained artifact hashes from Gradle cache.")
     return Component(id, verifications.toList())
 }
 
-private fun verifyComponentFilesInTestRepository(
-    config: Config,
-    id: DependencyCoordinates
-): Component? {
-    if (m2 == null) return null
-    val dir = with(id) {
-        File(URI.create(m2)).resolve("${group.replace(".", "/")}/$module/$version")
-    }
-    if (!dir.exists()) {
-        config.logger.log("$id: not found in m2 repository; tried $dir")
-        return null
-    }
-    val verifications = dir.walk().filter { it.isFile && it.name.startsWith(id.module) }.map { f ->
-        ArtifactMetadata(
-            f.name,
-            sha256 = Sha256(f.sha256())
-        )
-    }
-    config.logger.log("$id: obtained artifact hashes from test Maven repository.")
-    return Component(id, verifications.toList())
-}
-
-private fun maybeDownloadGradleModule(
-    logger: Logger,
-    component: Component,
-    repos: List<Repository>
-): ArtifactDownload<Pair<String, GradleModule>>? {
-    if (component.artifacts.none { it.name.endsWith(".module") }) return null
-    val filename = with(component.id) { "$module-$version.module" }
-    return maybeDownloadArtifact(logger, component.id, filename, repos)?.let { artifact ->
-        try {
-            ArtifactDownload(
-                filename to JsonFormat.decodeFromString<GradleModule>(artifact.artifact),
-                artifact.url,
-                artifact.hash
-            )
-        } catch (e: SerializationException) {
-            logger.warn("${component.id}: failed to parse Gradle module metadata from ${artifact.url}")
-            null
-        }
-    }
-}
-
-private fun maybeDownloadMavenPom(
-    logger: Logger,
-    component: Component,
-    repos: List<Repository>,
-    gradleModule: GradleModule?
-): Pair<String, ArtifactFile>? {
-    if (component.artifacts.any { it.name.endsWith(".pom") }) return null
-    val pomRepos = repos.filter { "mavenPom" in it.metadataSources }
-    if (pomRepos.isEmpty()) return null
-    val filename = with(component.id) { "$module-$version.pom" }
-
-    return maybeDownloadArtifact(logger, component.id, filename, pomRepos)?.let { artifact ->
-        filename to ArtifactFile(
-            urls = pomRepos.flatMap { repo ->
-                artifactUrls(logger, component.id, filename, repo, gradleModule)
-            }.distinct(),
-            hash = artifact.hash.toSri()
+private fun downloadArtifact(
+    urls: List<String>
+): Artifact? {
+    return maybeDownloadText(urls)?.let {
+        Artifact(
+            urls,
+            it.hash.toSri()
         )
     }
 }
 
-private fun maybeDownloadIvyDescriptor(
-    logger: Logger,
-    component: Component,
-    repos: List<Repository>,
-): Pair<String, ArtifactFile>? {
-    val ivyRepos = repos.filter { "ivyDescriptor" in it.metadataSources }
-    if (ivyRepos.isEmpty()) return null
-
-    val urls = ivyRepos
-        .flatMap { repo ->
-            val attributes = attributes(component.id, repo)
-            repo.metadataResources.mapNotNull { fill(it, attributes).takeIf(::isUrlComplete) }
-        }
-        .filter { url ->
-            component.artifacts.none { url.substringAfterLast('/') == it.name }
-        }
-
-    var artifact: ArtifactDownload<String>? = null
-
-    for (url in urls) {
-        try {
-            val source = HashingSource.sha256(URL(url).openStream().source())
-            val text = source.buffer().readUtf8()
-            val hash = source.hash
-            artifact = ArtifactDownload(text, url, Sha256(hash.hex()))
-            break
-        } catch (e: IOException) {
-            // Pass
-        }
-    }
-
-    if (artifact == null) {
-        logger.debug("ivy descriptor not found in urls: $urls")
-        return null
-    }
-    return artifact.artifact to ArtifactFile(
-        urls = urls,
-        hash = artifact.hash.toSri()
-    )
-}
-
-private fun maybeDownloadArtifact(
-    logger: Logger,
-    id: DependencyCoordinates,
-    filename: String,
-    repos: List<Repository>
+private fun maybeDownloadText(
+    urls: List<String>,
 ): ArtifactDownload<String>? {
-    val urls = repos.flatMap { artifactUrls(logger, id, filename, it, null)}
-
-    logger.debug("artifact $filename: $urls")
-
     for (url in urls) {
         try {
-            val source = HashingSource.sha256(URL(url).openStream().source())
+            val source = HashingSource.sha256(URI(url).toURL().openStream().source())
             val text = source.buffer().readUtf8()
             val hash = source.hash
             return ArtifactDownload(text, url, Sha256(hash.hex()))
@@ -270,8 +121,6 @@ private fun maybeDownloadArtifact(
             // Pass
         }
     }
-
-    logger.debug("artifact $filename not found in any repository")
     return null
 }
 
@@ -281,7 +130,7 @@ private fun File.sha256(): String {
     return source.hash.hex()
 }
 
-private fun Checksum.toSri(): String {
+internal fun Checksum.toSri(): String {
     val hash = value.decodeHex().base64()
     return when (this) {
         is Md5 -> "md5-$hash"
@@ -291,96 +140,13 @@ private fun Checksum.toSri(): String {
     }
 }
 
-private fun artifactUrls(
-    logger: Logger,
-    id: DependencyCoordinates,
-    filename: String,
-    repository: Repository,
-    module: GradleModule?,
-): List<String> {
-    val groupAsPath = id.group.replace(".", "/")
-
-    val repoFilename = module?.let { m ->
-        m.variants
-            .asSequence()
-            .flatMap(Variant::files)
-            .find { it.name == filename }
-    }?.url ?: filename
-
-    val attributes = mutableMapOf(
-        "organisation" to if (repository.m2Compatible) groupAsPath else id.group,
-        "module" to id.module,
-        "revision" to id.version,
-    ) + fileAttributes(repoFilename, id.version)
-
-    val resources = when (attributes["ext"]) {
-        "pom" -> if ("mavenPom" in repository.metadataSources) repository.metadataResources else repository.artifactResources
-        "xml" -> if ("ivyDescriptor" in repository.metadataSources) repository.metadataResources else repository.artifactResources
-        "module" -> if ("gradleMetadata" in repository.metadataSources || "ignoreGradleMetadataRedirection" !in repository.metadataSources) {
-            repository.metadataResources
-        } else {
-            repository.artifactResources
-        }
-        else -> repository.artifactResources
-    }.map { it.replaceFirst("-[revision]", "-${id.artifactVersion}") }
-
-    val urls = mutableListOf<String>()
-
-    for (resource in resources) {
-        val location = attributes.entries.fold(fill(resource, attributes)) { acc, (key, value) ->
-            acc.replace("[$key]", value)
-        }
-        if (location.none { it == '[' || it == ']' }) {
-            urls.add(location)
-        } else {
-            logger.warn("failed to construct artifact URL: $location")
-        }
-    }
-
-    return urls.distinct()
-}
-
-private val optionalRegex = Regex("\\(([^)]+)\\)")
-private val attrRegex = Regex("\\[([^]]+)]")
-
-private fun fill(template: String, attributes: Map<String, String>): String {
-    return optionalRegex.replace(template) { match ->
-        attrRegex.find(match.value)?.groupValues?.get(1)?.let { attr ->
-            attributes[attr]?.takeIf { it.isNotBlank() }?.let { value ->
-                match.groupValues[1].replace("[$attr]", value)
-            }
-        } ?: ""
-    }
-}
-
-private fun isUrlComplete(url: String): Boolean = !url.contains("[")
-
-private fun attributes(id: DependencyCoordinates, repository: Repository): Map<String, String> = buildMap {
-    put("organisation", if (repository.m2Compatible) id.group.replace(".", "/") else id.group)
-    put("module", id.module)
-    put("revision", id.version)
-}
-
-// Gradle persists artifacts with the Maven artifact pattern, which may not match the repository's pattern.
-private fun fileAttributes(file: String, version: String): Map<String, String> {
-    val parts = Regex("(.+)-$version(-([^.]+))?(\\.(.+))?").matchEntire(file) ?: return emptyMap()
-
-    val (artifact, _, classifier, _, ext) = parts.destructured
-
-    return buildMap {
-        put("artifact", artifact)
-        put("classifier", classifier)
-        put("ext", ext)
-    }
-}
-
-private data class MergedDependency(
-    val id: DependencyCoordinates,
-    val repositories: List<Repository>
-)
-
 private data class ArtifactDownload<T>(
     val artifact: T,
     val url: String,
     val hash: Checksum
 )
+
+private val coordinatesComparator: Comparator<DependencyCoordinates> = compareBy<DependencyCoordinates> { it.group }
+    .thenBy { it.artifact }
+    .thenByDescending { Version(it.version) }
+    .thenByDescending { it.timestamp }

app/src/main/kotlin/org/nixos/gradle2nix/Version.kt

@@ -0,0 +1,143 @@
+package org.nixos.gradle2nix
+
+import java.util.concurrent.ConcurrentHashMap
+
+class Version(val source: String, val parts: List<String>, base: Version?) : Comparable<Version> {
+
+    private val base: Version
+    val numericParts: List<Long?>
+
+    init {
+        this.base = base ?: this
+        this.numericParts = parts.map {
+            try { it.toLong() } catch (e: NumberFormatException) { null }
+        }
+    }
+
+    override fun compareTo(other: Version): Int = compare(this, other)
+
+    override fun toString(): String = source
+
+    override fun equals(other: Any?): Boolean = when {
+        other === this -> true
+        other == null || other !is Version -> false
+        else -> source == other.source
+    }
+
+    override fun hashCode(): Int = source.hashCode()
+
+    companion object {
+        private val SPECIAL_MEANINGS: Map<String, Int> = mapOf(
+            "dev" to -1,
+            "rc" to 1,
+            "snapshot" to 2,
+            "final" to 3,
+            "ga" to 4,
+            "release" to 5,
+            "sp" to 6
+        )
+
+        private val cache = ConcurrentHashMap<String, Version>()
+
+        // From org.gradle.api.internal.artifacts.ivyservice.ivyresolve.strategy.VersionParser
+        operator fun invoke(original: String): Version = cache.getOrPut(original) {
+            val parts = mutableListOf<String>()
+            var digit = false
+            var startPart = 0
+            var pos = 0
+            var endBase = 0
+            var endBaseStr = 0
+            while (pos < original.length) {
+                val ch = original[pos]
+                if (ch == '.' || ch == '_' || ch == '-' || ch == '+') {
+                    parts.add(original.substring(startPart, pos))
+                    startPart = pos + 1
+                    digit = false
+                    if (ch != '.' && endBaseStr == 0) {
+                        endBase = parts.size
+                        endBaseStr = pos
+                    }
+                } else if (ch in '0'..'9') {
+                    if (!digit && pos > startPart) {
+                        if (endBaseStr == 0) {
+                            endBase = parts.size + 1
+                            endBaseStr = pos
+                        }
+                        parts.add(original.substring(startPart, pos))
+                        startPart = pos
+                    }
+                    digit = true
+                } else {
+                    if (digit) {
+                        if (endBaseStr == 0) {
+                            endBase = parts.size + 1
+                            endBaseStr = pos
+                        }
+                        parts.add(original.substring(startPart, pos))
+                        startPart = pos
+                    }
+                    digit = false
+                }
+                pos++
+            }
+            if (pos > startPart) {
+                parts.add(original.substring(startPart, pos))
+            }
+            var base: Version? = null
+            if (endBaseStr > 0) {
+                base = Version(original.substring(0, endBaseStr), parts.subList(0, endBase), null)
+            }
+            Version(original, parts, base)
+        }
+
+        // From org.gradle.api.internal.artifacts.ivyservice.ivyresolve.strategy.StaticVersionComparator
+        private fun compare(version1: Version, version2: Version): Int {
+            if (version1 == version2) {
+                return 0
+            }
+
+            val parts1 = version1.parts
+            val parts2 = version2.parts
+            val numericParts1 = version1.numericParts
+            val numericParts2 = version2.numericParts
+            var lastIndex = -1
+
+            for (i in 0..<(minOf(parts1.size, parts2.size))) {
+                lastIndex = i
+
+                val part1 = parts1[i]
+                val part2 = parts2[i]
+
+                val numericPart1 = numericParts1[i]
+                val numericPart2 = numericParts2[i]
+
+                when {
+                    part1 == part2 -> continue
+                    numericPart1 != null && numericPart2 == null -> return 1
+                    numericPart2 != null && numericPart1 == null -> return -1
+                    numericPart1 != null && numericPart2 != null -> {
+                        val result = numericPart1.compareTo(numericPart2)
+                        if (result == 0) continue
+                        return result
+                    }
+                    else -> {
+                        // both are strings, we compare them taking into account special meaning
+                        val sm1 = SPECIAL_MEANINGS[part1.lowercase()]
+                        val sm2 = SPECIAL_MEANINGS[part2.lowercase()]
+                        if (sm1 != null) return sm1 - (sm2 ?: 0)
+                        if (sm2 != null) return -sm2
+                        return part1.compareTo(part2)
+                    }
+                }
+            }
+            if (lastIndex < parts1.size) {
+                return if (numericParts1[lastIndex] == null) -1 else 1
+            }
+            if (lastIndex < parts2.size) {
+                return if (numericParts2[lastIndex] == null) 1 else -1
+            }
+
+            return 0
+        }
+    }
+}

app/src/main/kotlin/org/nixos/gradle2nix/env/Env.kt

@@ -1,68 +1,22 @@
 package org.nixos.gradle2nix.env
 
-import kotlinx.serialization.KSerializer
 import kotlinx.serialization.Serializable
-import kotlinx.serialization.SerializationException
-import kotlinx.serialization.descriptors.PrimitiveKind
-import kotlinx.serialization.descriptors.PrimitiveSerialDescriptor
-import kotlinx.serialization.descriptors.SerialDescriptor
-import kotlinx.serialization.encoding.Decoder
-import kotlinx.serialization.encoding.Encoder
-import org.nixos.gradle2nix.model.Version
 
-typealias Env = Map<ModuleId, Module>
+typealias Env = Map<String, Map<String, Artifact>>
 
 @Serializable
-@JvmInline
-value class Module(
-    val versions: Map<Version, ArtifactSet>,
-)
-
-@Serializable
-@JvmInline
-value class ArtifactSet(
-    val files: Map<String, ArtifactFile>
-)
-
-@Serializable
-data class ArtifactFile internal constructor(
+data class Artifact internal constructor(
     val urls: List<String>,
     val hash: String,
 ) {
 
     companion object {
-        operator fun invoke(urls: List<String>, hash: String) = ArtifactFile(urls.sorted(), hash)
-    }
-}
-
-@Serializable(ModuleId.Serializer::class)
-data class ModuleId(
-    val group: String,
-    val name: String,
-) : Comparable<ModuleId> {
-
-    override fun compareTo(other: ModuleId): Int =
-        compareValuesBy(this, other, ModuleId::group, ModuleId::name)
-
-    override fun toString(): String = "$group:$name"
-
-    companion object Serializer : KSerializer<ModuleId> {
-        override val descriptor: SerialDescriptor get() = PrimitiveSerialDescriptor(
-            ModuleId::class.qualifiedName!!,
-            PrimitiveKind.STRING
+        operator fun invoke(
+            urls: List<String>,
+            hash: String
+        ) = Artifact(
+            urls.sorted(),
+            hash
         )
-
-        override fun serialize(encoder: Encoder, value: ModuleId) {
-            encoder.encodeString(value.toString())
-        }
-
-        override fun deserialize(decoder: Decoder): ModuleId {
-            val encoded = decoder.decodeString()
-            val parts = encoded.split(":")
-            if (parts.size != 2 || parts.any(String::isBlank)) {
-                throw SerializationException("invalid module id: $encoded")
-            }
-            return ModuleId(parts[0], parts[1])
-        }
     }
 }

app/src/main/kotlin/org/nixos/gradle2nix/gradle/GradleModule.kt

@@ -1,4 +1,4 @@
-package org.nixos.gradle2nix.module
+package org.nixos.gradle2nix.gradle
 
 import kotlinx.serialization.SerialName
 import kotlinx.serialization.Serializable

app/src/main/kotlin/org/nixos/gradle2nix/metadata/VerificationMetadata.kt

@@ -10,6 +10,7 @@ import nl.adaptivity.xmlutil.serialization.XmlSerialName
 import nl.adaptivity.xmlutil.xmlStreaming
 import org.nixos.gradle2nix.Logger
 import org.nixos.gradle2nix.model.DependencyCoordinates
+import org.nixos.gradle2nix.model.impl.DefaultDependencyCoordinates
 
 sealed interface Coordinates {
     val group: String?
@@ -40,10 +41,10 @@ data class Configuration(
 
 @Serializable
 sealed interface Checksum {
-    abstract val value: String
-    abstract val origin: String?
-    abstract val reason: String?
-    abstract val alternatives: List<String>
+    val value: String
+    val origin: String?
+    val reason: String?
+    val alternatives: List<String>
 }
 
 @Serializable
@@ -107,11 +108,11 @@ data class Component(
     val timestamp: String? = null,
     val artifacts: List<Artifact> = emptyList(),
 ) {
-    val id: DependencyCoordinates get() = DependencyCoordinates(group, name, version, timestamp)
+    val id: DependencyCoordinates get() = DefaultDependencyCoordinates(group, name, version, timestamp)
 
     constructor(id: DependencyCoordinates, artifacts: List<Artifact>) : this(
         id.group,
-        id.module,
+        id.artifact,
         id.version,
         id.timestamp,
         artifacts

app/src/test/kotlin/org/nixos/gradle2nix/GoldenTest.kt

@@ -1,8 +1,11 @@
 package org.nixos.gradle2nix
 
+import io.kotest.core.extensions.install
 import io.kotest.core.spec.style.FunSpec
 
 class GoldenTest : FunSpec({
+    install(MavenRepo)
+
     context("basic") {
         golden("basic/basic-java-project")
         golden("basic/basic-kotlin-project")

app/src/test/kotlin/org/nixos/gradle2nix/TestUtil.kt

@@ -1,20 +1,37 @@
 package org.nixos.gradle2nix
 
 import io.kotest.assertions.fail
+import io.kotest.assertions.withClue
 import io.kotest.common.ExperimentalKotest
 import io.kotest.common.KotestInternal
+import io.kotest.core.extensions.MountableExtension
+import io.kotest.core.listeners.AfterSpecListener
 import io.kotest.core.names.TestName
 import io.kotest.core.source.sourceRef
+import io.kotest.core.spec.Spec
 import io.kotest.core.test.NestedTest
 import io.kotest.core.test.TestScope
 import io.kotest.core.test.TestType
 import io.kotest.matchers.equals.beEqual
 import io.kotest.matchers.file.shouldBeAFile
+import io.kotest.matchers.nulls.shouldNotBeNull
 import io.kotest.matchers.should
+import io.kotest.matchers.shouldBe
+import io.ktor.http.ContentType
+import io.ktor.http.Url
+import io.ktor.server.engine.embeddedServer
+import io.ktor.server.http.content.staticFiles
+import io.ktor.server.netty.Netty
+import io.ktor.server.netty.NettyApplicationEngine
+import io.ktor.server.routing.routing
 import java.io.File
 import java.io.FileFilter
 import java.nio.file.Files
 import java.nio.file.Paths
+import kotlin.random.Random
+import kotlinx.coroutines.CoroutineScope
+import kotlinx.coroutines.Dispatchers
+import kotlinx.coroutines.launch
 import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.SerializationException
 import kotlinx.serialization.encodeToString
@@ -23,6 +40,7 @@ import kotlinx.serialization.json.decodeFromStream
 import kotlinx.serialization.json.encodeToStream
 import okio.use
 import org.nixos.gradle2nix.env.Env
+import org.nixos.gradle2nix.metadata.parseVerificationMetadata
 
 private val app = Gradle2Nix()
 
@@ -32,7 +50,7 @@ private val json = Json {
     prettyPrintIndent = "  "
 }
 
-val testLogger = Logger(verbose = true, stacktrace = true)
+val testLogger = Logger(logLevel = LogLevel.debug, stacktrace = true)
 
 fun fixture(path: String): File {
     return Paths.get("../fixtures", path).toFile()
@@ -42,10 +60,10 @@ fun fixture(path: String): File {
 suspend fun TestScope.fixture(
     project: String,
     vararg args: String,
-    test: suspend TestScope.(Env) -> Unit
+    test: suspend TestScope.(File, Env) -> Unit
 ) {
     val tmp = Paths.get("build/tmp/gradle2nix").apply { toFile().mkdirs() }
-    val baseDir = Paths.get("../fixtures", project).toFile()
+    val baseDir = Paths.get("../fixtures/projects", project).toFile()
     val children = baseDir.listFiles(FileFilter { it.isDirectory && (it.name == "groovy" || it.name == "kotlin") })
         ?.toList()
     val cases = if (children.isNullOrEmpty()) {
@@ -73,13 +91,22 @@ suspend fun TestScope.fixture(
                 if (!tempDir.resolve("settings.gradle").exists() && !tempDir.resolve("settings.gradle.kts").exists()) {
                     Files.createFile(tempDir.resolve("settings.gradle").toPath())
                 }
-                app.main(listOf("-d", tempDir.toString()) + listOf("--debug") + args.withM2() + "-Dorg.gradle.internal.operations.trace=${tempDir.resolve("build").absolutePath}")
+                app.main(
+                    listOf(
+                        "-d", tempDir.toString(),
+                        "--log", "debug",
+                        "--stacktrace",
+                        "--dump-events",
+                        "--",
+                        "-Dorg.nixos.gradle2nix.m2=$m2"
+                    ) + args
+                )
                 val file = tempDir.resolve("${app.envFile}.json")
                 file.shouldBeAFile()
                 val env: Env = file.inputStream().buffered().use { input ->
                     Json.decodeFromStream(input)
                 }
-                test(env)
+                test(tempDir, env)
             }
         )
     }
@@ -92,7 +119,7 @@ suspend fun TestScope.golden(
     project: String,
     vararg args: String,
 ) {
-    fixture(project, *args) { env ->
+    fixture(project, *args) { dir, env ->
         val filename = "${testCase.name.testName}.json"
         val goldenFile = File("../fixtures/golden/$filename")
         if (updateGolden) {
@@ -111,14 +138,83 @@ suspend fun TestScope.golden(
             }
             json.encodeToString(env) should beEqual(goldenData)
         }
+
+        val metadata = parseVerificationMetadata(
+            testLogger,
+            dir.resolve("gradle/verification-metadata.xml")
+        )!!
+
+        for (component in metadata.components) {
+            val componentId = component.id.id
+
+            withClue("env should contain component $componentId") {
+                env[componentId].shouldNotBeNull()
+            }
+        }
     }
 }
 
-val m2 = System.getProperty("org.nixos.gradle2nix.m2")
+val m2: String = requireNotNull(System.getProperty("org.nixos.gradle2nix.m2"))
 
-private fun Array<out String>.withM2(): List<String> {
-    val args = toMutableList()
-    if (args.indexOf("--") < 0) args.add("--")
-    args.add("-Dorg.nixos.gradle2nix.m2=$m2")
-    return args
+object MavenRepo : MountableExtension<MavenRepo.Config, NettyApplicationEngine>, AfterSpecListener {
+    class Config {
+        var repository: File = File("../fixtures/repositories/m2")
+        var path: String = ""
+        var port: Int? = null
+        var host: String = DEFAULT_HOST
+    }
+
+    const val DEFAULT_HOST = "0.0.0.0"
+
+    private val coroutineScope = CoroutineScope(Dispatchers.Default)
+    private var server: NettyApplicationEngine? = null
+    private val config = Config()
+
+    init {
+        require(config.repository.exists()) {
+            "test repository doesn't exist: ${config.repository}"
+        }
+        val m2Url = Url(m2)
+        config.path = m2Url.encodedPath
+        config.host = m2Url.host
+        config.port = m2Url.port
+    }
+
+    override fun mount(configure: Config.() -> Unit): NettyApplicationEngine {
+        config.configure()
+        // try 3 times to find a port if random
+        return tryStart(3).also { this.server = it }
+    }
+
+    private fun tryStart(attempts: Int): NettyApplicationEngine {
+        return try {
+            val p = config.port ?: Random.nextInt(10000, 65000)
+            val s = embeddedServer(Netty, port = p, host = config.host) {
+                routing {
+                    staticFiles(
+                        remotePath = config.path,
+                        dir = config.repository,
+                        index = null,
+                    ) {
+                        enableAutoHeadResponse()
+                        contentType { path ->
+                            when (path.extension) {
+                                "pom", "xml" -> ContentType.Text.Xml
+                                "jar" -> ContentType("application", "java-archive")
+                                else -> ContentType.Text.Plain
+                            }
+                        }
+                    }
+                }
+            }
+            coroutineScope.launch { s.start(wait = true) }
+            s
+        } catch (e: Throwable) {
+            if (config.port == null && attempts > 0) tryStart(attempts - 1) else throw e
+        }
+    }
+
+    override suspend fun afterSpec(spec: Spec) {
+        server?.stop()
+    }
 }