Use custom dependency resolution

- Use Apache Ivy to resolve artifact URLs
- Update build model with full artifact IDs
- Generate Maven module metadata to support dynamic version constraints
- Resolve snapshot versions and generate snapshot metadata
- Add test fixtures and rewrite Gradle plugin tests
- Update dependencies

app/src/dist/share/gradle-env.nix

@@ -19,7 +19,7 @@
 #         '';
 #       }
 
-{ stdenv, lib, buildEnv, fetchurl, gradleGen, writeText }:
+{ stdenv, buildEnv, fetchurl, gradleGen, writeText, writeTextDir }:
 
 { envSpec
 , pname ? null
@@ -27,9 +27,18 @@
 , enableParallelBuilding ? true
 , gradleFlags ? [ "build" ]
 , gradlePackage ? null
+, enableDebug ? false
 , ... } @ args:
 
 let
+  inherit (builtins)
+    filter sort replaceStrings attrValues match fromJSON
+    concatStringsSep;
+
+  inherit (stdenv.lib)
+    versionOlder unique mapAttrs last concatMapStringsSep removeSuffix
+    optionalString groupBy' readFile hasSuffix;
+
   mkDep = depSpec: stdenv.mkDerivation {
     inherit (depSpec) name;
 
@@ -41,40 +50,171 @@ let
 
     installPhase = ''
       mkdir -p $out/${depSpec.path}
-      ln -s $src $out/${depSpec.path}/${depSpec.filename}
+      ln -s $src $out/${depSpec.path}/${depSpec.name}
     '';
   };
 
+  mkModuleMetadata = deps:
+    let
+      ids = filter
+        (id: id.type == "pom")
+        (map (dep: dep.id) deps);
+
+      modules = groupBy'
+        (meta: id:
+          let
+            isNewer = versionOlder meta.latest id.version;
+            isNewerRelease =
+              !(hasSuffix "-SNAPSHOT" id.version) &&
+              versionOlder meta.release id.version;
+          in {
+            groupId = id.group;
+            artifactId = id.name;
+            latest = if isNewer then id.version else meta.latest;
+            release = if isNewerRelease then id.version else meta.release;
+            versions = meta.versions ++ [id.version];
+          }
+        )
+        {
+          latest = "";
+          release = "";
+          versions = [];
+        }
+        (id: "${replaceStrings ["."] ["/"] id.group}/${id.name}/maven-metadata.xml")
+        ids;
+
+    in
+      attrValues (mapAttrs (path: meta:
+        let
+          versions' = sort versionOlder (unique meta.versions);
+        in
+          with meta; writeTextDir path ''
+            <?xml version="1.0" encoding="UTF-8"?>
+            <metadata modelVersion="1.1">
+              <groupId>${groupId}</groupId>
+              <artifactId>${artifactId}</artifactId>
+              <versioning>
+                ${optionalString (latest != "") "<latest>${latest}</latest>"}
+                ${optionalString (release != "") "<release>${release}</release>"}
+                <versions>
+                  ${concatMapStringsSep "\n    " (v: "<version>${v}</version>") versions'}
+                </versions>
+              </versioning>
+            </metadata>
+          ''
+      ) modules);
+
+  mkSnapshotMetadata = deps:
+    let
+      snapshotDeps = filter (dep: dep ? build && dep ? timestamp) deps;
+
+      modules = groupBy'
+        (meta: dep:
+          let
+            id = dep.id;
+            isNewer = dep.build > meta.buildNumber;
+            # Timestamp values can be bogus, e.g. jitpack.io
+            updated = if (match "[0-9]{8}\.[0-9]{6}" dep.timestamp) != null
+                      then replaceStrings ["."] [""] dep.timestamp
+                      else "";
+          in {
+            groupId = id.group;
+            artifactId = id.name;
+            version = id.version;
+            timestamp = if isNewer then dep.timestamp else meta.timestamp;
+            buildNumber = if isNewer then dep.build else meta.buildNumber;
+            lastUpdated = if isNewer then updated else meta.lastUpdated;
+            versions = meta.versions or [] ++ [{
+              classifier = id.classifier or "";
+              extension = id.extension;
+              value = "${removeSuffix "-SNAPSHOT" id.version}-${dep.timestamp}-${toString dep.build}";
+              updated = updated;
+            }];
+          }
+        )
+        {
+          timestamp = "";
+          buildNumber = -1;
+          lastUpdated = "";
+        }
+        (dep: "${replaceStrings ["."] ["/"] dep.id.group}/${dep.id.name}/${dep.id.version}/maven-metadata.xml")
+        snapshotDeps;
+
+      mkSnapshotVersion = version: ''
+        <snapshotVersion>
+          ${optionalString (version.classifier != "") "<classifier>${version.classifier}</classifier>"}
+          <extension>${version.extension}</extension>
+          <value>${version.value}</value>
+          ${optionalString (version.updated != "") "<updated>${version.updated}</updated>"}
+        </snapshotVersion>
+      '';
+
+    in
+      attrValues (mapAttrs (path: meta:
+        with meta; writeTextDir path ''
+          <?xml version="1.0" encoding="UTF-8"?>
+          <metadata modelVersion="1.1">
+            <groupId>${groupId}</groupId>
+            <artifactId>${artifactId}</artifactId>
+            <version>${version}</version>
+            <versioning>
+              <snapshot>
+                ${optionalString (timestamp != "") "<timestamp>${timestamp}</timestamp>"}
+                ${optionalString (buildNumber != -1) "<buildNumber>${toString buildNumber}</buildNumber>"}
+              </snapshot>
+              ${optionalString (lastUpdated != "") "<lastUpdated>${lastUpdated}</lastUpdated>"}
+              <snapshotVersions>
+                ${concatMapStringsSep "\n    " mkSnapshotVersion versions}
+              </snapshotVersions>
+            </versioning>
+          </metadata>
+        ''
+      ) modules);
+
   mkRepo = project: type: deps: buildEnv {
     name = "${project}-gradle-${type}-env";
-    paths = map mkDep deps;
+    paths = map mkDep deps ++ mkModuleMetadata deps ++ mkSnapshotMetadata deps;
   };
 
   mkInitScript = projectSpec:
     let
-      repos = builtins.mapAttrs (mkRepo projectSpec.name) projectSpec.dependencies;
+      repos = mapAttrs (mkRepo projectSpec.name) projectSpec.dependencies;
     in
       writeText "init.gradle" ''
+        static def offlineRepo(RepositoryHandler repositories, String env, String path) {
+            repositories.clear()
+            repositories.maven {
+                name "Nix''${env.capitalize()}MavenOffline"
+                url path
+                metadataSources {
+                    it.gradleMetadata()
+                    it.mavenPom()
+                    it.artifact()
+                }
+            }
+            repositories.ivy {
+                name "Nix''${env.capitalize()}IvyOffline"
+                url path
+                layout "maven"
+                metadataSources {
+                    it.gradleMetadata()
+                    it.ivyDescriptor()
+                    it.artifact()
+                }
+            }
+        }
+
         gradle.settingsEvaluated {
-          it.pluginManagement.repositories {
-            clear()
-            maven { url = uri("${repos.plugin}") }
-          }
+            offlineRepo(it.pluginManagement.repositories, "plugin", "${repos.plugin}")
         }
 
         gradle.projectsLoaded {
-          allprojects {
-            buildscript {
-              repositories {
-                clear()
-                maven { url = uri("${repos.buildscript}") }
-              }
+            allprojects {
+                buildscript {
+                    offlineRepo(repositories, "buildscript", "${repos.buildscript}")
+                }
+                offlineRepo(repositories, "project", "${repos.project}")
             }
-            repositories {
-              clear()
-              maven { url = uri("${repos.project}") }
-            }
-          }
         }
       '';
 
@@ -95,9 +235,9 @@ let
     gradle = args.gradlePackage or mkGradle projectSpec.gradle;
   };
 
-  gradleEnv = builtins.mapAttrs
+  gradleEnv = mapAttrs
     (_: p: mkProjectEnv p)
-    (builtins.fromJSON (builtins.readFile envSpec));
+    (fromJSON (readFile envSpec));
 
   projectEnv = gradleEnv."";
   pname = args.pname or projectEnv.name;
@@ -118,9 +258,10 @@ in stdenv.mkDerivation (args // {
       "GRADLE_USER_HOME=$(mktemp -d)" \
       gradle --offline --no-daemon --no-build-cache \
         --info --full-stacktrace --warning-mode=all \
-        ${lib.optionalString enableParallelBuilding "--parallel"} \
+        ${optionalString enableParallelBuilding "--parallel"} \
+        ${optionalString enableDebug "-Dorg.gradle.debug=true"} \
         --init-script ${projectEnv.initScript} \
-        ${builtins.concatStringsSep " " gradleFlags}
+        ${concatStringsSep " " gradleFlags}
     )
 
     runHook postBuild

app/src/main/kotlin/org/nixos/gradle2nix/Env.kt

@@ -8,16 +8,7 @@ data class NixGradleEnv(
     val version: String,
     val path: String,
     val gradle: DefaultGradle,
-    val dependencies: Map<String, List<Dependency>>
-)
-
-@JsonClass(generateAdapter = true)
-data class Dependency(
-    val name: String,
-    val filename: String,
-    val path: String,
-    val urls: List<String>,
-    val sha256: String
+    val dependencies: Map<String, List<DefaultArtifact>>
 )
 
 fun buildEnv(builds: Map<String, DefaultBuild>): Map<String, NixGradleEnv> =
@@ -28,58 +19,31 @@ fun buildEnv(builds: Map<String, DefaultBuild>): Map<String, NixGradleEnv> =
             path = path,
             gradle = build.gradle,
             dependencies = mapOf(
-                "plugin" to buildRepo(build.pluginDependencies).values.toList(),
-                "buildscript" to build.rootProject.collectDependencies(DefaultProject::buildscriptDependencies)
-                    .values.toList(),
+                "plugin" to build.pluginDependencies,
+                "buildscript" to build.rootProject.collectDependencies(DefaultProject::buildscriptDependencies),
                 "project" to build.rootProject.collectDependencies(DefaultProject::projectDependencies)
-                    .values.toList()
             )
         )
     }
 
-private fun DefaultProject.collectDependencies(chooser: DefaultProject.() -> DefaultDependencies): Map<DefaultArtifact, Dependency> {
-    val result = mutableMapOf<DefaultArtifact, Dependency>()
-    mergeRepo(result, buildRepo(chooser()))
+private fun DefaultProject.collectDependencies(
+    chooser: DefaultProject.() -> List<DefaultArtifact>
+): List<DefaultArtifact> {
+    val result = mutableMapOf<ArtifactIdentifier, DefaultArtifact>()
+    mergeRepo(result, chooser())
     for (child in children) {
         mergeRepo(result, child.collectDependencies(chooser))
     }
-    return result
+    return result.values.toList()
 }
 
-private fun buildRepo(deps: DefaultDependencies): Map<DefaultArtifact, Dependency> =
-    deps.artifacts.associate { artifact ->
-        val name = with(artifact) {
-            buildString {
-                append("$groupId-$artifactId-$version")
-                if (classifier.isNotEmpty()) append("-$classifier")
-                append("-$extension")
-                replace(Regex("[^A-Za-z0-9+\\-._?=]"), "_")
-            }
-        }
-        val filename = with(artifact) {
-            buildString {
-                append("$artifactId-$version")
-                if (classifier.isNotEmpty()) append("-$classifier")
-                append(".$extension")
-            }
-        }
-        val path = with(artifact) { "${groupId.replace(".", "/")}/$artifactId/$version" }
-        val dep = Dependency(
-            name = name,
-            filename = filename,
-            path = path,
-            urls = deps.repositories.maven.flatMap { repo ->
-                repo.urls.map { "${it.removeSuffix("/")}/$path/$filename"  }
-            },
-            sha256 = artifact.sha256
-        )
-        artifact to dep
-    }
-
-private fun mergeRepo(base: MutableMap<DefaultArtifact, Dependency>, extra: Map<DefaultArtifact, Dependency>) {
-    extra.forEach { (artifact, dep) ->
-        base.merge(artifact, dep) { old, new ->
-            old.copy(urls = old.urls + (new.urls - old.urls))
+private fun mergeRepo(
+    base: MutableMap<ArtifactIdentifier, DefaultArtifact>,
+    extra: List<DefaultArtifact>
+) {
+    extra.forEach { artifact ->
+        base.merge(artifact.id, artifact) { old, new ->
+            old.copy(urls = old.urls.union(new.urls).toList())
         }
     }
 }

app/src/main/kotlin/org/nixos/gradle2nix/GradleRunner.kt

@@ -21,8 +21,10 @@ fun ProjectConnection.getBuildModel(config: Config, path: String): DefaultBuild
             "-Porg.nixos.gradle2nix.configurations=${config.configurations.joinToString(",")}",
             "-Porg.nixos.gradle2nix.subprojects=${config.subprojects.joinToString(",")}"
         )
+        if (config.gradleArgs != null) addArguments(config.gradleArgs)
         if (path.isNotEmpty()) addArguments("--project-dir=$path")
         if (!config.quiet) {
+            setColorOutput(true)
             setStandardOutput(System.err)
             setStandardError(System.err)
         }

app/src/main/kotlin/org/nixos/gradle2nix/Logger.kt

@@ -4,7 +4,8 @@ import java.io.PrintStream
 
 class Logger(
     val out: PrintStream = System.err,
-    val verbose: Boolean) {
+    val verbose: Boolean
+) {
 
     val log: (String) -> Unit = { if (verbose) out.println(it) }
     val warn: (String) -> Unit = { out.println("Warning: $it")}

app/src/main/kotlin/org/nixos/gradle2nix/Main.kt

@@ -8,7 +8,12 @@ import com.github.ajalt.clikt.parameters.arguments.ProcessedArgument
 import com.github.ajalt.clikt.parameters.arguments.argument
 import com.github.ajalt.clikt.parameters.arguments.convert
 import com.github.ajalt.clikt.parameters.arguments.default
-import com.github.ajalt.clikt.parameters.options.*
+import com.github.ajalt.clikt.parameters.options.default
+import com.github.ajalt.clikt.parameters.options.flag
+import com.github.ajalt.clikt.parameters.options.multiple
+import com.github.ajalt.clikt.parameters.options.option
+import com.github.ajalt.clikt.parameters.options.split
+import com.github.ajalt.clikt.parameters.options.validate
 import com.github.ajalt.clikt.parameters.types.file
 import com.squareup.moshi.Moshi
 import com.squareup.moshi.Types
@@ -20,6 +25,7 @@ val shareDir: String = System.getProperty("org.nixos.gradle2nix.share")
 
 data class Config(
     val gradleVersion: String?,
+    val gradleArgs: String?,
     val configurations: List<String>,
     val projectDir: File,
     val includes: List<File>,
@@ -37,6 +43,10 @@ class Main : CliktCommand(
         metavar = "VERSION",
         help = "Use a specific Gradle version")
 
+    private val gradleArgs: String? by option("--gradle-args", "-a",
+        metavar = "ARGS",
+        help = "Extra arguments to pass to Gradle")
+
     private val configurations: List<String> by option("--configuration", "-c",
         metavar = "NAME",
         help = "Add a configuration to resolve (default: all configurations)")
@@ -96,7 +106,16 @@ class Main : CliktCommand(
     }
 
     override fun run() {
-        val config = Config(gradleVersion, configurations, projectDir, includes, subprojects, buildSrc, quiet)
+        val config = Config(
+            gradleVersion,
+            gradleArgs,
+            configurations,
+            projectDir,
+            includes,
+            subprojects,
+            buildSrc,
+            quiet
+        )
         val (log, _, _) = Logger(verbose = !config.quiet)
 
         val paths = resolveProjects(config).map { p ->