plugin: Support S3 repositories

plugin/src/compatTest/kotlin/org/nixos/gradle2nix/S3Test.kt

@@ -0,0 +1,52 @@
+package org.nixos.gradle2nix
+
+import dev.minutest.Tests
+import dev.minutest.junit.JUnit5Minutests
+import dev.minutest.rootContext
+import dev.minutest.test
+import strikt.api.expectThat
+import strikt.assertions.containsExactly
+import strikt.assertions.flatMap
+import strikt.assertions.map
+
+class S3Test : JUnit5Minutests {
+    @Tests
+    fun tests() = rootContext("s3 tests") {
+        withBucket("repositories") {
+            withFixture("s3/maven") {
+
+                test("dependency from s3 maven repo") {
+                    expectThat(build()) {
+                        get("root project dependencies") { rootProject.projectDependencies }.and {
+                            ids.containsExactly(
+                                "org.apache:test:1.0.0@jar",
+                                "org.apache:test:1.0.0@pom"
+                            )
+                            flatMap { it.urls }.containsExactly(
+                                "s3://repositories/m2/org/apache/test/1.0.0/test-1.0.0.jar",
+                                "s3://repositories/m2/org/apache/test/1.0.0/test-1.0.0.pom"
+                            )
+                        }
+                    }
+                }
+            }
+
+            withFixture("s3/maven-snapshot") {
+                test("snapshot dependency from s3 maven repo") {
+                    expectThat(build()) {
+                        get("root project dependencies") { rootProject.projectDependencies }.and {
+                            ids.containsExactly(
+                                "org.apache:test-SNAPSHOT1:2.0.0-SNAPSHOT@jar",
+                                "org.apache:test-SNAPSHOT1:2.0.0-SNAPSHOT@pom"
+                            )
+                            flatMap { it.urls }.containsExactly(
+                                "s3://repositories/m2/org/apache/test-SNAPSHOT1/2.0.0-SNAPSHOT/test-SNAPSHOT1-2.0.0-20070310.181613-3.jar",
+                                "s3://repositories/m2/org/apache/test-SNAPSHOT1/2.0.0-SNAPSHOT/test-SNAPSHOT1-2.0.0-20070310.181613-3.pom"
+                            )
+                        }
+                    }
+                }
+            }
+        }
+    }
+}

plugin/src/compatTest/kotlin/org/nixos/gradle2nix/TestUtil.kt

@@ -1,5 +1,8 @@
 package org.nixos.gradle2nix
 
+import com.adobe.testing.s3mock.S3MockApplication
+import com.adobe.testing.s3mock.junit5.S3MockExtension
+import com.adobe.testing.s3mock.testsupport.common.S3MockStarter
 import com.squareup.moshi.Moshi
 import dev.minutest.ContextBuilder
 import dev.minutest.MinutestFixture
@@ -13,6 +16,7 @@ import dev.minutest.given
 import dev.minutest.givenClosable
 import dev.minutest.given_
 import io.javalin.Javalin
+import io.javalin.http.staticfiles.Location
 import okio.buffer
 import okio.source
 import org.gradle.internal.classpath.DefaultClassPath
@@ -25,10 +29,12 @@ import strikt.assertions.map
 import java.io.Closeable
 import java.io.File
 import java.io.StringWriter
-import java.nio.file.Paths
+import java.util.concurrent.atomic.AtomicBoolean
 
 private val moshi = Moshi.Builder().build()
 
+val fixtureRoot = File(System.getProperty("fixtures"))
+
 val gradleVersion = System.getProperty("compat.gradle.version")
     ?.let(GradleVersion::version)
     ?: GradleVersion.current()
@@ -69,7 +75,8 @@ fun File.buildKotlin(
 
 private fun File.build(
     configurations: List<String>,
-    subprojects: List<String>
+    subprojects: List<String>,
+    extraArguments: List<String> = emptyList()
 ): DefaultBuild {
     val log = StringWriter()
 
@@ -83,7 +90,8 @@ private fun File.build(
             "--init-script=${initscript()}",
             "--stacktrace",
             "-Porg.nixos.gradle2nix.configurations=${configurations.joinToString(",")}",
-            "-Porg.nixos.gradle2nix.subprojects=${subprojects.joinToString(",")}"
+            "-Porg.nixos.gradle2nix.subprojects=${subprojects.joinToString(",")}",
+            *(extraArguments.toTypedArray())
         )
         .runCatching { build() }
 
@@ -107,15 +115,64 @@ val <T : Iterable<Artifact>> Assertion.Builder<T>.ids: Assertion.Builder<Iterabl
 
 private fun File.parents() = generateSequence(parentFile) { it.parentFile }
 
+abstract class ArgumentsSupplier(private val parent: ArgumentsSupplier? = null) {
+    open val arguments: List<String> = emptyList()
+
+    val extraArguments: List<String> get() = (parent?.extraArguments ?: emptyList()) + arguments
+}
+
 @MinutestFixture
-class RepositoryFixture(private val server: Javalin) : Closeable {
+class RepositoryFixture(
+    private val server: Javalin,
+    parent: ArgumentsSupplier? = null
+) : ArgumentsSupplier(parent), Closeable {
     override fun close() {
         server.stop()
     }
 }
 
 @MinutestFixture
-class TestFixture(val name: String, val source: File) : Closeable {
+class S3Fixture(
+    private val name: String,
+    parent: ArgumentsSupplier? = null
+) : ArgumentsSupplier(parent), Closeable {
+    private val s3mock = S3Mock(
+        initialBuckets = listOf(name),
+        secureConnection = false
+    )
+
+    override val arguments: List<String> get() = listOf(
+        "-Dorg.gradle.s3.endpoint=${s3mock.serviceEndpoint}",
+        "-Dorg.nixos.gradle2nix.s3test=true"
+    )
+
+    init {
+        s3mock.startServer()
+
+        val s3root = fixtureRoot.resolve(name)
+        val s3client = s3mock.createS3Client()
+        require(s3root.exists() && s3root.isDirectory) {
+            "$name: S3 fixture not found: $s3root"
+        }
+        s3root.walkTopDown()
+            .filter { it.isFile }
+            .forEach { file ->
+                val key = file.toRelativeString(s3root)
+                s3client.putObject(name, key, file)
+            }
+    }
+
+    override fun close() {
+        s3mock.stopServer()
+    }
+}
+
+@MinutestFixture
+class TestFixture(
+    val name: String,
+    val source: File,
+    parent: ArgumentsSupplier? = null
+) : ArgumentsSupplier(parent), Closeable {
     val dest: File
 
     init {
@@ -131,7 +188,10 @@ class TestFixture(val name: String, val source: File) : Closeable {
 }
 
 @MinutestFixture
-class ProjectFixture(private val parent: TestFixture, private val source: File) : Closeable {
+data class ProjectFixture(
+    private val parent: TestFixture,
+    private val source: File
+) : Closeable {
     private val dest: File
 
     init {
@@ -142,30 +202,48 @@ class ProjectFixture(private val parent: TestFixture, private val source: File)
         dest = parent.dest.resolve(rel)
     }
 
-    fun copy() {
+    fun copySource() {
         source.copyRecursively(dest, true)
     }
 
     fun build(
         configurations: List<String> = emptyList(),
         subprojects: List<String> = emptyList()
-    ) = dest.build(configurations, subprojects)
+    ) = dest.build(configurations, subprojects, parent.extraArguments)
 
     override fun close() {
         dest.deleteRecursively()
     }
 }
 
+fun ContextBuilder<*>.withBucket(
+    name: String,
+    block: TestContextBuilder<*, S3Fixture>.() -> Unit
+) = derivedContext<S3Fixture>("with s3 bucket: $name") {
+    given_ { parent ->
+        S3Fixture(name, parent as? ArgumentsSupplier)
+    }
+
+    afterEach { it.close() }
+
+    block()
+}
+
 fun ContextBuilder<*>.withRepository(
     name: String,
     block: TestContextBuilder<*, RepositoryFixture>.() -> Unit
 ) = derivedContext<RepositoryFixture>("with repository: $name") {
-    givenClosable {
-        RepositoryFixture(Javalin.create { config ->
-            config.addStaticFiles("/repositories/$name")
-        }.start(9999))
+    given_ { parent ->
+        RepositoryFixture(
+            server = Javalin.create { config ->
+                config.addStaticFiles("${fixtureRoot}/repositories/$name", Location.EXTERNAL)
+            }.start(9999),
+            parent = parent as? ArgumentsSupplier
+        )
     }
 
+    afterEach { it.close() }
+
     block()
 }
 
@@ -173,24 +251,50 @@ fun ContextBuilder<*>.withFixture(
     name: String,
     block: TestContextBuilder<*, ProjectFixture>.() -> Unit
 ) = derivedContext<TestFixture>(name) {
-    val url = checkNotNull(Thread.currentThread().contextClassLoader.getResource(name)?.toURI()) {
-        "$name: No test fixture found"
+
+    val projectRoot = fixtureRoot.resolve(name).also {
+        check(it.exists()) { "$name: project fixture not found: $it" }
     }
-    val fixtureRoot = Paths.get(url).toFile().absoluteFile
 
-    given { TestFixture(name, fixtureRoot) }
+    given_ { parent ->
+        TestFixture(name, projectRoot, parent as? ArgumentsSupplier)
+    }
 
-    val testRoots = fixtureRoot.listFiles()!!
+    val testRoots = projectRoot.listFiles()!!
         .filter { it.isDirectory }
         .map { it.absoluteFile }
         .toList()
 
     testRoots.forEach { testRoot ->
         derivedContext<ProjectFixture>(testRoot.name) {
-            given_ { ProjectFixture(it, testRoot) }
-            beforeEach { copy() }
+            given_ { parent -> ProjectFixture(parent, testRoot) }
+            beforeEach { copySource() }
             afterEach { close() }
             block()
         }
     }
 }
+
+class S3Mock(
+    initialBuckets: List<String> = emptyList(),
+    secureConnection: Boolean = true
+) : S3MockStarter(
+    mapOf(
+        S3MockApplication.PROP_INITIAL_BUCKETS to initialBuckets.joinToString(","),
+        S3MockApplication.PROP_SECURE_CONNECTION to secureConnection
+    )
+) {
+    private val running = AtomicBoolean()
+
+    fun startServer() {
+        if (running.compareAndSet(false, true)) {
+            start()
+        }
+    }
+
+    fun stopServer() {
+        if (running.compareAndSet(true, false)) {
+            stop()
+        }
+    }
+}

plugin/src/compatTest/kotlin/org/nixos/gradle2nix/WrapperTest.kt

@@ -1,6 +1,7 @@
 package org.nixos.gradle2nix
 
 import dev.minutest.Tests
+import dev.minutest.given
 import dev.minutest.junit.JUnit5Minutests
 import dev.minutest.rootContext
 import dev.minutest.test
@@ -11,7 +12,7 @@ import java.io.File
 class WrapperTest : JUnit5Minutests {
     @Tests
     fun tests() = rootContext<File>("wrapper tests") {
-        fixture { createTempDir("gradle2nix") }
+        given { createTempDir("gradle2nix") }
 
         test("resolves gradle wrapper version") {
             expectThat(buildKotlin("""

plugin/src/main/kotlin/org/nixos/gradle2nix/ConfigurationResolver.kt

@@ -62,7 +62,7 @@ internal class ConfigurationResolver(
     private val resolvers: List<RepositoryResolver>,
     private val dependencies: DependencyHandler
 ) {
-    private val failed = mutableListOf<ArtifactIdentifier>()
+    private val failed = mutableSetOf<ArtifactIdentifier>()
     private val ivy = Ivy.newInstance(ivySettings)
 
     val unresolved: List<ArtifactIdentifier> = failed.toList()

plugin/src/main/kotlin/org/nixos/gradle2nix/RepositoryResolver.kt

@@ -1,5 +1,7 @@
 package org.nixos.gradle2nix
 
+import com.amazonaws.auth.BasicAWSCredentials
+import com.amazonaws.auth.BasicSessionCredentials
 import org.apache.ivy.core.LogOptions
 import org.apache.ivy.core.cache.ArtifactOrigin
 import org.apache.ivy.core.cache.CacheResourceOptions
@@ -11,6 +13,7 @@ import org.apache.ivy.core.resolve.DownloadOptions
 import org.apache.ivy.core.settings.IvySettings
 import org.apache.ivy.core.settings.TimeoutConstraint
 import org.apache.ivy.plugins.repository.Repository
+import org.apache.ivy.plugins.repository.Resource
 import org.apache.ivy.plugins.repository.url.URLRepository
 import org.apache.ivy.plugins.repository.url.URLResource
 import org.apache.ivy.plugins.resolver.AbstractResolver
@@ -33,6 +36,7 @@ import org.gradle.api.logging.Logger
 import org.gradle.api.logging.Logging
 import org.gradle.authentication.aws.AwsImAuthentication
 import org.gradle.internal.authentication.AllSchemesAuthentication
+import org.gradle.kotlin.dsl.getCredentials
 import java.io.IOException
 import java.net.URI
 import org.apache.ivy.core.module.descriptor.Artifact as IvyArtifact
@@ -82,8 +86,18 @@ internal class MavenResolver(
 
     override fun resolve(artifactId: DefaultArtifactIdentifier, sha256: String?): DefaultArtifact? {
         val ivyArtifact: IvyArtifact = artifactId.toArtifact()
-        val origin = ivyResolver.locate(ivyArtifact)?.takeIf(ArtifactOrigin::isExists) ?: return null
-        val hash = sha256 ?: ivyResolver.download(origin, downloadOptions).localFile?.sha256() ?: return null
+        val origin = ivyResolver.locate(ivyArtifact)
+        if (origin == null || !origin.isExists) return null
+        val hash = if (sha256 != null) sha256 else {
+            val report = ivyResolver.download(origin, downloadOptions)
+            report.localFile?.sha256().also {
+                if (it == null) log.error(report.toString())
+            }
+        }
+        if (hash == null) {
+            log.error("Failed to download '$artifactId' from repository '${ivyResolver.repository.name}'")
+            return null
+        }
         val snapshotVersion: SnapshotVersion? = artifactId.version.snapshotVersion()?.let {
             findSnapshotVersion(artifactId, it)
         }
@@ -103,9 +117,9 @@ internal class MavenResolver(
         snapshotVersion: SnapshotVersion
     ): SnapshotVersion {
         if (snapshotVersion.timestamp != null) return snapshotVersion
-        val metadataLocation = "${ivyResolver.root}${artifactId.repoPath()}/maven-metadata.xml".toUrl()
+        val metadataLocation = "${ivyResolver.root}${artifactId.repoPath()}/maven-metadata.xml"
         val metadataFile = ivyResolver.repositoryCacheManager.downloadRepositoryResource(
-            URLResource(metadataLocation, ivyResolver.timeoutConstraint),
+            ivyResolver.repository.getResource(metadataLocation),
             "maven-metadata",
             "maven-metadata",
             "xml",
@@ -160,7 +174,16 @@ internal class IvyResolver(
     override fun resolve(artifactId: DefaultArtifactIdentifier, sha256: String?): DefaultArtifact? {
         val ivyArtifact: IvyArtifact = artifactId.toArtifact()
         val origin = ivyResolver.locate(ivyArtifact)?.takeIf(ArtifactOrigin::isExists) ?: return null
-        val hash = sha256 ?: ivyResolver.download(origin, downloadOptions).localFile?.sha256() ?: return null
+        val hash = if (sha256 != null) sha256 else {
+            val report = ivyResolver.download(origin, downloadOptions)
+            report.localFile?.sha256().also {
+                if (it == null) log.error(report.toString())
+            }
+        }
+        if (hash == null) {
+            log.error("Failed to download '$artifactId' from repository '${ivyResolver.repository.name}'")
+            return null
+        }
         return DefaultArtifact(
             id = DefaultArtifactIdentifier(artifactId),
             name = artifactId.filename(null),
@@ -180,7 +203,9 @@ private fun cacheManager(
     return DefaultRepositoryCacheManager(
         "${scope.name.toLowerCase()}-${repository.name}-cache",
         ivySettings,
-        project.buildDir.resolve("tmp/gradle2nix/${scope.name.toLowerCase()}/${repository.name}")
+        project.buildDir.resolve("tmp/gradle2nix/${repository.name}").also {
+            it.mkdirs()
+        }
     ).also {
         ivySettings.addRepositoryCacheManager(it)
     }
@@ -236,31 +261,42 @@ private fun ArtifactIdentifier.filename(
     append(".", extension)
 }
 
-private val downloadOptions = DownloadOptions().apply { log = LogOptions.LOG_QUIET }
+private val downloadOptions = DownloadOptions().apply {
+    log = LogOptions.LOG_DEFAULT
+}
 
 private fun <T> AbstractResolver.resolverRepository(
     repository: T
 ) : Repository
-where T : UrlArtifactRepository,
+where T : ArtifactRepository,
       T : AuthenticationSupported =
     when (val scheme = repository.url.scheme) {
-        "s3" -> s3Repository(repository.authentication, LazyTimeoutConstraint(this))
+        "s3" -> s3Repository(
+            repository.getCredentials(AwsCredentials::class),
+            LazyTimeoutConstraint(this)
+        )
         "http", "https" -> URLRepository(LazyTimeoutConstraint(this))
         else -> throw IllegalStateException("Unknown repository URL scheme: $scheme")
     }
 
 private fun s3Repository(
-    authContainer: AuthenticationContainer,
+    credentials: AwsCredentials?,
     timeoutConstraint: TimeoutConstraint
 ): Repository {
-    val auth = authContainer.firstOrNull { auth ->
-        auth is AllSchemesAuthentication || auth is AwsImAuthentication
+    val awsCredentials = credentials?.let {
+        if (it.sessionToken == null) {
+            BasicAWSCredentials(it.accessKey, it.secretKey)
+        } else {
+            BasicSessionCredentials(it.accessKey, it.secretKey, it.sessionToken)
+        }
     }
-    checkNotNull(auth) { "S3 resource should either specify AwsImAuthentication or provide some AwsCredentials." }
     return S3Repository(
-        credentials = (auth as? AllSchemesAuthentication)?.credentials as? AwsCredentials,
-        endpoint = System.getProperty("org.gradle.s3.endpoint")?.let { URI(it) }
-    )
+        credentials = awsCredentials,
+        endpoint = System.getProperty("org.gradle.s3.endpoint")?.let { URI(it) },
+        timeoutConstraint = timeoutConstraint
+    ).apply {
+        name = "AWS S3"
+    }
 }
 
 private class LazyTimeoutConstraint(
@@ -272,3 +308,10 @@ private class LazyTimeoutConstraint(
     override fun getReadTimeout(): Int =
         resolver.timeoutConstraint?.readTimeout ?: -1
 }
+
+// Compatibility shim as UrlArtifactRepository was added in Gradle 6.0
+private val ArtifactRepository.url: URI get() = when (this) {
+    is MavenArtifactRepository -> url
+    is IvyArtifactRepository -> url
+    else -> throw IllegalStateException("Unhandled repository type: ${this::class.simpleName}")
+}

plugin/src/main/kotlin/org/nixos/gradle2nix/Util.kt

@@ -1,6 +1,7 @@
 package org.nixos.gradle2nix
 
 import java.io.File
+import java.net.URI
 import java.net.URL
 import java.security.MessageDigest
 
@@ -19,3 +20,5 @@ private fun ByteArray.sha256() = buildString {
 }
 
 internal fun String.toUrl(): URL = URL(this)
+
+internal fun String.toUri(): URI = URI(this)