sbl/gradle2nix
1
0
Fork 0
mirror of https://github.com/tadfisher/gradle2nix.git synced 2026-01-11 23:40:37 -05:00
Code Issues Packages Projects Releases Wiki Activity

Vastly simplify artifact extraction

Browse Source
This commit is contained in:
Tad Fisher
2024-05-23 17:26:32 -07:00
parent 43c4b71928
commit 4910251482
48 changed files with 1248 additions and 9576 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
app/src/dist/share/gradle.nix vendored
View File

@@ -119,14 +119,12 @@ let
# Fetch urls using the scheme for the first entry only; there isn't a
# straightforward way to tell Nix to try multiple fetchers in turn
# and short-circuit on the first successful fetch.
fetch = name: { urls, hash }:
fetch = name: { url, hash }:
let
first = head urls;
scheme = head (builtins.match "([a-z0-9+.-]+)://.*" first);
scheme = head (builtins.match "([a-z0-9+.-]+)://.*" url);
fetch' = getAttr scheme fetchers';
urls' = filter (hasPrefix scheme) urls;
in
fetch' { urls = urls'; inherit hash; };
fetch' { inherit url hash; };
mkModule = id: artifacts:
let

11
app/src/main/kotlin/org/nixos/gradle2nix/Env.kt Normal file
View File

@@ -0,0 +1,11 @@
package org.nixos.gradle2nix
import kotlinx.serialization.Serializable
typealias Env = Map<String, Map<String, Artifact>>
@Serializable
data class Artifact internal constructor(
val url: String,
val hash: String,
)

3
app/src/main/kotlin/org/nixos/gradle2nix/GradleRunner.kt
View File

@@ -62,13 +62,12 @@ suspend fun ProjectConnection.build(config: Config): DependencySet = suspendCanc
"--refresh-dependencies",
"--gradle-user-home=${config.gradleHome}",
"--init-script=${config.appHome}/init.gradle",
"--write-verification-metadata", "sha256"
)
.apply {
if (config.logger.stacktrace) {
addArguments("--stacktrace")
}
if (config.logger.logLevel <= LogLevel.debug) {
if (config.logger.logLevel < LogLevel.error) {
setStandardOutput(System.err)
setStandardError(System.err)
}

2
app/src/main/kotlin/org/nixos/gradle2nix/Main.kt
View File

@@ -76,7 +76,7 @@ class Gradle2Nix : CliktCommand(
.file(canBeFile = false, canBeDir = true)
.defaultLazy("<project>") { projectDir }
private val lockFile: String by option(
internal val lockFile: String by option(
"--lock-file", "-l",
metavar = "FILENAME",
help = "Name of the generated lock file"

110
app/src/main/kotlin/org/nixos/gradle2nix/Process.kt
View File

@@ -1,24 +1,6 @@
package org.nixos.gradle2nix
import org.nixos.gradle2nix.metadata.Artifact as ArtifactMetadata
import java.io.File
import java.io.IOException
import java.net.URI
import okio.ByteString.Companion.decodeHex
import okio.HashingSource
import okio.blackholeSink
import okio.buffer
import okio.source
import org.nixos.gradle2nix.env.Artifact
import org.nixos.gradle2nix.env.Env
import org.nixos.gradle2nix.metadata.Checksum
import org.nixos.gradle2nix.metadata.Component
import org.nixos.gradle2nix.metadata.Md5
import org.nixos.gradle2nix.metadata.Sha1
import org.nixos.gradle2nix.metadata.Sha256
import org.nixos.gradle2nix.metadata.Sha512
import org.nixos.gradle2nix.metadata.VerificationMetadata
import org.nixos.gradle2nix.metadata.parseVerificationMetadata
import org.nixos.gradle2nix.model.DependencyCoordinates
import org.nixos.gradle2nix.model.DependencySet
@@ -26,12 +8,9 @@ fun processDependencies(
config: Config,
dependencySets: Iterable<DependencySet>
): Env {
val verificationMetadata = readVerificationMetadata(config)
val verificationComponents = verificationMetadata?.components?.associateBy { it.id.id } ?: emptyMap()
return buildMap<DependencyCoordinates, Map<String, Artifact>> {
for (dependencySet in dependencySets) {
val env = dependencySet.toEnv(config, verificationComponents)
val env = dependencySet.toEnv()
for ((id, artifacts) in env) {
merge(id, artifacts) { a, b ->
@@ -47,10 +26,7 @@ fun processDependencies(
""".trimIndent())
}
Artifact(
(aa.urls + ba.urls).distinct().sorted(),
aa.hash
)
aa
}
}
}
@@ -63,89 +39,19 @@ fun processDependencies(
.mapKeys { (coordinates, _) -> coordinates.id }
}
private fun DependencySet.toEnv(config: Config, verificationComponents: Map<String, Component>): Map<DependencyCoordinates, Map<String, Artifact>> {
private fun DependencySet.toEnv(): Map<DependencyCoordinates, Map<String, Artifact>> {
return dependencies.associate { dep ->
val component = verificationComponents[dep.coordinates.id]
?: verifyComponentFilesInCache(config, dep.coordinates)
?: config.logger.error("${dep.coordinates}: no dependency metadata found")
dep.coordinates to dep.artifacts.mapNotNull { resolvedArtifact ->
val artifact = component.artifacts.find { it.name == resolvedArtifact.name }
?.let { Artifact(resolvedArtifact.urls.sorted(), it.checksums.first().toSri()) }
?: downloadArtifact(resolvedArtifact.urls.sorted())
artifact?.let { resolvedArtifact.filename to it }
}.sortedBy { it.first }.toMap()
}
}
private fun readVerificationMetadata(config: Config): VerificationMetadata? {
return parseVerificationMetadata(config.logger, config.projectDir.resolve("gradle/verification-metadata.xml"))
}
private fun verifyComponentFilesInCache(
config: Config,
id: DependencyCoordinates,
): Component? {
val cacheDir = with(id) { config.gradleHome.resolve("caches/modules-2/files-2.1/$group/$artifact/$version") }
if (!cacheDir.exists()) {
return null
}
val verifications = cacheDir.walk().filter { it.isFile }.map { f ->
ArtifactMetadata(f.name.replaceFirst(id.version, id.timestampedVersion), sha256 = Sha256(f.sha256()))
}
config.logger.info("${id.id}: obtained artifact hashes from Gradle cache.")
return Component(id, verifications.toList())
}
private fun downloadArtifact(
urls: List<String>
): Artifact? {
return maybeDownloadText(urls)?.let {
Artifact(
urls,
it.hash.toSri()
)
}
}
private fun maybeDownloadText(
urls: List<String>,
): ArtifactDownload<String>? {
for (url in urls) {
try {
val source = HashingSource.sha256(URI(url).toURL().openStream().source())
val text = source.buffer().readUtf8()
val hash = source.hash
return ArtifactDownload(text, url, Sha256(hash.hex()))
} catch (e: IOException) {
// Pass
dep.coordinates to dep.artifacts.associate {
it.name to Artifact(it.url, it.hash.toSri())
}
}
return null
}
private fun File.sha256(): String {
val source = HashingSource.sha256(source())
source.buffer().readAll(blackholeSink())
return source.hash.hex()
internal fun String.toSri(): String {
val hash = decodeHex().base64()
return "sha256-$hash"
}
internal fun Checksum.toSri(): String {
val hash = value.decodeHex().base64()
return when (this) {
is Md5 -> "md5-$hash"
is Sha1 -> "sha1-$hash"
is Sha256 -> "sha256-$hash"
is Sha512 -> "sha512-$hash"
}
}
private data class ArtifactDownload<T>(
val artifact: T,
val url: String,
val hash: Checksum
)
private val coordinatesComparator: Comparator<DependencyCoordinates> = compareBy<DependencyCoordinates> { it.group }
.thenBy { it.artifact }
.thenByDescending { Version(it.version) }

22
app/src/main/kotlin/org/nixos/gradle2nix/env/Env.kt vendored
View File

@@ -1,22 +0,0 @@
package org.nixos.gradle2nix.env
import kotlinx.serialization.Serializable
typealias Env = Map<String, Map<String, Artifact>>
@Serializable
data class Artifact internal constructor(
val urls: List<String>,
val hash: String,
) {
companion object {
operator fun invoke(
urls: List<String>,
hash: String
) = Artifact(
urls.sorted(),
hash
)
}
}

102
app/src/main/kotlin/org/nixos/gradle2nix/gradle/GradleModule.kt
View File

@@ -1,102 +0,0 @@
package org.nixos.gradle2nix.gradle
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonObject
@Serializable
data class GradleModule(
val formatVersion: String,
val component: Component? = null,
val createdBy: CreatedBy? = null,
val variants: List<Variant> = emptyList(),
)
@Serializable
data class Component(
val group: String,
val module: String,
val version: String,
val url: String? = null,
)
@Serializable
data class Gradle(
val version: String,
val buildId: String? = null
)
@Serializable
data class CreatedBy(
val gradle: Gradle? = null
)
@Serializable
data class Variant(
val name: String,
val attributes: JsonObject? = null,
@SerialName("available-at") val availableAt: AvailableAt? = null,
val dependencies: List<Dependency> = emptyList(),
val dependencyConstraints: List<DependencyConstraint> = emptyList(),
val files: List<VariantFile> = emptyList(),
val capabilities: List<Capability> = emptyList()
)
@Serializable
data class AvailableAt(
val url: String,
val group: String,
val module: String,
val version: String,
)
@Serializable
data class Dependency(
val group: String,
val module: String,
val version: JsonObject? = null,
val excludes: List<Exclude> = emptyList(),
val reason: String? = null,
val attributes: JsonObject? = null,
val requestedCapabilities: List<Capability> = emptyList(),
val endorseStrictVersions: Boolean = false,
val thirdPartyCompatibility: ThirdPartyCompatibility? = null,
)
@Serializable
data class DependencyConstraint(
val group: String,
val module: String,
val version: JsonObject? = null,
val reason: String? = null,
val attributes: JsonObject? = null,
)
@Serializable
data class VariantFile(
val name: String,
val url: String,
val size: Long,
val sha1: String? = null,
val sha256: String? = null,
val sha512: String? = null,
val md5: String? = null,
)
@Serializable
data class Capability(
val group: String,
val name: String,
val version: String? = null,
)
@Serializable
data class Exclude(
val group: String,
val module: String,
)
@Serializable
data class ThirdPartyCompatibility(
val artifactSelector: String
)

147
app/src/main/kotlin/org/nixos/gradle2nix/metadata/VerificationMetadata.kt
View File

@@ -1,147 +0,0 @@
package org.nixos.gradle2nix.metadata
import java.io.File
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import nl.adaptivity.xmlutil.serialization.XML
import nl.adaptivity.xmlutil.serialization.XmlChildrenName
import nl.adaptivity.xmlutil.serialization.XmlElement
import nl.adaptivity.xmlutil.serialization.XmlSerialName
import nl.adaptivity.xmlutil.xmlStreaming
import org.nixos.gradle2nix.Logger
import org.nixos.gradle2nix.model.DependencyCoordinates
import org.nixos.gradle2nix.model.impl.DefaultDependencyCoordinates
sealed interface Coordinates {
val group: String?
val name: String?
val version: String?
val regex: Boolean
val file: String?
}
@Serializable
@SerialName("trust")
data class Trust(
override val group: String? = null,
override val name: String? = null,
override val version: String? = null,
override val regex: Boolean = false,
override val file: String? = null,
val reason: String? = null,
) : Coordinates
@Serializable
@SerialName("configuration")
data class Configuration(
@SerialName("verify-metadata") @XmlElement(true) val verifyMetadata: Boolean = false,
@SerialName("verify-signatures") @XmlElement(true) val verifySignatures: Boolean = false,
@SerialName("trusted-artifacts") @XmlChildrenName("trusted-artifacts") val trustedArtifacts: List<Trust> = emptyList()
)
@Serializable
sealed interface Checksum {
val value: String
val origin: String?
val reason: String?
val alternatives: List<String>
}
@Serializable
@SerialName("md5")
data class Md5(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("sha1")
data class Sha1(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("sha256")
data class Sha256(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("sha512")
data class Sha512(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("artifact")
data class Artifact(
val name: String,
val md5: Md5? = null,
val sha1: Sha1? = null,
val sha256: Sha256? = null,
val sha512: Sha512? = null,
) {
val checksums: List<Checksum> by lazy { listOfNotNull(sha512, sha256, sha1, md5) }
}
@Serializable
@SerialName("component")
data class Component(
val group: String,
val name: String,
val version: String,
val timestamp: String? = null,
val artifacts: List<Artifact> = emptyList(),
) {
val id: DependencyCoordinates get() = DefaultDependencyCoordinates(group, name, version, timestamp)
constructor(id: DependencyCoordinates, artifacts: List<Artifact>) : this(
id.group,
id.artifact,
id.version,
id.timestamp,
artifacts
)
}
@Serializable
@XmlSerialName(
"verification-metadata",
namespace = "https://schema.gradle.org/dependency-verification",
prefix = ""
)
data class VerificationMetadata(
val configuration: Configuration = Configuration(),
@XmlChildrenName("components", "https://schema.gradle.org/dependency-verification") val components: List<Component> = emptyList()
)
val XmlFormat = XML {
autoPolymorphic = true
recommended()
}
fun parseVerificationMetadata(logger: Logger, metadata: File): VerificationMetadata? {
return try {
metadata.reader().buffered().let(xmlStreaming::newReader).use { input ->
XmlFormat.decodeFromReader(input)
}
} catch (e: Exception) {
logger.warn("$metadata: failed to parse Gradle dependency verification metadata", e)
return null
}
}

25
app/src/test/kotlin/org/nixos/gradle2nix/TestUtil.kt
View File

@@ -1,7 +1,6 @@
package org.nixos.gradle2nix
import io.kotest.assertions.fail
import io.kotest.assertions.withClue
import io.kotest.common.ExperimentalKotest
import io.kotest.common.KotestInternal
import io.kotest.core.extensions.MountableExtension
@@ -14,9 +13,7 @@ import io.kotest.core.test.TestScope
import io.kotest.core.test.TestType
import io.kotest.matchers.equals.beEqual
import io.kotest.matchers.file.shouldBeAFile
import io.kotest.matchers.nulls.shouldNotBeNull
import io.kotest.matchers.should
import io.kotest.matchers.shouldBe
import io.ktor.http.ContentType
import io.ktor.http.Url
import io.ktor.server.engine.embeddedServer
@@ -39,8 +36,6 @@ import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import kotlinx.serialization.json.encodeToStream
import okio.use
import org.nixos.gradle2nix.env.Env
import org.nixos.gradle2nix.metadata.parseVerificationMetadata
private val app = Gradle2Nix()
@@ -93,15 +88,16 @@ suspend fun TestScope.fixture(
}
app.main(
listOf(
"-d", tempDir.toString(),
"-p", tempDir.toString(),
"--log", "debug",
"--stacktrace",
"--dump-events",
"--",
"-Dorg.nixos.gradle2nix.m2=$m2"
"-Dorg.nixos.gradle2nix.m2=$m2",
"--info"
) + args
)
val file = tempDir.resolve("${app.envFile}.json")
val file = tempDir.resolve(app.lockFile)
file.shouldBeAFile()
val env: Env = file.inputStream().buffered().use { input ->
Json.decodeFromStream(input)
@@ -138,19 +134,6 @@ suspend fun TestScope.golden(
}
json.encodeToString(env) should beEqual(goldenData)
}
val metadata = parseVerificationMetadata(
testLogger,
dir.resolve("gradle/verification-metadata.xml")
)!!
for (component in metadata.components) {
val componentId = component.id.id
withClue("env should contain component $componentId") {
env[componentId].shouldNotBeNull()
}
}
}
}

17
app/src/test/kotlin/org/nixos/gradle2nix/VerificationMetadataTest.kt
View File

@@ -1,17 +0,0 @@
package org.nixos.gradle2nix
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.nulls.beNull
import io.kotest.matchers.nulls.shouldNotBeNull
import io.kotest.matchers.shouldNot
import kotlinx.serialization.decodeFromString
import org.nixos.gradle2nix.metadata.Artifact
import org.nixos.gradle2nix.metadata.XmlFormat
import org.nixos.gradle2nix.metadata.parseVerificationMetadata
class VerificationMetadataTest : FunSpec({
test("parses verification metadata") {
val metadata = parseVerificationMetadata(testLogger, fixture("metadata/verification-metadata.xml"))
metadata shouldNot beNull()
}
})