Vastly simplify artifact extraction

This commit is contained in:
Tad Fisher
2024-05-23 17:26:32 -07:00
parent 43c4b71928
commit 4910251482
48 changed files with 1248 additions and 9576 deletions

View File

@@ -10,6 +10,7 @@ dependencies {
shadow(kotlin("stdlib-jdk8"))
shadow(kotlin("reflect"))
implementation(project(":model"))
implementation(libs.serialization.json)
testImplementation(libs.kotest.assertions)
testImplementation(libs.kotest.runner)
}

View File

@@ -11,17 +11,22 @@ import org.gradle.tooling.provider.model.ToolingModelBuilderRegistry
import org.nixos.gradle2nix.dependencygraph.DependencyExtractor
import org.nixos.gradle2nix.forceresolve.ForceDependencyResolutionPlugin
import org.nixos.gradle2nix.model.DependencySet
import org.nixos.gradle2nix.util.buildOperationAncestryTracker
import org.nixos.gradle2nix.util.artifactCachesProvider
import org.nixos.gradle2nix.util.buildOperationListenerManager
import org.nixos.gradle2nix.util.service
import org.nixos.gradle2nix.util.checksumService
import org.nixos.gradle2nix.util.fileStoreAndIndexProvider
abstract class Gradle2NixPlugin @Inject constructor(
private val toolingModelBuilderRegistry: ToolingModelBuilderRegistry
): Plugin<Gradle> {
override fun apply(gradle: Gradle) {
val dependencyExtractor = DependencyExtractor(
gradle.buildOperationAncestryTracker,
gradle.artifactCachesProvider,
gradle.checksumService,
gradle.fileStoreAndIndexProvider,
)
toolingModelBuilderRegistry.register(DependencySetModelBuilder(dependencyExtractor))
gradle.buildOperationListenerManager.addListener(dependencyExtractor)

View File

@@ -1,252 +1,115 @@
package org.nixos.gradle2nix.dependencygraph
import java.net.URI
import java.util.Collections
import java.io.File
import java.util.concurrent.ConcurrentHashMap
import kotlin.jvm.optionals.getOrNull
import org.gradle.api.internal.artifacts.DownloadArtifactBuildOperationType
import org.gradle.api.internal.artifacts.configurations.ResolveConfigurationDependenciesBuildOperationType
import org.gradle.api.logging.Logging
import org.gradle.internal.operations.BuildOperationAncestryTracker
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.JsonObject
import kotlinx.serialization.json.decodeFromStream
import kotlinx.serialization.json.jsonArray
import kotlinx.serialization.json.jsonObject
import kotlinx.serialization.json.jsonPrimitive
import org.gradle.api.internal.artifacts.ivyservice.ArtifactCachesProvider
import org.gradle.api.internal.artifacts.ivyservice.modulecache.FileStoreAndIndexProvider
import org.gradle.internal.hash.ChecksumService
import org.gradle.internal.operations.BuildOperationDescriptor
import org.gradle.internal.operations.BuildOperationListener
import org.gradle.internal.operations.OperationFinishEvent
import org.gradle.internal.operations.OperationIdentifier
import org.gradle.internal.operations.OperationProgressEvent
import org.gradle.internal.operations.OperationStartEvent
import org.gradle.internal.resource.ExternalResourceReadBuildOperationType
import org.gradle.internal.resource.ExternalResourceReadMetadataBuildOperationType
import org.nixos.gradle2nix.model.DependencyCoordinates
import org.nixos.gradle2nix.model.DependencySet
import org.nixos.gradle2nix.model.Repository
import org.nixos.gradle2nix.model.impl.DefaultDependencyCoordinates
import org.nixos.gradle2nix.model.impl.DefaultDependencySet
import org.nixos.gradle2nix.model.impl.DefaultRepository
import org.nixos.gradle2nix.model.impl.DefaultResolvedArtifact
import org.nixos.gradle2nix.model.impl.DefaultResolvedDependency
class DependencyExtractor(
private val ancestryTracker: BuildOperationAncestryTracker,
private val artifactCachesProvider: ArtifactCachesProvider,
private val checksumService: ChecksumService,
private val fileStoreAndIndexProvider: FileStoreAndIndexProvider,
) : BuildOperationListener {
// Repositories by ID
private val repositories: MutableMap<String, DefaultRepository> = ConcurrentHashMap()
private val urls = ConcurrentHashMap<String, Unit>()
private val thrownExceptions = Collections.synchronizedList(mutableListOf<Throwable>())
override fun started(buildOperation: BuildOperationDescriptor, startEvent: OperationStartEvent) {}
private val artifacts: MutableMap<
OperationIdentifier,
DownloadArtifactBuildOperationType.Details
> = ConcurrentHashMap()
override fun progress(operationIdentifier: OperationIdentifier, progressEvent: OperationProgressEvent) {}
private val files: MutableMap<
OperationIdentifier,
ExternalResourceReadMetadataBuildOperationType.Details
> = ConcurrentHashMap()
private val fileArtifacts: MutableMap<OperationIdentifier, OperationIdentifier> = ConcurrentHashMap()
override fun finished(buildOperation: BuildOperationDescriptor, finishEvent: OperationFinishEvent) {
when (val details = buildOperation.details) {
is ExternalResourceReadBuildOperationType.Details -> urls.computeIfAbsent(details.location) { Unit }
is ExternalResourceReadMetadataBuildOperationType.Details -> urls.computeIfAbsent(details.location) { Unit }
else -> null
} ?: return
}
fun buildDependencySet(): DependencySet {
println("DependencyExtractor: buildDependencySet (wtf)")
val files = mutableMapOf<DependencyCoordinates, MutableMap<File, String>>()
val mappings = mutableMapOf<DependencyCoordinates, Map<String, String>>()
val repoList = repositories.values.toList()
val dependencies = buildMap<DependencyCoordinates, MutableMap<String, MutableSet<Pair<String, MutableSet<String>>>>> {
for ((fileId, file) in files) {
val filename = file.location.substringAfterLast("/").substringBefore('#').substringBefore('?')
if (filename == "maven-metadata.xml") {
// Skip Maven metadata, we don't need it for the local repository
continue
}
val artifactOperationId = fileArtifacts[fileId]
val artifact = artifactOperationId?.let { artifacts[it] }
val artifactIdentifier = artifact?.artifactIdentifier?.let(::parseArtifactIdentifier)
var coords = artifactIdentifier?.first
var name = artifactIdentifier?.second
if (coords == null || name == null) {
val parsed = parseComponent(repoList, file.location)
if (parsed == null) {
LOGGER.info("Couldn't parse location for ${artifactIdentifier?.first?.toString() ?: name}: ${file.location}")
continue
}
coords = coords ?: parsed.first
name = name ?: parseArtifact(parsed.second, coords, file.location)
}
getOrPut(coords) { mutableMapOf() }
.getOrPut(name) { mutableSetOf() }
.run {
val existing = find { it.first == filename }
if (existing != null) {
existing.second.add(file.location)
} else {
add(filename to mutableSetOf(file.location))
artifactCachesProvider.writableCacheAccessCoordinator.useCache {
for ((url, _) in urls) {
fileStoreAndIndexProvider.externalResourceIndex.lookup(url)?.let { cached ->
cached.cachedFile?.let { file ->
cachedComponentId(file)?.let { componentId ->
files.getOrPut(componentId, ::mutableMapOf)[file] = url
if (file.extension == "module") {
parseFileMappings(file)?.let {
mappings[componentId] = it
}
}
}
}
}
}
}
return DefaultDependencySet(
dependencies = dependencies.map { (coords, artifacts) ->
DefaultResolvedDependency(
coords,
artifacts.flatMap { (name, files) ->
files.map { (filename, urls) ->
DefaultResolvedArtifact(name, filename, urls.toList())
dependencies = buildList {
for ((componentId, componentFiles) in files) {
add(DefaultResolvedDependency(
componentId,
buildList {
val remoteMappings = mappings[componentId]
for ((file, url) in componentFiles) {
add(DefaultResolvedArtifact(
remoteMappings?.get(file.name) ?: file.name,
checksumService.sha256(file).toString(),
url
))
}
}
}
)
}
)
}
override fun started(buildOperation: BuildOperationDescriptor, startEvent: OperationStartEvent) {
val id = buildOperation.id ?: return
when (val details = buildOperation.details) {
is ResolveConfigurationDependenciesBuildOperationType.Details -> {
for (repository in details.repositories.orEmpty()) {
addRepository(repository)
}
}
is DownloadArtifactBuildOperationType.Details -> {
artifacts[id] = details
}
is ExternalResourceReadMetadataBuildOperationType.Details -> {
files[id] = details
ancestryTracker.findClosestMatchingAncestor(id) { it in artifacts }.getOrNull()?.let {
fileArtifacts[id] = it
}
}
}
}
override fun progress(operationIdentifier: OperationIdentifier, progressEvent: OperationProgressEvent) {}
override fun finished(buildOperation: BuildOperationDescriptor, finishEvent: OperationFinishEvent) {}
private fun addRepository(
repository: ResolveConfigurationDependenciesBuildOperationType.Repository
): DefaultRepository {
@Suppress("UNCHECKED_CAST")
val candidate = DefaultRepository(
id = repository.id,
type = enumValueOf(repository.type),
metadataSources = (repository.properties["METADATA_SOURCES"] as? List<String>) ?: emptyList(),
metadataResources = metadataResources(repository),
artifactResources = artifactResources(repository),
)
// Repository IDs are not unique across the entire build, unfortunately.
val existing = repositories.values.find {
it.type == candidate.type &&
it.metadataSources == candidate.metadataSources &&
it.metadataResources == candidate.metadataResources &&
it.artifactResources == candidate.artifactResources
}
if (existing != null) return existing
var inc = 0
fun incId() = if (inc > 0) "${candidate.id}[$inc]" else candidate.id
while (incId() in repositories) inc++
val added = if (inc > 0) candidate else candidate.copy(id = incId())
repositories[added.id] = added
return added
}
companion object {
private val LOGGER = Logging.getLogger(DependencyExtractor::class.java)
internal const val M2_PATTERN =
"[organisation]/[module]/[revision]/[artifact]-[revision](-[classifier]).[ext]"
private const val IVY_ARTIFACT_PATTERN = "[organisation]/[module]/[revision]/[type]s/[artifact](.[ext])";
private fun resources(urls: List<URI>, patterns: List<String>): List<String> {
if (urls.isEmpty()) {
return patterns
}
if (patterns.isEmpty()) {
return urls.map { it.toString() }
}
return mutableListOf<String>().apply {
for (pattern in patterns) {
for (url in urls) {
add(
url.toString()
.removeSuffix("/")
.plus("/")
.plus(pattern.removePrefix("/"))
)
}
}
}
}
private fun metadataResources(
repository: ResolveConfigurationDependenciesBuildOperationType.Repository
): List<String> {
return when (repository.type) {
Repository.Type.MAVEN.name -> {
resources(
listOfNotNull(repository.properties["URL"] as? URI),
listOf(M2_PATTERN)
)
}
Repository.Type.IVY.name -> {
@Suppress("UNCHECKED_CAST")
val patterns = repository.properties["IVY_PATTERNS"] as? List<String>
?: listOf(IVY_ARTIFACT_PATTERN)
resources(
listOfNotNull(repository.properties["URL"] as? URI),
patterns
)
}
else -> emptyList()
}
}
private fun artifactResources(
repository: ResolveConfigurationDependenciesBuildOperationType.Repository
): List<String> {
return when (repository.type) {
Repository.Type.MAVEN.name -> {
@Suppress("UNCHECKED_CAST")
(resources(
listOfNotNull(repository.properties["URL"] as? URI)
.plus(repository.properties["ARTIFACT_URLS"] as? List<URI> ?: emptyList()),
listOf(M2_PATTERN)
))
}
Repository.Type.IVY.name -> {
@Suppress("UNCHECKED_CAST")
val patterns = repository.properties["ARTIFACT_PATTERNS"] as? List<String>
?: listOf(IVY_ARTIFACT_PATTERN)
resources(
listOfNotNull(repository.properties["URL"] as? URI),
patterns
)
}
else -> emptyList()
}
}
private val artifactRegex = Regex("(?<name>\\S+) \\((?<coordinates>\\S+)\\)")
private fun parseArtifactIdentifier(input: String): Pair<DependencyCoordinates, String>? {
val groups = artifactRegex.matchEntire(input)?.groups ?: return null.also {
LOGGER.warn("artifact regex didn't match $input")
}
val coords = groups["coordinates"]?.value?.let(DefaultDependencyCoordinates::parse) ?: return null
val name = groups["name"]?.value ?: return null
return coords to name
}
)
}
}
private fun cachedComponentId(file: File): DependencyCoordinates? {
val parts = file.invariantSeparatorsPath.split('/')
if (parts.size < 6) return null
if (parts[parts.size - 6] != "files-2.1") return null
return parts.dropLast(2).takeLast(3).joinToString(":").let(DefaultDependencyCoordinates::parse)
}
@OptIn(ExperimentalSerializationApi::class)
private fun parseFileMappings(file: File): Map<String, String>? = try {
Json.decodeFromStream<JsonObject>(file.inputStream())
.jsonObject["variants"]?.jsonArray
?.flatMap { it.jsonObject["files"]?.jsonArray ?: emptyList() }
?.map { it.jsonObject }
?.mapNotNull {
val name = it["name"]?.jsonPrimitive?.content ?: return@mapNotNull null
val url = it["url"]?.jsonPrimitive?.content ?: return@mapNotNull null
name to url
}
?.toMap()
?.takeUnless { it.isEmpty() }
} catch (e: Throwable) {
null
}

View File

@@ -1,158 +0,0 @@
package org.nixos.gradle2nix.dependencygraph
import java.util.concurrent.ConcurrentHashMap
import org.nixos.gradle2nix.model.DependencyCoordinates
import org.nixos.gradle2nix.model.Repository
import org.nixos.gradle2nix.model.impl.DefaultDependencyCoordinates
private val partRegex = Regex("\\[(?<attr>[^]]+)]|\\((?<optional>([^)]+))\\)")
private fun StringBuilder.appendPattern(
input: String,
seen: MutableList<String>,
) {
var literalStart = 0
partRegex.findAll(input).forEach { match ->
val literal = input.substring(literalStart, match.range.first)
if (literal.isNotEmpty()) {
append(Regex.escape(literal))
}
literalStart = match.range.last + 1
val optionalValue = match.groups["optional"]?.value
val attrValue = match.groups["attr"]?.value
if (optionalValue != null) {
append("(")
appendPattern(optionalValue, seen)
append(")?")
} else if (attrValue != null) {
if (attrValue !in seen) {
seen.add(attrValue)
append("(?<$attrValue>[^/]+)")
} else {
append("\\k<$attrValue>")
}
}
}
val tail = input.substring(literalStart)
if (tail.isNotEmpty()) {
append(Regex.escape(input.substring(literalStart)))
}
}
private fun String.replaceAttrs(
attrs: Map<String, String>
): String {
return partRegex.replace(this) { match ->
val optionalValue = match.groups["optional"]?.value
val attrValue = match.groups["attr"]?.value
if (optionalValue != null) {
val replaced = optionalValue.replaceAttrs(attrs)
if (replaced != optionalValue) replaced else match.value
} else if (attrValue != null) {
attrs[attrValue] ?: match.value
} else {
match.value
}
}
}
private fun interface ArtifactMatcher {
fun match(url: String): Map<String, String>?
}
private fun regexMatcher(regex: Regex, attrs: List<String>): ArtifactMatcher {
return ArtifactMatcher { url ->
regex.matchEntire(url)?.groups?.let { groups ->
buildMap {
for (attr in attrs) {
groups[attr]?.let { put(attr, it.value) }
}
}
}
}
}
private fun patternMatcher(pattern: String): ArtifactMatcher {
val attrs = mutableListOf<String>()
val exp = buildString { appendPattern(pattern, attrs) }.toRegex()
return regexMatcher(exp, attrs)
}
private fun mavenMatcher(pattern: String): ArtifactMatcher {
val attrs = mutableListOf<String>()
val exp = buildString { appendPattern(pattern.replaceAfterLast("/", ""), attrs) }
.replace("<organisation>[^/]+", "<organisation>.+")
.plus("[^/]+")
.toRegex()
return regexMatcher(exp, attrs)
}
private val matcherCache: MutableMap<String, ArtifactMatcher> = ConcurrentHashMap()
private fun matcher(
pattern: String,
): ArtifactMatcher = matcherCache.getOrPut(pattern) {
if (pattern.endsWith(DependencyExtractor.M2_PATTERN)) mavenMatcher(pattern) else patternMatcher(pattern)
}
fun parseComponent(
repositories: List<Repository>,
url: String,
): Pair<DependencyCoordinates, String>? {
for (repository in repositories) {
for (pattern in (repository.metadataResources + repository.artifactResources).distinct()) {
val matcher = matcher(pattern)
val attrs = matcher.match(url)
if (attrs != null) {
val group = attrs["organisation"]?.replace('/', '.') ?: continue
val artifact = attrs["module"] ?: continue
val revision = attrs["revision"] ?: continue
return DefaultDependencyCoordinates(group, artifact, revision) to pattern.replaceAttrs(attrs)
}
}
}
return null
}
fun parseArtifact(
resource: String,
component: DependencyCoordinates,
url: String
): String {
val attrs = mutableListOf<String>()
var pattern = buildString { appendPattern(resource, attrs) }
if (component.version.endsWith("-SNAPSHOT")) {
val base = component.version.substringBeforeLast("-SNAPSHOT", "")
pattern = pattern.replace("\\Q-${component.version}\\E", "\\Q-$base-\\E(?:.+)")
}
val values = regexMatcher(pattern.toRegex(), attrs).match(url)
val artifact = values?.get("artifact")
val classifier = values?.get("classifier")
val ext = values?.get("ext")
if (artifact == null) return artifactFromFilename(
url.substringAfterLast('/').substringBefore('#').substringBefore('?'),
component.version,
classifier
)
return buildString {
append("$artifact-${component.version}")
if (classifier != null) append("-$classifier")
if (ext != null) append(".$ext")
}
}
private fun artifactFromFilename(filename: String, version: String, classifier: String?): String {
val name = filename.substringBeforeLast('.')
val extension = filename.substringAfterLast('.', "")
return buildString {
append("$name-$version")
if (classifier != null) append("-$classifier")
if (extension.isNotEmpty()) append(".$extension")
}
}

View File

@@ -3,16 +3,25 @@ package org.nixos.gradle2nix.util
import java.lang.reflect.Method
import org.gradle.api.artifacts.Configuration
import org.gradle.api.internal.GradleInternal
import org.gradle.api.internal.artifacts.ivyservice.ArtifactCachesProvider
import org.gradle.api.internal.artifacts.ivyservice.modulecache.FileStoreAndIndexProvider
import org.gradle.api.invocation.Gradle
import org.gradle.internal.hash.ChecksumService
import org.gradle.internal.operations.BuildOperationAncestryTracker
import org.gradle.internal.operations.BuildOperationListenerManager
internal inline val Gradle.buildOperationAncestryTracker: BuildOperationAncestryTracker
internal inline val Gradle.artifactCachesProvider: ArtifactCachesProvider
get() = service()
internal inline val Gradle.buildOperationListenerManager: BuildOperationListenerManager
get() = service()
internal inline val Gradle.checksumService: ChecksumService
get() = service()
internal inline val Gradle.fileStoreAndIndexProvider: FileStoreAndIndexProvider
get() = service()
internal inline fun <reified T> Gradle.service(): T =
(this as GradleInternal).services.get(T::class.java)

View File

@@ -1,58 +0,0 @@
package org.nixos.gradle2nix.dependencygraph
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.nulls.shouldNotBeNull
import io.kotest.matchers.shouldBe
import org.nixos.gradle2nix.model.Repository
import org.nixos.gradle2nix.model.impl.DefaultDependencyCoordinates
import org.nixos.gradle2nix.model.impl.DefaultRepository
class DependencyUrlParserTest : FunSpec({
val mavenCentral = DefaultRepository(
"MavenRepo",
Repository.Type.MAVEN,
metadataSources = listOf("mavenPom"),
metadataResources = listOf("https://repo.maven.apache.org/maven2/${DependencyExtractor.M2_PATTERN}"),
artifactResources = listOf("https://repo.maven.apache.org/maven2/${DependencyExtractor.M2_PATTERN}")
)
test("parses maven url") {
val url = "https://repo.maven.apache.org/maven2/com/github/ajalt/clikt-metadata/2.8.0/clikt-metadata-2.8.0.jar"
val (coords, pattern) = parseComponent(listOf(mavenCentral), url).shouldNotBeNull()
coords shouldBe DefaultDependencyCoordinates("com.github.ajalt", "clikt-metadata", "2.8.0")
parseArtifact(pattern, coords, url) shouldBe "clikt-metadata-2.8.0.jar"
}
test("parses maven snapshot url") {
val url = "https://repo.maven.apache.org/maven2/org/apache/test-SNAPSHOT2/2.0.2-SNAPSHOT/test-SNAPSHOT2-2.0.2-SNAPSHOT.jar"
val (coords, pattern) = parseComponent(listOf(mavenCentral), url).shouldNotBeNull()
coords shouldBe DefaultDependencyCoordinates("org.apache", "test-SNAPSHOT2", "2.0.2-SNAPSHOT")
parseArtifact(pattern, coords, url) shouldBe "test-SNAPSHOT2-2.0.2-SNAPSHOT.jar"
}
test("parses maven timestamped snapshot url") {
val url = "https://repo.maven.apache.org/maven2/org/apache/test-SNAPSHOT1/2.0.2-SNAPSHOT/test-SNAPSHOT1-2.0.2-20070310.181613-3.jar"
val (coords, pattern) = parseComponent(listOf(mavenCentral), url).shouldNotBeNull()
coords shouldBe DefaultDependencyCoordinates("org.apache", "test-SNAPSHOT1", "2.0.2-SNAPSHOT")
parseArtifact(pattern, coords, url) shouldBe "test-SNAPSHOT1-2.0.2-SNAPSHOT.jar"
}
test("parses ivy descriptor url") {
val url = "https://asset.opendof.org/ivy2/org.opendof.core-java/dof-cipher-sms4/1.0/ivy.xml"
val (coords, pattern) = parseComponent(
listOf(
DefaultRepository(
"ivy",
Repository.Type.IVY,
metadataSources = listOf("ivyDescriptor"),
metadataResources = listOf("https://asset.opendof.org/ivy2/[organisation]/[module]/[revision]/ivy(.[platform]).xml"),
artifactResources = listOf("https://asset.opendof.org/artifact/[organisation]/[module]/[revision](/[platform])(/[type]s)/[artifact]-[revision](-[classifier]).[ext]")
)
),
url
).shouldNotBeNull()
coords shouldBe DefaultDependencyCoordinates("org.opendof.core-java", "dof-cipher-sms4", "1.0")
parseArtifact(pattern, coords, url) shouldBe "ivy-1.0.xml"
}
})