Update metadata parsing

This commit is contained in:
Tad Fisher
2023-10-11 13:29:21 -07:00
parent ba088f5bc6
commit 121e512a06
29 changed files with 5708 additions and 1135 deletions

View File

@@ -5,17 +5,35 @@ import kotlin.system.exitProcess
class Logger(
val out: PrintStream = System.err,
val verbose: Boolean
val verbose: Boolean,
val stacktrace: Boolean = false
) {
val log: (String) -> Unit = { if (verbose) out.println(it) }
val warn: (String) -> Unit = { out.println("Warning: $it")}
val error: (String) -> Nothing = {
out.println("Error: $it")
fun log(message: String, error: Throwable? = null) {
if (!verbose) return
out.println(message)
if (error == null) return
error.message?.let { println(" Cause: $it") }
if (stacktrace) error.printStackTrace(out)
}
fun warn(message: String, error: Throwable? = null) {
out.println("Warning: $message")
if (error == null) return
error.message?.let { println(" Cause: $it") }
if (stacktrace) error.printStackTrace(out)
}
fun error(message: String, error: Throwable? = null): Nothing {
out.println("Error: $message")
if (error != null) {
error.message?.let { println(" Cause: $it") }
if (stacktrace) error.printStackTrace(out)
}
exitProcess(1)
}
operator fun component1() = log
operator fun component2() = warn
operator fun component3() = error
operator fun component1() = ::log
operator fun component2() = ::warn
operator fun component3() = ::error
}

View File

@@ -30,7 +30,8 @@ data class Config(
)
@OptIn(ExperimentalSerializationApi::class)
private val JsonFormat = Json {
val JsonFormat = Json {
ignoreUnknownKeys = true
prettyPrint = true
prettyPrintIndent = " "
}
@@ -106,9 +107,6 @@ class Gradle2Nix : CliktCommand(
}
}
// Visible for testing
lateinit var config: Config
@OptIn(ExperimentalSerializationApi::class)
override fun run() {
val appHome = System.getProperty("org.nixos.gradle2nix.share")
@@ -118,7 +116,7 @@ class Gradle2Nix : CliktCommand(
val gradleHome = System.getenv("GRADLE_USER_HOME")?.let(::File) ?: File("${System.getProperty("user.home")}/.gradle")
val logger = Logger(verbose = !quiet)
config = Config(
val config = Config(
File(appHome),
gradleHome,
gradleVersion,
@@ -131,8 +129,6 @@ class Gradle2Nix : CliktCommand(
logger
)
val (log, _, error) = logger
val metadata = File("$projectDir/gradle/verification-metadata.xml")
if (metadata.exists()) {
val backup = metadata.resolveSibling("verification-metadata.xml.bak")
@@ -158,7 +154,7 @@ class Gradle2Nix : CliktCommand(
val outDir = outDir ?: projectDir
val json = outDir.resolve("$envFile.json")
log("Writing environment to $json")
logger.log("Writing environment to $json")
json.outputStream().buffered().use { output ->
JsonFormat.encodeToStream(dependencies, output)
}

View File

@@ -1,9 +1,12 @@
package org.nixos.gradle2nix
import org.nixos.gradle2nix.metadata.Artifact as ArtifactMetadata
import java.io.File
import java.io.FileFilter
import java.io.IOException
import java.net.URI
import java.net.URL
import kotlinx.serialization.ExperimentalSerializationApi
import kotlinx.serialization.SerializationException
import kotlinx.serialization.json.Json
import kotlinx.serialization.json.decodeFromStream
import okio.ByteString.Companion.decodeHex
@@ -11,16 +14,19 @@ import okio.HashingSource
import okio.blackholeSink
import okio.buffer
import okio.source
import org.nixos.gradle2nix.dependency.ModuleComponentIdentifier
import org.nixos.gradle2nix.dependencygraph.model.DependencyCoordinates
import org.nixos.gradle2nix.dependencygraph.model.Repository
import org.nixos.gradle2nix.dependencygraph.model.ResolvedConfiguration
import org.nixos.gradle2nix.metadata.ArtifactVerificationMetadata
import org.nixos.gradle2nix.metadata.Checksum
import org.nixos.gradle2nix.metadata.ChecksumKind
import org.nixos.gradle2nix.metadata.ComponentVerificationMetadata
import org.nixos.gradle2nix.metadata.DependencyVerificationsXmlReader
import org.nixos.gradle2nix.metadata.DependencyVerifier
import org.nixos.gradle2nix.metadata.Component
import org.nixos.gradle2nix.metadata.Md5
import org.nixos.gradle2nix.metadata.Sha1
import org.nixos.gradle2nix.metadata.Sha256
import org.nixos.gradle2nix.metadata.Sha512
import org.nixos.gradle2nix.metadata.VerificationMetadata
import org.nixos.gradle2nix.metadata.parseVerificationMetadata
import org.nixos.gradle2nix.module.GradleModule
import org.nixos.gradle2nix.module.Variant
// Local Maven repository for testing
private val m2 = System.getProperty("org.nixos.gradle2nix.m2")
@@ -31,7 +37,11 @@ private fun shouldSkipRepository(repository: Repository): Boolean {
}
fun processDependencies(config: Config): Map<String, Map<String, Artifact>> {
val verifier = readVerificationMetadata(config)
val verificationMetadata = readVerificationMetadata(config)
val verificationComponents = verificationMetadata?.components?.associateBy {
DependencyCoordinates(it.group, it.name, it.version)
} ?: emptyMap()
val moduleCache = mutableMapOf<DependencyCoordinates, GradleModule?>()
val configurations = readDependencyGraph(config)
val repositories = configurations
@@ -61,15 +71,10 @@ fun processDependencies(config: Config): Map<String, Map<String, Artifact>> {
return@mapNotNull null
}
val coordinates = deps.first().coordinates
val componentId = ModuleComponentIdentifier(
coordinates.group,
coordinates.module,
coordinates.version
)
val metadata = verifier.verificationMetadata[componentId]
?: verifyComponentFilesInCache(config, componentId)
?: verifyComponentFilesInTestRepository(config, componentId)
if (metadata == null) {
val component = verificationComponents[coordinates]
?: verifyComponentFilesInCache(config, coordinates)
?: verifyComponentFilesInTestRepository(config, coordinates)
if (component == null) {
config.logger.warn("$id: not present in metadata or cache; skipping")
return@mapNotNull null
}
@@ -85,23 +90,25 @@ fun processDependencies(config: Config): Map<String, Map<String, Artifact>> {
return@mapNotNull null
}
id to metadata.artifactVerifications.associate { meta ->
meta.artifactName to Artifact(
val gradleModule = moduleCache.getOrPut(coordinates) {
maybeGetGradleModule(config.logger, coordinates, repos)
}
id to component.artifacts.associate { meta ->
meta.name to Artifact(
urls = repos
.flatMap { repository -> artifactUrls(coordinates, meta, repository) }
.flatMap { repository -> artifactUrls(coordinates, meta.name, repository, gradleModule) }
.distinct(),
hash = meta.checksums.maxBy { c -> c.kind.ordinal }.toSri()
hash = meta.checksums.first().toSri()
)
}
}
.sortedBy { it.first }
.toMap()
}
private fun readVerificationMetadata(config: Config): DependencyVerifier {
return config.projectDir.resolve("gradle/verification-metadata.xml")
.inputStream()
.buffered()
.use { input -> DependencyVerificationsXmlReader.readFromXml(input) }
private fun readVerificationMetadata(config: Config): VerificationMetadata? {
return parseVerificationMetadata(config.logger, config.projectDir.resolve("gradle/verification-metadata.xml"))
}
@OptIn(ExperimentalSerializationApi::class)
@@ -114,40 +121,58 @@ private fun readDependencyGraph(config: Config): List<ResolvedConfiguration> {
private fun verifyComponentFilesInCache(
config: Config,
component: ModuleComponentIdentifier
): ComponentVerificationMetadata? {
val cacheDir = config.gradleHome.resolve("caches/modules-2/files-2.1/${component.group}/${component.module}/${component.version}")
coordinates: DependencyCoordinates,
): Component? {
val cacheDir = with(coordinates) { config.gradleHome.resolve("caches/modules-2/files-2.1/$group/$module/$version") }
if (!cacheDir.exists()) {
return null
}
val verifications = cacheDir.walk().filter { it.isFile }.map { f ->
ArtifactVerificationMetadata(
f.name,
listOf(Checksum(ChecksumKind.sha256, f.sha256()))
)
ArtifactMetadata(f.name, sha256 = Sha256(f.sha256()))
}
config.logger.log("$component: obtained artifact hashes from Gradle cache.")
return ComponentVerificationMetadata(component, verifications.toList())
config.logger.log("$coordinates: obtained artifact hashes from Gradle cache.")
return Component(coordinates, verifications.toList())
}
private fun verifyComponentFilesInTestRepository(
config: Config,
component: ModuleComponentIdentifier
): ComponentVerificationMetadata? {
coordinates: DependencyCoordinates
): Component? {
if (m2 == null) return null
val dir = File(URI.create(m2)).resolve("${component.group.replace(".", "/")}/${component.module}/${component.version}")
val dir = with(coordinates) {
File(URI.create(m2)).resolve("${group.replace(".", "/")}/$module/$version")
}
if (!dir.exists()) {
config.logger.log("$component: not found in m2 repository; tried $dir")
config.logger.log("$coordinates: not found in m2 repository; tried $dir")
return null
}
val verifications = dir.walk().filter { it.isFile && it.name.startsWith(component.module) }.map { f ->
ArtifactVerificationMetadata(
val verifications = dir.walk().filter { it.isFile && it.name.startsWith(coordinates.module) }.map { f ->
ArtifactMetadata(
f.name,
listOf(Checksum(ChecksumKind.sha256, f.sha256()))
sha256 = Sha256(f.sha256())
)
}
config.logger.log("$component: obtained artifact hashes from test Maven repository.")
return ComponentVerificationMetadata(component, verifications.toList())
config.logger.log("$coordinates: obtained artifact hashes from test Maven repository.")
return Component(coordinates, verifications.toList())
}
@OptIn(ExperimentalSerializationApi::class)
private fun maybeGetGradleModule(logger: Logger, coordinates: DependencyCoordinates, repos: List<Repository>): GradleModule? {
val filename = with(coordinates) { "$module-$version.module" }
for (url in repos.flatMap { artifactUrls(coordinates, filename, it, null)}) {
try {
return URL(url).openStream().buffered().use { input ->
JsonFormat.decodeFromStream(input)
}
} catch (e: SerializationException) {
logger.error("$coordinates: failed to parse Gradle module metadata ($url)", e)
} catch (e: IOException) {
// Pass
}
}
return null
}
private fun File.sha256(): String {
@@ -158,26 +183,34 @@ private fun File.sha256(): String {
private fun Checksum.toSri(): String {
val hash = value.decodeHex().base64()
return when (kind) {
ChecksumKind.md5 -> "md5-$hash"
ChecksumKind.sha1 -> "sha1-$hash"
ChecksumKind.sha256 -> "sha256-$hash"
ChecksumKind.sha512 -> "sha512-$hash"
return when (this) {
is Md5 -> "md5-$hash"
is Sha1 -> "sha1-$hash"
is Sha256 -> "sha256-$hash"
is Sha512 -> "sha512-$hash"
}
}
private fun artifactUrls(
coordinates: DependencyCoordinates,
metadata: ArtifactVerificationMetadata,
repository: Repository
filename: String,
repository: Repository,
module: GradleModule?
): List<String> {
val groupAsPath = coordinates.group.replace(".", "/")
val repoFilename = module?.let { m ->
m.variants
.asSequence()
.flatMap(Variant::files)
.find { it.name == filename }
}?.url ?: filename
val attributes = mutableMapOf(
"organisation" to if (repository.m2Compatible) groupAsPath else coordinates.group,
"module" to coordinates.module,
"revision" to coordinates.version,
) + fileAttributes(metadata.artifactName, coordinates.version)
) + fileAttributes(repoFilename, coordinates.version)
val resources = when (attributes["ext"]) {
"pom" -> if ("mavenPom" in repository.metadataSources) repository.metadataResources else repository.artifactResources

View File

@@ -1,31 +0,0 @@
/*
* Copyright 2014 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.dependency
/**
* An opaque immutable identifier for an artifact that belongs to some component instance.
*/
interface ComponentArtifactIdentifier {
/**
* Returns the id of the component that this artifact belongs to.
*/
val componentIdentifier: ComponentIdentifier
/**
* Returns some human-consumable display name for this artifact.
*/
val displayName: String
}

View File

@@ -1,28 +0,0 @@
/*
* Copyright 2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.dependency
/**
* An opaque immutable identifier for a component instance. There are various sub-interfaces that expose specific details about the identifier.
*/
interface ComponentIdentifier {
/**
* Returns a human-consumable display name for this identifier.
*
* @return Component identifier display name
*/
val displayName: String
}

View File

@@ -1,40 +0,0 @@
/*
* Copyright 2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.dependency
/**
* An immutable identifier for an artifact that belongs to some module version.
*/
interface ModuleComponentArtifactIdentifier : ComponentArtifactIdentifier {
/**
* Returns the id of the component that this artifact belongs to.
*/
override val componentIdentifier: ModuleComponentIdentifier
/**
* Returns a file base name that can be used for this artifact.
*/
val fileName: String
}
data class DefaultModuleComponentArtifactIdentifier(
override val componentIdentifier: ModuleComponentIdentifier,
override val fileName: String,
) : ModuleComponentArtifactIdentifier {
override val displayName: String get() = "$fileName ($componentIdentifier)"
override fun toString(): String = displayName
}

View File

@@ -1,76 +0,0 @@
/*
* Copyright 2013 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.dependency
/**
* An identifier for a component instance which is available as a module version.
*/
interface ModuleComponentIdentifier : ComponentIdentifier {
/**
* The module group of the component.
*
* @return Component group
*/
val group: String
/**
* The module name of the component.
*
* @return Component module
*/
val module: String
/**
* The module version of the component.
*
* @return Component version
*/
val version: String
/**
* The module identifier of the component. Returns the same information
* as [group] and [module].
*
* @return the module identifier
*/
val moduleIdentifier: ModuleIdentifier
}
data class DefaultModuleComponentIdentifier(
override val moduleIdentifier: ModuleIdentifier,
override val version: String,
) : ModuleComponentIdentifier {
override val group: String
get() = moduleIdentifier.group
override val module: String
get() = moduleIdentifier.name
override val displayName: String
get() = "$group:$module:$version"
override fun toString(): String = displayName
}
fun ModuleComponentIdentifier(
group: String,
module: String,
version: String
): ModuleComponentIdentifier = DefaultModuleComponentIdentifier(
DefaultModuleIdentifier(group, module),
version
)

View File

@@ -1,40 +0,0 @@
/*
* Copyright 2012 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.dependency
/**
* The identifier of a module.
*/
interface ModuleIdentifier {
/**
* The group of the module.
*
* @return module group
*/
val group: String
/**
* The name of the module.
*
* @return module name
*/
val name: String
}
data class DefaultModuleIdentifier(
override val group: String,
override val name: String,
) : ModuleIdentifier

View File

@@ -1,6 +0,0 @@
package org.nixos.gradle2nix.metadata
data class ArtifactVerificationMetadata(
val artifactName: String,
val checksums: List<Checksum>
)

View File

@@ -1,43 +0,0 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.metadata
/**
* Internal representation of a checksum, aimed at *verification*.
* A checksum consists of a kind (md5, sha1, ...), a value, but also
* provides *alternatives*. Alternatives are checksums which are
* deemed trusted, because sometimes in a single build we can see different
* checksums for the same module, because they are sourced from different
* repositories.
*
* In theory, this shouldn't be allowed. However, it's often the case that
* an artifact, in particular _metadata artifacts_ (POM files, ...) differ
* from one repository to the other (either by end of lines, additional line
* at the end of the file, ...). Because they are different doesn't mean that
* they are compromised, so this is a facility for the user to declare "I know
* I should use a single source of truth but the infrastructure is hard or
* impossible to fix so let's trust this source".
*
* In addition to the list of alternatives, a checksum also provides a source,
* which is documentation to explain where a checksum was found.
*/
data class Checksum(
val kind: ChecksumKind,
val value: String,
val alternatives: Set<String> = emptySet(),
val origin: String? = null,
val reason: String? = null
)

View File

@@ -1,30 +0,0 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.metadata
enum class ChecksumKind(val algorithm: String) {
md5("MD5"),
sha1("SHA1"),
sha256("SHA-256"),
sha512("SHA-512");
companion object {
private val SORTED_BY_SECURITY: List<ChecksumKind> = listOf(sha512, sha256, sha1, md5)
fun mostSecureFirst(): List<ChecksumKind> {
return SORTED_BY_SECURITY
}
}
}

View File

@@ -1,8 +0,0 @@
package org.nixos.gradle2nix.metadata
import org.nixos.gradle2nix.dependency.ModuleComponentIdentifier
data class ComponentVerificationMetadata(
val componentId: ModuleComponentIdentifier,
val artifactVerifications: List<ArtifactVerificationMetadata>,
)

View File

@@ -1,98 +0,0 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.metadata
import org.nixos.gradle2nix.dependency.ModuleComponentArtifactIdentifier
import org.nixos.gradle2nix.dependency.ModuleComponentIdentifier
class DependencyVerificationConfiguration(
val trustedArtifacts: List<TrustCoordinates> = emptyList(),
) {
data class TrustCoordinates internal constructor(
val group: String?,
val name: String?,
val version: String?,
val fileName: String?,
val isRegex: Boolean,
val reason: String?
) : Comparable<TrustCoordinates> {
fun matches(id: ModuleComponentArtifactIdentifier): Boolean {
val moduleComponentIdentifier: ModuleComponentIdentifier = id.componentIdentifier
return (matches(group, moduleComponentIdentifier.group)
&& matches(name, moduleComponentIdentifier.module)
&& matches(version, moduleComponentIdentifier.version)
&& matches(fileName, id.fileName))
}
private fun matches(value: String?, expr: String): Boolean {
if (value == null) {
return true
}
return if (!isRegex) {
expr == value
} else expr.matches(value.toRegex())
}
override fun compareTo(other: TrustCoordinates): Int {
val regexComparison = isRegex.compareTo(other.isRegex)
if (regexComparison != 0) {
return regexComparison
}
val groupComparison = compareNullableStrings(
group, other.group
)
if (groupComparison != 0) {
return groupComparison
}
val nameComparison = compareNullableStrings(
name, other.name
)
if (nameComparison != 0) {
return nameComparison
}
val versionComparison = compareNullableStrings(
version, other.version
)
if (versionComparison != 0) {
return versionComparison
}
val fileNameComparison = compareNullableStrings(
fileName, other.fileName
)
return if (fileNameComparison != 0) {
fileNameComparison
} else compareNullableStrings(
reason, other.reason
)
}
}
companion object {
private fun compareNullableStrings(first: String?, second: String?): Int {
if (first == null) {
return if (second == null) {
0
} else {
-1
}
} else if (second == null) {
return 1
}
return first.compareTo(second)
}
}
}

View File

@@ -1,48 +0,0 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.metadata
internal object DependencyVerificationXmlTags {
const val ALSO_TRUST = "also-trust"
const val ARTIFACT = "artifact"
const val COMPONENT = "component"
const val COMPONENTS = "components"
const val CONFIG = "configuration"
const val ENABLED = "enabled"
const val FILE = "file"
const val GROUP = "group"
const val ID = "id"
const val IGNORED_KEY = "ignored-key"
const val IGNORED_KEYS = "ignored-keys"
const val KEY_SERVER = "key-server"
const val KEY_SERVERS = "key-servers"
const val NAME = "name"
const val ORIGIN = "origin"
const val PGP = "pgp"
const val REASON = "reason"
const val REGEX = "regex"
const val TRUST = "trust"
const val TRUSTED_ARTIFACTS = "trusted-artifacts"
const val TRUSTED_KEY = "trusted-key"
const val TRUSTED_KEYS = "trusted-keys"
const val TRUSTING = "trusting"
const val URI = "uri"
const val VALUE = "value"
const val VERIFICATION_METADATA = "verification-metadata"
const val VERIFY_METADATA = "verify-metadata"
const val VERIFY_SIGNATURES = "verify-signatures"
const val VERSION = "version"
}

View File

@@ -1,421 +0,0 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.metadata
import java.io.IOException
import java.io.InputStream
import javax.xml.parsers.ParserConfigurationException
import javax.xml.parsers.SAXParser
import javax.xml.parsers.SAXParserFactory
import org.gradle.internal.UncheckedException
import org.nixos.gradle2nix.dependency.DefaultModuleComponentArtifactIdentifier
import org.nixos.gradle2nix.dependency.DefaultModuleComponentIdentifier
import org.nixos.gradle2nix.dependency.DefaultModuleIdentifier
import org.nixos.gradle2nix.dependency.ModuleComponentArtifactIdentifier
import org.nixos.gradle2nix.dependency.ModuleComponentIdentifier
import org.nixos.gradle2nix.dependency.ModuleIdentifier
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.ALSO_TRUST
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.ARTIFACT
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.COMPONENT
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.COMPONENTS
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.CONFIG
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.IGNORED_KEY
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.IGNORED_KEYS
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.KEY_SERVER
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.KEY_SERVERS
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.PGP
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.TRUST
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.TRUSTED_ARTIFACTS
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.TRUSTED_KEY
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.TRUSTED_KEYS
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.TRUSTING
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.VALUE
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.VERIFICATION_METADATA
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.VERIFY_METADATA
import org.nixos.gradle2nix.metadata.DependencyVerificationXmlTags.VERIFY_SIGNATURES
import org.xml.sax.Attributes
import org.xml.sax.InputSource
import org.xml.sax.SAXException
import org.xml.sax.ext.DefaultHandler2
object DependencyVerificationsXmlReader {
fun readFromXml(
input: InputStream,
builder: DependencyVerifierBuilder
) {
try {
val saxParser = createSecureParser()
val xmlReader = saxParser.xmlReader
val handler = VerifiersHandler(builder)
xmlReader.setProperty("http://xml.org/sax/properties/lexical-handler", handler)
xmlReader.contentHandler = handler
xmlReader.parse(InputSource(input))
} catch (e: Exception) {
throw IllegalStateException(
"Unable to read dependency verification metadata",
e
)
} finally {
try {
input.close()
} catch (e: IOException) {
throw UncheckedException.throwAsUncheckedException(e)
}
}
}
fun readFromXml(input: InputStream): DependencyVerifier {
val builder = DependencyVerifierBuilder()
readFromXml(input, builder)
return builder.build()
}
@Throws(ParserConfigurationException::class, SAXException::class)
private fun createSecureParser(): SAXParser {
val spf = SAXParserFactory.newInstance()
spf.setFeature("http://apache.org/xml/features/nonvalidating/load-external-dtd", false)
spf.setFeature("http://xml.org/sax/features/namespaces", false)
spf.setFeature("http://apache.org/xml/features/disallow-doctype-decl", true)
return spf.newSAXParser()
}
private class VerifiersHandler(private val builder: DependencyVerifierBuilder) : DefaultHandler2() {
private var inMetadata = false
private var inComponents = false
private var inConfiguration = false
private var inVerifyMetadata = false
private var inVerifySignatures = false
private var inTrustedArtifacts = false
private var inKeyServers = false
private var inIgnoredKeys = false
private var inTrustedKeys = false
private var inTrustedKey = false
private var currentTrustedKey: String? = null
private var currentComponent: ModuleComponentIdentifier? = null
private var currentArtifact: ModuleComponentArtifactIdentifier? =
null
private var currentChecksum: ChecksumKind? = null
override fun startElement(uri: String, localName: String, qName: String, attributes: Attributes) {
when (qName) {
CONFIG -> inConfiguration =
true
VERIFICATION_METADATA -> inMetadata =
true
COMPONENTS -> {
assertInMetadata()
inComponents = true
}
COMPONENT -> {
assertInComponents()
currentComponent = createComponentId(attributes)
}
ARTIFACT -> {
assertValidComponent()
currentArtifact = createArtifactId(attributes)
}
VERIFY_METADATA -> {
assertInConfiguration(VERIFY_METADATA)
inVerifyMetadata = true
}
VERIFY_SIGNATURES -> {
assertInConfiguration(VERIFY_SIGNATURES)
inVerifySignatures = true
}
TRUSTED_ARTIFACTS -> {
assertInConfiguration(TRUSTED_ARTIFACTS)
inTrustedArtifacts = true
}
TRUSTED_KEY -> {
assertContext(
inTrustedKeys,
TRUSTED_KEY,
TRUSTED_KEYS
)
inTrustedKey = true
}
TRUSTED_KEYS -> {
assertInConfiguration(TRUSTED_KEYS)
inTrustedKeys = true
}
TRUST -> {
assertInTrustedArtifacts()
addTrustedArtifact(attributes)
}
TRUSTING -> {
assertContext(
inTrustedKey,
TRUSTING,
TRUSTED_KEY
)
}
KEY_SERVERS -> {
assertInConfiguration(KEY_SERVERS)
inKeyServers = true
}
KEY_SERVER -> {
assertContext(
inKeyServers,
KEY_SERVER,
KEY_SERVERS
)
}
IGNORED_KEYS -> {
if (currentArtifact == null) {
assertInConfiguration(IGNORED_KEYS)
}
inIgnoredKeys = true
}
IGNORED_KEY -> {
assertContext(
inIgnoredKeys,
IGNORED_KEY,
IGNORED_KEYS
)
}
else -> if (currentChecksum != null && ALSO_TRUST == qName) {
builder.addChecksum(
currentArtifact!!,
currentChecksum!!,
getAttribute(attributes, VALUE),
null,
null
)
} else if (currentArtifact != null) {
if (PGP != qName) {
currentChecksum = enumValueOf<ChecksumKind>(qName)
builder.addChecksum(
currentArtifact!!,
currentChecksum!!,
getAttribute(
attributes,
VALUE
),
getNullableAttribute(
attributes,
DependencyVerificationXmlTags.ORIGIN
),
getNullableAttribute(
attributes,
DependencyVerificationXmlTags.REASON
)
)
}
}
}
}
private fun assertInTrustedArtifacts() {
assertContext(
inTrustedArtifacts,
TRUST,
TRUSTED_ARTIFACTS
)
}
private fun addTrustedArtifact(attributes: Attributes) {
var regex = false
val regexAttr = getNullableAttribute(
attributes,
DependencyVerificationXmlTags.REGEX
)
if (regexAttr != null) {
regex = regexAttr.toBoolean()
}
builder.addTrustedArtifact(
getNullableAttribute(
attributes,
DependencyVerificationXmlTags.GROUP
),
getNullableAttribute(
attributes,
DependencyVerificationXmlTags.NAME
),
getNullableAttribute(
attributes,
DependencyVerificationXmlTags.VERSION
),
getNullableAttribute(
attributes,
DependencyVerificationXmlTags.FILE
),
regex,
getNullableAttribute(
attributes,
DependencyVerificationXmlTags.REASON
)
)
}
private fun readBoolean(ch: CharArray, start: Int, length: Int): Boolean {
return String(ch, start, length).toBoolean()
}
private fun assertInConfiguration(tag: String) {
assertContext(
inConfiguration,
tag,
DependencyVerificationXmlTags.CONFIG
)
}
private fun assertInComponents() {
assertContext(
inComponents,
DependencyVerificationXmlTags.COMPONENT,
DependencyVerificationXmlTags.COMPONENTS
)
}
private fun assertInMetadata() {
assertContext(
inMetadata,
DependencyVerificationXmlTags.COMPONENTS,
DependencyVerificationXmlTags.VERIFICATION_METADATA
)
}
private fun assertValidComponent() {
assertContext(
currentComponent != null,
ARTIFACT,
DependencyVerificationXmlTags.COMPONENT
)
}
override fun endElement(uri: String, localName: String, qName: String) {
when (qName) {
DependencyVerificationXmlTags.CONFIG -> inConfiguration =
false
VERIFY_METADATA -> inVerifyMetadata =
false
VERIFY_SIGNATURES -> inVerifySignatures =
false
DependencyVerificationXmlTags.VERIFICATION_METADATA -> inMetadata =
false
DependencyVerificationXmlTags.COMPONENTS -> inComponents =
false
DependencyVerificationXmlTags.COMPONENT -> currentComponent =
null
TRUSTED_ARTIFACTS -> inTrustedArtifacts =
false
TRUSTED_KEYS -> inTrustedKeys =
false
TRUSTED_KEY -> {
inTrustedKey = false
currentTrustedKey = null
}
KEY_SERVERS -> inKeyServers =
false
ARTIFACT -> {
currentArtifact = null
currentChecksum = null
}
IGNORED_KEYS -> inIgnoredKeys =
false
}
}
private fun createArtifactId(attributes: Attributes): ModuleComponentArtifactIdentifier {
return DefaultModuleComponentArtifactIdentifier(
currentComponent!!,
getAttribute(
attributes,
DependencyVerificationXmlTags.NAME
)
)
}
private fun createComponentId(attributes: Attributes): ModuleComponentIdentifier {
return DefaultModuleComponentIdentifier(
createModuleId(attributes),
getAttribute(
attributes,
DependencyVerificationXmlTags.VERSION
)
)
}
private fun createModuleId(attributes: Attributes): ModuleIdentifier {
return DefaultModuleIdentifier(
getAttribute(
attributes,
DependencyVerificationXmlTags.GROUP
),
getAttribute(
attributes,
DependencyVerificationXmlTags.NAME
)
)
}
private fun getAttribute(attributes: Attributes, name: String): String {
val value = attributes.getValue(name)
assertContext(
value != null,
"Missing attribute: $name"
)
return value.intern()
}
private fun getNullableAttribute(attributes: Attributes, name: String): String? {
val value = attributes.getValue(name) ?: return null
return value.intern()
}
companion object {
private fun assertContext(test: Boolean, innerTag: String, outerTag: String) {
assertContext(
test,
"<$innerTag> must be found under the <$outerTag> tag"
)
}
private fun assertContext(test: Boolean, message: String) {
if (!test) {
throw IllegalStateException("Invalid dependency verification metadata file: $message")
}
}
}
}
}

View File

@@ -1,23 +0,0 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.metadata
import org.nixos.gradle2nix.dependency.ModuleComponentIdentifier
class DependencyVerifier internal constructor(
val verificationMetadata: Map<ModuleComponentIdentifier, ComponentVerificationMetadata>,
val configuration: DependencyVerificationConfiguration,
)

View File

@@ -1,162 +0,0 @@
/*
* Copyright 2019 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package org.nixos.gradle2nix.metadata
import org.nixos.gradle2nix.dependency.ModuleComponentArtifactIdentifier
import org.nixos.gradle2nix.dependency.ModuleComponentIdentifier
class DependencyVerifierBuilder {
private val byComponent: MutableMap<ModuleComponentIdentifier, ComponentVerificationsBuilder> = mutableMapOf()
private val trustedArtifacts: MutableList<DependencyVerificationConfiguration.TrustCoordinates> = mutableListOf()
fun addChecksum(
artifact: ModuleComponentArtifactIdentifier,
kind: ChecksumKind,
value: String,
origin: String?,
reason: String?
) {
val componentIdentifier: ModuleComponentIdentifier = artifact.componentIdentifier
byComponent.getOrPut(componentIdentifier) {
ComponentVerificationsBuilder(componentIdentifier)
}.addChecksum(artifact, kind, value, origin, reason)
}
@JvmOverloads
fun addTrustedArtifact(
group: String?,
name: String?,
version: String?,
fileName: String?,
regex: Boolean,
reason: String? = null
) {
validateUserInput(group, name, version, fileName)
trustedArtifacts.add(DependencyVerificationConfiguration.TrustCoordinates(group, name, version, fileName, regex, reason))
}
private fun validateUserInput(
group: String?,
name: String?,
version: String?,
fileName: String?
) {
// because this can be called from parsing XML, we need to perform additional verification
if (group == null && name == null && version == null && fileName == null) {
throw IllegalStateException("A trusted artifact must have at least one of group, name, version or file name not null")
}
}
fun build(): DependencyVerifier {
return DependencyVerifier(
byComponent
.toSortedMap(
compareBy<ModuleComponentIdentifier> { it.group }
.thenBy { it.module }
.thenBy { it.version }
)
.mapValues { it.value.build() },
DependencyVerificationConfiguration(trustedArtifacts),
)
}
private class ComponentVerificationsBuilder(private val component: ModuleComponentIdentifier) {
private val byArtifact: MutableMap<String, ArtifactVerificationBuilder> = mutableMapOf()
fun addChecksum(
artifact: ModuleComponentArtifactIdentifier,
kind: ChecksumKind,
value: String,
origin: String?,
reason: String?
) {
byArtifact.computeIfAbsent(artifact.fileName) { ArtifactVerificationBuilder() }
.addChecksum(kind, value, origin, reason)
}
fun build(): ComponentVerificationMetadata {
return ComponentVerificationMetadata(
component,
byArtifact
.map { ArtifactVerificationMetadata(it.key, it.value.buildChecksums()) }
.sortedBy { it.artifactName }
)
}
}
protected class ArtifactVerificationBuilder {
private val builder: MutableMap<ChecksumKind, ChecksumBuilder> = mutableMapOf()
fun addChecksum(kind: ChecksumKind, value: String, origin: String?, reason: String?) {
val builder = builder.getOrPut(kind) {
ChecksumBuilder(kind)
}
builder.addChecksum(value)
if (origin != null) {
builder.withOrigin(origin)
}
if (reason != null) {
builder.withReason(reason)
}
}
fun buildChecksums(): List<Checksum> {
return builder.values
.map(ChecksumBuilder::build)
.sortedBy { it.kind }
}
}
private class ChecksumBuilder(private val kind: ChecksumKind) {
private var value: String? = null
private var origin: String? = null
private var reason: String? = null
private var alternatives: MutableSet<String> = mutableSetOf()
/**
* Sets the origin, if not set already. This is
* mostly used for automatic generation of checksums
*/
fun withOrigin(origin: String?) {
this.origin = this.origin ?: origin
}
/**
* Sets the reason, if not set already.
*/
fun withReason(reason: String?) {
this.reason = this.reason ?: reason
}
fun addChecksum(checksum: String) {
if (value == null) {
value = checksum
} else if (value != checksum) {
alternatives.add(checksum)
}
}
fun build(): Checksum {
return Checksum(
kind,
checkNotNull(value) { "Checksum is null" },
alternatives,
origin,
reason
)
}
}
}

View File

@@ -0,0 +1,142 @@
package org.nixos.gradle2nix.metadata
import java.io.File
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import nl.adaptivity.xmlutil.XmlStreaming
import nl.adaptivity.xmlutil.serialization.XML
import nl.adaptivity.xmlutil.serialization.XmlChildrenName
import nl.adaptivity.xmlutil.serialization.XmlElement
import nl.adaptivity.xmlutil.serialization.XmlSerialName
import org.nixos.gradle2nix.Logger
import org.nixos.gradle2nix.dependencygraph.model.DependencyCoordinates
sealed interface Coordinates {
val group: String?
val name: String?
val version: String?
val regex: Boolean
val file: String?
}
@Serializable
@SerialName("trust")
data class Trust(
override val group: String? = null,
override val name: String? = null,
override val version: String? = null,
override val regex: Boolean = false,
override val file: String? = null,
val reason: String? = null,
) : Coordinates
@Serializable
@SerialName("configuration")
data class Configuration(
@SerialName("verify-metadata") @XmlElement(true) val verifyMetadata: Boolean = false,
@SerialName("verify-signatures") @XmlElement(true) val verifySignatures: Boolean = false,
@SerialName("trusted-artifacts") @XmlChildrenName("trusted-artifacts") val trustedArtifacts: List<Trust> = emptyList()
)
@Serializable
sealed interface Checksum {
abstract val value: String
abstract val origin: String?
abstract val reason: String?
abstract val alternatives: List<String>
}
@Serializable
@SerialName("md5")
data class Md5(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("sha1")
data class Sha1(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("sha256")
data class Sha256(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("sha512")
data class Sha512(
override val value: String,
override val origin: String? = null,
override val reason: String? = null,
@XmlChildrenName("also-trust")
override val alternatives: List<String> = emptyList()
) : Checksum
@Serializable
@SerialName("artifact")
data class Artifact(
val name: String,
val md5: Md5? = null,
val sha1: Sha1? = null,
val sha256: Sha256? = null,
val sha512: Sha512? = null,
) {
val checksums: List<Checksum> by lazy { listOfNotNull(sha512, sha256, sha1, md5) }
}
@Serializable
@SerialName("component")
data class Component(
val group: String,
val name: String,
val version: String,
val artifacts: List<Artifact> = emptyList(),
) {
constructor(coordinates: DependencyCoordinates, artifacts: List<Artifact>) : this(
coordinates.group,
coordinates.module,
coordinates.version,
artifacts
)
}
@Serializable
@XmlSerialName(
"verification-metadata",
namespace = "https://schema.gradle.org/dependency-verification",
prefix = ""
)
data class VerificationMetadata(
val configuration: Configuration = Configuration(),
@XmlChildrenName("components", "https://schema.gradle.org/dependency-verification") val components: List<Component> = emptyList()
)
val XmlFormat = XML {
autoPolymorphic = true
recommended()
}
fun parseVerificationMetadata(logger: Logger, metadata: File): VerificationMetadata? {
return try {
metadata.reader().buffered().let(XmlStreaming::newReader).use { input ->
XmlFormat.decodeFromReader(input)
}
} catch (e: Exception) {
logger.warn("$metadata: failed to parse Gradle dependency verification metadata", e)
return null
}
}

View File

@@ -0,0 +1,102 @@
package org.nixos.gradle2nix.module
import kotlinx.serialization.SerialName
import kotlinx.serialization.Serializable
import kotlinx.serialization.json.JsonObject
@Serializable
data class GradleModule(
val formatVersion: String,
val component: Component? = null,
val createdBy: CreatedBy? = null,
val variants: List<Variant> = emptyList(),
)
@Serializable
data class Component(
val group: String,
val module: String,
val version: String,
val url: String? = null,
)
@Serializable
data class Gradle(
val version: String,
val buildId: String? = null
)
@Serializable
data class CreatedBy(
val gradle: Gradle? = null
)
@Serializable
data class Variant(
val name: String,
val attributes: JsonObject? = null,
@SerialName("available-at") val availableAt: AvailableAt? = null,
val dependencies: List<Dependency> = emptyList(),
val dependencyConstraints: List<DependencyConstraint> = emptyList(),
val files: List<VariantFile> = emptyList(),
val capabilities: List<Capability> = emptyList()
)
@Serializable
data class AvailableAt(
val url: String,
val group: String,
val module: String,
val version: String,
)
@Serializable
data class Dependency(
val group: String,
val module: String,
val version: JsonObject? = null,
val excludes: List<Exclude> = emptyList(),
val reason: String? = null,
val attributes: JsonObject? = null,
val requestedCapabilities: List<Capability> = emptyList(),
val endorseStrictVersions: Boolean = false,
val thirdPartyCompatibility: ThirdPartyCompatibility? = null,
)
@Serializable
data class DependencyConstraint(
val group: String,
val module: String,
val version: JsonObject? = null,
val reason: String? = null,
val attributes: JsonObject? = null,
)
@Serializable
data class VariantFile(
val name: String,
val url: String,
val size: Long,
val sha1: String? = null,
val sha256: String? = null,
val sha512: String? = null,
val md5: String? = null,
)
@Serializable
data class Capability(
val group: String,
val name: String,
val version: String? = null,
)
@Serializable
data class Exclude(
val group: String,
val module: String,
)
@Serializable
data class ThirdPartyCompatibility(
val artifactSelector: String
)

View File

@@ -38,6 +38,12 @@ private val json = Json {
prettyPrintIndent = " "
}
val testLogger = Logger(verbose = true, stacktrace = true)
fun fixture(path: String): File {
return Paths.get("../fixtures", path).toFile()
}
@OptIn(ExperimentalKotest::class, ExperimentalSerializationApi::class, KotestInternal::class)
suspend fun TestScope.fixture(
project: String,

View File

@@ -0,0 +1,17 @@
package org.nixos.gradle2nix
import io.kotest.core.spec.style.FunSpec
import io.kotest.matchers.nulls.beNull
import io.kotest.matchers.nulls.shouldNotBeNull
import io.kotest.matchers.shouldNot
import kotlinx.serialization.decodeFromString
import org.nixos.gradle2nix.metadata.Artifact
import org.nixos.gradle2nix.metadata.XmlFormat
import org.nixos.gradle2nix.metadata.parseVerificationMetadata
class VerificationMetadataTest : FunSpec({
test("parses verification metadata") {
val metadata = parseVerificationMetadata(testLogger, fixture("metadata/verification-metadata.xml"))
metadata shouldNot beNull()
}
})