WIP

plugin/src/main/kotlin/org/nixos/gradle2nix/NixDependencyGraphRenderer.kt

@@ -5,7 +5,7 @@ import kotlinx.serialization.ExperimentalSerializationApi
 import kotlinx.serialization.json.Json
 import kotlinx.serialization.json.encodeToStream
 import org.nixos.gradle2nix.dependencygraph.DependencyGraphRenderer
-import org.nixos.gradle2nix.dependencygraph.model.ResolvedConfiguration
+import org.nixos.gradle2nix.model.ResolvedConfiguration
 
 @OptIn(ExperimentalSerializationApi::class)
 private val json = Json {

plugin/src/main/kotlin/org/nixos/gradle2nix/dependencygraph/AbstractDependencyExtractorPlugin.kt

@@ -1,15 +1,26 @@
 package org.nixos.gradle2nix.dependencygraph
 
 import org.gradle.api.Plugin
+import org.gradle.api.internal.GradleInternal
+import org.gradle.api.internal.project.DefaultProjectRegistry
+import org.gradle.api.internal.project.ProjectInternal
+import org.gradle.api.internal.project.ProjectRegistry
 import org.gradle.api.invocation.Gradle
+import org.gradle.api.logging.Logging
 import org.gradle.api.provider.Provider
+import org.gradle.api.services.internal.RegisteredBuildServiceProvider
+import org.gradle.internal.build.BuildProjectRegistry
 import org.gradle.internal.build.event.BuildEventListenerRegistryInternal
+import org.gradle.internal.composite.IncludedBuildInternal
+import org.gradle.internal.operations.BuildOperationAncestryTracker
+import org.gradle.internal.operations.BuildOperationListenerManager
 import org.gradle.util.GradleVersion
 import org.nixos.gradle2nix.dependencygraph.extractor.DependencyExtractor
 import org.nixos.gradle2nix.dependencygraph.extractor.DependencyExtractorBuildService
 import org.nixos.gradle2nix.dependencygraph.extractor.LegacyDependencyExtractor
 import org.nixos.gradle2nix.dependencygraph.util.buildDirCompat
 import org.nixos.gradle2nix.dependencygraph.util.service
+import org.nixos.gradle2nix.model.ConfigurationTarget
 
 abstract class AbstractDependencyExtractorPlugin : Plugin<Gradle> {
     // Register extension functions on `Gradle` type
@@ -39,6 +50,15 @@ abstract class AbstractDependencyExtractorPlugin : Plugin<Gradle> {
                 .rootProjectBuildDirectory = project.buildDirCompat
         }
 
+        val logger = Logging.getLogger(AbstractDependencyExtractorPlugin::class.java.name)
+
+        gradle.projectsLoaded {
+            (gradle as GradleInternal).let { g ->
+                logger.lifecycle("all projects: ${g.owner.projects.allProjects}")
+                logger.lifecycle("included projects: ${g.includedBuilds().flatMap { it.target.projects.allProjects }.joinToString { it.identityPath.path }}")
+            }
+        }
+
         // Register the service to listen for Build Events
         applicatorStrategy.registerExtractorListener(gradle, dependencyExtractorProvider)
 
@@ -122,7 +142,6 @@ abstract class AbstractDependencyExtractorPlugin : Plugin<Gradle> {
                 gradle: Gradle,
                 extractorServiceProvider: Provider<out DependencyExtractor>
             ) {
-                // No-op as DependencyExtractorService is Auto-Closable
             }
         }
     }

plugin/src/main/kotlin/org/nixos/gradle2nix/dependencygraph/DependencyGraphRenderer.kt

@@ -1,7 +1,7 @@
 package org.nixos.gradle2nix.dependencygraph
 
 import java.io.File
-import org.nixos.gradle2nix.dependencygraph.model.ResolvedConfiguration
+import org.nixos.gradle2nix.model.ResolvedConfiguration
 
 interface DependencyGraphRenderer {
     fun outputDependencyGraph(

plugin/src/main/kotlin/org/nixos/gradle2nix/dependencygraph/extractor/DependencyExtractor.kt

@@ -3,12 +3,27 @@ package org.nixos.gradle2nix.dependencygraph.extractor
 import java.io.File
 import java.net.URI
 import java.util.Collections
+import java.util.concurrent.ConcurrentHashMap
 import org.gradle.api.GradleException
+import org.gradle.api.artifacts.DependencyResolutionListener
+import org.gradle.api.artifacts.ResolvableDependencies
+import org.gradle.api.artifacts.component.BuildIdentifier
+import org.gradle.api.artifacts.component.ModuleComponentIdentifier
+import org.gradle.api.artifacts.query.ArtifactResolutionQuery
+import org.gradle.api.artifacts.result.ResolvedArtifactResult
 import org.gradle.api.artifacts.result.ResolvedComponentResult
 import org.gradle.api.artifacts.result.ResolvedDependencyResult
+import org.gradle.api.component.Artifact
+import org.gradle.api.internal.artifacts.DefaultModuleVersionIdentifier
 import org.gradle.api.internal.artifacts.DefaultProjectComponentIdentifier
 import org.gradle.api.internal.artifacts.configurations.ResolveConfigurationDependenciesBuildOperationType
+import org.gradle.api.internal.artifacts.repositories.resolver.MavenUniqueSnapshotComponentIdentifier
 import org.gradle.api.logging.Logging
+import org.gradle.configuration.ApplyScriptPluginBuildOperationType
+import org.gradle.configuration.ConfigurationTargetIdentifier
+import org.gradle.initialization.LoadBuildBuildOperationType
+import org.gradle.internal.component.external.model.DefaultModuleComponentIdentifier
+import org.gradle.internal.component.external.model.ModuleComponentArtifactIdentifier
 import org.gradle.internal.exceptions.DefaultMultiCauseException
 import org.gradle.internal.operations.BuildOperationDescriptor
 import org.gradle.internal.operations.BuildOperationListener
@@ -16,27 +31,44 @@ import org.gradle.internal.operations.OperationFinishEvent
 import org.gradle.internal.operations.OperationIdentifier
 import org.gradle.internal.operations.OperationProgressEvent
 import org.gradle.internal.operations.OperationStartEvent
-import org.nixos.gradle2nix.PARAM_INCLUDE_CONFIGURATIONS
-import org.nixos.gradle2nix.PARAM_INCLUDE_PROJECTS
-import org.nixos.gradle2nix.PARAM_REPORT_DIR
+import org.gradle.ivy.IvyDescriptorArtifact
+import org.gradle.jvm.JvmLibrary
+import org.gradle.language.base.artifact.SourcesArtifact
+import org.gradle.language.java.artifact.JavadocArtifact
+import org.gradle.maven.MavenPomArtifact
+import org.gradle.util.GradleVersion
 import org.nixos.gradle2nix.dependencygraph.DependencyGraphRenderer
-import org.nixos.gradle2nix.DependencyCoordinates
-import org.nixos.gradle2nix.dependencygraph.model.DependencySource
-import org.nixos.gradle2nix.dependencygraph.model.Repository
-import org.nixos.gradle2nix.dependencygraph.model.ResolvedConfiguration
-import org.nixos.gradle2nix.dependencygraph.model.ResolvedDependency
+import org.nixos.gradle2nix.dependencygraph.util.BuildOperationTracker
 import org.nixos.gradle2nix.dependencygraph.util.loadOptionalParam
+import org.nixos.gradle2nix.model.ConfigurationTarget
+import org.nixos.gradle2nix.model.DependencyCoordinates
+import org.nixos.gradle2nix.model.DependencySource
+import org.nixos.gradle2nix.model.PARAM_INCLUDE_CONFIGURATIONS
+import org.nixos.gradle2nix.model.PARAM_INCLUDE_PROJECTS
+import org.nixos.gradle2nix.model.PARAM_REPORT_DIR
+import org.nixos.gradle2nix.model.Repository
+import org.nixos.gradle2nix.model.ResolvedArtifact
+import org.nixos.gradle2nix.model.ResolvedConfiguration
+import org.nixos.gradle2nix.model.ResolvedDependency
 
 abstract class DependencyExtractor :
     BuildOperationListener,
     AutoCloseable {
 
+    private val configurations =
+        ConcurrentHashMap<
+                OperationIdentifier,
+                Pair<ResolveConfigurationDependenciesBuildOperationType.Details,
+                        ResolveConfigurationDependenciesBuildOperationType.Result>>()
+
     private val resolvedConfigurations = Collections.synchronizedList(mutableListOf<ResolvedConfiguration>())
 
     private val thrownExceptions = Collections.synchronizedList(mutableListOf<Throwable>())
 
     var rootProjectBuildDirectory: File? = null
 
+    private val operationTracker = BuildOperationTracker()
+
     // Properties are lazily initialized so that System Properties are initialized by the time
     // the values are used. This is required due to a bug in older Gradle versions. (https://github.com/gradle/gradle/issues/6825)
     private val configurationFilter by lazy {
@@ -52,37 +84,37 @@ abstract class DependencyExtractor :
 
     abstract fun getRendererClassName(): String
 
-    override fun started(buildOperation: BuildOperationDescriptor, startEvent: OperationStartEvent) {
-        // This method will never be called when registered in a `BuildServiceRegistry` (ie. Gradle 6.1 & higher)
-        // No-op
-    }
+    override fun started(buildOperation: BuildOperationDescriptor, startEvent: OperationStartEvent) {}
 
-    override fun progress(operationIdentifier: OperationIdentifier, progressEvent: OperationProgressEvent) {
-        // This method will never be called when registered in a `BuildServiceRegistry` (ie. Gradle 6.1 & higher)
-        // No-op
-    }
+    override fun progress(operationIdentifier: OperationIdentifier, progressEvent: OperationProgressEvent) {}
 
     override fun finished(buildOperation: BuildOperationDescriptor, finishEvent: OperationFinishEvent) {
-        handleBuildOperationType<
+        operationTracker.finished(buildOperation, finishEvent)
+
+        handleFinishBuildOperationType<
             ResolveConfigurationDependenciesBuildOperationType.Details,
             ResolveConfigurationDependenciesBuildOperationType.Result
-                >(buildOperation, finishEvent) { details, result -> extractConfigurationDependencies(details, result) }
+                >(buildOperation, finishEvent) { details, result ->
+            buildOperation.id?.let { operationId ->
+                configurations[operationId] = details to result
+            }
+        }
     }
 
-    private inline fun <reified D, reified R> handleBuildOperationType(
+    private inline fun <reified D, reified R> handleFinishBuildOperationType(
         buildOperation: BuildOperationDescriptor,
         finishEvent: OperationFinishEvent,
         handler: (details: D, result: R) -> Unit
     ) {
         try {
-            handleBuildOperationTypeRaw<D, R>(buildOperation, finishEvent, handler)
+            handleFinishBuildOperationTypeRaw<D, R>(buildOperation, finishEvent, handler)
         } catch (e: Throwable) {
             thrownExceptions.add(e)
             throw e
         }
     }
 
-    private inline fun <reified D, reified R> handleBuildOperationTypeRaw(
+    private inline fun <reified D, reified R> handleFinishBuildOperationTypeRaw(
         buildOperation: BuildOperationDescriptor,
         finishEvent: OperationFinishEvent,
         handler: (details: D, result: R) -> Unit
@@ -101,13 +133,28 @@ abstract class DependencyExtractor :
         handler(details, result)
     }
 
+    // This returns null for the root build, because the build operation won't complete until after close() is called.
+    private fun findBuildDetails(buildOperationId: OperationIdentifier?): LoadBuildBuildOperationType.Details? {
+        return operationTracker.findParent(buildOperationId) {
+            it.details as? LoadBuildBuildOperationType.Details
+        }
+    }
+
+    private fun processConfigurations() {
+        for ((operationId, data) in configurations) {
+            val (details, result) = data
+            extractConfigurationDependencies(operationId, details, result)
+        }
+    }
+
     private fun extractConfigurationDependencies(
+        operationId: OperationIdentifier,
         details: ResolveConfigurationDependenciesBuildOperationType.Details,
         result: ResolveConfigurationDependenciesBuildOperationType.Result
     ) {
         val repositories = details.repositories?.mapNotNull {
             @Suppress("UNCHECKED_CAST")
-            Repository(
+            (Repository(
                 id = it.id,
                 type = enumValueOf(it.type),
                 name = it.name,
@@ -115,35 +162,53 @@ abstract class DependencyExtractor :
                 metadataSources = (it.properties["METADATA_SOURCES"] as? List<String>) ?: emptyList(),
                 metadataResources = metadataResources(it),
                 artifactResources = artifactResources(it),
-            )
+            ))
         } ?: emptyList()
+
+        if (repositories.isEmpty()) {
+            return
+        }
+
         val rootComponent = result.rootComponent
 
         if (rootComponent.dependencies.isEmpty()) {
             // No dependencies to extract: can safely ignore
             return
         }
-        val projectIdentityPath = (rootComponent.id as? DefaultProjectComponentIdentifier)?.identityPath?.path
 
-        // TODO: At this point, any resolution not bound to a particular project will be assigned to the root "build :"
-        // This is because `details.buildPath` is always ':', which isn't correct in a composite build.
-        // It is possible to do better. By tracking the current build operation context, we can assign more precisely.
-        // See the Gradle Enterprise Build Scan Plugin: `ConfigurationResolutionCapturer_5_0`
-        val rootPath = projectIdentityPath ?: details.buildPath
-
-        if (!configurationFilter.include(rootPath, details.configurationName)) {
-            LOGGER.debug("Ignoring resolved configuration: $rootPath - ${details.configurationName}")
-            return
+        val source: DependencySource = when {
+            details.isScriptConfiguration -> {
+                val parent = operationTracker.findParent(operationId) {
+                    it.details as? ApplyScriptPluginBuildOperationType.Details
+                } ?: throw IllegalStateException("Couldn't find parent script operation for ${details.configurationName}")
+                DependencySource(
+                    targetType = when (parent.targetType) {
+                        ConfigurationTargetIdentifier.Type.GRADLE.label -> ConfigurationTarget.GRADLE
+                        ConfigurationTargetIdentifier.Type.SETTINGS.label -> ConfigurationTarget.SETTINGS
+                        ConfigurationTargetIdentifier.Type.PROJECT.label -> ConfigurationTarget.BUILDSCRIPT
+                        else -> throw IllegalStateException("Unknown configuration target type: ${parent.targetType}")
+                    },
+                    targetPath = parent.targetPath ?: ":",
+                    buildPath = parent.buildPath!!
+                )
+            }
+            else -> {
+                DependencySource(
+                    targetType = ConfigurationTarget.PROJECT,
+                    targetPath = details.projectPath!!,
+                    buildPath = details.buildPath
+                )
+            }
         }
 
-        val rootId = if (projectIdentityPath == null) "build $rootPath" else componentId(rootComponent)
-        val rootSource = DependencySource(rootId, rootPath)
-        val resolvedConfiguration = ResolvedConfiguration(rootSource, details.configurationName, repositories)
+
+        val resolvedConfiguration = ResolvedConfiguration(source, details.configurationName, repositories)
 
         for (directDependency in getResolvedDependencies(rootComponent)) {
+            val moduleComponentId = directDependency.id as? ModuleComponentIdentifier ?: continue
             val directDep = createComponentNode(
-                componentId(directDependency),
-                rootSource,
+                moduleComponentId,
+                source,
                 true,
                 directDependency,
                 result.getRepositoryId(directDependency)
@@ -167,16 +232,18 @@ abstract class DependencyExtractor :
 
         val dependencyComponents = getResolvedDependencies(component)
         for (dependencyComponent in dependencyComponents) {
-            val dependencyId = componentId(dependencyComponent)
-            if (!resolvedConfiguration.hasDependency(dependencyId)) {
-                val dependencyNode = createComponentNode(
-                    dependencyId,
-                    componentSource,
-                    direct,
-                    dependencyComponent,
-                    result.getRepositoryId(component)
-                )
-                resolvedConfiguration.addDependency(dependencyNode)
+            if (!resolvedConfiguration.hasDependency(componentId(dependencyComponent))) {
+                val moduleComponentId = dependencyComponent.id as? ModuleComponentIdentifier
+                if (moduleComponentId != null) {
+                    val dependencyNode = createComponentNode(
+                        moduleComponentId,
+                        componentSource,
+                        direct,
+                        dependencyComponent,
+                        result.getRepositoryId(component)
+                    )
+                    resolvedConfiguration.addDependency(dependencyNode)
+                }
 
                 walkComponentDependencies(result, dependencyComponent, componentSource, resolvedConfiguration)
             }
@@ -186,24 +253,40 @@ abstract class DependencyExtractor :
     private fun getSource(component: ResolvedComponentResult, source: DependencySource): DependencySource {
         val componentId = component.id
         if (componentId is DefaultProjectComponentIdentifier) {
-            return DependencySource(componentId(component), componentId.identityPath.path)
+            return DependencySource(
+                ConfigurationTarget.PROJECT,
+                componentId.projectPath,
+                componentId.build.buildPathCompat
+            )
         }
         return source
     }
 
+    private val BuildIdentifier.buildPathCompat: String
+        @Suppress("DEPRECATION")
+        get() = if (GradleVersion.current() < GradleVersion.version("8.2")) name else buildPath
+
     private fun getResolvedDependencies(component: ResolvedComponentResult): List<ResolvedComponentResult> {
         return component.dependencies.filterIsInstance<ResolvedDependencyResult>().map { it.selected }.filter { it != component }
     }
 
-    private fun createComponentNode(componentId: String, source: DependencySource, direct: Boolean, component: ResolvedComponentResult, repositoryId: String?): ResolvedDependency {
-        val componentDependencies = component.dependencies.filterIsInstance<ResolvedDependencyResult>().map { componentId(it.selected) }
+    private fun createComponentNode(
+        componentId: ModuleComponentIdentifier,
+        source: DependencySource,
+        direct: Boolean,
+        component: ResolvedComponentResult,
+        repositoryId: String?
+    ): ResolvedDependency {
+        val componentDependencies =
+            component.dependencies.filterIsInstance<ResolvedDependencyResult>().map { componentId(it.selected) }
+        val coordinates = coordinates(componentId)
         return ResolvedDependency(
-            componentId,
+            componentId.displayName,
             source,
             direct,
-            coordinates(component),
+            coordinates,
             repositoryId,
-            componentDependencies
+            componentDependencies,
         )
     }
 
@@ -211,16 +294,25 @@ abstract class DependencyExtractor :
         return component.id.displayName
     }
 
-    private fun coordinates(component: ResolvedComponentResult): DependencyCoordinates {
-        // TODO: Consider and handle null moduleVersion
-        val moduleVersionIdentifier = component.moduleVersion!!
+    private fun coordinates(componentId: ModuleComponentIdentifier): DependencyCoordinates {
         return DependencyCoordinates(
-            moduleVersionIdentifier.group,
-            moduleVersionIdentifier.name,
-            moduleVersionIdentifier.version
+            componentId.group,
+            componentId.module,
+            componentId.version,
+            (componentId as? MavenUniqueSnapshotComponentIdentifier)?.timestamp
         )
     }
 
+    private fun artifactType(type: Class<out Artifact>): ResolvedArtifact.Type? {
+        return when (type) {
+            SourcesArtifact::class.java -> ResolvedArtifact.Type.SOURCES
+            JavadocArtifact::class.java -> ResolvedArtifact.Type.JAVADOC
+            IvyDescriptorArtifact::class.java -> ResolvedArtifact.Type.IVY_DESCRIPTOR
+            MavenPomArtifact::class.java -> ResolvedArtifact.Type.MAVEN_POM
+            else -> null
+        }
+    }
+
     private fun writeDependencyGraph() {
         val outputDirectory = getOutputDir()
         outputDirectory.mkdirs()
@@ -248,6 +340,8 @@ abstract class DependencyExtractor :
     }
 
     override fun close() {
+        LOGGER.lifecycle("DependencyExtractor: CLOSE")
+
         if (thrownExceptions.isNotEmpty()) {
             throw DefaultMultiCauseException(
                 "The Gradle2Nix plugin encountered errors while extracting dependencies. " +
@@ -256,6 +350,10 @@ abstract class DependencyExtractor :
             )
         }
         try {
+            processConfigurations()
+
+            LOGGER.lifecycle("Resolved ${resolvedConfigurations.size} configurations.")
+
             writeDependencyGraph()
         } catch (e: RuntimeException) {
             throw GradleException(

plugin/src/main/kotlin/org/nixos/gradle2nix/dependencygraph/extractor/DependencyExtractorBuildService.kt

@@ -8,7 +8,6 @@ abstract class DependencyExtractorBuildService :
     DependencyExtractor(),
     BuildService<DependencyExtractorBuildService.Params>
 {
-    // Some parameters for the web server
     internal interface Params : BuildServiceParameters {
         val rendererClassName: Property<String>
     }

plugin/src/main/kotlin/org/nixos/gradle2nix/dependencygraph/util/BuildOperationTracker.kt

@@ -0,0 +1,64 @@
+package org.nixos.gradle2nix.dependencygraph.util
+
+import java.util.concurrent.ConcurrentHashMap
+import org.gradle.api.logging.Logging
+import org.gradle.internal.operations.BuildOperation
+import org.gradle.internal.operations.BuildOperationDescriptor
+import org.gradle.internal.operations.BuildOperationListener
+import org.gradle.internal.operations.OperationFinishEvent
+import org.gradle.internal.operations.OperationIdentifier
+import org.gradle.internal.operations.OperationProgressEvent
+import org.gradle.internal.operations.OperationStartEvent
+
+class BuildOperationTracker : BuildOperationListener {
+    private val _parents: MutableMap<OperationIdentifier, OperationIdentifier?> = ConcurrentHashMap()
+    private val _operations: MutableMap<OperationIdentifier, BuildOperationDescriptor> = ConcurrentHashMap()
+    private val _results: MutableMap<OperationIdentifier, Any> = ConcurrentHashMap()
+
+    val parents: Map<OperationIdentifier, OperationIdentifier?> get() = _parents
+    val operations: Map<OperationIdentifier, BuildOperationDescriptor> get() = _operations
+    val results: Map<OperationIdentifier, Any> get() = _results
+
+    override fun started(buildOperation: BuildOperationDescriptor, startEvent: OperationStartEvent) {
+    }
+
+    override fun progress(operationIdentifier: OperationIdentifier, progressEvent: OperationProgressEvent) {
+    }
+
+    override fun finished(buildOperation: BuildOperationDescriptor, finishEvent: OperationFinishEvent) {
+        val id = buildOperation.id ?: return
+        _parents[id] = buildOperation.parentId
+        _operations[id] = buildOperation
+    }
+
+    tailrec fun <T> findParent(id: OperationIdentifier?, block: (BuildOperationDescriptor) -> T?): T? {
+        if (id == null) return null
+        val operation = _operations[id] ?: return null.also {
+            LOGGER.lifecycle("no operation for $id")
+        }
+        return block(operation) ?: findParent(operation.parentId, block)
+    }
+
+    fun <T> findChild(id: OperationIdentifier?, block: (BuildOperationDescriptor) -> T?): T? {
+        if (id == null) return null
+        val operation = operations[id] ?: return null
+        block(operation)?.let { return it }
+        return children(id).firstNotNullOfOrNull { findChild(it, block) }
+    }
+
+    fun children(id: OperationIdentifier): Set<OperationIdentifier> {
+        return parents.filterValues { it == id }.keys
+    }
+
+    inline fun <reified T> getDetails(id: OperationIdentifier): T? {
+        return operations[id]?.details as? T
+    }
+
+    inline fun <reified T> getResult(id: OperationIdentifier): T? {
+        return results[id] as? T
+    }
+
+    companion object {
+        private val LOGGER = Logging.getLogger(BuildOperationTracker::class.qualifiedName!!)
+    }
+}

plugin/src/main/kotlin/org/nixos/gradle2nix/forceresolve/ForceDependencyResolutionPlugin.kt

@@ -6,8 +6,8 @@ import org.gradle.api.Task
 import org.gradle.api.invocation.Gradle
 import org.gradle.api.tasks.TaskProvider
 import org.gradle.util.GradleVersion
-import org.nixos.gradle2nix.RESOLVE_ALL_TASK
-import org.nixos.gradle2nix.RESOLVE_PROJECT_TASK
+import org.nixos.gradle2nix.model.RESOLVE_ALL_TASK
+import org.nixos.gradle2nix.model.RESOLVE_PROJECT_TASK
 
 // TODO: Rename these