Record the assembly and its references on every nugetfuzz finding

A finding named the package, the assembly's file name and the type, which is
not enough to reproduce it: the assembly lives in a version-specific cache
directory, and the references it was decompiled against are the whole question
whenever a warning turns out to be a reference problem rather than a defect.
That judgement had to be made by rerunning the package under NUGETFUZZ_VERBOSE
and reading the console, which the ledger of a catalog sweep cannot offer at
all.

The reference set is captured once the assembly is finished rather than when a
finding is first hit. The resolver keeps discovering references for as long as
it decompiles, so a finding from the first type would otherwise record a set
that is mostly still empty.

Ledger lines written before this field existed deserialize with it empty and
render as they did.

Assisted-by: Claude:claude-opus-5:Claude Code

TestTools/README.md

@@ -37,6 +37,24 @@ net4x. Getting these right matters - binding a WPF assembly against the stub fac
 `NETCore.App.Ref` collapses whole type hierarchies to `Unknown` and invents hundreds of bogus
 warnings, so treat a sudden warning spike as a reference problem until proven otherwise.
 
+Every finding therefore carries the assembly it came from and the references that assembly was
+decompiled against - the search directories in priority order, and what each reference resolved
+to - so that judgement can be made from the report rather than from a second run, and so a
+finding can be reproduced by hand:
+
+```
+assembly: ~/.cache/nugetfuzz/fluentvalidation/12.1.1/lib/net8.0/FluentValidation.dll
+  -r ~/.cache/nugetfuzz/fluentvalidation/12.1.1/lib/net8.0
+  -r ~/.cache/nugetfuzz/microsoft.netcore.app.ref/8.0.31/ref/net8.0
+  System.Runtime, Version=8.0.0.0, ... -> ~/.cache/nugetfuzz/microsoft.netcore.app.ref/8.0.31/ref/net8.0/System.Runtime.dll
+  Some.Package, Version=1.0.0.0, ...  -> NOT FOUND
+```
+
+The reference set is recorded once the assembly is finished, not when the finding is first hit:
+the resolver keeps discovering references for as long as it decompiles. This is the same data
+`NUGETFUZZ_VERBOSE` prints, and it is the bulk of a ledger line - findings dedupe, so it is
+carried once per distinct finding, not once per hit.
+
 Environment variables: `NUGETFUZZ_VERBOSE` (per-type progress), `NUGETFUZZ_DUMP=<dir>` (write the
 decompiled C#), `NUGETFUZZ_LEDGER=<file>` (append findings as JSONL instead of writing a
 per-run HTML report), `NUGETFUZZ_HTML=<file>` (report path), `NUGET_PACKAGES` (package cache).

TestTools/nugetfuzz.cs

@@ -137,6 +137,9 @@ var downloadOnly = args.Contains("--download-only");
 var pdbMode = args.Contains("--pdb");
 // Checks the PDB the assembly already ships with. Calibration, not a sweep.
 var pdbLint = args.Contains("--pdb-lint");
+// Findings first hit by the assembly being decompiled. They get their reference context
+// once it is done, because the resolver keeps discovering references until then.
+var pendingContext = new List<string>();
 var arguments = args
 	.Where(a => !a.StartsWith("--"))
 	.SelectMany(a => a.StartsWith('@') ? File.ReadAllLines(a[1..]) : new[] { a })
@@ -623,7 +626,7 @@ async Task DecompileAssembly(string pkg, string dllPath, List<string> searchDirs
 		if (pdbMode)
 		{
 			CheckGeneratedPdb(pkg, name, module, decompiler, dllPath, orderedDirs);
-			ReportResolutions(logResolver);
+			ReportResolutions(logResolver, dllPath, orderedDirs);
 			return;
 		}
 		foreach (var type in decompiler.TypeSystem.MainModule.TopLevelTypeDefinitions.ToList())
@@ -677,7 +680,7 @@ async Task DecompileAssembly(string pkg, string dllPath, List<string> searchDirs
 				Report(pkg, name, type.FullTypeName.ToString(), ex);
 			}
 		}
-		ReportResolutions(logResolver);
+		ReportResolutions(logResolver, dllPath, orderedDirs);
 	}
 }
 
@@ -688,10 +691,21 @@ string SyntaxTreeToString(SyntaxTree syntaxTree)
 	return w.ToString();
 }
 
-void ReportResolutions(LoggingResolver logResolver)
+void ReportResolutions(LoggingResolver logResolver, string dllPath, List<string> orderedDirs)
 {
 	var resolutions = logResolver.Resolutions;
 	var unresolved = resolutions.Where(kv => kv.Value == null).Select(kv => kv.Key).OrderBy(k => k).ToList();
+	// What it takes to reproduce a finding by hand: the assembly it came from and the
+	// references it was decompiled against, which are what the report is read for once
+	// a warning turns out to be a reference problem rather than a decompiler defect.
+	var context = new StringBuilder($"assembly: {dllPath}");
+	foreach (var dir in orderedDirs)
+		context.Append($"{Environment.NewLine}  -r {dir}");
+	foreach (var (refName, refPath) in resolutions.OrderBy(kv => kv.Key, StringComparer.Ordinal))
+		context.Append($"{Environment.NewLine}  {refName} -> {refPath ?? "NOT FOUND"}");
+	foreach (var key in pendingContext)
+		failures[key] = failures[key] with { Context = context.ToString() };
+	pendingContext.Clear();
 	refsTotal += resolutions.Count;
 	refsResolved += resolutions.Count - unresolved.Count;
 	Console.WriteLine($"    refs: {resolutions.Count - unresolved.Count}/{resolutions.Count} resolved");
@@ -1307,6 +1321,7 @@ void Report(string pkg, string asm, string type, Exception ex)
 	{
 		failures[key] = new Finding(kind, inner.GetType().Name, FirstLine(inner.Message),
 			FirstLine(topFrame), location, ex.ToString(), 1);
+		pendingContext.Add(key);
 		Console.WriteLine($"  [{kind}] {location}");
 		foreach (var line in ex.ToString().Split('\n').Take(30))
 			Console.WriteLine("      " + line.TrimEnd());
@@ -1327,7 +1342,7 @@ static void AppendToLedger(string path, IEnumerable<Finding> findings, int assem
 		// often than a decompiler defect ("might be due to ... missing references" is what
 		// the warning itself says), and the report separates the two on this basis.
 		lines.Add(JsonSerializer.Serialize(new LedgerEntry("finding", f.Kind, f.ExceptionType, f.Message,
-			f.Frame, f.FirstLocation, f.Detail, f.Count, 0, 0, refsResolved, refsTotal)));
+			f.Frame, f.FirstLocation, f.Detail, f.Count, 0, 0, refsResolved, refsTotal, f.Context)));
 	}
 	lines.Add(JsonSerializer.Serialize(new LedgerEntry("totals", "", "", "", "", "", "", 0,
 		assemblies, types, refsResolved, refsTotal)));
@@ -1388,7 +1403,7 @@ static void RenderLedger(string ledgerPath, string outPath)
 		merged[key] = merged.TryGetValue(key, out var existing)
 			? existing with { Count = existing.Count + entry.Count }
 			: new Finding(entry.Kind, entry.ExceptionType, entry.Message, entry.Frame,
-				entry.FirstLocation, entry.Detail, entry.Count);
+				entry.FirstLocation, entry.Detail, entry.Count, entry.Context);
 		// Ledger lines written before this attribution existed carry 0/0; treat those as
 		// unknown rather than clean, so they are never presented as confirmed defects.
 		(entry.RefsTotal > 0 && entry.RefsResolved == entry.RefsTotal ? clean : degraded).Add(key);
@@ -1453,7 +1468,8 @@ static void WriteHtmlReport(string path, List<Finding> findings, int assemblies,
 				: "";
 			html.AppendLine($"<details class={kind}><summary><span class=count>{f.Count}x</span> "
 				+ $"{Esc(f.ExceptionType)}: {Esc(f.Message)}{suspect}</summary>");
-			html.AppendLine($"<pre>first: {Esc(f.FirstLocation)}\nframe: {Esc(f.Frame)}\n\n{Esc(f.Detail)}</pre></details>");
+			var context = f.Context.Length > 0 ? $"{Esc(f.Context)}\n" : "";
+			html.AppendLine($"<pre>first: {Esc(f.FirstLocation)}\nframe: {Esc(f.Frame)}\n{context}\n{Esc(f.Detail)}</pre></details>");
 		}
 	}
 	html.AppendLine("""
@@ -1482,7 +1498,7 @@ static string FirstLine(string s)
 // One deduplicated defect: Count counts every location that hit it, Detail keeps the
 // full exception text of the first one for triage.
 record Finding(string Kind, string ExceptionType, string Message, string Frame,
-	string FirstLocation, string Detail, int Count)
+	string FirstLocation, string Detail, int Count, string Context = "")
 {
 	public string Describe()
 		=> $"[{Kind}] {ExceptionType}: {Message} @ {Frame}  (first: {FirstLocation})";
@@ -1491,7 +1507,7 @@ record Finding(string Kind, string ExceptionType, string Message, string Frame,
 // One line of the sweep ledger: either a deduplicated finding or a per-run totals record.
 record LedgerEntry(string Record, string Kind, string ExceptionType, string Message, string Frame,
 	string FirstLocation, string Detail, int Count, int Assemblies, int Types,
-	long RefsResolved, long RefsTotal);
+	long RefsResolved, long RefsTotal, string Context = "");
 
 record VersionIndex(string[] versions);