From b13214d7ae92e830b30e0d3f60eb473e5aa50f31 Mon Sep 17 00:00:00 2001 From: Siegfried Pammer Date: Wed, 9 Sep 2026 10:16:10 +0200 Subject: [PATCH] Record the assembly and its references on every nugetfuzz finding A finding named the package, the assembly's file name and the type, which is not enough to reproduce it: the assembly lives in a version-specific cache directory, and the references it was decompiled against are the whole question whenever a warning turns out to be a reference problem rather than a defect. That judgement had to be made by rerunning the package under NUGETFUZZ_VERBOSE and reading the console, which the ledger of a catalog sweep cannot offer at all. The reference set is captured once the assembly is finished rather than when a finding is first hit. The resolver keeps discovering references for as long as it decompiles, so a finding from the first type would otherwise record a set that is mostly still empty. Ledger lines written before this field existed deserialize with it empty and render as they did. Assisted-by: Claude:claude-opus-5:Claude Code --- TestTools/README.md | 18 ++++++++++++++++++ TestTools/nugetfuzz.cs | 32 ++++++++++++++++++++++++-------- 2 files changed, 42 insertions(+), 8 deletions(-) diff --git a/TestTools/README.md b/TestTools/README.md index 723a2d834..7bdbbfef1 100644 --- a/TestTools/README.md +++ b/TestTools/README.md @@ -37,6 +37,24 @@ net4x. Getting these right matters - binding a WPF assembly against the stub fac `NETCore.App.Ref` collapses whole type hierarchies to `Unknown` and invents hundreds of bogus warnings, so treat a sudden warning spike as a reference problem until proven otherwise. +Every finding therefore carries the assembly it came from and the references that assembly was +decompiled against - the search directories in priority order, and what each reference resolved +to - so that judgement can be made from the report rather than from a second run, and so a +finding can be reproduced by hand: + +``` +assembly: ~/.cache/nugetfuzz/fluentvalidation/12.1.1/lib/net8.0/FluentValidation.dll + -r ~/.cache/nugetfuzz/fluentvalidation/12.1.1/lib/net8.0 + -r ~/.cache/nugetfuzz/microsoft.netcore.app.ref/8.0.31/ref/net8.0 + System.Runtime, Version=8.0.0.0, ... -> ~/.cache/nugetfuzz/microsoft.netcore.app.ref/8.0.31/ref/net8.0/System.Runtime.dll + Some.Package, Version=1.0.0.0, ... -> NOT FOUND +``` + +The reference set is recorded once the assembly is finished, not when the finding is first hit: +the resolver keeps discovering references for as long as it decompiles. This is the same data +`NUGETFUZZ_VERBOSE` prints, and it is the bulk of a ledger line - findings dedupe, so it is +carried once per distinct finding, not once per hit. + Environment variables: `NUGETFUZZ_VERBOSE` (per-type progress), `NUGETFUZZ_DUMP=` (write the decompiled C#), `NUGETFUZZ_LEDGER=` (append findings as JSONL instead of writing a per-run HTML report), `NUGETFUZZ_HTML=` (report path), `NUGET_PACKAGES` (package cache). diff --git a/TestTools/nugetfuzz.cs b/TestTools/nugetfuzz.cs index b1e001393..a2e9f54cc 100644 --- a/TestTools/nugetfuzz.cs +++ b/TestTools/nugetfuzz.cs @@ -137,6 +137,9 @@ var downloadOnly = args.Contains("--download-only"); var pdbMode = args.Contains("--pdb"); // Checks the PDB the assembly already ships with. Calibration, not a sweep. var pdbLint = args.Contains("--pdb-lint"); +// Findings first hit by the assembly being decompiled. They get their reference context +// once it is done, because the resolver keeps discovering references until then. +var pendingContext = new List(); var arguments = args .Where(a => !a.StartsWith("--")) .SelectMany(a => a.StartsWith('@') ? File.ReadAllLines(a[1..]) : new[] { a }) @@ -623,7 +626,7 @@ async Task DecompileAssembly(string pkg, string dllPath, List searchDirs if (pdbMode) { CheckGeneratedPdb(pkg, name, module, decompiler, dllPath, orderedDirs); - ReportResolutions(logResolver); + ReportResolutions(logResolver, dllPath, orderedDirs); return; } foreach (var type in decompiler.TypeSystem.MainModule.TopLevelTypeDefinitions.ToList()) @@ -677,7 +680,7 @@ async Task DecompileAssembly(string pkg, string dllPath, List searchDirs Report(pkg, name, type.FullTypeName.ToString(), ex); } } - ReportResolutions(logResolver); + ReportResolutions(logResolver, dllPath, orderedDirs); } } @@ -688,10 +691,21 @@ string SyntaxTreeToString(SyntaxTree syntaxTree) return w.ToString(); } -void ReportResolutions(LoggingResolver logResolver) +void ReportResolutions(LoggingResolver logResolver, string dllPath, List orderedDirs) { var resolutions = logResolver.Resolutions; var unresolved = resolutions.Where(kv => kv.Value == null).Select(kv => kv.Key).OrderBy(k => k).ToList(); + // What it takes to reproduce a finding by hand: the assembly it came from and the + // references it was decompiled against, which are what the report is read for once + // a warning turns out to be a reference problem rather than a decompiler defect. + var context = new StringBuilder($"assembly: {dllPath}"); + foreach (var dir in orderedDirs) + context.Append($"{Environment.NewLine} -r {dir}"); + foreach (var (refName, refPath) in resolutions.OrderBy(kv => kv.Key, StringComparer.Ordinal)) + context.Append($"{Environment.NewLine} {refName} -> {refPath ?? "NOT FOUND"}"); + foreach (var key in pendingContext) + failures[key] = failures[key] with { Context = context.ToString() }; + pendingContext.Clear(); refsTotal += resolutions.Count; refsResolved += resolutions.Count - unresolved.Count; Console.WriteLine($" refs: {resolutions.Count - unresolved.Count}/{resolutions.Count} resolved"); @@ -1307,6 +1321,7 @@ void Report(string pkg, string asm, string type, Exception ex) { failures[key] = new Finding(kind, inner.GetType().Name, FirstLine(inner.Message), FirstLine(topFrame), location, ex.ToString(), 1); + pendingContext.Add(key); Console.WriteLine($" [{kind}] {location}"); foreach (var line in ex.ToString().Split('\n').Take(30)) Console.WriteLine(" " + line.TrimEnd()); @@ -1327,7 +1342,7 @@ static void AppendToLedger(string path, IEnumerable findings, int assem // often than a decompiler defect ("might be due to ... missing references" is what // the warning itself says), and the report separates the two on this basis. lines.Add(JsonSerializer.Serialize(new LedgerEntry("finding", f.Kind, f.ExceptionType, f.Message, - f.Frame, f.FirstLocation, f.Detail, f.Count, 0, 0, refsResolved, refsTotal))); + f.Frame, f.FirstLocation, f.Detail, f.Count, 0, 0, refsResolved, refsTotal, f.Context))); } lines.Add(JsonSerializer.Serialize(new LedgerEntry("totals", "", "", "", "", "", "", 0, assemblies, types, refsResolved, refsTotal))); @@ -1388,7 +1403,7 @@ static void RenderLedger(string ledgerPath, string outPath) merged[key] = merged.TryGetValue(key, out var existing) ? existing with { Count = existing.Count + entry.Count } : new Finding(entry.Kind, entry.ExceptionType, entry.Message, entry.Frame, - entry.FirstLocation, entry.Detail, entry.Count); + entry.FirstLocation, entry.Detail, entry.Count, entry.Context); // Ledger lines written before this attribution existed carry 0/0; treat those as // unknown rather than clean, so they are never presented as confirmed defects. (entry.RefsTotal > 0 && entry.RefsResolved == entry.RefsTotal ? clean : degraded).Add(key); @@ -1453,7 +1468,8 @@ static void WriteHtmlReport(string path, List findings, int assemblies, : ""; html.AppendLine($"
{f.Count}x " + $"{Esc(f.ExceptionType)}: {Esc(f.Message)}{suspect}"); - html.AppendLine($"
first: {Esc(f.FirstLocation)}\nframe: {Esc(f.Frame)}\n\n{Esc(f.Detail)}
"); + var context = f.Context.Length > 0 ? $"{Esc(f.Context)}\n" : ""; + html.AppendLine($"
first: {Esc(f.FirstLocation)}\nframe: {Esc(f.Frame)}\n{context}\n{Esc(f.Detail)}
"); } } html.AppendLine(""" @@ -1482,7 +1498,7 @@ static string FirstLine(string s) // One deduplicated defect: Count counts every location that hit it, Detail keeps the // full exception text of the first one for triage. record Finding(string Kind, string ExceptionType, string Message, string Frame, - string FirstLocation, string Detail, int Count) + string FirstLocation, string Detail, int Count, string Context = "") { public string Describe() => $"[{Kind}] {ExceptionType}: {Message} @ {Frame} (first: {FirstLocation})"; @@ -1491,7 +1507,7 @@ record Finding(string Kind, string ExceptionType, string Message, string Frame, // One line of the sweep ledger: either a deduplicated finding or a per-run totals record. record LedgerEntry(string Record, string Kind, string ExceptionType, string Message, string Frame, string FirstLocation, string Detail, int Count, int Assemblies, int Types, - long RefsResolved, long RefsTotal); + long RefsResolved, long RefsTotal, string Context = ""); record VersionIndex(string[] versions);