diff --git a/TestTools/README.md b/TestTools/README.md
index 723a2d834..7bdbbfef1 100644
--- a/TestTools/README.md
+++ b/TestTools/README.md
@@ -37,6 +37,24 @@ net4x. Getting these right matters - binding a WPF assembly against the stub fac
`NETCore.App.Ref` collapses whole type hierarchies to `Unknown` and invents hundreds of bogus
warnings, so treat a sudden warning spike as a reference problem until proven otherwise.
+Every finding therefore carries the assembly it came from and the references that assembly was
+decompiled against - the search directories in priority order, and what each reference resolved
+to - so that judgement can be made from the report rather than from a second run, and so a
+finding can be reproduced by hand:
+
+```
+assembly: ~/.cache/nugetfuzz/fluentvalidation/12.1.1/lib/net8.0/FluentValidation.dll
+ -r ~/.cache/nugetfuzz/fluentvalidation/12.1.1/lib/net8.0
+ -r ~/.cache/nugetfuzz/microsoft.netcore.app.ref/8.0.31/ref/net8.0
+ System.Runtime, Version=8.0.0.0, ... -> ~/.cache/nugetfuzz/microsoft.netcore.app.ref/8.0.31/ref/net8.0/System.Runtime.dll
+ Some.Package, Version=1.0.0.0, ... -> NOT FOUND
+```
+
+The reference set is recorded once the assembly is finished, not when the finding is first hit:
+the resolver keeps discovering references for as long as it decompiles. This is the same data
+`NUGETFUZZ_VERBOSE` prints, and it is the bulk of a ledger line - findings dedupe, so it is
+carried once per distinct finding, not once per hit.
+
Environment variables: `NUGETFUZZ_VERBOSE` (per-type progress), `NUGETFUZZ_DUMP=
` (write the
decompiled C#), `NUGETFUZZ_LEDGER=` (append findings as JSONL instead of writing a
per-run HTML report), `NUGETFUZZ_HTML=` (report path), `NUGET_PACKAGES` (package cache).
diff --git a/TestTools/nugetfuzz.cs b/TestTools/nugetfuzz.cs
index b1e001393..a2e9f54cc 100644
--- a/TestTools/nugetfuzz.cs
+++ b/TestTools/nugetfuzz.cs
@@ -137,6 +137,9 @@ var downloadOnly = args.Contains("--download-only");
var pdbMode = args.Contains("--pdb");
// Checks the PDB the assembly already ships with. Calibration, not a sweep.
var pdbLint = args.Contains("--pdb-lint");
+// Findings first hit by the assembly being decompiled. They get their reference context
+// once it is done, because the resolver keeps discovering references until then.
+var pendingContext = new List();
var arguments = args
.Where(a => !a.StartsWith("--"))
.SelectMany(a => a.StartsWith('@') ? File.ReadAllLines(a[1..]) : new[] { a })
@@ -623,7 +626,7 @@ async Task DecompileAssembly(string pkg, string dllPath, List searchDirs
if (pdbMode)
{
CheckGeneratedPdb(pkg, name, module, decompiler, dllPath, orderedDirs);
- ReportResolutions(logResolver);
+ ReportResolutions(logResolver, dllPath, orderedDirs);
return;
}
foreach (var type in decompiler.TypeSystem.MainModule.TopLevelTypeDefinitions.ToList())
@@ -677,7 +680,7 @@ async Task DecompileAssembly(string pkg, string dllPath, List searchDirs
Report(pkg, name, type.FullTypeName.ToString(), ex);
}
}
- ReportResolutions(logResolver);
+ ReportResolutions(logResolver, dllPath, orderedDirs);
}
}
@@ -688,10 +691,21 @@ string SyntaxTreeToString(SyntaxTree syntaxTree)
return w.ToString();
}
-void ReportResolutions(LoggingResolver logResolver)
+void ReportResolutions(LoggingResolver logResolver, string dllPath, List orderedDirs)
{
var resolutions = logResolver.Resolutions;
var unresolved = resolutions.Where(kv => kv.Value == null).Select(kv => kv.Key).OrderBy(k => k).ToList();
+ // What it takes to reproduce a finding by hand: the assembly it came from and the
+ // references it was decompiled against, which are what the report is read for once
+ // a warning turns out to be a reference problem rather than a decompiler defect.
+ var context = new StringBuilder($"assembly: {dllPath}");
+ foreach (var dir in orderedDirs)
+ context.Append($"{Environment.NewLine} -r {dir}");
+ foreach (var (refName, refPath) in resolutions.OrderBy(kv => kv.Key, StringComparer.Ordinal))
+ context.Append($"{Environment.NewLine} {refName} -> {refPath ?? "NOT FOUND"}");
+ foreach (var key in pendingContext)
+ failures[key] = failures[key] with { Context = context.ToString() };
+ pendingContext.Clear();
refsTotal += resolutions.Count;
refsResolved += resolutions.Count - unresolved.Count;
Console.WriteLine($" refs: {resolutions.Count - unresolved.Count}/{resolutions.Count} resolved");
@@ -1307,6 +1321,7 @@ void Report(string pkg, string asm, string type, Exception ex)
{
failures[key] = new Finding(kind, inner.GetType().Name, FirstLine(inner.Message),
FirstLine(topFrame), location, ex.ToString(), 1);
+ pendingContext.Add(key);
Console.WriteLine($" [{kind}] {location}");
foreach (var line in ex.ToString().Split('\n').Take(30))
Console.WriteLine(" " + line.TrimEnd());
@@ -1327,7 +1342,7 @@ static void AppendToLedger(string path, IEnumerable findings, int assem
// often than a decompiler defect ("might be due to ... missing references" is what
// the warning itself says), and the report separates the two on this basis.
lines.Add(JsonSerializer.Serialize(new LedgerEntry("finding", f.Kind, f.ExceptionType, f.Message,
- f.Frame, f.FirstLocation, f.Detail, f.Count, 0, 0, refsResolved, refsTotal)));
+ f.Frame, f.FirstLocation, f.Detail, f.Count, 0, 0, refsResolved, refsTotal, f.Context)));
}
lines.Add(JsonSerializer.Serialize(new LedgerEntry("totals", "", "", "", "", "", "", 0,
assemblies, types, refsResolved, refsTotal)));
@@ -1388,7 +1403,7 @@ static void RenderLedger(string ledgerPath, string outPath)
merged[key] = merged.TryGetValue(key, out var existing)
? existing with { Count = existing.Count + entry.Count }
: new Finding(entry.Kind, entry.ExceptionType, entry.Message, entry.Frame,
- entry.FirstLocation, entry.Detail, entry.Count);
+ entry.FirstLocation, entry.Detail, entry.Count, entry.Context);
// Ledger lines written before this attribution existed carry 0/0; treat those as
// unknown rather than clean, so they are never presented as confirmed defects.
(entry.RefsTotal > 0 && entry.RefsResolved == entry.RefsTotal ? clean : degraded).Add(key);
@@ -1453,7 +1468,8 @@ static void WriteHtmlReport(string path, List findings, int assemblies,
: "";
html.AppendLine($"{f.Count}x "
+ $"{Esc(f.ExceptionType)}: {Esc(f.Message)}{suspect}
");
- html.AppendLine($"first: {Esc(f.FirstLocation)}\nframe: {Esc(f.Frame)}\n\n{Esc(f.Detail)} ");
+ var context = f.Context.Length > 0 ? $"{Esc(f.Context)}\n" : "";
+ html.AppendLine($"first: {Esc(f.FirstLocation)}\nframe: {Esc(f.Frame)}\n{context}\n{Esc(f.Detail)}");
}
}
html.AppendLine("""
@@ -1482,7 +1498,7 @@ static string FirstLine(string s)
// One deduplicated defect: Count counts every location that hit it, Detail keeps the
// full exception text of the first one for triage.
record Finding(string Kind, string ExceptionType, string Message, string Frame,
- string FirstLocation, string Detail, int Count)
+ string FirstLocation, string Detail, int Count, string Context = "")
{
public string Describe()
=> $"[{Kind}] {ExceptionType}: {Message} @ {Frame} (first: {FirstLocation})";
@@ -1491,7 +1507,7 @@ record Finding(string Kind, string ExceptionType, string Message, string Frame,
// One line of the sweep ledger: either a deduplicated finding or a per-run totals record.
record LedgerEntry(string Record, string Kind, string ExceptionType, string Message, string Frame,
string FirstLocation, string Detail, int Count, int Assemblies, int Types,
- long RefsResolved, long RefsTotal);
+ long RefsResolved, long RefsTotal, string Context = "");
record VersionIndex(string[] versions);