From 7493c5f3d2fce54c3f026ba60215575f9e1136a4 Mon Sep 17 00:00:00 2001 From: Siegfried Pammer Date: Sat, 29 Aug 2026 22:24:41 +0200 Subject: [PATCH] Add nuget-top.ps1 to build a corpus from the most-downloaded packages Both tools need a corpus of real assemblies and neither had a way to get one: nugetfuzz-all.ps1 walks the catalog in publish order, which is fine for a crash sweep but makes a poor readability corpus, and the alternative was picking package ids by hand. The ids come from an empty search query, which orders by download count. Downloading them reuses nugetfuzz, which already resolves versions, matches target frameworks and walks the dependency closure into the same cache decompdiff reads; --download-only stops it before it decompiles, since the sweep is the expensive part and a corpus only needs the files. The result is a list of lib directories rather than a single root, because a package already restored on this machine is used from the machine-wide NuGet cache instead of being copied into ours, and a corpus that silently omitted those would misrepresent what was tested. Assisted-by: Claude:claude-opus-5[1m]:Claude Code --- TestTools/README.md | 25 +++++++++++++- TestTools/nuget-top.ps1 | 73 +++++++++++++++++++++++++++++++++++++++++ TestTools/nugetfuzz.cs | 21 ++++++++++-- 3 files changed, 115 insertions(+), 4 deletions(-) create mode 100644 TestTools/nuget-top.ps1 diff --git a/TestTools/README.md b/TestTools/README.md index e60639f6c..797736312 100644 --- a/TestTools/README.md +++ b/TestTools/README.md @@ -1,12 +1,13 @@ # TestTools -Two standalone tools that run the decompiler over real-world assemblies, to find defects the +Standalone tools that run the decompiler over real-world assemblies, to find defects the in-repo test suite cannot: it decompiles fixtures we wrote, these decompile what the world ships. | tool | question it answers | |---|---| | `nugetfuzz.cs` | Does the decompiler *crash* on real code? (asserts, exceptions, IL warnings) | | `decompdiff.cs` | Did a change make the *output* better or worse? (readability across two builds) | +| `nuget-top.ps1` | Where do I get a corpus? (downloads the most-downloaded packages) | Both are [file-based apps](https://learn.microsoft.com/dotnet/core/whats-new/dotnet-10/sdk#file-based-apps): single `.cs` files run directly by the SDK, no project, no solution entry. They are not built by @@ -55,6 +56,28 @@ including while the sweep is still running, with `--report`. Logs of failed runs `logs/`, successful ones are deleted. The package cache (`~/.cache/nugetfuzz`) is capped at 20 GB by default (`-CacheCapMB`) and pruned least-recently-used, because `decompdiff` uses it as a corpus. +## nuget-top + +Downloads the most-downloaded packages on nuget.org, with their dependency closures, and writes the +list of lib directories they resolved to - a ready-made corpus for either tool: + +```pwsh +./nuget-top.ps1 -Count 200 +./nuget-top.ps1 -Count 50 -Skip 200 # extend an existing corpus further down the ranking +./nuget-top.ps1 -Count 200 -ListOnly # just the ids, download nothing +``` + +The ids come from an empty query against the search service, which orders by download count. The +download itself is `nugetfuzz --download-only`, so package selection, TFM matching and the +dependency walk behave exactly as they do in a sweep - only the decompiling is skipped. + +The corpus is written as a list of directories rather than one root, because a package already +restored on this machine is used from the machine-wide NuGet cache instead of being copied: + +```pwsh +dotnet run decompdiff.cs -- --old master --new my-branch -o report $(cat crawl/top-200.corpus.txt) +``` + ## decompdiff Decompiles a corpus with **two** builds of `ICSharpCode.Decompiler` side by side (separate diff --git a/TestTools/nuget-top.ps1 b/TestTools/nuget-top.ps1 new file mode 100644 index 000000000..028ca4aef --- /dev/null +++ b/TestTools/nuget-top.ps1 @@ -0,0 +1,73 @@ +#!/usr/bin/env pwsh +# Downloads the N most-downloaded packages on nuget.org into the cache that decompdiff +# uses as its corpus (~/.cache/nugetfuzz), together with their dependency closures. +# +# The download, TFM selection and dependency walk all live in nugetfuzz.cs already, so +# this only picks the ids and hands them over: an empty query against the search service +# returns packages ordered by download count. +# +# usage: ./nuget-top.ps1 [-Count n] [-ListOnly] [-Out path] [-Skip n] +# ponytail: no per-id retry; a package that fails to download is reported and skipped + +[CmdletBinding()] +param( + [int]$Count = 100, + [int]$Skip = 0, # start further down the ranking, to extend an existing corpus + [switch]$ListOnly, # write the id list, download nothing + [string]$Out +) + +$ErrorActionPreference = 'Stop' +Set-Location $PSScriptRoot + +# Required by the local OpenSSL configuration to validate the SHA-1 signed packages. +if (-not $env:OPENSSL_ENABLE_SHA1_SIGNATURES) { + $env:OPENSSL_ENABLE_SHA1_SIGNATURES = '1' +} + +$state = Join-Path $PSScriptRoot 'crawl' +New-Item -ItemType Directory -Force -Path $state | Out-Null +if (-not $Out) { + $Out = Join-Path $state "top-$Count.txt" +} + +# The search service caps how far a caller may page; ask for the ids in blocks and stop +# as soon as it stops handing any back, so a too-large -Count truncates instead of failing. +$endpoint = 'https://azuresearch-usnc.nuget.org/query' +$ids = [System.Collections.Generic.List[string]]::new() +$page = 100 +while ($ids.Count -lt $Count) { + $take = [Math]::Min($page, $Count - $ids.Count) + $uri = "${endpoint}?q=&skip=$($Skip + $ids.Count)&take=$take&prerelease=false&semVerLevel=2.0.0" + $response = Invoke-RestMethod -Uri $uri + if (-not $response.data -or $response.data.Count -eq 0) { + Write-Warning "search returned nothing at skip=$($Skip + $ids.Count); stopping at $($ids.Count) ids" + break + } + foreach ($entry in $response.data) { + $ids.Add($entry.id) + } +} + +# Ranking order is worth keeping in the file: it says what a truncated corpus dropped. +Set-Content -Path $Out -Value $ids +Write-Host "$($ids.Count) package ids -> $Out" + +if ($ListOnly) { + return +} + +# A package already restored on this machine is used from the machine-wide NuGet cache +# rather than copied, so the downloaded set is not one directory that can be handed to +# decompdiff. nugetfuzz names the lib directory it settled on for each package, and that +# list IS the corpus - one entry per package, at the target framework it chose. +$corpusFile = [IO.Path]::ChangeExtension($Out, '.corpus.txt') +dotnet run nugetfuzz.cs -- --download-only "@$Out" | Tee-Object -Variable log +if ($LASTEXITCODE -ne 0) { + throw "nugetfuzz exited with $LASTEXITCODE" +} +$dirs = $log | ForEach-Object { if ($_ -match '^\s*cached:\s*(.+)$') { $Matches[1].Trim() } } +Set-Content -Path $corpusFile -Value $dirs +Write-Host "" +Write-Host "$($dirs.Count) lib directories -> $corpusFile" +Write-Host "decompdiff --old --new -o report `$(cat $corpusFile)" diff --git a/TestTools/nugetfuzz.cs b/TestTools/nugetfuzz.cs index 41c74a968..e49df6efb 100644 --- a/TestTools/nugetfuzz.cs +++ b/TestTools/nugetfuzz.cs @@ -25,7 +25,7 @@ // Microsoft.NETFramework.ReferenceAssemblies packages), then decompiles every // assembly type-by-type and reports Debug.Assert failures / exceptions. // -// usage: dotnet run nugetfuzz.cs -- ... | @packagelist.txt +// usage: dotnet run nugetfuzz.cs -- [--download-only] ... | @packagelist.txt using System.Diagnostics; using System.IO.Compression; @@ -108,14 +108,18 @@ if (args is ["--report", var ledgerPath, ..]) return 0; } +// Populates the cache without decompiling: the sweep is the slow part, and a corpus +// only needs the assemblies on disk. +var downloadOnly = args.Contains("--download-only"); var packages = args + .Where(a => a != "--download-only") .SelectMany(a => a.StartsWith('@') ? File.ReadAllLines(a[1..]) : new[] { a }) .Select(l => l.Trim()) .Where(l => l.Length > 0 && !l.StartsWith('#')) .ToList(); if (packages.Count == 0) { - Console.Error.WriteLine("usage: nugetfuzz ... | @packagelist.txt"); + Console.Error.WriteLine("usage: nugetfuzz [--download-only] ... | @packagelist.txt"); Console.Error.WriteLine(" nugetfuzz --report [out.html]"); return 1; } @@ -147,7 +151,11 @@ foreach (var entry in failures.Values.OrderByDescending(f => f.Count)) // shared ledger; `--report ` renders the aggregate. Without a ledger the run // reports only itself. var ledger = Environment.GetEnvironmentVariable("NUGETFUZZ_LEDGER"); -if (ledger != null) +if (downloadOnly) +{ + // Nothing was decompiled, so there are no findings to report on. +} +else if (ledger != null) { AppendToLedger(ledger, failures.Values, assemblyCount, typeCount, refsResolved, refsTotal); Console.WriteLine($"ledger: {Path.GetFullPath(ledger)}"); @@ -190,6 +198,13 @@ async Task ProcessPackage(string spec) var searchDirs = await CollectDependencies(dir, matchTarget, id); searchDirs.Insert(0, libDir); + if (downloadOnly) + { + // The package and its dependency closure are in the cache now, which is all a + // corpus needs; decompiling every type is what the sweep is for. + Console.WriteLine($" cached: {libDir}"); + return; + } // The installed runtime dir is a last-resort fallback only: Microsoft.NETCore.App // ships stub facades (e.g. WindowsBase.dll without System.Windows.Point) that must // not shadow the real assemblies from ref packs or dependency packages.