diff --git a/TestTools/README.md b/TestTools/README.md index e60639f6c..797736312 100644 --- a/TestTools/README.md +++ b/TestTools/README.md @@ -1,12 +1,13 @@ # TestTools -Two standalone tools that run the decompiler over real-world assemblies, to find defects the +Standalone tools that run the decompiler over real-world assemblies, to find defects the in-repo test suite cannot: it decompiles fixtures we wrote, these decompile what the world ships. | tool | question it answers | |---|---| | `nugetfuzz.cs` | Does the decompiler *crash* on real code? (asserts, exceptions, IL warnings) | | `decompdiff.cs` | Did a change make the *output* better or worse? (readability across two builds) | +| `nuget-top.ps1` | Where do I get a corpus? (downloads the most-downloaded packages) | Both are [file-based apps](https://learn.microsoft.com/dotnet/core/whats-new/dotnet-10/sdk#file-based-apps): single `.cs` files run directly by the SDK, no project, no solution entry. They are not built by @@ -55,6 +56,28 @@ including while the sweep is still running, with `--report`. Logs of failed runs `logs/`, successful ones are deleted. The package cache (`~/.cache/nugetfuzz`) is capped at 20 GB by default (`-CacheCapMB`) and pruned least-recently-used, because `decompdiff` uses it as a corpus. +## nuget-top + +Downloads the most-downloaded packages on nuget.org, with their dependency closures, and writes the +list of lib directories they resolved to - a ready-made corpus for either tool: + +```pwsh +./nuget-top.ps1 -Count 200 +./nuget-top.ps1 -Count 50 -Skip 200 # extend an existing corpus further down the ranking +./nuget-top.ps1 -Count 200 -ListOnly # just the ids, download nothing +``` + +The ids come from an empty query against the search service, which orders by download count. The +download itself is `nugetfuzz --download-only`, so package selection, TFM matching and the +dependency walk behave exactly as they do in a sweep - only the decompiling is skipped. + +The corpus is written as a list of directories rather than one root, because a package already +restored on this machine is used from the machine-wide NuGet cache instead of being copied: + +```pwsh +dotnet run decompdiff.cs -- --old master --new my-branch -o report $(cat crawl/top-200.corpus.txt) +``` + ## decompdiff Decompiles a corpus with **two** builds of `ICSharpCode.Decompiler` side by side (separate diff --git a/TestTools/nuget-top.ps1 b/TestTools/nuget-top.ps1 new file mode 100644 index 000000000..028ca4aef --- /dev/null +++ b/TestTools/nuget-top.ps1 @@ -0,0 +1,73 @@ +#!/usr/bin/env pwsh +# Downloads the N most-downloaded packages on nuget.org into the cache that decompdiff +# uses as its corpus (~/.cache/nugetfuzz), together with their dependency closures. +# +# The download, TFM selection and dependency walk all live in nugetfuzz.cs already, so +# this only picks the ids and hands them over: an empty query against the search service +# returns packages ordered by download count. +# +# usage: ./nuget-top.ps1 [-Count n] [-ListOnly] [-Out path] [-Skip n] +# ponytail: no per-id retry; a package that fails to download is reported and skipped + +[CmdletBinding()] +param( + [int]$Count = 100, + [int]$Skip = 0, # start further down the ranking, to extend an existing corpus + [switch]$ListOnly, # write the id list, download nothing + [string]$Out +) + +$ErrorActionPreference = 'Stop' +Set-Location $PSScriptRoot + +# Required by the local OpenSSL configuration to validate the SHA-1 signed packages. +if (-not $env:OPENSSL_ENABLE_SHA1_SIGNATURES) { + $env:OPENSSL_ENABLE_SHA1_SIGNATURES = '1' +} + +$state = Join-Path $PSScriptRoot 'crawl' +New-Item -ItemType Directory -Force -Path $state | Out-Null +if (-not $Out) { + $Out = Join-Path $state "top-$Count.txt" +} + +# The search service caps how far a caller may page; ask for the ids in blocks and stop +# as soon as it stops handing any back, so a too-large -Count truncates instead of failing. +$endpoint = 'https://azuresearch-usnc.nuget.org/query' +$ids = [System.Collections.Generic.List[string]]::new() +$page = 100 +while ($ids.Count -lt $Count) { + $take = [Math]::Min($page, $Count - $ids.Count) + $uri = "${endpoint}?q=&skip=$($Skip + $ids.Count)&take=$take&prerelease=false&semVerLevel=2.0.0" + $response = Invoke-RestMethod -Uri $uri + if (-not $response.data -or $response.data.Count -eq 0) { + Write-Warning "search returned nothing at skip=$($Skip + $ids.Count); stopping at $($ids.Count) ids" + break + } + foreach ($entry in $response.data) { + $ids.Add($entry.id) + } +} + +# Ranking order is worth keeping in the file: it says what a truncated corpus dropped. +Set-Content -Path $Out -Value $ids +Write-Host "$($ids.Count) package ids -> $Out" + +if ($ListOnly) { + return +} + +# A package already restored on this machine is used from the machine-wide NuGet cache +# rather than copied, so the downloaded set is not one directory that can be handed to +# decompdiff. nugetfuzz names the lib directory it settled on for each package, and that +# list IS the corpus - one entry per package, at the target framework it chose. +$corpusFile = [IO.Path]::ChangeExtension($Out, '.corpus.txt') +dotnet run nugetfuzz.cs -- --download-only "@$Out" | Tee-Object -Variable log +if ($LASTEXITCODE -ne 0) { + throw "nugetfuzz exited with $LASTEXITCODE" +} +$dirs = $log | ForEach-Object { if ($_ -match '^\s*cached:\s*(.+)$') { $Matches[1].Trim() } } +Set-Content -Path $corpusFile -Value $dirs +Write-Host "" +Write-Host "$($dirs.Count) lib directories -> $corpusFile" +Write-Host "decompdiff --old --new -o report `$(cat $corpusFile)" diff --git a/TestTools/nugetfuzz.cs b/TestTools/nugetfuzz.cs index 41c74a968..e49df6efb 100644 --- a/TestTools/nugetfuzz.cs +++ b/TestTools/nugetfuzz.cs @@ -25,7 +25,7 @@ // Microsoft.NETFramework.ReferenceAssemblies packages), then decompiles every // assembly type-by-type and reports Debug.Assert failures / exceptions. // -// usage: dotnet run nugetfuzz.cs -- ... | @packagelist.txt +// usage: dotnet run nugetfuzz.cs -- [--download-only] ... | @packagelist.txt using System.Diagnostics; using System.IO.Compression; @@ -108,14 +108,18 @@ if (args is ["--report", var ledgerPath, ..]) return 0; } +// Populates the cache without decompiling: the sweep is the slow part, and a corpus +// only needs the assemblies on disk. +var downloadOnly = args.Contains("--download-only"); var packages = args + .Where(a => a != "--download-only") .SelectMany(a => a.StartsWith('@') ? File.ReadAllLines(a[1..]) : new[] { a }) .Select(l => l.Trim()) .Where(l => l.Length > 0 && !l.StartsWith('#')) .ToList(); if (packages.Count == 0) { - Console.Error.WriteLine("usage: nugetfuzz ... | @packagelist.txt"); + Console.Error.WriteLine("usage: nugetfuzz [--download-only] ... | @packagelist.txt"); Console.Error.WriteLine(" nugetfuzz --report [out.html]"); return 1; } @@ -147,7 +151,11 @@ foreach (var entry in failures.Values.OrderByDescending(f => f.Count)) // shared ledger; `--report ` renders the aggregate. Without a ledger the run // reports only itself. var ledger = Environment.GetEnvironmentVariable("NUGETFUZZ_LEDGER"); -if (ledger != null) +if (downloadOnly) +{ + // Nothing was decompiled, so there are no findings to report on. +} +else if (ledger != null) { AppendToLedger(ledger, failures.Values, assemblyCount, typeCount, refsResolved, refsTotal); Console.WriteLine($"ledger: {Path.GetFullPath(ledger)}"); @@ -190,6 +198,13 @@ async Task ProcessPackage(string spec) var searchDirs = await CollectDependencies(dir, matchTarget, id); searchDirs.Insert(0, libDir); + if (downloadOnly) + { + // The package and its dependency closure are in the cache now, which is all a + // corpus needs; decompiling every type is what the sweep is for. + Console.WriteLine($" cached: {libDir}"); + return; + } // The installed runtime dir is a last-resort fallback only: Microsoft.NETCore.App // ships stub facades (e.g. WindowsBase.dll without System.Windows.Point) that must // not shadow the real assemblies from ref packs or dependency packages.