diff --git a/TestTools/README.md b/TestTools/README.md index cf93ce3a8..379b1334b 100644 --- a/TestTools/README.md +++ b/TestTools/README.md @@ -76,9 +76,14 @@ The report directory gets `summary.md`, a self-contained `index.html` with inlin changed types dumped under `old/` and `new/` for `git diff --no-index report/old report/new`. Each assembly is decompiled from a staging directory holding its neighbours plus the transitive -closure of its references, found in `--refs` directories, the NuGet cache and the .NET Framework -reference packs. Both sides read the same staging directory, so anything still unresolved degrades -them identically and the diff stays meaningful. Unresolved references are listed in the summary. +closure of its references, found in the reference-assembly pack matching each assembly's own target +framework (`Microsoft.NETCore.App.Ref` and the Windows-desktop / ASP.NET packs for .NET Core targets, +`Microsoft.NETFramework.ReferenceAssemblies` for classic net4x), then `--refs` directories and the +NuGet cache. Getting the pack right matters as much as it does for `nugetfuzz`: binding a net9.0 +assembly against the net4x packs resolves mscorlib but not `ValueTask` or the async method builders, +and every async method then decompiles as a raw state machine. Both sides read the same staging +directory, so anything still unresolved degrades them identically and the diff stays meaningful. +Unresolved references are listed in the summary. ## Windows notes diff --git a/TestTools/decompdiff.cs b/TestTools/decompdiff.cs index b8d5d62b2..d32221bce 100644 --- a/TestTools/decompdiff.cs +++ b/TestTools/decompdiff.cs @@ -38,7 +38,8 @@ // Reference handling: every assembly is decompiled out of a staging directory that // holds symlinks to itself, its original neighbours, and the transitive closure of // its references as found in --refs directories, the machine-wide NuGet cache, and -// the .NET Framework reference-assembly packs. UniversalAssemblyResolver searches +// the reference-assembly pack matching the assembly's own target framework. +// UniversalAssemblyResolver searches // that directory first (ResolveInternal -> SearchDirectory), so staging fixes both // classic failure modes - a sibling package that does not sit next to the assembly, // and the Windows-only mscorlib lookup that throws "Version not supported" on Linux @@ -299,70 +300,169 @@ static string MetricNotes(ChangedType c) } // Locates reference assemblies by simple name and stages them next to the assembly -// being decompiled. Sources, in order: the --refs directories (indexed once), the -// machine-wide NuGet cache (probed per name, so nothing scans ~40k packages), and -// the .NET Framework reference-assembly packs restored under it - the last one is -// what makes classic net4x assemblies decompilable on Linux at all, since their -// mscorlib otherwise only exists behind a Windows path lookup. +// being decompiled. Sources, in order: the reference-assembly pack for the assembly's +// own target framework, the --refs directories (indexed once), and the machine-wide +// NuGet cache (probed per name, so nothing scans ~40k packages). +// +// The pack has to be chosen per assembly and has to win: a net9.0 assembly resolved +// against the .NET Framework packs finds mscorlib but not ValueTask or the async +// method builders, which collapses whole type hierarchies to Unknown and leaves +// AsyncAwaitDecompiler unable to recognise a state machine at all. The packs are also +// what makes classic net4x assemblies decompilable on Linux, whose mscorlib otherwise +// only exists behind a Windows path lookup. sealed class RefIndex { readonly Dictionary byName = new(StringComparer.OrdinalIgnoreCase); readonly List probeRoots = new(); + readonly string nugetRoot; + // Reference assemblies are picked per corpus assembly, keyed by its TargetFrameworkAttribute: + // a net9.0 assembly resolved against the .NET Framework packs finds mscorlib but not ValueTask + // or the async method builders, which collapses whole type hierarchies to Unknown and leaves + // AsyncAwaitDecompiler unable to recognise a state machine at all. + readonly Dictionary> byFramework = new(StringComparer.OrdinalIgnoreCase); public string Description { get; } public RefIndex(List refDirs, List corpus) { foreach (var dir in refDirs.Concat(corpus).Where(Directory.Exists)) - IndexDirectory(dir); - var nugetRoot = Environment.GetEnvironmentVariable("NUGET_PACKAGES") + IndexDirectory(dir, byName); + nugetRoot = Environment.GetEnvironmentVariable("NUGET_PACKAGES") ?? Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.UserProfile), ".nuget", "packages"); if (Directory.Exists(nugetRoot)) probeRoots.Add(nugetRoot); - foreach (var pack in EnumerateFrameworkRefPacks(nugetRoot)) - IndexDirectory(pack); + Description = $"{byName.Count} assemblies indexed" + + (probeRoots.Count > 0 ? $", NuGet cache probe at {string.Join(", ", probeRoots)}" : ""); + } + + // The reference assemblies for one target framework moniker, indexed by simple name. + // Empty when no matching pack is installed, in which case resolution falls back to the + // --refs directories, the corpus and the NuGet cache probe. + public Dictionary IndexFor(string? targetFramework) + { + var key = targetFramework ?? ""; + if (byFramework.TryGetValue(key, out var index)) + return index; + index = new Dictionary(StringComparer.OrdinalIgnoreCase); + foreach (var dir in RefPackDirs(key)) + IndexDirectory(dir, index); + byFramework[key] = index; + return index; + } + + // Ref-pack directories for a TargetFrameworkAttribute value, most specific first. + IEnumerable RefPackDirs(string targetFramework) + { + var version = TfmVersion(targetFramework); + if (targetFramework.StartsWith(".NETCoreApp", StringComparison.OrdinalIgnoreCase) && version != null) + { + // The WindowsDesktop and AspNetCore packs come first: Microsoft.NETCore.App.Ref ships + // stub facades for their assemblies (its WindowsBase.dll has no DependencyObject), and + // whichever is indexed first wins. + foreach (var pack in new[] { "microsoft.windowsdesktop.app.ref", "microsoft.aspnetcore.app.ref", "microsoft.netcore.app.ref" }) + { + var dir = CorePackDir(pack, version); + if (dir != null) + yield return dir; + } + yield break; + } + // .NETFramework and .NETStandard both resolve against the classic reference assemblies, + // but only a .NETFramework version selects a pack: a .NETStandard version is not a net4x + // version, and feeding 2.0 to the picker would pick net45, whose Facades carry no + // netstandard.dll. netstandard targets take the newest pack, with the widest facade set. + var netFxVersion = targetFramework.StartsWith(".NETFramework", StringComparison.OrdinalIgnoreCase) + ? version : null; + foreach (var dir in EnumerateFrameworkRefPacks(nugetRoot, netFxVersion)) + yield return dir; // On Windows the same reference assemblies also ship with the targeting packs, so a // net4x corpus resolves there without restoring the NuGet package first. var installedFxRefs = Path.Combine(Environment.GetFolderPath(Environment.SpecialFolder.ProgramFilesX86), "Reference Assemblies", "Microsoft", "Framework", ".NETFramework"); if (Directory.Exists(installedFxRefs)) - IndexDirectory(installedFxRefs); - Description = $"{byName.Count} assemblies indexed" - + (probeRoots.Count > 0 ? $", NuGet cache probe at {string.Join(", ", probeRoots)}" : ""); + yield return installedFxRefs; + } + + static Version? TfmVersion(string targetFramework) + { + var i = targetFramework.IndexOf("Version=v", StringComparison.OrdinalIgnoreCase); + return i >= 0 && Version.TryParse(targetFramework[(i + 9)..], out var v) ? v : null; + } + + // ref/ of a .NET (Core) shared-framework pack. Never falls back to "newest available": + // the packs only go back to 3.0, so a netcoreapp1.x/2.x assembly would silently bind against + // a current BCL and decompile as a wall of Unknown. Better to resolve nothing and say so. + string? CorePackDir(string packId, Version version) + { + var root = Path.Combine(nugetRoot, packId); + if (!Directory.Exists(root)) + return null; + var versions = Directory.EnumerateDirectories(root) + .Select(d => (Dir: d, V: Version.TryParse(Path.GetFileName(d).Split('-')[0], out var v) ? v : null)) + .Where(x => x.V != null) + .OrderBy(x => x.V) + .ToList(); + var pick = versions.LastOrDefault(x => x.V!.Major == version.Major && x.V.Minor == version.Minor).Dir + ?? versions.LastOrDefault(x => x.V!.Major <= version.Major).Dir; + if (pick == null) + return null; + var refRoot = Path.Combine(pick, "ref"); + return Directory.Exists(refRoot) ? Directory.EnumerateDirectories(refRoot).FirstOrDefault() : null; } // Reference assemblies for .NET Framework targets; the newest pack wins, and its // Facades subdirectory carries the type-forwarding shims netstandard code needs. - static IEnumerable EnumerateFrameworkRefPacks(string nugetRoot) + // The single .NET Framework reference-assembly pack to bind against: the smallest one that + // is still a superset of the requested version, or the newest installed when no version is + // given (.NETStandard, which wants the widest set of facades). Indexing every installed pack + // instead would let the oldest one win by name and hide the newer BCL from every assembly. + static IEnumerable EnumerateFrameworkRefPacks(string nugetRoot, Version? requested) { if (!Directory.Exists(nugetRoot)) yield break; - foreach (var pkg in Directory.EnumerateDirectories(nugetRoot, "microsoft.netframework.referenceassemblies.*") - .OrderBy(d => d)) + var packs = Directory.EnumerateDirectories(nugetRoot, "microsoft.netframework.referenceassemblies.net4*") + .Select(d => (Dir: d, V: PackVersion(Path.GetFileName(d)))) + .Where(x => x.V != null) + .OrderBy(x => x.V) + .ToList(); + var pick = (requested != null ? packs.FirstOrDefault(x => x.V >= requested).Dir : null) + ?? packs.LastOrDefault().Dir; + if (pick == null) + yield break; + foreach (var dir in Directory.EnumerateDirectories(pick, "v*", SearchOption.AllDirectories)) { - foreach (var dir in Directory.EnumerateDirectories(pkg, "v*", SearchOption.AllDirectories)) - { - yield return dir; - var facades = Path.Combine(dir, "Facades"); - if (Directory.Exists(facades)) - yield return facades; - } + yield return dir; + var facades = Path.Combine(dir, "Facades"); + if (Directory.Exists(facades)) + yield return facades; } } - void IndexDirectory(string dir) + // "microsoft.netframework.referenceassemblies.net472" -> 4.7.2 + static Version? PackVersion(string packageId) + { + var tfm = packageId[(packageId.LastIndexOf('.') + 1)..]; + return tfm.Length > 3 && tfm.StartsWith("net", StringComparison.Ordinal) + && Version.TryParse(string.Join('.', tfm[3..].ToCharArray()), out var v) ? v : null; + } + + static void IndexDirectory(string dir, Dictionary index) { foreach (var dll in Directory.EnumerateFiles(dir, "*.dll", SearchOption.AllDirectories)) { var name = Path.GetFileNameWithoutExtension(dll); // First indexed wins: --refs directories are added before the corpus, so an // explicitly supplied reference is never shadowed by a corpus copy. - if (!byName.ContainsKey(name)) - byName[name] = dll; + if (!index.ContainsKey(name)) + index[name] = dll; } } - public string? Find(string simpleName) + public string? Find(string simpleName, Dictionary frameworkIndex) { + // The target framework's own reference assemblies outrank everything else: a corpus + // neighbour or a stray NuGet copy of System.Runtime would otherwise decide the BCL. + if (frameworkIndex.TryGetValue(simpleName, out var fxHit)) + return fxHit; if (byName.TryGetValue(simpleName, out var hit)) return hit; foreach (var root in probeRoots) @@ -391,6 +491,7 @@ sealed class RefIndex // plus the reference names nothing could supply. public static (string Staged, List Missing) Stage(string dll, string stageRoot, RefIndex refs) { + var frameworkIndex = refs.IndexFor(TargetFrameworkOf(dll)); var dir = Path.Combine(stageRoot, StageName(dll)); Directory.CreateDirectory(dir); // Whatever sat next to the assembly keeps sitting next to it, so staging never @@ -411,7 +512,7 @@ sealed class RefIndex var staged = Path.Combine(dir, name + ".dll"); if (!File.Exists(staged)) { - var found = refs.Find(name); + var found = refs.Find(name, frameworkIndex); if (found == null) { missing.Add(name); @@ -454,6 +555,42 @@ sealed class RefIndex } } + // The assembly's TargetFrameworkAttribute value (e.g. ".NETCoreApp,Version=v9.0"), or null + // when it carries none - which is normal for the .NET Framework era and for ref assemblies. + public static string? TargetFrameworkOf(string dll) + { + try + { + using var stream = File.OpenRead(dll); + using var pe = new PEReader(stream); + if (!pe.HasMetadata) + return null; + var md = pe.GetMetadataReader(); + foreach (var handle in md.GetAssemblyDefinition().GetCustomAttributes()) + { + var attr = md.GetCustomAttribute(handle); + if (attr.Constructor.Kind != HandleKind.MemberReference) + continue; + var ctor = md.GetMemberReference((MemberReferenceHandle)attr.Constructor); + if (ctor.Parent.Kind != HandleKind.TypeReference) + continue; + var type = md.GetTypeReference((TypeReferenceHandle)ctor.Parent); + if (md.GetString(type.Name) != "TargetFrameworkAttribute") + continue; + // blob: prolog (0x0001), then a SerString holding the moniker. + var reader = md.GetBlobReader(attr.Value); + if (reader.ReadUInt16() != 1) + return null; + return reader.ReadSerializedString(); + } + } + catch (Exception ex) when (ex is BadImageFormatException or IOException) + { + // Native or corrupt file: nothing to read. + } + return null; + } + public static List ReferencesOf(string dll) { var names = new List();