From 61cde6c2f8cb24f5999cfeb84fce7cb2d0a56a43 Mon Sep 17 00:00:00 2001 From: Daniel Grunwald Date: Sun, 5 Dec 2010 14:41:23 +0100 Subject: [PATCH] Add documentation providers (xml and binary). --- .../TypeSystem/CommonTypeInferenceTests.cs | 1 - .../TypeSystem/TestInterningProvider.cs | 20 +- .../Dom/GeneralScope/DelegateDeclaration.cs | 8 +- .../BinaryDocumentationProvider.cs | 231 ++++++++++++++++++ .../Documentation/XmlDocumentationProvider.cs | 179 ++++++++++++++ .../ICSharpCode.NRefactory.csproj | 6 +- .../TypeSystem/CecilLoader.cs | 25 +- .../TypeSystem/IDocumentationProvider.cs | 19 ++ .../Implementation/AbstractMember.cs | 16 +- .../Implementation/DefaultProperty.cs | 2 + 10 files changed, 492 insertions(+), 15 deletions(-) create mode 100644 ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs create mode 100644 ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs create mode 100644 ICSharpCode.NRefactory/TypeSystem/IDocumentationProvider.cs diff --git a/ICSharpCode.NRefactory.Tests/TypeSystem/CommonTypeInferenceTests.cs b/ICSharpCode.NRefactory.Tests/TypeSystem/CommonTypeInferenceTests.cs index 4573741647..9ee11aa2e6 100644 --- a/ICSharpCode.NRefactory.Tests/TypeSystem/CommonTypeInferenceTests.cs +++ b/ICSharpCode.NRefactory.Tests/TypeSystem/CommonTypeInferenceTests.cs @@ -5,7 +5,6 @@ using System; using System.Collections; using System.Collections.Generic; using System.Linq; -using System.Linq; using ICSharpCode.NRefactory.CSharp.Resolver; using NUnit.Framework; diff --git a/ICSharpCode.NRefactory.Tests/TypeSystem/TestInterningProvider.cs b/ICSharpCode.NRefactory.Tests/TypeSystem/TestInterningProvider.cs index 841108c57f..fcf21713d2 100644 --- a/ICSharpCode.NRefactory.Tests/TypeSystem/TestInterningProvider.cs +++ b/ICSharpCode.NRefactory.Tests/TypeSystem/TestInterningProvider.cs @@ -7,12 +7,15 @@ using System.Collections.Generic; using System.Collections.ObjectModel; using System.Linq; using System.Runtime.CompilerServices; +using System.Runtime.InteropServices; +using System.Runtime.Serialization; using ICSharpCode.NRefactory.TypeSystem.Implementation; +using ICSharpCode.NRefactory.Utils; using NUnit.Framework; namespace ICSharpCode.NRefactory.TypeSystem { - /* Not a real unit test + //* Not a real unit test [TestFixture] public class TestInterningProvider : IInterningProvider { @@ -143,6 +146,19 @@ namespace ICSharpCode.NRefactory.TypeSystem foreach (var element in stats) { Console.WriteLine(element.Type + ": " + element.PostCount + "/" + element.PreCount); } + Console.WriteLine(stats.Sum(r => r.PostCount * SizeOf(r.Type)) / 1024 + " KB / " + + stats.Sum(r => r.PreCount * SizeOf(r.Type)) / 1024 + " KB"); } - }*/ + + static int SizeOf(Type t) + { + if (t == typeof(string)) + return 16; + long start = GC.GetTotalMemory(true); + object o = FormatterServices.GetUninitializedObject(t); + long stop = GC.GetTotalMemory(true); + GC.KeepAlive(o); + return (int)Math.Max(8, stop - start); + } + }//*/ } diff --git a/ICSharpCode.NRefactory/CSharp/Dom/GeneralScope/DelegateDeclaration.cs b/ICSharpCode.NRefactory/CSharp/Dom/GeneralScope/DelegateDeclaration.cs index 83e0aaeec7..788c146bc6 100644 --- a/ICSharpCode.NRefactory/CSharp/Dom/GeneralScope/DelegateDeclaration.cs +++ b/ICSharpCode.NRefactory/CSharp/Dom/GeneralScope/DelegateDeclaration.cs @@ -1,4 +1,4 @@ -// +// // DelegateDeclaration.cs // // Author: @@ -69,12 +69,6 @@ namespace ICSharpCode.NRefactory.CSharp get { return (CSharpTokenNode)GetChildByRole (Roles.RPar) ?? CSharpTokenNode.Null; } } - public IEnumerable Attributes { - get { - return base.GetChildrenByRole (Roles.Attribute).Cast (); - } - } - public override S AcceptVisitor (DomVisitor visitor, T data) { return visitor.VisitDelegateDeclaration (this, data); diff --git a/ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs b/ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs new file mode 100644 index 0000000000..a824396c38 --- /dev/null +++ b/ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs @@ -0,0 +1,231 @@ +// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt) +// This code is distributed under MIT X11 license (for details please see \doc\license.txt) + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using ICSharpCode.NRefactory.TypeSystem; + +namespace ICSharpCode.NRefactory.Documentation +{ + /// + /// Provides xml documentation from a binary cache file. + /// This allows providing XML documentation without having to read the whole documentation into memory. + /// + public class BinaryDocumentationProvider : IDisposable, IDocumentationProvider + { + struct IndexEntry + { + public readonly int HashCode; + public readonly int FileLocation; + + public IndexEntry(int HashCode, int FileLocation) + { + this.HashCode = HashCode; + this.FileLocation = FileLocation; + } + } + + #region Save binary files + // FILE FORMAT FOR BINARY DOCUMENTATION + // long magic = 0x4244636f446c6d58 (identifies file type = 'XmlDocDB') + const long magic = 0x4244636f446c6d58; + // short version = 3 (file version) + const short version = 3; + // long fileDate (last change date of xml file in DateTime ticks) + // int testHashCode = magicTestString.GetHashCode() // (check if hash-code implementation is compatible) + const string magicTestString = "HashMe-XmlDocDB"; + // int entryCount (count of entries) + // int indexPointer (points to location where index starts in the file) + // { + // string key (documentation key as length-prefixed string) + // string docu (xml documentation as length-prefixed string) + // } + // indexPointer points to the start of the following section: + // { + // int hashcode + // int index (index where the docu string starts in the file) + // } + + + /// + /// Saves the xml documentation into a on-disk database file. + /// + /// Filename of the database + /// Last-modified date of the .xml file + /// The xml documentation that should be written to disk. + public static void Save(string fileName, DateTime fileDate, IEnumerable> xmlDocumentation) + { + if (fileName == null) + throw new ArgumentNullException("fileName"); + if (xmlDocumentation == null) + throw new ArgumentNullException("xmlDocumentation"); + using (FileStream fs = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None)) { + using (BinaryWriter w = new BinaryWriter(fs)) { + w.Write(magic); + w.Write(version); + w.Write(fileDate.Ticks); + w.Write(magicTestString.GetHashCode()); + + List index = new List(); + int indexLengthPos = (int)fs.Position; + w.Write(0); // skip 4 bytes for index length + w.Write(0); // skip 4 bytes for index pointer + + int i = 0; + foreach (KeyValuePair p in xmlDocumentation) { + index.Add(new IndexEntry(p.Key.GetHashCode(), (int)fs.Position)); + w.Write(p.Key); + w.Write(p.Value.Trim()); + i += 1; + } + + index.Sort((a,b) => a.HashCode.CompareTo(b.HashCode)); + + int indexStart = (int)fs.Position; + foreach (IndexEntry entry in index) { + w.Write(entry.HashCode); + w.Write(entry.FileLocation); + } + w.Seek(indexLengthPos, SeekOrigin.Begin); + w.Write(index.Count); // write index length + w.Write(indexStart); // write index count + } + } + } + #endregion + + BinaryReader loader; + FileStream fs; + + Dictionary xmlDescription = new Dictionary(); + IndexEntry[] index; // SORTED array of index entries + + const int cacheLength = 50; // number of strings to cache when working in file-mode + Queue keyCacheQueue = new Queue(cacheLength); + + #region Load binary files + private BinaryDocumentationProvider() {} + + /// + /// Loads binary documentation. + /// + /// + /// Don't forget to dispose the BinaryDocumentationProvider. + /// + /// The name of the binary cache file. + /// The file date of the original XML file. Loading will fail if the cached data was generated + /// from a different file date than the original XML file. + /// + /// The BinaryDocumentationProvider representing the file's content; or null if loading failed. + /// + public static BinaryDocumentationProvider Load(string fileName, DateTime fileDate) + { + BinaryDocumentationProvider doc = new BinaryDocumentationProvider(); + try { + doc.fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); + int len = (int)doc.fs.Length; + BinaryReader loader = doc.loader = new BinaryReader(doc.fs); + if (loader.ReadInt64() != magic) { + Debug.WriteLine("Cannot load XmlDoc: wrong magic"); + return null; + } + if (loader.ReadInt16() != version) { + Debug.WriteLine("Cannot load XmlDoc: wrong version"); + return null; + } + if (loader.ReadInt64() != fileDate.Ticks) { + Debug.WriteLine("Not loading XmlDoc: file changed since cache was created"); + return null; + } + int count = loader.ReadInt32(); + int indexStartPosition = loader.ReadInt32(); // go to start of index + if (indexStartPosition <= 0 || indexStartPosition >= len) { + Debug.WriteLine("XmlDoc: Cannot find index, cache invalid!"); + return null; + } + doc.fs.Position = indexStartPosition; + IndexEntry[] index = new IndexEntry[count]; + for (int i = 0; i < index.Length; i++) { + index[i] = new IndexEntry(loader.ReadInt32(), loader.ReadInt32()); + } + doc.index = index; + return doc; + } catch (IOException ex) { + Debug.WriteLine("Cannot load from cache" + ex.ToString()); + return null; + } + } + + string LoadDocumentation(string key) + { + if (keyCacheQueue.Count > cacheLength - 1) { + xmlDescription.Remove(keyCacheQueue.Dequeue()); + } + + int hashcode = key.GetHashCode(); + + // use binary search to find the item + string resultDocu = null; + + int m = Array.BinarySearch(index, new IndexEntry(hashcode, 0)); + if (m >= 0) { + // correct hash code found. + // possibly there are multiple items with the same hash, so go to the first. + while (--m >= 0 && index[m].HashCode == hashcode); + // go through all items that have the correct hash + while (++m < index.Length && index[m].HashCode == hashcode) { + fs.Position = index[m].FileLocation; + string keyInFile = loader.ReadString(); + if (keyInFile == key) { + //LoggingService.Debug("Got XML documentation for " + key); + resultDocu = loader.ReadString(); + break; + } else { + // this is a harmless hash collision, just continue reading + Debug.WriteLine("Found " + keyInFile + " instead of " + key); + } + } + } + + keyCacheQueue.Enqueue(key); + xmlDescription.Add(key, resultDocu); + + return resultDocu; + } + #endregion + + public string GetDocumentation(string key) + { + lock (xmlDescription) { + if (index == null) + throw new ObjectDisposedException("BinaryDocumentationProvider"); + string result; + if (xmlDescription.TryGetValue(key, out result)) + return result; + return LoadDocumentation(key); + } + } + + public string GetDocumentation(IEntity entity) + { + return GetDocumentation(XmlDocumentationProvider.GetDocumentationKey(entity)); + } + + public void Dispose() + { + lock (xmlDescription) { + if (loader != null) { + loader.Close(); + fs.Close(); + } + xmlDescription.Clear(); + index = null; + keyCacheQueue = null; + loader = null; + fs = null; + } + } + } +} diff --git a/ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs b/ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs new file mode 100644 index 0000000000..8fc369d41c --- /dev/null +++ b/ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs @@ -0,0 +1,179 @@ +// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt) +// This code is distributed under MIT X11 license (for details please see \doc\license.txt) + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Xml; + +using ICSharpCode.NRefactory.TypeSystem; + +namespace ICSharpCode.NRefactory.Documentation +{ + /// + /// Provides documentation from an .xml file (as generated by the Microsoft C# compiler). + /// + public class XmlDocumentationProvider : IDocumentationProvider + { + readonly IDictionary xmlDocumentation; + + #region Load From File + public XmlDocumentationProvider(string fileName) + { + if (fileName == null) + throw new ArgumentNullException("fileName"); + this.xmlDocumentation = new Dictionary(); + using (XmlTextReader xmlReader = new XmlTextReader(fileName)) { + xmlReader.MoveToContent(); + if (string.IsNullOrEmpty(xmlReader.GetAttribute("redirect"))) { + ReadXmlDoc(xmlReader); + } else { + string redirectionTarget = GetRedirectionTarget(xmlReader.GetAttribute("redirect")); + if (redirectionTarget != null) { + Debug.WriteLine("XmlDoc " + fileName + " is redirecting to " + redirectionTarget); + using (XmlTextReader redirectedXmlReader = new XmlTextReader(redirectionTarget)) { + ReadXmlDoc(redirectedXmlReader); + } + } else { + Debug.WriteLine("XmlDoc " + fileName + " is redirecting to " + xmlReader.GetAttribute("redirect") + ", but that file was not found."); + } + } + } + } + + static string GetRedirectionTarget(string target) + { + string programFilesDir = Environment.GetFolderPath(Environment.SpecialFolder.ProgramFiles); + programFilesDir = AppendDirectorySeparator(programFilesDir); + + string corSysDir = System.Runtime.InteropServices.RuntimeEnvironment.GetRuntimeDirectory(); + corSysDir = AppendDirectorySeparator(corSysDir); + + return LookupLocalizedXmlDoc(target.Replace("%PROGRAMFILESDIR%", programFilesDir) + .Replace("%CORSYSDIR%", corSysDir)); + } + + static string AppendDirectorySeparator(string dir) + { + if (dir.EndsWith("\\", StringComparison.Ordinal) || dir.EndsWith("/", StringComparison.Ordinal)) + return dir; + else + return dir + Path.DirectorySeparatorChar; + } + + internal static string LookupLocalizedXmlDoc(string fileName) + { + string xmlFileName = Path.ChangeExtension(fileName, ".xml"); + string currentCulture = System.Threading.Thread.CurrentThread.CurrentUICulture.TwoLetterISOLanguageName; + string localizedXmlDocFile = GetLocalizedName(xmlFileName, currentCulture); + + Debug.WriteLine("Try find XMLDoc @" + localizedXmlDocFile); + if (File.Exists(localizedXmlDocFile)) { + return localizedXmlDocFile; + } + Debug.WriteLine("Try find XMLDoc @" + xmlFileName); + if (File.Exists(xmlFileName)) { + return xmlFileName; + } + if (currentCulture != "en") { + string englishXmlDocFile = GetLocalizedName(xmlFileName, "en"); + Debug.WriteLine("Try find XMLDoc @" + englishXmlDocFile); + if (File.Exists(englishXmlDocFile)) { + return englishXmlDocFile; + } + } + return null; + } + + static string GetLocalizedName(string fileName, string language) + { + string localizedXmlDocFile = Path.GetDirectoryName(fileName); + localizedXmlDocFile = Path.Combine(localizedXmlDocFile, language); + localizedXmlDocFile = Path.Combine(localizedXmlDocFile, Path.GetFileName(fileName)); + return localizedXmlDocFile; + } + #endregion + + #region Load from XmlReader + public XmlDocumentationProvider(XmlReader reader) + { + if (reader == null) + throw new ArgumentNullException("reader"); + this.xmlDocumentation = new Dictionary(); + ReadXmlDoc(reader); + } + + public XmlDocumentationProvider(IDictionary dictionary) + { + if (dictionary == null) + throw new ArgumentNullException("dictionary"); + this.xmlDocumentation = dictionary; + } + + void ReadXmlDoc(XmlReader reader) + { + while (reader.Read()) { + if (reader.IsStartElement()) { + switch (reader.LocalName) { + case "members": + ReadMembersSection(reader); + break; + } + } + } + } + + void ReadMembersSection(XmlReader reader) + { + while (reader.Read()) { + switch (reader.NodeType) { + case XmlNodeType.EndElement: + if (reader.LocalName == "members") { + return; + } + break; + case XmlNodeType.Element: + if (reader.LocalName == "member") { + string memberAttr = reader.GetAttribute(0); + string innerXml = reader.ReadInnerXml(); + xmlDocumentation[memberAttr] = innerXml; + } + break; + } + } + } + #endregion + + /// + /// Gets all entries in the documentation file. + /// + public IDictionary XmlDocumentation { + get { return xmlDocumentation; } + } + + public string GetDocumentation(string key) + { + if (key == null) + throw new ArgumentNullException("key"); + + string result; + if (xmlDocumentation.TryGetValue(key, out result)) + return result; + else + return null; + } + + public string GetDocumentation(IEntity entity) + { + return GetDocumentation(GetDocumentationKey(entity)); + } + + public static string GetDocumentationKey(IEntity entity) + { + if (entity == null) + throw new ArgumentNullException("entity"); + throw new NotImplementedException(); + } + } +} diff --git a/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj b/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj index 0a18b5afe6..84063a2875 100644 --- a/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj +++ b/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj @@ -14,7 +14,7 @@ False False false - 1591 + 1591,0618 AnyCPU @@ -176,6 +176,8 @@ + + @@ -190,6 +192,7 @@ + @@ -315,6 +318,7 @@ + \ No newline at end of file diff --git a/ICSharpCode.NRefactory/TypeSystem/CecilLoader.cs b/ICSharpCode.NRefactory/TypeSystem/CecilLoader.cs index f9a8b7f536..ba64f7f3b7 100644 --- a/ICSharpCode.NRefactory/TypeSystem/CecilLoader.cs +++ b/ICSharpCode.NRefactory/TypeSystem/CecilLoader.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Collections.ObjectModel; +using System.Diagnostics; using System.Linq; using System.Runtime.CompilerServices; using System.Runtime.InteropServices; @@ -32,6 +33,11 @@ namespace ICSharpCode.NRefactory.TypeSystem /// Specifies whether to include internal members. The default is false. /// public bool IncludeInternalMembers { get; set; } + + /// + /// Gets/Sets the documentation provider that is used to retrive the XML documentation for all members. + /// + public IDocumentationProvider DocumentationProvider { get; set; } #endregion #region Load From AssemblyDefinition @@ -46,7 +52,7 @@ namespace ICSharpCode.NRefactory.TypeSystem assemblyAttributes.Add(ReadAttribute(attr)); } TypeStorage typeStorage = new TypeStorage(); - CecilProjectContent pc = new CecilProjectContent(typeStorage, assemblyDefinition.Name.FullName, assemblyAttributes.AsReadOnly()); + CecilProjectContent pc = new CecilProjectContent(typeStorage, assemblyDefinition.Name.FullName, assemblyAttributes.AsReadOnly(), this.DocumentationProvider); this.EarlyBindContext = CompositeTypeResolveContext.Combine(pc, this.EarlyBindContext); List types = new List(); @@ -96,16 +102,20 @@ namespace ICSharpCode.NRefactory.TypeSystem #endregion #region IProjectContent implementation - sealed class CecilProjectContent : ProxyTypeResolveContext, IProjectContent, ISynchronizedTypeResolveContext + sealed class CecilProjectContent : ProxyTypeResolveContext, IProjectContent, ISynchronizedTypeResolveContext, IDocumentationProvider { readonly string assemblyName; readonly ReadOnlyCollection assemblyAttributes; + readonly IDocumentationProvider documentationProvider; - public CecilProjectContent(TypeStorage types, string assemblyName, ReadOnlyCollection assemblyAttributes) + public CecilProjectContent(TypeStorage types, string assemblyName, ReadOnlyCollection assemblyAttributes, IDocumentationProvider documentationProvider) : base(types) { + Debug.Assert(assemblyName != null); + Debug.Assert(assemblyAttributes != null); this.assemblyName = assemblyName; this.assemblyAttributes = assemblyAttributes; + this.documentationProvider = documentationProvider; } public IList AssemblyAttributes { @@ -125,6 +135,15 @@ namespace ICSharpCode.NRefactory.TypeSystem public void Dispose() { + // Disposibng the synchronization context has no effect + } + + string IDocumentationProvider.GetDocumentation(IEntity entity) + { + if (documentationProvider != null) + return documentationProvider.GetDocumentation(entity); + else + return null; } } #endregion diff --git a/ICSharpCode.NRefactory/TypeSystem/IDocumentationProvider.cs b/ICSharpCode.NRefactory/TypeSystem/IDocumentationProvider.cs new file mode 100644 index 0000000000..fbd74b75c3 --- /dev/null +++ b/ICSharpCode.NRefactory/TypeSystem/IDocumentationProvider.cs @@ -0,0 +1,19 @@ +// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt) +// This code is distributed under MIT X11 license (for details please see \doc\license.txt) + +using System; +using System.IO; + +namespace ICSharpCode.NRefactory.TypeSystem +{ + /// + /// Provides XML documentation for members. + /// + public interface IDocumentationProvider + { + /// + /// Gets the XML documentation for the specified entity. + /// + string GetDocumentation(IEntity entity); + } +} diff --git a/ICSharpCode.NRefactory/TypeSystem/Implementation/AbstractMember.cs b/ICSharpCode.NRefactory/TypeSystem/Implementation/AbstractMember.cs index 289feb6703..cf4a3b48cd 100644 --- a/ICSharpCode.NRefactory/TypeSystem/Implementation/AbstractMember.cs +++ b/ICSharpCode.NRefactory/TypeSystem/Implementation/AbstractMember.cs @@ -12,6 +12,10 @@ namespace ICSharpCode.NRefactory.TypeSystem.Implementation /// public abstract class AbstractMember : AbstractFreezable, IMember, ISupportsInterning { + // possible optimizations to reduce the memory usage of AbstractMember: + // - put 'bool isFrozen' into flags + // - store regions in more compact form (e.g. assume both file names are identical; use ushort for columns) + ITypeDefinition declaringTypeDefinition; ITypeReference returnType = SharedTypes.UnknownType; IList attributes; @@ -162,7 +166,17 @@ namespace ICSharpCode.NRefactory.TypeSystem.Implementation } public virtual string Documentation { - get { return null; } + get { + // To save memory, we don't store the documentation provider within the member, + // but simply use our declaring type definition as documentation provider. + // If that fails, we try if the project content is a documentation provider: + IDocumentationProvider provider = declaringTypeDefinition as IDocumentationProvider + ?? declaringTypeDefinition.ProjectContent as IDocumentationProvider; + if (provider != null) + return provider.GetDocumentation(this); + else + return null; + } } public Accessibility Accessibility { diff --git a/ICSharpCode.NRefactory/TypeSystem/Implementation/DefaultProperty.cs b/ICSharpCode.NRefactory/TypeSystem/Implementation/DefaultProperty.cs index 48cc56a740..4e1945d31f 100644 --- a/ICSharpCode.NRefactory/TypeSystem/Implementation/DefaultProperty.cs +++ b/ICSharpCode.NRefactory/TypeSystem/Implementation/DefaultProperty.cs @@ -37,6 +37,8 @@ namespace ICSharpCode.NRefactory.TypeSystem.Implementation public override void PrepareForInterning(IInterningProvider provider) { base.PrepareForInterning(provider); + getter = provider.Intern(getter); + setter = provider.Intern(setter); parameters = provider.InternList(parameters); }