From 4995f487fca97a10f3d33af75f13a1987962893b Mon Sep 17 00:00:00 2001 From: Daniel Grunwald Date: Fri, 15 Apr 2011 00:24:15 +0200 Subject: [PATCH] New XmlDocumentationProvider implementation: Keep only a small index in memory, and read the relevant portion of the .xml file again when the documentation is requested. This means we no longer need to store the documentation in temporary binary files for efficient access. --- .../BinaryDocumentationProvider.cs | 231 ------------------ .../Documentation/XmlDocumentationProvider.cs | 201 ++++++++++++--- .../ICSharpCode.NRefactory.csproj | 1 - 3 files changed, 162 insertions(+), 271 deletions(-) delete mode 100644 ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs diff --git a/ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs b/ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs deleted file mode 100644 index a824396c38..0000000000 --- a/ICSharpCode.NRefactory/Documentation/BinaryDocumentationProvider.cs +++ /dev/null @@ -1,231 +0,0 @@ -// Copyright (c) AlphaSierraPapa for the SharpDevelop Team (for details please see \doc\copyright.txt) -// This code is distributed under MIT X11 license (for details please see \doc\license.txt) - -using System; -using System.Collections.Generic; -using System.Diagnostics; -using System.IO; -using ICSharpCode.NRefactory.TypeSystem; - -namespace ICSharpCode.NRefactory.Documentation -{ - /// - /// Provides xml documentation from a binary cache file. - /// This allows providing XML documentation without having to read the whole documentation into memory. - /// - public class BinaryDocumentationProvider : IDisposable, IDocumentationProvider - { - struct IndexEntry - { - public readonly int HashCode; - public readonly int FileLocation; - - public IndexEntry(int HashCode, int FileLocation) - { - this.HashCode = HashCode; - this.FileLocation = FileLocation; - } - } - - #region Save binary files - // FILE FORMAT FOR BINARY DOCUMENTATION - // long magic = 0x4244636f446c6d58 (identifies file type = 'XmlDocDB') - const long magic = 0x4244636f446c6d58; - // short version = 3 (file version) - const short version = 3; - // long fileDate (last change date of xml file in DateTime ticks) - // int testHashCode = magicTestString.GetHashCode() // (check if hash-code implementation is compatible) - const string magicTestString = "HashMe-XmlDocDB"; - // int entryCount (count of entries) - // int indexPointer (points to location where index starts in the file) - // { - // string key (documentation key as length-prefixed string) - // string docu (xml documentation as length-prefixed string) - // } - // indexPointer points to the start of the following section: - // { - // int hashcode - // int index (index where the docu string starts in the file) - // } - - - /// - /// Saves the xml documentation into a on-disk database file. - /// - /// Filename of the database - /// Last-modified date of the .xml file - /// The xml documentation that should be written to disk. - public static void Save(string fileName, DateTime fileDate, IEnumerable> xmlDocumentation) - { - if (fileName == null) - throw new ArgumentNullException("fileName"); - if (xmlDocumentation == null) - throw new ArgumentNullException("xmlDocumentation"); - using (FileStream fs = new FileStream(fileName, FileMode.Create, FileAccess.Write, FileShare.None)) { - using (BinaryWriter w = new BinaryWriter(fs)) { - w.Write(magic); - w.Write(version); - w.Write(fileDate.Ticks); - w.Write(magicTestString.GetHashCode()); - - List index = new List(); - int indexLengthPos = (int)fs.Position; - w.Write(0); // skip 4 bytes for index length - w.Write(0); // skip 4 bytes for index pointer - - int i = 0; - foreach (KeyValuePair p in xmlDocumentation) { - index.Add(new IndexEntry(p.Key.GetHashCode(), (int)fs.Position)); - w.Write(p.Key); - w.Write(p.Value.Trim()); - i += 1; - } - - index.Sort((a,b) => a.HashCode.CompareTo(b.HashCode)); - - int indexStart = (int)fs.Position; - foreach (IndexEntry entry in index) { - w.Write(entry.HashCode); - w.Write(entry.FileLocation); - } - w.Seek(indexLengthPos, SeekOrigin.Begin); - w.Write(index.Count); // write index length - w.Write(indexStart); // write index count - } - } - } - #endregion - - BinaryReader loader; - FileStream fs; - - Dictionary xmlDescription = new Dictionary(); - IndexEntry[] index; // SORTED array of index entries - - const int cacheLength = 50; // number of strings to cache when working in file-mode - Queue keyCacheQueue = new Queue(cacheLength); - - #region Load binary files - private BinaryDocumentationProvider() {} - - /// - /// Loads binary documentation. - /// - /// - /// Don't forget to dispose the BinaryDocumentationProvider. - /// - /// The name of the binary cache file. - /// The file date of the original XML file. Loading will fail if the cached data was generated - /// from a different file date than the original XML file. - /// - /// The BinaryDocumentationProvider representing the file's content; or null if loading failed. - /// - public static BinaryDocumentationProvider Load(string fileName, DateTime fileDate) - { - BinaryDocumentationProvider doc = new BinaryDocumentationProvider(); - try { - doc.fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete); - int len = (int)doc.fs.Length; - BinaryReader loader = doc.loader = new BinaryReader(doc.fs); - if (loader.ReadInt64() != magic) { - Debug.WriteLine("Cannot load XmlDoc: wrong magic"); - return null; - } - if (loader.ReadInt16() != version) { - Debug.WriteLine("Cannot load XmlDoc: wrong version"); - return null; - } - if (loader.ReadInt64() != fileDate.Ticks) { - Debug.WriteLine("Not loading XmlDoc: file changed since cache was created"); - return null; - } - int count = loader.ReadInt32(); - int indexStartPosition = loader.ReadInt32(); // go to start of index - if (indexStartPosition <= 0 || indexStartPosition >= len) { - Debug.WriteLine("XmlDoc: Cannot find index, cache invalid!"); - return null; - } - doc.fs.Position = indexStartPosition; - IndexEntry[] index = new IndexEntry[count]; - for (int i = 0; i < index.Length; i++) { - index[i] = new IndexEntry(loader.ReadInt32(), loader.ReadInt32()); - } - doc.index = index; - return doc; - } catch (IOException ex) { - Debug.WriteLine("Cannot load from cache" + ex.ToString()); - return null; - } - } - - string LoadDocumentation(string key) - { - if (keyCacheQueue.Count > cacheLength - 1) { - xmlDescription.Remove(keyCacheQueue.Dequeue()); - } - - int hashcode = key.GetHashCode(); - - // use binary search to find the item - string resultDocu = null; - - int m = Array.BinarySearch(index, new IndexEntry(hashcode, 0)); - if (m >= 0) { - // correct hash code found. - // possibly there are multiple items with the same hash, so go to the first. - while (--m >= 0 && index[m].HashCode == hashcode); - // go through all items that have the correct hash - while (++m < index.Length && index[m].HashCode == hashcode) { - fs.Position = index[m].FileLocation; - string keyInFile = loader.ReadString(); - if (keyInFile == key) { - //LoggingService.Debug("Got XML documentation for " + key); - resultDocu = loader.ReadString(); - break; - } else { - // this is a harmless hash collision, just continue reading - Debug.WriteLine("Found " + keyInFile + " instead of " + key); - } - } - } - - keyCacheQueue.Enqueue(key); - xmlDescription.Add(key, resultDocu); - - return resultDocu; - } - #endregion - - public string GetDocumentation(string key) - { - lock (xmlDescription) { - if (index == null) - throw new ObjectDisposedException("BinaryDocumentationProvider"); - string result; - if (xmlDescription.TryGetValue(key, out result)) - return result; - return LoadDocumentation(key); - } - } - - public string GetDocumentation(IEntity entity) - { - return GetDocumentation(XmlDocumentationProvider.GetDocumentationKey(entity)); - } - - public void Dispose() - { - lock (xmlDescription) { - if (loader != null) { - loader.Close(); - fs.Close(); - } - xmlDescription.Clear(); - index = null; - keyCacheQueue = null; - loader = null; - fs = null; - } - } - } -} diff --git a/ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs b/ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs index 8fc369d41c..61d3ecbc3d 100644 --- a/ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs +++ b/ICSharpCode.NRefactory/Documentation/XmlDocumentationProvider.cs @@ -14,29 +14,90 @@ namespace ICSharpCode.NRefactory.Documentation /// /// Provides documentation from an .xml file (as generated by the Microsoft C# compiler). /// + /// + /// This class first creates an in-memory index of the .xml file, and then uses that to read only the requested members. + /// This way, we avoid keeping all the documentation in memory. + /// public class XmlDocumentationProvider : IDocumentationProvider { - readonly IDictionary xmlDocumentation; + #region Cache + sealed class XmlDocumentationCache + { + readonly KeyValuePair[] entries; + int pos; + + public XmlDocumentationCache(int size = 50) + { + if (size <= 0) + throw new ArgumentOutOfRangeException("size", size, "Value must be positive"); + this.entries = new KeyValuePair[size]; + } + + internal string Get(string key) + { + foreach (var pair in entries) { + if (pair.Key == key) + return pair.Value; + } + return null; + } + + internal void Add(string key, string value) + { + entries[pos++] = new KeyValuePair(key, value); + if (pos == entries.Length) + pos = 0; + } + } + #endregion + + struct IndexEntry : IComparable + { + internal readonly int HashCode; + internal readonly int PositionInFile; + + internal IndexEntry(int hashCode, int positionInFile) + { + this.HashCode = hashCode; + this.PositionInFile = positionInFile; + } + + public int CompareTo(IndexEntry other) + { + return this.HashCode.CompareTo(other.HashCode); + } + } - #region Load From File + readonly XmlDocumentationCache cache = new XmlDocumentationCache(); + readonly string fileName; + IndexEntry[] index; // SORTED array of index entries + + #region Constructor / Redirection support + /// + /// Creates a new XmlDocumentationProvider. + /// + /// Name of the .xml file. public XmlDocumentationProvider(string fileName) { if (fileName == null) throw new ArgumentNullException("fileName"); - this.xmlDocumentation = new Dictionary(); + using (XmlTextReader xmlReader = new XmlTextReader(fileName)) { + xmlReader.XmlResolver = null; // no DTD resolving xmlReader.MoveToContent(); if (string.IsNullOrEmpty(xmlReader.GetAttribute("redirect"))) { + this.fileName = fileName; ReadXmlDoc(xmlReader); } else { string redirectionTarget = GetRedirectionTarget(xmlReader.GetAttribute("redirect")); if (redirectionTarget != null) { Debug.WriteLine("XmlDoc " + fileName + " is redirecting to " + redirectionTarget); using (XmlTextReader redirectedXmlReader = new XmlTextReader(redirectionTarget)) { + this.fileName = redirectionTarget; ReadXmlDoc(redirectedXmlReader); } } else { - Debug.WriteLine("XmlDoc " + fileName + " is redirecting to " + xmlReader.GetAttribute("redirect") + ", but that file was not found."); + throw new XmlException("XmlDoc " + fileName + " is redirecting to " + xmlReader.GetAttribute("redirect") + ", but that file was not found."); } } } @@ -95,36 +156,53 @@ namespace ICSharpCode.NRefactory.Documentation } #endregion - #region Load from XmlReader - public XmlDocumentationProvider(XmlReader reader) - { - if (reader == null) - throw new ArgumentNullException("reader"); - this.xmlDocumentation = new Dictionary(); - ReadXmlDoc(reader); - } - - public XmlDocumentationProvider(IDictionary dictionary) + #region Load / Create Index + void ReadXmlDoc(XmlTextReader reader) { - if (dictionary == null) - throw new ArgumentNullException("dictionary"); - this.xmlDocumentation = dictionary; + // Open up a second file stream for the line<->position mapping + using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { + LinePositionMapper linePosMapper = new LinePositionMapper(fs); + List indexList = new List(); + while (reader.Read()) { + if (reader.IsStartElement()) { + switch (reader.LocalName) { + case "members": + ReadMembersSection(reader, linePosMapper, indexList); + break; + } + } + } + indexList.Sort(); + this.index = indexList.ToArray(); + } } - void ReadXmlDoc(XmlReader reader) + sealed class LinePositionMapper { - while (reader.Read()) { - if (reader.IsStartElement()) { - switch (reader.LocalName) { - case "members": - ReadMembersSection(reader); - break; + readonly FileStream fs; + int currentLine = 1; + + public LinePositionMapper(FileStream fs) + { + this.fs = fs; + } + + public int GetPositionForLine(int line) + { + Debug.Assert(line >= currentLine); + while (line > currentLine) { + int b = fs.ReadByte(); + if (b < 0) + throw new EndOfStreamException(); + if (b == '\n') { + currentLine++; } } + return checked((int)fs.Position); } } - void ReadMembersSection(XmlReader reader) + void ReadMembersSection(XmlTextReader reader, LinePositionMapper linePosMapper, List indexList) { while (reader.Read()) { switch (reader.NodeType) { @@ -135,9 +213,11 @@ namespace ICSharpCode.NRefactory.Documentation break; case XmlNodeType.Element: if (reader.LocalName == "member") { - string memberAttr = reader.GetAttribute(0); - string innerXml = reader.ReadInnerXml(); - xmlDocumentation[memberAttr] = innerXml; + int pos = linePosMapper.GetPositionForLine(reader.LineNumber) + Math.Max(reader.LinePosition - 2, 0); + string memberAttr = reader.GetAttribute("name"); + if (memberAttr != null) + indexList.Add(new IndexEntry(memberAttr.GetHashCode(), pos)); + reader.Skip(); } break; } @@ -145,35 +225,78 @@ namespace ICSharpCode.NRefactory.Documentation } #endregion - /// - /// Gets all entries in the documentation file. - /// - public IDictionary XmlDocumentation { - get { return xmlDocumentation; } + #region GetDocumentation + /// + public string GetDocumentation(IEntity entity) + { + return GetDocumentation(GetDocumentationKey(entity)); } + /// + /// Get the documentation for the member with the specified documentation key. + /// public string GetDocumentation(string key) { if (key == null) throw new ArgumentNullException("key"); - string result; - if (xmlDocumentation.TryGetValue(key, out result)) - return result; - else + int hashcode = key.GetHashCode(); + // index is sorted, so we can use binary search + int m = Array.BinarySearch(index, new IndexEntry(hashcode, 0)); + if (m < 0) return null; + // correct hash code found. + // possibly there are multiple items with the same hash, so go to the first. + while (--m >= 0 && index[m].HashCode == hashcode); + // m is now 1 before the first item with the correct hash + + lock (cache) { + string val = cache.Get(key); + if (val == null) { + // go through all items that have the correct hash + while (++m < index.Length && index[m].HashCode == hashcode) { + val = LoadDocumentation(key, index[m].PositionInFile); + if (val != null) + break; + } + // cache the result (even if it is null) + cache.Add(key, val); + } + return val; + } } + #endregion - public string GetDocumentation(IEntity entity) + #region Load / Read XML + string LoadDocumentation(string key, int positionInFile) { - return GetDocumentation(GetDocumentationKey(entity)); + using (FileStream fs = new FileStream(fileName, FileMode.Open, FileAccess.Read, FileShare.Read | FileShare.Delete)) { + fs.Position = positionInFile; + using (XmlTextReader r = new XmlTextReader(fs, XmlNodeType.Element, null)) { + r.XmlResolver = null; // no DTD resolving + while (r.Read()) { + if (r.NodeType == XmlNodeType.Element) { + string memberAttr = r.GetAttribute("name"); + if (memberAttr == key) { + return r.ReadInnerXml(); + } else { + return null; + } + } + } + return null; + } + } } + #endregion + #region GetDocumentationKey public static string GetDocumentationKey(IEntity entity) { if (entity == null) throw new ArgumentNullException("entity"); throw new NotImplementedException(); } + #endregion } } diff --git a/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj b/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj index 0ac90b2d37..1f7bc4cc0d 100644 --- a/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj +++ b/ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj @@ -197,7 +197,6 @@ -