Implemented incremental tag soup parser.

ICSharpCode.NRefactory.ConsistencyCheck/Program.cs

@@ -36,9 +36,9 @@ namespace ICSharpCode.NRefactory.ConsistencyCheck
 			@"C:\Program Files (x86)\GtkSharp\2.12\lib\Mono.Posix",
 			@"C:\work\SD\src\Tools\NUnit"
 		};
-		//public const string SolutionFile = @"C:\work\NRefactory\NRefactory.sln";
+		public const string SolutionFile = @"C:\work\NRefactory\NRefactory.sln";
 		//public const string SolutionFile = @"C:\work\SD\SharpDevelop.sln";
-		public const string SolutionFile = @"C:\work\ILSpy\ILSpy.sln";
+		//public const string SolutionFile = @"C:\work\ILSpy\ILSpy.sln";
 		
 		public const string TempPath = @"C:\temp";
 		
@@ -46,7 +46,7 @@ namespace ICSharpCode.NRefactory.ConsistencyCheck
 		
 		public static void Main(string[] args)
 		{
-			TagSoupIncrementalTests.Run("c:\\temp\\ClosedXML.xml");
+			TagSoupIncrementalTests.Run("ICSharpCode.NRefactory.xml");
 			
 			using (new Timer("Loading solution... ")) {
 				solution = new Solution(SolutionFile);

ICSharpCode.NRefactory.ConsistencyCheck/Xml/TagSoupIncrementalTests.cs

@@ -52,17 +52,36 @@ namespace ICSharpCode.NRefactory.ConsistencyCheck
 			StringBuilder b = new StringBuilder(originalXmlFile.Text);
 			IncrementalParserState parserState = null;
 			TestTextSourceVersion version = new TestTextSourceVersion();
-			for (int iteration = 0; iteration < 100; iteration++) {
+			int totalCharactersParsed = 0;
+			int totalCharactersChanged = originalXmlFile.TextLength;
+			TimeSpan incrementalParseTime = TimeSpan.Zero;
+			TimeSpan nonIncrementalParseTime = TimeSpan.Zero;
+			Stopwatch w = new Stopwatch();
+			for (int iteration = 0; iteration < 500; iteration++) {
+				totalCharactersParsed += b.Length;
 				var textSource = new TextSourceWithVersion(new StringTextSource(b.ToString()), version);
+				w.Restart();
 				var incrementalResult = parser.ParseIncremental(parserState, textSource, out parserState);
+				w.Stop();
+				incrementalParseTime += w.Elapsed;
+				w.Restart();
 				var nonIncrementalResult = parser.Parse(textSource);
+				w.Stop();
+				nonIncrementalParseTime += w.Elapsed;
 				CompareResults(incrementalResult, nonIncrementalResult);
+				
+				new ValidationVisitor(textSource).Visit(incrementalResult);
+				
 				// Randomly mutate the file:
 				
 				List<TextChangeEventArgs> changes = new List<TextChangeEventArgs>();
-				int modifications = rnd.Next(0, 10);
+				int modifications = rnd.Next(0, 25);
+				int offset = 0;
 				for (int i = 0; i < modifications; i++) {
-					int offset = rnd.Next(0, b.Length);
+					if (i == 0 || rnd.Next(0, 10) == 0)
+						offset = rnd.Next(0, b.Length);
+					else
+						offset += rnd.Next(0, Math.Min(10, b.Length - offset));
 					int originalOffset = rnd.Next(0, originalXmlFile.TextLength);
 					int insertionLength;
 					int removalLength;
@@ -72,7 +91,7 @@ namespace ICSharpCode.NRefactory.ConsistencyCheck
 							insertionLength = rnd.Next(0, Math.Min(50, originalXmlFile.TextLength - originalOffset));
 							break;
 						case 1:
-							removalLength = rnd.Next(0, Math.Min(10, b.Length - offset));
+							removalLength = rnd.Next(0, Math.Min(20, b.Length - offset));
 							insertionLength = rnd.Next(0, Math.Min(20, originalXmlFile.TextLength - originalOffset));
 							break;
 						default:
@@ -85,9 +104,12 @@ namespace ICSharpCode.NRefactory.ConsistencyCheck
 					string insertedText = originalXmlFile.GetText(originalOffset, insertionLength);
 					b.Insert(offset, insertedText);
 					changes.Add(new TextChangeEventArgs(offset, removedText, insertedText));
+					totalCharactersChanged += insertionLength;
 				}
 				version = new TestTextSourceVersion(version, changes);
 			}
+			Console.WriteLine("Incremental parse time:     " + incrementalParseTime + " for " + totalCharactersChanged + " characters changed");
+			Console.WriteLine("Non-Incremental parse time: " + nonIncrementalParseTime + " for " + totalCharactersParsed + " characters");
 		}
 		
 		static void CompareResults(IList<AXmlObject> result1, IList<AXmlObject> result2)
@@ -148,5 +170,45 @@ namespace ICSharpCode.NRefactory.ConsistencyCheck
 			
 			CompareResults(obj1.Children, obj2.Children);
 		}
+		
+		sealed class ValidationVisitor : IAXmlVisitor
+		{
+			readonly ITextSource textSource;
+			
+			public ValidationVisitor(ITextSource textSource)
+			{
+				this.textSource = textSource;
+			}
+			
+			public void Visit(IList<AXmlObject> objects)
+			{
+				for (int i = 0; i < objects.Count; i++) {
+					objects[i].AcceptVisitor(this);
+				}
+			}
+			
+			public void VisitTag(AXmlTag tag)
+			{
+				if (textSource.GetText(tag.StartOffset, tag.OpeningBracket.Length) != tag.OpeningBracket)
+					throw new InvalidOperationException();
+				if (textSource.GetText(tag.NameSegment) != tag.Name)
+					throw new InvalidOperationException();
+				if (textSource.GetText(tag.EndOffset - tag.ClosingBracket.Length, tag.ClosingBracket.Length) != tag.ClosingBracket)
+					throw new InvalidOperationException();
+				Visit(tag.Children);
+			}
+			
+			public void VisitAttribute(AXmlAttribute attribute)
+			{
+				if (textSource.GetText(attribute.NameSegment) != attribute.Name)
+					throw new InvalidOperationException();
+				Visit(attribute.Children);
+			}
+			
+			public void VisitText(AXmlText text)
+			{
+				Visit(text.Children);
+			}
+		}
 	}
 }

ICSharpCode.NRefactory.Xml/IncrementalParserState.cs

@@ -17,6 +17,9 @@
 // DEALINGS IN THE SOFTWARE.
 
 using System;
+using System.Collections.Generic;
+using System.Diagnostics;
+using ICSharpCode.NRefactory.Editor;
 
 namespace ICSharpCode.NRefactory.Xml
 {
@@ -25,8 +28,88 @@ namespace ICSharpCode.NRefactory.Xml
 	/// </summary>
 	public class IncrementalParserState
 	{
-		internal IncrementalParserState()
+		internal readonly int TextLength;
+		internal readonly ITextSourceVersion Version;
+		internal readonly InternalObject[] Objects;
+		
+		internal IncrementalParserState(int textLength, ITextSourceVersion version, InternalObject[] objects)
 		{
+			this.TextLength = textLength;
+			this.Version = version;
+			this.Objects = objects;
+		}
+		
+		internal List<UnchangedSegment> GetReuseMapTo(ITextSourceVersion newVersion)
+		{
+			ITextSourceVersion oldVersion = this.Version;
+			if (oldVersion == null || newVersion == null)
+				return null;
+			if (!oldVersion.BelongsToSameDocumentAs(newVersion))
+				return null;
+			List<UnchangedSegment> reuseMap = new List<UnchangedSegment>();
+			reuseMap.Add(new UnchangedSegment(0, 0, this.TextLength));
+			foreach (var change in oldVersion.GetChangesTo(newVersion)) {
+				bool needsSegmentRemoval = false;
+				for (int i = 0; i < reuseMap.Count; i++) {
+					UnchangedSegment segment = reuseMap[i];
+					if (segment.NewOffset + segment.Length <= change.Offset) {
+						// change is completely after this segment
+						continue;
+					}
+					if (change.Offset + change.RemovalLength <= segment.NewOffset) {
+						// change is completely before this segment
+						segment.NewOffset += change.InsertionLength - change.RemovalLength;
+						reuseMap[i] = segment;
+						continue;
+					}
+					// Change is overlapping segment.
+					// Split segment into two parts: the part before change, and the part after change.
+					var segmentBefore = new UnchangedSegment(segment.OldOffset, segment.NewOffset, change.Offset - segment.NewOffset);
+					Debug.Assert(segmentBefore.Length < segment.Length);
+					
+					int lengthAtEnd = segment.NewOffset + segment.Length - (change.Offset + change.RemovalLength);
+					var segmentAfter = new UnchangedSegment(
+						segment.OldOffset + segment.Length - lengthAtEnd,
+						change.Offset + change.InsertionLength,
+						lengthAtEnd);
+					Debug.Assert(segmentAfter.Length < segment.Length);
+					Debug.Assert(segmentBefore.Length + segmentAfter.Length <= segment.Length);
+					Debug.Assert(segmentBefore.NewOffset + segmentBefore.Length <= segmentAfter.NewOffset);
+					Debug.Assert(segmentBefore.OldOffset + segmentBefore.Length <= segmentAfter.OldOffset);
+					if (segmentBefore.Length > 0 && segmentAfter.Length > 0) {
+						reuseMap[i] = segmentBefore;
+						reuseMap.Insert(++i, segmentAfter);
+					} else if (segmentBefore.Length > 0) {
+						reuseMap[i] = segmentBefore;
+					} else {
+						reuseMap[i] = segmentAfter;
+						if (segmentAfter.Length <= 0)
+							needsSegmentRemoval = true;
+					}
+				}
+				if (needsSegmentRemoval)
+					reuseMap.RemoveAll(s => s.Length <= 0);
+			}
+			return reuseMap;
+		}
+	}
+	
+	struct UnchangedSegment
+	{
+		public int OldOffset;
+		public int NewOffset;
+		public int Length;
+		
+		public UnchangedSegment(int oldOffset, int newOffset, int length)
+		{
+			this.OldOffset = oldOffset;
+			this.NewOffset = newOffset;
+			this.Length = length;
+		}
+		
+		public override string ToString()
+		{
+			return string.Format("[UnchangedSegment OldOffset={0}, NewOffset={1}, Length={2}]", OldOffset, NewOffset, Length);
 		}
 	}
 }

ICSharpCode.NRefactory.Xml/InternalDocument.cs

@@ -20,14 +20,16 @@ using System;
 
 namespace ICSharpCode.NRefactory.Xml
 {
-	abstract class InternalObject
+	internal abstract class InternalObject
 	{
-		internal int StartRelativeToParent;
-		internal int Length;
-		internal InternalSyntaxError[] SyntaxErrors;
-		internal InternalObject[] NestedObjects;
+		public int StartRelativeToParent;
+		public int Length;
+		/// <summary>Length that was touched to parsed this object.</summary>
+		public int LengthTouched;
+		public InternalSyntaxError[] SyntaxErrors;
+		public InternalObject[] NestedObjects;
 		
-		internal InternalObject SetStartRelativeToParent(int newStartRelativeToParent)
+		public InternalObject SetStartRelativeToParent(int newStartRelativeToParent)
 		{
 			if (newStartRelativeToParent == StartRelativeToParent)
 				return this;
@@ -41,9 +43,9 @@ namespace ICSharpCode.NRefactory.Xml
 	
 	sealed class InternalText : InternalObject
 	{
-		internal TextType Type;
-		internal bool ContainsOnlyWhitespace;
-		internal string Value;
+		public TextType Type;
+		public bool ContainsOnlyWhitespace;
+		public string Value;
 		
 		public override AXmlObject CreatePublicObject(AXmlObject parent, int parentStartOffset)
 		{
@@ -53,10 +55,10 @@ namespace ICSharpCode.NRefactory.Xml
 	
 	sealed class InternalTag : InternalObject
 	{
-		internal string OpeningBracket;
-		internal int RelativeNameStart;
-		internal string Name;
-		internal string ClosingBracket;
+		public string OpeningBracket;
+		public int RelativeNameStart;
+		public string Name;
+		public string ClosingBracket;
 		
 		/// <summary> True if tag starts with "&lt;" </summary>
 		public bool IsStartOrEmptyTag       { get { return OpeningBracket == "<"; } }
@@ -81,9 +83,9 @@ namespace ICSharpCode.NRefactory.Xml
 	
 	struct InternalSyntaxError
 	{
-		internal readonly int RelativeStart;
-		internal readonly int RelativeEnd;
-		internal readonly string Description;
+		public readonly int RelativeStart;
+		public readonly int RelativeEnd;
+		public readonly string Description;
 		
 		public InternalSyntaxError(int relativeStart, int relativeEnd, string description)
 		{
@@ -95,9 +97,9 @@ namespace ICSharpCode.NRefactory.Xml
 	
 	class InternalAttribute : InternalObject
 	{
-		internal string Name;
-		internal int EqualsSignLength; // length of equals sign including the surrounding whitespace
-		internal string Value;
+		public string Name;
+		public int EqualsSignLength; // length of equals sign including the surrounding whitespace
+		public string Value;
 		
 		public override AXmlObject CreatePublicObject(AXmlObject parent, int parentStartOffset)
 		{

ICSharpCode.NRefactory.Xml/TagReader.cs

@@ -18,6 +18,7 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.Linq;
 using System.Text;
@@ -39,6 +40,54 @@ namespace ICSharpCode.NRefactory.Xml
 			while (HasMoreData()) {
 				ReadObject();
 			}
+			for (int i = 0; i < objects.Count; i++) {
+				objects[i].StartRelativeToParent = 0;
+			}
+			var arr = objects.ToArray();
+			objects.Clear();
+			return arr;
+		}
+		
+		public InternalObject[] ReadAllObjectsIncremental(InternalObject[] oldObjects, List<UnchangedSegment> reuseMap)
+		{
+			int oldObjectIndex = 0;
+			int oldObjectPosition = 0;
+			int reuseMapIndex = 0;
+			while (reuseMapIndex < reuseMap.Count) {
+				var reuseEntry = reuseMap[reuseMapIndex];
+				while (this.CurrentLocation < reuseEntry.NewOffset) {
+					ReadObject();
+				}
+				if (this.CurrentLocation >= reuseEntry.NewOffset + reuseEntry.Length) {
+					reuseMapIndex++;
+					continue;
+				}
+				Debug.Assert(reuseEntry.NewOffset <= this.CurrentLocation && this.CurrentLocation < reuseEntry.NewOffset + reuseEntry.Length);
+				// reuse the nodes within this reuseEntry starting at oldOffset:
+				int oldOffset = this.CurrentLocation - reuseEntry.NewOffset + reuseEntry.OldOffset;
+				// seek to oldOffset in the oldObjects array:
+				while (oldObjectPosition < oldOffset && oldObjectIndex < oldObjects.Length) {
+					oldObjectPosition += oldObjects[oldObjectIndex++].Length;
+				}
+				if (oldObjectPosition == oldOffset) {
+					// reuse old objects within this reuse entry:
+					int reuseEnd = reuseEntry.OldOffset + reuseEntry.Length;
+					while ((oldObjectIndex < oldObjects.Length) && (oldObjectPosition + oldObjects[oldObjectIndex].LengthTouched < reuseEnd)) {
+						var oldObject = oldObjects[oldObjectIndex++];
+						Debug.Assert(oldObject.StartRelativeToParent == 0);
+						oldObjectPosition += oldObject.Length;
+						objects.Add(oldObject);
+						Skip(oldObject.Length);
+					}
+				}
+				reuseMapIndex++;
+			}
+			while (HasMoreData()) {
+				ReadObject();
+			}
+			for (int i = 0; i < objects.Count; i++) {
+				objects[i].StartRelativeToParent = 0;
+			}
 			var arr = objects.ToArray();
 			objects.Clear();
 			return arr;
@@ -93,7 +142,8 @@ namespace ICSharpCode.NRefactory.Xml
 		
 		void EndInternalObject(InternalObjectFrame frame, bool storeNewObject = true)
 		{
-			frame.InternalObject.Length = this.CurrentRelativeLocation;
+			frame.InternalObject.Length = this.CurrentLocation - internalObjectStartPosition;
+			frame.InternalObject.LengthTouched = this.MaxTouchedLocation - internalObjectStartPosition;
 			frame.InternalObject.SyntaxErrors = GetSyntaxErrors();
 			if (storeNewObject)
 				objects.Add(frame.InternalObject);

ICSharpCode.NRefactory.Xml/TagSoupParser.cs

@@ -18,7 +18,10 @@
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
+using System.Xml.Linq;
 using ICSharpCode.NRefactory.Editor;
+using ICSharpCode.NRefactory.Utils;
 
 namespace ICSharpCode.NRefactory.Xml
 {
@@ -42,6 +45,11 @@ namespace ICSharpCode.NRefactory.Xml
 				throw new ArgumentNullException("textSource");
 			var reader = new TagReader(this, textSource);
 			var internalObjects = reader.ReadAllObjects();
+			return CreatePublic(internalObjects);
+		}
+		
+		IList<AXmlObject> CreatePublic(InternalObject[] internalObjects)
+		{
 			var publicObjects = new AXmlObject[internalObjects.Length];
 			int pos = 0;
 			for (int i = 0; i < internalObjects.Length; i++) {
@@ -62,9 +70,16 @@ namespace ICSharpCode.NRefactory.Xml
 		{
 			if (newTextSource == null)
 				throw new ArgumentNullException("newTextSource");
-			// TODO: incremental parser
-			newParserState = null;
-			return Parse(newTextSource);
+			var reader = new TagReader(this, newTextSource);
+			var reuseMap = oldParserState != null ? oldParserState.GetReuseMapTo(newTextSource.Version) : null;
+			
+			InternalObject[] internalObjects;
+			if (reuseMap != null)
+				internalObjects = reader.ReadAllObjectsIncremental(oldParserState.Objects, reuseMap);
+			else
+				internalObjects = reader.ReadAllObjects();
+			newParserState = new IncrementalParserState(newTextSource.TextLength, newTextSource.Version, internalObjects);
+			return CreatePublic(internalObjects);
 		}
 	}
 }

ICSharpCode.NRefactory.Xml/TokenReader.cs

@@ -45,7 +45,8 @@ namespace ICSharpCode.NRefactory.Xml
 		}
 		
 		public int MaxTouchedLocation {
-			get { return Math.Max(currentLocation, maxTouchedLocation); }
+			// add 1 to currentLocation because single-char-peek does not increment maxTouchedLocation
+			get { return Math.Max(currentLocation + 1, maxTouchedLocation); }
 		}
 		
 		public TokenReader(ITextSource input)
@@ -86,7 +87,8 @@ namespace ICSharpCode.NRefactory.Xml
 		protected void GoBack(int oldLocation)
 		{
 			Log.Assert(oldLocation <= currentLocation, "Trying to move forward");
-			maxTouchedLocation = Math.Max(maxTouchedLocation, currentLocation);
+			// add 1 because single-char-peek does not increment maxTouchedLocation
+			maxTouchedLocation = Math.Max(maxTouchedLocation, currentLocation + 1);
 			currentLocation = oldLocation;
 		}
 		
@@ -150,7 +152,7 @@ namespace ICSharpCode.NRefactory.Xml
 		{
 			if (!TryPeek(text[0])) return false; // Early exit
 			
-			maxTouchedLocation = Math.Max(maxTouchedLocation, currentLocation + (text.Length - 1));
+			maxTouchedLocation = Math.Max(maxTouchedLocation, currentLocation + text.Length);
 			// The following comparison 'touches' the end of file - it does depend on the end being there
 			if (currentLocation + text.Length > inputLength) return false;
 			
@@ -214,7 +216,7 @@ namespace ICSharpCode.NRefactory.Xml
 			if (currentLocation == inputLength) return false;
 			int index = input.IndexOf(text, currentLocation, inputLength - currentLocation, StringComparison.Ordinal);
 			if (index != -1) {
-				maxTouchedLocation = index + text.Length - 1;
+				maxTouchedLocation = index + text.Length;
 				currentLocation = index;
 				return true;
 			} else {

ICSharpCode.NRefactory/ICSharpCode.NRefactory.csproj

@@ -207,6 +207,7 @@
     <Compile Include="Utils\FastSerializer.cs" />
     <Compile Include="Utils\GraphVizGraph.cs" />
     <Compile Include="Utils\ImmutableStack.cs" />
+    <Compile Include="Utils\KeyComparer.cs" />
     <Compile Include="Utils\LazyInit.cs" />
     <Compile Include="Utils\Platform.cs" />
     <Compile Include="Utils\ProjectedList.cs" />

ICSharpCode.NRefactory/Utils/KeyComparer.cs

@@ -0,0 +1,60 @@
+// Copyright (c) AlphaSierraPapa for the SharpDevelop Team
+// 
+// Permission is hereby granted, free of charge, to any person obtaining a copy of this
+// software and associated documentation files (the "Software"), to deal in the Software
+// without restriction, including without limitation the rights to use, copy, modify, merge,
+// publish, distribute, sublicense, and/or sell copies of the Software, and to permit persons
+// to whom the Software is furnished to do so, subject to the following conditions:
+// 
+// The above copyright notice and this permission notice shall be included in all copies or
+// substantial portions of the Software.
+// 
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR IMPLIED,
+// INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, FITNESS FOR A PARTICULAR
+// PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE
+// FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR
+// OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER
+// DEALINGS IN THE SOFTWARE.
+
+using System;
+using System.Collections.Generic;
+
+namespace ICSharpCode.NRefactory.Utils
+{
+	public static class KeyComparer
+	{
+		public static KeyComparer<TElement, TKey> Create<TElement, TKey>(Func<TElement, TKey> keySelector)
+		{
+			return new KeyComparer<TElement, TKey>(keySelector, Comparer<TKey>.Default, EqualityComparer<TKey>.Default);
+		}
+	}
+	
+	public class KeyComparer<TElement, TKey> : IComparer<TElement>, IEqualityComparer<TElement>
+	{
+		readonly Func<TElement, TKey> keySelector;
+		readonly IComparer<TKey> keyComparer;
+		readonly IEqualityComparer<TKey> keyEqualityComparer;
+		
+		public KeyComparer(Func<TElement, TKey> keySelector, IComparer<TKey> keyComparer, IEqualityComparer<TKey> keyEqualityComparer)
+		{
+			this.keySelector = keySelector;
+			this.keyComparer = keyComparer;
+			this.keyEqualityComparer = keyEqualityComparer;
+		}
+		
+		public int Compare(TElement x, TElement y)
+		{
+			return keyComparer.Compare(keySelector(x), keySelector(y));
+		}
+		
+		public bool Equals(TElement x, TElement y)
+		{
+			return keyEqualityComparer.Equals(keySelector(x), keySelector(y));
+		}
+		
+		public int GetHashCode(TElement obj)
+		{
+			return keyEqualityComparer.GetHashCode(keySelector(obj));
+		}
+	}
+}

ICSharpCode.NRefactory/Utils/ReferenceComparer.cs

@@ -22,7 +22,7 @@ using System.Runtime.CompilerServices;
 
 namespace ICSharpCode.NRefactory.Utils
 {
-	sealed class ReferenceComparer : IEqualityComparer<object>
+	public sealed class ReferenceComparer : IEqualityComparer<object>
 	{
 		public readonly static ReferenceComparer Instance = new ReferenceComparer();