implemented detection of implicit line continuations

git-svn-id: svn://svn.sharpdevelop.net/sharpdevelop/branches/vbnet@6024 1ccf3a8d-04fe-1044-b7c0-cef0b8235c61

src/Libraries/NRefactory/Project/Src/Lexer/AbstractLexer.cs

@@ -228,7 +228,6 @@ namespace ICSharpCode.NRefactory.Parser
 //			Console.WriteLine("Call to Peek");
 			if (peekToken.next == null) {
 				peekToken.next = Next();
-				specialTracker.InformToken(peekToken.next.kind);
 			}
 			peekToken = peekToken.next;
 			return peekToken;
@@ -242,7 +241,6 @@ namespace ICSharpCode.NRefactory.Parser
 		{
 			if (curToken == null) {
 				curToken = Next();
-				specialTracker.InformToken(curToken.kind);
 				//Console.WriteLine(ICSharpCode.NRefactory.Parser.CSharp.Tokens.GetTokenString(curToken.kind) + " -- " + curToken.val + "(" + curToken.kind + ")");
 				return curToken;
 			}
@@ -251,9 +249,6 @@ namespace ICSharpCode.NRefactory.Parser
 			
 			if (curToken.next == null) {
 				curToken.next = Next();
-				if (curToken.next != null) {
-					specialTracker.InformToken(curToken.next.kind);
-				}
 			}
 			
 			curToken  = curToken.next;

src/Libraries/NRefactory/Project/Src/Lexer/Special/SpecialTracker.cs

@@ -26,11 +26,6 @@ namespace ICSharpCode.NRefactory.Parser
 			}
 		}
 		
-		public void InformToken(int kind)
-		{
-			
-		}
-		
 		/// <summary>
 		/// Gets the specials from the SpecialTracker and resets the lists.
 		/// </summary>

src/Libraries/NRefactory/Project/Src/Lexer/VBNet/Lexer.cs

@@ -1,12 +1,13 @@
 // <file>
 //     <copyright see="prj:///doc/copyright.txt"/>
 //     <license see="prj:///doc/license.txt"/>
-//     <owner name="Andrea Paatz" email="andrea@icsharpcode.net"/>
+//     <owner name="Siegfried Pammer" email="siegfriedpammer@gmail.com" />
 //     <version>$Revision$</version>
 // </file>
 
 using System;
 using System.Collections.Generic;
+using System.Diagnostics;
 using System.Globalization;
 using System.IO;
 using System.Linq;
@@ -22,43 +23,12 @@ namespace ICSharpCode.NRefactory.Parser.VB
 	{
 		bool lineEnd = true;
 		bool isAtLineBegin = false; // TODO: handle line begin, if neccessarry
+		bool misreadExclamationMarkAsTypeCharacter;
+		bool encounteredLineContinuation;
 		
 		ExpressionFinder ef;
 		
-		public Lexer(TextReader reader) : base(reader)
+		#region XmlModeStackInfo
-		{
-			ef = new ExpressionFinder();
-		}
-		
-		public override Token NextToken()
-		{
-			if (curToken == null) { // first call of NextToken()
-				curToken = Next();
-				specialTracker.InformToken(curToken.kind);
-				//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
-				return curToken;
-			}
-			
-			lastToken = curToken;
-			
-			if (curToken.next == null) {
-				curToken.next = Next();
-				specialTracker.InformToken(curToken.next.kind);
-			}
-			
-			curToken = curToken.next;
-			
-			if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
-				curToken = new Token(Tokens.EOL, curToken.col, curToken.line, string.Empty);
-				specialTracker.InformToken(curToken.kind);
-				curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, string.Empty);
-				specialTracker.InformToken(curToken.next.kind);
-			}
-			//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
-			return curToken;
-		}
-		
-		bool misreadExclamationMarkAsTypeCharacter;
 		bool inXmlMode;
 		
 		class XmlModeStackInfo {
@@ -73,6 +43,12 @@ namespace ICSharpCode.NRefactory.Parser.VB
 		}
 		
 		Stack<XmlModeStackInfo> xmlModeStack = new Stack<XmlModeStackInfo>();
+		#endregion
+		
+		public Lexer(TextReader reader) : base(reader)
+		{
+			ef = new ExpressionFinder();
+		}
 		
 		Token NextInternal()
 		{
@@ -202,6 +178,7 @@ namespace ICSharpCode.NRefactory.Parser.VB
 								lineEnd = false;
 								return new Token(Tokens.Identifier, x, y, s);
 							}
+							encounteredLineContinuation = true;
 							ch = (char)ReaderRead();
 							
 							bool oldLineEnd = lineEnd;
@@ -389,82 +366,128 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			}
 		}
 		
-		Token ReadXmlProcessingInstruction(int x, int y)
+		Token prevToken;
-		{
-			sb.Length = 0;
-			int nextChar = -1;
-			
-			while (ReaderPeek() != '?' || ReaderPeek(1) != '>') {
-				nextChar = ReaderRead();
-				if (nextChar == -1)
-					break;
-				sb.Append((char)nextChar);
-			}
-			
-			ReaderSkip("?>".Length);
-			
-			return new Token(Tokens.XmlProcessingInstruction, new Location(x, y), new Location(Col, Line), sb.ToString(), null, LiteralFormat.None);
-		}
 		
-		Token ReadXmlCommentOrCData(int x, int y)
+		protected override Token Next()
 		{
-			sb.Length = 0;
+			Token t = NextInternal();
-			int nextChar = -1;
+			if (t.kind == Tokens.EOL) {
-			
+				Debug.Assert(t.next == null); // NextInternal() must return only 1 token
-			if (string.CompareOrdinal(ReaderPeekString("--".Length), "--") == 0) {
+				t.next = NextInternal();
-				ReaderSkip("--".Length);
+				Debug.Assert(t.next.next == null);
-				while ((nextChar = ReaderRead()) != -1) {
+				if (SkipEOL(prevToken, t.next)) {
-					sb.Append((char)nextChar);
+					t = t.next;
-					if (string.CompareOrdinal(ReaderPeekString("-->".Length), "-->") == 0) {
-						ReaderSkip("-->".Length);
-						return new Token(Tokens.XmlComment, new Location(x, y), new Location(Col, Line), sb.ToString(), null, LiteralFormat.None);
-					}
 				}
 			}
-			
+			// inform EF only once we're sure it's really a token
-			if (string.CompareOrdinal(ReaderPeekString("[CDATA[".Length), "[CDATA[") == 0) {
+			// this means we inform it about EOL tokens "1 token too late", but that's not a problem because
-				ReaderSkip("[CDATA[".Length);
+			// XML literals cannot start immediately after an EOL token
-				while ((nextChar = ReaderRead()) != -1) {
+			ef.InformToken(t);
-					sb.Append((char)nextChar);
+			if (t.next != null) {
-					if (string.CompareOrdinal(ReaderPeekString("]]>".Length), "]]>") == 0) {
+				// Next() isn't called again when it returns 2 tokens, so we need to process both tokens
-						ReaderSkip("]]>".Length);
+				ef.InformToken(t.next);
-						return new Token(Tokens.XmlCData, new Location(x, y), new Location(Col, Line), sb.ToString(), null, LiteralFormat.None);
+				prevToken = t.next;
-					}
+			} else {
-				}
+				prevToken = t;
 			}
-			
+			ef.Advance();
-			return null;
+			return t;
 		}
 		
-		string ReadXmlContent(char ch)
+		/// <remarks>see VB language specification 10; pg. 6</remarks>
+		bool SkipEOL(Token prevToken, Token nextToken)
 		{
-			sb.Length = 0;
+			// exception directly after _
-			while (true) {
+			if (encounteredLineContinuation) {
-				sb.Append(ch);
+				return encounteredLineContinuation = false;
-				int next = ReaderPeek();
-				
-				if (next == -1 || next == '<')
-					break;
-				ch = (char)ReaderRead();
 			}
 			
-			return sb.ToString();
+			// 1st rule
+			// after a comma (,), open parenthesis ((), open curly brace ({), or open embedded expression (<%=)
+			if (new[] { Tokens.Comma, Tokens.OpenParenthesis, Tokens.OpenCurlyBrace, Tokens.XmlStartInlineVB }
+			    .Contains(prevToken.kind))
+				return true;
+			
+			// 2nd rule
+			// after a member qualifier (. or .@ or ...), provided that something is being qualified (i.e. is not
+			// using an implicit With context)
+			if (new[] { Tokens.Dot, Tokens.DotAt, Tokens.TripleDot }.Contains(prevToken.kind)
+			    && !ef.WasQualifierTokenAtStart)
+				return true;
+			
+			// 3rd rule
+			// before a close parenthesis ()), close curly brace (}), or close embedded expression (%>)
+			if (new[] { Tokens.CloseParenthesis, Tokens.CloseCurlyBrace, Tokens.XmlEndInlineVB }
+			    .Contains(nextToken.kind))
+				return true;
+			
+			// 4th rule
+			// after a less-than (<) in an attribute context
+			if (prevToken.kind == Tokens.LessThan && ef.CurrentBlock.context == Context.Attribute)
+				return true;
+			
+			// 5th rule
+			// before a greater-than (>) in an attribute context
+			if (nextToken.kind == Tokens.GreaterThan && ef.CurrentBlock.context == Context.Attribute)
+				return true;
+			
+			// 6th rule
+			// after a greater-than (>) in a non-file-level attribute context
+			if (prevToken.kind == Tokens.GreaterThan && ef.CurrentBlock.context == Context.Attribute)
+				return true;
+			
+			// 7th rule
+			// before and after query operators (Where, Order, Select, etc.)
+			var queryOperators = new int[] { Tokens.From, Tokens.Aggregate, Tokens.Select, Tokens.Distinct,
+				Tokens.Where, Tokens.Order, Tokens.By, Tokens.Ascending, Tokens.Descending, Tokens.Take,
+				Tokens.Skip, Tokens.Let, Tokens.Group, Tokens.Into, Tokens.On, Tokens.While, Tokens.Join };
+			if ((queryOperators.Contains(prevToken.kind) || queryOperators.Contains(nextToken.kind))
+			    && ef.CurrentBlock.context == Context.Query)
+				return true;
+			
+			// 8th rule
+			// after binary operators (+, -, /, *, etc.) in an expression context
+			if (new[] { Tokens.Plus, Tokens.Minus, Tokens.Div, Tokens.DivInteger, Tokens.Times, Tokens.Mod, Tokens.Power,
+			   Tokens.Assign, Tokens.NotEqual, Tokens.LessThan, Tokens.LessEqual, Tokens.GreaterThan, Tokens.GreaterEqual,
+			   Tokens.Like, Tokens.ConcatString, Tokens.AndAlso, Tokens.OrElse, Tokens.And, Tokens.Or, Tokens.Xor, 
+			   Tokens.ShiftLeft, Tokens.ShiftRight }.Contains(prevToken.kind) && ef.CurrentBlock.context == Context.Expression)
+				return true;
+			
+			// 9th rule
+			// after assignment operators (=, :=, +=, -=, etc.) in any context.
+			if (new[] { Tokens.Assign, Tokens.ColonAssign, Tokens.ConcatStringAssign, Tokens.DivAssign,
+			    	Tokens.DivIntegerAssign, Tokens.MinusAssign, Tokens.PlusAssign, Tokens.PowerAssign,
+			    	Tokens.ShiftLeftAssign, Tokens.ShiftRightAssign, Tokens.TimesAssign }.Contains(prevToken.kind))
+				return true;
+			
+			return false;
 		}
 		
-		protected override Token Next()
+		public override Token NextToken()
 		{
-			Token t = NextInternal();
+			if (curToken == null) { // first call of NextToken()
-			ef.InformToken(t);
+				curToken = Next();
-//			if (ef.Errors.Any()) {
+				//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
-//				foreach (Token token in ef.Errors) {
+				return curToken;
-//					if (token != null)
+			}
-//						errors.Error(token.Location.Line, token.Location.Column, "unexpected: " + token.ToString());
+			
-//				}
+			lastToken = curToken;
-//			}
+			
-			ef.Advance();
+			if (curToken.next == null) {
-			return t;
+				curToken.next = Next();
+			}
+			
+			curToken = curToken.next;
+			
+			if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
+				curToken = new Token(Tokens.EOL, curToken.col, curToken.line, string.Empty);
+				curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, string.Empty);
+			}
+			//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
+			return curToken;
 		}
 		
+		#region VB Readers
 		string ReadIdent(char ch)
 		{
 			char typeCharacter;
@@ -504,25 +527,6 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			return sb.ToString();
 		}
 		
-		string ReadXmlIdent(char ch)
-		{
-			sb.Length = 0;
-			sb.Append(ch);
-			
-			int peek;
-			
-			while ((peek = ReaderPeek()) != -1 && (peek == ':' || XmlConvert.IsNCNameChar((char)peek))) {
-				sb.Append((char)ReaderRead());
-			}
-			
-			return sb.ToString();
-		}
-		
-		char PeekUpperChar()
-		{
-			return Char.ToUpper((char)ReaderPeek(), CultureInfo.InvariantCulture);
-		}
-		
 		[System.Diagnostics.CodeAnalysis.SuppressMessage("Microsoft.Performance", "CA1818:DoNotConcatenateStringsInsideLoops")]
 		Token ReadDigit(char ch, int x)
 		{
@@ -809,27 +813,6 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			return sb.ToString();
 		}
 		
-		string ReadXmlString(char terminator)
-		{
-			char ch = '\0';
-			sb.Length = 0;
-			int nextChar;
-			while ((nextChar = ReaderRead()) != -1) {
-				ch = (char)nextChar;
-				if (ch == terminator) {
-					break;
-				} else if (ch == '\n') {
-					errors.Error(Line, Col, String.Format("No return allowed inside String literal"));
-				} else {
-					sb.Append(ch);
-				}
-			}
-			if (ch != terminator) {
-				errors.Error(Line, Col, String.Format("End of File reached before String terminated "));
-			}
-			return sb.ToString();
-		}
-		
 		void ReadComment()
 		{
 			Location startPos = new Location(Col, Line);
@@ -1028,6 +1011,111 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			}
 			return null;
 		}
+		#endregion
+		
+		#region XML Readers
+		Token ReadXmlProcessingInstruction(int x, int y)
+		{
+			sb.Length = 0;
+			int nextChar = -1;
+			
+			while (ReaderPeek() != '?' || ReaderPeek(1) != '>') {
+				nextChar = ReaderRead();
+				if (nextChar == -1)
+					break;
+				sb.Append((char)nextChar);
+			}
+			
+			ReaderSkip("?>".Length);
+			
+			return new Token(Tokens.XmlProcessingInstruction, new Location(x, y), new Location(Col, Line), sb.ToString(), null, LiteralFormat.None);
+		}
+		
+		Token ReadXmlCommentOrCData(int x, int y)
+		{
+			sb.Length = 0;
+			int nextChar = -1;
+			
+			if (string.CompareOrdinal(ReaderPeekString("--".Length), "--") == 0) {
+				ReaderSkip("--".Length);
+				while ((nextChar = ReaderRead()) != -1) {
+					sb.Append((char)nextChar);
+					if (string.CompareOrdinal(ReaderPeekString("-->".Length), "-->") == 0) {
+						ReaderSkip("-->".Length);
+						return new Token(Tokens.XmlComment, new Location(x, y), new Location(Col, Line), sb.ToString(), null, LiteralFormat.None);
+					}
+				}
+			}
+			
+			if (string.CompareOrdinal(ReaderPeekString("[CDATA[".Length), "[CDATA[") == 0) {
+				ReaderSkip("[CDATA[".Length);
+				while ((nextChar = ReaderRead()) != -1) {
+					sb.Append((char)nextChar);
+					if (string.CompareOrdinal(ReaderPeekString("]]>".Length), "]]>") == 0) {
+						ReaderSkip("]]>".Length);
+						return new Token(Tokens.XmlCData, new Location(x, y), new Location(Col, Line), sb.ToString(), null, LiteralFormat.None);
+					}
+				}
+			}
+			
+			return null;
+		}
+		
+		string ReadXmlContent(char ch)
+		{
+			sb.Length = 0;
+			while (true) {
+				sb.Append(ch);
+				int next = ReaderPeek();
+				
+				if (next == -1 || next == '<')
+					break;
+				ch = (char)ReaderRead();
+			}
+			
+			return sb.ToString();
+		}
+		
+		string ReadXmlString(char terminator)
+		{
+			char ch = '\0';
+			sb.Length = 0;
+			int nextChar;
+			while ((nextChar = ReaderRead()) != -1) {
+				ch = (char)nextChar;
+				if (ch == terminator) {
+					break;
+				} else if (ch == '\n') {
+					errors.Error(Line, Col, String.Format("No return allowed inside String literal"));
+				} else {
+					sb.Append(ch);
+				}
+			}
+			if (ch != terminator) {
+				errors.Error(Line, Col, String.Format("End of File reached before String terminated "));
+			}
+			return sb.ToString();
+		}
+		
+		string ReadXmlIdent(char ch)
+		{
+			sb.Length = 0;
+			sb.Append(ch);
+			
+			int peek;
+			
+			while ((peek = ReaderPeek()) != -1 && (peek == ':' || XmlConvert.IsNCNameChar((char)peek))) {
+				sb.Append((char)ReaderRead());
+			}
+			
+			return sb.ToString();
+		}
+		#endregion
+		
+		char PeekUpperChar()
+		{
+			return Char.ToUpper((char)ReaderPeek(), CultureInfo.InvariantCulture);
+		}
 		
 		public override void SkipCurrentBlock(int targetToken)
 		{

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/ExpressionFinder.atg

@@ -415,11 +415,13 @@ StatementTerminatorAndBlock =
 Expression
 	(.NamedState:startOfExpression.)
 =
+	(. PushContext(Context.Expression, t); .)
 	(.
 		if (la != null)
 			CurrentBlock.lastExpressionStart = la.Location;
 	.)
 	SimpleExpressionWithSuffix { BinaryOperator SimpleExpressionWithSuffix }
+	(. PopContext(); .)
 .
 
 BinaryOperator =
@@ -447,7 +449,7 @@ SimpleExpression =
     (    Literal
     |    ( "(" Expression ")" )
     |    IdentifierForExpressionStart
-    |    ( "." | "!" | ".@" | "..." ) (. nextTokenIsStartOfImportsOrAccessExpression = true; .) [ XmlOpenTag ] IdentifierOrKeyword [ XmlCloseTag ]
+    |    ( "." | "!" | ".@" | "..." ) (. nextTokenIsStartOfImportsOrAccessExpression = true; wasQualifierTokenAtStart = true; .) [ XmlOpenTag ] IdentifierOrKeyword [ XmlCloseTag ]
     |    "GetType" "(" TypeName ")"
     |    "GetXmlNamespace" "(" (. readXmlIdentifier = true; .) Identifier ")"
     |    XmlLiteral
@@ -546,9 +548,12 @@ FunctionLambdaExpression =
 	( GREEDY Expression | [ "As" TypeName ] StatementTerminatorAndBlock "End" "Function" )
 .
 
-QueryExpression =
+QueryExpression
+	(. PushContext(Context.Query, t); .)
+=
 	( FromQueryOperator | AggregateQueryOperator )
 	{ QueryOperator }
+	(. PopContext(); .)
 .
  
 QueryOperator =

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/ExpressionFinder.cs

@@ -113,6 +113,10 @@ namespace ICSharpCode.NRefactory.Parser.VBNet.Experimental
 			get { return nextTokenIsStartOfImportsOrAccessExpression; }
 		}
 		
+		public bool WasQualifierTokenAtStart {
+			get { return wasQualifierTokenAtStart; }
+		}
+		
 		public List<Token> Errors {
 			get { return errors; }
 		}
@@ -127,6 +131,8 @@ namespace ICSharpCode.NRefactory.Parser.VBNet.Experimental
 		Body,
 		Xml,
 		Attribute,
+		Query,
+		Expression,
 		Debug,
 		Default
 	}

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/PushParser.frame

@@ -34,6 +34,7 @@ partial class ExpressionFinder {
 
 -->declarations
 	readonly Stack<int> stateStack = new Stack<int>();
+	bool wasQualifierTokenAtStart = false;
 	bool nextTokenIsPotentialStartOfXmlMode = false;
 	bool readXmlIdentifier = false;
 	bool nextTokenIsStartOfImportsOrAccessExpression = false;
@@ -67,6 +68,7 @@ partial class ExpressionFinder {
 		nextTokenIsPotentialStartOfXmlMode = false;
 		readXmlIdentifier = false;
 		nextTokenIsStartOfImportsOrAccessExpression = false;
+		wasQualifierTokenAtStart = false;
 		-->informToken
 		ApplyToken(la);
 		if (la != null) t = la;

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/Test/ImplicitLineContinuationTests.cs

@@ -0,0 +1,113 @@
+// <file>
+//     <copyright see="prj:///doc/copyright.txt"/>
+//     <license see="prj:///doc/license.txt"/>
+//     <owner name="Siegfried Pammer" email="siegfriedpammer@gmail.com" />
+//     <version>$Revision$</version>
+// </file>
+
+using System;
+using System.IO;
+using ICSharpCode.NRefactory;
+using ICSharpCode.NRefactory.Parser;
+using ICSharpCode.NRefactory.Parser.VB;
+using NUnit.Framework;
+
+namespace VBParserExperiment
+{
+	[TestFixture]
+	public class ImplicitLineContinuationTests
+	{
+		[Test]
+		public void Example1()
+		{
+			string code = @"Module Test
+    Sub Print(
+        Param1 As Integer,
+        Param2 As Integer)
+
+        If (Param1 < Param2) Or
+           (Param1 > Param2) Then
+            Console.WriteLine(""Not equal"")
+        End If
+    End Sub
+End Module";
+			
+			ILexer lexer = GenerateLexer(new StringReader(code));
+			
+			CheckTokens(lexer, Tokens.Module, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Sub, Tokens.Identifier, Tokens.OpenParenthesis,
+			            Tokens.Identifier, Tokens.As, Tokens.Integer, Tokens.Comma,
+			            Tokens.Identifier, Tokens.As, Tokens.Integer, Tokens.CloseParenthesis, Tokens.EOL,
+			            Tokens.If, Tokens.OpenParenthesis, Tokens.Identifier, Tokens.LessThan, Tokens.Identifier, Tokens.CloseParenthesis, Tokens.Or,
+			            Tokens.OpenParenthesis, Tokens.Identifier, Tokens.GreaterThan, Tokens.Identifier, Tokens.CloseParenthesis, Tokens.Then, Tokens.EOL,
+			            Tokens.Identifier, Tokens.Dot, Tokens.Identifier, Tokens.OpenParenthesis, Tokens.LiteralString, Tokens.CloseParenthesis, Tokens.EOL,
+			            Tokens.End, Tokens.If, Tokens.EOL,
+			            Tokens.End, Tokens.Sub, Tokens.EOL,
+			            Tokens.End, Tokens.Module);
+		}
+		
+		[Test]
+		public void QualifierInWith()
+		{
+			string code = @"Module Test
+	Sub Print
+		With xml
+			Dim a = b.
+				d
+			Dim c = .
+				Count
+		End With
+	End Sub
+End Module";
+			
+			ILexer lexer = GenerateLexer(new StringReader(code));
+			
+			CheckTokens(lexer, Tokens.Module, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Sub, Tokens.Identifier, Tokens.EOL,
+			            Tokens.With, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Dim, Tokens.Identifier, Tokens.Assign, Tokens.Identifier, Tokens.Dot, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Dim, Tokens.Identifier, Tokens.Assign, Tokens.Dot, Tokens.EOL,
+			            Tokens.Identifier, Tokens.EOL,
+			            Tokens.End, Tokens.With, Tokens.EOL,
+			            Tokens.End, Tokens.Sub, Tokens.EOL,
+			            Tokens.End, Tokens.Module);
+		}
+		
+		[Test]
+		public void Example2()
+		{
+			string code = @"Module Test
+	Sub Print
+		Dim a = _
+			
+			y
+	End Sub
+End Module";
+			
+			ILexer lexer = GenerateLexer(new StringReader(code));
+			
+			CheckTokens(lexer, Tokens.Module, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Sub, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Dim, Tokens.Identifier, Tokens.Assign, Tokens.EOL, Tokens.Identifier, Tokens.EOL,
+			            Tokens.End, Tokens.Sub, Tokens.EOL,
+			            Tokens.End, Tokens.Module);
+		}
+		
+		#region Helpers
+		ILexer GenerateLexer(StringReader sr)
+		{
+			return ParserFactory.CreateLexer(SupportedLanguage.VBNet, sr);
+		}
+		
+		void CheckTokens(ILexer lexer, params int[] tokens)
+		{
+			for (int i = 0; i < tokens.Length; i++) {
+				int token = tokens[i];
+				Token t = lexer.NextToken();
+				int next = t.Kind;
+				Assert.AreEqual(token, next, "{2} of {3}: {0} != {1}; at {4}", Tokens.GetTokenString(token), Tokens.GetTokenString(next), i + 1, tokens.Length, t.Location);
+			}
+		}
+		#endregion
+	}
+}

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/Test/ParserTests.cs

@@ -5,7 +5,6 @@
 //     <version>$Revision$</version>
 // </file>
 
-
 using System;
 using System.IO;
 using ICSharpCode.NRefactory;
@@ -50,8 +49,10 @@ End Class
 			enter Body
 				enter IdentifierExpected
 				exit IdentifierExpected
-				enter Xml
+				enter Expression
-				exit Xml
+					enter Xml
+					exit Xml
+				exit Expression
 			exit Body
 		exit Member
 	exit Type
@@ -79,8 +80,10 @@ End Class
 		enter Member
 			enter IdentifierExpected
 			exit IdentifierExpected
-			enter Xml
+			enter Expression
-			exit Xml
+				enter Xml
+				exit Xml
+			exit Expression
 		exit Member
 		enter Member
 			enter IdentifierExpected
@@ -88,8 +91,10 @@ End Class
 			enter Body
 				enter IdentifierExpected
 				exit IdentifierExpected
-				enter Xml
+				enter Expression
-				exit Xml
+					enter Xml
+					exit Xml
+				exit Expression
 			exit Body
 		exit Member
 	exit Type
@@ -121,6 +126,8 @@ End Class
 			enter Body
 				enter IdentifierExpected
 				exit IdentifierExpected
+				enter Expression
+				exit Expression
 			exit Body
 		exit Member
 	exit Type
@@ -152,6 +159,8 @@ End Class
 			enter Body
 				enter IdentifierExpected
 				exit IdentifierExpected
+				enter Expression
+				exit Expression
 			exit Body
 		exit Member
 	exit Type
@@ -183,6 +192,8 @@ End Class
 			enter Body
 				enter IdentifierExpected
 				exit IdentifierExpected
+				enter Expression
+				exit Expression
 			exit Body
 		exit Member
 	exit Type
@@ -211,6 +222,8 @@ End Class
 			enter IdentifierExpected
 			exit IdentifierExpected
 			enter Body
+				enter Expression
+				exit Expression
 				enter Body
 				exit Body
 			exit Body
@@ -253,10 +266,14 @@ End Class
 			enter Body
 				enter IdentifierExpected
 				exit IdentifierExpected
+				enter Expression
+				exit Expression
 				enter Body
 				exit Body
 				enter IdentifierExpected
 				exit IdentifierExpected
+				enter Expression
+				exit Expression
 				enter Body
 				exit Body
 				enter Body

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/Test/VBParserExperiment.csproj

@@ -47,6 +47,7 @@
     </Reference>
   </ItemGroup>
   <ItemGroup>
+    <Compile Include="ImplicitLineContinuationTests.cs" />
     <Compile Include="LATextReaderTests.cs" />
     <Compile Include="ParserTests.cs" />
     <Compile Include="Program.cs" />