Lexer can read simple XML literals

git-svn-id: svn://svn.sharpdevelop.net/sharpdevelop/branches/vbnet@5899 1ccf3a8d-04fe-1044-b7c0-cef0b8235c61

src/Libraries/NRefactory/NRefactory.sln

@@ -1,7 +1,7 @@
 
 Microsoft Visual Studio Solution File, Format Version 11.00
 # Visual Studio 2010
-# SharpDevelop 4.0.0.5826
+# SharpDevelop 4.0.0.5897
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NRefactory", "Project\NRefactory.csproj", "{3A9AE6AA-BC07-4A2F-972C-581E3AE2F195}"
 EndProject
 Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "NRefactoryTests", "Test\NRefactoryTests.csproj", "{870115DD-960A-4406-A6B9-600BCDC36A03}"

src/Libraries/NRefactory/Project/NRefactory.csproj

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="3.5">
+<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0">
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@@ -24,25 +24,26 @@
     <FileAlignment>4096</FileAlignment>
     <RunCodeAnalysis>False</RunCodeAnalysis>
     <CodeAnalysisRules>-Microsoft.Design#CA1002;-Microsoft.Design#CA1020;-Microsoft.Design#CA1051;-Microsoft.Design#CA1062;-Microsoft.Globalization#CA1303;-Microsoft.Globalization#CA1305;-Microsoft.Naming#CA1704;-Microsoft.Performance#CA1800;-Microsoft.Performance#CA1805;-Microsoft.Usage#CA2211;-Microsoft.Usage#CA2227</CodeAnalysisRules>
-    <TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">
     <Optimize>False</Optimize>
     <CheckForOverflowUnderflow>True</CheckForOverflowUnderflow>
-    <DefineConstants>TEST;DEBUG;NET35</DefineConstants>
+    <DefineConstants>TEST;DEBUG</DefineConstants>
     <OutputPath>..\..\..\..\bin\</OutputPath>
     <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Release|AnyCPU' ">
     <Optimize>True</Optimize>
     <CheckForOverflowUnderflow>False</CheckForOverflowUnderflow>
-    <DefineConstants>TEST;NET35</DefineConstants>
+    <DefineConstants>TEST</DefineConstants>
     <OutputPath>..\..\..\..\bin\</OutputPath>
     <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <DebugType>Full</DebugType>
     <DebugSymbols>true</DebugSymbols>
+    <StartAction>Project</StartAction>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
     <DebugSymbols>false</DebugSymbols>

src/Libraries/NRefactory/Project/NRefactory.csproj.user

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="3.5">
+<Project xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0">
   <PropertyGroup>
     <LastOpenVersion>8.0.41115</LastOpenVersion>
     <ProjectView>ShowAllFiles</ProjectView>

src/Libraries/NRefactory/Project/Src/AstBuilder/ExpressionBuilder.cs

@@ -11,7 +11,6 @@ using ICSharpCode.NRefactory.Ast;
 
 namespace ICSharpCode.NRefactory.AstBuilder
 {
-	#if NET35
 	/// <summary>
 	/// Extension methods for NRefactory.Ast.Expression.
 	/// </summary>
@@ -94,5 +93,4 @@ namespace ICSharpCode.NRefactory.AstBuilder
 			return new BinaryOperatorExpression(left, op, right);
 		}
 	}
-	#endif
 }

src/Libraries/NRefactory/Project/Src/Lexer/VBNet/Lexer.cs

@@ -31,7 +31,6 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			if (curToken == null) { // first call of NextToken()
 				curToken = Next();
 				specialTracker.InformToken(curToken.kind);
-				ef.InformToken(curToken);
 				//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
 				return curToken;
 			}
@@ -41,7 +40,6 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			if (curToken.next == null) {
 				curToken.next = Next();
 				specialTracker.InformToken(curToken.next.kind);
-				ef.InformToken(curToken);
 			}
 			
 			curToken = curToken.next;
@@ -49,10 +47,8 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			if (curToken.kind == Tokens.EOF && !(lastToken.kind == Tokens.EOL)) { // be sure that before EOF there is an EOL token
 				curToken = new Token(Tokens.EOL, curToken.col, curToken.line, string.Empty);
 				specialTracker.InformToken(curToken.kind);
-				ef.InformToken(curToken);
 				curToken.next = new Token(Tokens.EOF, curToken.col, curToken.line, string.Empty);
 				specialTracker.InformToken(curToken.next.kind);
-				ef.InformToken(curToken);
 			}
 			//Console.WriteLine("Tok:" + Tokens.GetTokenString(curToken.kind) + " --- " + curToken.val);
 			return curToken;
@@ -62,7 +58,7 @@ namespace ICSharpCode.NRefactory.Parser.VB
 		bool inXmlMode, expectXmlIdentifier, inXmlTag, inXmlCloseTag;
 		int level = 0;
 		
-		protected override Token Next()
+		Token NextInternal()
 		{
 			if (misreadExclamationMarkAsTypeCharacter) {
 				misreadExclamationMarkAsTypeCharacter = false;
@@ -89,22 +85,28 @@ namespace ICSharpCode.NRefactory.Parser.VB
 						switch (ch) {
 							case '<':
 								if (ReaderPeek() == '/') {
+									ReaderRead();
 									inXmlCloseTag = true;
 									return new Token(Tokens.XmlOpenEndTag, Col - 1, Line);
 								}
 								if (ReaderPeek() == '%') {
 									// TODO : suspend xml mode tracking
+									ReaderRead();
 									return new Token(Tokens.XmlStartInlineVB, Col - 1, Line);
 								}
 								
 								return new Token(Tokens.XmlOpenTag, Col - 1, Line);
 							case '/':
-								if (ReaderPeek() == '>')
+								if (ReaderPeek() == '>') {
+									ReaderRead();
+									level--;
 									return new Token(Tokens.XmlCloseTagEmptyElement, Col - 1, Line);
+								}
 								break;
 							case '%':
 								if (ReaderPeek() == '>') {
 									// TODO : resume xml mode tracking
+									ReaderRead();
 									return new Token(Tokens.XmlEndInlineVB, Col - 1, Line);
 								}
 								break;
@@ -113,7 +115,7 @@ namespace ICSharpCode.NRefactory.Parser.VB
 									level--;
 								return new Token(Tokens.XmlCloseTag, Col - 1, Line);
 							case '=':
-								return new Token(Tokens.Equals, Col - 1, Line);
+								return new Token(Tokens.Assign, Col - 1, Line);
 							case '\'':
 							case '"':
 								int x = Col - 1;
@@ -122,6 +124,8 @@ namespace ICSharpCode.NRefactory.Parser.VB
 								return new Token(Tokens.LiteralString, Col - 1, Line, '"' + s + '"', s, LiteralFormat.StringLiteral);
 							default:
 								// TODO : can be either identifier or xml content
+								if (XmlConvert.IsWhitespaceChar(ch))
+									continue;
 								return new Token(Tokens.Identifier, Col - 1, Line, ReadXmlIdent(ch));
 						}
 					} else {
@@ -303,6 +307,14 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			}
 		}
 		
+		protected override Token Next()
+		{
+			Token t = NextInternal();
+			ef.InformToken(t);
+			ef.Advance();
+			return t;
+		}
+		
 		string ReadIdent(char ch)
 		{
 			char typeCharacter;
@@ -337,7 +349,7 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			return sb.ToString();
 		}
 		
-		string ReadXmlIndent(char ch)
+		string ReadXmlIdent(char ch)
 		{
 			sb.Length = 0;
 			sb.Append(ch);
@@ -345,8 +357,7 @@ namespace ICSharpCode.NRefactory.Parser.VB
 			int peek;
 			
 			while ((peek = ReaderPeek()) != -1 && (peek == ':' || XmlConvert.IsNCNameChar((char)peek))) {
-				ReaderRead();
+				sb.Append((char)ReaderRead());
-				sb.Append(ch);
 			}
 			
 			return sb.ToString();

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/PushParser.frame

@@ -57,6 +57,11 @@ partial class ExpressionFinder {
 		-->informToken
 	}
 	
+	public void Advance()
+	{
+		InformToken(null);
+	}
+	
 	static readonly bool[,] set = {
 -->initialization
 	};

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/Test/XmlModeLexerTests.cs

@@ -33,15 +33,62 @@ namespace VBParserExperiment
 		}
 		
 		[Test]
-		public void SimpleTag()
+		public void SimpleEmptyTag()
 		{
 			ILexer lexer = GenerateLexer(new StringReader(TestStatement("Dim x = <Test />")));
 			
+			CheckHead(lexer);
+			
+			Assert.AreEqual(Tokens.Dim, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Assign, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.XmlOpenTag, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.XmlCloseTagEmptyElement, lexer.NextToken().Kind);
+			
+			CheckFoot(lexer);
+		}
+		
+		[Test]
+		public void SimpleTag()
+		{
+			ILexer lexer = GenerateLexer(new StringReader(TestStatement("Dim x = <Test></Test>")));
+			
+			CheckHead(lexer);
+			
+			Assert.AreEqual(Tokens.Dim, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Assign, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.XmlOpenTag, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.XmlCloseTag, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.XmlOpenEndTag, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.XmlCloseTag, lexer.NextToken().Kind);
+			
+			CheckFoot(lexer);
+		}
+
+		void CheckFoot(ILexer lexer)
+		{
+			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.End, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Sub, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.End, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.Class, lexer.NextToken().Kind);
+		}
+
+		void CheckHead(ILexer lexer)
+		{
 			Assert.AreEqual(Tokens.Class, lexer.NextToken().Kind);
 			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
 			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
 			Assert.AreEqual(Tokens.Sub, lexer.NextToken().Kind);
 			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.OpenParenthesis, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.CloseParenthesis, lexer.NextToken().Kind);
+			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
 		}
 	}
 }

src/Libraries/NRefactory/Test/NRefactoryTests.csproj

@@ -1,5 +1,5 @@
 <?xml version="1.0" encoding="utf-8"?>
-<Project ToolsVersion="3.5" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
+<Project ToolsVersion="4.0" DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003">
   <PropertyGroup>
     <Configuration Condition=" '$(Configuration)' == '' ">Debug</Configuration>
     <Platform Condition=" '$(Platform)' == '' ">AnyCPU</Platform>
@@ -17,7 +17,7 @@
     <AllowUnsafeBlocks>False</AllowUnsafeBlocks>
     <PlatformTarget>x86</PlatformTarget>
     <WarningLevel>4</WarningLevel>
-    <TargetFrameworkVersion>v3.5</TargetFrameworkVersion>
+    <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
     <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)|$(Platform)' == 'Debug|AnyCPU' ">