implemented attributes, XML comments and CDATA sections

git-svn-id: svn://svn.sharpdevelop.net/sharpdevelop/branches/vbnet@5905 1ccf3a8d-04fe-1044-b7c0-cef0b8235c61

src/Libraries/NRefactory/Project/Src/Lexer/VBNet/Lexer.cs

@@ -55,7 +55,7 @@ namespace ICSharpCode.NRefactory.Parser.VB
 		}
 		
 		bool misreadExclamationMarkAsTypeCharacter;
-		bool inXmlMode, inXmlTag, inXmlCloseTag;
+		bool inXmlMode, inXmlTag, inXmlCloseTag, wasComment;
 		int level = 0;
 		
 		Token NextInternal()
@@ -71,71 +71,92 @@ namespace ICSharpCode.NRefactory.Parser.VB
 					if (nextChar == -1)
 						return new Token(Tokens.EOF, Col, Line, string.Empty);
 					char ch = (char)nextChar;
-					if (inXmlMode && level <= 0) {
+					#region XML mode
-//						int peek;
+					if (inXmlMode && level <= 0 && !wasComment) {
-//						while ((peek = ReaderPeek()) != -1 && XmlConvert.IsWhitespaceChar((char)peek)) ;
+						int peek;
-//
+						while (true) {
-//						inXmlMode = ReaderPeek() == '<' &&
+							while ((peek = ReaderPeek()) != -1 && XmlConvert.IsWhitespaceChar((char)peek))
-//							ReaderPeek() == '!' &&
+								ReaderRead();
-//							ReaderPeek() == '-' &&
+
-//							ReaderPeek() == '-';
+							if (ReaderPeek() == '<') {
+								ReaderRead();
+								if (ReaderPeek() == '!') {
+									ReaderRead();
+									Token token = ReadXmlCommentOrCData(Col - 1, Line);
+									ReaderRead();
+									return token;
+								}
+							}
+							
+							break;
+						}
 						inXmlMode = false;
 					}
 					if (inXmlMode) {
+						int x = Col - 1;
+						int y = Line;
 						switch (ch) {
 							case '<':
 								if (ReaderPeek() == '/') {
 									ReaderRead();
 									inXmlCloseTag = true;
-									return new Token(Tokens.XmlOpenEndTag, Col - 1, Line);
+									return new Token(Tokens.XmlOpenEndTag, x, y);
 								}
 								if (ReaderPeek() == '%') {
 									// TODO : suspend xml mode tracking
 									ReaderRead();
-									return new Token(Tokens.XmlStartInlineVB, Col - 1, Line);
+									return new Token(Tokens.XmlStartInlineVB, x, y);
+								}
+								if (ReaderPeek() == '!') {
+									ReaderRead();
+									Token token = ReadXmlCommentOrCData(x, y);
+									wasComment = token.Kind == Tokens.XmlComment;
+									ReaderRead();
+									return token;
 								}
 								level++;
 								inXmlTag = true;
-								return new Token(Tokens.XmlOpenTag, Col - 1, Line);
+								return new Token(Tokens.XmlOpenTag, x, y);
 							case '/':
 								if (ReaderPeek() == '>') {
 									ReaderRead();
 									inXmlTag = false;
 									level--;
-									return new Token(Tokens.XmlCloseTagEmptyElement, Col - 1, Line);
+									return new Token(Tokens.XmlCloseTagEmptyElement, x, y);
 								}
 								break;
 							case '%':
 								if (ReaderPeek() == '>') {
 									// TODO : resume xml mode tracking
 									ReaderRead();
-									return new Token(Tokens.XmlEndInlineVB, Col - 1, Line);
+									return new Token(Tokens.XmlEndInlineVB, x, y);
 								}
 								break;
-							case '>':
+								case '>':                   /* workaround for XML Imports */
-								if (inXmlCloseTag)
+								if (inXmlCloseTag || (inXmlTag && ef.CurrentContext == Context.Global))
 									level--;
+								wasComment = false;
 								inXmlTag = inXmlCloseTag = false;
-								return new Token(Tokens.XmlCloseTag, Col - 1, Line);
+								return new Token(Tokens.XmlCloseTag, x, y);
 							case '=':
-								return new Token(Tokens.Assign, Col - 1, Line);
+								return new Token(Tokens.Assign, x, y);
 							case '\'':
 							case '"':
-								int x = Col - 1;
-								int y = Line;
 								string s = ReadXmlString(ch);
-								return new Token(Tokens.LiteralString, Col - 1, Line, ch + s + ch, s, LiteralFormat.StringLiteral);
+								return new Token(Tokens.LiteralString, x, y, ch + s + ch, s, LiteralFormat.StringLiteral);
 							default:
 								// TODO : can be either identifier or xml content
 								if (inXmlCloseTag || inXmlTag) {
 									if (XmlConvert.IsWhitespaceChar(ch))
 										continue;
-									return new Token(Tokens.Identifier, Col - 1, Line, ReadXmlIdent(ch));
+									return new Token(Tokens.Identifier, x, y, ReadXmlIdent(ch));
 								} else {
-									return new Token(Tokens.XmlContent, Col - 1, Line, ReadXmlContent(ch));
+									return new Token(Tokens.XmlContent, x, y, ReadXmlContent(ch));
 								}
 						}
+						#endregion
 					} else {
+						#region Standard Mode
 						if (Char.IsWhiteSpace(ch)) {
 							if (HandleLineEnd(ch)) {
 								if (lineEnd) {
@@ -295,11 +316,32 @@ namespace ICSharpCode.NRefactory.Parser.VB
 							}
 							return new Token(Tokens.LiteralString, x, y, '"' + s + '"', s, LiteralFormat.StringLiteral);
 						}
-						
+						#endregion
 						if (ch == '<' && ef.NextTokenIsPotentialStartOfXmlMode) {
-							inXmlMode = inXmlTag = true;
+							int x = Col - 1;
+							int y = Line;
+							inXmlMode = true;
+							level = 0;
+							if (ReaderPeek() == '/') {
+								ReaderRead();
+								inXmlCloseTag = true;
+								return new Token(Tokens.XmlOpenEndTag, x, y);
+							}
+							if (ReaderPeek() == '%') {
+								// TODO : suspend xml mode tracking
+								ReaderRead();
+								return new Token(Tokens.XmlStartInlineVB, x, y);
+							}
+							if (ReaderPeek() == '!') {
+								ReaderRead();
+								Token t = ReadXmlCommentOrCData(x, y);
+								wasComment = t.Kind == Tokens.XmlComment;
+								ReaderRead();
+								return t;
+							}
+							inXmlTag = true;
 							level = 1;
-							return new Token(Tokens.XmlOpenTag, Col - 1, Line);
+							return new Token(Tokens.XmlOpenTag, x, y);
 						}
 						Token token = ReadOperator(ch);
 						if (token != null) {
@@ -307,13 +349,48 @@ namespace ICSharpCode.NRefactory.Parser.VB
 							return token;
 						}
 					}
-
+					
-
 					errors.Error(Line, Col, String.Format("Unknown char({0}) which can't be read", ch));
 				}
 			}
 		}
 		
+		Token ReadXmlCommentOrCData(int x, int y)
+		{
+			sb.Length = 0;
+			int nextChar = -1;
+			
+			for (int i = 0; i < 7; i++) {
+				nextChar = ReaderRead();
+				if (nextChar > -1)
+					sb.Append((char)nextChar);
+			}
+			
+			if (sb.ToString().StartsWith("--")) {
+				sb.Length = 0;
+				while ((nextChar = ReaderRead()) != -1) {
+					sb.Append((char)nextChar);
+					if (ReaderPeek() == '>' && sb.ToString().EndsWith("--")) {
+						string text = sb.Remove(sb.Length - 2, 2).ToString();
+						return new Token(Tokens.XmlComment, x, y, text);
+					}
+				}
+			}
+			
+			if (sb.ToString().StartsWith("[CDATA[")) {
+				sb.Length = 0;
+				while ((nextChar = ReaderRead()) != -1) {
+					sb.Append((char)nextChar);
+					if (ReaderPeek() == '>' && sb.ToString().EndsWith("]]")) {
+						string text = sb.Remove(sb.Length - 2, 2).ToString();
+						return new Token(Tokens.XmlCData, x, y, text);
+					}
+				}
+			}
+			
+			return null;
+		}
+		
 		string ReadXmlContent(char ch)
 		{
 			sb.Length = 0;

src/Libraries/NRefactory/Project/Src/Parser/VBNet/Experimental/Test/XmlModeLexerTests.cs

@@ -289,6 +289,27 @@ namespace VBParserExperiment
 			CheckFoot(lexer);
 		}
 		
+		[Test]
+		public void SimpleXmlWithComments()
+		{
+			ILexer lexer = GenerateLexer(new StringReader(TestStatement(@"Dim x = <!-- Test file -->
+			                                                                      <Test>
+			                                                                        <!-- Test data -->
+			                                                                        <Data />
+			                                                                      </Test>
+			                                                                      <!-- eof -->
+			                                                                      <!-- hey, wait! -->")));
+			
+			CheckHead(lexer);
+			
+			CheckTokens(lexer, Tokens.Dim, Tokens.Identifier, Tokens.Assign,
+			            Tokens.XmlComment, Tokens.XmlContent, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTag,
+			            Tokens.XmlContent, Tokens.XmlComment, Tokens.XmlContent, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTagEmptyElement,
+			            Tokens.XmlContent, Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlComment, Tokens.XmlComment);
+			
+			CheckFoot(lexer);
+		}
+		
 		[Test]
 		public void SimpleEmptyTag()
 		{
@@ -296,12 +317,8 @@ namespace VBParserExperiment
 			
 			CheckHead(lexer);
 			
-			Assert.AreEqual(Tokens.Dim, lexer.NextToken().Kind);
+			CheckTokens(lexer, Tokens.Dim, Tokens.Identifier, Tokens.Assign,
-			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			            Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTagEmptyElement);
-			Assert.AreEqual(Tokens.Assign, lexer.NextToken().Kind);
-			Assert.AreEqual(Tokens.XmlOpenTag, lexer.NextToken().Kind);
-			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
-			Assert.AreEqual(Tokens.XmlCloseTagEmptyElement, lexer.NextToken().Kind);
 			
 			CheckFoot(lexer);
 		}
@@ -313,39 +330,131 @@ namespace VBParserExperiment
 			
 			CheckHead(lexer);
 			
-			Assert.AreEqual(Tokens.Dim, lexer.NextToken().Kind);
+			CheckTokens(lexer, Tokens.Dim, Tokens.Identifier, Tokens.Assign, Tokens.XmlOpenTag,
-			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			            Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlOpenEndTag,
-			Assert.AreEqual(Tokens.Assign, lexer.NextToken().Kind);
+			            Tokens.Identifier, Tokens.XmlCloseTag);
-			Assert.AreEqual(Tokens.XmlOpenTag, lexer.NextToken().Kind);
+			
-			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			CheckFoot(lexer);
-			Assert.AreEqual(Tokens.XmlCloseTag, lexer.NextToken().Kind);
+		}
-			Assert.AreEqual(Tokens.XmlOpenEndTag, lexer.NextToken().Kind);
+		
-			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+		[Test]
-			Assert.AreEqual(Tokens.XmlCloseTag, lexer.NextToken().Kind);
+		public void XmlImport()
+		{
+			string code = @"Imports System
+Imports System.Linq
+
+Imports <xmlns='http://icsharpcode.net/sharpdevelop/avalonedit'>
+Imports <xmlns:h='http://www.w3.org/TR/html4/'>
+
+Class TestClass
+	Sub TestSub()
+		Dim xml = <h:table>
+					<h:tr>
+						<h:td>1. Cell</h:td>
+					</h:tr>
+				  </h:table>
+	End Sub
+End Class";
+			
+			ILexer lexer = GenerateLexer(new StringReader(code));
+			
+			CheckTokens(lexer, Tokens.Imports, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Imports, Tokens.Identifier, Tokens.Dot, Tokens.Identifier, Tokens.EOL,
+			            Tokens.Imports, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.Assign, Tokens.LiteralString, Tokens.XmlCloseTag, Tokens.EOL,
+			            Tokens.Imports, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.Assign, Tokens.LiteralString, Tokens.XmlCloseTag, Tokens.EOL,
+			            Tokens.Class, Tokens.Identifier, Tokens.EOL, Tokens.Sub, Tokens.Identifier, Tokens.OpenParenthesis, Tokens.CloseParenthesis, Tokens.EOL,
+			            Tokens.Dim, Tokens.Identifier, Tokens.Assign, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTag,
+			            Tokens.XmlContent, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTag,
+			            Tokens.XmlContent, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlContent, Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag,
+			            Tokens.XmlContent, Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlContent, Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag,
+			            Tokens.EOL, Tokens.End, Tokens.Sub, Tokens.EOL, Tokens.End, Tokens.Class
+			           );
+		}
+		
+		[Test]
+		public void CDataSection()
+		{
+			string xml = @"Dim xml = <template>
+				<name>test</name>
+				<language>VB</languge>
+				<file language='XAML'>
+					<![CDATA[<Window x:Class='DefaultNamespace.Window1'
+	xmlns='http://schemas.microsoft.com/winfx/2006/xaml/presentation'
+	xmlns:x='http://schemas.microsoft.com/winfx/2006/xaml'
+	Title='DefaultNamespace' Height='300' Width='300'>
+	<Grid>
+		
+	</Grid>
+</Window>]]>
+				</file>
+				<file language='CSharp'>
+				<![CDATA[using System;
+using System.Collections.Generic;
+using System.Text;
+using System.Windows;
+using System.Windows.Controls;
+using System.Windows.Data;
+using System.Windows.Documents;
+using System.Windows.Input;
+using System.Windows.Media;
+
+namespace DefaultNamespace
+{
+	/// <summary>
+	/// Interaction logic for Window1.xaml
+	/// </summary>
+	public partial class Window1 : Window
+	{
+		public Window1()
+		{
+			InitializeComponent();
+		}
+	}
+}]]>
+				</file>
+			</template>
+			";
+			
+			ILexer lexer = GenerateLexer(new StringReader(TestStatement(xml)));
+			
+			CheckHead(lexer);
+			
+			CheckTokens(lexer, Tokens.Dim, Tokens.Identifier, Tokens.Assign, // 2
+			            Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlContent, // 6
+			            Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlContent, // 10
+			            Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlContent, // 14
+			            Tokens.XmlOpenTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlContent, // 18
+			            Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag, Tokens.XmlContent, // 22
+			            Tokens.XmlOpenTag, Tokens.Identifier, Tokens.Identifier, Tokens.Assign, Tokens.LiteralString, Tokens.XmlCloseTag, // 28
+			            Tokens.XmlContent, Tokens.XmlCData, Tokens.XmlContent, Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag, // 34
+			            Tokens.XmlContent, Tokens.XmlOpenTag, Tokens.Identifier, Tokens.Identifier, Tokens.Assign, Tokens.LiteralString, Tokens.XmlCloseTag,
+			            Tokens.XmlContent, Tokens.XmlCData, Tokens.XmlContent, Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag,
+			            Tokens.XmlContent, Tokens.XmlOpenEndTag, Tokens.Identifier, Tokens.XmlCloseTag
+			           );
+			
 			
 			CheckFoot(lexer);
 		}
 
 		void CheckFoot(ILexer lexer)
 		{
-			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
+			CheckTokens(lexer, Tokens.EOL, Tokens.End, Tokens.Sub, Tokens.EOL, Tokens.End, Tokens.Class);
-			Assert.AreEqual(Tokens.End, lexer.NextToken().Kind);
-			Assert.AreEqual(Tokens.Sub, lexer.NextToken().Kind);
-			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
-			Assert.AreEqual(Tokens.End, lexer.NextToken().Kind);
-			Assert.AreEqual(Tokens.Class, lexer.NextToken().Kind);
 		}
 
 		void CheckHead(ILexer lexer)
 		{
-			Assert.AreEqual(Tokens.Class, lexer.NextToken().Kind);
+			CheckTokens(lexer, Tokens.Class, Tokens.Identifier, Tokens.EOL,
-			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+			            Tokens.Sub, Tokens.Identifier, Tokens.OpenParenthesis,
-			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
+			            Tokens.CloseParenthesis, Tokens.EOL);
-			Assert.AreEqual(Tokens.Sub, lexer.NextToken().Kind);
+		}
-			Assert.AreEqual(Tokens.Identifier, lexer.NextToken().Kind);
+		
-			Assert.AreEqual(Tokens.OpenParenthesis, lexer.NextToken().Kind);
+		void CheckTokens(ILexer lexer, params int[] tokens)
-			Assert.AreEqual(Tokens.CloseParenthesis, lexer.NextToken().Kind);
+		{
-			Assert.AreEqual(Tokens.EOL, lexer.NextToken().Kind);
+			for (int i = 0; i < tokens.Length; i++) {
+				int token = tokens[i];
+				int next = lexer.NextToken().Kind;
+				Assert.AreEqual(token, next, "{2} {0} != {1}", Tokens.GetTokenString(token), Tokens.GetTokenString(next), i);
+			}
 		}
 	}
 }