From 6a9debd3ad49da4e063ae0143f81723bb14d5af8 Mon Sep 17 00:00:00 2001 From: Andrea Paatz Date: Sun, 31 Jul 2005 13:47:56 +0000 Subject: [PATCH] C# lexer ready, begin with vb lexer. git-svn-id: svn://svn.sharpdevelop.net/sharpdevelop/trunk@289 1ccf3a8d-04fe-1044-b7c0-cef0b8235c61 --- .../Project/Src/Lexer/AbstractLexer.cs | 4 +- .../Project/Src/Lexer/CSharp/Lexer.cs | 62 ++++++------- .../Project/Src/Lexer/VBNet/Lexer.cs | 90 +++++++++---------- .../GlobalScope/NamespaceDeclarationTests.cs | 5 +- 4 files changed, 81 insertions(+), 80 deletions(-) diff --git a/src/Libraries/NRefactory/Project/Src/Lexer/AbstractLexer.cs b/src/Libraries/NRefactory/Project/Src/Lexer/AbstractLexer.cs index c68b8afc4c..90560fb98d 100644 --- a/src/Libraries/NRefactory/Project/Src/Lexer/AbstractLexer.cs +++ b/src/Libraries/NRefactory/Project/Src/Lexer/AbstractLexer.cs @@ -19,7 +19,6 @@ namespace ICSharpCode.NRefactory.Parser public abstract class AbstractLexer : ILexer { protected TextReader reader; - protected int col = 1; protected int line = 1; @@ -103,6 +102,7 @@ namespace ICSharpCode.NRefactory.Parser /// public Token Token { get { + Console.WriteLine("Call to Token"); return lastToken; } } @@ -112,6 +112,7 @@ namespace ICSharpCode.NRefactory.Parser /// public Token LookAhead { get { + Console.WriteLine("Call to LookAhead"); return curToken; } } @@ -151,6 +152,7 @@ namespace ICSharpCode.NRefactory.Parser /// An object. public virtual Token Peek() { + Console.WriteLine("Call to Peek"); if (peekToken.next == null) { peekToken.next = Next(); specialTracker.InformToken(peekToken.next.kind); diff --git a/src/Libraries/NRefactory/Project/Src/Lexer/CSharp/Lexer.cs b/src/Libraries/NRefactory/Project/Src/Lexer/CSharp/Lexer.cs index ee4d640473..65c143ed65 100644 --- a/src/Libraries/NRefactory/Project/Src/Lexer/CSharp/Lexer.cs +++ b/src/Libraries/NRefactory/Project/Src/Lexer/CSharp/Lexer.cs @@ -34,8 +34,8 @@ namespace ICSharpCode.NRefactory.Parser.CSharp } if (Char.IsLetter(ch) || ch == '_') { - int x = col - 1; // col was incremented above, but we want the start of the identifier - int y = line; + int x = Col - 1; // Col was incremented above, but we want the start of the identifier + int y = Line; string s = ReadIdent(ch); int keyWordToken = Keywords.GetToken(s); if (keyWordToken >= 0) { @@ -45,7 +45,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp } if (Char.IsDigit(ch)) { - return ReadDigit(ch, col - 1); + return ReadDigit(ch, Col - 1); } switch (ch) { @@ -57,7 +57,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp } break; case '#': - Point start = new Point(col - 1, line); + Point start = new Point(Col - 1, Line); string directive = ReadIdent('#'); string argument = ReadToEOL(); this.specialTracker.AddPreProcessingDirective(directive, argument.Trim(), start, new Point(start.X + directive.Length + argument.Length, start.Y)); @@ -69,10 +69,10 @@ namespace ICSharpCode.NRefactory.Parser.CSharp case '@': int next = ReaderRead(); if (next == -1) { - errors.Error(line, col, String.Format("EOF after @")); + errors.Error(Line, Col, String.Format("EOF after @")); } else { - int x = col; - int y = line; + int x = Col; + int y = Line; ch = (char)next; if (ch == '"') { return ReadVerbatimString(); @@ -94,7 +94,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp return token; } - return new Token(Tokens.EOF, col, line, String.Empty); + return new Token(Tokens.EOF, Col, Line, String.Empty); } // The C# compiler has a fixed size length therefore we'll use a fixed size char array for identifiers @@ -113,7 +113,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp if (curPos < MAX_IDENTIFIER_LENGTH) { identBuffer[curPos++] = ch; } else { - errors.Error(line, col, String.Format("Identifier too long")); + errors.Error(Line, Col, String.Format("Identifier too long")); while ((peek = ReaderPeek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) { ReaderRead(); } @@ -126,7 +126,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp Token ReadDigit(char ch, int x) { unchecked { // prevent exception when ReaderPeek() = -1 is cast to char - int y = line; + int y = Line; sb.Length = 0; sb.Append(ch); string prefix = null; @@ -319,8 +319,8 @@ namespace ICSharpCode.NRefactory.Parser.CSharp Token ReadString() { - int x = col - 1; - int y = line; + int x = Col - 1; + int y = Line; sb.Length = 0; originalValue.Length = 0; @@ -358,8 +358,8 @@ namespace ICSharpCode.NRefactory.Parser.CSharp Token ReadVerbatimString() { - int x = col; - int y = line; + int x = Col; + int y = Line; int nextChar; sb.Length = 0; originalValue.Length = 0; @@ -397,7 +397,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp { int nextChar = ReaderRead(); if (nextChar == -1) { - errors.Error(line, col, String.Format("End of file reached inside escape sequence")); + errors.Error(Line, Col, String.Format("End of file reached inside escape sequence")); ch = '\0'; return String.Empty; } @@ -445,7 +445,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp escapeSequenceBuffer[curPos++] = c; if (number < 0) { - errors.Error(line, col - 1, String.Format("Invalid char in literal : {0}", c)); + errors.Error(Line, Col - 1, String.Format("Invalid char in literal : {0}", c)); } for (int i = 0; i < 3; ++i) { if (IsHex((char)ReaderPeek())) { @@ -460,7 +460,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp ch = (char)number; break; default: - errors.Error(line, col, String.Format("Unexpected escape sequence : {0}", c)); + errors.Error(Line, Col, String.Format("Unexpected escape sequence : {0}", c)); ch = '\0'; break; } @@ -469,8 +469,8 @@ namespace ICSharpCode.NRefactory.Parser.CSharp Token ReadChar() { - int x = col - 1; - int y = line; + int x = Col - 1; + int y = Line; int nextChar = ReaderRead(); if (nextChar == -1) { errors.Error(y, x, String.Format("End of file reached inside character literal")); @@ -493,8 +493,8 @@ namespace ICSharpCode.NRefactory.Parser.CSharp Token ReadOperator(char ch) { - int x = col; - int y = line; + int x = Col; + int y = Line; switch (ch) { case '+': switch (ReaderPeek()) { @@ -640,7 +640,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp // Prevent OverflowException when ReaderPeek returns -1 int tmp = ReaderPeek(); if (tmp > 0 && Char.IsDigit((char)tmp)) { - return ReadDigit('.', col - 1); + return ReadDigit('.', Col - 1); } return new Token(Tokens.Dot, x, y); case ')': @@ -675,7 +675,7 @@ namespace ICSharpCode.NRefactory.Parser.CSharp } break; default: - errors.Error(line, col, String.Format("Error while reading comment")); + errors.Error(Line, Col, String.Format("Error while reading comment")); break; } } @@ -701,9 +701,9 @@ namespace ICSharpCode.NRefactory.Parser.CSharp string tag = curWord.ToString(); curWord.Length = 0; if (specialCommentHash.ContainsKey(tag)) { - Point p = new Point(col, line); + Point p = new Point(Col, Line); string comment = ReadToEOL(); - tagComments.Add(new TagComment(tag, comment, p, new Point(col, line))); + tagComments.Add(new TagComment(tag, comment, p, new Point(Col, Line))); sb.Append(comment); break; } @@ -715,14 +715,14 @@ namespace ICSharpCode.NRefactory.Parser.CSharp void ReadSingleLineComment(CommentType commentType) { - specialTracker.StartComment(commentType, new Point(col, line)); + specialTracker.StartComment(commentType, new Point(Col, Line)); specialTracker.AddString(ReadCommentToEOL()); - specialTracker.FinishComment(new Point(col, line)); + specialTracker.FinishComment(new Point(Col, Line)); } void ReadMultiLineComment() { - specialTracker.StartComment(CommentType.Block, new Point(col, line)); + specialTracker.StartComment(CommentType.Block, new Point(Col, Line)); int nextChar; while ((nextChar = ReaderRead()) != -1) { char ch = (char)nextChar; @@ -735,14 +735,14 @@ namespace ICSharpCode.NRefactory.Parser.CSharp // End of multiline comment reached ? if (ch == '*' && ReaderPeek() == '/') { ReaderRead(); - specialTracker.FinishComment(new Point(col, line)); + specialTracker.FinishComment(new Point(Col, Line)); return; } specialTracker.AddChar(ch); } - specialTracker.FinishComment(new Point(col, line)); + specialTracker.FinishComment(new Point(Col, Line)); // Reached EOF before end of multiline comment. - errors.Error(line, col, String.Format("Reached EOF before the end of a multiline comment")); + errors.Error(Line, Col, String.Format("Reached EOF before the end of a multiline comment")); } /// diff --git a/src/Libraries/NRefactory/Project/Src/Lexer/VBNet/Lexer.cs b/src/Libraries/NRefactory/Project/Src/Lexer/VBNet/Lexer.cs index 628e7125e8..02f6017521 100644 --- a/src/Libraries/NRefactory/Project/Src/Lexer/VBNet/Lexer.cs +++ b/src/Libraries/NRefactory/Project/Src/Lexer/VBNet/Lexer.cs @@ -65,7 +65,7 @@ namespace ICSharpCode.NRefactory.Parser.VB int y = line; ++line; col = 1; - if (reader.Peek() == '\r') { + if (ReaderPeek() == '\r') { reader.Read(); if (!lineEnd) { lineEnd = true; @@ -81,12 +81,12 @@ namespace ICSharpCode.NRefactory.Parser.VB } if (ch == '_') { - if (reader.Peek() == -1) { + if (ReaderPeek() == -1) { errors.Error(line, col, String.Format("No EOF expected after _")); return new Token(Tokens.EOF); } ++col; - if (!Char.IsWhiteSpace((char)reader.Peek())) { + if (!Char.IsWhiteSpace((char)ReaderPeek())) { --col; int x = col; int y = line; @@ -103,7 +103,7 @@ namespace ICSharpCode.NRefactory.Parser.VB col = 1; break; } - if (reader.Peek() != -1) { + if (ReaderPeek() != -1) { ch = (char)reader.Read(); ++col; } @@ -115,11 +115,11 @@ namespace ICSharpCode.NRefactory.Parser.VB } if (ch == '#') { - while (Char.IsWhiteSpace((char)reader.Peek())) { + while (Char.IsWhiteSpace((char)ReaderPeek())) { ++col; reader.Read(); } - if (Char.IsDigit((char)reader.Peek())) { + if (Char.IsDigit((char)ReaderPeek())) { int x = col; int y = line; string s = ReadDate(); @@ -138,7 +138,7 @@ namespace ICSharpCode.NRefactory.Parser.VB if (ch == '[') { // Identifier lineEnd = false; - if (reader.Peek() == -1) { + if (ReaderPeek() == -1) { errors.Error(line, col, String.Format("Identifier expected")); } ch = (char)reader.Read(); @@ -149,7 +149,7 @@ namespace ICSharpCode.NRefactory.Parser.VB int x = col - 1; int y = line; string s = ReadIdent(ch); - if (reader.Peek() == -1) { + if (ReaderPeek() == -1) { errors.Error(line, col, String.Format("']' expected")); } ch = (char)reader.Read(); @@ -189,10 +189,10 @@ namespace ICSharpCode.NRefactory.Parser.VB } if (ch == '&') { lineEnd = false; - if (reader.Peek() == -1) { + if (ReaderPeek() == -1) { return ReadOperator('&'); } - ch = (char)reader.Peek(); + ch = (char)ReaderPeek(); ++col; if (Char.ToUpper(ch) == 'H' || Char.ToUpper(ch) == 'O') { --col; @@ -215,7 +215,7 @@ namespace ICSharpCode.NRefactory.Parser.VB int x = col - 1; int y = line; string s = ReadString(); - if (reader.Peek() != -1 && (reader.Peek() == 'C' || reader.Peek() == 'c')) { + if (ReaderPeek() != -1 && (ReaderPeek() == 'C' || ReaderPeek() == 'c')) { reader.Read(); ++col; if (s.Length != 1) { @@ -241,7 +241,7 @@ namespace ICSharpCode.NRefactory.Parser.VB sb.Length = 0; sb.Append(ch); int peek; - while ((peek = reader.Peek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) { + while ((peek = ReaderPeek()) != -1 && (Char.IsLetterOrDigit(ch = (char)peek) || ch == '_')) { reader.Read(); ++col; sb.Append(ch.ToString()); @@ -277,40 +277,40 @@ namespace ICSharpCode.NRefactory.Parser.VB bool isdouble = false; bool isdecimal = false; - if (reader.Peek() == -1) { + if (ReaderPeek() == -1) { if (ch == '&') { errors.Error(line, col, String.Format("digit expected")); } return new Token(Tokens.LiteralInteger, x, y, sb.ToString() ,ch - '0'); } if (ch == '.') { - if (Char.IsDigit((char)reader.Peek())) { + if (Char.IsDigit((char)ReaderPeek())) { isdouble = true; // double is default if (ishex || isokt) { errors.Error(line, col, String.Format("No hexadecimal or oktadecimal floating point values allowed")); } ++col; - while (reader.Peek() != -1 && Char.IsDigit((char)reader.Peek())){ // read decimal digits beyond the dot + while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot digit += (char)reader.Read(); ++col; } } - } else if (ch == '&' && Char.ToUpper((char)reader.Peek()) == 'H') { + } else if (ch == '&' && Char.ToUpper((char)ReaderPeek()) == 'H') { const string hex = "0123456789ABCDEF"; sb.Append((char)reader.Read()); // skip 'H' ++col; - while (reader.Peek() != -1 && hex.IndexOf(Char.ToUpper((char)reader.Peek())) != -1) { + while (ReaderPeek() != -1 && hex.IndexOf(Char.ToUpper((char)ReaderPeek())) != -1) { ch = (char)reader.Read(); sb.Append(ch); digit += Char.ToUpper(ch); ++col; } ishex = true; - } else if (reader.Peek() != -1 && ch == '&' && Char.ToUpper((char)reader.Peek()) == 'O') { + } else if (ReaderPeek() != -1 && ch == '&' && Char.ToUpper((char)ReaderPeek()) == 'O') { const string okt = "01234567"; sb.Append((char)reader.Read()); // skip 'O' ++col; - while (reader.Peek() != -1 && okt.IndexOf(Char.ToUpper((char)reader.Peek())) != -1) { + while (ReaderPeek() != -1 && okt.IndexOf(Char.ToUpper((char)ReaderPeek())) != -1) { ch = (char)reader.Read(); sb.Append(ch); digit += Char.ToUpper(ch); @@ -318,7 +318,7 @@ namespace ICSharpCode.NRefactory.Parser.VB } isokt = true; } else { - while (reader.Peek() != -1 && Char.IsDigit((char)reader.Peek())) { + while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { ch = (char)reader.Read();; digit += ch; sb.Append(ch); @@ -326,13 +326,13 @@ namespace ICSharpCode.NRefactory.Parser.VB } } - if (reader.Peek() != -1 && ("%&SILU".IndexOf(Char.ToUpper((char)reader.Peek())) != -1 || ishex || isokt)) { - ch = (char)reader.Peek(); + if (ReaderPeek() != -1 && ("%&SILU".IndexOf(Char.ToUpper((char)ReaderPeek())) != -1 || ishex || isokt)) { + ch = (char)ReaderPeek(); sb.Append(ch); ch = Char.ToUpper(ch); bool unsigned = ch == 'U'; if (unsigned) { - ch = (char)reader.Peek(); + ch = (char)ReaderPeek(); sb.Append(ch); ch = Char.ToUpper(ch); if (ch != 'I' && ch != 'L' && ch != 'S') { @@ -398,16 +398,16 @@ namespace ICSharpCode.NRefactory.Parser.VB } } Token nextToken = null; // if we accedently read a 'dot' - if (!isdouble && reader.Peek() == '.') { // read floating point number + if (!isdouble && ReaderPeek() == '.') { // read floating point number reader.Read(); - if (reader.Peek() != -1 && Char.IsDigit((char)reader.Peek())) { + if (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { isdouble = true; // double is default if (ishex || isokt) { errors.Error(line, col, String.Format("No hexadecimal or oktadecimal floating point values allowed")); } digit += '.'; ++col; - while (reader.Peek() != -1 && Char.IsDigit((char)reader.Peek())){ // read decimal digits beyond the dot + while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())){ // read decimal digits beyond the dot digit += (char)reader.Read(); ++col; } @@ -416,22 +416,22 @@ namespace ICSharpCode.NRefactory.Parser.VB } } - if (reader.Peek() != -1 && Char.ToUpper((char)reader.Peek()) == 'E') { // read exponent + if (ReaderPeek() != -1 && Char.ToUpper((char)ReaderPeek()) == 'E') { // read exponent isdouble = true; digit += (char)reader.Read(); ++col; - if (reader.Peek() != -1 && (reader.Peek() == '-' || reader.Peek() == '+')) { + if (ReaderPeek() != -1 && (ReaderPeek() == '-' || ReaderPeek() == '+')) { digit += (char)reader.Read(); ++col; } - while (reader.Peek() != -1 && Char.IsDigit((char)reader.Peek())) { // read exponent value + while (ReaderPeek() != -1 && Char.IsDigit((char)ReaderPeek())) { // read exponent value digit += (char)reader.Read(); ++col; } } - if (reader.Peek() != -1) { - switch (char.ToUpper((char)reader.Peek())) { + if (ReaderPeek() != -1) { + switch (char.ToUpper((char)ReaderPeek())) { case 'R': case '#': reader.Read(); @@ -531,7 +531,7 @@ namespace ICSharpCode.NRefactory.Parser.VB ch = (char)nextChar; ++col; if (ch == '"') { - if (reader.Peek() != -1 && reader.Peek() == '"') { + if (ReaderPeek() != -1 && ReaderPeek() == '"') { sb.Append('"'); reader.Read(); ++col; @@ -610,7 +610,7 @@ namespace ICSharpCode.NRefactory.Parser.VB int y = line; switch(ch) { case '+': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -620,7 +620,7 @@ namespace ICSharpCode.NRefactory.Parser.VB } return new Token(Tokens.Plus, x, y); case '-': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -630,7 +630,7 @@ namespace ICSharpCode.NRefactory.Parser.VB } return new Token(Tokens.Minus, x, y); case '*': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -640,7 +640,7 @@ namespace ICSharpCode.NRefactory.Parser.VB } return new Token(Tokens.Times, x, y, "*"); case '/': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -650,7 +650,7 @@ namespace ICSharpCode.NRefactory.Parser.VB } return new Token(Tokens.Div, x, y); case '\\': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -660,7 +660,7 @@ namespace ICSharpCode.NRefactory.Parser.VB } return new Token(Tokens.DivInteger, x, y); case '&': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -670,7 +670,7 @@ namespace ICSharpCode.NRefactory.Parser.VB } return new Token(Tokens.ConcatString, x, y); case '^': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -684,7 +684,7 @@ namespace ICSharpCode.NRefactory.Parser.VB case '=': return new Token(Tokens.Assign, x, y); case '<': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; @@ -695,7 +695,7 @@ namespace ICSharpCode.NRefactory.Parser.VB return new Token(Tokens.NotEqual, x, y); case '<': reader.Read(); - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); col += 2; @@ -708,15 +708,15 @@ namespace ICSharpCode.NRefactory.Parser.VB } return new Token(Tokens.LessThan, x, y); case '>': - switch (reader.Peek()) { + switch (ReaderPeek()) { case '=': reader.Read(); ++col; return new Token(Tokens.GreaterEqual, x, y); case '>': reader.Read(); - if (reader.Peek() != -1) { - switch (reader.Peek()) { + if (ReaderPeek() != -1) { + switch (ReaderPeek()) { case '=': reader.Read(); col += 2; @@ -733,7 +733,7 @@ namespace ICSharpCode.NRefactory.Parser.VB return new Token(Tokens.Comma, x, y); case '.': // Prevent OverflowException when Peek returns -1 - int tmp = reader.Peek(); + int tmp = ReaderPeek(); if (tmp > 0 && Char.IsDigit((char)tmp)) { --col; return ReadDigit('.', col); diff --git a/src/Libraries/NRefactory/Test/Parser/GlobalScope/NamespaceDeclarationTests.cs b/src/Libraries/NRefactory/Test/Parser/GlobalScope/NamespaceDeclarationTests.cs index 6c19dfe57d..1abf670fcd 100644 --- a/src/Libraries/NRefactory/Test/Parser/GlobalScope/NamespaceDeclarationTests.cs +++ b/src/Libraries/NRefactory/Test/Parser/GlobalScope/NamespaceDeclarationTests.cs @@ -53,9 +53,8 @@ namespace ICSharpCode.NRefactory.Tests.AST [Test] public void VBNetSimpleNamespaceTest() { - string program = "Namespace TestNamespace\n" + - "End Namespace\n"; - + string program = "Namespace TestNamespace" + Environment.NewLine + + "End Namespace" +Environment.NewLine; NamespaceDeclaration ns = (NamespaceDeclaration)ParseUtilVBNet.ParseGlobal(program, typeof(NamespaceDeclaration)); Assert.AreEqual("TestNamespace", ns.Name); }