Fixed small lexer problem.

git-svn-id: svn://svn.sharpdevelop.net/sharpdevelop/trunk@908 1ccf3a8d-04fe-1044-b7c0-cef0b8235c61

src/Libraries/NRefactory/Project/NRefactory.csproj

@@ -15,7 +15,7 @@
     <OutputType>Library</OutputType>
     <SignAssembly>true</SignAssembly>
     <AssemblyOriginatorKeyFile>Resources\ICSharpCode.NRefactory.snk</AssemblyOriginatorKeyFile>
-    <AllowUnsafeBlocks>True</AllowUnsafeBlocks>
+    <AllowUnsafeBlocks>False</AllowUnsafeBlocks>
     <RegisterForComInterop>False</RegisterForComInterop>
     <GenerateSerializationAssemblies>Auto</GenerateSerializationAssemblies>
     <BaseAddress>4194304</BaseAddress>
@@ -34,14 +34,15 @@
     <CheckForOverflowUnderflow>False</CheckForOverflowUnderflow>
     <DefineConstants>TEST</DefineConstants>
     <OutputPath>..\..\..\..\bin\</OutputPath>
-    <TreatWarningsAsErrors>False</TreatWarningsAsErrors>
+    <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
     <DebugType>Full</DebugType>
     <DebugSymbols>true</DebugSymbols>
   </PropertyGroup>
   <PropertyGroup Condition=" '$(Configuration)' == 'Release' ">
-    <DebugSymbols>False</DebugSymbols>
+    <DebugSymbols>false</DebugSymbols>
+    <DebugType>None</DebugType>
   </PropertyGroup>
   <ItemGroup>
     <Reference Include="System" />

src/Libraries/NRefactory/Project/Src/Lexer/AbstractLexer.cs

@@ -152,7 +152,7 @@ namespace ICSharpCode.NRefactory.Parser
 		/// <summary>
 		/// Must be called before a peek operation.
 		/// </summary>
-		public virtual void StartPeek()
+		public void StartPeek()
 		{
 			peekToken = curToken;
 		}
@@ -161,7 +161,7 @@ namespace ICSharpCode.NRefactory.Parser
 		/// Gives back the next token. A second call to Peek() gives the next token after the last call for Peek() and so on.
 		/// </summary>
 		/// <returns>An <see cref="Token"/> object.</returns>
-		public virtual Token Peek()
+		public Token Peek()
 		{
 //			Console.WriteLine("Call to Peek");
 			if (peekToken.next == null) {
@@ -214,15 +214,9 @@ namespace ICSharpCode.NRefactory.Parser
 		
 		protected bool IsIdentifierPart(int ch)
 		{
-			// char.IsLetter is slow, so optimize for raw ASCII
-			if (ch < 48)  return false; // 48 = '0'
-			if (ch <= 57) return true;  // 57 = '9'
-			if (ch < 65)  return false; // 65 = 'A'
-			if (ch <= 90) return true;  // 90 = 'Z'
 			if (ch == 95) return true;  // 95 = '_'
-			if (ch < 97)  return false; // 97 = 'a'
-			if (ch <= 122) return true; // 97 = 'z'
-			return char.IsLetter((char)ch); // accept unicode letters
+			if (ch == -1) return false;
+			return char.IsLetterOrDigit((char)ch); // accept unicode letters
 		}
 		
 		protected bool IsHex(char digit)

src/Libraries/NRefactory/Project/Src/Lexer/CSharp/Lexer.cs

@@ -25,30 +25,18 @@ namespace ICSharpCode.NRefactory.Parser.CSharp
 		protected override Token Next()
 		{
 			int nextChar;
+			char ch;
 			while ((nextChar = ReaderRead()) != -1) {
-				char ch = (char)nextChar;
-				if (ch == ' ' || ch == '\t')
-					continue;
-				
-				if (Char.IsWhiteSpace(ch)) {
-					HandleLineEnd(ch);
-					continue;
-				}
-				
-				if (Char.IsLetter(ch) || ch == '_') {
-					int x = Col - 1; // Col was incremented above, but we want the start of the identifier
-					int y = Line;
-					string s = ReadIdent(ch);
-					int keyWordToken = Keywords.GetToken(s);
-					if (keyWordToken >= 0) {
-						return new Token(keyWordToken, x, y);
-					}
-					return new Token(Tokens.Identifier, x, y, s);
-				}
-				
 				Token token;
 				
-				switch (ch) {
+				switch (nextChar) {
+					case ' ':
+					case '\t':
+						continue;
+					case '\r':
+					case '\n':
+						HandleLineEnd((char)nextChar);
+						continue;
 					case '/':
 						int peek = ReaderPeek();
 						if (peek == '/' || peek == '*') {
@@ -90,7 +78,17 @@ namespace ICSharpCode.NRefactory.Parser.CSharp
 						}
 						break;
 					default:
-						if (Char.IsDigit(ch)) {
+						ch = (char)nextChar;
+						if (Char.IsLetter(ch) || ch == '_') {
+							int x = Col - 1; // Col was incremented above, but we want the start of the identifier
+							int y = Line;
+							string s = ReadIdent(ch);
+							int keyWordToken = Keywords.GetToken(s);
+							if (keyWordToken >= 0) {
+								return new Token(keyWordToken, x, y);
+							}
+							return new Token(Tokens.Identifier, x, y, s);
+						} else if (Char.IsDigit(ch)) {
 							token = ReadDigit(ch, Col - 1);
 						} else {
 							token = ReadOperator(ch);