From 62b5ddf53199dbc3dddcf318af9ad1fb077605f5 Mon Sep 17 00:00:00 2001 From: Siegfried Pammer Date: Sun, 16 May 2010 14:45:45 +0000 Subject: [PATCH] replaced BuildKeywords.pl by KeywordGenerator.cs git-svn-id: svn://svn.sharpdevelop.net/sharpdevelop/branches/vbnet@5828 1ccf3a8d-04fe-1044-b7c0-cef0b8235c61 --- .../KeywordGenerator.cs | 398 ++++++++++++++++++ .../NRefactory/NRefactoryASTGenerator/Main.cs | 8 +- .../NRefactoryASTGenerator.csproj | 5 +- .../Project/Src/Lexer/BuildKeywords.pl | 340 --------------- 4 files changed, 408 insertions(+), 343 deletions(-) create mode 100644 src/Libraries/NRefactory/NRefactoryASTGenerator/KeywordGenerator.cs delete mode 100644 src/Libraries/NRefactory/Project/Src/Lexer/BuildKeywords.pl diff --git a/src/Libraries/NRefactory/NRefactoryASTGenerator/KeywordGenerator.cs b/src/Libraries/NRefactory/NRefactoryASTGenerator/KeywordGenerator.cs new file mode 100644 index 0000000000..f69e30721f --- /dev/null +++ b/src/Libraries/NRefactory/NRefactoryASTGenerator/KeywordGenerator.cs @@ -0,0 +1,398 @@ +// +// +// +// +// $Revision$ +// + +using System; +using System.Collections.Generic; +using System.Diagnostics; +using System.IO; +using System.Linq; +using System.Security.Principal; +using System.Text; +using System.Text.RegularExpressions; +using System.Threading.Tasks; +using NRefactoryASTGenerator.Ast; + +namespace NRefactoryASTGenerator +{ + static class KeywordGenerator + { + static readonly string baseDir = "../../../Project/Src/Lexer/"; + static readonly string testBaseDir = "../../../Test/Lexer/"; + static readonly string parserBaseDir = "../../../Project/Src/Parser/"; + + public static void Generate() + { + Generate("CSharp"); + Generate("VBNet"); + } + + static void Generate(string language) + { + try { + Dictionary properties = new Dictionary(); + Dictionary sets = new Dictionary(); + List keywords = new List(); + List terminals = new List(); + Dictionary specialChars = new Dictionary(); + + ReadFile(properties, sets, keywords, terminals, specialChars, language); + + GenerateFiles(properties, sets, keywords, terminals, specialChars, language); + } catch (Exception e) { + Debug.Print(e.ToString()); + } + } + + static void GenerateFiles(Dictionary properties, Dictionary sets, + List keywords, List terminals, Dictionary specialChars, + string language) + { + GenerateKeywords(properties, keywords, language); + GenerateTokens(properties, sets, keywords, terminals, specialChars, language); + GenerateTests(keywords, specialChars, language); + GenerateKeywordSection(keywords, terminals, specialChars, language); + } + + static void GenerateKeywordSection(List keywords, List terminals, Dictionary specialChars, string language) + { + string sourceDir = Path.Combine(parserBaseDir, language, (language == "CSharp" ? "cs" : language) + ".atg"); + StringBuilder builder = new StringBuilder(); + + builder.AppendLine("/* START AUTOGENERATED TOKENS SECTION */"); + builder.AppendLine("TOKENS"); + builder.AppendLine("\t/* ----- terminal classes ----- */"); + builder.AppendLine("\t/* EOF is 0 */"); + + foreach (string terminal in terminals) { + if (terminal == "EOF") + continue; + if (terminal == "Identifier") { + builder.AppendLine("\tident"); + continue; + } + builder.AppendLine("\t" + terminal); + } + + builder.AppendLine(); + builder.AppendLine("\t/* ----- special character ----- */"); + foreach (string specialChar in specialChars.Values) { + builder.AppendLine("\t" + specialChar); + } + + builder.AppendLine(); + builder.AppendLine("\t/* ----- keywords ----- */"); + foreach (string keyword in keywords) { + builder.AppendLine("\t\"" + keyword + "\""); + } + + builder.AppendLine("/* END AUTOGENERATED TOKENS SECTION */"); + + string[] generatedLines = builder.ToString().Split(new string[] { Environment.NewLine }, StringSplitOptions.None); + string[] lines = File.ReadAllLines(sourceDir); + + var newContent = lines + .TakeWhile(l => l != "/* START AUTOGENERATED TOKENS SECTION */") + .Concat(generatedLines) + .Concat(lines.SkipWhile(l => l != "/* END AUTOGENERATED TOKENS SECTION */").Skip(2)) + .ToArray(); + + File.WriteAllLines(sourceDir, newContent); + } + + static void GenerateTests(List keywords, Dictionary specialChars, string language) + { + string sourceDir = Path.Combine(testBaseDir, language, "LexerTests.cs"); + using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) { + writer.WriteLine("using System;"); + writer.WriteLine("using System.IO;"); + writer.WriteLine("using NUnit.Framework;"); + writer.WriteLine("using ICSharpCode.NRefactory.Parser;"); + writer.WriteLine("using ICSharpCode.NRefactory.Parser.{0};", language == "VBNet" ? "VB" : language); + writer.WriteLine("using ICSharpCode.NRefactory.PrettyPrinter;\n"); + writer.WriteLine(); + writer.WriteLine("namespace ICSharpCode.NRefactory.Tests.Lexer.{0}", language == "VBNet" ? "VB" : language); + writer.WriteLine("{"); + writer.WriteLine("\t[TestFixture]"); + writer.WriteLine("\tpublic sealed class LexerTests"); + writer.WriteLine("\t{"); + writer.WriteLine("\t\tILexer GenerateLexer(StringReader sr)"); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\treturn ParserFactory.CreateLexer(SupportedLanguage.CSharp, sr);"); + writer.WriteLine("\t\t}"); + for (int i = 0; i < specialChars.Values.Count; i++) { + writer.WriteLine(); + writer.WriteLine("\t\t[Test]"); + writer.WriteLine("\t\tpublic void Test{0}()", specialChars.Keys.ElementAt(i)); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\tILexer lexer = GenerateLexer(new StringReader({0}));", specialChars.Values.ElementAt(i)); + writer.WriteLine("\t\t\tAssert.AreEqual(Tokens.{0}, lexer.NextToken().Kind);", specialChars.Keys.ElementAt(i)); + writer.WriteLine("\t\t}"); + } + foreach (string keyword in keywords) { + writer.WriteLine(); + writer.WriteLine("\t\t[Test]"); + writer.WriteLine("\t\tpublic void Test{0}()", UpperCaseFirst(keyword)); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\tILexer lexer = GenerateLexer(new StringReader(\"{0}\"));", keyword); + writer.WriteLine("\t\t\tAssert.AreEqual(Tokens.{0}, lexer.NextToken().Kind);", UpperCaseFirst(keyword)); + writer.WriteLine("\t\t}"); + } + writer.WriteLine("\t}"); + writer.WriteLine("}"); + } + } + + static void GenerateTokens(Dictionary properties, Dictionary sets, List keywords, List terminals, Dictionary specialChars, string language) + { + string sourceDir = Path.Combine(baseDir, language, "Tokens.cs"); + using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) { + writer.WriteLine("// this file was autogenerated by a tool."); + writer.WriteLine("using System;"); + writer.WriteLine("using System.Collections;"); + writer.WriteLine(); + writer.WriteLine("namespace {0}", properties["Namespace"]); + writer.WriteLine("{"); + writer.WriteLine("\tpublic static class Tokens"); + writer.WriteLine("\t{"); + writer.WriteLine("\t\t// ----- terminal classes -----"); + int tokenValue = 0; + foreach (string terminal in terminals) + writer.WriteToken(terminal, ref tokenValue); + writer.WriteLine(); + writer.WriteLine("\t\t// ----- special character -----"); + foreach (string specialChar in specialChars.Keys) + writer.WriteToken(specialChar, ref tokenValue); + writer.WriteLine(); + writer.WriteLine("\t\t// ----- keywords -----"); + foreach (string keyword in keywords) + writer.WriteToken(keyword, ref tokenValue); + writer.WriteLine(); + writer.WriteLine("\t\tpublic const int MaxToken = {0};", tokenValue); + if (sets.Any()) { + writer.WriteLine("\t\tstatic BitArray NewSet(params int[] values)"); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\tBitArray bitArray = new BitArray(MaxToken);"); + writer.WriteLine("\t\t\tforeach (int val in values) {"); + writer.WriteLine("\t\t\tbitArray[val] = true;"); + writer.WriteLine("\t\t\t}"); + writer.WriteLine("\t\t\treturn bitArray;"); + writer.WriteLine("\t\t}"); + foreach (var pair in sets) { + StringBuilder builder = new StringBuilder(); + PrintList(pair.Value, builder, sets, specialChars); + writer.WriteLine("\t\tpublic static BitArray {0} = NewSet({1});", pair.Key, builder.ToString()); + } + writer.WriteLine(); + } + + // write token number --> string function. + writer.WriteLine("\t\tstatic string[] tokenList = new string[] {"); + + writer.WriteLine("\t\t\t// ----- terminal classes -----"); + foreach (string terminal in terminals) + writer.WriteLine("\t\t\t\"<{0}>\",", terminal); + + writer.WriteLine("\t\t\t// ----- special character -----"); + foreach (string specialChar in specialChars.Values) + writer.WriteLine("\t\t\t{0},", specialChar); + + writer.WriteLine("\t\t\t// ----- keywords -----"); + foreach (string keyword in keywords) + writer.WriteLine("\t\t\t\"{0}\",", keyword); + + writer.WriteLine("\t\t};"); + + writer.WriteLine("\t\tpublic static string GetTokenString(int token)"); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\tif (token >= 0 && token < tokenList.Length) {"); + writer.WriteLine("\t\t\t\treturn tokenList[token];"); + writer.WriteLine("\t\t\t}"); + writer.WriteLine("\t\t\tthrow new System.NotSupportedException(\"Unknown token:\" + token);"); + writer.WriteLine("\t\t}"); + + writer.WriteLine("\t}"); + writer.WriteLine("}"); + } + } + + static void PrintList(string[] value, StringBuilder builder, Dictionary sets, Dictionary specialChars) + { + for (int i = 0; i < value.Length; i++) { + string item = value[i]; + if (Regex.IsMatch(item, "\\\"(\\w+)\\\"")) // keywords + builder.Append(UpperCaseFirst(item.Trim('"', ' ', '\t'))); + else if (Regex.IsMatch(item, "\\\"(\\W+)\\\"")) // special chars + builder.Append(specialChars.Keys.ElementAt(specialChars.Values.FindIndex(it => item == it))); + else if (Regex.IsMatch(item, "@(\\w+)")) // other list + PrintList(sets[item.Substring(1)], builder, sets, specialChars); + else + builder.Append(item); + if (i + 1 < value.Length) + builder.Append(", "); + } + } + + static int FindIndex(this IEnumerable items, Func f) + { + int index = -1; + foreach (T item in items) { + index++; + if (f(item)) + return index; + } + + return -1; + } + + static void GenerateKeywords(Dictionary properties, List keywords, string language) + { + string sourceDir = Path.Combine(baseDir, language, "Keywords.cs"); + using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) { + writer.WriteLine("// this file was autogenerated by a tool."); + writer.WriteLine("using System;"); + writer.WriteLine(); + writer.WriteLine("namespace {0}", properties["Namespace"]); + writer.WriteLine("{"); + writer.WriteLine("\tpublic static class Keywords"); + writer.WriteLine("\t{"); + writer.WriteLine("\t\tstatic readonly string[] keywordList = {"); + for (int i = 0; i < keywords.Count; i++) { + writer.Write("\t\t\t\"{0}\"", properties["UpperCaseKeywords"] == "True" ? keywords[i].ToUpperInvariant() : keywords[i]); + if (i + 1 < keywords.Count) + writer.Write(","); + writer.WriteLine(); + } + writer.WriteLine("\t\t};"); + writer.WriteLine("\t\t"); + writer.WriteLine("\t\tstatic LookupTable keywords = new LookupTable({0});", properties["UpperCaseKeywords"] == "True" ? "false" : "true"); + writer.WriteLine("\t\t"); + writer.WriteLine("\t\tstatic Keywords()"); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\tfor (int i = 0; i < keywordList.Length; ++i) {"); + writer.WriteLine("\t\t\t\tkeywords[keywordList[i]] = i + Tokens.{0};", UpperCaseFirst(keywords[0])); + writer.WriteLine("\t\t\t}"); + writer.WriteLine("\t\t}"); + writer.WriteLine("\t\t"); + writer.WriteLine("\t\tpublic static int GetToken(string keyword)"); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\treturn keywords[keyword];"); + writer.WriteLine("\t\t}"); + writer.WriteLine("\t\t"); + writer.WriteLine("\t\tpublic static bool IsNonIdentifierKeyword(string word)"); + writer.WriteLine("\t\t{"); + writer.WriteLine("\t\t\tint token = GetToken(word);"); + writer.WriteLine("\t\t\tif (token < 0)"); + writer.WriteLine("\t\t\t\treturn false;"); + writer.WriteLine("\t\t\treturn !Tokens.IdentifierTokens[token];"); + writer.WriteLine("\t\t}"); + + writer.WriteLine("\t}"); + writer.WriteLine("}"); + + writer.Close(); + } + } + + #region input + static void ReadFile(Dictionary properties, Dictionary sets, + List keywords, List terminals, Dictionary specialChars, + string language) + { + string sourceDir = Path.Combine(baseDir, language, "KeywordList.txt"); + + using (StreamReader reader = new StreamReader(new FileStream(sourceDir, FileMode.Open))) { + string line = reader.ReadLine(); + while (line != null) { + ReadProperty(properties, line); + ReadKeyword(keywords, line); + ReadSet(sets, line); + ReadTerminalSymbol(terminals, line); + ReadSpecialChar(specialChars, line); + line = reader.ReadLine(); + } + reader.Close(); + } + } + + static void ReadProperty(Dictionary properties, string line) + { + // properties form: $PROPERTY = "VALUE" + Match match = Regex.Match(line, @"^\s*\$(\w+)\s*=\s*(\S+)\s*$"); + + if (match.Success) { + properties.Add(match.Groups[1].Value, match.Groups[2].Value); + } + } + + static void ReadKeyword(List keywords, string line) + { + // special keywords form: "VALUE" + Match match = Regex.Match(line, "^\\s*\\\"(\\S+)\\s*\\\"\\s*$"); + + if (match.Success) { + keywords.Add(match.Groups[1].Value); + } + } + + static void ReadSet(Dictionary sets, string line) + { + // sets form: NAME(comma separated list) + Match match = Regex.Match(line, @"^\s*(\w+)\s*\((.*)\)\s*$"); + + if (match.Success) { + sets.Add( + match.Groups[1].Value, + match.Groups[2].Value.Split(new[] {", "}, StringSplitOptions.None) + ); + } + } + + static void ReadTerminalSymbol(List terminals, string line) + { + // special terminal classes form: name + Match match = Regex.Match(line, @"^\s*(\w+)\s*$"); + + if (match.Success) { + terminals.Add(match.Groups[1].Value); + } + } + + static void ReadSpecialChar(Dictionary specialChars, string line) + { + // special characters form: name = "VALUE" + Match match = Regex.Match(line, @"^\s*(\w+)\s*=\s*(\S+)\s*$"); + + if (match.Success) { + specialChars.Add(match.Groups[1].Value, match.Groups[2].Value); + } + } + #endregion + + #region helpers + static string UpperCaseFirst(string keyword) + { + return char.ToUpperInvariant(keyword[0]) + keyword.Substring(1); + } + + static void WriteToken(this StreamWriter writer, string tokenName, ref int tokenValue) + { + string formattedName = UpperCaseFirst(tokenName).PadRight(20); + if (tokenName == "GetType" || tokenName.ToLowerInvariant() == "equals") + writer.WriteLine("\t\tnew public const int {0} = {1};", formattedName, tokenValue); + else + writer.WriteLine("\t\tpublic const int {0} = {1};", formattedName, tokenValue); + tokenValue++; + } + + + static void WriteList(this StreamWriter writer, Tuple data) + { + + } + #endregion + } +} diff --git a/src/Libraries/NRefactory/NRefactoryASTGenerator/Main.cs b/src/Libraries/NRefactory/NRefactoryASTGenerator/Main.cs index eac2e44ce3..edb328a002 100644 --- a/src/Libraries/NRefactory/NRefactoryASTGenerator/Main.cs +++ b/src/Libraries/NRefactory/NRefactoryASTGenerator/Main.cs @@ -159,6 +159,12 @@ namespace NRefactoryASTGenerator File.WriteAllText(visitorsDir + "NotImplementedAstVisitor.cs", NormalizeNewLines(writer)); } Debug.WriteLine("AST Generator done!"); + + Debug.WriteLine("start keyword list generation..."); + + KeywordGenerator.Generate(); + + Debug.WriteLine("keyword list generation done!"); } static string NormalizeNewLines(StringWriter writer) @@ -585,4 +591,4 @@ namespace NRefactoryASTGenerator return td; } } -} +} \ No newline at end of file diff --git a/src/Libraries/NRefactory/NRefactoryASTGenerator/NRefactoryASTGenerator.csproj b/src/Libraries/NRefactory/NRefactoryASTGenerator/NRefactoryASTGenerator.csproj index 03473aa40a..c711ad7e89 100644 --- a/src/Libraries/NRefactory/NRefactoryASTGenerator/NRefactoryASTGenerator.csproj +++ b/src/Libraries/NRefactory/NRefactoryASTGenerator/NRefactoryASTGenerator.csproj @@ -1,5 +1,5 @@  - + Exe NRefactoryASTGenerator @@ -17,7 +17,7 @@ 4 0169 false - v3.5 + v4.0 bin\Debug\ @@ -44,6 +44,7 @@ + diff --git a/src/Libraries/NRefactory/Project/Src/Lexer/BuildKeywords.pl b/src/Libraries/NRefactory/Project/Src/Lexer/BuildKeywords.pl deleted file mode 100644 index a34bdde9a7..0000000000 --- a/src/Libraries/NRefactory/Project/Src/Lexer/BuildKeywords.pl +++ /dev/null @@ -1,340 +0,0 @@ -#!/bin/perl - -# File names -$keyword_file = "KeywordList.txt"; -$keywords_outfile = "Keywords.cs"; -$tokens_outfile = "Tokens.cs"; -$unittests_outfile = "LexerTests.cs"; -$ATGTokensSection = "ATGTokensSection.gen"; - -#read infile -print "\n"; -print "Reading keyword definition from '$keyword_file'.\n"; -open(DAT, $keyword_file) || die("Could not open file '$keyword_file': $!"); -@raw_data=; -close(DAT); -print "done.\n"; - -#analyse infile -print "starting analysis... this could take a few seconds.\n"; - -foreach (@raw_data) { - if ($_=~/^\s*\$(\w+)\s*=\s*(\S+)\s*$/) { - #properties form: $PROPERTY = "VALUE" - $properties{$1} = $2; - } elsif ($_=~/^\s*(\w+)\s*=\s*(\S+)\s*$/) { - #special characters form: name = "VALUE" - $specialCharLookup{$2} = $1; - $special_chars[$#special_chars + 1] = $1; - $special_values[$#special_values + 1] = $2; - } elsif ($_=~/^\s*\"(\S+)\s*\"\s*$/) { - #special keywords form: "VALUE" - $keywords[$#keywords + 1] = $1; - } elsif ($_=~/^\s*(\w+)\s*\((.*)\)\s*$/) { - $sets[$#sets + 1] = $1; - #Split set values (comma separated list) - $_ = $2; - @vals = split/\s*,\s*/; - - push @$setValues, [@vals]; - } elsif ($_=~/^\s*(\w+)\s*$/) { - #special terminal classes form: name - $terminals[$#terminals + 1] = $1 - } elsif ($_=~/^\s*(#.*)?$/) { - #ignore empty line - } else { - print "unknown line: $_"; - } -} - - -for ($i=0; $i <= $#keywords; $i++) { - $upperKeywords[$i] = uc $keywords[$i]; -} -sort (ascend @upperKeywords); - - -sort (ascend @keywords); -print "done.\n"; - -#write output -print "writing output files.\nIf your computer doesn’t respond, then press \"Ctrl-Alt-Delete\"\n"; -print "\n"; -&write_keywordfile; -print "\n"; -&write_tokensfile; -print "\n"; -&write_atgtokensfile; -print "\n"; -&write_unittests; -print "finished.\n"; - -sub write_keywordfile -{ - print " ->Generating Keywords class to file '$keywords_outfile'\n"; - open(DAT,">$keywords_outfile") || die("Cannot Open File"); - print DAT "// this file was autogenerated by a tool.\n"; - print DAT "using System;\n"; - print DAT "\n"; - print DAT "namespace " . $properties{'Namespace'} . "\n"; - print DAT "{\n"; - print DAT "\tpublic static class Keywords\n"; - print DAT "\t{\n"; - print DAT "\t\tstatic readonly string[] keywordList = {\n"; - if ($properties{'UpperCaseKeywords'} eq "True") { - for ($i=0; $i <= $#upperKeywords; $i++) { - print DAT "\t\t\t\"$upperKeywords[$i]\""; - if ($i + 1 <= $#upperKeywords) { - print DAT ","; - } - print DAT "\n"; - } - } else { - for ($i=0; $i <= $#keywords; $i++) { - print DAT "\t\t\t\"$keywords[$i]\""; - if ($i + 1 <= $#keywords) { - print DAT ","; - } - print DAT "\n"; - } - } - - print DAT "\t\t};\n"; - print DAT "\t\t\n"; - if ($properties{'UpperCaseKeywords'} eq "True") { - print DAT "\t\tstatic LookupTable keywords = new LookupTable(false);\n"; - } else { - print DAT "\t\tstatic LookupTable keywords = new LookupTable(true);\n"; - } - - print DAT "\t\t\n"; - print DAT "\t\tstatic Keywords()\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\tfor (int i = 0; i < keywordList.Length; ++i) {\n"; - print DAT "\t\t\t\tkeywords[keywordList[i]] = i + Tokens." . ucfirst $keywords[0] . ";\n"; - print DAT "\t\t\t}\n"; - print DAT "\t\t}\n"; - print DAT "\t\t\n"; - print DAT "\t\tpublic static int GetToken(string keyword)\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\treturn keywords[keyword];\n"; - print DAT "\t\t}\n"; - print DAT "\t\t\n"; - print DAT "\t\tpublic static bool IsNonIdentifierKeyword(string word)\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\tint token = GetToken(word);\n"; - print DAT "\t\t\tif (token < 0)\n"; - print DAT "\t\t\t\treturn false;\n"; - print DAT "\t\t\treturn !Tokens.IdentifierTokens[token];\n"; - print DAT "\t\t}\n"; - print DAT "\t}\n"; - print DAT "}\n"; - close(DAT); - print " ->done.\n"; -} - -sub write_token { - $formattedString = sprintf("%-20s", ucfirst $tokenName); - if (($tokenName eq "GetType") or ($tokenName eq "equals") or ($tokenName eq "Equals")) { - print DAT "\t\tnew public const int $formattedString = $tokenValue;\n"; - } else { - print DAT "\t\tpublic const int $formattedString = $tokenValue;\n"; - } - $tokenValue++; - -} - -sub print_list { - local ($j, $k, $max, $index); - - $index = $_[0]; - $max = $#{$setValues->[$index]}; - - for ($j=0; $j <= $max; $j++) { - $_ = $setValues->[$index][$j]; - if (/\"(\w+)\"/) { # Keywords - print DAT ucfirst $1; - } elsif (/\"(\W+)\"/) { # special chars - print DAT $specialCharLookup{$_}; - } elsif (/@(\w+)/) { # @otherList - for ($k=0; $k <= $#sets; $k++) { - if ($sets[$k] eq $1) { - print_list($k); - } - } - } else { - print DAT $_; - } - - if ($j + 1 <= $max) { - print DAT ", "; - } - } -} - - -sub write_tokensfile { - print " ->Generating Tokens class to file '$tokens_outfile'\n"; - open(DAT,">$tokens_outfile") || die("Cannot Open File"); - print DAT "// this file was autogenerated by a tool.\n"; - print DAT "using System;\n"; - print DAT "using System.Collections;\n"; - print DAT "\n"; - print DAT "namespace " . $properties{'Namespace'} . "\n"; - print DAT "{\n"; - print DAT "\tpublic static class Tokens\n"; - print DAT "\t{\n"; - $tokenValue = 0; - - print DAT "\t\t// ----- terminal classes -----\n"; - foreach (@terminals) { - $tokenName = $_; - write_token(); - } - print DAT "\n"; - print DAT "\t\t// ----- special character -----\n"; - foreach (@special_chars) { - $tokenName = $_; - write_token(); - } - print DAT "\n"; - print DAT "\t\t// ----- keywords -----\n"; - foreach (@keywords) { - $tokenName = $_; - write_token(); - } - print DAT "\n"; - - print DAT "\t\tpublic const int MaxToken = " . $tokenValue . ";\n"; - - #write sets. - if ($#sets > 0) { - print DAT "\t\tstatic BitArray NewSet(params int[] values)\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\tBitArray bitArray = new BitArray(MaxToken);\n"; - print DAT "\t\t\tforeach (int val in values) {\n"; - print DAT "\t\t\tbitArray[val] = true;\n"; - print DAT "\t\t\t}\n"; - print DAT "\t\t\treturn bitArray;\n"; - print DAT "\t\t}\n"; - for ($i=0; $i <= $#sets; $i++) { - print DAT "\t\tpublic static BitArray ". $sets[$i] . " = NewSet("; - print_list($i); - print DAT ");\n"; - } - print DAT "\n"; - } - - #write token number --> string function. - print DAT "\t\tstatic string[] tokenList = new string[] {\n"; - print DAT "\t\t\t// ----- terminal classes -----\n"; - foreach (@terminals) { - print DAT "\t\t\t\"<$_>\",\n"; - } - print DAT "\t\t\t// ----- special character -----\n"; - foreach (@special_values) { - print DAT "\t\t\t$_,\n"; - } - print DAT "\t\t\t// ----- keywords -----\n"; - foreach (@keywords) { - print DAT "\t\t\t\"$_\",\n"; - } - print DAT "\t\t};\n"; - - - print DAT "\t\tpublic static string GetTokenString(int token)\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\tif (token >= 0 && token < tokenList.Length) {\n"; - print DAT "\t\t\t\treturn tokenList[token];\n"; - print DAT "\t\t\t}\n"; - print DAT "\t\t\tthrow new System.NotSupportedException(\"Unknown token:\" + token);\n"; - print DAT "\t\t}\n"; - print DAT "\t}\n"; - - - - print DAT "}\n"; - close(DAT); - print " ->done.\n"; -} - -sub write_unittests { - open(DAT,">$unittests_outfile") || die("Cannot Open File"); - print DAT "using System;\n"; - print DAT "using System.IO;\n"; - print DAT "using NUnit.Framework;\n"; - print DAT "using ICSharpCode.NRefactory.Parser;\n"; - print DAT "using ICSharpCode.NRefactory.PrettyPrinter;\n"; - - print DAT "\n"; - print DAT "namespace ICSharpCode.NRefactory.Tests.Lexer\n"; - print DAT "{\n"; - print DAT "\t[TestFixture]\n"; - print DAT "\tpublic sealed class LexerTests\n"; - print DAT "\t{\n"; - print DAT "\t\tILexer GenerateLexer(StringReader sr)\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\treturn ParserFactory.CreateLexer(SupportedLanguage.CSharp, sr);\n"; - print DAT "\t\t}\n\n"; - - for ($i=0; $i <= $#special_values; $i++) { - - print DAT "\t\t[Test]\n"; - print DAT "\t\tpublic void Test" . $special_chars[$i] ."()\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\tILexer lexer = GenerateLexer(new StringReader(" . $special_values[$i] . "));\n"; - print DAT "\t\t\tAssert.AreEqual(Tokens." . $special_chars[$i] . ", lexer.NextToken().kind);\n"; - print DAT "\t\t}\n\n"; - - } - - foreach (@keywords) { - print DAT "\t\t[Test()]\n"; - print DAT "\t\tpublic void Test" . ucfirst $_ ."()\n"; - print DAT "\t\t{\n"; - print DAT "\t\t\tILexer lexer = GenerateLexer(new StringReader(\"" . $_ . "\"));\n"; - print DAT "\t\t\tAssert.AreEqual(Tokens." . ucfirst $_ . ", lexer.NextToken().kind);\n"; - print DAT "\t\t}\n"; - } - - print DAT "\t}\n"; - print DAT "}\n"; - - - close(DAT); -} - -sub write_atgtokensfile { - print " ->Generating ATG TOKENS section and writing it to file '$ATGTokensSection'\n"; - open(DAT,">$ATGTokensSection") || die("Cannot Open File"); - print DAT "/* START AUTOGENERATED TOKENS SECTION */\n"; - print DAT "TOKENS\n"; - - print DAT "\t/* ----- terminal classes ----- */\n"; - print DAT "\t/* EOF is 0 */\n"; - foreach $term (@terminals) { - if ($term eq "EOF") { - } elsif ($term eq "Identifier") { - print DAT "\tident\n"; - } else { - print DAT "\t$term\n"; - } - - } - - print DAT "\n"; - print DAT "\t/* ----- special character ----- */\n"; - foreach (@special_values) { - print DAT "\t$_\n"; - } - print DAT "\n"; - print DAT "\t/* ----- keywords ----- */\n"; - foreach (@keywords) { - print DAT "\t\"$_\"\n"; - } - - print DAT "/* END AUTOGENERATED TOKENS SECTION */\n"; - close(DAT); - print " ->done.\n"; -} -