pimhwang
/
SharpDevelop
mirror of https://github.com/icsharpcode/SharpDevelop.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

replaced BuildKeywords.pl by KeywordGenerator.cs 

git-svn-id: svn://svn.sharpdevelop.net/sharpdevelop/branches/vbnet@5828 1ccf3a8d-04fe-1044-b7c0-cef0b8235c61
pull/1/head
Siegfried Pammer 16 years ago
parent
96488a9045
commit
62b5ddf531
4 changed files with 408 additions and 343 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 398
      src/Libraries/NRefactory/NRefactoryASTGenerator/KeywordGenerator.cs
  2. 8
      src/Libraries/NRefactory/NRefactoryASTGenerator/Main.cs
  3. 5
      src/Libraries/NRefactory/NRefactoryASTGenerator/NRefactoryASTGenerator.csproj
  4. 340
      src/Libraries/NRefactory/Project/Src/Lexer/BuildKeywords.pl

398
src/Libraries/NRefactory/NRefactoryASTGenerator/KeywordGenerator.cs
Unescape View File

@ -0,0 +1,398 @@
// <file>
// <copyright see="prj:///doc/copyright.txt"/>
// <license see="prj:///doc/license.txt"/>
// <owner name="Siegfried Pammer" email="siegfriedpammer@gmail.com" />
// <version>$Revision$</version>
// </file>
using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.IO;
using System.Linq;
using System.Security.Principal;
using System.Text;
using System.Text.RegularExpressions;
using System.Threading.Tasks;
using NRefactoryASTGenerator.Ast;
namespace NRefactoryASTGenerator
{
static class KeywordGenerator
{
static readonly string baseDir = "../../../Project/Src/Lexer/";
static readonly string testBaseDir = "../../../Test/Lexer/";
static readonly string parserBaseDir = "../../../Project/Src/Parser/";
public static void Generate()
{
Generate("CSharp");
Generate("VBNet");
}
static void Generate(string language)
{
try {
Dictionary<string, string> properties = new Dictionary<string, string>();
Dictionary<string, string[]> sets = new Dictionary<string, string[]>();
List<string> keywords = new List<string>();
List<string> terminals = new List<string>();
Dictionary<string, string> specialChars = new Dictionary<string, string>();
ReadFile(properties, sets, keywords, terminals, specialChars, language);
GenerateFiles(properties, sets, keywords, terminals, specialChars, language);
} catch (Exception e) {
Debug.Print(e.ToString());
}
}
static void GenerateFiles(Dictionary<string, string> properties, Dictionary<string, string[]> sets,
List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars,
string language)
{
GenerateKeywords(properties, keywords, language);
GenerateTokens(properties, sets, keywords, terminals, specialChars, language);
GenerateTests(keywords, specialChars, language);
GenerateKeywordSection(keywords, terminals, specialChars, language);
}
static void GenerateKeywordSection(List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars, string language)
{
string sourceDir = Path.Combine(parserBaseDir, language, (language == "CSharp" ? "cs" : language) + ".atg");
StringBuilder builder = new StringBuilder();
builder.AppendLine("/* START AUTOGENERATED TOKENS SECTION */");
builder.AppendLine("TOKENS");
builder.AppendLine("\t/* ----- terminal classes ----- */");
builder.AppendLine("\t/* EOF is 0 */");
foreach (string terminal in terminals) {
if (terminal == "EOF")
continue;
if (terminal == "Identifier") {
builder.AppendLine("\tident");
continue;
}
builder.AppendLine("\t" + terminal);
}
builder.AppendLine();
builder.AppendLine("\t/* ----- special character ----- */");
foreach (string specialChar in specialChars.Values) {
builder.AppendLine("\t" + specialChar);
}
builder.AppendLine();
builder.AppendLine("\t/* ----- keywords ----- */");
foreach (string keyword in keywords) {
builder.AppendLine("\t\"" + keyword + "\"");
}
builder.AppendLine("/* END AUTOGENERATED TOKENS SECTION */");
string[] generatedLines = builder.ToString().Split(new string[] { Environment.NewLine }, StringSplitOptions.None);
string[] lines = File.ReadAllLines(sourceDir);
var newContent = lines
.TakeWhile(l => l != "/* START AUTOGENERATED TOKENS SECTION */")
.Concat(generatedLines)
.Concat(lines.SkipWhile(l => l != "/* END AUTOGENERATED TOKENS SECTION */").Skip(2))
.ToArray();
File.WriteAllLines(sourceDir, newContent);
}
static void GenerateTests(List<string> keywords, Dictionary<string, string> specialChars, string language)
{
string sourceDir = Path.Combine(testBaseDir, language, "LexerTests.cs");
using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) {
writer.WriteLine("using System;");
writer.WriteLine("using System.IO;");
writer.WriteLine("using NUnit.Framework;");
writer.WriteLine("using ICSharpCode.NRefactory.Parser;");
writer.WriteLine("using ICSharpCode.NRefactory.Parser.{0};", language == "VBNet" ? "VB" : language);
writer.WriteLine("using ICSharpCode.NRefactory.PrettyPrinter;\n");
writer.WriteLine();
writer.WriteLine("namespace ICSharpCode.NRefactory.Tests.Lexer.{0}", language == "VBNet" ? "VB" : language);
writer.WriteLine("{");
writer.WriteLine("\t[TestFixture]");
writer.WriteLine("\tpublic sealed class LexerTests");
writer.WriteLine("\t{");
writer.WriteLine("\t\tILexer GenerateLexer(StringReader sr)");
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\treturn ParserFactory.CreateLexer(SupportedLanguage.CSharp, sr);");
writer.WriteLine("\t\t}");
for (int i = 0; i < specialChars.Values.Count; i++) {
writer.WriteLine();
writer.WriteLine("\t\t[Test]");
writer.WriteLine("\t\tpublic void Test{0}()", specialChars.Keys.ElementAt(i));
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\tILexer lexer = GenerateLexer(new StringReader({0}));", specialChars.Values.ElementAt(i));
writer.WriteLine("\t\t\tAssert.AreEqual(Tokens.{0}, lexer.NextToken().Kind);", specialChars.Keys.ElementAt(i));
writer.WriteLine("\t\t}");
}
foreach (string keyword in keywords) {
writer.WriteLine();
writer.WriteLine("\t\t[Test]");
writer.WriteLine("\t\tpublic void Test{0}()", UpperCaseFirst(keyword));
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\tILexer lexer = GenerateLexer(new StringReader(\"{0}\"));", keyword);
writer.WriteLine("\t\t\tAssert.AreEqual(Tokens.{0}, lexer.NextToken().Kind);", UpperCaseFirst(keyword));
writer.WriteLine("\t\t}");
}
writer.WriteLine("\t}");
writer.WriteLine("}");
}
}
static void GenerateTokens(Dictionary<string, string> properties, Dictionary<string, string[]> sets, List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars, string language)
{
string sourceDir = Path.Combine(baseDir, language, "Tokens.cs");
using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) {
writer.WriteLine("// this file was autogenerated by a tool.");
writer.WriteLine("using System;");
writer.WriteLine("using System.Collections;");
writer.WriteLine();
writer.WriteLine("namespace {0}", properties["Namespace"]);
writer.WriteLine("{");
writer.WriteLine("\tpublic static class Tokens");
writer.WriteLine("\t{");
writer.WriteLine("\t\t// ----- terminal classes -----");
int tokenValue = 0;
foreach (string terminal in terminals)
writer.WriteToken(terminal, ref tokenValue);
writer.WriteLine();
writer.WriteLine("\t\t// ----- special character -----");
foreach (string specialChar in specialChars.Keys)
writer.WriteToken(specialChar, ref tokenValue);
writer.WriteLine();
writer.WriteLine("\t\t// ----- keywords -----");
foreach (string keyword in keywords)
writer.WriteToken(keyword, ref tokenValue);
writer.WriteLine();
writer.WriteLine("\t\tpublic const int MaxToken = {0};", tokenValue);
if (sets.Any()) {
writer.WriteLine("\t\tstatic BitArray NewSet(params int[] values)");
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\tBitArray bitArray = new BitArray(MaxToken);");
writer.WriteLine("\t\t\tforeach (int val in values) {");
writer.WriteLine("\t\t\tbitArray[val] = true;");
writer.WriteLine("\t\t\t}");
writer.WriteLine("\t\t\treturn bitArray;");
writer.WriteLine("\t\t}");
foreach (var pair in sets) {
StringBuilder builder = new StringBuilder();
PrintList(pair.Value, builder, sets, specialChars);
writer.WriteLine("\t\tpublic static BitArray {0} = NewSet({1});", pair.Key, builder.ToString());
}
writer.WriteLine();
}
// write token number --> string function.
writer.WriteLine("\t\tstatic string[] tokenList = new string[] {");
writer.WriteLine("\t\t\t// ----- terminal classes -----");
foreach (string terminal in terminals)
writer.WriteLine("\t\t\t\"<{0}>\",", terminal);
writer.WriteLine("\t\t\t// ----- special character -----");
foreach (string specialChar in specialChars.Values)
writer.WriteLine("\t\t\t{0},", specialChar);
writer.WriteLine("\t\t\t// ----- keywords -----");
foreach (string keyword in keywords)
writer.WriteLine("\t\t\t\"{0}\",", keyword);
writer.WriteLine("\t\t};");
writer.WriteLine("\t\tpublic static string GetTokenString(int token)");
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\tif (token >= 0 && token < tokenList.Length) {");
writer.WriteLine("\t\t\t\treturn tokenList[token];");
writer.WriteLine("\t\t\t}");
writer.WriteLine("\t\t\tthrow new System.NotSupportedException(\"Unknown token:\" + token);");
writer.WriteLine("\t\t}");
writer.WriteLine("\t}");
writer.WriteLine("}");
}
}
static void PrintList(string[] value, StringBuilder builder, Dictionary<string, string[]> sets, Dictionary<string, string> specialChars)
{
for (int i = 0; i < value.Length; i++) {
string item = value[i];
if (Regex.IsMatch(item, "\\\"(\\w+)\\\"")) // keywords
builder.Append(UpperCaseFirst(item.Trim('"', ' ', '\t')));
else if (Regex.IsMatch(item, "\\\"(\\W+)\\\"")) // special chars
builder.Append(specialChars.Keys.ElementAt(specialChars.Values.FindIndex(it => item == it)));
else if (Regex.IsMatch(item, "@(\\w+)")) // other list
PrintList(sets[item.Substring(1)], builder, sets, specialChars);
else
builder.Append(item);
if (i + 1 < value.Length)
builder.Append(", ");
}
}
static int FindIndex<T>(this IEnumerable<T> items, Func<T, bool> f)
{
int index = -1;
foreach (T item in items) {
index++;
if (f(item))
return index;
}
return -1;
}
static void GenerateKeywords(Dictionary<string, string> properties, List<string> keywords, string language)
{
string sourceDir = Path.Combine(baseDir, language, "Keywords.cs");
using (StreamWriter writer = new StreamWriter(new FileStream(sourceDir, FileMode.Create))) {
writer.WriteLine("// this file was autogenerated by a tool.");
writer.WriteLine("using System;");
writer.WriteLine();
writer.WriteLine("namespace {0}", properties["Namespace"]);
writer.WriteLine("{");
writer.WriteLine("\tpublic static class Keywords");
writer.WriteLine("\t{");
writer.WriteLine("\t\tstatic readonly string[] keywordList = {");
for (int i = 0; i < keywords.Count; i++) {
writer.Write("\t\t\t\"{0}\"", properties["UpperCaseKeywords"] == "True" ? keywords[i].ToUpperInvariant() : keywords[i]);
if (i + 1 < keywords.Count)
writer.Write(",");
writer.WriteLine();
}
writer.WriteLine("\t\t};");
writer.WriteLine("\t\t");
writer.WriteLine("\t\tstatic LookupTable keywords = new LookupTable({0});", properties["UpperCaseKeywords"] == "True" ? "false" : "true");
writer.WriteLine("\t\t");
writer.WriteLine("\t\tstatic Keywords()");
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\tfor (int i = 0; i < keywordList.Length; ++i) {");
writer.WriteLine("\t\t\t\tkeywords[keywordList[i]] = i + Tokens.{0};", UpperCaseFirst(keywords[0]));
writer.WriteLine("\t\t\t}");
writer.WriteLine("\t\t}");
writer.WriteLine("\t\t");
writer.WriteLine("\t\tpublic static int GetToken(string keyword)");
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\treturn keywords[keyword];");
writer.WriteLine("\t\t}");
writer.WriteLine("\t\t");
writer.WriteLine("\t\tpublic static bool IsNonIdentifierKeyword(string word)");
writer.WriteLine("\t\t{");
writer.WriteLine("\t\t\tint token = GetToken(word);");
writer.WriteLine("\t\t\tif (token < 0)");
writer.WriteLine("\t\t\t\treturn false;");
writer.WriteLine("\t\t\treturn !Tokens.IdentifierTokens[token];");
writer.WriteLine("\t\t}");
writer.WriteLine("\t}");
writer.WriteLine("}");
writer.Close();
}
}
#region input
static void ReadFile(Dictionary<string, string> properties, Dictionary<string, string[]> sets,
List<string> keywords, List<string> terminals, Dictionary<string, string> specialChars,
string language)
{
string sourceDir = Path.Combine(baseDir, language, "KeywordList.txt");
using (StreamReader reader = new StreamReader(new FileStream(sourceDir, FileMode.Open))) {
string line = reader.ReadLine();
while (line != null) {
ReadProperty(properties, line);
ReadKeyword(keywords, line);
ReadSet(sets, line);
ReadTerminalSymbol(terminals, line);
ReadSpecialChar(specialChars, line);
line = reader.ReadLine();
}
reader.Close();
}
}
static void ReadProperty(Dictionary<string, string> properties, string line)
{
// properties form: $PROPERTY = "VALUE"
Match match = Regex.Match(line, @"^\s*\$(\w+)\s*=\s*(\S+)\s*$");
if (match.Success) {
properties.Add(match.Groups[1].Value, match.Groups[2].Value);
}
}
static void ReadKeyword(List<string> keywords, string line)
{
// special keywords form: "VALUE"
Match match = Regex.Match(line, "^\\s*\\\"(\\S+)\\s*\\\"\\s*$");
if (match.Success) {
keywords.Add(match.Groups[1].Value);
}
}
static void ReadSet(Dictionary<string, string[]> sets, string line)
{
// sets form: NAME(comma separated list)
Match match = Regex.Match(line, @"^\s*(\w+)\s*\((.*)\)\s*$");
if (match.Success) {
sets.Add(
match.Groups[1].Value,
match.Groups[2].Value.Split(new[] {", "}, StringSplitOptions.None)
);
}
}
static void ReadTerminalSymbol(List<string> terminals, string line)
{
// special terminal classes form: name
Match match = Regex.Match(line, @"^\s*(\w+)\s*$");
if (match.Success) {
terminals.Add(match.Groups[1].Value);
}
}
static void ReadSpecialChar(Dictionary<string, string> specialChars, string line)
{
// special characters form: name = "VALUE"
Match match = Regex.Match(line, @"^\s*(\w+)\s*=\s*(\S+)\s*$");
if (match.Success) {
specialChars.Add(match.Groups[1].Value, match.Groups[2].Value);
}
}
#endregion
#region helpers
static string UpperCaseFirst(string keyword)
{
return char.ToUpperInvariant(keyword[0]) + keyword.Substring(1);
}
static void WriteToken(this StreamWriter writer, string tokenName, ref int tokenValue)
{
string formattedName = UpperCaseFirst(tokenName).PadRight(20);
if (tokenName == "GetType" || tokenName.ToLowerInvariant() == "equals")
writer.WriteLine("\t\tnew public const int {0} = {1};", formattedName, tokenValue);
else
writer.WriteLine("\t\tpublic const int {0} = {1};", formattedName, tokenValue);
tokenValue++;
}
static void WriteList(this StreamWriter writer, Tuple<string, string[]> data)
{
}
#endregion
}
}

8
src/Libraries/NRefactory/NRefactoryASTGenerator/Main.cs
Unescape View File

@ -159,6 +159,12 @@ namespace NRefactoryASTGenerator
File.WriteAllText(visitorsDir + "NotImplementedAstVisitor.cs", NormalizeNewLines(writer)); File.WriteAllText(visitorsDir + "NotImplementedAstVisitor.cs", NormalizeNewLines(writer));
} }
Debug.WriteLine("AST Generator done!"); Debug.WriteLine("AST Generator done!");
Debug.WriteLine("start keyword list generation...");
KeywordGenerator.Generate();
Debug.WriteLine("keyword list generation done!");
} }
static string NormalizeNewLines(StringWriter writer) static string NormalizeNewLines(StringWriter writer)
@ -585,4 +591,4 @@ namespace NRefactoryASTGenerator
return td; return td;
} }
} }
} }

5
src/Libraries/NRefactory/NRefactoryASTGenerator/NRefactoryASTGenerator.csproj
Unescape View File

@ -1,5 +1,5 @@
<?xml version="1.0" encoding="utf-8"?> <?xml version="1.0" encoding="utf-8"?>
<Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="3.5"> <Project DefaultTargets="Build" xmlns="http://schemas.microsoft.com/developer/msbuild/2003" ToolsVersion="4.0">
<PropertyGroup> <PropertyGroup>
<OutputType>Exe</OutputType> <OutputType>Exe</OutputType>
<RootNamespace>NRefactoryASTGenerator</RootNamespace> <RootNamespace>NRefactoryASTGenerator</RootNamespace>
@ -17,7 +17,7 @@
<WarningLevel>4</WarningLevel> <WarningLevel>4</WarningLevel>
<NoWarn>0169</NoWarn> <NoWarn>0169</NoWarn>
<TreatWarningsAsErrors>false</TreatWarningsAsErrors> <TreatWarningsAsErrors>false</TreatWarningsAsErrors>
<TargetFrameworkVersion>v3.5</TargetFrameworkVersion> <TargetFrameworkVersion>v4.0</TargetFrameworkVersion>
</PropertyGroup> </PropertyGroup>
<PropertyGroup Condition=" '$(Configuration)' == 'Debug' "> <PropertyGroup Condition=" '$(Configuration)' == 'Debug' ">
<OutputPath>bin\Debug\</OutputPath> <OutputPath>bin\Debug\</OutputPath>
@ -44,6 +44,7 @@
<Reference Include="System.Xml" /> <Reference Include="System.Xml" />
</ItemGroup> </ItemGroup>
<ItemGroup> <ItemGroup>
<Compile Include="KeywordGenerator.cs" />
<Compile Include="Main.cs" /> <Compile Include="Main.cs" />
<Compile Include="AssemblyInfo.cs" /> <Compile Include="AssemblyInfo.cs" />
<Compile Include="AST\Expressions.cs" /> <Compile Include="AST\Expressions.cs" />

340
src/Libraries/NRefactory/Project/Src/Lexer/BuildKeywords.pl
Unescape View File

@ -1,340 +0,0 @@
#!/bin/perl
# File names
$keyword_file = "KeywordList.txt";
$keywords_outfile = "Keywords.cs";
$tokens_outfile = "Tokens.cs";
$unittests_outfile = "LexerTests.cs";
$ATGTokensSection = "ATGTokensSection.gen";
#read infile
print "\n";
print "Reading keyword definition from '$keyword_file'.\n";
open(DAT, $keyword_file) || die("Could not open file '$keyword_file': $!");
@raw_data=<DAT>;
close(DAT);
print "done.\n";
#analyse infile
print "starting analysis... this could take a few seconds.\n";
foreach (@raw_data) {
if ($_=~/^\s*\$(\w+)\s*=\s*(\S+)\s*$/) {
#properties form: $PROPERTY = "VALUE"
$properties{$1} = $2;
} elsif ($_=~/^\s*(\w+)\s*=\s*(\S+)\s*$/) {
#special characters form: name = "VALUE"
$specialCharLookup{$2} = $1;
$special_chars[$#special_chars + 1] = $1;
$special_values[$#special_values + 1] = $2;
} elsif ($_=~/^\s*\"(\S+)\s*\"\s*$/) {
#special keywords form: "VALUE"
$keywords[$#keywords + 1] = $1;
} elsif ($_=~/^\s*(\w+)\s*\((.*)\)\s*$/) {
$sets[$#sets + 1] = $1;
#Split set values (comma separated list)
$_ = $2;
@vals = split/\s*,\s*/;
push @$setValues, [@vals];
} elsif ($_=~/^\s*(\w+)\s*$/) {
#special terminal classes form: name
$terminals[$#terminals + 1] = $1
} elsif ($_=~/^\s*(#.*)?$/) {
#ignore empty line
} else {
print "unknown line: $_";
}
}
for ($i=0; $i <= $#keywords; $i++) {
$upperKeywords[$i] = uc $keywords[$i];
}
sort (ascend @upperKeywords);
sort (ascend @keywords);
print "done.\n";
#write output
print "writing output files.\nIf your computer doesn’t respond, then press \"Ctrl-Alt-Delete\"\n";
print "\n";
&write_keywordfile;
print "\n";
&write_tokensfile;
print "\n";
&write_atgtokensfile;
print "\n";
&write_unittests;
print "finished.\n";
sub write_keywordfile
{
print " ->Generating Keywords class to file '$keywords_outfile'\n";
open(DAT,">$keywords_outfile") || die("Cannot Open File");
print DAT "// this file was autogenerated by a tool.\n";
print DAT "using System;\n";
print DAT "\n";
print DAT "namespace " . $properties{'Namespace'} . "\n";
print DAT "{\n";
print DAT "\tpublic static class Keywords\n";
print DAT "\t{\n";
print DAT "\t\tstatic readonly string[] keywordList = {\n";
if ($properties{'UpperCaseKeywords'} eq "True") {
for ($i=0; $i <= $#upperKeywords; $i++) {
print DAT "\t\t\t\"$upperKeywords[$i]\"";
if ($i + 1 <= $#upperKeywords) {
print DAT ",";
}
print DAT "\n";
}
} else {
for ($i=0; $i <= $#keywords; $i++) {
print DAT "\t\t\t\"$keywords[$i]\"";
if ($i + 1 <= $#keywords) {
print DAT ",";
}
print DAT "\n";
}
}
print DAT "\t\t};\n";
print DAT "\t\t\n";
if ($properties{'UpperCaseKeywords'} eq "True") {
print DAT "\t\tstatic LookupTable keywords = new LookupTable(false);\n";
} else {
print DAT "\t\tstatic LookupTable keywords = new LookupTable(true);\n";
}
print DAT "\t\t\n";
print DAT "\t\tstatic Keywords()\n";
print DAT "\t\t{\n";
print DAT "\t\t\tfor (int i = 0; i < keywordList.Length; ++i) {\n";
print DAT "\t\t\t\tkeywords[keywordList[i]] = i + Tokens." . ucfirst $keywords[0] . ";\n";
print DAT "\t\t\t}\n";
print DAT "\t\t}\n";
print DAT "\t\t\n";
print DAT "\t\tpublic static int GetToken(string keyword)\n";
print DAT "\t\t{\n";
print DAT "\t\t\treturn keywords[keyword];\n";
print DAT "\t\t}\n";
print DAT "\t\t\n";
print DAT "\t\tpublic static bool IsNonIdentifierKeyword(string word)\n";
print DAT "\t\t{\n";
print DAT "\t\t\tint token = GetToken(word);\n";
print DAT "\t\t\tif (token < 0)\n";
print DAT "\t\t\t\treturn false;\n";
print DAT "\t\t\treturn !Tokens.IdentifierTokens[token];\n";
print DAT "\t\t}\n";
print DAT "\t}\n";
print DAT "}\n";
close(DAT);
print " ->done.\n";
}
sub write_token {
$formattedString = sprintf("%-20s", ucfirst $tokenName);
if (($tokenName eq "GetType") or ($tokenName eq "equals") or ($tokenName eq "Equals")) {
print DAT "\t\tnew public const int $formattedString = $tokenValue;\n";
} else {
print DAT "\t\tpublic const int $formattedString = $tokenValue;\n";
}
$tokenValue++;
}
sub print_list {
local ($j, $k, $max, $index);
$index = $_[0];
$max = $#{$setValues->[$index]};
for ($j=0; $j <= $max; $j++) {
$_ = $setValues->[$index][$j];
if (/\"(\w+)\"/) { # Keywords
print DAT ucfirst $1;
} elsif (/\"(\W+)\"/) { # special chars
print DAT $specialCharLookup{$_};
} elsif (/@(\w+)/) { # @otherList
for ($k=0; $k <= $#sets; $k++) {
if ($sets[$k] eq $1) {
print_list($k);
}
}
} else {
print DAT $_;
}
if ($j + 1 <= $max) {
print DAT ", ";
}
}
}
sub write_tokensfile {
print " ->Generating Tokens class to file '$tokens_outfile'\n";
open(DAT,">$tokens_outfile") || die("Cannot Open File");
print DAT "// this file was autogenerated by a tool.\n";
print DAT "using System;\n";
print DAT "using System.Collections;\n";
print DAT "\n";
print DAT "namespace " . $properties{'Namespace'} . "\n";
print DAT "{\n";
print DAT "\tpublic static class Tokens\n";
print DAT "\t{\n";
$tokenValue = 0;
print DAT "\t\t// ----- terminal classes -----\n";
foreach (@terminals) {
$tokenName = $_;
write_token();
}
print DAT "\n";
print DAT "\t\t// ----- special character -----\n";
foreach (@special_chars) {
$tokenName = $_;
write_token();
}
print DAT "\n";
print DAT "\t\t// ----- keywords -----\n";
foreach (@keywords) {
$tokenName = $_;
write_token();
}
print DAT "\n";
print DAT "\t\tpublic const int MaxToken = " . $tokenValue . ";\n";
#write sets.
if ($#sets > 0) {
print DAT "\t\tstatic BitArray NewSet(params int[] values)\n";
print DAT "\t\t{\n";
print DAT "\t\t\tBitArray bitArray = new BitArray(MaxToken);\n";
print DAT "\t\t\tforeach (int val in values) {\n";
print DAT "\t\t\tbitArray[val] = true;\n";
print DAT "\t\t\t}\n";
print DAT "\t\t\treturn bitArray;\n";
print DAT "\t\t}\n";
for ($i=0; $i <= $#sets; $i++) {
print DAT "\t\tpublic static BitArray ". $sets[$i] . " = NewSet(";
print_list($i);
print DAT ");\n";
}
print DAT "\n";
}
#write token number --> string function.
print DAT "\t\tstatic string[] tokenList = new string[] {\n";
print DAT "\t\t\t// ----- terminal classes -----\n";
foreach (@terminals) {
print DAT "\t\t\t\"<$_>\",\n";
}
print DAT "\t\t\t// ----- special character -----\n";
foreach (@special_values) {
print DAT "\t\t\t$_,\n";
}
print DAT "\t\t\t// ----- keywords -----\n";
foreach (@keywords) {
print DAT "\t\t\t\"$_\",\n";
}
print DAT "\t\t};\n";
print DAT "\t\tpublic static string GetTokenString(int token)\n";
print DAT "\t\t{\n";
print DAT "\t\t\tif (token >= 0 && token < tokenList.Length) {\n";
print DAT "\t\t\t\treturn tokenList[token];\n";
print DAT "\t\t\t}\n";
print DAT "\t\t\tthrow new System.NotSupportedException(\"Unknown token:\" + token);\n";
print DAT "\t\t}\n";
print DAT "\t}\n";
print DAT "}\n";
close(DAT);
print " ->done.\n";
}
sub write_unittests {
open(DAT,">$unittests_outfile") || die("Cannot Open File");
print DAT "using System;\n";
print DAT "using System.IO;\n";
print DAT "using NUnit.Framework;\n";
print DAT "using ICSharpCode.NRefactory.Parser;\n";
print DAT "using ICSharpCode.NRefactory.PrettyPrinter;\n";
print DAT "\n";
print DAT "namespace ICSharpCode.NRefactory.Tests.Lexer\n";
print DAT "{\n";
print DAT "\t[TestFixture]\n";
print DAT "\tpublic sealed class LexerTests\n";
print DAT "\t{\n";
print DAT "\t\tILexer GenerateLexer(StringReader sr)\n";
print DAT "\t\t{\n";
print DAT "\t\t\treturn ParserFactory.CreateLexer(SupportedLanguage.CSharp, sr);\n";
print DAT "\t\t}\n\n";
for ($i=0; $i <= $#special_values; $i++) {
print DAT "\t\t[Test]\n";
print DAT "\t\tpublic void Test" . $special_chars[$i] ."()\n";
print DAT "\t\t{\n";
print DAT "\t\t\tILexer lexer = GenerateLexer(new StringReader(" . $special_values[$i] . "));\n";
print DAT "\t\t\tAssert.AreEqual(Tokens." . $special_chars[$i] . ", lexer.NextToken().kind);\n";
print DAT "\t\t}\n\n";
}
foreach (@keywords) {
print DAT "\t\t[Test()]\n";
print DAT "\t\tpublic void Test" . ucfirst $_ ."()\n";
print DAT "\t\t{\n";
print DAT "\t\t\tILexer lexer = GenerateLexer(new StringReader(\"" . $_ . "\"));\n";
print DAT "\t\t\tAssert.AreEqual(Tokens." . ucfirst $_ . ", lexer.NextToken().kind);\n";
print DAT "\t\t}\n";
}
print DAT "\t}\n";
print DAT "}\n";
close(DAT);
}
sub write_atgtokensfile {
print " ->Generating ATG TOKENS section and writing it to file '$ATGTokensSection'\n";
open(DAT,">$ATGTokensSection") || die("Cannot Open File");
print DAT "/* START AUTOGENERATED TOKENS SECTION */\n";
print DAT "TOKENS\n";
print DAT "\t/* ----- terminal classes ----- */\n";
print DAT "\t/* EOF is 0 */\n";
foreach $term (@terminals) {
if ($term eq "EOF") {
} elsif ($term eq "Identifier") {
print DAT "\tident\n";
} else {
print DAT "\t$term\n";
}
}
print DAT "\n";
print DAT "\t/* ----- special character ----- */\n";
foreach (@special_values) {
print DAT "\t$_\n";
}
print DAT "\n";
print DAT "\t/* ----- keywords ----- */\n";
foreach (@keywords) {
print DAT "\t\"$_\"\n";
}
print DAT "/* END AUTOGENERATED TOKENS SECTION */\n";
close(DAT);
print " ->done.\n";
}
Write Preview
Loading…
Cancel
Save