@ -16,6 +16,7 @@ using System.Text;
@@ -16,6 +16,7 @@ using System.Text;
using System.Collections.Generic ;
using System.Globalization ;
using System.Diagnostics ;
using System.Collections ;
namespace Mono.CSharp
{
@ -66,32 +67,62 @@ namespace Mono.CSharp
@@ -66,32 +67,62 @@ namespace Mono.CSharp
}
//
// This class has to be used in the parser only, it reuses token
// details after each parse
// This class has to be used by parser only, it reuses token
// details after each file parse completion
//
public class LocatedToken
{
int row , column ;
string value ;
public int row , column ;
public string value ;
public SourceFile file ;
static LocatedToken [ ] buffer = new LocatedToken [ 0 ] ;
static int pos ;
private LocatedToken ( )
public LocatedToken ( )
{
}
public static LocatedToken Create ( int row , int column )
public LocatedToken ( string value , Location loc )
{
return Create ( null , row , column ) ;
this . value = value ;
file = loc . SourceFile ;
row = loc . Row ;
column = loc . Column ;
}
public static LocatedToken Create ( string value , Location loc )
public override string ToString ( )
{
return Create ( value , loc . Row , loc . C olumn) ;
return string . Format ( "Token '{0}' at {1},{2}" , Value , row , c olumn) ;
}
public static LocatedToken Create ( string value , int row , int column )
public Location Location {
get { return new Location ( file , row , column ) ; }
}
public string Value {
get { return value ; }
}
}
public class LocatedTokenBuffer
{
readonly LocatedToken [ ] buffer ;
public int pos ;
public LocatedTokenBuffer ( )
{
buffer = new LocatedToken [ 0 ] ;
}
public LocatedTokenBuffer ( LocatedToken [ ] buffer )
{
this . buffer = buffer ? ? new LocatedToken [ 0 ] ;
}
public LocatedToken Create ( SourceFile file , int row , int column )
{
return Create ( null , file , row , column ) ;
}
public LocatedToken Create ( string value , SourceFile file , int row , int column )
{
//
// TODO: I am not very happy about the logic but it's the best
@ -105,15 +136,16 @@ namespace Mono.CSharp
@@ -105,15 +136,16 @@ namespace Mono.CSharp
if ( pos > = buffer . Length ) {
entry = new LocatedToken ( ) ;
} else {
entry = buffer [ pos ] ;
entry = buffer [ pos ] ;
if ( entry = = null ) {
entry = new LocatedToken ( ) ;
buffer [ pos ] = entry ;
buffer [ pos ] = entry ;
}
+ + pos ;
}
entry . value = value ;
entry . file = file ;
entry . row = row ;
entry . column = column ;
return entry ;
@ -123,31 +155,9 @@ namespace Mono.CSharp
@@ -123,31 +155,9 @@ namespace Mono.CSharp
// Used for token not required by expression evaluator
//
[Conditional ("FULL_AST")]
public static void CreateOptional ( int row , int col , ref object token )
public void CreateOptional ( SourceFile file , int row , int col , ref object token )
{
token = Create ( row , col ) ;
}
public static void Initialize ( )
{
#if !FULL_AST
if ( buffer . Length = = 0 )
buffer = new LocatedToken [ 1 5 0 0 0 ] ;
#endif
pos = 0 ;
}
public override string ToString ( )
{
return string . Format ( "Token '{0}' at {1},{2}" , Value , row , column ) ;
}
public Location Location {
get { return new Location ( row , column ) ; }
}
public string Value {
get { return value ; }
token = Create ( file , row , col ) ;
}
}
@ -174,13 +184,7 @@ namespace Mono.CSharp
@@ -174,13 +184,7 @@ namespace Mono.CSharp
readonly SeekableStreamReader reader ;
readonly CompilationSourceFile source_file ;
public CompilationSourceFile SourceFile {
get {
return source_file ;
}
}
public CompilationSourceFile SourceFile { get { return source_file ; } }
readonly CompilerContext context ;
SourceFile current_source ;
@ -199,6 +203,7 @@ namespace Mono.CSharp
@@ -199,6 +203,7 @@ namespace Mono.CSharp
List < Location > escaped_identifiers ;
int parsing_generic_less_than ;
readonly bool doc_processing ;
readonly LocatedTokenBuffer ltb ;
//
// Used mainly for parser optimizations. Some expressions for instance
@ -362,27 +367,15 @@ namespace Mono.CSharp
@@ -362,27 +367,15 @@ namespace Mono.CSharp
//
Stack < int > ifstack ;
const int max_id_size = 5 1 2 ;
const int max_number_size = 5 1 2 ;
#if FULL_AST
readonly char [ ] id_builder = new char [ max_id_size ] ;
Dictionary < char [ ] , string > [ ] identifiers = new Dictionary < char [ ] , string > [ max_id_size + 1 ] ;
public const int MaxIdentifierLength = 5 1 2 ;
public const int MaxNumberLength = 5 1 2 ;
char [ ] number_builder = new char [ max_number_size ] ;
readonly char [ ] id_builder ;
readonly Dictionary < char [ ] , string > [ ] identifiers ;
readonly char [ ] number_builder ;
int number_pos ;
char [ ] value_builder = new char [ 2 5 6 ] ;
#else
static readonly char [ ] id_builder = new char [ max_id_size ] ;
static Dictionary < char [ ] , string > [ ] identifiers = new Dictionary < char [ ] , string > [ max_id_size + 1 ] ;
static char [ ] number_builder = new char [ max_number_size ] ;
static int number_pos ;
static char [ ] value_builder = new char [ 2 5 6 ] ;
#endif
char [ ] value_builder = new char [ 6 4 ] ;
public int Line {
get {
@ -437,11 +430,15 @@ namespace Mono.CSharp
@@ -437,11 +430,15 @@ namespace Mono.CSharp
}
}
public Tokenizer ( SeekableStreamReader input , CompilationSourceFile file )
public Tokenizer ( SeekableStreamReader input , CompilationSourceFile file , ParserSession session )
{
this . source_file = file ;
this . context = file . Compiler ;
this . current_source = file . SourceFile ;
this . identifiers = session . Identifiers ;
this . id_builder = session . IDBuilder ;
this . number_builder = session . NumberBuilder ;
this . ltb = new LocatedTokenBuffer ( session . LocatedTokens ) ;
reader = input ;
@ -451,8 +448,6 @@ namespace Mono.CSharp
@@ -451,8 +448,6 @@ namespace Mono.CSharp
doc_processing = context . Settings . DocumentationFile ! = null ;
tab_size = context . Settings . TabSize ;
Mono . CSharp . Location . Push ( current_source ) ;
}
public void PushPosition ( )
@ -836,11 +831,13 @@ namespace Mono.CSharp
@@ -836,11 +831,13 @@ namespace Mono.CSharp
PushPosition ( ) ;
xtoken ( ) ;
if ( xtoken ( ) ! = Token . ARROW )
res = - 1 ;
goto default ;
PopPosition ( ) ;
break ;
default :
// peek_token could overwrite id_buffer
id_builder [ 0 ] = 'a' ; id_builder [ 1 ] = 's' ; id_builder [ 2 ] = 'y' ; id_builder [ 3 ] = 'n' ; id_builder [ 4 ] = 'c' ;
res = - 1 ;
break ;
}
@ -898,7 +895,7 @@ namespace Mono.CSharp
@@ -898,7 +895,7 @@ namespace Mono.CSharp
public Location Location {
get {
return new Location ( ref_line , col ) ;
return new Location ( current_source , ref_line , col ) ;
}
}
@ -1111,9 +1108,14 @@ namespace Mono.CSharp
@@ -1111,9 +1108,14 @@ namespace Mono.CSharp
start :
int the_token = token ( ) ;
if ( the_token = = Token . OPEN_BRACKET ) {
do {
while ( true ) {
the_token = token ( ) ;
} while ( the_token ! = Token . CLOSE_BRACKET ) ;
if ( the_token = = Token . EOF )
return true ;
if ( the_token = = Token . CLOSE_BRACKET )
break ;
}
the_token = token ( ) ;
} else if ( the_token = = Token . IN | | the_token = = Token . OUT ) {
the_token = token ( ) ;
@ -1326,7 +1328,7 @@ namespace Mono.CSharp
@@ -1326,7 +1328,7 @@ namespace Mono.CSharp
bool seen_digits = false ;
if ( c ! = - 1 ) {
if ( number_pos = = max_number_size )
if ( number_pos = = MaxNumberLength )
Error_NumericConstantTooLong ( ) ;
number_builder [ number_pos + + ] = ( char ) c ;
}
@ -1337,7 +1339,7 @@ namespace Mono.CSharp
@@ -1337,7 +1339,7 @@ namespace Mono.CSharp
//
while ( ( d = peek_char2 ( ) ) ! = - 1 ) {
if ( d > = '0' & & d < = '9' ) {
if ( number_pos = = max_number_size )
if ( number_pos = = MaxNumberLength )
Error_NumericConstantTooLong ( ) ;
number_builder [ number_pos + + ] = ( char ) d ;
get_char ( ) ;
@ -1602,23 +1604,23 @@ namespace Mono.CSharp
@@ -1602,23 +1604,23 @@ namespace Mono.CSharp
if ( c = = 'e' | | c = = 'E' ) {
is_real = true ;
if ( number_pos = = max_number_size )
if ( number_pos = = MaxNumberLength )
Error_NumericConstantTooLong ( ) ;
number_builder [ number_pos + + ] = ( char ) c ;
c = get_char ( ) ;
if ( c = = '+' ) {
if ( number_pos = = max_number_size )
if ( number_pos = = MaxNumberLength )
Error_NumericConstantTooLong ( ) ;
number_builder [ number_pos + + ] = '+' ;
c = - 1 ;
} else if ( c = = '-' ) {
if ( number_pos = = max_number_size )
if ( number_pos = = MaxNumberLength )
Error_NumericConstantTooLong ( ) ;
number_builder [ number_pos + + ] = '-' ;
c = - 1 ;
} else {
if ( number_pos = = max_number_size )
if ( number_pos = = MaxNumberLength )
Error_NumericConstantTooLong ( ) ;
number_builder [ number_pos + + ] = '+' ;
}
@ -1967,7 +1969,7 @@ namespace Mono.CSharp
@@ -1967,7 +1969,7 @@ namespace Mono.CSharp
}
if ( pos ! = 0 ) {
if ( pos > max_id_size )
if ( pos > MaxIdentifierLength )
arg = new string ( value_builder , 0 , pos ) ;
else
arg = InternIdentifier ( value_builder , pos ) ;
@ -2002,7 +2004,6 @@ namespace Mono.CSharp
@@ -2002,7 +2004,6 @@ namespace Mono.CSharp
}
//ref_line = line;
Location . Push ( current_source ) ;
return true ;
}
@ -2089,7 +2090,6 @@ namespace Mono.CSharp
@@ -2089,7 +2090,6 @@ namespace Mono.CSharp
if ( new_file_name ! = null ) {
current_source = context . LookupFile ( source_file , new_file_name ) ;
source_file . AddIncludeFile ( current_source ) ;
Location . Push ( current_source ) ;
}
if ( ! hidden_block_start . IsNull ) {
@ -2187,6 +2187,8 @@ namespace Mono.CSharp
@@ -2187,6 +2187,8 @@ namespace Mono.CSharp
//
// The syntax is ` "foo.txt" "{guid}" "hash"'
//
// guid is predefined hash algorithm guid {406ea660-64cf-4c82-b6f0-42d48172a799} for md5
//
int c = get_char ( ) ;
if ( c ! = '"' )
@ -2249,6 +2251,7 @@ namespace Mono.CSharp
@@ -2249,6 +2251,7 @@ namespace Mono.CSharp
// Any length of checksum
List < byte > checksum_bytes = new List < byte > ( 1 6 ) ;
var checksum_location = Location ;
c = peek_char ( ) ;
while ( c ! = '"' & & c ! = - 1 ) {
checksum_bytes . Add ( read_hex ( out error ) ) ;
@ -2264,14 +2267,23 @@ namespace Mono.CSharp
@@ -2264,14 +2267,23 @@ namespace Mono.CSharp
return false ;
}
file . SetChecksum ( guid_bytes , checksum_bytes . ToArray ( ) ) ;
current_source . AutoGenerated = true ;
if ( context . Settings . GenerateDebugInfo ) {
var chsum = checksum_bytes . ToArray ( ) ;
if ( file . HasChecksum ) {
if ( ! ArrayComparer . IsEqual ( file . Checksum , chsum ) ) {
// TODO: Report.SymbolRelatedToPreviousError
Report . Warning ( 1 6 9 7 , 1 , checksum_location , "Different checksum values specified for file `{0}'" , file . Name ) ;
}
}
file . SetChecksum ( guid_bytes , chsum ) ;
current_source . AutoGenerated = true ;
}
return true ;
}
#if !FULL_AST
static
#endif
bool IsTokenIdentifierEqual ( char [ ] identifier )
{
for ( int i = 0 ; i < identifier . Length ; + + i ) {
@ -3013,7 +3025,7 @@ namespace Mono.CSharp
@@ -3013,7 +3025,7 @@ namespace Mono.CSharp
if ( id_builder [ 0 ] > = '_' & & ! quoted ) {
int keyword = GetKeyword ( id_builder , pos ) ;
if ( keyword ! = - 1 ) {
val = LocatedToken . Create ( keyword = = Token . AWAIT ? "await" : null , ref_line , column ) ;
val = ltb . Create ( keyword = = Token . AWAIT ? "await" : null , current_source , ref_line , column ) ;
return keyword ;
}
}
@ -3021,12 +3033,12 @@ namespace Mono.CSharp
@@ -3021,12 +3033,12 @@ namespace Mono.CSharp
string s = InternIdentifier ( id_builder , pos ) ;
#if FULL_AST
if ( quoted ) {
val = LocatedToken . Create ( "@" + s , ref_line , column - 1 ) ;
val = ltb . Create ( "@" + s , current_source , ref_line , column - 1 ) ;
} else {
val = LocatedToken . Create ( s , ref_line , column ) ;
val = ltb . Create ( s , current_source , ref_line , column ) ;
}
#else
val = LocatedToken . Create ( s , ref_line , column ) ;
val = ltb . Create ( s , current_source , ref_line , column ) ;
#endif
if ( quoted & & parsing_attribute_section )
AddEscapedIdentifier ( ( ( LocatedToken ) val ) . Location ) ;
@ -3034,9 +3046,6 @@ namespace Mono.CSharp
@@ -3034,9 +3046,6 @@ namespace Mono.CSharp
return Token . IDENTIFIER ;
}
#if !FULL_AST
static
#endif
string InternIdentifier ( char [ ] charBuffer , int length )
{
//
@ -3102,17 +3111,17 @@ namespace Mono.CSharp
@@ -3102,17 +3111,17 @@ namespace Mono.CSharp
return consume_identifier ( c ) ;
case '{' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
return Token . OPEN_BRACE ;
case '}' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
return Token . CLOSE_BRACE ;
case '[' :
// To block doccomment inside attribute declaration.
if ( doc_state = = XmlCommentState . Allowed )
doc_state = XmlCommentState . NotAllowed ;
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
if ( parsing_block = = 0 | | lambda_arguments_parsing )
return Token . OPEN_BRACKET ;
@ -3138,10 +3147,10 @@ namespace Mono.CSharp
@@ -3138,10 +3147,10 @@ namespace Mono.CSharp
return Token . OPEN_BRACKET_EXPR ;
}
case ']' :
LocatedToken . CreateOptional ( ref_line , col , ref val ) ;
ltb . CreateOptional ( current_source , ref_line , col , ref val ) ;
return Token . CLOSE_BRACKET ;
case '(' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
//
// An expression versions of parens can appear in block context only
//
@ -3186,29 +3195,29 @@ namespace Mono.CSharp
@@ -3186,29 +3195,29 @@ namespace Mono.CSharp
return Token . OPEN_PARENS ;
case ')' :
LocatedToken . CreateOptional ( ref_line , col , ref val ) ;
ltb . CreateOptional ( current_source , ref_line , col , ref val ) ;
return Token . CLOSE_PARENS ;
case ',' :
LocatedToken . CreateOptional ( ref_line , col , ref val ) ;
ltb . CreateOptional ( current_source , ref_line , col , ref val ) ;
return Token . COMMA ;
case ';' :
LocatedToken . CreateOptional ( ref_line , col , ref val ) ;
ltb . CreateOptional ( current_source , ref_line , col , ref val ) ;
return Token . SEMICOLON ;
case '~' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
return Token . TILDE ;
case '?' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
return TokenizePossibleNullableType ( ) ;
case '<' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
if ( parsing_generic_less_than + + > 0 )
return Token . OP_GENERICS_LT ;
return TokenizeLessThan ( ) ;
case '>' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
d = peek_char ( ) ;
if ( d = = '=' ) {
@ -3235,7 +3244,7 @@ namespace Mono.CSharp
@@ -3235,7 +3244,7 @@ namespace Mono.CSharp
return Token . OP_GT ;
case '+' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
d = peek_char ( ) ;
if ( d = = '+' ) {
d = Token . OP_INC ;
@ -3248,7 +3257,7 @@ namespace Mono.CSharp
@@ -3248,7 +3257,7 @@ namespace Mono.CSharp
return d ;
case '-' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
d = peek_char ( ) ;
if ( d = = '-' ) {
d = Token . OP_DEC ;
@ -3263,15 +3272,15 @@ namespace Mono.CSharp
@@ -3263,15 +3272,15 @@ namespace Mono.CSharp
return d ;
case '!' :
val = LocatedToken . Create ( ref_line , col ) ;
if ( peek_char ( ) = = '=' ) {
val = ltb . Create ( current_source , ref_line , col ) ;
if ( peek_char ( ) = = '=' ) {
get_char ( ) ;
return Token . OP_NE ;
}
return Token . BANG ;
case '=' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
d = peek_char ( ) ;
if ( d = = '=' ) {
get_char ( ) ;
@ -3285,7 +3294,7 @@ namespace Mono.CSharp
@@ -3285,7 +3294,7 @@ namespace Mono.CSharp
return Token . ASSIGN ;
case '&' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
d = peek_char ( ) ;
if ( d = = '&' ) {
get_char ( ) ;
@ -3298,7 +3307,7 @@ namespace Mono.CSharp
@@ -3298,7 +3307,7 @@ namespace Mono.CSharp
return Token . BITWISE_AND ;
case '|' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
d = peek_char ( ) ;
if ( d = = '|' ) {
get_char ( ) ;
@ -3311,8 +3320,8 @@ namespace Mono.CSharp
@@ -3311,8 +3320,8 @@ namespace Mono.CSharp
return Token . BITWISE_OR ;
case '*' :
val = LocatedToken . Create ( ref_line , col ) ;
if ( peek_char ( ) = = '=' ) {
val = ltb . Create ( current_source , ref_line , col ) ;
if ( peek_char ( ) = = '=' ) {
get_char ( ) ;
return Token . OP_MULT_ASSIGN ;
}
@ -3320,8 +3329,8 @@ namespace Mono.CSharp
@@ -3320,8 +3329,8 @@ namespace Mono.CSharp
case '/' :
d = peek_char ( ) ;
if ( d = = '=' ) {
val = LocatedToken . Create ( ref_line , col ) ;
if ( d = = '=' ) {
val = ltb . Create ( current_source , ref_line , col ) ;
get_char ( ) ;
return Token . OP_DIV_ASSIGN ;
}
@ -3430,11 +3439,11 @@ namespace Mono.CSharp
@@ -3430,11 +3439,11 @@ namespace Mono.CSharp
update_formatted_doc_comment ( current_comment_start ) ;
continue ;
}
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
return Token . DIV ;
case '%' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
if ( peek_char ( ) = = '=' ) {
get_char ( ) ;
return Token . OP_MOD_ASSIGN ;
@ -3442,7 +3451,7 @@ namespace Mono.CSharp
@@ -3442,7 +3451,7 @@ namespace Mono.CSharp
return Token . PERCENT ;
case '^' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
if ( peek_char ( ) = = '=' ) {
get_char ( ) ;
return Token . OP_XOR_ASSIGN ;
@ -3450,7 +3459,7 @@ namespace Mono.CSharp
@@ -3450,7 +3459,7 @@ namespace Mono.CSharp
return Token . CARRET ;
case ':' :
val = LocatedToken . Create ( ref_line , col ) ;
val = ltb . Create ( current_source , ref_line , col ) ;
if ( peek_char ( ) = = ':' ) {
get_char ( ) ;
return Token . DOUBLE_COLON ;
@ -3474,7 +3483,7 @@ namespace Mono.CSharp
@@ -3474,7 +3483,7 @@ namespace Mono.CSharp
if ( d > = '0' & & d < = '9' )
return is_number ( c ) ;
LocatedToken . CreateOptional ( ref_line , col , ref val ) ;
ltb . CreateOptional ( current_source , ref_line , col , ref val ) ;
return Token . DOT ;
case '#' :