The parser is now able to emit new lines in the AST. However the output visitor needs to be adjusted. (otherwise too many new lines may be inserted)

ICSharpCode.NRefactory.CSharp/Ast/GeneralScope/NewLineNode.cs

@@ -76,6 +76,11 @@ namespace ICSharpCode.NRefactory.CSharp
 		{
 		}
 
+		public override string GetText(CSharpFormattingOptions formattingOptions)
+		{
+			return "\n";
+		}
+
 		protected internal override bool DoMatch(AstNode other, PatternMatching.Match match)
 		{
 			var o = other as UnixNewLine;
@@ -99,6 +104,11 @@ namespace ICSharpCode.NRefactory.CSharp
 		{
 		}
 
+		public override string GetText(CSharpFormattingOptions formattingOptions)
+		{
+			return "\r\n";
+		}
+
 		protected internal override bool DoMatch(AstNode other, PatternMatching.Match match)
 		{
 			var o = other as WindowsNewLine;
@@ -122,6 +132,11 @@ namespace ICSharpCode.NRefactory.CSharp
 		{
 		}
 
+		public override string GetText(CSharpFormattingOptions formattingOptions)
+		{
+			return "\r";
+		}
+
 		protected internal override bool DoMatch(AstNode other, PatternMatching.Match match)
 		{
 			var o = other as MacNewLine;

ICSharpCode.NRefactory.CSharp/Formatter/AstFormattingVisitor.cs

@@ -1043,6 +1043,8 @@ namespace ICSharpCode.NRefactory.CSharp
 					child.AcceptVisitor(this);
 				} else if (child is Comment) {
 					child.AcceptVisitor(this);
+				} else if (child is NewLineNode) {
+					// ignore
 				} else {
 					// pre processor directives at line start, if they are there.
 					if (child.StartLocation.Column > 1)

ICSharpCode.NRefactory.CSharp/Parser/CSharpParser.cs

@@ -3630,6 +3630,15 @@ namespace ICSharpCode.NRefactory.CSharp
 							Argument = directive.Arg,
 							Take = directive.Take
 						};
+					} else {
+	/*					var newLine = special as SpecialsBag.NewLineToken;
+						if (newLine != null) {
+							if (newLine.NewLine == SpecialsBag.NewLine.Unix) {
+								newLeaf = new UnixNewLine (new TextLocation (newLine.Line, newLine.Col));
+							} else {
+								newLeaf = new WindowsNewLine (new TextLocation (newLine.Line, newLine.Col));
+							}
+						}*/
 					}
 				}
 				if (newLeaf == null)
@@ -3644,7 +3653,9 @@ namespace ICSharpCode.NRefactory.CSharp
 							leaf = node;
 							node = node.Parent;
 						}
-						if (newLeaf is Comment) {
+						if (newLeaf is NewLineNode) {
+							node.InsertChildBefore (leaf, (NewLineNode)newLeaf, Roles.NewLine);
+						} else if (newLeaf is Comment) {
 							node.InsertChildBefore (leaf, (Comment)newLeaf, Roles.Comment);
 						} else {
 							node.InsertChildBefore (leaf, (PreProcessorDirective)newLeaf, Roles.PreProcessorDirective);
@@ -3656,7 +3667,9 @@ namespace ICSharpCode.NRefactory.CSharp
 					// insert comment at the end
 					if (nextLeaf == null) {
 						var node = leaf.Parent ?? conversionVisitor.Unit;
-						if (newLeaf is Comment) {
+						if (newLeaf is NewLineNode) {
+							node.AddChild ((NewLineNode)newLeaf, Roles.NewLine);
+						} else if (newLeaf is Comment) {
 							node.AddChild ((Comment)newLeaf, Roles.Comment);
 						} else {
 							node.AddChild ((PreProcessorDirective)newLeaf, Roles.PreProcessorDirective);
@@ -3668,7 +3681,9 @@ namespace ICSharpCode.NRefactory.CSharp
 					// comment is between 2 nodes
 					if (leaf.EndLocation <= newLeaf.StartLocation && newLeaf.StartLocation <= nextLeaf.StartLocation) {
 						var node = leaf.Parent ?? conversionVisitor.Unit;
-						if (newLeaf is Comment) {
+						if (newLeaf is NewLineNode) {
+							node.InsertChildAfter (leaf, (NewLineNode)newLeaf, Roles.NewLine);
+						} else if (newLeaf is Comment) {
 							node.InsertChildAfter (leaf, (Comment)newLeaf, Roles.Comment);
 						} else {
 							node.InsertChildAfter (leaf, (PreProcessorDirective)newLeaf, Roles.PreProcessorDirective);
@@ -3884,6 +3899,9 @@ namespace ICSharpCode.NRefactory.CSharp
 		IEnumerable<EntityDeclaration> ParseTypeMembers (string code, int initialLine, int initialColumn)
 		{
 			const string prefix = "unsafe partial class MyClass { ";
+			Console.WriteLine("---");
+			Console.WriteLine(prefix + code + "}");
+			Console.WriteLine("---");
 			var syntaxTree = Parse (new StringTextSource (prefix + code + "}"), "parsed.cs", initialLine, initialColumn - prefix.Length);
 			if (syntaxTree == null)
 				return Enumerable.Empty<EntityDeclaration> ();

ICSharpCode.NRefactory.CSharp/Parser/mcs/cs-tokenizer.cs

@@ -1797,40 +1797,23 @@ namespace Mono.CSharp
 			if (x == '\r') {
 				if (peek_char () == '\n') {
 					putback_char = -1;
+					advance_line (SpecialsBag.NewLine.Windows);
+				} else {
+					advance_line (SpecialsBag.NewLine.Unix);
 				}
-
 				x = '\n';
-				advance_line ();
 			} else if (x == '\n') {
-				advance_line ();
-			} else {
-				col++;
-			}
-			return x;
-		}
+				advance_line (SpecialsBag.NewLine.Unix);
 
-		int get_char_withwithoutskippingwindowseol ()
-		{
-			int x;
-			if (putback_char != -1) {
-				x = putback_char;
-				putback_char = -1;
-			} else {
-				x = reader.Read ();
-			}
-			
-			if (x == '\r') {
-
-			} else if (x == '\n') {
-				advance_line ();
 			} else {
 				col++;
 			}
 			return x;
 		}
 
-		void advance_line ()
+		void advance_line (SpecialsBag.NewLine newLine)
 		{
+			sbag.AddNewLine (line, col, newLine);
 			line++;
 			ref_line++;
 			previous_col = col;
@@ -2921,8 +2904,16 @@ namespace Mono.CSharp
 #endif
 
 			while (true){
-				c = get_char_withwithoutskippingwindowseol ();
+				// Cannot use get_char because of \r in quoted strings
+				if (putback_char != -1) {
+					c = putback_char;
+					putback_char = -1;
+				} else {
+					c = reader.Read ();
+				}
+
 				if (c == '"') {
+					++col;
 					if (quoted && peek_char () == '"') {
 						if (pos == value_builder.Length)
 							Array.Resize (ref value_builder, pos * 2);
@@ -2954,9 +2945,21 @@ namespace Mono.CSharp
 				if (c == '\n') {
 					if (!quoted) {
 						Report.Error (1010, Location, "Newline in constant");
+
+
+						// Don't add \r to string literal
+						if (pos > 1 && value_builder [pos - 1] == '\r') {
+							advance_line (SpecialsBag.NewLine.Windows);
+							--pos;
+						} else {
+							advance_line (SpecialsBag.NewLine.Unix);
+						}
+
 						val = new StringLiteral (context.BuiltinTypes, new string (value_builder, 0, pos), start_location);
 						return Token.LITERAL;
 					}
+
+					advance_line (SpecialsBag.NewLine.Unix);
 				} else if (c == '\\' && !quoted) {
 					int surrogate;
 					c = escape (c, out surrogate);
@@ -2972,6 +2975,8 @@ namespace Mono.CSharp
 				} else if (c == -1) {
 					Report.Error (1039, Location, "Unterminated string literal");
 					return Token.EOF;
+				} else {
+					++col;
 				}
 
 				if (pos == value_builder.Length)

ICSharpCode.NRefactory.CSharp/Parser/mcs/location.cs

@@ -414,8 +414,25 @@ if (checkpoints.Length <= CheckpointIndex) throw new Exception (String.Format ("
 			Documentation,
 			InactiveCode
 		}
+
+		public class SpecialVisitor
+		{
+			public virtual void Visit (Comment comment)
+			{
+			}
+			public virtual void Visit (NewLineToken newLineToken)
+			{
+			}
+			public virtual void Visit (PreProcessorDirective preProcessorDirective)
+			{
+			}
+		}
+		public abstract class SpecialBase
+		{
+			public abstract void Accept (SpecialVisitor visitor);
+		}
 		
-		public class Comment
+		public class Comment : SpecialBase
 		{
 			public readonly CommentType CommentType;
 			public readonly bool StartsLine;
@@ -440,9 +457,33 @@ if (checkpoints.Length <= CheckpointIndex) throw new Exception (String.Format ("
 			{
 				return string.Format ("[Comment: CommentType={0}, Line={1}, Col={2}, EndLine={3}, EndCol={4}, Content={5}]", CommentType, Line, Col, EndLine, EndCol, Content);
 			}
+
+			public override void Accept (SpecialVisitor visitor)
+			{
+				visitor.Visit (this);
+			}
 		}
-		
-		public class PreProcessorDirective
+
+		public class NewLineToken : SpecialBase
+		{
+			public readonly int Line;
+			public readonly int Col;
+			public readonly NewLine NewLine;
+
+			public NewLineToken (int line, int col, NewLine newLine)
+			{
+				this.Line = line;
+				this.Col = col;
+				this.NewLine = newLine;
+			}
+
+			public override void Accept (SpecialVisitor visitor)
+			{
+				visitor.Visit (this);
+			}
+		}
+
+		public class PreProcessorDirective : SpecialBase
 		{
 			public readonly int Line;
 			public readonly int Col;
@@ -464,13 +505,18 @@ if (checkpoints.Length <= CheckpointIndex) throw new Exception (String.Format ("
 				this.Arg = arg;
 			}
 			
+			public override void Accept (SpecialVisitor visitor)
+			{
+				visitor.Visit (this);
+			}
+
 			public override string ToString ()
 			{
 				return string.Format ("[PreProcessorDirective: Line={0}, Col={1}, EndLine={2}, EndCol={3}, Cmd={4}, Arg={5}]", Line, Col, EndLine, EndCol, Cmd, Arg);
 			}
 		}
 		
-		public readonly List<object> Specials = new List<object> ();
+		public readonly List<SpecialBase> Specials = new List<SpecialBase> ();
 		
 		CommentType curComment;
 		bool startsLine;
@@ -519,6 +565,14 @@ if (checkpoints.Length <= CheckpointIndex) throw new Exception (String.Format ("
 			Specials.Add (new PreProcessorDirective (startLine, startCol, endLine, endColumn, cmd, arg));
 		}
 
+		public enum NewLine { Unix, Windows }
+
+		[Conditional ("FULL_AST")]
+		public void AddNewLine (int line, int col, NewLine newLine)
+		{
+			Specials.Add (new NewLineToken (line, col, newLine));
+		}
+
 		public void SkipIf ()
 		{
 			if (Specials.Count > 0) {