diff --git a/ILSpy/ILSpy.csproj b/ILSpy/ILSpy.csproj
index f61c643c4..1f3fc6f81 100644
--- a/ILSpy/ILSpy.csproj
+++ b/ILSpy/ILSpy.csproj
@@ -50,7 +50,7 @@
   </ItemGroup>
 
   <ItemGroup>
-    <PackageReference Include="AvalonEdit" Version="5.0.3" />
+    <PackageReference Include="AvalonEdit" Version="5.0.4" />
   </ItemGroup>
 
   <ItemGroup>
diff --git a/ILSpy/TextView/DecompilerTextView.cs b/ILSpy/TextView/DecompilerTextView.cs
index ad52a7046..b8fea2606 100644
--- a/ILSpy/TextView/DecompilerTextView.cs
+++ b/ILSpy/TextView/DecompilerTextView.cs
@@ -106,7 +106,8 @@ namespace ICSharpCode.ILSpy.TextView
 			textEditor.Options.RequireControlModifierForHyperlinkClick = false;
 			textEditor.TextArea.TextView.MouseHover += TextViewMouseHover;
 			textEditor.TextArea.TextView.MouseHoverStopped += TextViewMouseHoverStopped;
-			textEditor.TextArea.TextView.MouseDown += TextViewMouseDown;
+			textEditor.TextArea.PreviewMouseDown += TextAreaMouseDown;
+			textEditor.TextArea.PreviewMouseUp += TextAreaMouseUp;
 			textEditor.SetBinding(Control.FontFamilyProperty, new Binding { Source = DisplaySettingsPanel.CurrentDisplaySettings, Path = new PropertyPath("SelectedFont") });
 			textEditor.SetBinding(Control.FontSizeProperty, new Binding { Source = DisplaySettingsPanel.CurrentDisplaySettings, Path = new PropertyPath("SelectedFontSize") });
 			textEditor.SetBinding(TextEditor.WordWrapProperty, new Binding { Source = DisplaySettingsPanel.CurrentDisplaySettings, Path = new PropertyPath("EnableWordWrap") });
@@ -602,12 +603,36 @@ namespace ICSharpCode.ILSpy.TextView
 			MainWindow.Instance.JumpToReference(reference);
 		}
 
-		void TextViewMouseDown(object sender, MouseButtonEventArgs e)
+		Point? mouseDownPos;
+
+		void TextAreaMouseDown(object sender, MouseButtonEventArgs e)
 		{
-			if (GetReferenceSegmentAtMousePosition() == null)
-				ClearLocalReferenceMarks();
+			mouseDownPos = e.GetPosition(this);
 		}
-
+		
+		void TextAreaMouseUp(object sender, MouseButtonEventArgs e)
+		{
+			if (mouseDownPos == null)
+				return;
+			Vector dragDistance = e.GetPosition(this) - mouseDownPos.Value;
+			if (Math.Abs(dragDistance.X) < SystemParameters.MinimumHorizontalDragDistance
+				&& Math.Abs(dragDistance.Y) < SystemParameters.MinimumVerticalDragDistance
+				&& e.ChangedButton == MouseButton.Left)
+			{
+				// click without moving mouse
+				var referenceSegment = GetReferenceSegmentAtMousePosition();
+				if (referenceSegment == null) {
+					ClearLocalReferenceMarks();
+				} else {
+					JumpToReference(referenceSegment);
+					textEditor.TextArea.ClearSelection();
+				}
+				// cancel mouse selection to avoid AvalonEdit selecting between the new
+				// cursor position and the mouse position.
+				textEditor.TextArea.MouseSelectionMode = MouseSelectionMode.None;
+			}
+		}
+		
 		void ClearLocalReferenceMarks()
 		{
 			foreach (var mark in localReferenceMarks) {
diff --git a/ILSpy/TextView/ReferenceElementGenerator.cs b/ILSpy/TextView/ReferenceElementGenerator.cs
index 3e4e88ca0..262f7d598 100644
--- a/ILSpy/TextView/ReferenceElementGenerator.cs
+++ b/ILSpy/TextView/ReferenceElementGenerator.cs
@@ -105,16 +105,6 @@ namespace ICSharpCode.ILSpy.TextView
 			e.Cursor = referenceSegment.IsLocal ? Cursors.Arrow : Cursors.Hand;
 		}
 		
-		/// <inheritdoc/>
-		protected override void OnMouseDown(MouseButtonEventArgs e)
-		{
-			if (e.ChangedButton == MouseButton.Left && !e.Handled) {
-				parent.JumpToReference(referenceSegment);
-				if(!referenceSegment.IsLocal)
-					e.Handled = true;
-			}
-		}
-		
 		/// <inheritdoc/>
 		protected override VisualLineText CreateInstance(int length)
 		{