Fix character reference consumption for consecutive & in text and correct numeric NCR edge handling

src/main/java/org/htmlunit/cyberneko/HTMLScanner.java

@@ -1467,6 +1467,16 @@ protected int scanEntityRef(final XMLString str, final XMLString plainValue, fin
             }
             return returnEntityRefString(str, content);
         }
+
+        if ('&' == nextChar) {
+            fCurrentEntity.rewind(1);
+            if (plainValue != null) {
+                plainValue.append('&');
+            }
+            str.clearAndAppend('&');
+            return returnEntityRefString(str, content);
+        }
+
         str.append((char) nextChar);
 
         if ('#' == nextChar) {

src/main/java/org/htmlunit/cyberneko/HTMLUnicodeEntitiesParser.java

@@ -83,6 +83,7 @@ public void setMatchFromCode() {
         // a surrogate-character-reference parse error. Set the character reference code to 0xFFFD
         if (Character.isSurrogate((char) code_)) {
             match_ = "\uFFFD";
+            matchLength_ = consumedCount_;
             return;
         }
 
@@ -196,7 +197,7 @@ public void setMatchFromCode() {
                 return;
 
             case 0x98:
-                match_ = "\u20DC";
+                match_ = "\u02DC";
                 matchLength_ = consumedCount_;
                 return;
 

src/test/java/org/htmlunit/cyberneko/GeneralTest.java

@@ -44,7 +44,7 @@
     private static final String[] FEATURES = {"http://cyberneko.org/html/features/augmentations"};
 
     @Test
     public void newlineInAttributeCrLf() throws Exception {
         String expected =
             "[synth](html" + NL
             + "[synth](head" + NL
@@ -87,7 +87,7 @@
     }
 
     @Test
     public void newlineInAttributeLf() throws Exception {
         String expected =
             "[synth](html" + NL
             + "[synth](head" + NL
@@ -133,7 +133,7 @@
     }
 
     @Test
     public void newlineInPiCrLf() throws Exception {
         String expected =
             "[synth](html" + NL
             + "[synth](head" + NL
@@ -170,7 +170,7 @@
     }
 
     @Test
     public void newlineInPiLf() throws Exception {
         String expected =
             "[synth](html" + NL
             + "[synth](head" + NL
@@ -239,7 +239,7 @@
     }
 
     @Test
     public void parseInputSourceReplacement() throws Exception {
         final DOMParser parser = new DOMParser(HTMLDocumentImpl.class);
 
         final StringWriter out = new StringWriter();
@@ -262,7 +262,7 @@
     }
 
     @Test
     public void parseInputSourceResolvesToReplacement() throws Exception {
         final DOMParser parser = new DOMParser(HTMLDocumentImpl.class);
 
         final StringWriter out = new StringWriter();
@@ -285,7 +285,7 @@
     }
 
     @Test
     public void parseInputSourceIANAEncoding() throws Exception {
         final DOMParser parser = new DOMParser(HTMLDocumentImpl.class);
 
         final StringWriter out = new StringWriter();
@@ -315,4 +315,40 @@
                 + ")html";
         assertEquals(expected.trim(), out.toString().trim());
     }
+
+    @Test
+    public void textContentConsecutiveAmpersandsBeforeNamedEntity() throws Exception {
+        final String expected = "(html" + NL
+                + "(head" + NL
+                + ")head" + NL
+                + "(body" + NL
+                + "\"FOO&&&>BAR" + NL
+                + ")body" + NL
+                + ")html";
+        doTest("FOO&&&>BAR", null, expected);
+    }
+
+    @Test
+    public void textContentSurrogateNumericReference() throws Exception {
+        final String expected = "(html" + NL
+                + "(head" + NL
+                + ")head" + NL
+                + "(body" + NL
+                + "\"FOO\uFFFDZOO" + NL
+                + ")body" + NL
+                + ")html";
+        doTest("FOO�ZOO", null, expected);
+    }
+
+    @Test
+    public void textContentWindows1252ControlMapping() throws Exception {
+        final String expected = "(html" + NL
+                + "(head" + NL
+                + ")head" + NL
+                + "(body" + NL
+                + "\"FOO\u02DCZOO" + NL
+                + ")body" + NL
+                + ")html";
+        doTest("FOO˜ZOO", null, expected);
+    }
 }

src/test/java/org/htmlunit/cyberneko/HTMLUnicodeEntitiesParserTest.java

@@ -26,9 +26,9 @@
  * @author René Schwietzke
  * @author Ronald Brill
  */
 public class HTMLUnicodeEntitiesParserTest {
     @Test
     public void parseEuro() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "x80;";
@@ -42,7 +42,7 @@
     }
 
     @Test
     public void parseEuroUppercase() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "X80;";
@@ -56,7 +56,7 @@
     }
 
     @Test
     public void parseBroken() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "A80;";
@@ -70,7 +70,7 @@
     }
 
     @Test
     public void parseLTAsDecimal() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "60;";
@@ -84,7 +84,7 @@
     }
 
     @Test
     public void parseLTAsDecimalBroken() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "60 ";
@@ -98,7 +98,7 @@
     }
 
     @Test
     public void parseEuroMissingSemicolon() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "x80<";
@@ -112,7 +112,7 @@
     }
 
     @Test
     public void parseNullChar() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "x00;";
@@ -126,7 +126,7 @@
     }
 
     @Test
     public void parseOverflowRange() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "x11FFFF;";
@@ -140,7 +140,7 @@
     }
 
     @Test
     public void parseSurrogate() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "xD800;";
@@ -150,11 +150,25 @@
         }
 
         assertEquals("\uFFFD", parser.getMatch());
-        assertEquals(6, parser.getRewindCount());
+        assertEquals(0, parser.getRewindCount());
+    }
+
+    @Test
+    public void parseWindows1252SmallTilde() {
+        final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
+
+        final String input = "x98;";
+        int i = 0;
+        while (parser.parseNumeric(input.charAt(i))) {
+            i++;
+        }
+
+        assertEquals("\u02DC", parser.getMatch());
+        assertEquals(0, parser.getRewindCount());
     }
 
     @Test
     public void parseNonCharacterLow() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "x80;";
@@ -168,7 +182,7 @@
     }
 
     @Test
     public void parseNonCharacterHighLowercase() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "x9f;";
@@ -182,7 +196,7 @@
     }
 
     @Test
     public void parseNonCharacterHighUppercase() {
         final HTMLUnicodeEntitiesParser parser = new HTMLUnicodeEntitiesParser();
 
         final String input = "x9F;";