From 5346086a54970066524ab7d83bed61d0511705d4 Mon Sep 17 00:00:00 2001
From: Matt Van Horn <455140+mvanhorn@users.noreply.github.com>
Date: Thu, 22 Oct 2026 03:26:10 -0700
Subject: [PATCH] Add backward movement and search to Scanner
prev steps back one UTF-16 code unit, including across the synthetic
newline between lines, and leaves the position unchanged at the start of
the input. findPrev searches backward for a character or a CharMatcher,
with overloads that take an inclusive lower position so the search can
stop at a saved opener. The current character is included, the end
sentinel is skipped and never matches, a hit returns how many code units
were moved, and a miss returns -1 while staying at the start of the
input or at the bound. A bound after the current position or outside the
input throws IllegalArgumentException before the position changes.
Fixes #395
---
.../org/commonmark/parser/beta/Scanner.java | 152 +++++-
.../parser/InlineContentParserTest.java | 91 ++++
.../commonmark/parser/beta/ScannerTest.java | 467 ++++++++++++++++++
3 files changed, 709 insertions(+), 1 deletion(-)
diff --git a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
index 1acf39ced..91aaa6015 100644
--- a/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
+++ b/commonmark/src/main/java/org/commonmark/parser/beta/Scanner.java
@@ -13,7 +13,8 @@ public class Scanner {
* "previous" methods).
*
* Note that we can use NULL to represent this because CommonMark does not allow those in the
- * input (we replace them in the beginning of parsing).
+ * input (we replace them in the beginning of parsing). It is not a searchable source character:
+ * {@link #find(char)} and {@link #findPrev(char)} skip it instead of matching it.
*/
public static final char END = '\0';
@@ -119,6 +120,30 @@ public void next() {
}
}
+ /**
+ * Move back by one UTF-16 code unit.
+ *
+ *
This reverses {@link #next()}. Lines yield a synthetic newline between lines and no
+ * newline after the final line: from index zero the position moves to the previous line's
+ * length (that newline), and from the end of input it moves to the last character, or across
+ * an empty final line to the preceding newline. At the beginning of the input, including an
+ * empty input, the position is left unchanged.
+ */
+ public void prev() {
+ if (index > 0) {
+ index--;
+ } else if (lineIndex > 0) {
+ lineIndex--;
+ if (lineIndex < lines.size()) {
+ setLine(lines.get(lineIndex));
+ } else {
+ // Landed past the end (after next() at EOF); same representation as next().
+ setLine(SourceLine.of("", null));
+ }
+ index = lineLength;
+ }
+ }
+
/**
* Check if the specified char is next and advance the position.
*
@@ -221,6 +246,100 @@ public int find(CharMatcher matcher) {
}
}
+ /**
+ * Find the nearest occurrence of {@code c} at or before the current position.
+ *
+ *
The current character is included, matching {@link #find(char)}, so an immediate match
+ * returns {@code 0}. {@link #END} is skipped and never matches. From the end of input a match
+ * on the character immediately before the end returns {@code 1}. A newline between lines can
+ * be matched. If {@code c} does not occur, the scanner is left at the beginning of the input
+ * and {@code -1} is returned.
+ *
+ * @param c character to find (including a newline)
+ * @return the number of UTF-16 code units moved backward, or {@code -1} if not found
+ */
+ public int findPrev(char c) {
+ return findPrev(c, null, 0, 0);
+ }
+
+ /**
+ * Find the nearest character matched by {@code matcher} at or before the current position.
+ *
+ *
See {@link #findPrev(char)} for the current position, {@link #END}, and a search that
+ * reaches the beginning of the input. {@link #END} is not passed to {@code matcher}.
+ *
+ * @param matcher matcher applied to each character
+ * @return the number of UTF-16 code units moved backward, or {@code -1} if not found
+ */
+ public int findPrev(CharMatcher matcher) {
+ return findPrev(END, matcher, 0, 0);
+ }
+
+ /**
+ * Find the nearest occurrence of {@code c} at or before the current position and at or after
+ * {@code bound}.
+ *
+ *
{@code bound} is inclusive: the character there is examined, and the search never moves
+ * before it. If {@code bound} is the current position, only that character is checked. If there
+ * is no match, the scanner is left at {@code bound} and {@code -1} is returned. {@link #END} is
+ * skipped and never matches. On an empty input the search returns {@code -1}; the position at
+ * the beginning of that input is a valid bound.
+ *
+ *
The bound is checked before the scanner moves. It must have valid coordinates for this
+ * scanner and must not be after the current position. As with {@link #setPosition(Position)},
+ * positions are expected to come from the same scanner.
+ *
+ * @param c character to find (including a newline)
+ * @param bound inclusive lower bound
+ * @return the number of UTF-16 code units moved backward, or {@code -1} if not found
+ * @throws IllegalArgumentException if {@code bound} is after the current position or outside
+ * the input
+ */
+ public int findPrev(char c, Position bound) {
+ checkLowerBound(bound);
+ return findPrev(c, null, bound.lineIndex, bound.index);
+ }
+
+ /**
+ * Find the nearest character matched by {@code matcher} at or before the current position and
+ * at or after {@code bound}.
+ *
+ *
See {@link #findPrev(char, Position)} for the bound, {@link #END}, and a search with no
+ * match. {@link #END} is not passed to {@code matcher}.
+ *
+ * @param matcher matcher applied to each character
+ * @param bound inclusive lower bound
+ * @return the number of UTF-16 code units moved backward, or {@code -1} if not found
+ * @throws IllegalArgumentException if {@code bound} is after the current position or outside
+ * the input
+ */
+ public int findPrev(CharMatcher matcher, Position bound) {
+ checkLowerBound(bound);
+ return findPrev(END, matcher, bound.lineIndex, bound.index);
+ }
+
+ // matcher == null matches c exactly, so the char overloads allocate nothing per character.
+ private int findPrev(char c, CharMatcher matcher, int boundLine, int boundIndex) {
+ int count = 0;
+ while (true) {
+ char cur = peek();
+ if (cur != END && (matcher != null ? matcher.matches(cur) : cur == c)) {
+ return count;
+ }
+ if (lineIndex == boundLine && index == boundIndex) {
+ return -1;
+ }
+ int lineBefore = lineIndex;
+ int indexBefore = index;
+ prev();
+ // Beginning of input: prev() is a no-op, so stop instead of spinning.
+ if (lineIndex == lineBefore && index == indexBefore) {
+ return -1;
+ }
+ count++;
+ }
+ }
+
// Don't expose the int index, because it would be good if we could switch input to a
// List of lines later instead of one contiguous String.
public Position position() {
@@ -280,4 +399,35 @@ private void checkPosition(int lineIndex, int index) {
"Index " + index + " out of range, line length: " + line.getContent().length());
}
}
+
+ private void checkLowerBound(Position position) {
+ // Empty input has no lines; its only position is the canonical (0, 0) cursor.
+ if (lines.isEmpty()) {
+ if (position.lineIndex != 0) {
+ throw new IllegalArgumentException(
+ "Line index "
+ + position.lineIndex
+ + " out of range, number of lines: "
+ + lines.size());
+ }
+ if (position.index != 0) {
+ throw new IllegalArgumentException(
+ "Index " + position.index + " out of range, line length: 0");
+ }
+ } else {
+ checkPosition(position.lineIndex, position.index);
+ }
+ if (position.lineIndex > lineIndex
+ || (position.lineIndex == lineIndex && position.index > index)) {
+ throw new IllegalArgumentException(
+ "Bound line "
+ + position.lineIndex
+ + " index "
+ + position.index
+ + " is after the current position line "
+ + lineIndex
+ + " index "
+ + index);
+ }
+ }
}
diff --git a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java
index 167f74f25..fe4d03a17 100644
--- a/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java
+++ b/commonmark/src/test/java/org/commonmark/parser/InlineContentParserTest.java
@@ -9,6 +9,8 @@
import org.commonmark.parser.beta.InlineParserState;
import org.commonmark.parser.beta.ParsedInline;
import org.commonmark.test.Nodes;
+import org.commonmark.text.AsciiMatcher;
+import org.commonmark.text.CharMatcher;
import org.junit.jupiter.api.Test;
class InlineContentParserTest {
@@ -55,6 +57,46 @@ void bangInlineContentParser() {
assertThat(((Text) image.getNext().getNext().getNext()).getLiteral()).isEqualTo("notimage");
}
+ @Test
+ void backwardSearchInlineContentParser() {
+ var parser =
+ Parser.builder()
+ .customInlineContentParserFactory(new DestinationInlineParser.Factory())
+ .build();
+
+ var doc = parser.parse("img(not-a-link-)more-text) more text");
+ var inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(((Text) inline.getPrevious()).getLiteral()).isEqualTo("img");
+ assertThat(inline.getDestination()).isEqualTo("not-a-link-)more-text");
+ assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text");
+
+ doc = parser.parse("img(not-a-link-)more-text)");
+ inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(inline.getDestination()).isEqualTo("not-a-link-)more-text");
+ assertThat(inline.getNext()).isNull();
+
+ doc = parser.parse("img(not-a-link) more text");
+ inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(inline.getDestination()).isEqualTo("not-a-link");
+ assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text");
+
+ doc = parser.parse("img() more text");
+ inline = Nodes.find(doc, DestinationInline.class);
+ assertThat(inline.getDestination()).isEqualTo("");
+ assertThat(((Text) inline.getNext()).getLiteral()).isEqualTo(" more text");
+
+ doc = parser.parse("img(not-a-link more text");
+ assertThat(Nodes.tryFind(doc, DestinationInline.class)).isNull();
+ assertThat(((Text) doc.getFirstChild().getFirstChild()).getLiteral())
+ .isEqualTo("img(not-a-link more text");
+
+ // The ')' before the saved opening position is outside the inclusive bound.
+ doc = parser.parse("before)img(not-a-link more text");
+ assertThat(Nodes.tryFind(doc, DestinationInline.class)).isNull();
+ assertThat(((Text) doc.getFirstChild().getFirstChild()).getLiteral())
+ .isEqualTo("before)img(not-a-link more text");
+ }
+
private static class DollarInline extends CustomNode {
private final String literal;
private final int index;
@@ -105,6 +147,55 @@ public InlineContentParser create() {
}
}
+ /**
+ * Custom image-like syntax {@code (destination)} whose destination may itself contain {@code
+ * )}. The closer is the last {@code )} before whitespace or the end of input, and it must not
+ * be before the opening parenthesis.
+ */
+ private static class DestinationInline extends CustomNode {
+ private final String destination;
+
+ public DestinationInline(String destination) {
+ this.destination = destination;
+ }
+
+ public String getDestination() {
+ return destination;
+ }
+ }
+
+ private static class DestinationInlineParser implements InlineContentParser {
+
+ private static final CharMatcher WHITESPACE =
+ AsciiMatcher.builder().anyOf(" \t\n\u000B\f\r").build();
+
+ @Override
+ public ParsedInline tryParse(InlineParserState inlineParserState) {
+ var scanner = inlineParserState.scanner();
+ scanner.next();
+ var contentStart = scanner.position();
+ scanner.find(WHITESPACE);
+ if (scanner.findPrev(')', contentStart) == -1) {
+ return ParsedInline.none();
+ }
+ var destination = scanner.getSource(contentStart, scanner.position()).getContent();
+ scanner.next();
+ return ParsedInline.of(new DestinationInline(destination), scanner.position());
+ }
+
+ static class Factory implements InlineContentParserFactory {
+ @Override
+ public Set getTriggerCharacters() {
+ return Set.of('(');
+ }
+
+ @Override
+ public InlineContentParser create() {
+ return new DestinationInlineParser();
+ }
+ }
+ }
+
private static class BangInline extends CustomNode {}
private static class BangInlineParser implements InlineContentParser {
diff --git a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
index 437812866..44f94c957 100644
--- a/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
+++ b/commonmark/src/test/java/org/commonmark/parser/beta/ScannerTest.java
@@ -1,11 +1,13 @@
package org.commonmark.parser.beta;
import static org.assertj.core.api.Assertions.assertThat;
+import static org.assertj.core.api.Assertions.assertThatThrownBy;
import java.util.List;
import org.commonmark.node.SourceSpan;
import org.commonmark.parser.SourceLine;
import org.commonmark.parser.SourceLines;
+import org.commonmark.text.CharMatcher;
import org.junit.jupiter.api.Test;
class ScannerTest {
@@ -144,6 +146,471 @@ private void assertSourceLines(
assertThat(sourceLines.getSourceSpans()).isEqualTo(List.of(expectedSourceSpans));
}
+ @Test
+ void testPrev() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abc", null)));
+ Position start = scanner.position();
+ scanner.next();
+ Position b = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('b');
+ scanner.next();
+ Position c = scanner.position();
+ scanner.next();
+ Position eof = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.hasNext()).isFalse();
+
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(c);
+ assertThat(scanner.peek()).isEqualTo('c');
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(b);
+ assertThat(scanner.peek()).isEqualTo('b');
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.peek()).isEqualTo('a');
+
+ // No-op at the beginning.
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.peek()).isEqualTo('a');
+
+ scanner.next();
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.peek()).isEqualTo('a');
+
+ scanner.setPosition(eof);
+ scanner.prev();
+ scanner.next();
+ assertThat(scanner.position()).isEqualTo(eof);
+ assertThat(scanner.peek()).isEqualTo('\0');
+
+ Scanner empty = Scanner.of(SourceLines.empty());
+ Position emptyPos = empty.position();
+ empty.prev();
+ assertThat(empty.position()).isEqualTo(emptyPos);
+ assertThat(empty.peek()).isEqualTo('\0');
+ assertThat(empty.hasNext()).isFalse();
+
+ Scanner emptyLine = Scanner.of(SourceLines.of(SourceLine.of("", null)));
+ Position emptyLinePos = emptyLine.position();
+ assertThat(emptyLine.peek()).isEqualTo('\0');
+ assertThat(emptyLine.hasNext()).isFalse();
+ emptyLine.prev();
+ assertThat(emptyLine.position()).isEqualTo(emptyLinePos);
+ assertThat(emptyLine.peek()).isEqualTo('\0');
+ }
+
+ @Test
+ void testPrevAcrossLines() {
+ Scanner scanner =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("ab", null), SourceLine.of("c", null))));
+ Position start = scanner.position();
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('b');
+ scanner.next();
+ Position newline = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\n');
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('c');
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.hasNext()).isFalse();
+
+ scanner.prev();
+ assertThat(scanner.peek()).isEqualTo('c');
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(newline);
+ assertThat(scanner.peek()).isEqualTo('\n');
+ scanner.prev();
+ assertThat(scanner.peek()).isEqualTo('b');
+ scanner.next();
+ assertThat(scanner.position()).isEqualTo(newline);
+ assertThat(scanner.peek()).isEqualTo('\n');
+ scanner.prev();
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.peek()).isEqualTo('a');
+ scanner.prev();
+ assertThat(scanner.position()).isEqualTo(start);
+
+ // Unequal lengths: cached line content/length must follow the line we step onto.
+ Scanner uneven =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("a", null), SourceLine.of("bbb", null))));
+ uneven.next();
+ uneven.next();
+ assertThat(uneven.peek()).isEqualTo('b');
+ uneven.prev();
+ assertThat(uneven.peek()).isEqualTo('\n');
+ uneven.prev();
+ assertThat(uneven.peek()).isEqualTo('a');
+ uneven.next();
+ assertThat(uneven.peek()).isEqualTo('\n');
+ uneven.next();
+ uneven.next();
+ uneven.next();
+ uneven.next();
+ assertThat(uneven.peek()).isEqualTo('\0');
+ uneven.prev();
+ assertThat(uneven.peek()).isEqualTo('b');
+
+ // Consecutive empty lines, including an empty final line: no newline after the last line.
+ Scanner blanks =
+ Scanner.of(
+ SourceLines.of(
+ List.of(
+ SourceLine.of("", null),
+ SourceLine.of("", null),
+ SourceLine.of("", null))));
+ Position blanksStart = blanks.position();
+ assertThat(blanks.peek()).isEqualTo('\n');
+ blanks.next();
+ assertThat(blanks.peek()).isEqualTo('\n');
+ blanks.next();
+ assertThat(blanks.peek()).isEqualTo('\0');
+ assertThat(blanks.hasNext()).isFalse();
+ blanks.prev();
+ assertThat(blanks.peek()).isEqualTo('\n');
+ blanks.prev();
+ assertThat(blanks.position()).isEqualTo(blanksStart);
+ assertThat(blanks.peek()).isEqualTo('\n');
+ blanks.next();
+ blanks.next();
+ assertThat(blanks.peek()).isEqualTo('\0');
+
+ // Empty final line steps back onto the preceding newline, not a made-up terminal one.
+ Scanner trailingBlank =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("a", null), SourceLine.of("", null))));
+ Position trailingStart = trailingBlank.position();
+ trailingBlank.next();
+ Position precedingNewline = trailingBlank.position();
+ assertThat(trailingBlank.peek()).isEqualTo('\n');
+ trailingBlank.next();
+ assertThat(trailingBlank.peek()).isEqualTo('\0');
+ assertThat(trailingBlank.hasNext()).isFalse();
+ trailingBlank.prev();
+ assertThat(trailingBlank.position()).isEqualTo(precedingNewline);
+ assertThat(trailingBlank.peek()).isEqualTo('\n');
+ trailingBlank.next();
+ assertThat(trailingBlank.peek()).isEqualTo('\0');
+ trailingBlank.prev();
+ trailingBlank.prev();
+ assertThat(trailingBlank.position()).isEqualTo(trailingStart);
+ assertThat(trailingBlank.peek()).isEqualTo('a');
+ }
+
+ @Test
+ void testFindPrev() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("abXcdX", null)));
+ Position start = scanner.position();
+ assertThat(scanner.findPrev('a')).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.findPrev(c -> c == 'a')).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(start);
+
+ scanner.next();
+ scanner.next();
+ Position firstX = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('X');
+ assertThat(scanner.findPrev('X')).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(firstX);
+
+ scanner.next();
+ scanner.next();
+ // Between the two matches: the nearer (earlier) X, not one after the cursor.
+ assertThat(scanner.peek()).isEqualTo('d');
+ assertThat(scanner.findPrev('X')).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(firstX);
+
+ scanner.next();
+ scanner.next();
+ scanner.next();
+ Position secondX = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('X');
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ // Match immediately before EOF counts the step off the sentinel.
+ assertThat(scanner.findPrev('X')).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(secondX);
+ assertThat(scanner.peek()).isEqualTo('X');
+
+ scanner.next();
+ assertThat(scanner.findPrev(c -> c == 'X')).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(secondX);
+
+ assertThat(scanner.findPrev('z')).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+ assertThat(scanner.peek()).isEqualTo('a');
+
+ scanner.setPosition(secondX);
+ scanner.next();
+ assertThat(scanner.findPrev(Scanner.END)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+ scanner.setPosition(secondX);
+ scanner.next();
+ assertThat(scanner.findPrev(c -> c == Scanner.END)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(start);
+
+ CharMatcher any = c -> true;
+ scanner.setPosition(secondX);
+ assertThat(scanner.findPrev(any)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(secondX);
+ scanner.next();
+ assertThat(scanner.findPrev(any)).isEqualTo(1);
+ assertThat(scanner.position()).isEqualTo(secondX);
+ assertThat(scanner.peek()).isEqualTo('X');
+
+ Scanner lines =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("ab", null), SourceLine.of("c", null))));
+ lines.next();
+ lines.next();
+ Position newline = lines.position();
+ lines.next();
+ lines.next();
+ assertThat(lines.peek()).isEqualTo('\0');
+ assertThat(lines.findPrev('\n')).isEqualTo(2);
+ assertThat(lines.position()).isEqualTo(newline);
+ assertThat(lines.peek()).isEqualTo('\n');
+ assertThat(lines.findPrev(c -> c == '\n')).isEqualTo(0);
+ assertThat(lines.position()).isEqualTo(newline);
+
+ Scanner empty = Scanner.of(SourceLines.empty());
+ Position emptyPos = empty.position();
+ assertThat(empty.findPrev('a')).isEqualTo(-1);
+ assertThat(empty.findPrev(Scanner.END)).isEqualTo(-1);
+ assertThat(empty.findPrev(any)).isEqualTo(-1);
+ assertThat(empty.findPrev(c -> c == Scanner.END)).isEqualTo(-1);
+ assertThat(empty.position()).isEqualTo(emptyPos);
+ assertThat(empty.peek()).isEqualTo('\0');
+
+ Scanner text = Scanner.of(SourceLines.of(SourceLine.of("a", null)));
+ text.next();
+ assertThat(text.findPrev(any)).isEqualTo(1);
+ assertThat(text.peek()).isEqualTo('a');
+ }
+
+ @Test
+ void testFindPrevBounds() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("aXbYc", null)));
+ Position start = scanner.position();
+ scanner.next();
+ Position x = scanner.position();
+ scanner.next();
+ scanner.next();
+ Position y = scanner.position();
+ scanner.next();
+ scanner.next();
+ Position eof = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\0');
+
+ // Inclusive bound on the same line, including a match sitting on the bound.
+ assertThat(scanner.findPrev('Y', x)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(y);
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('Y', y)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.peek()).isEqualTo('Y');
+
+ // A match before the bound is excluded; a miss stays on the bound.
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('X', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.peek()).isEqualTo('Y');
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('z', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+
+ // Start and bound are the same position: only that character is checked.
+ assertThat(scanner.findPrev('Y', y)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.findPrev('X', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev('c', eof)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(eof);
+ assertThat(scanner.peek()).isEqualTo('\0');
+
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev(c -> c == 'Y', x)).isEqualTo(2);
+ assertThat(scanner.position()).isEqualTo(y);
+ scanner.setPosition(eof);
+ assertThat(scanner.findPrev(c -> c == 'X', y)).isEqualTo(-1);
+ assertThat(scanner.position()).isEqualTo(y);
+ assertThat(scanner.findPrev(c -> c == 'Y', y)).isEqualTo(0);
+
+ // Across lines the bound is still inclusive and stops the search.
+ Scanner lines =
+ Scanner.of(
+ SourceLines.of(
+ List.of(SourceLine.of("aXb", null), SourceLine.of("cYd", null))));
+ Position lineStart = lines.position();
+ lines.next();
+ Position lineX = lines.position();
+ lines.next();
+ lines.next();
+ Position newline = lines.position();
+ assertThat(lines.peek()).isEqualTo('\n');
+ lines.next();
+ Position secondStart = lines.position();
+ lines.next();
+ Position lineY = lines.position();
+ lines.next();
+ lines.next();
+ Position linesEof = lines.position();
+
+ assertThat(lines.findPrev('Y', newline)).isEqualTo(2);
+ assertThat(lines.position()).isEqualTo(lineY);
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('\n', lineStart)).isEqualTo(4);
+ assertThat(lines.position()).isEqualTo(newline);
+ assertThat(lines.peek()).isEqualTo('\n');
+
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('X', newline)).isEqualTo(-1);
+ assertThat(lines.position()).isEqualTo(newline);
+ assertThat(lines.peek()).isEqualTo('\n');
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('a', secondStart)).isEqualTo(-1);
+ assertThat(lines.position()).isEqualTo(secondStart);
+
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev('a', lineStart)).isEqualTo(7);
+ assertThat(lines.position()).isEqualTo(lineStart);
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev(c -> c == '\n', newline)).isEqualTo(4);
+ assertThat(lines.position()).isEqualTo(newline);
+ lines.setPosition(linesEof);
+ assertThat(lines.findPrev(c -> c == 'X', secondStart)).isEqualTo(-1);
+ assertThat(lines.position()).isEqualTo(secondStart);
+
+ // Future and out-of-range bounds throw before the cursor moves.
+ lines.setPosition(lineX);
+ assertThat(lines.peek()).isEqualTo('X');
+ assertThatThrownBy(() -> lines.findPrev('a', lineY))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev(c -> true, linesEof))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(-1, 0)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(0, -1)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(0, 4)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> lines.findPrev('a', new Position(2, 0)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThat(lines.position()).isEqualTo(lineX);
+ assertThat(lines.peek()).isEqualTo('X');
+
+ Scanner empty = Scanner.of(SourceLines.empty());
+ Position origin = empty.position();
+ assertThat(empty.findPrev('x', origin)).isEqualTo(-1);
+ assertThat(empty.findPrev(c -> true, origin)).isEqualTo(-1);
+ assertThat(empty.findPrev(Scanner.END, origin)).isEqualTo(-1);
+ assertThat(empty.position()).isEqualTo(origin);
+ assertThatThrownBy(() -> empty.findPrev('x', new Position(0, 1)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThatThrownBy(() -> empty.findPrev(c -> true, new Position(1, 0)))
+ .isInstanceOf(IllegalArgumentException.class);
+ assertThat(empty.position()).isEqualTo(origin);
+ assertThat(empty.peek()).isEqualTo('\0');
+
+ // Same-line bound equal to the start still accepts the character there.
+ scanner.setPosition(start);
+ assertThat(scanner.findPrev('a', start)).isEqualTo(0);
+ assertThat(scanner.position()).isEqualTo(start);
+ }
+
+ @Test
+ void testCodeUnitMovement() {
+ Scanner scanner = Scanner.of(SourceLines.of(SourceLine.of("A\uD83D\uDE0AZ", null)));
+ assertThat(scanner.peek()).isEqualTo('A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('A');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\0');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\uD83D');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('Z');
+ // Previous code point reassembles the surrogate pair behind this code unit.
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo(128522);
+ scanner.next();
+ assertThat(scanner.hasNext()).isFalse();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\0');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('Z');
+
+ scanner.prev();
+ assertThat(scanner.peek()).isEqualTo('Z');
+ scanner.prev();
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('\uD83D');
+ scanner.prev();
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A');
+
+ scanner.next();
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ scanner.prev();
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+
+ while (scanner.hasNext()) {
+ scanner.next();
+ }
+ assertThat(scanner.findPrev('\uDE0A')).isEqualTo(2);
+ assertThat(scanner.peek()).isEqualTo('\uDE0A');
+ assertThat(scanner.peekCodePoint()).isEqualTo('\uDE0A');
+ while (scanner.hasNext()) {
+ scanner.next();
+ }
+ assertThat(scanner.findPrev('\uD83D')).isEqualTo(3);
+ assertThat(scanner.peek()).isEqualTo('\uD83D');
+ assertThat(scanner.peekCodePoint()).isEqualTo(128522);
+ assertThat(scanner.peekPreviousCodePoint()).isEqualTo('A');
+ }
+
+ @Test
+ void testFindPrevSource() {
+ Scanner scanner =
+ new Scanner(
+ List.of(
+ SourceLine.of("ab", SourceSpan.of(10, 3, 13, 2)),
+ SourceLine.of("cde", SourceSpan.of(11, 4, 20, 3))),
+ 0,
+ 0);
+ while (scanner.hasNext()) {
+ scanner.next();
+ }
+ Position end = scanner.position();
+ assertThat(scanner.peek()).isEqualTo('\0');
+ assertThat(scanner.findPrev('b')).isEqualTo(5);
+ assertThat(scanner.peek()).isEqualTo('b');
+ assertSourceLines(
+ scanner.getSource(scanner.position(), end),
+ "b\ncde",
+ SourceSpan.of(10, 4, 14, 1),
+ SourceSpan.of(11, 4, 20, 3));
+ }
+
@Test
void nextString() {
Scanner scanner =