Buffered parsing: add parse() method, and return Tokens from take_while() and take()

.github/workflows/ci.yml

@@ -57,7 +57,7 @@ jobs:
         uses: Swatinem/[email protected]
 
       - name: Check internal documentation links
-        run: RUSTDOCFLAGS="--deny broken_intra_doc_links" cargo doc --verbose --workspace --no-deps --document-private-items
+        run: RUSTDOCFLAGS="--deny rustdoc::broken_intra_doc_links" cargo doc --verbose --workspace --no-deps --document-private-items
 
   tests:
     name: Run tests

README.md

@@ -68,25 +68,24 @@ fn parse_op(t: &mut impl Tokens<Item=char>) -> Option<Op> {
 
 // We also get other useful functions..
 fn parse_digits(t: &mut impl Tokens<Item=char>) -> Option<u32> {
-    let s: String = t
-        .tokens_while(|c| c.is_digit(10))
-        .collect();
-    s.parse().ok()
+    t.take_while(|c| c.is_digit(10))
+     .parse::<u32, String>()
+     .ok()
 }
 
 // As well as combinator functions like `sep_by_all` and `surrounded_by`..
 let op_or_digit = tokens.sep_by_all(
     |t| t.surrounded_by(
         |t| parse_digits(t).map(OpOrDigit::Digit),
-        |t| { t.skip_tokens_while(|c| c.is_ascii_whitespace()); }
+        |t| { t.skip_while(|c| c.is_ascii_whitespace()); }
     ),
     |t| parse_op(t).map(OpOrDigit::Op)
 );
 
 // Now we've parsed our input into OpOrDigits, let's calculate the result..
 let mut current_op = Op::Plus;
 let mut current_digit = 0;
-for d in op_or_digit {
+for d in op_or_digit.into_iter() {
     match d {
         OpOrDigit::Op(op) => {
             current_op = op

examples/json.rs

@@ -316,7 +316,7 @@ fn number(toks: &mut impl Tokens<Item = char>) -> Option<Result<f64, Error>> {
 
     // Now, skip over digits. If none, then this isn't an number unless
     // the char above was a digit too.
-    let num_skipped = toks.skip_tokens_while(|c| c.is_numeric());
+    let num_skipped = toks.skip_while(|c| c.is_numeric());
     if num_skipped == 0 && !is_fst_number {
         let loc = toks.location();
         return Some(Err(ErrorKind::DigitExpectedNext.at(start, loc)));
@@ -328,7 +328,7 @@ fn number(toks: &mut impl Tokens<Item = char>) -> Option<Result<f64, Error>> {
         if !toks.token('.') {
             return None;
         }
-        let num_digits = toks.skip_tokens_while(|c| c.is_numeric());
+        let num_digits = toks.skip_while(|c| c.is_numeric());
         if num_digits == 0 {
             let loc = toks.location();
             Some(Err(ErrorKind::DigitExpectedNext.at(start.clone(), loc)))
@@ -342,14 +342,20 @@ fn number(toks: &mut impl Tokens<Item = char>) -> Option<Result<f64, Error>> {
         return Some(Err(e));
     }
 
-    // If we get this far, we saw a valid number. Just let Rust parse it for us..
+    // If we get this far, we saw a valid number. Just let Rust parse it for us.
+    // We use a `String` buffer to accumulate the slice of tokens before parsing
+    // by default, although `StrTokens` is optimised to avoid using it in this case.
     let end = toks.location();
-    let n_str: String = toks.slice(start, end).as_iter().collect();
-    Some(Ok(n_str.parse().expect("valid number expected here")))
+    let n = toks
+        .slice(start, end)
+        .parse::<f64, String>()
+        .expect("valid number expected here");
+
+    Some(Ok(n))
 }
 
 fn skip_whitespace(toks: &mut impl Tokens<Item = char>) {
-    toks.skip_tokens_while(|c| c.is_ascii_whitespace());
+    toks.skip_while(|c| c.is_ascii_whitespace());
 }
 
 fn field_separator(toks: &mut impl Tokens<Item = char>) -> bool {

src/chars.rs

@@ -0,0 +1,27 @@
+//! Helper functions for working with `Tokens<Item=char>`.
+
+use crate::Tokens;
+
+/// Parses a line ending of either `\n` (like on linux)  or `\r\n` (like on windows).
+/// Returns a static string equal to the line ending parsed, or `None` if no line
+/// ending is seen at this location.
+///
+/// # Example
+///
+/// ```
+/// use yap::{Tokens, IntoTokens, chars};
+///
+/// let mut toks = "\r\n abc".into_tokens();
+///
+/// assert_eq!(chars::line_ending(&mut toks), Some("\r\n"));
+/// assert_eq!(toks.remaining(), " abc");
+///
+/// let mut toks = "\n abc".into_tokens();
+///
+/// assert_eq!(chars::line_ending(&mut toks), Some("\n"));
+/// assert_eq!(toks.remaining(), " abc");
+/// ```
+pub fn line_ending<T: Tokens<Item = char>>(t: &mut T) -> Option<&'static str> {
+    t.optional(|t| t.token('\n').then_some("\n"))
+        .or_else(|| t.optional(|t| t.tokens("\r\n".chars()).then_some("\r\n")))
+}

src/lib.rs

@@ -65,25 +65,24 @@ fn parse_op(t: &mut impl Tokens<Item=char>) -> Option<Op> {
 
 // We also get other useful functions..
 fn parse_digits(t: &mut impl Tokens<Item=char>) -> Option<u32> {
-    let s: String = t
-        .tokens_while(|c| c.is_digit(10))
-        .collect();
-    s.parse().ok()
+    t.take_while(|c| c.is_digit(10))
+     .parse::<u32, String>()
+     .ok()
 }
 
 // As well as combinator functions like `sep_by_all` and `surrounded_by`..
 let op_or_digit = tokens.sep_by_all(
     |t| t.surrounded_by(
         |t| parse_digits(t).map(OpOrDigit::Digit),
-        |t| { t.skip_tokens_while(|c| c.is_ascii_whitespace()); }
+        |t| { t.skip_while(|c| c.is_ascii_whitespace()); }
     ),
     |t| parse_op(t).map(OpOrDigit::Op)
 );
 
 // Now we've parsed our input into OpOrDigits, let's calculate the result..
 let mut current_op = Op::Plus;
 let mut current_digit = 0;
-for d in op_or_digit {
+for d in op_or_digit.into_iter() {
     match d {
         OpOrDigit::Op(op) => {
             current_op = op
@@ -115,5 +114,6 @@ assert_eq!(remaining, ",foobar");
 mod one_of;
 mod tokens;
 
+pub mod chars;
 pub mod types;
 pub use tokens::{IntoTokens, TokenLocation, Tokens};