winnow/token/
mod.rs

1//! Parsers extracting tokens from the stream
2
3#[cfg(all(test, feature = "ascii"))]
4mod tests;
5
6use crate::combinator::trace;
7use crate::combinator::DisplayDebug;
8use crate::error::Needed;
9use crate::error::ParserError;
10use crate::stream::Range;
11use crate::stream::{Compare, CompareResult, ContainsToken, FindSlice, Stream};
12use crate::stream::{StreamIsPartial, ToUsize};
13use crate::Parser;
14use crate::Result;
15use core::result::Result::Ok;
16
17/// Matches one token
18///
19/// *Complete version*: Will return an error if there's not enough input data.
20///
21/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
22///
23/// # Effective Signature
24///
25/// Assuming you are parsing a `&str` [Stream]:
26/// ```rust
27/// # use winnow::prelude::*;;
28/// pub fn any(input: &mut &str) -> ModalResult<char>
29/// # {
30/// #     winnow::token::any.parse_next(input)
31/// # }
32/// ```
33///
34/// # Example
35///
36/// ```rust
37/// # use winnow::{token::any, error::ErrMode, error::ContextError};
38/// # use winnow::prelude::*;
39/// fn parser(input: &mut &str) -> ModalResult<char> {
40///     any.parse_next(input)
41/// }
42///
43/// assert_eq!(parser.parse_peek("abc"), Ok(("bc",'a')));
44/// assert!(parser.parse_peek("").is_err());
45/// ```
46///
47/// ```rust
48/// # use winnow::{token::any, error::ErrMode, error::ContextError, error::Needed};
49/// # use winnow::prelude::*;
50/// # use winnow::Partial;
51/// assert_eq!(any::<_, ErrMode<ContextError>>.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"),'a')));
52/// assert_eq!(any::<_, ErrMode<ContextError>>.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
53/// ```
54#[inline(always)]
55#[doc(alias = "token")]
56pub fn any<Input, Error>(input: &mut Input) -> Result<<Input as Stream>::Token, Error>
57where
58    Input: StreamIsPartial + Stream,
59    Error: ParserError<Input>,
60{
61    trace("any", move |input: &mut Input| {
62        if <Input as StreamIsPartial>::is_partial_supported() {
63            any_::<_, _, true>(input)
64        } else {
65            any_::<_, _, false>(input)
66        }
67    })
68    .parse_next(input)
69}
70
71fn any_<I, E: ParserError<I>, const PARTIAL: bool>(input: &mut I) -> Result<<I as Stream>::Token, E>
72where
73    I: StreamIsPartial,
74    I: Stream,
75{
76    input.next_token().ok_or_else(|| {
77        if PARTIAL && input.is_partial() {
78            ParserError::incomplete(input, Needed::new(1))
79        } else {
80            ParserError::from_input(input)
81        }
82    })
83}
84
85/// Recognizes a literal
86///
87/// The input data will be compared to the literal combinator's argument and will return the part of
88/// the input that matches the argument
89///
90/// It will return `Err(ErrMode::Backtrack(_))` if the input doesn't match the literal
91///
92/// <div class="warning">
93///
94/// **Note:** [`Parser`] is implemented for strings and byte strings as a convenience (complete
95/// only)
96///
97/// </div>
98///
99/// # Effective Signature
100///
101/// Assuming you are parsing a `&str` [Stream]:
102/// ```rust
103/// # use winnow::prelude::*;;
104/// # use winnow::error::ContextError;
105/// pub fn literal(literal: &str) -> impl Parser<&str, &str, ContextError>
106/// # {
107/// #     winnow::token::literal(literal)
108/// # }
109/// ```
110///
111/// # Example
112/// ```rust
113/// # use winnow::prelude::*;
114/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
115/// #
116/// fn parser<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
117///   "Hello".parse_next(s)
118/// }
119///
120/// assert_eq!(parser.parse_peek("Hello, World!"), Ok((", World!", "Hello")));
121/// assert!(parser.parse_peek("Something").is_err());
122/// assert!(parser.parse_peek("").is_err());
123/// ```
124///
125/// ```rust
126/// # use winnow::prelude::*;
127/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
128/// # use winnow::Partial;
129///
130/// fn parser<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
131///   "Hello".parse_next(s)
132/// }
133///
134/// assert_eq!(parser.parse_peek(Partial::new("Hello, World!")), Ok((Partial::new(", World!"), "Hello")));
135/// assert!(parser.parse_peek(Partial::new("Something")).is_err());
136/// assert!(parser.parse_peek(Partial::new("S")).is_err());
137/// assert_eq!(parser.parse_peek(Partial::new("H")), Err(ErrMode::Incomplete(Needed::Unknown)));
138/// ```
139///
140/// ```rust
141/// # #[cfg(feature = "ascii")] {
142/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
143/// # use winnow::prelude::*;
144/// use winnow::token::literal;
145/// use winnow::ascii::Caseless;
146///
147/// fn parser<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
148///   literal(Caseless("hello")).parse_next(s)
149/// }
150///
151/// assert_eq!(parser.parse_peek("Hello, World!"), Ok((", World!", "Hello")));
152/// assert_eq!(parser.parse_peek("hello, World!"), Ok((", World!", "hello")));
153/// assert_eq!(parser.parse_peek("HeLlO, World!"), Ok((", World!", "HeLlO")));
154/// assert!(parser.parse_peek("Something").is_err());
155/// assert!(parser.parse_peek("").is_err());
156/// # }
157/// ```
158#[inline(always)]
159#[doc(alias = "tag")]
160#[doc(alias = "bytes")]
161#[doc(alias = "just")]
162pub fn literal<Literal, Input, Error>(
163    literal: Literal,
164) -> impl Parser<Input, <Input as Stream>::Slice, Error>
165where
166    Input: StreamIsPartial + Stream + Compare<Literal>,
167    Literal: Clone + core::fmt::Debug,
168    Error: ParserError<Input>,
169{
170    trace(DisplayDebug(literal.clone()), move |i: &mut Input| {
171        let t = literal.clone();
172        if <Input as StreamIsPartial>::is_partial_supported() {
173            literal_::<_, _, _, true>(i, t)
174        } else {
175            literal_::<_, _, _, false>(i, t)
176        }
177    })
178}
179
180fn literal_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
181    i: &mut I,
182    t: T,
183) -> Result<<I as Stream>::Slice, Error>
184where
185    I: StreamIsPartial,
186    I: Stream + Compare<T>,
187    T: core::fmt::Debug,
188{
189    match i.compare(t) {
190        CompareResult::Ok(len) => Ok(i.next_slice(len)),
191        CompareResult::Incomplete if PARTIAL && i.is_partial() => {
192            Err(ParserError::incomplete(i, Needed::Unknown))
193        }
194        CompareResult::Incomplete | CompareResult::Error => Err(ParserError::from_input(i)),
195    }
196}
197
198/// Recognize a token that matches a [set of tokens][ContainsToken]
199///
200/// <div class="warning">
201///
202/// **Note:** [`Parser`] is implemented as a convenience (complete
203/// only) for
204/// - `u8`
205/// - `char`
206///
207/// </div>
208///
209/// *Complete version*: Will return an error if there's not enough input data.
210///
211/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
212///
213/// # Effective Signature
214///
215/// Assuming you are parsing a `&str` [Stream]:
216/// ```rust
217/// # use winnow::prelude::*;;
218/// # use winnow::stream::ContainsToken;
219/// # use winnow::error::ContextError;
220/// pub fn one_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
221/// # {
222/// #     winnow::token::one_of(set)
223/// # }
224/// ```
225///
226/// # Example
227///
228/// ```rust
229/// # use winnow::prelude::*;
230/// # use winnow::{error::ErrMode, error::ContextError};
231/// # use winnow::token::one_of;
232/// assert_eq!(one_of::<_, _, ContextError>(['a', 'b', 'c']).parse_peek("b"), Ok(("", 'b')));
233/// assert!(one_of::<_, _, ContextError>('a').parse_peek("bc").is_err());
234/// assert!(one_of::<_, _, ContextError>('a').parse_peek("").is_err());
235///
236/// fn parser_fn(i: &mut &str) -> ModalResult<char> {
237///     one_of(|c| c == 'a' || c == 'b').parse_next(i)
238/// }
239/// assert_eq!(parser_fn.parse_peek("abc"), Ok(("bc", 'a')));
240/// assert!(parser_fn.parse_peek("cd").is_err());
241/// assert!(parser_fn.parse_peek("").is_err());
242/// ```
243///
244/// ```rust
245/// # use winnow::prelude::*;
246/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
247/// # use winnow::Partial;
248/// # use winnow::token::one_of;
249/// assert_eq!(one_of::<_, _, ErrMode<ContextError>>(['a', 'b', 'c']).parse_peek(Partial::new("b")), Ok((Partial::new(""), 'b')));
250/// assert!(one_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("bc")).is_err());
251/// assert_eq!(one_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
252///
253/// fn parser_fn(i: &mut Partial<&str>) -> ModalResult<char> {
254///     one_of(|c| c == 'a' || c == 'b').parse_next(i)
255/// }
256/// assert_eq!(parser_fn.parse_peek(Partial::new("abc")), Ok((Partial::new("bc"), 'a')));
257/// assert!(parser_fn.parse_peek(Partial::new("cd")).is_err());
258/// assert_eq!(parser_fn.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
259/// ```
260#[inline(always)]
261#[doc(alias = "char")]
262#[doc(alias = "token")]
263#[doc(alias = "satisfy")]
264pub fn one_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
265where
266    Input: StreamIsPartial + Stream,
267    <Input as Stream>::Token: Clone,
268    Set: ContainsToken<<Input as Stream>::Token>,
269    Error: ParserError<Input>,
270{
271    trace(
272        "one_of",
273        any.verify(move |t: &<Input as Stream>::Token| set.contains_token(t.clone())),
274    )
275}
276
277/// Recognize a token that does not match a [set of tokens][ContainsToken]
278///
279/// *Complete version*: Will return an error if there's not enough input data.
280///
281/// *[Partial version][crate::_topic::partial]*: Will return `Err(winnow::error::ErrMode::Incomplete(_))` if there's not enough input data.
282///
283/// # Effective Signature
284///
285/// Assuming you are parsing a `&str` [Stream]:
286/// ```rust
287/// # use winnow::prelude::*;;
288/// # use winnow::stream::ContainsToken;
289/// # use winnow::error::ContextError;
290/// pub fn none_of<'i>(set: impl ContainsToken<char>) -> impl Parser<&'i str, char, ContextError>
291/// # {
292/// #     winnow::token::none_of(set)
293/// # }
294/// ```
295///
296/// # Example
297///
298/// ```rust
299/// # use winnow::{error::ErrMode, error::ContextError};
300/// # use winnow::prelude::*;
301/// # use winnow::token::none_of;
302/// assert_eq!(none_of::<_, _, ContextError>(['a', 'b', 'c']).parse_peek("z"), Ok(("", 'z')));
303/// assert!(none_of::<_, _, ContextError>(['a', 'b']).parse_peek("a").is_err());
304/// assert!(none_of::<_, _, ContextError>('a').parse_peek("").is_err());
305/// ```
306///
307/// ```rust
308/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
309/// # use winnow::prelude::*;
310/// # use winnow::Partial;
311/// # use winnow::token::none_of;
312/// assert_eq!(none_of::<_, _, ErrMode<ContextError>>(['a', 'b', 'c']).parse_peek(Partial::new("z")), Ok((Partial::new(""), 'z')));
313/// assert!(none_of::<_, _, ErrMode<ContextError>>(['a', 'b']).parse_peek(Partial::new("a")).is_err());
314/// assert_eq!(none_of::<_, _, ErrMode<ContextError>>('a').parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
315/// ```
316#[inline(always)]
317pub fn none_of<Input, Set, Error>(set: Set) -> impl Parser<Input, <Input as Stream>::Token, Error>
318where
319    Input: StreamIsPartial + Stream,
320    <Input as Stream>::Token: Clone,
321    Set: ContainsToken<<Input as Stream>::Token>,
322    Error: ParserError<Input>,
323{
324    trace(
325        "none_of",
326        any.verify(move |t: &<Input as Stream>::Token| !set.contains_token(t.clone())),
327    )
328}
329
330/// Recognize the longest input slice (bound by `occurrences`) that matches a [set of tokens][ContainsToken]
331///
332/// It will return an `ErrMode::Backtrack(_)` if the set of tokens wasn't met or is out
333/// of `occurrences` range.
334///
335/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if a member of the set of tokens reaches the end of the input or is too short.
336///
337/// To take a series of tokens, use [`repeat`][crate::combinator::repeat] to [`Accumulate`][crate::stream::Accumulate] into a `()` and then [`Parser::take`].
338///
339/// # Effective Signature
340///
341/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
342/// ```rust
343/// # use std::ops::RangeFrom;
344/// # use winnow::prelude::*;
345/// # use winnow::stream::ContainsToken;
346/// # use winnow::error::ContextError;
347/// pub fn take_while<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
348/// # {
349/// #     winnow::token::take_while(occurrences, set)
350/// # }
351/// ```
352///
353/// # Example
354///
355/// Zero or more tokens:
356/// ```rust
357/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
358/// # use winnow::prelude::*;
359/// use winnow::token::take_while;
360/// use winnow::stream::AsChar;
361///
362/// fn alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
363///   take_while(0.., AsChar::is_alpha).parse_next(s)
364/// }
365///
366/// assert_eq!(alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
367/// assert_eq!(alpha.parse_peek(b"12345"), Ok((&b"12345"[..], &b""[..])));
368/// assert_eq!(alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
369/// assert_eq!(alpha.parse_peek(b""), Ok((&b""[..], &b""[..])));
370/// ```
371///
372/// ```rust
373/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
374/// # use winnow::prelude::*;
375/// # use winnow::Partial;
376/// use winnow::token::take_while;
377/// use winnow::stream::AsChar;
378///
379/// fn alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
380///   take_while(0.., AsChar::is_alpha).parse_next(s)
381/// }
382///
383/// assert_eq!(alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
384/// assert_eq!(alpha.parse_peek(Partial::new(b"12345")), Ok((Partial::new(&b"12345"[..]), &b""[..])));
385/// assert_eq!(alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
386/// assert_eq!(alpha.parse_peek(Partial::new(b"")), Err(ErrMode::Incomplete(Needed::new(1))));
387/// ```
388///
389/// One or more tokens:
390/// ```rust
391/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
392/// # use winnow::prelude::*;
393/// use winnow::token::take_while;
394/// use winnow::stream::AsChar;
395///
396/// fn alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
397///   take_while(1.., AsChar::is_alpha).parse_next(s)
398/// }
399///
400/// assert_eq!(alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
401/// assert_eq!(alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
402/// assert!(alpha.parse_peek(b"12345").is_err());
403///
404/// fn hex<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
405///   take_while(1.., ('0'..='9', 'A'..='F')).parse_next(s)
406/// }
407///
408/// assert_eq!(hex.parse_peek("123 and voila"), Ok((" and voila", "123")));
409/// assert_eq!(hex.parse_peek("DEADBEEF and others"), Ok((" and others", "DEADBEEF")));
410/// assert_eq!(hex.parse_peek("BADBABEsomething"), Ok(("something", "BADBABE")));
411/// assert_eq!(hex.parse_peek("D15EA5E"), Ok(("", "D15EA5E")));
412/// assert!(hex.parse_peek("").is_err());
413/// ```
414///
415/// ```rust
416/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
417/// # use winnow::prelude::*;
418/// # use winnow::Partial;
419/// use winnow::token::take_while;
420/// use winnow::stream::AsChar;
421///
422/// fn alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
423///   take_while(1.., AsChar::is_alpha).parse_next(s)
424/// }
425///
426/// assert_eq!(alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
427/// assert_eq!(alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
428/// assert!(alpha.parse_peek(Partial::new(b"12345")).is_err());
429///
430/// fn hex<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
431///   take_while(1.., ('0'..='9', 'A'..='F')).parse_next(s)
432/// }
433///
434/// assert_eq!(hex.parse_peek(Partial::new("123 and voila")), Ok((Partial::new(" and voila"), "123")));
435/// assert_eq!(hex.parse_peek(Partial::new("DEADBEEF and others")), Ok((Partial::new(" and others"), "DEADBEEF")));
436/// assert_eq!(hex.parse_peek(Partial::new("BADBABEsomething")), Ok((Partial::new("something"), "BADBABE")));
437/// assert_eq!(hex.parse_peek(Partial::new("D15EA5E")), Err(ErrMode::Incomplete(Needed::new(1))));
438/// assert_eq!(hex.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
439/// ```
440///
441/// Arbitrary amount of tokens:
442/// ```rust
443/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
444/// # use winnow::prelude::*;
445/// use winnow::token::take_while;
446/// use winnow::stream::AsChar;
447///
448/// fn short_alpha<'i>(s: &mut &'i [u8]) -> ModalResult<&'i [u8]> {
449///   take_while(3..=6, AsChar::is_alpha).parse_next(s)
450/// }
451///
452/// assert_eq!(short_alpha.parse_peek(b"latin123"), Ok((&b"123"[..], &b"latin"[..])));
453/// assert_eq!(short_alpha.parse_peek(b"lengthy"), Ok((&b"y"[..], &b"length"[..])));
454/// assert_eq!(short_alpha.parse_peek(b"latin"), Ok((&b""[..], &b"latin"[..])));
455/// assert!(short_alpha.parse_peek(b"ed").is_err());
456/// assert!(short_alpha.parse_peek(b"12345").is_err());
457/// ```
458///
459/// ```rust
460/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
461/// # use winnow::prelude::*;
462/// # use winnow::Partial;
463/// use winnow::token::take_while;
464/// use winnow::stream::AsChar;
465///
466/// fn short_alpha<'i>(s: &mut Partial<&'i [u8]>) -> ModalResult<&'i [u8]> {
467///   take_while(3..=6, AsChar::is_alpha).parse_next(s)
468/// }
469///
470/// assert_eq!(short_alpha.parse_peek(Partial::new(b"latin123")), Ok((Partial::new(&b"123"[..]), &b"latin"[..])));
471/// assert_eq!(short_alpha.parse_peek(Partial::new(b"lengthy")), Ok((Partial::new(&b"y"[..]), &b"length"[..])));
472/// assert_eq!(short_alpha.parse_peek(Partial::new(b"latin")), Err(ErrMode::Incomplete(Needed::new(1))));
473/// assert_eq!(short_alpha.parse_peek(Partial::new(b"ed")), Err(ErrMode::Incomplete(Needed::new(1))));
474/// assert!(short_alpha.parse_peek(Partial::new(b"12345")).is_err());
475/// ```
476#[inline(always)]
477#[doc(alias = "is_a")]
478#[doc(alias = "take_while0")]
479#[doc(alias = "take_while1")]
480pub fn take_while<Set, Input, Error>(
481    occurrences: impl Into<Range>,
482    set: Set,
483) -> impl Parser<Input, <Input as Stream>::Slice, Error>
484where
485    Input: StreamIsPartial + Stream,
486    Set: ContainsToken<<Input as Stream>::Token>,
487    Error: ParserError<Input>,
488{
489    let Range {
490        start_inclusive,
491        end_inclusive,
492    } = occurrences.into();
493    trace("take_while", move |i: &mut Input| {
494        match (start_inclusive, end_inclusive) {
495            (0, None) => {
496                if <Input as StreamIsPartial>::is_partial_supported() {
497                    take_till0::<_, _, _, true>(i, |c| !set.contains_token(c))
498                } else {
499                    take_till0::<_, _, _, false>(i, |c| !set.contains_token(c))
500                }
501            }
502            (1, None) => {
503                if <Input as StreamIsPartial>::is_partial_supported() {
504                    take_till1::<_, _, _, true>(i, |c| !set.contains_token(c))
505                } else {
506                    take_till1::<_, _, _, false>(i, |c| !set.contains_token(c))
507                }
508            }
509            (start, end) => {
510                let end = end.unwrap_or(usize::MAX);
511                if <Input as StreamIsPartial>::is_partial_supported() {
512                    take_till_m_n::<_, _, _, true>(i, start, end, |c| !set.contains_token(c))
513                } else {
514                    take_till_m_n::<_, _, _, false>(i, start, end, |c| !set.contains_token(c))
515                }
516            }
517        }
518    })
519}
520
521fn take_till0<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
522    input: &mut I,
523    predicate: P,
524) -> Result<<I as Stream>::Slice, E>
525where
526    P: Fn(I::Token) -> bool,
527{
528    let offset = match input.offset_for(predicate) {
529        Some(offset) => offset,
530        None if PARTIAL && input.is_partial() => {
531            return Err(ParserError::incomplete(input, Needed::new(1)));
532        }
533        None => input.eof_offset(),
534    };
535    Ok(input.next_slice(offset))
536}
537
538fn take_till1<P, I: StreamIsPartial + Stream, E: ParserError<I>, const PARTIAL: bool>(
539    input: &mut I,
540    predicate: P,
541) -> Result<<I as Stream>::Slice, E>
542where
543    P: Fn(I::Token) -> bool,
544{
545    let offset = match input.offset_for(predicate) {
546        Some(offset) => offset,
547        None if PARTIAL && input.is_partial() => {
548            return Err(ParserError::incomplete(input, Needed::new(1)));
549        }
550        None => input.eof_offset(),
551    };
552    if offset == 0 {
553        Err(ParserError::from_input(input))
554    } else {
555        Ok(input.next_slice(offset))
556    }
557}
558
559fn take_till_m_n<P, I, Error: ParserError<I>, const PARTIAL: bool>(
560    input: &mut I,
561    m: usize,
562    n: usize,
563    predicate: P,
564) -> Result<<I as Stream>::Slice, Error>
565where
566    I: StreamIsPartial,
567    I: Stream,
568    P: Fn(I::Token) -> bool,
569{
570    if n < m {
571        return Err(ParserError::assert(
572            input,
573            "`occurrences` should be ascending, rather than descending",
574        ));
575    }
576
577    let mut final_count = 0;
578    for (processed, (offset, token)) in input.iter_offsets().enumerate() {
579        if predicate(token) {
580            if processed < m {
581                return Err(ParserError::from_input(input));
582            } else {
583                return Ok(input.next_slice(offset));
584            }
585        } else {
586            if processed == n {
587                return Ok(input.next_slice(offset));
588            }
589            final_count = processed + 1;
590        }
591    }
592    if PARTIAL && input.is_partial() {
593        if final_count == n {
594            Ok(input.finish())
595        } else {
596            let needed = if m > input.eof_offset() {
597                m - input.eof_offset()
598            } else {
599                1
600            };
601            Err(ParserError::incomplete(input, Needed::new(needed)))
602        }
603    } else {
604        if m <= final_count {
605            Ok(input.finish())
606        } else {
607            Err(ParserError::from_input(input))
608        }
609    }
610}
611
612/// Recognize the longest input slice  (bound by `occurrences`) till a member of a [set of tokens][ContainsToken] is found.
613///
614/// It doesn't consume the terminating token from the set.
615///
616/// It will return an `ErrMode::Backtrack(_)` if the set of tokens wasn't met or is out
617/// of `occurrences` range.
618///
619/// *[Partial version][crate::_topic::partial]* will return a `ErrMode::Incomplete(Needed::new(1))` if the match reaches the
620/// end of input or if there was not match.
621///
622/// See also
623/// - [`take_until`] for recognizing up-to a [`literal`] (w/ optional simd optimizations)
624/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
625///
626/// # Effective Signature
627///
628/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
629/// ```rust
630/// # use std::ops::RangeFrom;
631/// # use winnow::prelude::*;
632/// # use winnow::stream::ContainsToken;
633/// # use winnow::error::ContextError;
634/// pub fn take_till<'i>(occurrences: RangeFrom<usize>, set: impl ContainsToken<char>) -> impl Parser<&'i str, &'i str, ContextError>
635/// # {
636/// #     winnow::token::take_till(occurrences, set)
637/// # }
638/// ```
639///
640/// # Example
641///
642/// ```rust
643/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
644/// # use winnow::prelude::*;
645/// use winnow::token::take_till;
646///
647/// fn till_colon<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
648///   take_till(0.., |c| c == ':').parse_next(s)
649/// }
650///
651/// assert_eq!(till_colon.parse_peek("latin:123"), Ok((":123", "latin")));
652/// assert_eq!(till_colon.parse_peek(":empty matched"), Ok((":empty matched", ""))); //allowed
653/// assert_eq!(till_colon.parse_peek("12345"), Ok(("", "12345")));
654/// assert_eq!(till_colon.parse_peek(""), Ok(("", "")));
655/// ```
656///
657/// ```rust
658/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
659/// # use winnow::prelude::*;
660/// # use winnow::Partial;
661/// use winnow::token::take_till;
662///
663/// fn till_colon<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
664///   take_till(0.., |c| c == ':').parse_next(s)
665/// }
666///
667/// assert_eq!(till_colon.parse_peek(Partial::new("latin:123")), Ok((Partial::new(":123"), "latin")));
668/// assert_eq!(till_colon.parse_peek(Partial::new(":empty matched")), Ok((Partial::new(":empty matched"), ""))); //allowed
669/// assert_eq!(till_colon.parse_peek(Partial::new("12345")), Err(ErrMode::Incomplete(Needed::new(1))));
670/// assert_eq!(till_colon.parse_peek(Partial::new("")), Err(ErrMode::Incomplete(Needed::new(1))));
671/// ```
672#[inline(always)]
673#[doc(alias = "is_not")]
674pub fn take_till<Set, Input, Error>(
675    occurrences: impl Into<Range>,
676    set: Set,
677) -> impl Parser<Input, <Input as Stream>::Slice, Error>
678where
679    Input: StreamIsPartial + Stream,
680    Set: ContainsToken<<Input as Stream>::Token>,
681    Error: ParserError<Input>,
682{
683    let Range {
684        start_inclusive,
685        end_inclusive,
686    } = occurrences.into();
687    trace("take_till", move |i: &mut Input| {
688        match (start_inclusive, end_inclusive) {
689            (0, None) => {
690                if <Input as StreamIsPartial>::is_partial_supported() {
691                    take_till0::<_, _, _, true>(i, |c| set.contains_token(c))
692                } else {
693                    take_till0::<_, _, _, false>(i, |c| set.contains_token(c))
694                }
695            }
696            (1, None) => {
697                if <Input as StreamIsPartial>::is_partial_supported() {
698                    take_till1::<_, _, _, true>(i, |c| set.contains_token(c))
699                } else {
700                    take_till1::<_, _, _, false>(i, |c| set.contains_token(c))
701                }
702            }
703            (start, end) => {
704                let end = end.unwrap_or(usize::MAX);
705                if <Input as StreamIsPartial>::is_partial_supported() {
706                    take_till_m_n::<_, _, _, true>(i, start, end, |c| set.contains_token(c))
707                } else {
708                    take_till_m_n::<_, _, _, false>(i, start, end, |c| set.contains_token(c))
709                }
710            }
711        }
712    })
713}
714
715/// Recognize an input slice containing the first N input elements (I[..N]).
716///
717/// *Complete version*: It will return `Err(ErrMode::Backtrack(_))` if the input is shorter than the argument.
718///
719/// *[Partial version][crate::_topic::partial]*: if the input has less than N elements, `take` will
720/// return a `ErrMode::Incomplete(Needed::new(M))` where M is the number of
721/// additional bytes the parser would need to succeed.
722/// It is well defined for `&[u8]` as the number of elements is the byte size,
723/// but for types like `&str`, we cannot know how many bytes correspond for
724/// the next few chars, so the result will be `ErrMode::Incomplete(Needed::Unknown)`
725///
726/// # Effective Signature
727///
728/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` ranges:
729/// ```rust
730/// # use std::ops::RangeFrom;
731/// # use winnow::prelude::*;
732/// # use winnow::stream::ContainsToken;
733/// # use winnow::error::ContextError;
734/// pub fn take<'i>(token_count: usize) -> impl Parser<&'i str, &'i str, ContextError>
735/// # {
736/// #     winnow::token::take(token_count)
737/// # }
738/// ```
739///
740/// # Example
741///
742/// ```rust
743/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
744/// # use winnow::prelude::*;
745/// use winnow::token::take;
746///
747/// fn take6<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
748///   take(6usize).parse_next(s)
749/// }
750///
751/// assert_eq!(take6.parse_peek("1234567"), Ok(("7", "123456")));
752/// assert_eq!(take6.parse_peek("things"), Ok(("", "things")));
753/// assert!(take6.parse_peek("short").is_err());
754/// assert!(take6.parse_peek("").is_err());
755/// ```
756///
757/// The units that are taken will depend on the input type. For example, for a
758/// `&str` it will take a number of `char`'s, whereas for a `&[u8]` it will
759/// take that many `u8`'s:
760///
761/// ```rust
762/// # use winnow::prelude::*;
763/// use winnow::error::ContextError;
764/// use winnow::token::take;
765///
766/// assert_eq!(take::<_, _, ContextError>(1usize).parse_peek("💙"), Ok(("", "💙")));
767/// assert_eq!(take::<_, _, ContextError>(1usize).parse_peek("💙".as_bytes()), Ok((b"\x9F\x92\x99".as_ref(), b"\xF0".as_ref())));
768/// ```
769///
770/// ```rust
771/// # use winnow::prelude::*;
772/// # use winnow::error::{ErrMode, ContextError, Needed};
773/// # use winnow::Partial;
774/// use winnow::token::take;
775///
776/// fn take6<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
777///   take(6usize).parse_next(s)
778/// }
779///
780/// assert_eq!(take6.parse_peek(Partial::new("1234567")), Ok((Partial::new("7"), "123456")));
781/// assert_eq!(take6.parse_peek(Partial::new("things")), Ok((Partial::new(""), "things")));
782/// // `Unknown` as we don't know the number of bytes that `count` corresponds to
783/// assert_eq!(take6.parse_peek(Partial::new("short")), Err(ErrMode::Incomplete(Needed::Unknown)));
784/// ```
785#[inline(always)]
786pub fn take<UsizeLike, Input, Error>(
787    token_count: UsizeLike,
788) -> impl Parser<Input, <Input as Stream>::Slice, Error>
789where
790    Input: StreamIsPartial + Stream,
791    UsizeLike: ToUsize,
792    Error: ParserError<Input>,
793{
794    let c = token_count.to_usize();
795    trace("take", move |i: &mut Input| {
796        if <Input as StreamIsPartial>::is_partial_supported() {
797            take_::<_, _, true>(i, c)
798        } else {
799            take_::<_, _, false>(i, c)
800        }
801    })
802}
803
804fn take_<I, Error: ParserError<I>, const PARTIAL: bool>(
805    i: &mut I,
806    c: usize,
807) -> Result<<I as Stream>::Slice, Error>
808where
809    I: StreamIsPartial,
810    I: Stream,
811{
812    match i.offset_at(c) {
813        Ok(offset) => Ok(i.next_slice(offset)),
814        Err(e) if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, e)),
815        Err(_needed) => Err(ParserError::from_input(i)),
816    }
817}
818
819/// Recognize the input slice (bound by `occurrences`) up to the first occurrence of a [literal].
820///
821/// Feature `simd` will enable the use of [`memchr`](https://docs.rs/memchr/latest/memchr/).
822///
823/// It doesn't consume the literal.
824///
825/// It will return an `ErrMode::Backtrack(_)` if the set of tokens wasn't met or is out
826/// of `occurrences` range.
827///
828/// *Complete version*: It will return `Err(ErrMode::Backtrack(_))`
829/// if the literal wasn't met.
830///
831/// *[Partial version][crate::_topic::partial]*: will return a `ErrMode::Incomplete(Needed::new(N))` if the input doesn't
832/// contain the literal or if the input is smaller than the literal.
833///
834/// See also
835/// - [`take_till`] for recognizing up-to a [set of tokens][ContainsToken]
836/// - [`repeat_till`][crate::combinator::repeat_till] with [`Parser::take`] for taking tokens up to a [`Parser`]
837///
838/// # Effective Signature
839///
840/// Assuming you are parsing a `&str` [Stream] with `0..` or `1..` [ranges][Range]:
841/// ```rust
842/// # use std::ops::RangeFrom;
843/// # use winnow::prelude::*;;
844/// # use winnow::error::ContextError;
845/// pub fn take_until(occurrences: RangeFrom<usize>, literal: &str) -> impl Parser<&str, &str, ContextError>
846/// # {
847/// #     winnow::token::take_until(occurrences, literal)
848/// # }
849/// ```
850///
851/// # Example
852///
853/// ```rust
854/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
855/// # use winnow::prelude::*;
856/// use winnow::token::take_until;
857///
858/// fn until_eof<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
859///   take_until(0.., "eof").parse_next(s)
860/// }
861///
862/// assert_eq!(until_eof.parse_peek("hello, worldeof"), Ok(("eof", "hello, world")));
863/// assert!(until_eof.parse_peek("hello, world").is_err());
864/// assert!(until_eof.parse_peek("").is_err());
865/// assert_eq!(until_eof.parse_peek("1eof2eof"), Ok(("eof2eof", "1")));
866/// ```
867///
868/// ```rust
869/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
870/// # use winnow::prelude::*;
871/// # use winnow::Partial;
872/// use winnow::token::take_until;
873///
874/// fn until_eof<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
875///   take_until(0.., "eof").parse_next(s)
876/// }
877///
878/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
879/// assert_eq!(until_eof.parse_peek(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
880/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
881/// assert_eq!(until_eof.parse_peek(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
882/// ```
883///
884/// ```rust
885/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
886/// # use winnow::prelude::*;
887/// use winnow::token::take_until;
888///
889/// fn until_eof<'i>(s: &mut &'i str) -> ModalResult<&'i str> {
890///   take_until(1.., "eof").parse_next(s)
891/// }
892///
893/// assert_eq!(until_eof.parse_peek("hello, worldeof"), Ok(("eof", "hello, world")));
894/// assert!(until_eof.parse_peek("hello, world").is_err());
895/// assert!(until_eof.parse_peek("").is_err());
896/// assert_eq!(until_eof.parse_peek("1eof2eof"), Ok(("eof2eof", "1")));
897/// assert!(until_eof.parse_peek("eof").is_err());
898/// ```
899///
900/// ```rust
901/// # use winnow::{error::ErrMode, error::ContextError, error::Needed};
902/// # use winnow::prelude::*;
903/// # use winnow::Partial;
904/// use winnow::token::take_until;
905///
906/// fn until_eof<'i>(s: &mut Partial<&'i str>) -> ModalResult<&'i str> {
907///   take_until(1.., "eof").parse_next(s)
908/// }
909///
910/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeof")), Ok((Partial::new("eof"), "hello, world")));
911/// assert_eq!(until_eof.parse_peek(Partial::new("hello, world")), Err(ErrMode::Incomplete(Needed::Unknown)));
912/// assert_eq!(until_eof.parse_peek(Partial::new("hello, worldeo")), Err(ErrMode::Incomplete(Needed::Unknown)));
913/// assert_eq!(until_eof.parse_peek(Partial::new("1eof2eof")), Ok((Partial::new("eof2eof"), "1")));
914/// assert!(until_eof.parse_peek(Partial::new("eof")).is_err());
915/// ```
916#[inline(always)]
917pub fn take_until<Literal, Input, Error>(
918    occurrences: impl Into<Range>,
919    literal: Literal,
920) -> impl Parser<Input, <Input as Stream>::Slice, Error>
921where
922    Input: StreamIsPartial + Stream + FindSlice<Literal>,
923    Literal: Clone,
924    Error: ParserError<Input>,
925{
926    let Range {
927        start_inclusive,
928        end_inclusive,
929    } = occurrences.into();
930    trace("take_until", move |i: &mut Input| {
931        match (start_inclusive, end_inclusive) {
932            (0, None) => {
933                if <Input as StreamIsPartial>::is_partial_supported() {
934                    take_until0_::<_, _, _, true>(i, literal.clone())
935                } else {
936                    take_until0_::<_, _, _, false>(i, literal.clone())
937                }
938            }
939            (1, None) => {
940                if <Input as StreamIsPartial>::is_partial_supported() {
941                    take_until1_::<_, _, _, true>(i, literal.clone())
942                } else {
943                    take_until1_::<_, _, _, false>(i, literal.clone())
944                }
945            }
946            (start, end) => {
947                let end = end.unwrap_or(usize::MAX);
948                if <Input as StreamIsPartial>::is_partial_supported() {
949                    take_until_m_n_::<_, _, _, true>(i, start, end, literal.clone())
950                } else {
951                    take_until_m_n_::<_, _, _, false>(i, start, end, literal.clone())
952                }
953            }
954        }
955    })
956}
957
958fn take_until0_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
959    i: &mut I,
960    t: T,
961) -> Result<<I as Stream>::Slice, Error>
962where
963    I: StreamIsPartial,
964    I: Stream + FindSlice<T>,
965{
966    match i.find_slice(t) {
967        Some(range) => Ok(i.next_slice(range.start)),
968        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
969        None => Err(ParserError::from_input(i)),
970    }
971}
972
973fn take_until1_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
974    i: &mut I,
975    t: T,
976) -> Result<<I as Stream>::Slice, Error>
977where
978    I: StreamIsPartial,
979    I: Stream + FindSlice<T>,
980{
981    match i.find_slice(t) {
982        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
983        None => Err(ParserError::from_input(i)),
984        Some(range) => {
985            if range.start == 0 {
986                Err(ParserError::from_input(i))
987            } else {
988                Ok(i.next_slice(range.start))
989            }
990        }
991    }
992}
993
994fn take_until_m_n_<T, I, Error: ParserError<I>, const PARTIAL: bool>(
995    i: &mut I,
996    start: usize,
997    end: usize,
998    t: T,
999) -> Result<<I as Stream>::Slice, Error>
1000where
1001    I: StreamIsPartial,
1002    I: Stream + FindSlice<T>,
1003{
1004    if end < start {
1005        return Err(ParserError::assert(
1006            i,
1007            "`occurrences` should be ascending, rather than descending",
1008        ));
1009    }
1010
1011    match i.find_slice(t) {
1012        Some(range) => {
1013            let start_offset = i.offset_at(start);
1014            let end_offset = i.offset_at(end).unwrap_or_else(|_err| i.eof_offset());
1015            if start_offset.map(|s| range.start < s).unwrap_or(true) {
1016                if PARTIAL && i.is_partial() {
1017                    return Err(ParserError::incomplete(i, Needed::Unknown));
1018                } else {
1019                    return Err(ParserError::from_input(i));
1020                }
1021            }
1022            if end_offset < range.start {
1023                return Err(ParserError::from_input(i));
1024            }
1025            Ok(i.next_slice(range.start))
1026        }
1027        None if PARTIAL && i.is_partial() => Err(ParserError::incomplete(i, Needed::Unknown)),
1028        None => Err(ParserError::from_input(i)),
1029    }
1030}
1031
1032/// Return the remaining input.
1033///
1034/// # Effective Signature
1035///
1036/// Assuming you are parsing a `&str` [Stream]:
1037/// ```rust
1038/// # use winnow::prelude::*;;
1039/// pub fn rest<'i>(input: &mut &'i str) -> ModalResult<&'i str>
1040/// # {
1041/// #     winnow::token::rest.parse_next(input)
1042/// # }
1043/// ```
1044///
1045/// # Example
1046///
1047/// ```rust
1048/// # use winnow::prelude::*;
1049/// # use winnow::error::ContextError;
1050/// use winnow::token::rest;
1051/// assert_eq!(rest::<_,ContextError>.parse_peek("abc"), Ok(("", "abc")));
1052/// assert_eq!(rest::<_,ContextError>.parse_peek(""), Ok(("", "")));
1053/// ```
1054#[inline]
1055pub fn rest<Input, Error>(input: &mut Input) -> Result<<Input as Stream>::Slice, Error>
1056where
1057    Input: Stream,
1058    Error: ParserError<Input>,
1059{
1060    trace("rest", move |input: &mut Input| Ok(input.finish())).parse_next(input)
1061}
1062
1063/// Return the length of the remaining input.
1064///
1065/// <div class="warning">
1066///
1067/// Note: this does not advance the [`Stream`]
1068///
1069/// </div>
1070///
1071/// # Effective Signature
1072///
1073/// Assuming you are parsing a `&str` [Stream]:
1074/// ```rust
1075/// # use winnow::prelude::*;;
1076/// pub fn rest_len(input: &mut &str) -> ModalResult<usize>
1077/// # {
1078/// #     winnow::token::rest_len.parse_next(input)
1079/// # }
1080/// ```
1081///
1082/// # Example
1083///
1084/// ```rust
1085/// # use winnow::prelude::*;
1086/// # use winnow::error::ContextError;
1087/// use winnow::token::rest_len;
1088/// assert_eq!(rest_len::<_,ContextError>.parse_peek("abc"), Ok(("abc", 3)));
1089/// assert_eq!(rest_len::<_,ContextError>.parse_peek(""), Ok(("", 0)));
1090/// ```
1091#[inline]
1092pub fn rest_len<Input, Error>(input: &mut Input) -> Result<usize, Error>
1093where
1094    Input: Stream,
1095    Error: ParserError<Input>,
1096{
1097    trace("rest_len", move |input: &mut Input| {
1098        let len = input.eof_offset();
1099        Ok(len)
1100    })
1101    .parse_next(input)
1102}
winnow/token/mod.rs

winnow/token/
mod.rs
