cargo fmt
This commit is contained in:
parent
daa4da8037
commit
108297b966
|
@ -1,16 +1,16 @@
|
||||||
//! convert text into a token stream
|
//! convert text into a token stream
|
||||||
|
|
||||||
use std::io;
|
|
||||||
use std::fmt;
|
|
||||||
use super::{identifier, ws0, IResult, Span};
|
use super::{identifier, ws0, IResult, Span};
|
||||||
use nom::{
|
use nom::{
|
||||||
branch::alt,
|
branch::alt,
|
||||||
bytes::complete::tag,
|
bytes::complete::tag,
|
||||||
character::complete::{digit1, anychar},
|
character::complete::{anychar, digit1},
|
||||||
combinator::{consumed, map, recognize},
|
combinator::{consumed, map, recognize},
|
||||||
multi::many0,
|
|
||||||
error::ParseError,
|
error::ParseError,
|
||||||
|
multi::many0,
|
||||||
};
|
};
|
||||||
|
use std::fmt;
|
||||||
|
use std::io;
|
||||||
|
|
||||||
pub struct Token<'a> {
|
pub struct Token<'a> {
|
||||||
span: Span<'a>,
|
span: Span<'a>,
|
||||||
|
@ -19,7 +19,13 @@ pub struct Token<'a> {
|
||||||
|
|
||||||
impl fmt::Debug for Token<'_> {
|
impl fmt::Debug for Token<'_> {
|
||||||
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
|
||||||
write!(f, "{:?} @{} {:?}", self.kind, self.span.location_offset(), self.span.fragment())?;
|
write!(
|
||||||
|
f,
|
||||||
|
"{:?} @{} {:?}",
|
||||||
|
self.kind,
|
||||||
|
self.span.location_offset(),
|
||||||
|
self.span.fragment()
|
||||||
|
)?;
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -33,7 +39,7 @@ impl<'a> Token<'a> {
|
||||||
pub fn pretty_tokens(mut w: impl io::Write, toks: &[Token]) -> io::Result<()> {
|
pub fn pretty_tokens(mut w: impl io::Write, toks: &[Token]) -> io::Result<()> {
|
||||||
for tok in toks {
|
for tok in toks {
|
||||||
writeln!(w, "{:?}", tok)?;
|
writeln!(w, "{:?}", tok)?;
|
||||||
};
|
}
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -75,8 +81,12 @@ pub struct TokenSpan<'a> {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl<'a> TokenSpan<'a> {
|
impl<'a> TokenSpan<'a> {
|
||||||
pub fn new(rest: &'a [Token<'a>]) -> Self { Self { rest, pos: 0 } }
|
pub fn new(rest: &'a [Token<'a>]) -> Self {
|
||||||
pub fn with_pos(rest: &'a [Token<'a>], pos: usize) -> Self { Self { rest, pos } }
|
Self { rest, pos: 0 }
|
||||||
|
}
|
||||||
|
pub fn with_pos(rest: &'a [Token<'a>], pos: usize) -> Self {
|
||||||
|
Self { rest, pos }
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
impl nom::InputTake for TokenSpan<'_> {
|
impl nom::InputTake for TokenSpan<'_> {
|
||||||
|
@ -86,7 +96,10 @@ impl nom::InputTake for TokenSpan<'_> {
|
||||||
|
|
||||||
fn take_split(&self, count: usize) -> (Self, Self) {
|
fn take_split(&self, count: usize) -> (Self, Self) {
|
||||||
let (head, tail) = &self.rest.split_at(count);
|
let (head, tail) = &self.rest.split_at(count);
|
||||||
(TokenSpan::with_pos(&head, self.pos), TokenSpan::with_pos(&tail, self.pos + count))
|
(
|
||||||
|
TokenSpan::with_pos(&head, self.pos),
|
||||||
|
TokenSpan::with_pos(&tail, self.pos + count),
|
||||||
|
)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
@ -97,17 +110,22 @@ impl nom_greedyerror::Position for TokenSpan<'_> {
|
||||||
}
|
}
|
||||||
|
|
||||||
/// combinator that matches a token kind
|
/// combinator that matches a token kind
|
||||||
pub fn token<'a, E>(kind: TokenKind) -> impl FnMut(TokenSpan<'a>) -> nom::IResult<TokenSpan, &Token, E>
|
pub fn token<'a, E>(
|
||||||
where E: ParseError<TokenSpan<'a>>
|
kind: TokenKind,
|
||||||
|
) -> impl FnMut(TokenSpan<'a>) -> nom::IResult<TokenSpan, &Token, E>
|
||||||
|
where
|
||||||
|
E: ParseError<TokenSpan<'a>>,
|
||||||
{
|
{
|
||||||
move |input: TokenSpan| {
|
move |input: TokenSpan| {
|
||||||
let next = &input.rest[0];
|
let next = &input.rest[0];
|
||||||
if next.kind == kind.clone() {
|
if next.kind == kind.clone() {
|
||||||
let rest = TokenSpan::new(&input.rest[1..]);
|
let rest = TokenSpan::new(&input.rest[1..]);
|
||||||
Ok((rest, next))
|
Ok((rest, next))
|
||||||
}
|
} else {
|
||||||
else {
|
Err(nom::Err::Error(E::from_error_kind(
|
||||||
Err(nom::Err::Error(E::from_error_kind(input, nom::error::ErrorKind::Tag)))
|
input,
|
||||||
|
nom::error::ErrorKind::Tag,
|
||||||
|
)))
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@ -118,7 +136,9 @@ pub fn lex(input: Span) -> IResult<Span, Vec<Token>> {
|
||||||
lex_literals,
|
lex_literals,
|
||||||
lex_braces,
|
lex_braces,
|
||||||
lex_punctuation,
|
lex_punctuation,
|
||||||
map(recognize(anychar), |span| Token::new(span, TokenKind::Error)),
|
map(recognize(anychar), |span| {
|
||||||
|
Token::new(span, TokenKind::Error)
|
||||||
|
}),
|
||||||
))))(input)
|
))))(input)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
Loading…
Reference in New Issue