diff --git a/src/protocol/parser/pass_tokenizer.rs b/src/protocol/parser/pass_tokenizer.rs index d6f457a90a4963c90a0f36bb011df94335f6e695..d41b6f65939a7267f7a38fb635d78b51fb634e1d 100644 --- a/src/protocol/parser/pass_tokenizer.rs +++ b/src/protocol/parser/pass_tokenizer.rs @@ -786,77 +786,3 @@ fn maybe_number_remaining(c: u8) -> bool { (c >= b'0' && c <= b'9') || (c >= b'A' && c <= b'F') || (c >= b'a' && c <= b'f') || c == b'_'; } - -#[cfg(test)] -mod tests { - use super::*; - - // TODO: Remove at some point - #[test] - fn test_tokenizer() { - let mut source = InputSource::new_test(" - - #version 500 - # hello 2 - - import std.reo::*; - - struct Thing { - int a: 5, - } - enum Hello { - A, - B - } - - // Hello hello, is it me you are looking for? - // I can seee it in your eeeyes - - func something(int a, int b, int c) -> byte { - int a = 5; - struct Inner { - int a - } - struct City { - int b - } - /* Waza - How are you doing - Things in here yo - /* */ */ - - a = a + 5 * 2; - struct Pressure { - int d - } - } - "); - let mut t = PassTokenizer::new(); - let mut buffer = TokenBuffer::new(); - t.tokenize(&mut source, &mut buffer).expect("tokenize"); - - println!("Ranges:\n"); - for (idx, range) in buffer.ranges.iter().enumerate() { - println!("[{}] {:?}", idx, range) - } - - println!("Tokens:\n"); - let mut iter = buffer.tokens.iter().enumerate(); - while let Some((idx, token)) = iter.next() { - match token.kind { - TokenKind::Ident | TokenKind::Pragma | TokenKind::Integer | - TokenKind::String | TokenKind::Character | TokenKind::LineComment | - TokenKind::BlockComment => { - let (_, end) = iter.next().unwrap(); - println!("[{}] {:?} ......", idx, token.kind); - assert_eq!(end.kind, TokenKind::SpanEnd); - let text = source.section_at_pos(token.pos, end.pos); - println!("{}", String::from_utf8_lossy(text)); - }, - _ => { - println!("[{}] {:?}", idx, token.kind); - } - } - } - } -} \ No newline at end of file