diff --git a/src/protocol/token_writer.rs b/src/protocol/token_writer.rs new file mode 100644 index 0000000000000000000000000000000000000000..a6028e9b5a315217ffc3d38ff94bf4076b908393 --- /dev/null +++ b/src/protocol/token_writer.rs @@ -0,0 +1,102 @@ +#![allow(dead_code)] + +use std::fmt::{Write, Error as FmtError}; +use std::io::Write as IOWrite; + +use crate::protocol::input_source::{InputSource, InputSpan}; +use crate::protocol::parser::Module; +use crate::protocol::tokens::{Token, TokenKind, TokenMarker}; + +pub(crate) struct TokenWriter { + buffer: String, +} + +impl TokenWriter { + pub(crate) fn new() -> Self { + return Self{ + buffer: String::with_capacity(4096), + } + } + + pub(crate) fn write(&mut self, w: &mut W, modules: &[Module]) { + self.buffer.clear(); + for module in modules { + self.write_module_tokens(module); + } + + w.write_all(self.buffer.as_bytes()).expect("write tokens"); + } + + fn write_module_tokens(&mut self, module: &Module) { + self.write_dashed_indent(0); + + match &module.name { + Some(name) => writeln!(self.buffer, "Module: {}", name.1.as_str()).unwrap(), + None => self.buffer.push_str("Unnamed module\n"), + } + + self.write_marker_array(&module.tokens.markers, 1).expect("write markers"); + self.write_token_array(&module.source, &module.tokens.tokens, 1).expect("write tokens"); + } + + fn write_marker_array(&mut self, markers: &[TokenMarker], indent: u32) -> Result<(), FmtError> { + self.write_indent(indent); + writeln!(self.buffer, "Markers: [")?; + + let marker_indent = indent + 1; + for marker in markers { + self.write_indent(marker_indent); + writeln!(self.buffer, "{:?}", marker)?; + } + + self.write_indent(indent); + writeln!(self.buffer, "]")?; + + return Ok(()); + } + + fn write_token_array(&mut self, source: &InputSource, tokens: &[Token], indent: u32) -> Result<(), FmtError> { + self.write_indent(indent); + writeln!(self.buffer, "Tokens: [")?; + + let num_tokens = tokens.len(); + let token_indent = indent + 1; + for token_index in 0..num_tokens { + // Skip uninteresting tokens + let token = &tokens[token_index]; + if token.kind == TokenKind::SpanEnd { + continue; + } + + self.write_indent(token_indent); + write!(self.buffer, "{:?} (index {})", token.kind, token_index)?; + if token.kind.has_span_end() { + let token_start = token.pos; + let token_end = tokens[token_index + 1].pos; + let section = source.section_at_span(InputSpan::from_positions(token_start, token_end)); + writeln!(self.buffer, " text: {}", String::from_utf8_lossy(section))?; + } else { + self.buffer.push('\n'); + } + } + + self.write_indent(indent); + writeln!(self.buffer, "]")?; + + return Ok(()); + } + + fn write_dashed_indent(&mut self, indent: u32) { + for _ in 0..indent * 2 { + self.buffer.push(' '); + } + self.buffer.push('-'); + self.buffer.push(' '); + } + + fn write_indent(&mut self, indent: u32) { + for _ in 0..(indent + 1)*2 { + self.buffer.push(' '); + } + } +} \ No newline at end of file