diff --git a/src/protocol/token_writer.rs b/src/protocol/token_writer.rs index ac28761f48e5a136126b1c0addd1b37782400b73..a6028e9b5a315217ffc3d38ff94bf4076b908393 100644 --- a/src/protocol/token_writer.rs +++ b/src/protocol/token_writer.rs @@ -1,11 +1,11 @@ #![allow(dead_code)] -use std::fmt::Write; +use std::fmt::{Write, Error as FmtError}; use std::io::Write as IOWrite; use crate::protocol::input_source::{InputSource, InputSpan}; use crate::protocol::parser::Module; -use crate::protocol::tokens::{Token, TokenKind, TokenRange}; +use crate::protocol::tokens::{Token, TokenKind, TokenMarker}; pub(crate) struct TokenWriter { buffer: String, @@ -35,77 +35,33 @@ impl TokenWriter { None => self.buffer.push_str("Unnamed module\n"), } - - let mut range_index = -1; - if !module.tokens.ranges.is_empty() { - range_index = 0; - } - - while range_index >= 0 { - range_index = self.write_token_range( - &module.source, &module.tokens.tokens, &module.tokens.ranges, range_index, 1 - ); - } + self.write_marker_array(&module.tokens.markers, 1).expect("write markers"); + self.write_token_array(&module.source, &module.tokens.tokens, 1).expect("write tokens"); } - /// Writes a single token range. Recurses if there are any child ranges. - /// Returns the next token range index to iterate over (or a negative - /// number, if there are no more sibling ranges). - fn write_token_range(&mut self, source: &InputSource, tokens: &[Token], ranges: &[TokenRange], range_index: i32, indent: u32) -> i32 { - // Write range kind - let range = &ranges[range_index as usize]; - self.write_dashed_indent(indent); - writeln!(self.buffer, "Range: {:?}", range.range_kind); - - // Write tokens/lines it spans - let first_token_pos = tokens[range.start as usize].pos; - - let last_token_pos = if (range.end as usize) < tokens.len() { - tokens[range.end as usize].pos - } else { - tokens.last().unwrap().pos - }; - let first_source_col = source.get_column(first_token_pos); - let last_source_col = source.get_column(last_token_pos); - + fn write_marker_array(&mut self, markers: &[TokenMarker], indent: u32) -> Result<(), FmtError> { self.write_indent(indent); - writeln!( - self.buffer, "Source: token {} to {}, file {}:{}:{} to {}:{}", - range.start, range.end, source.filename, - first_token_pos.line, first_source_col, - last_token_pos.line, last_source_col - ); - - let next_sibling_index = range.next_sibling_idx; - if range.num_child_ranges == 0 { - // No child ranges, so dump the tokens here - debug_assert!(range.first_child_idx < 0); - self.write_token_array(source, tokens, range, indent); - } else { - // Child ranges - debug_assert!(range.first_child_idx >= 0); - self.write_indent(indent); - writeln!(self.buffer, "Children: ["); - - let mut range_index = range.first_child_idx; - while range_index >= 0 { - range_index = self.write_token_range(source, tokens, ranges, range_index, indent + 1); - } + writeln!(self.buffer, "Markers: [")?; - self.write_indent(indent); - writeln!(self.buffer, "]"); + let marker_indent = indent + 1; + for marker in markers { + self.write_indent(marker_indent); + writeln!(self.buffer, "{:?}", marker)?; } - // Wrote everything, return the next sibling token range - return next_sibling_index; + self.write_indent(indent); + writeln!(self.buffer, "]")?; + + return Ok(()); } - fn write_token_array(&mut self, source: &InputSource, tokens: &[Token], range: &TokenRange, indent: u32) { + fn write_token_array(&mut self, source: &InputSource, tokens: &[Token], indent: u32) -> Result<(), FmtError> { self.write_indent(indent); - writeln!(self.buffer, "Tokens: ["); + writeln!(self.buffer, "Tokens: [")?; + let num_tokens = tokens.len(); let token_indent = indent + 1; - for token_index in range.start as usize..range.end as usize { + for token_index in 0..num_tokens { // Skip uninteresting tokens let token = &tokens[token_index]; if token.kind == TokenKind::SpanEnd { @@ -113,19 +69,21 @@ impl TokenWriter { } self.write_indent(token_indent); - write!(self.buffer, "{:?} (index {})", token.kind, token_index); + write!(self.buffer, "{:?} (index {})", token.kind, token_index)?; if token.kind.has_span_end() { let token_start = token.pos; let token_end = tokens[token_index + 1].pos; let section = source.section_at_span(InputSpan::from_positions(token_start, token_end)); - writeln!(self.buffer, " text: {}", String::from_utf8_lossy(section)); + writeln!(self.buffer, " text: {}", String::from_utf8_lossy(section))?; } else { self.buffer.push('\n'); } } self.write_indent(indent); - writeln!(self.buffer, "]"); + writeln!(self.buffer, "]")?; + + return Ok(()); } fn write_dashed_indent(&mut self, indent: u32) {