Files
@ 6d6c5b5f07ae
Branch filter:
Location: CSY/reowolf/src/protocol/token_writer.rs
6d6c5b5f07ae
4.8 KiB
application/rls-services+xml
Attempting to fix token tree construction
1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 | #![allow(dead_code)]
use std::fmt::Write;
use std::io::Write as IOWrite;
use crate::protocol::input_source::{InputSource, InputSpan};
use crate::protocol::parser::Module;
use crate::protocol::tokens::{Token, TokenBuffer, TokenKind, TokenRange, TokenIter, TokenRangeKind};
pub(crate) struct TokenWriter {
buffer: String,
}
impl TokenWriter {
pub(crate) fn new() -> Self {
return Self{
buffer: String::with_capacity(4096),
}
}
pub(crate) fn write<W: IOWrite>(&mut self, w: &mut W, modules: &[Module]) {
self.buffer.clear();
for module in modules {
self.write_module_tokens(module);
}
w.write_all(self.buffer.as_bytes()).expect("write tokens");
}
fn write_module_tokens(&mut self, module: &Module) {
self.write_dashed_indent(0);
match &module.name {
Some(name) => writeln!(self.buffer, "Module: {}", name.1.as_str()).unwrap(),
None => self.buffer.push_str("Unnamed module\n"),
}
let mut range_index = -1;
if !module.tokens.ranges.is_empty() {
range_index = 0;
}
while range_index >= 0 {
range_index = self.write_token_range(
&module.source, &module.tokens.tokens, &module.tokens.ranges, range_index, 1
);
}
}
/// Writes a single token range. Recurses if there are any child ranges.
/// Returns the next token range index to iterate over (or a negative
/// number, if there are no more sibling ranges).
fn write_token_range(&mut self, source: &InputSource, tokens: &[Token], ranges: &[TokenRange], range_index: i32, indent: u32) -> i32 {
// Write range kind
let range = &ranges[range_index as usize];
self.write_dashed_indent(indent);
writeln!(self.buffer, "Range: {:?}", range.range_kind);
// Write tokens/lines it spans
let first_token_pos = tokens[range.start as usize].pos;
let last_token_pos = if (range.end as usize) < tokens.len() {
tokens[range.end as usize].pos
} else {
tokens.last().unwrap().pos
};
let first_source_col = source.get_column(first_token_pos);
let last_source_col = source.get_column(last_token_pos);
self.write_indent(indent);
writeln!(
self.buffer, "Source: token {} to {}, file {}:{}:{} to {}:{}",
range.start, range.end, source.filename,
first_token_pos.line, first_source_col,
last_token_pos.line, last_source_col
);
let next_sibling_index = range.next_sibling_idx;
if range.num_child_ranges == 0 {
// No child ranges, so dump the tokens here
debug_assert!(range.first_child_idx < 0);
self.write_token_array(source, tokens, range, indent);
} else {
// Child ranges
debug_assert!(range.first_child_idx >= 0);
self.write_indent(indent);
writeln!(self.buffer, "Children: [");
let mut range_index = range.first_child_idx;
while range_index >= 0 {
range_index = self.write_token_range(source, tokens, ranges, range_index, indent + 1);
}
self.write_indent(indent);
writeln!(self.buffer, "]");
}
// Wrote everything, return the next sibling token range
return next_sibling_index;
}
fn write_token_array(&mut self, source: &InputSource, tokens: &[Token], range: &TokenRange, indent: u32) {
self.write_indent(indent);
writeln!(self.buffer, "Tokens: [");
let token_indent = indent + 1;
for token_index in range.start as usize..range.end as usize {
// Skip uninteresting tokens
let token = &tokens[token_index];
if token.kind == TokenKind::SpanEnd {
continue;
}
self.write_indent(token_indent);
write!(self.buffer, "{:?} (index {})", token.kind, token_index);
if token.kind.has_span_end() {
let token_start = token.pos;
let token_end = tokens[token_index + 1].pos;
let section = source.section_at_span(InputSpan::from_positions(token_start, token_end));
writeln!(self.buffer, " text: {}", String::from_utf8_lossy(section));
} else {
self.buffer.push('\n');
}
}
self.write_indent(indent);
writeln!(self.buffer, "]");
}
fn write_dashed_indent(&mut self, indent: u32) {
for _ in 0..indent * 2 {
self.buffer.push(' ');
}
self.buffer.push('-');
self.buffer.push(' ');
}
fn write_indent(&mut self, indent: u32) {
for _ in 0..(indent + 1)*2 {
self.buffer.push(' ');
}
}
}
|