CSY/reowolf Changeset - fc85ab427e9c · Centrum Wiskunde & Informatica (CWI)

Changeset - fc85ab427e9c

Parent rev.

Child rev.

[Not reviewed]

0 4 2

MH - 4 years ago 2022-03-25 15:38:45
contact@maxhenger.nl

Add debugging tool for token ranges

6 files changed with 662 insertions and 1 deletions:

src/protocol/input_source.rs

src/protocol/mod.rs

src/protocol/parser/mod.rs

src/protocol/parser/tokens.rs

src/protocol/token_writer.rs

144

tokens.txt

500

0 comments (0 inline, 0 general)

src/protocol/input_source.rs

➞

Show inline comments

 use std::fmt;
 use std::sync::{RwLock, RwLockReadGuard};
 use std::fmt::Write;
 #[derive(Debug, Clone, Copy)]
 pub struct InputPosition {
     pub line: u32,
     pub offset: u32,
+}
 impl InputPosition {
     pub(crate) fn with_offset(&self, offset: u32) -> Self {
         InputPosition { line: self.line, offset: self.offset + offset }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub struct InputSpan {
     pub begin: InputPosition,
     pub end: InputPosition,
+}
 impl InputSpan {
     // This must only be used if you're sure that the span will not be involved
     // in creating an error message.
     #[inline]
     pub const fn new() -> InputSpan {
         InputSpan{ begin: InputPosition{ line: 0, offset: 0 }, end: InputPosition{ line: 0, offset: 0 }}
+    }
     #[inline]
     pub fn from_positions(begin: InputPosition, end: InputPosition) -> Self {
         Self { begin, end }
+    }
+}
 /// Wrapper around source file with optional filename. Ensures that the file is
 /// only scanned once.
 pub struct InputSource {
     pub(crate) filename: String,
     pub(crate) input: Vec<u8>,
     // Iteration
     line: u32,
     offset: usize,
     // State tracking
     pub(crate) had_error: Option<ParseError>,
     // The offset_lookup is built on-demand upon attempting to report an error.
     // Only one procedure will actually create the lookup, afterwards only read
     // locks will be held.
     offset_lookup: RwLock<Vec<u32>>,
+}
 impl InputSource {
     pub fn new(filename: String, input: Vec<u8>) -> Self {
         Self{
             filename,
             input,
             line: 1,
             offset: 0,
             had_error: None,
             offset_lookup: RwLock::new(Vec::new()),
+        }
+    }
     #[inline]
     pub fn pos(&self) -> InputPosition {
         InputPosition { line: self.line, offset: self.offset as u32 }
+    }
     pub fn next(&self) -> Option<u8> {
         if self.offset < self.input.len() {
             Some(self.input[self.offset])
         } else {
             None
+        }
+    }
     pub fn lookahead(&self, offset: usize) -> Option<u8> {
         let offset_pos = self.offset + offset;
         if offset_pos < self.input.len() {
             Some(self.input[offset_pos])
         } else {
             None
+        }
+    }
     #[inline]
     pub fn section_at_pos(&self, start: InputPosition, end: InputPosition) -> &[u8] {
         &self.input[start.offset as usize..end.offset as usize]
+    }
     #[inline]
     pub fn section_at_span(&self, span: InputSpan) -> &[u8] {
         &self.input[span.begin.offset as usize..span.end.offset as usize]
+    }
     // Consumes the next character. Will check well-formedness of newlines: \r
     // must be followed by a \n, because this is used for error reporting. Will
     // not check for ascii-ness of the file, better left to a tokenizer.
     pub fn consume(&mut self) {
         match self.next() {
             Some(b'\r') => {
                 if Some(b'\n') == self.lookahead(1) {
                     // Well formed file
                     self.offset += 1;
                 } else {
                     // Not a well-formed file, pretend like we can continue
                     self.offset += 1;
                     self.set_error("Encountered carriage-feed without a following newline");
+                }
             },
             Some(b'\n') => {
                 self.line += 1;
                 self.offset += 1;
             },
             Some(_) => {
                 self.offset += 1;
+            }
             None => {}
+        }
         // Maybe we actually want to check this in release mode. Then again:
         // a 4 gigabyte source file... Really?
         debug_assert!(self.offset < u32::max_value() as usize);
+    }
     fn set_error(&mut self, msg: &str) {
         if self.had_error.is_none() {
             self.had_error = Some(ParseError::new_error_str_at_pos(self, self.pos(), msg));
+        }
+    }
     fn get_lookup(&self) -> RwLockReadGuard<Vec<u32>> {
         // Once constructed the lookup always contains one element. We use this
         // to see if it is constructed already.
+        {
             let lookup = self.offset_lookup.read().unwrap();
             if !lookup.is_empty() {
                 return lookup;
+            }
+        }
         // Lookup was not constructed yet
         let mut lookup = self.offset_lookup.write().unwrap();
         if !lookup.is_empty() {
             // Somebody created it before we had the chance
             drop(lookup);
             let lookup = self.offset_lookup.read().unwrap();
             return lookup;
+        }
         // Build the line number (!) to offset lookup, so offset by 1. We
         // assume the entire source file is scanned (most common case) for
         // preallocation.
         lookup.reserve(self.line as usize + 2);
         lookup.push(0); // line 0: never used
         lookup.push(0); // first line: first character
         for char_idx in 0..self.input.len() {
             if self.input[char_idx] == b'\n' {
                 lookup.push(char_idx as u32 + 1);
+            }
+        }
         lookup.push(self.input.len() as u32 + 1); // for lookup_line_end, intentionally adding one character
         // Return created lookup
         drop(lookup);
         let lookup = self.offset_lookup.read().unwrap();
         return lookup;
+    }
     /// Retrieves the column associated with a line. Calling this incurs a read
     /// lock, so don't spam it in happy-path compiler code.
     pub(crate) fn get_column(&self, pos: InputPosition) -> u32 {
         let line_start = self.lookup_line_start_offset(pos.line);
         return pos.offset - line_start + 1;
+    }
     /// Retrieves offset at which line starts (right after newline)
     fn lookup_line_start_offset(&self, line_number: u32) -> u32 {
         let lookup = self.get_lookup();
         lookup[line_number as usize]
+    }
     /// Retrieves offset at which line ends (at the newline character or the
     /// preceding carriage feed for \r\n-encoded newlines)
     fn lookup_line_end_offset(&self, line_number: u32) -> u32 {
         let lookup = self.get_lookup();
         let offset = lookup[(line_number + 1) as usize] - 1;
         let offset_usize = offset as usize;
         // Compensate for newlines and a potential carriage feed. Note that the
         // end position is exclusive. So we only need to compensate for a
         // "\r\n"
         if offset_usize > 0 && offset_usize < self.input.len() && self.input[offset_usize] == b'\n' && self.input[offset_usize - 1] == b'\r' {
             offset - 1
         } else {
             offset
+        }
+    }
+}
 #[derive(Debug)]
 pub enum StatementKind {
     Info,
     Error
+}
 #[derive(Debug)]
 pub enum ContextKind {
     SingleLine,
     MultiLine,
+}
 #[derive(Debug)]
 pub struct ErrorStatement {
     pub(crate) statement_kind: StatementKind,
     pub(crate) context_kind: ContextKind,
     pub(crate) start_line: u32,
     pub(crate) start_column: u32,
     pub(crate) end_line: u32,
     pub(crate) end_column: u32,
     pub(crate) filename: String,
     pub(crate) context: String,
     pub(crate) message: String,
+}
 impl ErrorStatement {
     fn from_source_at_pos(statement_kind: StatementKind, source: &InputSource, position: InputPosition, message: String) -> Self {
         // Seek line start and end
         let line_start = source.lookup_line_start_offset(position.line);
         let line_end = source.lookup_line_end_offset(position.line);
         let context = Self::create_context(source, line_start as usize, line_end as usize);
         debug_assert!(position.offset >= line_start);
         let column = position.offset - line_start + 1;
         Self{
             statement_kind,
             context_kind: ContextKind::SingleLine,
             start_line: position.line,
             start_column: column,
             end_line: position.line,
             end_column: column + 1,
             filename: source.filename.clone(),
             context,
             message,
+        }
+    }
     pub(crate) fn from_source_at_span(statement_kind: StatementKind, source: &InputSource, span: InputSpan, message: String) -> Self {
         debug_assert!(span.end.line >= span.begin.line);
         debug_assert!(span.end.offset >= span.begin.offset);
         let first_line_start = source.lookup_line_start_offset(span.begin.line);
         let last_line_start = source.lookup_line_start_offset(span.end.line);
         let last_line_end = source.lookup_line_end_offset(span.end.line);
         let context = Self::create_context(source, first_line_start as usize, last_line_end as usize);
         debug_assert!(span.begin.offset >= first_line_start);
         let start_column = span.begin.offset - first_line_start + 1;
         let end_column = span.end.offset - last_line_start + 1;
         let context_kind = if span.begin.line == span.end.line {
             ContextKind::SingleLine
         } else {
             ContextKind::MultiLine
         };
         Self{
             statement_kind,
             context_kind,
             start_line: span.begin.line,
             start_column,
             end_line: span.end.line,
             end_column,
             filename: source.filename.clone(),
             context,
             message,
+        }
+    }
     /// Produces context from source
     fn create_context(source: &InputSource, start: usize, end: usize) -> String {
         let context_raw = &source.input[start..end];
         String::from_utf8_lossy(context_raw).to_string()
+    }
+}
 impl fmt::Display for ErrorStatement {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         // Write kind of statement and message
         match self.statement_kind {
             StatementKind::Info => f.write_str(" INFO: ")?,
             StatementKind::Error => f.write_str("ERROR: ")?,
+        }
         f.write_str(&self.message)?;
         f.write_char('\n')?;
         // Write originating file/line/column
         f.write_str(" +- ")?;
         if !self.filename.is_empty() {
             write!(f, "in {} ", self.filename)?;
+        }
         match self.context_kind {
             ContextKind::SingleLine => writeln!(f, " at {}:{}", self.start_line, self.start_column),
             ContextKind::MultiLine => writeln!(
                 f, " from {}:{} to {}:{}",
                 self.start_line, self.start_column, self.end_line, self.end_column
+            )
         }?;
         // Helper function for writing context: converting tabs into 4 spaces
         // (oh, the controversy!) and creating an annotated line
         fn transform_context(source: &str, target: &mut String) {
             for char in source.chars() {
                 if char == '\t' {
                     target.push_str("    ");
                 } else {
                     target.push(char);
+                }
+            }
+        }
         fn extend_annotation(first_col: u32, last_col: u32, source: &str, target: &mut String, extend_char: char) {
             debug_assert!(first_col > 0 && last_col > first_col);
             // If the first index exceeds the size of the context then we should
             // have a message placed at the newline character
             let first_idx = first_col as usize - 1;
             let last_idx = last_col as usize - 1;
             if first_idx >= source.len() {
                 // If any of these fail then the logic behind reporting errors
                 // is incorrect.
                 debug_assert_eq!(first_idx, source.len());
                 debug_assert_eq!(first_idx + 1, last_idx);
                 target.push(extend_char);
             } else {
                 for (char_idx, char) in source.chars().enumerate().skip(first_idx) {
                     if char_idx == last_idx as usize {
                         break;
+                    }
                     if char == '\t' {
                         for _ in 0..4 { target.push(extend_char); }
                     } else {
                         target.push(extend_char);
+                    }
+                }
+            }
+        }
         // Write source context
         writeln!(f, " | ")?;
         let mut context = String::with_capacity(128);
         let mut annotation = String::with_capacity(128);
         match self.context_kind {
             ContextKind::SingleLine => {
                 // Write single line of context with indicator for the offending
                 // span underneath.
                 context.push_str(" |  ");
                 transform_context(&self.context, &mut context);
                 context.push('\n');
                 f.write_str(&context)?;
                 annotation.push_str(" | ");
                 extend_annotation(1, self.start_column + 1, &self.context, &mut annotation, ' ');
                 extend_annotation(self.start_column, self.end_column, &self.context, &mut annotation, '~');
                 annotation.push('\n');
                 f.write_str(&annotation)?;
             },
             ContextKind::MultiLine => {
                 // Annotate all offending lines
                 // - first line
                 let mut lines = self.context.lines();
                 let first_line = lines.next().unwrap();
                 transform_context(first_line, &mut context);
                 writeln!(f, " |- {}", &context)?;
                 // - remaining lines
                 let mut last_line = first_line;
                 while let Some(cur_line) = lines.next() {
                     context.clear();
                     transform_context(cur_line, &mut context);
                     writeln!(f, " |  {}", &context)?;
                     last_line = cur_line;
+                }
                 // - underline beneath last line
                 annotation.push_str(" \\__");
                 extend_annotation(1, self.end_column, &last_line, &mut annotation, '_');
                 annotation.push_str("/\n");
                 f.write_str(&annotation)?;
+            }
+        }
         Ok(())
+    }
+}
 #[derive(Debug)]
 pub struct ParseError {
     pub(crate) statements: Vec<ErrorStatement>
+}
 impl fmt::Display for ParseError {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         if self.statements.is_empty() {
             return Ok(())
+        }
         self.statements[0].fmt(f)?;
         for statement in self.statements.iter().skip(1) {
             writeln!(f)?;
             statement.fmt(f)?;
+        }
         Ok(())
+    }
+}
 impl ParseError {
     pub fn new_error_at_pos(source: &InputSource, position: InputPosition, message: String) -> Self {
         Self{ statements: vec!(ErrorStatement::from_source_at_pos(
             StatementKind::Error, source, position, message
         )) }
+    }
     pub fn new_error_str_at_pos(source: &InputSource, position: InputPosition, message: &str) -> Self {
         Self{ statements: vec!(ErrorStatement::from_source_at_pos(
             StatementKind::Error, source, position, message.to_string()
         )) }
+    }
     pub fn new_error_at_span(source: &InputSource, span: InputSpan, message: String) -> Self {
         Self{ statements: vec!(ErrorStatement::from_source_at_span(
             StatementKind::Error, source, span, message
         )) }
+    }
     pub fn new_error_str_at_span(source: &InputSource, span: InputSpan, message: &str) -> Self {
         Self{ statements: vec!(ErrorStatement::from_source_at_span(
             StatementKind::Error, source, span, message.to_string()
         )) }
+    }
     pub fn with_at_span(mut self, error_type: StatementKind, source: &InputSource, span: InputSpan, message: String) -> Self {
         self.statements.push(ErrorStatement::from_source_at_span(error_type, source, span, message.to_string()));
         self
+    }
     pub fn with_info_at_span(self, source: &InputSource, span: InputSpan, msg: String) -> Self {
         self.with_at_span(StatementKind::Info, source, span, msg)
+    }
     pub fn with_info_str_at_pos(self, source: &InputSource, pos: InputPosition, msg: &str) -> Self {
         self.with_at_span(
             StatementKind::Info, source,
             InputSpan::from_positions(pos, pos.with_offset(1)),
             msg.to_string()
+        )
+    }
     pub fn with_info_str_at_span(self, source: &InputSource, span: InputSpan, msg: &str) -> Self {
         self.with_at_span(StatementKind::Info, source, span, msg.to_string())
+    }
+}

src/protocol/mod.rs

➞

Show inline comments

 mod arena;
 pub(crate) mod eval;
 pub(crate) mod input_source;
 mod parser;
 #[cfg(test)] mod tests;
 pub(crate) mod ast;
 pub(crate) mod ast_writer;
 mod token_writer;
 use std::sync::Mutex;
 use crate::collections::{StringPool, StringRef};
 pub use crate::protocol::ast::*;
 use crate::protocol::eval::*;
 use crate::protocol::input_source::*;
 use crate::protocol::parser::*;
 use crate::protocol::type_table::*;
 pub use parser::type_table::TypeId;
 /// A protocol description module
 pub struct Module {
     pub(crate) source: InputSource,
     pub(crate) root_id: RootId,
     pub(crate) name: Option<StringRef<'static>>,
+}
 /// Description of a protocol object, used to configure new connectors.
 #[repr(C)]
 pub struct ProtocolDescription {
     pub(crate) modules: Vec<Module>,
     pub(crate) heap: Heap,
     pub(crate) types: TypeTable,
     pub(crate) pool: Mutex<StringPool>,
+}
 #[derive(Debug, Clone)]
 pub(crate) struct ComponentState {
     pub(crate) prompt: Prompt,
+}
 #[derive(Debug)]
 pub enum ComponentCreationError {
     ModuleDoesntExist,
     DefinitionDoesntExist,
     DefinitionNotComponent,
     InvalidNumArguments,
     InvalidArgumentType(usize),
     UnownedPort,
     InSync,
+}
 impl ProtocolDescription {
     pub fn parse(buffer: &[u8]) -> Result<Self, String> {
         let source = InputSource::new(String::new(), Vec::from(buffer));
         let mut parser = Parser::new()?;
         parser.feed(source).expect("failed to feed source");
         if let Err(err) = parser.parse() {
             println!("ERROR:\n{}", err);
             return Err(format!("{}", err))
+        }
         debug_assert_eq!(parser.modules.len(), 1, "only supporting one module here for now");
         let modules: Vec<Module> = parser.modules.into_iter()
             .map(|module| Module{
                 source: module.source,
                 root_id: module.root_id,
                 name: module.name.map(|(_, name)| name)
             })
             .collect();
         return Ok(ProtocolDescription {
             modules,
             heap: parser.heap,
             types: parser.type_table,
             pool: Mutex::new(parser.string_pool),
         });
+    }
     pub(crate) fn new_component(
         &self, module_name: &[u8], identifier: &[u8], arguments: ValueGroup
     ) -> Result<Prompt, ComponentCreationError> {
         // Find the module in which the definition can be found
         let module_root = self.lookup_module_root(module_name);
         if module_root.is_none() {
             return Err(ComponentCreationError::ModuleDoesntExist);
+        }
         let module_root = module_root.unwrap();
         let root = &self.heap[module_root];
         let definition_id = root.get_definition_ident(&self.heap, identifier);
         if definition_id.is_none() {
             return Err(ComponentCreationError::DefinitionDoesntExist);
+        }
         let definition_id = definition_id.unwrap();
         let ast_definition = &self.heap[definition_id];
         if !ast_definition.is_procedure() {
             return Err(ComponentCreationError::DefinitionNotComponent);
+        }
         // Make sure that the types of the provided value group matches that of
         // the expected types.
         let ast_definition = ast_definition.as_procedure();
         if !ast_definition.poly_vars.is_empty() || ast_definition.kind == ProcedureKind::Function {
             return Err(ComponentCreationError::DefinitionNotComponent);
+        }
         // - check number of arguments by retrieving the one instantiated
         //   monomorph
         let concrete_type = ConcreteType{ parts: vec![ConcreteTypePart::Component(ast_definition.this, 0)] };
         let procedure_type_id = self.types.get_procedure_monomorph_type_id(&definition_id, &concrete_type.parts).unwrap();
         let procedure_monomorph_index = self.types.get_monomorph(procedure_type_id).variant.as_procedure().monomorph_index;
         let monomorph_info = &ast_definition.monomorphs[procedure_monomorph_index as usize];
         if monomorph_info.argument_types.len() != arguments.values.len() {
             return Err(ComponentCreationError::InvalidNumArguments);
+        }
         // - for each argument try to make sure the types match
         for arg_idx in 0..arguments.values.len() {
             let expected_type_id = monomorph_info.argument_types[arg_idx];
             let expected_type = &self.types.get_monomorph(expected_type_id).concrete_type;
             let provided_value = &arguments.values[arg_idx];
             if !self.verify_same_type(expected_type, 0, &arguments, provided_value) {
                 return Err(ComponentCreationError::InvalidArgumentType(arg_idx));
+            }
+        }
         // By now we're sure that all of the arguments are correct. So create
         // the connector.
         return Ok(Prompt::new(&self.types, &self.heap, ast_definition.this, procedure_type_id, arguments));
+    }
     fn lookup_module_root(&self, module_name: &[u8]) -> Option<RootId> {
         for module in self.modules.iter() {
             match &module.name {
                 Some(name) => if name.as_bytes() == module_name {
                     return Some(module.root_id);
                 },
                 None => if module_name.is_empty() {
                     return Some(module.root_id);
+                }
+            }
+        }
         return None;
+    }
     fn verify_same_type(&self, expected: &ConcreteType, expected_idx: usize, arguments: &ValueGroup, argument: &Value) -> bool {
         use ConcreteTypePart as CTP;
         match &expected.parts[expected_idx] {
             CTP::Void | CTP::Message | CTP::Slice | CTP::Pointer | CTP::Function(_, _) | CTP::Component(_, _) => unreachable!(),
             CTP::Bool => if let Value::Bool(_) = argument { true } else { false },
             CTP::UInt8 => if let Value::UInt8(_) = argument { true } else { false },
             CTP::UInt16 => if let Value::UInt16(_) = argument { true } else { false },
             CTP::UInt32 => if let Value::UInt32(_) = argument { true } else { false },
             CTP::UInt64 => if let Value::UInt64(_) = argument { true } else { false },
             CTP::SInt8 => if let Value::SInt8(_) = argument { true } else { false },
             CTP::SInt16 => if let Value::SInt16(_) = argument { true } else { false },
             CTP::SInt32 => if let Value::SInt32(_) = argument { true } else { false },
             CTP::SInt64 => if let Value::SInt64(_) = argument { true } else { false },
             CTP::Character => if let Value::Char(_) = argument { true } else { false },
             CTP::String => {
                 // Match outer string type and embedded character types
                 if let Value::String(heap_pos) = argument {
                     for element in &arguments.regions[*heap_pos as usize] {
                         if let Value::Char(_) = element {} else {
                             return false;
+                        }
+                    }
                 } else {
                     return false;
+                }
                 return true;
             },
             CTP::Array => {
                 if let Value::Array(heap_pos) = argument {
                     let heap_pos = *heap_pos;
                     for element in &arguments.regions[heap_pos as usize] {
                         if !self.verify_same_type(expected, expected_idx + 1, arguments, element) {
                             return false;
+                        }
+                    }
                     return true;
                 } else {
                     return false;
+                }
             },
             CTP::Input => if let Value::Input(_) = argument { true } else { false },
             CTP::Output => if let Value::Output(_) = argument { true } else { false },
             CTP::Tuple(_) => todo!("implement full type checking on user-supplied arguments"),
             CTP::Instance(definition_id, _num_embedded) => {
                 let definition = self.types.get_base_definition(definition_id).unwrap();
                 match &definition.definition {
                     DefinedTypeVariant::Enum(definition) => {
                         if let Value::Enum(variant_value) = argument {
                             let is_valid = definition.variants.iter()
                                 .any(|v| v.value == *variant_value);
                             return is_valid;
+                        }
                     },
                     _ => todo!("implement full type checking on user-supplied arguments"),
+                }
                 return false;
             },
+        }
+    }
+}
 pub trait RunContext {
     fn performed_put(&mut self, port: PortId) -> bool;
     fn performed_get(&mut self, port: PortId) -> Option<ValueGroup>; // None if still waiting on message
     fn fires(&mut self, port: PortId) -> Option<Value>; // None if not yet branched
     fn performed_fork(&mut self) -> Option<bool>; // None if not yet forked
     fn created_channel(&mut self) -> Option<(Value, Value)>; // None if not yet prepared
     fn performed_select_wait(&mut self) -> Option<u32>; // None if not yet notified runtime of select blocker
+}
 pub struct ProtocolDescriptionBuilder {
     parser: Parser,
+}
 impl ProtocolDescriptionBuilder {
     pub fn new() -> Result<Self, String> {
         return Ok(Self{
             parser: Parser::new()?,
         })
+    }
     pub fn add(&mut self, filename: String, buffer: Vec<u8>) -> Result<(), ParseError> {
         let input = InputSource::new(filename, buffer);
         self.parser.feed(input)?;
         return Ok(())
+    }
     pub fn compile(mut self) -> Result<ProtocolDescription, ParseError> {
         self.parser.parse()?;
         let modules: Vec<Module> = self.parser.modules.into_iter()
             .map(|module| Module{
                 source: module.source,
                 root_id: module.root_id,
                 name: module.name.map(|(_, name)| name)
             })
             .collect();
         return Ok(ProtocolDescription {
             modules,
             heap: self.parser.heap,
             types: self.parser.type_table,
             pool: Mutex::new(self.parser.string_pool),
         });
+    }
+}

src/protocol/parser/mod.rs

➞

Show inline comments

 #[macro_use] mod visitor;
 pub(crate) mod symbol_table;
 pub(crate) mod type_table;
 pub(crate) mod tokens;
 pub(crate) mod token_parsing;
 pub(crate) mod pass_tokenizer;
 pub(crate) mod pass_symbols;
 pub(crate) mod pass_imports;
 pub(crate) mod pass_definitions;
 pub(crate) mod pass_definitions_types;
 pub(crate) mod pass_validation_linking;
 pub(crate) mod pass_rewriting;
 pub(crate) mod pass_typing;
 pub(crate) mod pass_stack_size;
 use tokens::*;
 use crate::collections::*;
 use visitor::Visitor;
 use pass_tokenizer::PassTokenizer;
 use pass_symbols::PassSymbols;
 use pass_imports::PassImport;
 use pass_definitions::PassDefinitions;
 use pass_validation_linking::PassValidationLinking;
 use pass_typing::{PassTyping, ResolveQueue};
 use pass_rewriting::PassRewriting;
 use pass_stack_size::PassStackSize;
 use symbol_table::*;
 use type_table::*;
 use crate::protocol::ast::*;
 use crate::protocol::input_source::*;
 use crate::protocol::ast_writer::ASTWriter;
 use crate::protocol::parser::type_table::PolymorphicVariable;
 use crate::protocol::token_writer::TokenWriter;
 const REOWOLF_PATH_ENV: &'static str = "REOWOLF_ROOT"; // first lookup reowolf path
 const REOWOLF_PATH_DIR: &'static str = "std"; // then try folder in current working directory
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum ModuleCompilationPhase {
     Tokenized,              // source is tokenized
     SymbolsScanned,         // all definitions are linked to their type class
     ImportsResolved,        // all imports are added to the symbol table
     DefinitionsParsed,      // produced the AST for the entire module
     TypesAddedToTable,      // added all definitions to the type table
     ValidatedAndLinked,     // AST is traversed and has linked the required AST nodes
     Typed,                  // Type inference and checking has been performed
     Rewritten,              // Special AST nodes are rewritten into regular AST nodes
     // When we continue with the compiler:
     // StackSize
+}
 pub struct Module {
     // Buffers
     pub source: InputSource,
     pub tokens: TokenBuffer,
     // Identifiers
     pub root_id: RootId,
     pub name: Option<(PragmaId, StringRef<'static>)>,
     pub version: Option<(PragmaId, i64)>,
     pub phase: ModuleCompilationPhase,
+}
 pub struct TargetArch {
     pub void_type_id: TypeId,
     pub message_type_id: TypeId,
     pub bool_type_id: TypeId,
     pub uint8_type_id: TypeId,
     pub uint16_type_id: TypeId,
     pub uint32_type_id: TypeId,
     pub uint64_type_id: TypeId,
     pub sint8_type_id: TypeId,
     pub sint16_type_id: TypeId,
     pub sint32_type_id: TypeId,
     pub sint64_type_id: TypeId,
     pub char_type_id: TypeId,
     pub string_type_id: TypeId,
     pub array_type_id: TypeId,
     pub slice_type_id: TypeId,
     pub input_type_id: TypeId,
     pub output_type_id: TypeId,
     pub pointer_type_id: TypeId,
+}
 impl TargetArch {
     fn new() -> Self {
         return Self{
             void_type_id: TypeId::new_invalid(),
             bool_type_id: TypeId::new_invalid(),
             message_type_id: TypeId::new_invalid(),
             uint8_type_id: TypeId::new_invalid(),
             uint16_type_id: TypeId::new_invalid(),
             uint32_type_id: TypeId::new_invalid(),
             uint64_type_id: TypeId::new_invalid(),
             sint8_type_id: TypeId::new_invalid(),
             sint16_type_id: TypeId::new_invalid(),
             sint32_type_id: TypeId::new_invalid(),
             sint64_type_id: TypeId::new_invalid(),
             char_type_id: TypeId::new_invalid(),
             string_type_id: TypeId::new_invalid(),
             array_type_id: TypeId::new_invalid(),
             slice_type_id: TypeId::new_invalid(),
             input_type_id: TypeId::new_invalid(),
             output_type_id: TypeId::new_invalid(),
             pointer_type_id: TypeId::new_invalid(),
+        }
+    }
+}
 pub struct PassCtx<'a> {
     heap: &'a mut Heap,
     symbols: &'a mut SymbolTable,
     pool: &'a mut StringPool,
     arch: &'a TargetArch,
+}
 pub struct Parser {
     // Storage of all information created/gathered during compilation.
     pub(crate) heap: Heap,
     pub(crate) string_pool: StringPool, // Do not deallocate, holds all strings
     pub(crate) modules: Vec<Module>,
     pub(crate) symbol_table: SymbolTable,
     pub(crate) type_table: TypeTable,
     pub(crate) global_module_index: usize, // contains globals, implicitly imported everywhere
     // Compiler passes, used as little state machine that keep their memory
     // around.
     pass_tokenizer: PassTokenizer,
     pass_symbols: PassSymbols,
     pass_import: PassImport,
     pass_definitions: PassDefinitions,
     pass_validation: PassValidationLinking,
     pass_typing: PassTyping,
     pass_rewriting: PassRewriting,
     pass_stack_size: PassStackSize,
     // Compiler options
     pub write_tokens_to: Option<String>,
     pub write_ast_to: Option<String>,
     pub(crate) arch: TargetArch,
+}
 impl Parser {
     pub fn new() -> Result<Self, String> {
         let mut parser = Parser{
             heap: Heap::new(),
             string_pool: StringPool::new(),
             modules: Vec::new(),
             symbol_table: SymbolTable::new(),
             type_table: TypeTable::new(),
             global_module_index: 0,
             pass_tokenizer: PassTokenizer::new(),
             pass_symbols: PassSymbols::new(),
             pass_import: PassImport::new(),
             pass_definitions: PassDefinitions::new(),
             pass_validation: PassValidationLinking::new(),
             pass_typing: PassTyping::new(),
             pass_rewriting: PassRewriting::new(),
             pass_stack_size: PassStackSize::new(),
             write_tokens_to: Some("tokens.txt".to_string()),
             write_ast_to: None,
             arch: TargetArch::new(),
         };
         parser.symbol_table.insert_scope(None, SymbolScope::Global);
         // Insert builtin types
         // TODO: At some point use correct values for size/alignment
         parser.arch.void_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Void], false, 0, 1);
         parser.arch.message_type_id = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Message], false, 24, 8);
         parser.arch.bool_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Bool], false, 1, 1);
         parser.arch.uint8_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt8], false, 1, 1);
         parser.arch.uint16_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt16], false, 2, 2);
         parser.arch.uint32_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt32], false, 4, 4);
         parser.arch.uint64_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt64], false, 8, 8);
         parser.arch.sint8_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt8], false, 1, 1);
         parser.arch.sint16_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt16], false, 2, 2);
         parser.arch.sint32_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt32], false, 4, 4);
         parser.arch.sint64_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt64], false, 8, 8);
         parser.arch.char_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Character], false, 4, 4);
         parser.arch.string_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::String], false, 24, 8);
         parser.arch.array_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Array, ConcreteTypePart::Void], true, 24, 8);
         parser.arch.slice_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Slice, ConcreteTypePart::Void], true, 16, 4);
         parser.arch.input_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Input, ConcreteTypePart::Void], true, 8, 8);
         parser.arch.output_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Output, ConcreteTypePart::Void], true, 8, 8);
         parser.arch.pointer_type_id = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Pointer, ConcreteTypePart::Void], true, 8, 8);
         // Parse standard library
         parser.feed_standard_library()?;
         return Ok(parser)
+    }
     /// Feeds a new InputSource to the parser, which will tokenize it and store
     /// it internally for later parsing (when all modules are present). Returns
     /// the index of the new module.
     pub fn feed(&mut self, mut source: InputSource) -> Result<usize, ParseError> {
         let mut token_buffer = TokenBuffer::new();
         self.pass_tokenizer.tokenize(&mut source, &mut token_buffer)?;
         let module = Module{
             source,
             tokens: token_buffer,
             root_id: RootId::new_invalid(),
             name: None,
             version: None,
             phase: ModuleCompilationPhase::Tokenized,
         };
         let module_index = self.modules.len();
         self.modules.push(module);
         return Ok(module_index);
+    }
     pub fn parse(&mut self) -> Result<(), ParseError> {
         let mut pass_ctx = PassCtx{
             heap: &mut self.heap,
             symbols: &mut self.symbol_table,
             pool: &mut self.string_pool,
             arch: &self.arch,
         };
         if let Some(filename) = &self.write_tokens_to {
             let mut writer = TokenWriter::new();
             let mut file = std::fs::File::create(std::path::Path::new(filename)).unwrap();
             writer.write(&mut file, &self.modules);
+        }
         // Advance all modules to the phase where all symbols are scanned
         for module_idx in 0..self.modules.len() {
             self.pass_symbols.parse(&mut self.modules, module_idx, &mut pass_ctx)?;
+        }
         // With all symbols scanned, perform further compilation until we can
         // add all base types to the type table.
         for module_idx in 0..self.modules.len() {
             self.pass_import.parse(&mut self.modules, module_idx, &mut pass_ctx)?;
             self.pass_definitions.parse(&mut self.modules, module_idx, &mut pass_ctx)?;
+        }
         // Add every known type to the type table
         self.type_table.build_base_types(&mut self.modules, &mut pass_ctx)?;
         // Continue compilation with the remaining phases now that the types
         // are all in the type table
         for module_idx in 0..self.modules.len() {
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_validation.visit_module(&mut ctx)?;
+        }
         // Perform typechecking on all modules
         let mut queue = ResolveQueue::new();
         for module_idx in 0..self.modules.len() {
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_typing.queue_module_definitions(&mut ctx, &mut queue);
         };
         while !queue.is_empty() {
             let top = queue.pop_front().unwrap();
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx: top.root_id.index as usize,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_typing.handle_module_definition(&mut ctx, &mut queue, top)?;
+        }
         // Rewrite nodes in tree, then prepare for execution of code
         for module_idx in 0..self.modules.len() {
             self.modules[module_idx].phase = ModuleCompilationPhase::Typed;
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_rewriting.visit_module(&mut ctx)?;
             self.pass_stack_size.visit_module(&mut ctx)?;
+        }
         // Write out desired information
         if let Some(filename) = &self.write_ast_to {
             let mut writer = ASTWriter::new();
             let mut file = std::fs::File::create(std::path::Path::new(filename)).unwrap();
             writer.write_ast(&mut file, &self.heap);
+        }
         Ok(())
+    }
     /// Tries to find the standard library and add the files for parsing.
     fn feed_standard_library(&mut self) -> Result<(), String> {
         use std::env;
         use std::path::{Path, PathBuf};
         use std::fs;
         const FILES: [&'static str; 1] = [
             "std.global.pdl",
         ];
         // Determine base directory
         let (base_path, from_env) = if let Ok(path) = env::var(REOWOLF_PATH_ENV) {
             // Path variable is set
             (path, true)
         } else {
             let mut path = String::with_capacity(REOWOLF_PATH_DIR.len() + 2);
             path.push_str("./");
             path.push_str(REOWOLF_PATH_DIR);
             (path, false)
         };
         // Make sure directory exists
         let path = Path::new(&base_path);
         if !path.exists() {
             return Err(format!("std lib root directory '{}' does not exist", base_path));
+        }
         // Try to load all standard library files. We might need a more unified
         // way to do this in the future (i.e. a "std" package, containing all
         // of the modules)
         let mut file_path = PathBuf::new();
         let mut first_file = true;
         for file in FILES {
             file_path.push(path);
             file_path.push(file);
             let source = fs::read(file_path.as_path());
             if let Err(err) = source {
                 return Err(format!(
                     "failed to read std lib file '{}' in root directory '{}', because: {}",
                     file, base_path, err
                 ));
+            }
             let source = source.unwrap();
             let input_source = InputSource::new(file.to_string(), source);
             let module_index = self.feed(input_source);
             if let Err(err) = module_index {
                 // A bit of a hack, but shouldn't really happen anyway: the
                 // compiler should ship with a decent standard library (at some
                 // point)
                 return Err(format!("{}", err));
+            }
             let module_index = module_index.unwrap();
             if first_file {
                 self.global_module_index = module_index;
                 first_file = false;
+            }
+        }
         return Ok(())
+    }
+}
 fn insert_builtin_type(type_table: &mut TypeTable, parts: Vec<ConcreteTypePart>, has_poly_var: bool, size: usize, alignment: usize) -> TypeId {
     const POLY_VARS: [PolymorphicVariable; 1] = [PolymorphicVariable{
         identifier: Identifier::new_empty(InputSpan::new()),
         is_in_use: false,
     }];
     let concrete_type = ConcreteType{ parts };
     let poly_var = if has_poly_var {
         POLY_VARS.as_slice()
     } else {
         &[]
     };
     return type_table.add_builtin_data_type(concrete_type, poly_var, size, alignment);
+}
@@ \ No newline at end of file @@

src/protocol/parser/tokens.rs

➞

Show inline comments

 use crate::protocol::input_source::{
     InputPosition as InputPosition,
     InputSpan
 };
 /// Represents a particular kind of token. Some tokens represent
 /// variable-character tokens. Such a token is always followed by a
 /// `TokenKind::SpanEnd` token.
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub enum TokenKind {
     // Variable-character tokens, followed by a SpanEnd token
     Ident,          // regular identifier
     Pragma,         // identifier with prefixed `#`, range includes `#`
     Integer,        // integer literal
     String,         // string literal, range includes `"`
     Character,      // character literal, range includes `'`
     LineComment,    // line comment, range includes leading `//`, but not newline
     BlockComment,   // block comment, range includes leading `/*` and trailing `*/`
     // Punctuation (single character)
     Exclamation,    // !
     Question,       // ?
     Pound,          // #
     OpenAngle,      // <
     OpenCurly,      // {
     OpenParen,      // (
     OpenSquare,     // [
     CloseAngle,     // >
     CloseCurly,     // }
     CloseParen,     // )
     CloseSquare,    // ]
     Colon,          // :
     Comma,          // ,
     Dot,            // .
     SemiColon,      // ;
     // Operator-like (single character)
     At,             // @
     Plus,           // +
     Minus,          // -
     Star,           // *
     Slash,          // /
     Percent,        // %
     Caret,          // ^
     And,            // &
     Or,             // |
     Tilde,          // ~
     Equal,          // =
     // Punctuation (two characters)
     ColonColon,     // ::
     DotDot,         // ..
     ArrowRight,     // ->
     // Operator-like (two characters)
     AtEquals,       // @=
     PlusPlus,       // ++
     PlusEquals,     // +=
     MinusMinus,     // --
     MinusEquals,    // -=
     StarEquals,     // *=
     SlashEquals,    // /=
     PercentEquals,  // %=
     CaretEquals,    // ^=
     AndAnd,         // &&
     AndEquals,      // &=
     OrOr,           // ||
     OrEquals,       // |=
     EqualEqual,     // ==
     NotEqual,       // !=
     ShiftLeft,      // <<
     LessEquals,     // <=
     ShiftRight,     // >>
     GreaterEquals,  // >=
     // Operator-like (three characters)
     ShiftLeftEquals,// <<=
     ShiftRightEquals, // >>=
     // Special marker token to indicate end of variable-character tokens
     SpanEnd,
+}
 impl TokenKind {
     /// Returns true if the next expected token is the special `TokenKind::SpanEnd` token. This is
     /// the case for tokens of variable length (e.g. an identifier).
     fn has_span_end(&self) -> bool {
+    pub(crate) fn has_span_end(&self) -> bool {
         return *self <= TokenKind::BlockComment
+    }
     /// Returns the number of characters associated with the token. May only be called on tokens
     /// that do not have a variable length.
     fn num_characters(&self) -> u32 {
         debug_assert!(!self.has_span_end() && *self != TokenKind::SpanEnd);
         if *self <= TokenKind::Equal {
         } else if *self <= TokenKind::GreaterEquals {
         } else {
+        }
+    }
     /// Returns the characters that are represented by the token, may only be called on tokens that
     /// do not have a variable length.
     pub fn token_chars(&self) -> &'static str {
         debug_assert!(!self.has_span_end() && *self != TokenKind::SpanEnd);
         use TokenKind as TK;
         match self {
             TK::Exclamation => "!",
             TK::Question => "?",
             TK::Pound => "#",
             TK::OpenAngle => "<",
             TK::OpenCurly => "{",
             TK::OpenParen => "(",
             TK::OpenSquare => "[",
             TK::CloseAngle => ">",
             TK::CloseCurly => "}",
             TK::CloseParen => ")",
             TK::CloseSquare => "]",
             TK::Colon => ":",
             TK::Comma => ",",
             TK::Dot => ".",
             TK::SemiColon => ";",
             TK::At => "@",
             TK::Plus => "+",
             TK::Minus => "-",
             TK::Star => "*",
             TK::Slash => "/",
             TK::Percent => "%",
             TK::Caret => "^",
             TK::And => "&",
             TK::Or => "|",
             TK::Tilde => "~",
             TK::Equal => "=",
             TK::ColonColon => "::",
             TK::DotDot => "..",
             TK::ArrowRight => "->",
             TK::AtEquals => "@=",
             TK::PlusPlus => "++",
             TK::PlusEquals => "+=",
             TK::MinusMinus => "--",
             TK::MinusEquals => "-=",
             TK::StarEquals => "*=",
             TK::SlashEquals => "/=",
             TK::PercentEquals => "%=",
             TK::CaretEquals => "^=",
             TK::AndAnd => "&&",
             TK::AndEquals => "&=",
             TK::OrOr => "||",
             TK::OrEquals => "|=",
             TK::EqualEqual => "==",
             TK::NotEqual => "!=",
             TK::ShiftLeft => "<<",
             TK::LessEquals => "<=",
             TK::ShiftRight => ">>",
             TK::GreaterEquals => ">=",
             TK::ShiftLeftEquals => "<<=",
             TK::ShiftRightEquals => ">>=",
             // Lets keep these in explicitly for now, in case we want to add more symbols
             TK::Ident | TK::Pragma | TK::Integer | TK::String | TK::Character |
             TK::LineComment | TK::BlockComment | TK::SpanEnd => unreachable!(),
+        }
+    }
+}
 /// Represents a single token at a particular position.
 pub struct Token {
     pub kind: TokenKind,
     pub pos: InputPosition,
+}
 impl Token {
     pub(crate) fn new(kind: TokenKind, pos: InputPosition) -> Self {
         Self{ kind, pos }
+    }
+}
 /// The kind of token ranges that are specially parsed by the tokenizer.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum TokenRangeKind {
     Module,
     Pragma,
     Import,
     Definition,
     Code,
+}
 pub const NO_RELATION: i32 = -1;
 pub const NO_SIBLING: i32 = NO_RELATION;
 /// A range of tokens with a specific meaning. Such a range is part of a tree
 /// where each parent tree envelops all of its children.
 #[derive(Debug)]
 pub struct TokenRange {
     // Index of parent in `TokenBuffer.ranges`, does not have a parent if the
     // range kind is Module, in that case the parent index is -1.
     pub parent_idx: i32,
     pub range_kind: TokenRangeKind,
     pub curly_depth: u32,
     // Offsets into `TokenBuffer.ranges`: the tokens belonging to this range.
     pub start: u32,             // first token (inclusive index)
     pub end: u32,               // last token (exclusive index)
     // Child ranges
     pub num_child_ranges: u32,  // Number of subranges
     pub first_child_idx: i32,   // First subrange (or -1 if no subranges)
     pub last_child_idx: i32,    // Last subrange (or -1 if no subranges)
     pub next_sibling_idx: i32,  // Next subrange (or -1 if no next subrange)
+}
 pub struct TokenBuffer {
     pub tokens: Vec<Token>,
     pub ranges: Vec<TokenRange>,
+}
 impl TokenBuffer {
     pub(crate) fn new() -> Self {
         Self{ tokens: Vec::new(), ranges: Vec::new() }
+    }
     pub(crate) fn iter_range<'a>(&'a self, range: &TokenRange) -> TokenIter<'a> {
         TokenIter::new(self, range.start as usize, range.end as usize)
+    }
     pub(crate) fn start_pos(&self, range: &TokenRange) -> InputPosition {
         self.tokens[range.start as usize].pos
+    }
     pub(crate) fn end_pos(&self, range: &TokenRange) -> InputPosition {
         let last_token = &self.tokens[range.end as usize - 1];
         if last_token.kind == TokenKind::SpanEnd {
             return last_token.pos
         } else {
             debug_assert!(!last_token.kind.has_span_end());
             return last_token.pos.with_offset(last_token.kind.num_characters());
+        }
+    }
+}
 /// Iterator over tokens within a specific `TokenRange`.
 pub(crate) struct TokenIter<'a> {
     tokens: &'a Vec<Token>,
     cur: usize,
     end: usize,
+}
 impl<'a> TokenIter<'a> {
     fn new(buffer: &'a TokenBuffer, start: usize, end: usize) -> Self {
         Self{ tokens: &buffer.tokens, cur: start, end }
+    }
     /// Returns the next token (may include comments), or `None` if at the end
     /// of the range.
     pub(crate) fn next_including_comments(&self) -> Option<TokenKind> {
         if self.cur >= self.end {
             return None;
+        }
         let token = &self.tokens[self.cur];
         Some(token.kind)
+    }
     /// Returns the next token (but skips over comments), or `None` if at the
     /// end of the range
     pub(crate) fn next(&mut self) -> Option<TokenKind> {
         while let Some(token_kind) = self.next_including_comments() {
             if token_kind != TokenKind::LineComment && token_kind != TokenKind::BlockComment {
                 return Some(token_kind);
+            }
             self.consume();
+        }
         return None
+    }
     /// Peeks ahead by one token (i.e. the one that comes after `next()`), and
     /// skips over comments
     pub(crate) fn peek(&self) -> Option<TokenKind> {
         for next_idx in self.cur + 1..self.end {
             let next_kind = self.tokens[next_idx].kind;
             if next_kind != TokenKind::LineComment && next_kind != TokenKind::BlockComment && next_kind != TokenKind::SpanEnd {
                 return Some(next_kind);
+            }
+        }
         return None;
+    }
     /// Returns the start position belonging to the token returned by `next`. If
     /// there is not a next token, then we return the end position of the
     /// previous token.
     pub(crate) fn last_valid_pos(&self) -> InputPosition {
         if self.cur < self.end {
             // Return token position
             return self.tokens[self.cur].pos
+        }
         // Return previous token end
         let token = &self.tokens[self.cur - 1];
         return if token.kind == TokenKind::SpanEnd {
             token.pos
         } else {
             token.pos.with_offset(token.kind.num_characters())
         };
+    }
     /// Assumes the token is not at the end and returns the starting position
     /// belonging to the token returned by `next`.
     pub(crate) fn next_start_position(&self) -> InputPosition {
         debug_assert!(self.cur < self.end);
         return self.tokens[self.cur].pos;
+    }
     /// Returns the token range belonging to the token returned by `next`. This
     /// assumes that we're not at the end of the range we're iterating over.
     pub(crate) fn next_positions(&self) -> (InputPosition, InputPosition) {
         debug_assert!(self.cur < self.end);
         let token = &self.tokens[self.cur];
         if token.kind.has_span_end() {
             let span_end = &self.tokens[self.cur + 1];
             debug_assert_eq!(span_end.kind, TokenKind::SpanEnd);
             (token.pos, span_end.pos)
         } else {
             let offset = token.kind.num_characters();
             (token.pos, token.pos.with_offset(offset))
+        }
+    }
     /// See `next_positions`
     pub(crate) fn next_span(&self) -> InputSpan {
         let (begin, end) = self.next_positions();
         return InputSpan::from_positions(begin, end)
+    }
     /// Advances the iterator to the next (meaningful) token.
     pub(crate) fn consume(&mut self) {
         if let Some(kind) = self.next_including_comments() {
             if kind.has_span_end() {
                 self.cur += 2;
             } else {
                 self.cur += 1;
+            }
+        }
+    }
     /// Saves the current iteration position, may be passed to `load` to return
     /// the iterator to a previous position.
     pub(crate) fn save(&self) -> (usize, usize) {
         (self.cur, self.end)
+    }
     pub(crate) fn load(&mut self, saved: (usize, usize)) {
         self.cur = saved.0;
         self.end = saved.1;
+    }
+}
@@ \ No newline at end of file @@

src/protocol/token_writer.rs

➞

Show inline comments

@@ new file 100644 @@
 #![allow(dead_code)]
 use std::fmt::Write;
 use std::io::Write as IOWrite;
 use crate::protocol::input_source::{InputSource, InputSpan};
 use crate::protocol::parser::Module;
 use crate::protocol::tokens::{Token, TokenBuffer, TokenKind, TokenRange, TokenIter, TokenRangeKind};
 pub(crate) struct TokenWriter {
     buffer: String,
+}
 impl TokenWriter {
     pub(crate) fn new() -> Self {
         return Self{
             buffer: String::with_capacity(4096),
+        }
+    }
     pub(crate) fn write<W: IOWrite>(&mut self, w: &mut W, modules: &[Module]) {
         self.buffer.clear();
         for module in modules {
             self.write_module_tokens(module);
+        }
         w.write_all(self.buffer.as_bytes()).expect("write tokens");
+    }
     fn write_module_tokens(&mut self, module: &Module) {
         self.write_dashed_indent(0);
         match &module.name {
             Some(name) => writeln!(self.buffer, "Module: {}", name.1.as_str()).unwrap(),
             None => self.buffer.push_str("Unnamed module\n"),
+        }
         let mut range_index = -1;
         if !module.tokens.ranges.is_empty() {
             range_index = 0;
+        }
         while range_index >= 0 {
             range_index = self.write_token_range(
                 &module.source, &module.tokens.tokens, &module.tokens.ranges, range_index, 1
             );
+        }
+    }
     /// Writes a single token range. Recurses if there are any child ranges.
     /// Returns the next token range index to iterate over (or a negative
     /// number, if there are no more sibling ranges).
     fn write_token_range(&mut self, source: &InputSource, tokens: &[Token], ranges: &[TokenRange], range_index: i32, indent: u32) -> i32 {
         // Write range kind
         let range = &ranges[range_index as usize];
         self.write_dashed_indent(indent);
         writeln!(self.buffer, "Range: {:?}", range.range_kind);
         // Write tokens/lines it spans
         let first_token_pos = tokens[range.start as usize].pos;
         let last_token_pos = if (range.end as usize) < tokens.len() {
             tokens[range.end as usize].pos
         } else {
             tokens.last().unwrap().pos
         };
         let first_source_col = source.get_column(first_token_pos);
         let last_source_col = source.get_column(last_token_pos);
         self.write_indent(indent);
         writeln!(
             self.buffer, "Source: token {} to {}, file {}:{}:{} to {}:{}",
             range.start, range.end, source.filename,
             first_token_pos.line, first_source_col,
             last_token_pos.line, last_source_col
         );
         let next_sibling_index = range.next_sibling_idx;
         if range.num_child_ranges == 0 {
             // No child ranges, so dump the tokens here
             debug_assert!(range.first_child_idx < 0);
             self.write_token_array(source, tokens, range, indent);
         } else {
             // Child ranges
             debug_assert!(range.first_child_idx >= 0);
             self.write_indent(indent);
             writeln!(self.buffer, "Children: [");
             let mut range_index = range.first_child_idx;
             while range_index >= 0 {
                 range_index = self.write_token_range(source, tokens, ranges, range_index, indent + 1);
+            }
             self.write_indent(indent);
             writeln!(self.buffer, "]");
+        }
         // Wrote everything, return the next sibling token range
         return next_sibling_index;
+    }
     fn write_token_array(&mut self, source: &InputSource, tokens: &[Token], range: &TokenRange, indent: u32) {
         self.write_indent(indent);
         writeln!(self.buffer, "Tokens: [");
         let token_indent = indent + 1;
         for token_index in range.start as usize..range.end as usize {
             // Skip uninteresting tokens
             let token = &tokens[token_index];
             if token.kind == TokenKind::SpanEnd {
                 continue;
+            }
             self.write_indent(token_indent);
             write!(self.buffer, "{:?} (index {})", token.kind, token_index);
             if token.kind.has_span_end() {
                 let token_start = token.pos;
                 let token_end = tokens[token_index + 1].pos;
                 let section = source.section_at_span(InputSpan::from_positions(token_start, token_end));
                 writeln!(self.buffer, " text: {}", String::from_utf8_lossy(section));
             } else {
                 self.buffer.push('\n');
+            }
+        }
         self.write_indent(indent);
         writeln!(self.buffer, "]");
+    }
     fn write_dashed_indent(&mut self, indent: u32) {
         for _ in 0..indent * 2 {
             self.buffer.push(' ');
+        }
         self.buffer.push('-');
         self.buffer.push(' ');
+    }
     fn write_indent(&mut self, indent: u32) {
         for _ in 0..(indent + 1)*2 {
             self.buffer.push(' ');
+        }
+    }
+}
@@ \ No newline at end of file @@

tokens.txt

➞

Show inline comments

@@ new file 100644 @@
 - Unnamed module
   - Range: Module
     Source: token 0 to 167, file std.global.pdl:1:1 to 9:53
     Children: [
     - Range: Pragma
       Source: token 0 to 7, file std.global.pdl:1:1 to 3:1
       Tokens: [
         Pragma (index 0) text: #module
         Ident (index 2) text: std
         Dot (index 4)
         Ident (index 5) text: global
+      ]
     - Range: Definition
       Source: token 7 to 167, file std.global.pdl:3:1 to 9:53
       Children: [
       - Range: Code
         Source: token 7 to 29, file std.global.pdl:3:1 to 3:33
         Tokens: [
           Ident (index 7) text: func
           Ident (index 9) text: get
           OpenAngle (index 11)
           Ident (index 12) text: T
           CloseAngle (index 14)
           OpenParen (index 15)
           Ident (index 16) text: in
           OpenAngle (index 18)
           Ident (index 19) text: T
           CloseAngle (index 21)
           Ident (index 22) text: input
           CloseParen (index 24)
           ArrowRight (index 25)
           Ident (index 26) text: T
           OpenCurly (index 28)
+        ]
       - Range: Pragma
         Source: token 29 to 32, file std.global.pdl:3:33 to 4:1
         Tokens: [
           Pragma (index 29) text: #builtin
           CloseCurly (index 31)
+        ]
       - Range: Definition
         Source: token 32 to 167, file std.global.pdl:4:1 to 9:53
         Children: [
         - Range: Code
           Source: token 32 to 56, file std.global.pdl:4:1 to 4:40
           Tokens: [
             Ident (index 32) text: func
             Ident (index 34) text: put
             OpenAngle (index 36)
             Ident (index 37) text: T
             CloseAngle (index 39)
             OpenParen (index 40)
             Ident (index 41) text: out
             OpenAngle (index 43)
             Ident (index 44) text: T
             CloseAngle (index 46)
             Ident (index 47) text: output
             Comma (index 49)
             Ident (index 50) text: T
             Ident (index 52) text: value
             CloseParen (index 54)
             ArrowRight (index 55)
+          ]
         - Range: Pragma
           Source: token 56 to 167, file std.global.pdl:4:40 to 9:53
           Children: [
           - Range: Code
             Source: token 56 to 59, file std.global.pdl:4:40 to 4:53
             Tokens: [
               Pragma (index 56) text: #type_void
               OpenCurly (index 58)
+            ]
           - Range: Pragma
             Source: token 59 to 62, file std.global.pdl:4:53 to 5:1
             Tokens: [
               Pragma (index 59) text: #builtin
               CloseCurly (index 61)
+            ]
           - Range: Definition
             Source: token 62 to 167, file std.global.pdl:5:1 to 9:53
             Children: [
             - Range: Code
               Source: token 62 to 71, file std.global.pdl:5:1 to 5:15
               Tokens: [
                 Ident (index 62) text: func
                 Ident (index 64) text: fires
                 OpenAngle (index 66)
                 Ident (index 67) text: T
                 CloseAngle (index 69)
                 OpenParen (index 70)
+              ]
             - Range: Pragma
               Source: token 71 to 167, file std.global.pdl:5:15 to 9:53
               Children: [
               - Range: Code
                 Source: token 71 to 82, file std.global.pdl:5:15 to 5:44
                 Tokens: [
                   Pragma (index 71) text: #type_portlike
                   OpenAngle (index 73)
                   Ident (index 74) text: T
                   CloseAngle (index 76)
                   CloseParen (index 77)
                   ArrowRight (index 78)
                   Ident (index 79) text: bool
                   OpenCurly (index 81)
+                ]
               - Range: Pragma
                 Source: token 82 to 85, file std.global.pdl:5:44 to 6:1
                 Tokens: [
                   Pragma (index 82) text: #builtin
                   CloseCurly (index 84)
+                ]
               - Range: Definition
                 Source: token 85 to 167, file std.global.pdl:6:1 to 9:53
                 Children: [
                 - Range: Code
                   Source: token 85 to 94, file std.global.pdl:6:1 to 6:16
                   Tokens: [
                     Ident (index 85) text: func
                     Ident (index 87) text: create
                     OpenAngle (index 89)
                     Ident (index 90) text: T
                     CloseAngle (index 92)
                     OpenParen (index 93)
+                  ]
                 - Range: Pragma
                   Source: token 94 to 167, file std.global.pdl:6:16 to 9:53
                   Children: [
                   - Range: Code
                     Source: token 94 to 105, file std.global.pdl:6:16 to 6:51
                     Tokens: [
                       Pragma (index 94) text: #type_integerlike
                       Ident (index 96) text: length
                       CloseParen (index 98)
                       ArrowRight (index 99)
                       Ident (index 100) text: T
                       OpenSquare (index 102)
                       CloseSquare (index 103)
                       OpenCurly (index 104)
+                    ]
                   - Range: Pragma
                     Source: token 105 to 108, file std.global.pdl:6:51 to 7:1
                     Tokens: [
                       Pragma (index 105) text: #builtin
                       CloseCurly (index 107)
+                    ]
                   - Range: Definition
                     Source: token 108 to 167, file std.global.pdl:7:1 to 9:53
                     Children: [
                     - Range: Code
                       Source: token 108 to 117, file std.global.pdl:7:1 to 7:16
                       Tokens: [
                         Ident (index 108) text: func
                         Ident (index 110) text: length
                         OpenAngle (index 112)
                         Ident (index 113) text: T
                         CloseAngle (index 115)
                         OpenParen (index 116)
+                      ]
                     - Range: Pragma
                       Source: token 117 to 167, file std.global.pdl:7:16 to 9:53
                       Children: [
                       - Range: Code
                         Source: token 117 to 130, file std.global.pdl:7:16 to 7:51
                         Tokens: [
                           Pragma (index 117) text: #type_arraylike
                           OpenAngle (index 119)
                           Ident (index 120) text: T
                           CloseAngle (index 122)
                           Ident (index 123) text: array
                           CloseParen (index 125)
                           ArrowRight (index 126)
                           Ident (index 127) text: u32
                           OpenCurly (index 129)
+                        ]
                       - Range: Pragma
                         Source: token 130 to 133, file std.global.pdl:7:51 to 8:1
                         Tokens: [
                           Pragma (index 130) text: #builtin
                           CloseCurly (index 132)
+                        ]
                       - Range: Definition
                         Source: token 133 to 167, file std.global.pdl:8:1 to 9:53
                         Children: [
                         - Range: Code
                           Source: token 133 to 144, file std.global.pdl:8:1 to 8:32
                           Tokens: [
                             Ident (index 133) text: func
                             Ident (index 135) text: assert
                             OpenParen (index 137)
                             Ident (index 138) text: bool
                             Ident (index 140) text: condition
                             CloseParen (index 142)
                             ArrowRight (index 143)
+                          ]
                         - Range: Pragma
                           Source: token 144 to 167, file std.global.pdl:8:32 to 9:53
                           Children: [
                           - Range: Code
                             Source: token 144 to 147, file std.global.pdl:8:32 to 8:45
                             Tokens: [
                               Pragma (index 144) text: #type_void
                               OpenCurly (index 146)
+                            ]
                           - Range: Pragma
                             Source: token 147 to 150, file std.global.pdl:8:45 to 9:1
                             Tokens: [
                               Pragma (index 147) text: #builtin
                               CloseCurly (index 149)
+                            ]
                           - Range: Definition
                             Source: token 150 to 167, file std.global.pdl:9:1 to 9:53
                             Children: [
                             - Range: Code
                               Source: token 150 to 161, file std.global.pdl:9:1 to 9:31
                               Tokens: [
                                 Ident (index 150) text: func
                                 Ident (index 152) text: print
                                 OpenParen (index 154)
                                 Ident (index 155) text: string
                                 Ident (index 157) text: message
                                 CloseParen (index 159)
                                 ArrowRight (index 160)
+                              ]
                             - Range: Pragma
                               Source: token 161 to 167, file std.global.pdl:9:31 to 9:53
                               Children: [
                               - Range: Code
                                 Source: token 161 to 164, file std.global.pdl:9:31 to 9:44
                                 Tokens: [
                                   Pragma (index 161) text: #type_void
                                   OpenCurly (index 163)
+                                ]
                               - Range: Pragma
                                 Source: token 164 to 167, file std.global.pdl:9:44 to 9:53
                                 Tokens: [
                                   Pragma (index 164) text: #builtin
                                   CloseCurly (index 166)
+                                ]
+                              ]
+                            ]
+                          ]
+                        ]
+                      ]
+                    ]
+                  ]
+                ]
+              ]
+            ]
+          ]
+        ]
+      ]
+    ]
 - Unnamed module
   - Range: Module
     Source: token 0 to 331, file :2:5 to 43:5
     Children: [
     - Range: Definition
       Source: token 0 to 47, file :2:5 to 7:5
       Tokens: [
         Ident (index 0) text: func
         Ident (index 2) text: infinite_assert
         OpenAngle (index 4)
         Ident (index 5) text: T
         CloseAngle (index 7)
         OpenParen (index 8)
         Ident (index 9) text: T
         Ident (index 11) text: val
         Comma (index 13)
         Ident (index 14) text: T
         Ident (index 16) text: expected
         CloseParen (index 18)
         ArrowRight (index 19)
         OpenParen (index 20)
         CloseParen (index 21)
         OpenCurly (index 22)
         Ident (index 23) text: while
         OpenParen (index 25)
         Ident (index 26) text: val
         NotEqual (index 28)
         Ident (index 29) text: expected
         CloseParen (index 31)
         OpenCurly (index 32)
         Ident (index 33) text: print
         OpenParen (index 35)
         String (index 36) text: "nope!"
         CloseParen (index 38)
         SemiColon (index 39)
         CloseCurly (index 40)
         Ident (index 41) text: return
         OpenParen (index 43)
         CloseParen (index 44)
         SemiColon (index 45)
         CloseCurly (index 46)
+      ]
     - Range: Definition
       Source: token 47 to 200, file :7:5 to 26:5
       Tokens: [
         Ident (index 47) text: primitive
         Ident (index 49) text: receiver
         OpenParen (index 51)
         Ident (index 52) text: in
         OpenAngle (index 54)
         Ident (index 55) text: u32
         CloseAngle (index 57)
         Ident (index 58) text: in_a
         Comma (index 60)
         Ident (index 61) text: in
         OpenAngle (index 63)
         Ident (index 64) text: u32
         CloseAngle (index 66)
         Ident (index 67) text: in_b
         Comma (index 69)
         Ident (index 70) text: u32
         Ident (index 72) text: num_sends
         CloseParen (index 74)
         OpenCurly (index 75)
         Ident (index 76) text: auto
         Ident (index 78) text: num_from_a
         Equal (index 80)
         Integer (index 81) text: 0
         SemiColon (index 83)
         Ident (index 84) text: auto
         Ident (index 86) text: num_from_b
         Equal (index 88)
         Integer (index 89) text: 0
         SemiColon (index 91)
         Ident (index 92) text: while
         OpenParen (index 94)
         Ident (index 95) text: num_from_a
         Plus (index 97)
         Ident (index 98) text: num_from_b
         OpenAngle (index 100)
         Integer (index 101) text: 2
         Star (index 103)
         Ident (index 104) text: num_sends
         CloseParen (index 106)
         OpenCurly (index 107)
         Ident (index 108) text: sync
         Ident (index 110) text: select
         OpenCurly (index 112)
         Ident (index 113) text: auto
         Ident (index 115) text: v
         Equal (index 117)
         Ident (index 118) text: get
         OpenParen (index 120)
         Ident (index 121) text: in_a
         CloseParen (index 123)
         ArrowRight (index 124)
         OpenCurly (index 125)
         Ident (index 126) text: print
         OpenParen (index 128)
         String (index 129) text: "got something from A"
         CloseParen (index 131)
         SemiColon (index 132)
         Ident (index 133) text: auto
         Ident (index 135) text: _
         Equal (index 137)
         Ident (index 138) text: infinite_assert
         OpenParen (index 140)
         Ident (index 141) text: v
         Comma (index 143)
         Ident (index 144) text: num_from_a
         CloseParen (index 146)
         SemiColon (index 147)
         Ident (index 148) text: num_from_a
         PlusEquals (index 150)
         Integer (index 151) text: 1
         SemiColon (index 153)
         CloseCurly (index 154)
         Ident (index 155) text: auto
         Ident (index 157) text: v
         Equal (index 159)
         Ident (index 160) text: get
         OpenParen (index 162)
         Ident (index 163) text: in_b
         CloseParen (index 165)
         ArrowRight (index 166)
         OpenCurly (index 167)
         Ident (index 168) text: print
         OpenParen (index 170)
         String (index 171) text: "got something from B"
         CloseParen (index 173)
         SemiColon (index 174)
         Ident (index 175) text: auto
         Ident (index 177) text: _
         Equal (index 179)
         Ident (index 180) text: infinite_assert
         OpenParen (index 182)
         Ident (index 183) text: v
         Comma (index 185)
         Ident (index 186) text: num_from_b
         CloseParen (index 188)
         SemiColon (index 189)
         Ident (index 190) text: num_from_b
         PlusEquals (index 192)
         Integer (index 193) text: 1
         SemiColon (index 195)
         CloseCurly (index 196)
         CloseCurly (index 197)
         CloseCurly (index 198)
         CloseCurly (index 199)
+      ]
     - Range: Definition
       Source: token 200 to 260, file :26:5 to 36:5
       Tokens: [
         Ident (index 200) text: primitive
         Ident (index 202) text: sender
         OpenParen (index 204)
         Ident (index 205) text: out
         OpenAngle (index 207)
         Ident (index 208) text: u32
         CloseAngle (index 210)
         Ident (index 211) text: tx
         Comma (index 213)
         Ident (index 214) text: u32
         Ident (index 216) text: num_sends
         CloseParen (index 218)
         OpenCurly (index 219)
         Ident (index 220) text: auto
         Ident (index 222) text: index
         Equal (index 224)
         Integer (index 225) text: 0
         SemiColon (index 227)
         Ident (index 228) text: while
         OpenParen (index 230)
         Ident (index 231) text: index
         OpenAngle (index 233)
         Ident (index 234) text: num_sends
         CloseParen (index 236)
         OpenCurly (index 237)
         Ident (index 238) text: sync
         OpenCurly (index 240)
         Ident (index 241) text: put
         OpenParen (index 243)
         Ident (index 244) text: tx
         Comma (index 246)
         Ident (index 247) text: index
         CloseParen (index 249)
         SemiColon (index 250)
         Ident (index 251) text: index
         PlusEquals (index 253)
         Integer (index 254) text: 1
         SemiColon (index 256)
         CloseCurly (index 257)
         CloseCurly (index 258)
         CloseCurly (index 259)
+      ]
     - Range: Definition
       Source: token 260 to 331, file :36:5 to 43:5
       Tokens: [
         Ident (index 260) text: composite
         Ident (index 262) text: constructor
         OpenParen (index 264)
         CloseParen (index 265)
         OpenCurly (index 266)
         Ident (index 267) text: auto
         Ident (index 269) text: num_sends
         Equal (index 271)
         Integer (index 272) text: 15
         SemiColon (index 274)
         Ident (index 275) text: channel
         Ident (index 277) text: tx_a
         ArrowRight (index 279)
         Ident (index 280) text: rx_a
         SemiColon (index 282)
         Ident (index 283) text: channel
         Ident (index 285) text: tx_b
         ArrowRight (index 287)
         Ident (index 288) text: rx_b
         SemiColon (index 290)
         Ident (index 291) text: new
         Ident (index 293) text: sender
         OpenParen (index 295)
         Ident (index 296) text: tx_a
         Comma (index 298)
         Ident (index 299) text: num_sends
         CloseParen (index 301)
         SemiColon (index 302)
         Ident (index 303) text: new
         Ident (index 305) text: receiver
         OpenParen (index 307)
         Ident (index 308) text: rx_a
         Comma (index 310)
         Ident (index 311) text: rx_b
         Comma (index 313)
         Ident (index 314) text: num_sends
         CloseParen (index 316)
         SemiColon (index 317)
         Ident (index 318) text: new
         Ident (index 320) text: sender
         OpenParen (index 322)
         Ident (index 323) text: tx_b
         Comma (index 325)
         Ident (index 326) text: num_sends
         CloseParen (index 328)
         SemiColon (index 329)
         CloseCurly (index 330)
+      ]
+    ]

0 comments (0 inline, 0 general)