CSY/reowolf Changeset - ddddcd3cc9aa · Centrum Wiskunde & Informatica (CWI)

Changeset - ddddcd3cc9aa

Parent rev.

Child rev.

[Not reviewed]

2 9 6

MH - 4 years ago 2021-04-18 10:47:33
contact@maxhenger.nl

WIP on compiler rearchitecting

14 files changed:

src/collections/string_pool.rs

src/protocol/ast.rs

src/protocol/ast_printer.rs

src/protocol/input_source2.rs

src/protocol/lexer.rs

src/protocol/lexer2.rs

664

src/protocol/mod.rs

src/protocol/parser/mod.rs

src/protocol/parser/pass_definitions.rs

199

src/protocol/parser/pass_imports.rs

308

src/protocol/parser/pass_symbols.rs

250

src/protocol/parser/pass_tokenizer.rs

315

src/protocol/parser/symbol_table.rs

src/protocol/parser/symbol_table2.rs

Changeset was too big and was cut off... Show full diff anyway

0 comments (0 inline, 0 general)

src/collections/string_pool.rs

➞

Show inline comments

 use std::ptr::null_mut;
 use std::collections::hash_map::DefaultHasher;
 use std::hash::{Hash, Hasher};
 use std::marker::PhantomData;
 const SLAB_SIZE: usize = u16::max_value() as usize;
 #[derive(Clone)]
 pub struct StringRef<'a> {
     data: *const u8,
     length: usize,
     _phantom: PhantomData<&'a [u8]>,
+}
 impl<'a> StringRef<'a> {
     /// `new` constructs a new StringRef whose data is not owned by the
     /// `StringPool`, hence cannot have a `'static` lifetime.
     pub(crate) fn new(data: &'a [u8]) -> StringRef<'a> {
         // This is an internal (compiler) function: so debug_assert that the
-        // string is valid UTF8. Most commonly the input will come from the
+        // string is valid ascii. Most commonly the input will come from the
         // code's source file, which is checked for ASCII-ness anyway.
         debug_assert!(data.is_ascii());
         let length = data.len();
         let data = data.as_ptr();
         StringRef{ data, length, _phantom: PhantomData }
+    }
     pub fn as_str(&self) -> &'a str {
         unsafe {
             let slice = std::slice::from_raw_parts::<'a, u8>(self.data, self.length);
             std::str::from_utf8_unchecked(slice)
+        }
+    }
     pub fn as_bytes(&self) -> &'a [u8] {
         unsafe {
             std::slice::from_raw_parts::<'a, u8>(self.data, self.length)
+        }
+    }
+}
 impl PartialEq for StringRef {
     fn eq(&self, other: &StringRef) -> bool {
         self.as_str() == other.as_str()
+    }
+}
 impl Eq for StringRef {}
 impl Hash for StringRef {
     fn hash<H: Hasher>(&self, state: &mut H) {
         unsafe{
             state.write(std::slice::from_raw_parts(self.data, self.length));
+        }
+    }
+}
 struct StringPoolSlab {
     prev: *mut StringPoolSlab,
     data: Vec<u8>,
     remaining: usize,
+}
 impl StringPoolSlab {
     fn new(prev: *mut StringPoolSlab) -> Self {
         Self{ prev, data: Vec::with_capacity(SLAB_SIZE), remaining: SLAB_SIZE }
+    }
+}
 /// StringPool is a ever-growing pool of strings. Strings have a maximum size
 /// equal to the slab size. The slabs are essentially a linked list to maintain
 /// pointer-stability of the strings themselves.
 /// All `StringRef` instances are invalidated when the string pool is dropped
 pub(crate) struct StringPool {
     last: *mut StringPoolSlab,
+}
 impl StringPool {
     pub(crate) fn new() -> Self {
         // To have some stability we just turn a box into a raw ptr.
         let initial_slab = Box::new(StringPoolSlab::new(null_mut()));
         let initial_slab = Box::into_raw(initial_slab);
         StringPool{
             last: initial_slab,
+        }
+    }
     // Interns a string to the StringPool, returning a reference to it
     /// Interns a string to the `StringPool`, returning a reference to it. The
     /// pointer owned by `StringRef` is `'static` as the `StringPool` doesn't
     /// reallocate/deallocate until dropped (which only happens at the end of
     /// the program.)
     pub(crate) fn intern(&mut self, data: &[u8]) -> StringRef<'static> {
         // TODO: Large string allocations, if ever needed.
         let data_len = data.len();
         assert!(data_len <= SLAB_SIZE, "string is too large for slab");
         debug_assert!(std::str::from_utf8(data).is_ok(), "string to intern is not valid UTF-8 encoded");
         let mut last = unsafe{&mut *self.last};
         if data.len() > last.remaining {
             // Doesn't fit: allocate new slab
             self.alloc_new_slab();
             last = unsafe{&mut *self.last};
+        }
         // Must fit now, compute hash and put in buffer
         debug_assert!(data_len <= last.remaining);
         let range_start = last.data.len();
         last.data.extend_from_slice(data);
         last.remaining -= data_len;
         debug_assert_eq!(range_start + data_len, last.data.len());
         unsafe {
             let start = last.data.as_ptr().offset(range_start as isize);
             StringRef{ data: start, length: data_len, _phantom: PhantomData }
+        }
+    }
     fn alloc_new_slab(&mut self) {
         let new_slab = Box::new(StringPoolSlab::new(self.last));
         let new_slab = Box::into_raw(new_slab);
         self.last = new_slab;
+    }
+}
 impl Drop for StringPool {
     fn drop(&mut self) {
         let mut new_slab = self.last;
         while !new_slab.is_null() {
             let cur_slab = new_slab;
             unsafe {
                 new_slab = (*cur_slab).prev;
                 Box::from_raw(cur_slab); // consume and deallocate
+            }
+        }
+    }
+}
 #[cfg(test)]
 mod tests {
     use super::*;
     #[test]
     fn test_string_just_fits() {
         let large = "0".repeat(SLAB_SIZE);
         let mut pool = StringPool::new();
         let interned = pool.intern(large.as_bytes());
         assert_eq!(interned.as_str(), large);
+    }
     #[test]
     #[should_panic]
     fn test_string_too_large() {
         let large = "0".repeat(SLAB_SIZE + 1);
         let mut pool = StringPool::new();
         let _interned = pool.intern(large.as_bytes());
+    }
     #[test]
     fn test_lots_of_small_allocations() {
         const NUM_PER_SLAB: usize = 32;
         const NUM_SLABS: usize = 4;
         let to_intern = "0".repeat(SLAB_SIZE / NUM_PER_SLAB);
         let mut pool = StringPool::new();
         let mut last_slab = pool.last;
         let mut all_refs = Vec::new();
         // Fill up first slab
         for _alloc_idx in 0..NUM_PER_SLAB {
             let interned = pool.intern(to_intern.as_bytes());
             all_refs.push(interned);
             assert!(std::ptr::eq(last_slab, pool.last));
+        }
         for _slab_idx in 0..NUM_SLABS-1 {
             for alloc_idx in 0..NUM_PER_SLAB {
                 let interned = pool.intern(to_intern.as_bytes());
                 all_refs.push(interned);
                 if alloc_idx == 0 {
                     // First allocation produces a new slab
                     assert!(!std::ptr::eq(last_slab, pool.last));
                     last_slab = pool.last;
                 } else {
                     assert!(std::ptr::eq(last_slab, pool.last));
+                }
+            }
+        }
         // All strings are still correct
         for string_ref in all_refs {
             assert_eq!(string_ref.as_str(), to_intern);
+        }
+    }
+}
@@ \ No newline at end of file @@

src/protocol/ast.rs

➞

Show inline comments

 // TODO: @cleanup, rigorous cleanup of dead code and silly object-oriented
 //  trait impls where I deem them unfit.
 use std::fmt;
 use std::fmt::{Debug, Display, Formatter};
 use std::ops::{Index, IndexMut};
 use super::arena::{Arena, Id};
 use crate::collections::StringRef;
 use crate::protocol::inputsource::*;
 use crate::protocol::input_source2::{InputPosition2, InputSpan};
 /// Global limits to the AST, should be checked by lexer and parser. Some are
 /// arbitrary
 const MAX_LEVEL: usize = 128;
 const MAX_NAMESPACES: usize = 64;
 /// Helper macro that defines a type alias for a AST element ID. In this case
 /// only used to alias the `Id<T>` types.
 macro_rules! define_aliased_ast_id {
     // Variant where we just defined the alias, without any indexing
     ($name:ident, $parent:ty) => {
         pub type $name = $parent;
     };
     // Variant where we define the type, and the Index and IndexMut traits
+    (
         $name:ident, $parent:ty,
         index($indexed_type:ty, $indexed_arena:ident)
     ) => {
         define_aliased_ast_id!($name, $parent);
         impl Index<$name> for Heap {
             type Output = $indexed_type;
             fn index(&self, index: $name) -> &Self::Output {
                 &self.$indexed_arena[index]
+            }
+        }
         impl IndexMut<$name> for Heap {
             fn index_mut(&mut self, index: $name) -> &mut Self::Output {
                 &mut self.$indexed_arena[index]
+            }
+        }
     };
     // Variant where we define type, Index(Mut) traits and an allocation function
+    (
         $name:ident, $parent:ty,
         index($indexed_type:ty, $indexed_arena:ident),
         alloc($fn_name:ident)
     ) => {
         define_aliased_ast_id!($name, $parent, index($indexed_type, $indexed_arena));
         impl Heap {
             pub fn $fn_name(&mut self, f: impl FnOnce($name) -> $indexed_type) -> $name {
                 self.$indexed_arena.alloc_with_id(|id| f(id))
+            }
+        }
     };
+}
 /// Helper macro that defines a wrapper type for a particular variant of an AST
 /// element ID. Only used to define single-wrapping IDs.
 macro_rules! define_new_ast_id {
     // Variant where we just defined the new type, without any indexing
     ($name:ident, $parent:ty) => {
         #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)]
         pub struct $name (pub(crate) $parent);
         impl $name {
             pub(crate) fn new_invalid() -> Self     { Self($parent::new_invalid()) }
             pub(crate) fn is_invalid(&self) -> bool { self.0.is_invalid() }
             pub fn upcast(self) -> $parent          { self.0 }
+        }
     };
     // Variant where we define the type, and the Index and IndexMut traits
+    (
         $name:ident, $parent:ty,
         index($indexed_type:ty, $wrapper_type:path, $indexed_arena:ident)
     ) => {
         define_new_ast_id!($name, $parent);
         impl Index<$name> for Heap {
             type Output = $indexed_type;
             fn index(&self, index: $name) -> &Self::Output {
                 if let $wrapper_type(v) = &self.$indexed_arena[index.0] {
+                    v
                 } else {
                     unreachable!()
+                }
+            }
+        }
         impl IndexMut<$name> for Heap {
             fn index_mut(&mut self, index: $name) -> &mut Self::Output {
                 if let $wrapper_type(v) = &mut self.$indexed_arena[index.0] {
+                    v
                 } else {
                     unreachable!()
+                }
+            }
+        }
     };
     // Variant where we define the type, the Index and IndexMut traits, and an allocation function
+    (
         $name:ident, $parent:ty,
         index($indexed_type:ty, $wrapper_type:path, $indexed_arena:ident),
         alloc($fn_name:ident)
     ) => {
         define_new_ast_id!($name, $parent, index($indexed_type, $wrapper_type, $indexed_arena));
         impl Heap {
             pub fn $fn_name(&mut self, f: impl FnOnce($name) -> $indexed_type) -> $name {
                 $name(
                     self.$indexed_arena.alloc_with_id(|id| {
                         $wrapper_type(f($name(id)))
                     })
+                )
+            }
+        }
+    }
+}
 define_aliased_ast_id!(RootId, Id<Root>, index(Root, protocol_descriptions), alloc(alloc_protocol_description));
 define_aliased_ast_id!(PragmaId, Id<Pragma>, index(Pragma, pragmas), alloc(alloc_pragma));
 define_aliased_ast_id!(ImportId, Id<Import>, index(Import, imports), alloc(alloc_import));
 define_aliased_ast_id!(ParserTypeId, Id<ParserType>, index(ParserType, parser_types), alloc(alloc_parser_type));
 define_aliased_ast_id!(VariableId, Id<Variable>, index(Variable, variables));
 define_new_ast_id!(ParameterId, VariableId, index(Parameter, Variable::Parameter, variables), alloc(alloc_parameter));
 define_new_ast_id!(LocalId, VariableId, index(Local, Variable::Local, variables), alloc(alloc_local));
 define_aliased_ast_id!(DefinitionId, Id<Definition>, index(Definition, definitions));
 define_new_ast_id!(StructDefinitionId, DefinitionId, index(StructDefinition, Definition::Struct, definitions), alloc(alloc_struct_definition));
 define_new_ast_id!(EnumDefinitionId, DefinitionId, index(EnumDefinition, Definition::Enum, definitions), alloc(alloc_enum_definition));
 define_new_ast_id!(UnionDefinitionId, DefinitionId, index(UnionDefinition, Definition::Union, definitions), alloc(alloc_union_definition));
 define_new_ast_id!(ComponentDefinitionId, DefinitionId, index(ComponentDefinition, Definition::Component, definitions), alloc(alloc_component_definition));
 define_new_ast_id!(FunctionDefinitionId, DefinitionId, index(FunctionDefinition, Definition::Function, definitions), alloc(alloc_function_definition));
 define_aliased_ast_id!(StatementId, Id<Statement>, index(Statement, statements));
 define_new_ast_id!(BlockStatementId, StatementId, index(BlockStatement, Statement::Block, statements), alloc(alloc_block_statement));
 define_new_ast_id!(LocalStatementId, StatementId, index(LocalStatement, Statement::Local, statements), alloc(alloc_local_statement));
 define_new_ast_id!(MemoryStatementId, LocalStatementId);
 define_new_ast_id!(ChannelStatementId, LocalStatementId);
 define_new_ast_id!(SkipStatementId, StatementId, index(SkipStatement, Statement::Skip, statements), alloc(alloc_skip_statement));
 define_new_ast_id!(LabeledStatementId, StatementId, index(LabeledStatement, Statement::Labeled, statements), alloc(alloc_labeled_statement));
 define_new_ast_id!(IfStatementId, StatementId, index(IfStatement, Statement::If, statements), alloc(alloc_if_statement));
 define_new_ast_id!(EndIfStatementId, StatementId, index(EndIfStatement, Statement::EndIf, statements), alloc(alloc_end_if_statement));
 define_new_ast_id!(WhileStatementId, StatementId, index(WhileStatement, Statement::While, statements), alloc(alloc_while_statement));
 define_new_ast_id!(EndWhileStatementId, StatementId, index(EndWhileStatement, Statement::EndWhile, statements), alloc(alloc_end_while_statement));
 define_new_ast_id!(BreakStatementId, StatementId, index(BreakStatement, Statement::Break, statements), alloc(alloc_break_statement));
 define_new_ast_id!(ContinueStatementId, StatementId, index(ContinueStatement, Statement::Continue, statements), alloc(alloc_continue_statement));
 define_new_ast_id!(SynchronousStatementId, StatementId, index(SynchronousStatement, Statement::Synchronous, statements), alloc(alloc_synchronous_statement));
 define_new_ast_id!(EndSynchronousStatementId, StatementId, index(EndSynchronousStatement, Statement::EndSynchronous, statements), alloc(alloc_end_synchronous_statement));
 define_new_ast_id!(ReturnStatementId, StatementId, index(ReturnStatement, Statement::Return, statements), alloc(alloc_return_statement));
 define_new_ast_id!(AssertStatementId, StatementId, index(AssertStatement, Statement::Assert, statements), alloc(alloc_assert_statement));
 define_new_ast_id!(GotoStatementId, StatementId, index(GotoStatement, Statement::Goto, statements), alloc(alloc_goto_statement));
 define_new_ast_id!(NewStatementId, StatementId, index(NewStatement, Statement::New, statements), alloc(alloc_new_statement));
 define_new_ast_id!(ExpressionStatementId, StatementId, index(ExpressionStatement, Statement::Expression, statements), alloc(alloc_expression_statement));
 define_aliased_ast_id!(ExpressionId, Id<Expression>, index(Expression, expressions));
 define_new_ast_id!(AssignmentExpressionId, ExpressionId, index(AssignmentExpression, Expression::Assignment, expressions), alloc(alloc_assignment_expression));
 define_new_ast_id!(BindingExpressionId, ExpressionId, index(BindingExpression, Expression::Binding, expressions), alloc(alloc_binding_expression));
 define_new_ast_id!(ConditionalExpressionId, ExpressionId, index(ConditionalExpression, Expression::Conditional, expressions), alloc(alloc_conditional_expression));
 define_new_ast_id!(BinaryExpressionId, ExpressionId, index(BinaryExpression, Expression::Binary, expressions), alloc(alloc_binary_expression));
 define_new_ast_id!(UnaryExpressionId, ExpressionId, index(UnaryExpression, Expression::Unary, expressions), alloc(alloc_unary_expression));
 define_new_ast_id!(IndexingExpressionId, ExpressionId, index(IndexingExpression, Expression::Indexing, expressions), alloc(alloc_indexing_expression));
 define_new_ast_id!(SlicingExpressionId, ExpressionId, index(SlicingExpression, Expression::Slicing, expressions), alloc(alloc_slicing_expression));
 define_new_ast_id!(SelectExpressionId, ExpressionId, index(SelectExpression, Expression::Select, expressions), alloc(alloc_select_expression));
 define_new_ast_id!(ArrayExpressionId, ExpressionId, index(ArrayExpression, Expression::Array, expressions), alloc(alloc_array_expression));
 define_new_ast_id!(LiteralExpressionId, ExpressionId, index(LiteralExpression, Expression::Literal, expressions), alloc(alloc_literal_expression));
 define_new_ast_id!(CallExpressionId, ExpressionId, index(CallExpression, Expression::Call, expressions), alloc(alloc_call_expression));
 define_new_ast_id!(VariableExpressionId, ExpressionId, index(VariableExpression, Expression::Variable, expressions), alloc(alloc_variable_expression));
 // TODO: @cleanup - pub qualifiers can be removed once done
 #[derive(Debug)]
 pub struct Heap {
     // Root arena, contains the entry point for different modules. Each root
     // contains lists of IDs that correspond to the other arenas.
     pub(crate) protocol_descriptions: Arena<Root>,
     // Contents of a file, these are the elements the `Root` elements refer to
     pragmas: Arena<Pragma>,
     pub(crate) imports: Arena<Import>,
     identifiers: Arena<Identifier>,
     pub(crate) parser_types: Arena<ParserType>,
     pub(crate) variables: Arena<Variable>,
     pub(crate) definitions: Arena<Definition>,
     pub(crate) statements: Arena<Statement>,
     pub(crate) expressions: Arena<Expression>,
+}
 impl Heap {
     pub fn new() -> Heap {
         Heap {
             // string_alloc: StringAllocator::new(),
             protocol_descriptions: Arena::new(),
             pragmas: Arena::new(),
             imports: Arena::new(),
             identifiers: Arena::new(),
             parser_types: Arena::new(),
             variables: Arena::new(),
             definitions: Arena::new(),
             statements: Arena::new(),
             expressions: Arena::new(),
+        }
+    }
     pub fn alloc_memory_statement(
         &mut self,
         f: impl FnOnce(MemoryStatementId) -> MemoryStatement,
     ) -> MemoryStatementId {
         MemoryStatementId(LocalStatementId(self.statements.alloc_with_id(|id| {
             Statement::Local(LocalStatement::Memory(
                 f(MemoryStatementId(LocalStatementId(id)))
             ))
         })))
+    }
     pub fn alloc_channel_statement(
         &mut self,
         f: impl FnOnce(ChannelStatementId) -> ChannelStatement,
     ) -> ChannelStatementId {
         ChannelStatementId(LocalStatementId(self.statements.alloc_with_id(|id| {
             Statement::Local(LocalStatement::Channel(
                 f(ChannelStatementId(LocalStatementId(id)))
             ))
         })))
+    }
+}
 impl Index<MemoryStatementId> for Heap {
     type Output = MemoryStatement;
     fn index(&self, index: MemoryStatementId) -> &Self::Output {
         &self.statements[index.0.0].as_memory()
+    }
+}
 impl Index<ChannelStatementId> for Heap {
     type Output = ChannelStatement;
     fn index(&self, index: ChannelStatementId) -> &Self::Output {
         &self.statements[index.0.0].as_channel()
+    }
+}
 #[derive(Debug, Clone)]
 pub struct Root {
     pub this: RootId,
     // Phase 1: parser
     // pub position: InputPosition,
     pub pragmas: Vec<PragmaId>,
     pub imports: Vec<ImportId>,
     pub definitions: Vec<DefinitionId>,
+}
 impl Root {
     pub fn get_definition_ident(&self, h: &Heap, id: &[u8]) -> Option<DefinitionId> {
         for &def in self.definitions.iter() {
             if h[def].identifier().value == id {
                 return Some(def);
+            }
+        }
         None
+    }
+}
 impl SyntaxElement for Root {
     fn position(&self) -> InputPosition {
         self.position
+    }
+}
 #[derive(Debug, Clone)]
 pub enum Pragma {
     Version(PragmaVersion),
     Module(PragmaModule),
+}
 impl Pragma {
     pub(crate) fn as_module(&self) -> &PragmaModule {
         match self {
             Pragma::Module(pragma) => pragma,
             _ => unreachable!("Tried to obtain {:?} as PragmaModule", self),
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct PragmaVersion {
     pub this: PragmaId,
     // Phase 1: parser
     pub span: InputSpan, // of full pragma
     pub version: u64,
+}
 #[derive(Debug, Clone)]
 pub struct PragmaModule {
     pub this: PragmaId,
     // Phase 1: parser
     pub span: InputSpan, // of full pragma
     pub value: Identifier,
+}
 #[derive(Debug, Clone)]
 pub enum Import {
     Module(ImportModule),
     Symbols(ImportSymbols)
+}
 impl Import {
     pub(crate) fn as_module(&self) -> &ImportModule {
         match self {
             Import::Module(m) => m,
-            _ => panic!("Unable to cast 'Import' to 'ImportModule'")
+            _ => unreachable!("Unable to cast 'Import' to 'ImportModule'")
+        }
+    }
     pub(crate) fn as_symbols(&self) -> &ImportSymbols {
         match self {
             Import::Symbols(m) => m,
             _ => panic!("Unable to cast 'Import' to 'ImportSymbols'")
             _ => unreachable!("Unable to cast 'Import' to 'ImportSymbols'")
+        }
+    }
     pub(crate) fn as_symbols_mut(&mut self) -> &mut ImportSymbols {
         match self {
             Import::Symbols(m) => m,
             _ => unreachable!("Unable to cast 'Import' to 'ImportSymbols'")
+        }
+    }
+}
 impl SyntaxElement for Import {
     fn position(&self) -> InputPosition {
         match self {
             Import::Module(m) => m.position,
             Import::Symbols(m) => m.position
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct ImportModule {
     pub this: ImportId,
     // Phase 1: parser
     pub span: InputSpan,
-    pub module_name: Identifier,
+    pub module: Identifier,
     pub alias: Identifier,
     pub module_id: RootId,
+}
 #[derive(Debug, Clone)]
 pub struct AliasedSymbol {
     // Phase 1: parser
     pub position: InputPosition,
     pub name: Identifier,
     pub alias: Identifier,
+    pub alias: Option<Identifier>,
     pub definition_id: DefinitionId,
+}
 #[derive(Debug, Clone)]
 pub struct ImportSymbols {
     pub this: ImportId,
     // Phase 1: parser
     pub span: InputSpan,
     pub module_name: Vec<u8>,
     pub module_id: Option<RootId>,
     pub module: Identifier,
     pub module_id: RootId,
     pub symbols: Vec<AliasedSymbol>,
+}
 #[derive(Debug, Clone)]
 pub struct Identifier {
     pub span: InputSpan,
     pub value: StringRef<'static>,
+}
 impl PartialEq for Identifier {
     fn eq(&self, other: &Self) -> bool {
         return self.value == other.value
+    }
+}
 impl Display for Identifier {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         // A source identifier is in ASCII range.
         write!(f, "{}", String::from_utf8_lossy(&self.value))
+    }
+}
 #[derive(Debug, Clone)]
 pub enum NamespacedIdentifierPart {
     // Regular identifier
     Identifier{start: u16, end: u16},
     // Polyargs associated with a preceding identifier
     PolyArgs{start: u16, end: u16},
+}
 impl NamespacedIdentifierPart {
     pub(crate) fn is_identifier(&self) -> bool {
         match self {
             NamespacedIdentifierPart::Identifier{..} => true,
             NamespacedIdentifierPart::PolyArgs{..} => false,
+        }
+    }
     pub(crate) fn as_identifier(&self) -> (u16, u16) {
         match self {
             NamespacedIdentifierPart::Identifier{start, end} => (*start, *end),
             NamespacedIdentifierPart::PolyArgs{..} => {
                 unreachable!("Tried to obtain {:?} as Identifier", self);
+            }
+        }
+    }
     pub(crate) fn as_poly_args(&self) -> (u16, u16) {
         match self {
             NamespacedIdentifierPart::PolyArgs{start, end} => (*start, *end),
             NamespacedIdentifierPart::Identifier{..} => {
                 unreachable!("Tried to obtain {:?} as PolyArgs", self)
+            }
+        }
+    }
+}
 /// An identifier with optional namespaces and polymorphic variables. Note that
 /// we allow each identifier to be followed by polymorphic arguments during the
 /// parsing phase (e.g. Foo<A,B>::Bar<C,D>::Qux). But in our current language
 /// implementation we can only have valid namespaced identifier that contain one
 /// set of polymorphic arguments at the appropriate position.
 /// TODO: @tokens Reimplement/rename once we have a tokenizer
 #[derive(Debug, Clone)]
 pub struct NamespacedIdentifier {
     pub position: InputPosition,
     pub value: Vec<u8>, // Full name as it resides in the input source
     pub poly_args: Vec<ParserTypeId>, // All poly args littered throughout the namespaced identifier
     pub parts: Vec<NamespacedIdentifierPart>, // Indices into value/poly_args
+}
 impl NamespacedIdentifier {
     /// Returns the identifier value without any of the specific polymorphic
     /// arguments.
     pub fn strip_poly_args(&self) -> Vec<u8> {
         debug_assert!(!self.parts.is_empty() && self.parts[0].is_identifier());
         let mut result = Vec::with_capacity(self.value.len());
         let mut iter = self.iter();
         let (first_ident, _) = iter.next().unwrap();
         result.extend(first_ident);
         for (ident, _) in iter.next() {
             result.push(b':');
             result.push(b':');
             result.extend(ident);
+        }
         result
+    }
     /// Returns an iterator of the elements in the namespaced identifier
     pub fn iter(&self) -> NamespacedIdentifierIter {
         return NamespacedIdentifierIter{
             identifier: self,
             element_idx: 0
+        }
+    }
     pub fn get_poly_args(&self) -> Option<&[ParserTypeId]> {
         let has_poly_args = self.parts.iter().any(|v| !v.is_identifier());
         if has_poly_args {
             Some(&self.poly_args)
         } else {
             None
+        }
+    }
     // Check if two namespaced identifiers match eachother when not considering
     // the polymorphic arguments
     pub fn matches_namespaced_identifier(&self, other: &Self) -> bool {
         let mut iter_self = self.iter();
         let mut iter_other = other.iter();
         loop {
             let val_self = iter_self.next();
             let val_other = iter_other.next();
             if val_self.is_some() != val_other.is_some() {
                 // One is longer than the other
                 return false;
+            }
             if val_self.is_none() {
                 // Both are none
                 return true;
+            }
             // Both are something
             let (val_self, _) = val_self.unwrap();
             let (val_other, _) = val_other.unwrap();
             if val_self != val_other { return false; }
+        }
+    }
     // Check if the namespaced identifier matches an identifier when not
     // considering the polymorphic arguments
     pub fn matches_identifier(&self, other: &Identifier) -> bool {
         let mut iter = self.iter();
         let (first_ident, _) = iter.next().unwrap();
         if first_ident != other.value {
             return false;
+        }
         if iter.next().is_some() {
             return false;
+        }
         return true;
+    }
+}
 /// Iterator over elements of the namespaced identifier. The element index will
 /// only ever be at the start of an identifier element.
 #[derive(Debug)]
 pub struct NamespacedIdentifierIter<'a> {
     identifier: &'a NamespacedIdentifier,
     element_idx: usize,
+}
 impl<'a> Iterator for NamespacedIdentifierIter<'a> {
     type Item = (&'a [u8], Option<&'a [ParserTypeId]>);
     fn next(&mut self) -> Option<Self::Item> {
         match self.get(self.element_idx) {
             Some((ident, poly)) => {
                 self.element_idx += 1;
                 if poly.is_some() {
                     self.element_idx += 1;
+                }
                 Some((ident, poly))
             },
             None => None
+        }
+    }
+}
 impl<'a> NamespacedIdentifierIter<'a> {
     /// Returns number of parts iterated over, may not correspond to number of
     /// times one called `next()` because returning an identifier with
     /// polymorphic arguments increments the internal counter by 2.
     pub fn num_returned(&self) -> usize {
         return self.element_idx;
+    }
     pub fn num_remaining(&self) -> usize {
         return self.identifier.parts.len() - self.element_idx;
+    }
     pub fn returned_section(&self) -> &[u8] {
         if self.element_idx == 0 { return &self.identifier.value[0..0]; }
         let last_idx = match &self.identifier.parts[self.element_idx - 1] {
             NamespacedIdentifierPart::Identifier{end, ..} => *end,
             NamespacedIdentifierPart::PolyArgs{end, ..} => *end,
         };
         return &self.identifier.value[..last_idx as usize];
+    }
     /// Returns a specific element from the namespaced identifier
     pub fn get(&self, idx: usize) -> Option<<Self as Iterator>::Item> {
         if idx >= self.identifier.parts.len() {
             return None
+        }
         let cur_part = &self.identifier.parts[idx];
         let next_part = self.identifier.parts.get(idx + 1);
         let (ident_start, ident_end) = cur_part.as_identifier();
         let poly_slice = match next_part {
             Some(part) => match part {
                 NamespacedIdentifierPart::Identifier{..} => None,
                 NamespacedIdentifierPart::PolyArgs{start, end} => Some(
                     &self.identifier.poly_args[*start as usize..*end as usize]
                 ),
             },
             None => None
         };
         Some((
             &self.identifier.value[ident_start as usize..ident_end as usize],
             poly_slice
         ))
+    }
     /// Returns the previously returend index into the parts array of the
     /// identifier.
     pub fn prev_idx(&self) -> Option<usize> {
         if self.element_idx == 0 {
             return None;
         };
         if self.identifier.parts[self.element_idx - 1].is_identifier() {
             return Some(self.element_idx - 1);
+        }
         // Previous part had polymorphic arguments, so the one before that must
         // be an identifier (if well formed)
         debug_assert!(self.element_idx >= 2 && self.identifier.parts[self.element_idx - 2].is_identifier());
         return Some(self.element_idx - 2)
+    }
     /// Returns the previously returned result from `next()`
     pub fn prev(&self) -> Option<<Self as Iterator>::Item> {
         match self.prev_idx() {
             None => None,
             Some(idx) => self.get(idx)
+        }
+    }
+}
 /// TODO: @types Remove the Message -> Byte hack at some point...
 #[derive(Debug, Clone)]
 pub enum ParserTypeVariant {
     // Basic builtin
     Message,
     Bool,
     Byte,
     Short,
     Int,
     Long,
     String,
     UInt8, Uint16, UInt32, UInt64,
     SInt8, SInt16, SInt32, SInt64,
     Character, String,
     // Literals (need to get concrete builtin type during typechecking)
     IntegerLiteral,
     Inferred,
     // Complex builtins
     Array(ParserTypeId), // array of a type
     Input(ParserTypeId), // typed input endpoint of a channel
     Output(ParserTypeId), // typed output endpoint of a channel
     Symbolic(SymbolicParserType), // symbolic type (definition or polyarg)
+}
 impl ParserTypeVariant {
     pub(crate) fn supports_polymorphic_args(&self) -> bool {
         use ParserTypeVariant::*;
         match self {
             Message | Bool | Byte | Short | Int | Long | String | IntegerLiteral | Inferred => false,
             _ => true
+        }
+    }
+}
 /// ParserType is a specification of a type during the parsing phase and initial
 /// linker/validator phase of the compilation process. These types may be
 /// (partially) inferred or represent literals (e.g. a integer whose bytesize is
 /// not yet determined).
 #[derive(Debug, Clone)]
 pub struct ParserType {
     pub this: ParserTypeId,
     pub pos: InputPosition,
     pub variant: ParserTypeVariant,
+}
 /// SymbolicParserType is the specification of a symbolic type. During the
 /// parsing phase we will only store the identifier of the type. During the
 /// validation phase we will determine whether it refers to a user-defined type,
 /// or a polymorphic argument. After the validation phase it may still be the
 /// case that the resulting `variant` will not pass the typechecker.
 #[derive(Debug, Clone)]
 pub struct SymbolicParserType {
     // Phase 1: parser
     pub identifier: NamespacedIdentifier,
     // Phase 2: validation/linking (for types in function/component bodies) and
     //  type table construction (for embedded types of structs/unions)
     pub poly_args2: Vec<ParserTypeId>, // taken from identifier or inferred
     pub variant: Option<SymbolicParserTypeVariant>
+}
 /// Specifies whether the symbolic type points to an actual user-defined type,
 /// or whether it points to a polymorphic argument within the definition (e.g.
 /// a defined variable `T var` within a function `int func<T>()`
 #[derive(Debug, Clone)]
 pub enum SymbolicParserTypeVariant {
     Definition(DefinitionId),
     // TODO: figure out if I need the DefinitionId here
     PolyArg(DefinitionId, usize), // index of polyarg in the definition
+}
 /// ConcreteType is the representation of a type after resolving symbolic types
 /// and performing type inference
 #[derive(Debug, Clone, Copy, Eq, PartialEq)]
 pub enum ConcreteTypePart {
     // Markers for the use of polymorphic types within a procedure's body that
     // refer to polymorphic variables on the procedure's definition. Different
     // from markers in the `InferenceType`, these will not contain nested types.
     Marker(usize),
     // Special types (cannot be explicitly constructed by the programmer)
     Void,
     // Builtin types without nested types
     Message,
     Bool,
     Byte,
     Short,
     Int,
     Long,
     String,
     // Builtin types with one nested type
     Array,
     Slice,
     Input,
     Output,
     // User defined type with any number of nested types
     Instance(DefinitionId, usize),
+}
 #[derive(Debug, Clone, Eq, PartialEq)]
 pub struct ConcreteType {
     pub(crate) parts: Vec<ConcreteTypePart>
+}
 impl Default for ConcreteType {
     fn default() -> Self {
         Self{ parts: Vec::new() }
+    }
+}
 impl ConcreteType {
     pub(crate) fn has_marker(&self) -> bool {
         self.parts
             .iter()
             .any(|p| {
                 if let ConcreteTypePart::Marker(_) = p { true } else { false }
             })
+    }
+}
 // TODO: Remove at some point
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum PrimitiveType {
     Unassigned,
     Input,
     Output,
     Message,
     Boolean,
     Byte,
     Short,
     Int,
     Long,
+}
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub struct Type {
     pub primitive: PrimitiveType,
     pub array: bool,
+}
 #[allow(dead_code)]
 impl Type {
     pub const UNASSIGNED: Type = Type { primitive: PrimitiveType::Unassigned, array: false };
     pub const INPUT: Type = Type { primitive: PrimitiveType::Input, array: false };
     pub const OUTPUT: Type = Type { primitive: PrimitiveType::Output, array: false };
     pub const MESSAGE: Type = Type { primitive: PrimitiveType::Message, array: false };
     pub const BOOLEAN: Type = Type { primitive: PrimitiveType::Boolean, array: false };
     pub const BYTE: Type = Type { primitive: PrimitiveType::Byte, array: false };
     pub const SHORT: Type = Type { primitive: PrimitiveType::Short, array: false };
     pub const INT: Type = Type { primitive: PrimitiveType::Int, array: false };
     pub const LONG: Type = Type { primitive: PrimitiveType::Long, array: false };
     pub const INPUT_ARRAY: Type = Type { primitive: PrimitiveType::Input, array: true };
     pub const OUTPUT_ARRAY: Type = Type { primitive: PrimitiveType::Output, array: true };
     pub const MESSAGE_ARRAY: Type = Type { primitive: PrimitiveType::Message, array: true };
     pub const BOOLEAN_ARRAY: Type = Type { primitive: PrimitiveType::Boolean, array: true };
     pub const BYTE_ARRAY: Type = Type { primitive: PrimitiveType::Byte, array: true };
     pub const SHORT_ARRAY: Type = Type { primitive: PrimitiveType::Short, array: true };
     pub const INT_ARRAY: Type = Type { primitive: PrimitiveType::Int, array: true };
     pub const LONG_ARRAY: Type = Type { primitive: PrimitiveType::Long, array: true };
+}
 impl Display for Type {
     fn fmt(&self, f: &mut Formatter<'_>) -> fmt::Result {
         match &self.primitive {
             PrimitiveType::Unassigned => {
                 write!(f, "unassigned")?;
+            }
             PrimitiveType::Input => {
                 write!(f, "in")?;
+            }
             PrimitiveType::Output => {
                 write!(f, "out")?;
+            }
             PrimitiveType::Message => {
                 write!(f, "msg")?;
+            }
             PrimitiveType::Boolean => {
                 write!(f, "boolean")?;
+            }
             PrimitiveType::Byte => {
                 write!(f, "byte")?;
+            }
             PrimitiveType::Short => {
                 write!(f, "short")?;
+            }
             PrimitiveType::Int => {
                 write!(f, "int")?;
+            }
             PrimitiveType::Long => {
                 write!(f, "long")?;
+            }
+        }
         if self.array {
             write!(f, "[]")
         } else {
             Ok(())
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub enum Field {
     Length,
     Symbolic(FieldSymbolic),
+}
 impl Field {
@@ @@ -890,385 +890,384 @@ impl Variable { @@
             _ => false,
+        }
+    }
     pub fn as_parameter(&self) -> &Parameter {
         match self {
             Variable::Parameter(result) => result,
             _ => panic!("Unable to cast `Variable` to `Parameter`"),
+        }
+    }
     pub fn as_local(&self) -> &Local {
         match self {
             Variable::Local(result) => result,
             _ => panic!("Unable to cast `Variable` to `Local`"),
+        }
+    }
     pub fn as_local_mut(&mut self) -> &mut Local {
         match self {
             Variable::Local(result) => result,
             _ => panic!("Unable to cast 'Variable' to 'Local'"),
+        }
+    }
+}
 impl SyntaxElement for Variable {
     fn position(&self) -> InputPosition {
         match self {
             Variable::Parameter(decl) => decl.position(),
             Variable::Local(decl) => decl.position(),
+        }
+    }
+}
 /// TODO: Remove distinction between parameter/local and add an enum to indicate
 ///     the distinction between the two
 #[derive(Debug, Clone)]
 pub struct Parameter {
     pub this: ParameterId,
     // Phase 1: parser
     pub position: InputPosition,
     pub parser_type: ParserTypeId,
     pub identifier: Identifier,
+}
 impl SyntaxElement for Parameter {
     fn position(&self) -> InputPosition {
         self.position
+    }
+}
 #[derive(Debug, Clone)]
 pub struct Local {
     pub this: LocalId,
     // Phase 1: parser
     pub position: InputPosition,
     pub parser_type: ParserTypeId,
     pub identifier: Identifier,
     // Phase 2: linker
     pub relative_pos_in_block: u32,
+}
 impl SyntaxElement for Local {
     fn position(&self) -> InputPosition {
         self.position
+    }
+}
 #[derive(Debug, Clone)]
 pub enum Definition {
     Struct(StructDefinition),
     Enum(EnumDefinition),
     Union(UnionDefinition),
     Component(ComponentDefinition),
     Function(FunctionDefinition),
+}
 impl Definition {
     pub fn is_struct(&self) -> bool {
         match self {
             Definition::Struct(_) => true,
             _ => false
+        }
+    }
     pub fn as_struct(&self) -> &StructDefinition {
         match self {
             Definition::Struct(result) => result,
             _ => panic!("Unable to cast 'Definition' to 'StructDefinition'"),
+        }
+    }
     pub fn is_enum(&self) -> bool {
         match self {
             Definition::Enum(_) => true,
             _ => false,
+        }
+    }
     pub fn as_enum(&self) -> &EnumDefinition {
         match self {
             Definition::Enum(result) => result,
             _ => panic!("Unable to cast 'Definition' to 'EnumDefinition'"),
+        }
+    }
     pub fn is_union(&self) -> bool {
         match self {
             Definition::Union(_) => true,
             _ => false,
+        }
+    }
     pub fn as_union(&self) -> &UnionDefinition {
         match self {
             Definition::Union(result) => result,
             _ => panic!("Unable to cast 'Definition' to 'UnionDefinition'"),
+        }
+    }
     pub fn is_component(&self) -> bool {
         match self {
             Definition::Component(_) => true,
             _ => false,
+        }
+    }
     pub fn as_component(&self) -> &ComponentDefinition {
         match self {
             Definition::Component(result) => result,
             _ => panic!("Unable to cast `Definition` to `Component`"),
+        }
+    }
     pub fn is_function(&self) -> bool {
         match self {
             Definition::Function(_) => true,
             _ => false,
+        }
+    }
     pub fn as_function(&self) -> &FunctionDefinition {
         match self {
             Definition::Function(result) => result,
             _ => panic!("Unable to cast `Definition` to `Function`"),
+        }
+    }
     pub fn identifier(&self) -> &Identifier {
         match self {
             Definition::Struct(def) => &def.identifier,
             Definition::Enum(def) => &def.identifier,
             Definition::Union(def) => &def.identifier,
             Definition::Component(def) => &def.identifier,
             Definition::Function(def) => &def.identifier,
+        }
+    }
     pub fn parameters(&self) -> &Vec<ParameterId> {
         // TODO: Fix this
         static EMPTY_VEC: Vec<ParameterId> = Vec::new();
         match self {
             Definition::Component(com) => &com.parameters,
             Definition::Function(fun) => &fun.parameters,
             _ => &EMPTY_VEC,
+        }
+    }
     pub fn body(&self) -> StatementId {
         // TODO: Fix this
         match self {
             Definition::Component(com) => com.body,
             Definition::Function(fun) => fun.body,
             _ => panic!("cannot retrieve body (for EnumDefinition or StructDefinition)")
+        }
+    }
+}
 impl SyntaxElement for Definition {
     fn position(&self) -> InputPosition {
         match self {
             Definition::Struct(def) => def.position,
             Definition::Enum(def) => def.position,
             Definition::Union(def) => def.position,
             Definition::Component(def) => def.position(),
             Definition::Function(def) => def.position(),
+        }
+    }
+}
 impl VariableScope for Definition {
     fn parent_scope(&self, _h: &Heap) -> Option<Scope> {
         None
+    }
     fn get_variable(&self, h: &Heap, id: &Identifier) -> Option<VariableId> {
         for &parameter_id in self.parameters().iter() {
             let parameter = &h[parameter_id];
             if parameter.identifier == *id {
                 return Some(parameter_id.0);
+            }
+        }
         None
+    }
+}
 #[derive(Debug, Clone)]
 pub struct StructFieldDefinition {
     pub position: InputPosition,
     pub field: Identifier,
     pub parser_type: ParserTypeId,
+}
 #[derive(Debug, Clone)]
 pub struct StructDefinition {
     pub this: StructDefinitionId,
     // Phase 1: parser
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     pub fields: Vec<StructFieldDefinition>
+}
 impl StructDefinition {
     pub(crate) fn new_empty(this: StructDefinitionId, span: InputSpan, identifier: Identifier) -> Self {
         Self{ this, span, identifier, poly_vars: Vec::new(), fields: Vec::new() }
+    }
+}
 #[derive(Debug, Clone)]
 pub enum EnumVariantValue {
     None,
     Integer(i64),
+}
 #[derive(Debug, Clone)]
 pub struct EnumVariantDefinition {
     pub position: InputPosition,
     pub identifier: Identifier,
     pub value: EnumVariantValue,
+}
 #[derive(Debug, Clone)]
 pub struct EnumDefinition {
     pub this: EnumDefinitionId,
     // Phase 1: parser
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     pub variants: Vec<EnumVariantDefinition>,
+}
 impl EnumDefinition {
     pub(crate) fn new_empty(this: EnumDefinitionId, span: InputSpan, identifier: Identifier) -> Self {
         Self{ this, span, identifier, poly_vars: Vec::new(), variants: Vec::new() }
+    }
+}
 #[derive(Debug, Clone)]
 pub enum UnionVariantValue {
     None,
     Embedded(Vec<ParserTypeId>),
+}
 #[derive(Debug, Clone)]
 pub struct UnionVariantDefinition {
     pub position: InputPosition,
     pub identifier: Identifier,
     pub value: UnionVariantValue,
+}
 #[derive(Debug, Clone)]
 pub struct UnionDefinition {
     pub this: UnionDefinitionId,
     // Phase 1: parser
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     pub variants: Vec<UnionVariantDefinition>,
+}
 impl UnionDefinition {
     pub(crate) fn new_empty(this: UnionDefinitionId, span: InputSpan, identifier: Identifier) -> Self {
         Self{ this, span, identifier, poly_vars: Vec::new(), variants: Vec::new() }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub enum ComponentVariant {
     Primitive,
     Composite,
+}
 #[derive(Debug, Clone)]
 pub struct ComponentDefinition {
     pub this: ComponentDefinitionId,
     // Phase 1: parser
     pub span: InputSpan,
     pub variant: ComponentVariant,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     pub parameters: Vec<ParameterId>,
     pub body: StatementId,
+}
 impl ComponentDefinition {
     pub(crate) fn new_empty(this: ComponentDefinitionId, span: InputSpan, variant: ComponentVariant, identifier: Identifier) -> Self {
         Self{
             this, span, variant, identifier,
             poly_vars: Vec::new(),
             parameters: Vec::new(),
             body: StatementId::new_invalid()
+        }
+    }
+}
 impl SyntaxElement for ComponentDefinition {
     fn position(&self) -> InputPosition {
         self.position
+    }
+}
 #[derive(Debug, Clone)]
 pub struct FunctionDefinition {
     pub this: FunctionDefinitionId,
     // Phase 1: parser
     pub span: InputSpan,
     pub return_type: ParserTypeId,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     pub parameters: Vec<ParameterId>,
     pub body: StatementId,
+}
 impl FunctionDefinition {
     pub(crate) fn new_empty(this: FunctionDefinitionId, span: InputSpan, identifier: Identifier) -> Self {
         Self {
             this, span, identifier,
             return_type: ParserTypeId::new_invalid(),
             poly_vars: Vec::new(),
             parameters: Vec::new(),
             body: StatementId::new_invalid(),
+        }
+    }
+}
 impl SyntaxElement for FunctionDefinition {
     fn position(&self) -> InputPosition {
         self.position
+    }
+}
 #[derive(Debug, Clone)]
 pub enum Statement {
     Block(BlockStatement),
     Local(LocalStatement),
     Skip(SkipStatement),
     Labeled(LabeledStatement),
     If(IfStatement),
     EndIf(EndIfStatement),
     While(WhileStatement),
     EndWhile(EndWhileStatement),
     Break(BreakStatement),
     Continue(ContinueStatement),
     Synchronous(SynchronousStatement),
     EndSynchronous(EndSynchronousStatement),
     Return(ReturnStatement),
     Assert(AssertStatement),
     Goto(GotoStatement),
     New(NewStatement),
     Expression(ExpressionStatement),
+}
 impl Statement {
     pub fn as_block(&self) -> &BlockStatement {
         match self {
             Statement::Block(result) => result,
             _ => panic!("Unable to cast `Statement` to `BlockStatement`"),
+        }
+    }
     pub fn as_block_mut(&mut self) -> &mut BlockStatement {
         match self {
             Statement::Block(result) => result,
             _ => panic!("Unable to cast `Statement` to `BlockStatement`"),
+        }
+    }
     pub fn as_local(&self) -> &LocalStatement {
         match self {
             Statement::Local(result) => result,
             _ => panic!("Unable to cast `Statement` to `LocalStatement`"),
+        }
+    }
     pub fn as_memory(&self) -> &MemoryStatement {
         self.as_local().as_memory()
+    }
     pub fn as_channel(&self) -> &ChannelStatement {
         self.as_local().as_channel()
+    }
     pub fn as_skip(&self) -> &SkipStatement {
         match self {
             Statement::Skip(result) => result,

src/protocol/ast_printer.rs

➞

Show inline comments

@@ @@ -49,394 +49,394 @@ const PREFIX_INDEXING_EXPR_ID: &'static str = "EIdx"; @@
 const PREFIX_SLICING_EXPR_ID: &'static str = "ESli";
 const PREFIX_SELECT_EXPR_ID: &'static str = "ESel";
 const PREFIX_ARRAY_EXPR_ID: &'static str = "EArr";
 const PREFIX_CONST_EXPR_ID: &'static str = "ECns";
 const PREFIX_CALL_EXPR_ID: &'static str = "ECll";
 const PREFIX_VARIABLE_EXPR_ID: &'static str = "EVar";
 struct KV<'a> {
     buffer: &'a mut String,
     prefix: Option<(&'static str, u32)>,
     indent: usize,
     temp_key: &'a mut String,
     temp_val: &'a mut String,
+}
 impl<'a> KV<'a> {
     fn new(buffer: &'a mut String, temp_key: &'a mut String, temp_val: &'a mut String, indent: usize) -> Self {
         temp_key.clear();
         temp_val.clear();
         KV{
             buffer,
             prefix: None,
             indent,
             temp_key,
             temp_val
+        }
+    }
     fn with_id(mut self, prefix: &'static str, id: u32) -> Self {
         self.prefix = Some((prefix, id));
         self
+    }
     fn with_s_key(self, key: &str) -> Self {
         self.temp_key.push_str(key);
         self
+    }
     fn with_d_key<D: Display>(self, key: &D) -> Self {
         self.temp_key.push_str(&key.to_string());
         self
+    }
     fn with_s_val(self, val: &str) -> Self {
         self.temp_val.push_str(val);
         self
+    }
     fn with_disp_val<D: Display>(self, val: &D) -> Self {
         self.temp_val.push_str(&format!("{}", val));
         self
+    }
     fn with_debug_val<D: Debug>(self, val: &D) -> Self {
         self.temp_val.push_str(&format!("{:?}", val));
         self
+    }
     fn with_ascii_val(self, val: &[u8]) -> Self {
         self.temp_val.push_str(&*String::from_utf8_lossy(val));
         self
+    }
     fn with_opt_disp_val<D: Display>(self, val: Option<&D>) -> Self {
         match val {
             Some(v) => { self.temp_val.push_str(&format!("Some({})", v)); },
             None => { self.temp_val.push_str("None"); }
+        }
         self
+    }
     fn with_opt_ascii_val(self, val: Option<&[u8]>) -> Self {
         match val {
             Some(v) => {
                 self.temp_val.push_str("Some(");
                 self.temp_val.push_str(&*String::from_utf8_lossy(v));
                 self.temp_val.push(')');
             },
             None => {
                 self.temp_val.push_str("None");
+            }
+        }
         self
+    }
     fn with_custom_val<F: Fn(&mut String)>(mut self, val_fn: F) -> Self {
         val_fn(&mut self.temp_val);
         self
+    }
+}
 impl<'a> Drop for KV<'a> {
     fn drop(&mut self) {
         // Prefix and indent
         if let Some((prefix, id)) = &self.prefix {
             self.buffer.push_str(&format!("{}[{:04}]", prefix, id));
         } else {
             self.buffer.push_str("           ");
+        }
         for _ in 0..self.indent * INDENT {
             self.buffer.push(' ');
+        }
         // Leading dash
         self.buffer.push_str("- ");
         // Key and value
         self.buffer.push_str(self.temp_key);
         if self.temp_val.is_empty() {
             self.buffer.push(':');
         } else {
             self.buffer.push_str(": ");
             self.buffer.push_str(&self.temp_val);
+        }
         self.buffer.push('\n');
+    }
+}
 pub(crate) struct ASTWriter {
     cur_definition: Option<DefinitionId>,
     buffer: String,
     temp1: String,
     temp2: String,
+}
 impl ASTWriter {
     pub(crate) fn new() -> Self {
         Self{
             cur_definition: None,
             buffer: String::with_capacity(4096),
             temp1: String::with_capacity(256),
             temp2: String::with_capacity(256),
+        }
+    }
     pub(crate) fn write_ast<W: IOWrite>(&mut self, w: &mut W, heap: &Heap) {
         for root_id in heap.protocol_descriptions.iter().map(|v| v.this) {
             self.write_module(heap, root_id);
             w.write_all(self.buffer.as_bytes()).expect("flush buffer");
             self.buffer.clear();
+        }
+    }
     //--------------------------------------------------------------------------
     // Top-level module writing
     //--------------------------------------------------------------------------
     fn write_module(&mut self, heap: &Heap, root_id: RootId) {
         self.kv(0).with_id(PREFIX_ROOT_ID, root_id.index)
             .with_s_key("Module");
         let root = &heap[root_id];
         self.kv(1).with_s_key("Pragmas");
         for pragma_id in &root.pragmas {
             self.write_pragma(heap, *pragma_id, 2);
+        }
         self.kv(1).with_s_key("Imports");
         for import_id in &root.imports {
             self.write_import(heap, *import_id, 2);
+        }
         self.kv(1).with_s_key("Definitions");
         for def_id in &root.definitions {
             self.write_definition(heap, *def_id, 2);
+        }
+    }
     fn write_pragma(&mut self, heap: &Heap, pragma_id: PragmaId, indent: usize) {
         match &heap[pragma_id] {
             Pragma::Version(pragma) => {
                 self.kv(indent).with_id(PREFIX_PRAGMA_ID, pragma.this.index)
                     .with_s_key("PragmaVersion")
                     .with_disp_val(&pragma.version);
             },
             Pragma::Module(pragma) => {
                 self.kv(indent).with_id(PREFIX_PRAGMA_ID, pragma.this.index)
                     .with_s_key("PragmaModule")
                     .with_ascii_val(&pragma.value);
+            }
+        }
+    }
     fn write_import(&mut self, heap: &Heap, import_id: ImportId, indent: usize) {
         let import = &heap[import_id];
         let indent2 = indent + 1;
         match import {
             Import::Module(import) => {
                 self.kv(indent).with_id(PREFIX_IMPORT_ID, import.this.index)
                     .with_s_key("ImportModule");
-                self.kv(indent2).with_s_key("Name").with_ascii_val(&import.module_name);
+                self.kv(indent2).with_s_key("Name").with_ascii_val(&import.module);
                 self.kv(indent2).with_s_key("Alias").with_ascii_val(&import.alias.value);
                 self.kv(indent2).with_s_key("Target")
                     .with_opt_disp_val(import.module_id.as_ref().map(|v| &v.index));
             },
             Import::Symbols(import) => {
                 self.kv(indent).with_id(PREFIX_IMPORT_ID, import.this.index)
                     .with_s_key("ImportSymbol");
-                self.kv(indent2).with_s_key("Name").with_ascii_val(&import.module_name);
+                self.kv(indent2).with_s_key("Name").with_ascii_val(&import.module);
                 self.kv(indent2).with_s_key("Target")
                     .with_opt_disp_val(import.module_id.as_ref().map(|v| &v.index));
                 self.kv(indent2).with_s_key("Symbols");
                 let indent3 = indent2 + 1;
                 let indent4 = indent3 + 1;
                 for symbol in &import.symbols {
                     self.kv(indent3).with_s_key("AliasedSymbol");
                     self.kv(indent4).with_s_key("Name").with_ascii_val(&symbol.name.value);
                     self.kv(indent4).with_s_key("Alias").with_ascii_val(&symbol.alias.value);
                     self.kv(indent4).with_s_key("Definition")
                         .with_opt_disp_val(symbol.definition_id.as_ref().map(|v| &v.index));
+                }
+            }
+        }
+    }
     //--------------------------------------------------------------------------
     // Top-level definition writing
     //--------------------------------------------------------------------------
     fn write_definition(&mut self, heap: &Heap, def_id: DefinitionId, indent: usize) {
         self.cur_definition = Some(def_id);
         let indent2 = indent + 1;
         let indent3 = indent2 + 1;
         let indent4 = indent3 + 1;
         match &heap[def_id] {
             Definition::Struct(def) => {
                 self.kv(indent).with_id(PREFIX_STRUCT_ID, def.this.0.index)
                     .with_s_key("DefinitionStruct");
                 self.kv(indent2).with_s_key("Name").with_ascii_val(&def.identifier.value);
                 for poly_var_id in &def.poly_vars {
                     self.kv(indent3).with_s_key("PolyVar").with_ascii_val(&poly_var_id.value);
+                }
                 self.kv(indent2).with_s_key("Fields");
                 for field in &def.fields {
                     self.kv(indent3).with_s_key("Field");
                     self.kv(indent4).with_s_key("Name")
                         .with_ascii_val(&field.field.value);
                     self.kv(indent4).with_s_key("Type")
                         .with_custom_val(|s| write_parser_type(s, heap, &heap[field.parser_type]));
+                }
             },
             Definition::Enum(def) => {
                 self.kv(indent).with_id(PREFIX_ENUM_ID, def.this.0.index)
                     .with_s_key("DefinitionEnum");
                 self.kv(indent2).with_s_key("Name").with_ascii_val(&def.identifier.value);
                 for poly_var_id in &def.poly_vars {
                     self.kv(indent3).with_s_key("PolyVar").with_ascii_val(&poly_var_id.value);
+                }
                 self.kv(indent2).with_s_key("Variants");
                 for variant in &def.variants {
                     self.kv(indent3).with_s_key("Variant");
                     self.kv(indent4).with_s_key("Name")
                         .with_ascii_val(&variant.identifier.value);
                     let variant_value = self.kv(indent4).with_s_key("Value");
                     match &variant.value {
                         EnumVariantValue::None => variant_value.with_s_val("None"),
                         EnumVariantValue::Integer(value) => variant_value.with_disp_val(value),
                     };
+                }
             },
             Definition::Union(def) => {
                 self.kv(indent).with_id(PREFIX_UNION_ID, def.this.0.index)
                     .with_s_key("DefinitionUnion");
                 self.kv(indent2).with_s_key("Name").with_ascii_val(&def.identifier.value);
                 for poly_var_id in &def.poly_vars {
                     self.kv(indent3).with_s_key("PolyVar").with_ascii_val(&poly_var_id.value);
+                }
                 self.kv(indent2).with_s_key("Variants");
                 for variant in &def.variants {
                     self.kv(indent3).with_s_key("Variant");
                     self.kv(indent4).with_s_key("Name")
                         .with_ascii_val(&variant.identifier.value);
                     match &variant.value {
                         UnionVariantValue::None => {
                             self.kv(indent4).with_s_key("Value").with_s_val("None");
+                        }
                         UnionVariantValue::Embedded(embedded) => {
                             self.kv(indent4).with_s_key("Values");
                             for embedded in embedded {
                                 self.kv(indent4+1).with_s_key("Value")
                                     .with_custom_val(|v| write_parser_type(v, heap, &heap[*embedded]));
+                            }
+                        }
+                    }
+                }
+            }
             Definition::Function(def) => {
                 self.kv(indent).with_id(PREFIX_FUNCTION_ID, def.this.0.index)
                     .with_s_key("DefinitionFunction");
                 self.kv(indent2).with_s_key("Name").with_ascii_val(&def.identifier.value);
                 for poly_var_id in &def.poly_vars {
                     self.kv(indent3).with_s_key("PolyVar").with_ascii_val(&poly_var_id.value);
+                }
                 self.kv(indent2).with_s_key("ReturnParserType").with_custom_val(|s| write_parser_type(s, heap, &heap[def.return_type]));
                 self.kv(indent2).with_s_key("Parameters");
                 for param_id in &def.parameters {
                     self.write_parameter(heap, *param_id, indent3);
+                }
                 self.kv(indent2).with_s_key("Body");
                 self.write_stmt(heap, def.body, indent3);
             },
             Definition::Component(def) => {
                 self.kv(indent).with_id(PREFIX_COMPONENT_ID,def.this.0.index)
                     .with_s_key("DefinitionComponent");
                 self.kv(indent2).with_s_key("Name").with_ascii_val(&def.identifier.value);
                 self.kv(indent2).with_s_key("Variant").with_debug_val(&def.variant);
                 self.kv(indent2).with_s_key("PolymorphicVariables");
                 for poly_var_id in &def.poly_vars {
                     self.kv(indent3).with_s_key("PolyVar").with_ascii_val(&poly_var_id.value);
+                }
                 self.kv(indent2).with_s_key("Parameters");
                 for param_id in &def.parameters {
                     self.write_parameter(heap, *param_id, indent3)
+                }
                 self.kv(indent2).with_s_key("Body");
                 self.write_stmt(heap, def.body, indent3);
+            }
+        }
+    }
     fn write_parameter(&mut self, heap: &Heap, param_id: ParameterId, indent: usize) {
         let indent2 = indent + 1;
         let param = &heap[param_id];
         self.kv(indent).with_id(PREFIX_PARAMETER_ID, param_id.0.index)
             .with_s_key("Parameter");
         self.kv(indent2).with_s_key("Name").with_ascii_val(&param.identifier.value);
         self.kv(indent2).with_s_key("ParserType").with_custom_val(|w| write_parser_type(w, heap, &heap[param.parser_type]));
+    }
     fn write_poly_args(&mut self, heap: &Heap, poly_args: &[ParserTypeId], indent: usize) {
         if poly_args.is_empty() {
             return
+        }
         let indent2 = indent + 1;
         self.kv(indent).with_s_key("PolymorphicArguments");
         for poly_arg in poly_args {
             self.kv(indent2).with_s_key("Argument")
                 .with_custom_val(|v| write_parser_type(v, heap, &heap[*poly_arg]));
+        }
+    }
     fn write_stmt(&mut self, heap: &Heap, stmt_id: StatementId, indent: usize) {
         let stmt = &heap[stmt_id];
         let indent2 = indent + 1;
         let indent3 = indent2 + 1;
         match stmt {
             Statement::Block(stmt) => {
                 self.kv(indent).with_id(PREFIX_BLOCK_STMT_ID, stmt.this.0.index)
                     .with_s_key("Block");
                 for stmt_id in &stmt.statements {
                     self.write_stmt(heap, *stmt_id, indent2);
+                }
             },
             Statement::Local(stmt) => {
                 match stmt {
                     LocalStatement::Channel(stmt) => {
                         self.kv(indent).with_id(PREFIX_CHANNEL_STMT_ID, stmt.this.0.0.index)
                             .with_s_key("LocalChannel");
                         self.kv(indent2).with_s_key("From");
                         self.write_local(heap, stmt.from, indent3);
                         self.kv(indent2).with_s_key("To");
                         self.write_local(heap, stmt.to, indent3);
                         self.kv(indent2).with_s_key("Next")
                             .with_opt_disp_val(stmt.next.as_ref().map(|v| &v.index));
                     },
                     LocalStatement::Memory(stmt) => {
                         self.kv(indent).with_id(PREFIX_MEM_STMT_ID, stmt.this.0.0.index)
                             .with_s_key("LocalMemory");

src/protocol/input_source2.rs

➞

Show inline comments

 use std::fmt;
 use std::cell::{Ref, RefCell};
 use std::fmt::Write;
 #[derive(Debug, Clone, Copy)]
 pub struct InputPosition2 {
     pub line: u32,
     pub offset: u32,
+}
 impl InputPosition2 {
     pub(crate) fn with_offset(&self, offset: u32) -> Self {
         InputPosition2{ line: self.line, offset: self.offset + offset }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub struct InputSpan {
     pub begin: InputPosition2,
     pub end: InputPosition2,
+}
 impl InputSpan {
     #[inline]
     pub fn from_positions(begin: InputPosition2, end: InputPosition2) -> Self {
         Self { begin, end }
+    }
+}
 /// Wrapper around source file with optional filename. Ensures that the file is
 /// only scanned once.
 pub struct InputSource2 {
     pub(crate) filename: String,
     pub(crate) input: Vec<u8>,
     // Iteration
     line: u32,
     offset: usize,
     // State tracking
     pub(crate) had_error: Option<ParseError>,
     // The offset_lookup is built on-demand upon attempting to report an error.
     // As the compiler is currently not multithreaded, we simply put it in a
     // RefCell to allow interior mutability.
     offset_lookup: RefCell<Vec<u32>>,
+}
 impl InputSource2 {
     pub fn new(filename: String, input: Vec<u8>) -> Self {
         Self{
             filename,
             input,
             line: 1,
             offset: 0,
             had_error: None,
             offset_lookup: RefCell::new(Vec::new()),
+        }
+    }
     #[cfg(test)]
     pub fn new_test(input: &str) -> Self {
         let bytes = Vec::from(input.as_bytes());
         return Self::new(String::from("test"), bytes)
+    }
     #[inline]
     pub fn pos(&self) -> InputPosition2 {
         InputPosition2{ line: self.line, offset: self.offset as u32 }
+    }
     pub fn next(&self) -> Option<u8> {
         if self.offset < self.input.len() {
             Some(self.input[self.offset])
         } else {
             None
+        }
+    }
     pub fn lookahead(&self, offset: usize) -> Option<u8> {
         let offset_pos = self.offset + offset;
         if offset_pos < self.input.len() {
             Some(self.input[offset_pos])
         } else {
             None
+        }
+    }
     pub fn section(&self, start: InputPosition2, end: InputPosition2) -> &[u8] {
     #[inline]
     pub fn section_at_pos(&self, start: InputPosition2, end: InputPosition2) -> &[u8] {
         &self.input[start.offset as usize..end.offset as usize]
+    }
     #[inline]
     pub fn section_at_span(&self, span: InputSpan) -> &[u8] {
         &self.input[span.begin.offset as usize..span.end.offset as usize]
+    }
     // Consumes the next character. Will check well-formedness of newlines: \r
     // must be followed by a \n, because this is used for error reporting. Will
     // not check for ascii-ness of the file, better left to a tokenizer.
     pub fn consume(&mut self) {
         match self.next() {
             Some(b'\r') => {
                 if Some(b'\n') == self.lookahead(1) {
                     // Well formed file
                     self.offset += 1;
                 } else {
                     // Not a well-formed file, pretend like we can continue
                     self.offset += 1;
                     self.set_error("Encountered carriage-feed without a following newline");
+                }
             },
             Some(b'\n') => {
                 self.line += 1;
                 self.offset += 1;
             },
             Some(_) => {
                 self.offset += 1;
+            }
             None => {}
+        }
         // Maybe we actually want to check this in release mode. Then again:
         // a 4 gigabyte source file... Really?
         debug_assert!(self.offset < u32::max_value() as usize);
+    }
     fn set_error(&mut self, msg: &str) {
         if self.had_error.is_none() {
             self.had_error = Some(ParseError::new_error(self, self.pos(), msg));
+        }
+    }
     fn get_lookup(&self) -> Ref<Vec<u32>> {
         // Once constructed the lookup always contains one element. We use this
         // to see if it is constructed already.
         let lookup = self.offset_lookup.borrow();
         if !lookup.is_empty() {
             return lookup;
+        }
         // Build the line number (!) to offset lookup, so offset by 1. We
         // assume the entire source file is scanned (most common case) for
         // preallocation.
         let mut lookup = self.offset_lookup.borrow_mut();
         lookup.reserve(self.line as usize + 2);
         lookup.push(0); // line 0: never used
         lookup.push(0); // first line: first character
         for char_idx in 0..self.input.len() {
             if self.input[char_idx] == b'\n' {
                 lookup.push(char_idx as u32 + 1);
+            }
+        }
         lookup.push(self.input.len() as u32); // for lookup_line_end
         debug_assert_eq!(self.line as usize + 2, lookup.len(), "remove me: i am a testing assert and sometimes invalid");
         // Return created lookup
         let lookup = self.offset_lookup.borrow();
         return lookup;
+    }
     /// Retrieves offset at which line starts (right after newline)
     fn lookup_line_start_offset(&self, line_number: u32) -> u32 {
         let lookup = self.get_lookup();
         lookup[line_number as usize]
+    }
     /// Retrieves offset at which line ends (at the newline character or the
     /// preceding carriage feed for \r\n-encoded newlines)
     fn lookup_line_end_offset(&self, line_number: u32) -> u32 {
         let lookup = self.get_lookup();
         let offset = lookup[(line_number + 1) as usize] - 1;
         let offset_usize = offset as usize;
         // Compensate for newlines and a potential carriage feed
         if self.input[offset_usize] == b'\n' {
             if offset_usize > 0 && self.input[offset_usize] == b'\r' {
                 offset - 2
             } else {
                 offset - 1
+            }
         } else {
             offset
+        }
+    }
+}
 #[derive(Debug)]
 pub enum StatementKind {
     Info,
     Error
+}
 #[derive(Debug)]
 pub enum ContextKind {
     SingleLine,
     MultiLine,
+}
 #[derive(Debug)]
 pub struct ParseErrorStatement {
     pub(crate) statement_kind: StatementKind,
     pub(crate) context_kind: ContextKind,
     pub(crate) start_line: u32,
     pub(crate) start_column: u32,
     pub(crate) end_line: u32,
     pub(crate) end_column: u32,
     pub(crate) filename: String,
     pub(crate) context: String,
     pub(crate) message: String,
+}
 impl ParseErrorStatement {
     fn from_source_at_pos(statement_kind: StatementKind, source: &InputSource2, position: InputPosition2, message: String) -> Self {
         // Seek line start and end
         let line_start = source.lookup_line_start_offset(position.line);
         let line_end = source.lookup_line_end_offset(position.line);
         let context = Self::create_context(source, line_start as usize, line_end as usize);
         debug_assert!(position.offset >= line_start);
         let column = position.offset - line_start + 1;
         Self{
             statement_kind,
             context_kind: ContextKind::SingleLine,
             start_line: position.line,
             start_column: column,
             end_line: position.line,
             end_column: column + 1,
             filename: source.filename.clone(),
             context,
             message,
+        }
+    }
     fn from_source_at_span(statement_kind: StatementKind, source: &InputSource2, span: InputSpan, message: String) -> Self {
         debug_assert!(span.end.line >= span.begin.line);
         debug_assert!(span.end.offset >= span.begin.offset);
         let first_line_start = source.lookup_line_start_offset(span.begin.line);
         let last_line_start = source.lookup_line_start_offset(span.end.line);
         let last_line_end = source.lookup_line_end_offset(span.end.line);
         let context = Self::create_context(source, first_line_start as usize, last_line_end as usize);
         debug_assert!(span.begin.offset >= first_line_start);
         let start_column = span.begin.offset - first_line_start + 1;
         let end_column = span.end.offset - last_line_start + 1;
         let context_kind = if span.begin.line == span.end.line {
             ContextKind::SingleLine
         } else {
             ContextKind::MultiLine
         };
         Self{
             statement_kind,
             context_kind,
             start_line: first_line_start,
             start_column,
             end_line: last_line_start,
             end_column,
             filename: source.filename.clone(),
             context,
             message,
+        }
+    }
     /// Produces context from source
     fn create_context(source: &InputSource2, start: usize, end: usize) -> String {
         let context_raw = &source.input[start..end];
         String::from_utf8_lossy(context_raw).to_string()
+    }
+}
 impl fmt::Display for ParseErrorStatement {
     fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result {
         // Write kind of statement and message
         match self.statement_kind {
             StatementKind::Info => f.write_str(" INFO: ")?,
             StatementKind::Error => f.write_str("ERROR: ")?,
+        }
         f.write_str(&self.message)?;
         f.write_char('\n')?;
         // Write originating file/line/column
         f.write_str(" +- ")?;
         if !self.filename.is_empty() {
             write!(f, "in {} ", self.filename)?;
+        }

src/protocol/lexer.rs

➞

Show inline comments

@@ @@ -2337,334 +2337,334 @@ impl Lexer<'_> { @@
                 };
                 Ok(UnionVariantDefinition{ position, identifier, value })
+            }
         )? {
             Some(variants) => variants,
             None => return Err(ParseError::new_error(
                 self.source, union_pos,
                 "A union definition must be followed by its variants"
             )),
         };
         Ok(h.alloc_union_definition(|this| UnionDefinition{
             this,
             position: union_pos,
             identifier: union_ident,
             poly_vars,
             variants,
         }))
+    }
     fn consume_component_definition(&mut self, h: &mut Heap) -> Result<ComponentId, ParseError> {
         // TODO: Cleanup
         if self.has_keyword(b"composite") {
             Ok(self.consume_composite_definition(h)?)
         } else {
             Ok(self.consume_primitive_definition(h)?)
+        }
+    }
     fn consume_composite_definition(&mut self, h: &mut Heap) -> Result<ComponentId, ParseError> {
         // Parse keyword, optional polyvars and the identifier
         let position = self.source.pos();
         self.consume_keyword(b"composite")?;
         self.consume_whitespace(true)?;
         let identifier = self.consume_identifier()?;
         self.consume_whitespace(false)?;
         let poly_vars = self.consume_polymorphic_vars(h)?;
         self.consume_whitespace(false)?;
         // Consume parameters
         let parameters = self.consume_parameters(h)?;
         self.consume_whitespace(false)?;
         // Parse body
         let body = self.consume_block_statement(h)?;
         Ok(h.alloc_component(|this| ComponentDefinition {
             this,
             variant: ComponentVariant::Composite,
             position,
             identifier,
             poly_vars,
             parameters,
             body
         }))
+    }
     fn consume_primitive_definition(&mut self, h: &mut Heap) -> Result<ComponentId, ParseError> {
         // Consume keyword, optional polyvars and identifier
         let position = self.source.pos();
         self.consume_keyword(b"primitive")?;
         self.consume_whitespace(true)?;
         let identifier = self.consume_identifier()?;
         self.consume_whitespace(false)?;
         let poly_vars = self.consume_polymorphic_vars(h)?;
         self.consume_whitespace(false)?;
         // Consume parameters
         let parameters = self.consume_parameters(h)?;
         self.consume_whitespace(false)?;
         // Consume body
         let body = self.consume_block_statement(h)?;
         Ok(h.alloc_component(|this| ComponentDefinition {
             this,
             variant: ComponentVariant::Primitive,
             position,
             identifier,
             poly_vars,
             parameters,
             body
         }))
+    }
     fn consume_function_definition(&mut self, h: &mut Heap) -> Result<FunctionId, ParseError> {
         // Consume return type, optional polyvars and identifier
         let position = self.source.pos();
         let return_type = self.consume_type(h, false)?;
         self.consume_whitespace(true)?;
         let identifier = self.consume_identifier()?;
         self.consume_whitespace(false)?;
         let poly_vars = self.consume_polymorphic_vars(h)?;
         self.consume_whitespace(false)?;
         // Consume parameters
         let parameters = self.consume_parameters(h)?;
         self.consume_whitespace(false)?;
         // Consume body
         let body = self.consume_block_statement(h)?;
         Ok(h.alloc_function(|this| FunctionDefinition {
             this,
             position,
             return_type,
             identifier,
             poly_vars,
             parameters,
             body,
         }))
+    }
     fn has_pragma(&self) -> bool {
         if let Some(c) = self.source.next() {
             c == b'#'
         } else {
             false
+        }
+    }
     fn consume_pragma(&mut self, h: &mut Heap) -> Result<PragmaId, ParseError> {
         let position = self.source.pos();
         let next = self.source.next();
         if next != Some(b'#') {
             return Err(self.error_at_pos("Expected pragma"));
+        }
         self.source.consume();
         if !is_vchar(self.source.next()) {
             return Err(self.error_at_pos("Expected pragma"));
+        }
         if self.has_string(b"version") {
             self.consume_string(b"version")?;
             self.consume_whitespace(true)?;
             if !self.has_integer() {
                 return Err(self.error_at_pos("Expected integer constant"));
+            }
             let version = self.consume_integer()?;
             debug_assert!(version >= 0);
             return Ok(h.alloc_pragma(|this| Pragma::Version(PragmaVersion{
                 this, position, version: version as u64
             })))
         } else if self.has_string(b"module") {
             self.consume_string(b"module")?;
             self.consume_whitespace(true)?;
             if !self.has_identifier() {
                 return Err(self.error_at_pos("Expected identifier"));
+            }
             let mut value = Vec::new();
             let mut ident = self.consume_ident()?;
             value.append(&mut ident);
             while self.has_string(b".") {
                 self.consume_string(b".")?;
                 value.push(b'.');
                 ident = self.consume_ident()?;
                 value.append(&mut ident);
+            }
             return Ok(h.alloc_pragma(|this| Pragma::Module(PragmaModule{
                 this, position, value
             })));
         } else {
             return Err(self.error_at_pos("Unknown pragma"));
+        }
+    }
     fn has_import(&self) -> bool {
         self.has_keyword(b"import")
+    }
     fn consume_import(&mut self, h: &mut Heap) -> Result<ImportId, ParseError> {
         // Parse the word "import" and the name of the module
         let position = self.source.pos();
         self.consume_keyword(b"import")?;
         self.consume_whitespace(true)?;
         let mut value = Vec::new();
         let mut last_ident_pos = self.source.pos();
         let mut ident = self.consume_ident()?;
         value.append(&mut ident);
         let mut last_ident_start = 0;
         while self.has_string(b".") {
             self.consume_string(b".")?;
             value.push(b'.');
             last_ident_pos = self.source.pos();
             ident = self.consume_ident()?;
             last_ident_start = value.len();
             value.append(&mut ident);
+        }
         self.consume_whitespace(false)?;
         // Check for the potential aliasing or specific module imports
         let import = if self.has_string(b"as") {
             self.consume_string(b"as")?;
             self.consume_whitespace(true)?;
             let alias = self.consume_identifier()?;
             h.alloc_import(|this| Import::Module(ImportModule{
                 this,
                 position,
-                module_name: value,
+                module: value,
                 alias,
                 module_id: None,
             }))
         } else if self.has_string(b"::") {
             self.consume_string(b"::")?;
             self.consume_whitespace(false)?;
             let next = self.source.next();
             if Some(b'{') == next {
                 let symbols = match self.consume_comma_separated(
                     h, b'{', b'}', "Expected end of import list",
                     |lexer, _heap| {
                         // Symbol name
                         let position = lexer.source.pos();
                         let name = lexer.consume_identifier()?;
                         lexer.consume_whitespace(false)?;
                         // Symbol alias
                         if lexer.has_string(b"as") {
                             // With alias
                             lexer.consume_string(b"as")?;
                             lexer.consume_whitespace(true)?;
                             let alias = lexer.consume_identifier()?;
                             Ok(AliasedSymbol{
                                 position,
                                 name,
                                 alias,
                                 definition_id: None
                             })
                         } else {
                             // Without alias
                             Ok(AliasedSymbol{
                                 position,
                                 name: name.clone(),
                                 alias: name,
                                 definition_id: None
                             })
+                        }
+                    }
                 )? {
                     Some(symbols) => symbols,
                     None => unreachable!(), // because we checked for opening '{'
                 };
                 h.alloc_import(|this| Import::Symbols(ImportSymbols{
                     this,
                     position,
-                    module_name: value,
+                    module: value,
                     module_id: None,
                     symbols,
                 }))
             } else if Some(b'*') == next {
                 self.source.consume();
                 h.alloc_import(|this| Import::Symbols(ImportSymbols{
                     this,
                     position,
-                    module_name: value,
+                    module: value,
                     module_id: None,
                     symbols: Vec::new()
                 }))
             } else if self.has_identifier() {
                 let position = self.source.pos();
                 let name = self.consume_identifier()?;
                 self.consume_whitespace(false)?;
                 let alias = if self.has_string(b"as") {
                     self.consume_string(b"as")?;
                     self.consume_whitespace(true)?;
                     self.consume_identifier()?
                 } else {
                     name.clone()
                 };
                 h.alloc_import(|this| Import::Symbols(ImportSymbols{
                     this,
                     position,
-                    module_name: value,
+                    module: value,
                     module_id: None,
                     symbols: vec![AliasedSymbol{
                         position,
                         name,
                         alias,
                         definition_id: None
                     }]
                 }))
             } else {
                 return Err(self.error_at_pos("Expected '*', '{' or a symbol name"));
+            }
         } else {
             // No explicit alias or subimports, so implicit alias
             let alias_value = Vec::from(&value[last_ident_start..]);
             h.alloc_import(|this| Import::Module(ImportModule{
                 this,
                 position,
-                module_name: value,
+                module: value,
                 alias: Identifier{
                     position: last_ident_pos,
                     value: Vec::from(alias_value),
                 },
                 module_id: None,
             }))
         };
         self.consume_whitespace(false)?;
         self.consume_string(b";")?;
         Ok(import)
+    }
     pub fn consume_protocol_description(&mut self, h: &mut Heap) -> Result<RootId, ParseError> {
         let position = self.source.pos();
         let mut pragmas = Vec::new();
         let mut imports = Vec::new();
         let mut definitions = Vec::new();
         self.consume_whitespace(false)?;
         while self.has_pragma() {
             let pragma = self.consume_pragma(h)?;
             pragmas.push(pragma);
             self.consume_whitespace(false)?;
+        }
         while self.has_import() {
             let import = self.consume_import(h)?;
             imports.push(import);
             self.consume_whitespace(false)?;
+        }
         while self.has_symbol_definition() {
             let def = self.consume_symbol_definition(h)?;
             definitions.push(def);
             self.consume_whitespace(false)?;
+        }
         // end of file
         if !self.source.is_eof() {
             return Err(self.error_at_pos("Expected end of file"));
+        }
         Ok(h.alloc_protocol_description(|this| Root {
             this,
             position,
             pragmas,
             imports,
             definitions,
         }))
+    }
+}

src/protocol/lexer2.rs

➞

Show inline comments

deleted file

src/protocol/mod.rs

➞

Show inline comments

 mod arena;
 // mod ast;
 mod eval;
 pub(crate) mod inputsource;
 pub(crate) mod input_source2;
 // mod lexer;
 mod tokenizer;
 mod parser;
 #[cfg(test)] mod tests;
 // TODO: Remove when not benchmarking
 pub(crate) mod ast;
 pub(crate) mod ast_printer;
 pub(crate) mod lexer;
 pub(crate) mod lexer2;
 lazy_static::lazy_static! {
     /// Conveniently-provided protocol description initialized with a zero-length PDL string.
     /// Exposed to minimize repeated initializations of this common protocol description.
     pub static ref TRIVIAL_PD: std::sync::Arc<ProtocolDescription> = {
         std::sync::Arc::new(ProtocolDescription::parse(b"").unwrap())
     };
+}
 use crate::common::*;
 use crate::protocol::ast::*;
 use crate::protocol::eval::*;
 use crate::protocol::inputsource::*;
 use crate::protocol::parser::*;
 /// Description of a protocol object, used to configure new connectors.
 #[repr(C)]
 pub struct ProtocolDescription {
     heap: Heap,
     source: InputSource,
     root: RootId,
+}
 #[derive(Debug, Clone)]
 pub(crate) struct ComponentState {
     prompt: Prompt,
+}
 pub(crate) enum EvalContext<'a> {
     Nonsync(&'a mut NonsyncProtoContext<'a>),
     Sync(&'a mut SyncProtoContext<'a>),
     // None,
+}
 //////////////////////////////////////////////
 impl std::fmt::Debug for ProtocolDescription {
     fn fmt(&self, f: &mut std::fmt::Formatter) -> std::fmt::Result {
         write!(f, "(An opaque protocol description)")
+    }
+}
 impl ProtocolDescription {
     // TODO: Allow for multi-file compilation
     pub fn parse(buffer: &[u8]) -> Result<Self, String> {
         // TODO: @fixme, keep code compilable, but needs support for multiple
         //  input files.
         let source = InputSource::from_buffer(buffer).unwrap();
         let mut parser = Parser::new();
         parser.feed(source).expect("failed to lex source");
         if let Err(err) = parser.parse() {
             println!("ERROR:\n{}", err);
             return Err(format!("{}", err))
+        }
         debug_assert_eq!(parser.modules.len(), 1, "only supporting one module here for now");
         let root = parser.modules[0].root_id;
         return Ok(ProtocolDescription { heap: parser.heap, source: parser.modules[0].source.clone(), root });
+    }
     pub(crate) fn component_polarities(
         &self,
         identifier: &[u8],
     ) -> Result<Vec<Polarity>, AddComponentError> {
         use AddComponentError::*;
         let h = &self.heap;
         let root = &h[self.root];
         let def = root.get_definition_ident(h, identifier);
         if def.is_none() {
             return Err(NoSuchComponent);
+        }
         let def = &h[def.unwrap()];
         if !def.is_component() {
             return Err(NoSuchComponent);
+        }
         for &param in def.parameters().iter() {
             let param = &h[param];
             let parser_type = &h[param.parser_type];
             match parser_type.variant {
                 ParserTypeVariant::Input(_) | ParserTypeVariant::Output(_) => continue,
                 _ => {
                     return Err(NonPortTypeParameters);
+                }
+            }
+        }
         let mut result = Vec::new();
         for &param in def.parameters().iter() {
             let param = &h[param];
             let parser_type = &h[param.parser_type];
             if let ParserTypeVariant::Input(_) = parser_type.variant {
                 result.push(Polarity::Getter)
             } else if let ParserTypeVariant::Output(_) = parser_type.variant {
                 result.push(Polarity::Putter)
             } else {
                 unreachable!()
+            }
+        }
         Ok(result)
+    }
     // expects port polarities to be correct
     pub(crate) fn new_component(&self, identifier: &[u8], ports: &[PortId]) -> ComponentState {
         let mut args = Vec::new();
         for (&x, y) in ports.iter().zip(self.component_polarities(identifier).unwrap()) {
             match y {
                 Polarity::Getter => args.push(Value::Input(InputValue(x))),
                 Polarity::Putter => args.push(Value::Output(OutputValue(x))),
+            }
+        }
         let h = &self.heap;
         let root = &h[self.root];
         let def = root.get_definition_ident(h, identifier).unwrap();
         ComponentState { prompt: Prompt::new(h, def, &args) }
+    }
+}
 impl ComponentState {
     pub(crate) fn nonsync_run<'a: 'b, 'b>(
         &'a mut self,
         context: &'b mut NonsyncProtoContext<'b>,
         pd: &'a ProtocolDescription,
     ) -> NonsyncBlocker {
         let mut context = EvalContext::Nonsync(context);
         loop {
             let result = self.prompt.step(&pd.heap, &mut context);
             match result {
                 // In component definitions, there are no return statements
                 Ok(_) => unreachable!(),
                 Err(cont) => match cont {
                     EvalContinuation::Stepping => continue,
                     EvalContinuation::Inconsistent => return NonsyncBlocker::Inconsistent,
                     EvalContinuation::Terminal => return NonsyncBlocker::ComponentExit,
                     EvalContinuation::SyncBlockStart => return NonsyncBlocker::SyncBlockStart,
                     // Not possible to end sync block if never entered one
                     EvalContinuation::SyncBlockEnd => unreachable!(),
                     EvalContinuation::NewComponent(definition_id, args) => {
                         // Look up definition (TODO for now, assume it is a definition)
                         let h = &pd.heap;
                         let init_state = ComponentState { prompt: Prompt::new(h, definition_id, &args) };
                         context.new_component(&args, init_state);
                         // Continue stepping
                         continue;
+                    }
                     // Outside synchronous blocks, no fires/get/put happens
                     EvalContinuation::BlockFires(_) => unreachable!(),
                     EvalContinuation::BlockGet(_) => unreachable!(),
                     EvalContinuation::Put(_, _) => unreachable!(),
                 },
+            }
+        }
+    }
     pub(crate) fn sync_run<'a: 'b, 'b>(
         &'a mut self,
         context: &'b mut SyncProtoContext<'b>,
         pd: &'a ProtocolDescription,
     ) -> SyncBlocker {
         let mut context = EvalContext::Sync(context);
         loop {
             let result = self.prompt.step(&pd.heap, &mut context);
             match result {
                 // Inside synchronous blocks, there are no return statements
                 Ok(_) => unreachable!(),
                 Err(cont) => match cont {
                     EvalContinuation::Stepping => continue,
                     EvalContinuation::Inconsistent => return SyncBlocker::Inconsistent,
                     // First need to exit synchronous block before definition may end
                     EvalContinuation::Terminal => unreachable!(),
                     // No nested synchronous blocks
                     EvalContinuation::SyncBlockStart => unreachable!(),
                     EvalContinuation::SyncBlockEnd => return SyncBlocker::SyncBlockEnd,
                     // Not possible to create component in sync block
                     EvalContinuation::NewComponent(_, _) => unreachable!(),
                     EvalContinuation::BlockFires(port) => match port {
                         Value::Output(OutputValue(port)) => {
                             return SyncBlocker::CouldntCheckFiring(port);
+                        }
                         Value::Input(InputValue(port)) => {
                             return SyncBlocker::CouldntCheckFiring(port);
+                        }
                         _ => unreachable!(),
                     },
                     EvalContinuation::BlockGet(port) => match port {
                         Value::Output(OutputValue(port)) => {
                             return SyncBlocker::CouldntReadMsg(port);
+                        }
                         Value::Input(InputValue(port)) => {
                             return SyncBlocker::CouldntReadMsg(port);
+                        }
                         _ => unreachable!(),
                     },
                     EvalContinuation::Put(port, message) => {
                         let value;
                         match port {
                             Value::Output(OutputValue(port_value)) => {
                                 value = port_value;

src/protocol/parser/mod.rs

➞

Show inline comments

 mod depth_visitor;
 pub(crate) mod symbol_table;
 pub(crate) mod symbol_table2;
 pub(crate) mod type_table;
 pub(crate) mod tokens;
 pub(crate) mod token_parsing;
 pub(crate) mod pass_tokenizer;
 pub(crate) mod pass_symbols;
 pub(crate) mod pass_imports;
 pub(crate) mod pass_definitions;
 mod type_resolver;
 mod visitor;
 mod visitor_linker;
 mod utils;
 use depth_visitor::*;
 use symbol_table::SymbolTable;
 use tokens::*;
 use crate::collections::*;
 use symbol_table2::SymbolTable;
 use visitor::Visitor2;
 use visitor_linker::ValidityAndLinkerVisitor;
 use type_resolver::{TypeResolvingVisitor, ResolveQueue};
 use type_table::{TypeTable, TypeCtx};
 use crate::protocol::ast::*;
-use crate::protocol::inputsource::*;
+use crate::protocol::input_source2::{InputSource2 as InputSource};
 use crate::protocol::lexer::*;
 use std::collections::HashMap;
 use crate::protocol::ast_printer::ASTWriter;
 #[derive(PartialEq, Eq)]
 pub enum ModuleCompilationPhase {
     Source,                 // only source is set
     Tokenized,              // source is tokenized
     SymbolsScanned,         // all definitions are linked to their type class
     ImportsResolved,        // all imports are added to the symbol table
     DefinitionsParsed,      // produced the AST for the entire module
     ValidatedAndLinked,     // AST is traversed and has linked the required AST nodes
     Typed,                  // Type inference and checking has been performed
+}
 pub struct Module {
     // Buffers
     source: InputSource,
     tokens: TokenBuffer,
     // Identifiers
     root_id: RootId,
     name: Option<(PragmaId, StringRef<'static>)>,
     version: Option<(PragmaId, i64)>,
     phase: ModuleCompilationPhase,
+}
 pub struct PassCtx<'a> {
     heap: &'a mut Heap,
     symbols: &'a mut SymbolTable,
     pool: &'a mut StringPool,
+}
 // TODO: @fixme, pub qualifier
 pub(crate) struct LexedModule {
     pub(crate) source: InputSource,
     module_name: Vec<u8>,
     version: Option<u64>,
     pub(crate) root_id: RootId,
+}
 pub struct Parser {
     pub(crate) heap: Heap,
     pub(crate) modules: Vec<LexedModule>,
     pub(crate) module_lookup: HashMap<Vec<u8>, usize>, // from (optional) module name to `modules` idx
     pub(crate) symbol_table: SymbolTable,
     pub(crate) type_table: TypeTable,
+}
 impl Parser {
     pub fn new() -> Self {
         Parser{
             heap: Heap::new(),
             modules: Vec::new(),
             module_lookup: HashMap::new(),
             symbol_table: SymbolTable::new(),
             type_table: TypeTable::new(),
+        }
+    }
     pub fn feed(&mut self, mut source: InputSource) -> Result<RootId, ParseError> {
         // Lex the input source
         let mut lex = Lexer::new(&mut source);
         let pd = lex.consume_protocol_description(&mut self.heap)?;
         // Seek the module name and version
         let root = &self.heap[pd];
         let mut module_name_pos = InputPosition::default();
         let mut module_name = Vec::new();
         let mut module_version_pos = InputPosition::default();
         let mut module_version = None;
         for pragma in &root.pragmas {
             match &self.heap[*pragma] {
                 Pragma::Module(module) => {
                     if !module_name.is_empty() {
                         return Err(
                             ParseError::new_error(&source, module.position, "Double definition of module name in the same file")
                                 .with_postfixed_info(&source, module_name_pos, "Previous definition was here")
+                        )
+                    }
                     module_name_pos = module.position.clone();
                     module_name = module.value.clone();
                 },
                 Pragma::Version(version) => {
                     if module_version.is_some() {
                         return Err(
                             ParseError::new_error(&source, version.position, "Double definition of module version")
                                 .with_postfixed_info(&source, module_version_pos, "Previous definition was here")
+                        )
+                    }
                     module_version_pos = version.position.clone();
                     module_version = Some(version.version);
                 },
+            }
+        }
         // Add module to list of modules and prevent naming conflicts
         let cur_module_idx = self.modules.len();
         if let Some(prev_module_idx) = self.module_lookup.get(&module_name) {
             // Find `#module` statement in other module again
             let prev_module = &self.modules[*prev_module_idx];
             let prev_module_pos = self.heap[prev_module.root_id].pragmas
                 .iter()
                 .find_map(|p| {
                     match &self.heap[*p] {
                         Pragma::Module(module) => Some(module.position.clone()),
                         _ => None
+                    }
                 })
                 .unwrap_or(InputPosition::default());
             let module_name_msg = if module_name.is_empty() {
                 format!("a nameless module")
             } else {
                 format!("module '{}'", String::from_utf8_lossy(&module_name))
             };
             return Err(
                 ParseError::new_error(&source, module_name_pos, &format!("Double definition of {} across files", module_name_msg))
                     .with_postfixed_info(&prev_module.source, prev_module_pos, "Other definition was here")
             );
+        }
         self.modules.push(LexedModule{
             source,
             module_name: module_name.clone(),
             version: module_version,
             root_id: pd
         });
         self.module_lookup.insert(module_name, cur_module_idx);
         Ok(pd)
+    }
     fn resolve_symbols_and_types(&mut self) -> Result<(), ParseError> {
         // Construct the symbol table to resolve any imports and/or definitions,
         // then use the symbol table to actually annotate all of the imports.
         // If the type table is constructed correctly then all imports MUST be
         // resolvable.
         self.symbol_table.build(&self.heap, &self.modules)?;
         // Not pretty, but we need to work around rust's borrowing rules, it is
         // totally safe to mutate the contents of an AST element that we are
         // not borrowing anywhere else.
         let mut module_index = 0;
         let mut import_index = 0;
         loop {
             if module_index >= self.modules.len() {
                 break;
+            }
             let module_root_id = self.modules[module_index].root_id;
             let import_id = {
                 let root = &self.heap[module_root_id];
                 if import_index >= root.imports.len() {
                     module_index += 1;
                     import_index = 0;
                     continue
+                }
                 root.imports[import_index]
             };
             let import = &mut self.heap[import_id];
             match import {
                 Import::Module(import) => {
                     debug_assert!(import.module_id.is_none(), "module import already resolved");
-                    let target_module_id = self.symbol_table.resolve_module(&import.module_name)
+                    let target_module_id = self.symbol_table.resolve_module(&import.module)
                         .expect("module import is resolved by symbol table");
                     import.module_id = Some(target_module_id)
                 },
                 Import::Symbols(import) => {
                     debug_assert!(import.module_id.is_none(), "module of symbol import already resolved");
-                    let target_module_id = self.symbol_table.resolve_module(&import.module_name)
+                    let target_module_id = self.symbol_table.resolve_module(&import.module)
                         .expect("symbol import's module is resolved by symbol table");
                     import.module_id = Some(target_module_id);
                     for symbol in &mut import.symbols {
                         debug_assert!(symbol.definition_id.is_none(), "symbol import already resolved");
                         let (_, target_definition_id) = self.symbol_table.resolve_identifier(module_root_id, &symbol.alias)
                             .expect("symbol import is resolved by symbol table")
                             .as_definition()
                             .expect("symbol import does not resolve to namespace symbol");
                         symbol.definition_id = Some(target_definition_id);
+                    }
+                }
+            }
             import_index += 1;
+        }
         // All imports in the AST are now annotated. We now use the symbol table
         // to construct the type table.
         let mut type_ctx = TypeCtx::new(&self.symbol_table, &mut self.heap, &self.modules);
         self.type_table.build_base_types(&mut type_ctx)?;
         Ok(())
+    }
     pub fn parse(&mut self) -> Result<(), ParseError> {
         self.resolve_symbols_and_types()?;
         // Validate and link all modules
         let mut visit = ValidityAndLinkerVisitor::new();
         for module in &self.modules {
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 module,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
             };
             visit.visit_module(&mut ctx)?;
+        }
         // Perform typechecking on all modules
         let mut visit = TypeResolvingVisitor::new();
         let mut queue = ResolveQueue::new();
         for module in &self.modules {
             let ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 module,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
             };
             TypeResolvingVisitor::queue_module_definitions(&ctx, &mut queue);
         };
         while !queue.is_empty() {
             let top = queue.pop().unwrap();
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 module: &self.modules[top.root_id.index as usize],
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
             };
             visit.handle_module_definition(&mut ctx, &mut queue, top)?;
+        }
         // Perform remaining steps
         // TODO: Phase out at some point
         for module in &self.modules {
             let root_id = module.root_id;
             if let Err((position, message)) = Self::parse_inner(&mut self.heap, root_id) {
                 return Err(ParseError::new_error(&self.modules[0].source, position, &message))
+            }
+        }
         // let mut writer = ASTWriter::new();
         // let mut file = std::fs::File::create(std::path::Path::new("ast.txt")).unwrap();
         // writer.write_ast(&mut file, &self.heap);
         Ok(())
+    }
     pub fn parse_inner(h: &mut Heap, pd: RootId) -> VisitorResult {
         // TODO: @cleanup, slowly phasing out old compiler
         // NestedSynchronousStatements::new().visit_protocol_description(h, pd)?;
         // ChannelStatementOccurrences::new().visit_protocol_description(h, pd)?;
         // FunctionStatementReturns::new().visit_protocol_description(h, pd)?;
         // ComponentStatementReturnNew::new().visit_protocol_description(h, pd)?;
         // CheckBuiltinOccurrences::new().visit_protocol_description(h, pd)?;
         // BuildSymbolDeclarations::new().visit_protocol_description(h, pd)?;
         // LinkCallExpressions::new().visit_protocol_description(h, pd)?;
         // BuildScope::new().visit_protocol_description(h, pd)?;
         // ResolveVariables::new().visit_protocol_description(h, pd)?;
         LinkStatements::new().visit_protocol_description(h, pd)?;
         // BuildLabels::new().visit_protocol_description(h, pd)?;
         // ResolveLabels::new().visit_protocol_description(h, pd)?;
         AssignableExpressions::new().visit_protocol_description(h, pd)?;
         IndexableExpressions::new().visit_protocol_description(h, pd)?;
         SelectableExpressions::new().visit_protocol_description(h, pd)?;
         Ok(())
+    }
+}
@@ \ No newline at end of file @@

src/protocol/parser/pass_definitions.rs

➞

Show inline comments

@@ new file 100644 @@
 use crate::protocol::ast::*;
 use super::symbol_table2::*;
 use super::{Module, ModuleCompilationPhase, PassCtx};
 use super::tokens::*;
 use super::token_parsing::*;
 use crate::protocol::input_source2::{InputSource2 as InputSource, InputSpan, ParseError};
 use crate::collections::*;
 /// Parses all the tokenized definitions into actual AST nodes.
 pub(crate) struct PassDefinitions {
     identifiers: Vec<Identifier>,
     struct_fields: Vec<StructFieldDefinition>,
+}
 impl PassDefinitions {
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let module_range = &module.tokens.ranges[0];
         debug_assert_eq!(module.phase, ModuleCompilationPhase::ImportsResolved);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         let mut range_idx = module_range.first_child_idx;
         loop {
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
             if cur_range.range_kind == TokenRangeKind::Definition {
                 self.visit_definition_range(modules, module_idx, ctx, range_idx_usize)?;
+            }
             match cur_range.next_sibling_idx {
                 Some(idx) => { range_idx = idx; },
                 None => { break; },
+            }
+        }
         Ok(())
+    }
     fn visit_definition_range(
         &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize
     ) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let cur_range = &module.tokens.ranges[range_idx];
         debug_assert_eq!(cur_range.range_kind, TokenRangeKind::Definition);
         // Detect which definition we're parsing
         let mut iter = module.tokens.iter_range(cur_range);
         let keyword = peek_ident(&module.source, &mut iter).unwrap();
         match keyword {
             KW_STRUCT => {
             },
             KW_ENUM => {
             },
             KW_UNION => {
             },
             KW_FUNCTION => {
             },
             KW_PRIMITIVE => {
             },
             KW_COMPOSITE => {
             },
             _ => unreachable!("encountered keyword '{}' in definition range", String::from_utf8_lossy(keyword)),
         };
         Ok(())
+    }
     fn visit_struct_definition(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<(), ParseError> {
         // Consume struct and name of struct
         let struct_span = consume_exact_ident(&module.source, iter, b"struct");
         let (ident_text, _) = consume_ident(&module.source, iter)?;
         // We should have preallocated the definition in the heap, retrieve its identifier
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(SymbolScope::Module(module.root_id), ident_text)
             .unwrap().variant.as_definition().definition_id;
         consume_polymorphic_vars(source, iter, ctx, &mut self.identifiers)?;
         debug_assert!(self.struct_fields.is_empty());
         consume_comma_separated(
             TokenKind::OpenCurly, TokenKind::CloseCurly, source, iter,
             |source, iter| {
                 let field = consume_ident_interned(source, iter, ctx)?;
                 StructFieldDefinition{ field, parser_type }
             },
             &mut self.struct_fields, "a struct field", "a list of struct fields"
+        )
+    }
+}
 enum TypeKind {
     Message,
     Bool,
     UInt8, UInt16, UInt32, UInt64,
     SInt8, SInt16, SInt32, SInt64,
     Character, String,
     Inferred,
     Array,
     Input,
     Output,
     SymbolicDefinition(DefinitionId),
     SymbolicPolyArg(DefinitionId, usize),
+}
 /// Consumes a type. A type always starts with an identifier which may indicate
 /// a builtin type or a user-defined type. The fact that it may contain
 /// polymorphic arguments makes it a tree-like structure. Because we cannot rely
 /// on knowing the exact number of polymorphic arguments we do not check for
 /// these.
 fn consume_parser_type(
     source: &InputSource, iter: &mut TokenIter, ctx: &mut PassCtx, poly_vars: &[Identifier]
 ) -> Result<(), ParseError> {
     struct StackEntry {
         angle_depth: i32,
+    }
     let mut type_stack = Vec::new();
     Ok(())
+}
 fn consume_parser_type_ident(
     source: &InputSource, iter: &mut TokenIter, ctx: &mut PassCtx,
     mut scope: SymbolScope, wrapping_definition: DefinitionId, poly_vars: &[Identifier]
 ) -> Result<(TypeKind, InputSpan), ParseError> {
     let (type_text, type_span) = consume_any_ident(source, iter)?;
     let type_kind = match type_text {
         KW_TYPE_MESSAGE => TypeKind::Message,
         KW_TYPE_BOOL => TypeKind::Bool,
         KW_TYPE_UINT8 => TypeKind::UInt8,
         KW_TYPE_UINT16 => TypeKind::UInt16,
         KW_TYPE_UINT32 => TypeKind::UInt32,
         KW_TYPE_UINT64 => TypeKind::UInt64,
         KW_TYPE_SINT8 => TypeKind::SInt8,
         KW_TYPE_SINT16 => TypeKind::SInt16,
         KW_TYPE_SINT32 => TypeKind::SInt32,
         KW_TYPE_SINT64 => TypeKind::SInt64,
         KW_TYPE_IN_PORT => TypeKind::Input,
         KW_TYPE_OUT_PORT => TypeKind::Output,
         _ => {
             // Must be some kind of symbolic type
             let mut type_kind = None;
             for (poly_idx, poly_var) in poly_vars.iter().enumerate() {
                 if poly_var.value.as_bytes() == type_text {
                     type_kind = Some(TypeKind::SymbolicPolyArg(wrapping_definition, poly_idx));
+                }
+            }
             if type_kind.is_none() {
                 // Check symbol table for definition
                 let last_symbol = ctx.symbols.get_symbol_by_name(scope, type_text);
                 if last_symbol.is_none() {
                     return Err(ParseError::new_error_str_at_span(source, type_span, "unknown type"));
+                }
                 let last_symbol = last_symbol.unwrap();
                 match last_symbol.variant {
                     SymbolVariant::Module(symbol_module) => {
                         // Keep seeking
                     },
                     SymbolVariant::Definition(symbol_definition) => {
+                    }
+                }
+            }
+        }
+    }
     Ok(())
+}
 /// Consumes polymorphic variables (i.e. a list of identifiers). If the list is
 /// absent then we simply return an empty array.
 fn consume_polymorphic_vars(
     source: &InputSource, iter: &mut TokenIter, ctx: &mut PassCtx, target: &mut Vec<Identifier>
 ) -> Result<(), ParseError> {
     // Note: because this is just a list of identifiers, we don't have to take
     // two `TokenKind::CloseAngle` interpreted as `TokenKind::ShiftRight` into
     // account.
     debug_assert!(target.is_empty());
     maybe_consume_comma_separated(
         TokenKind::OpenAngle, TokenKind::CloseAngle, source, iter,
         |source, iter| consume_ident_interned(source, iter, ctx),
         target, "a polymorphic variable"
     )?;
     Ok(())
+}
@@ \ No newline at end of file @@

src/protocol/parser/pass_imports.rs

➞

Show inline comments

@@ new file 100644 @@
 use crate::protocol::ast::*;
 use super::symbol_table2::*;
 use super::{Module, ModuleCompilationPhase, PassCtx};
 use super::tokens::*;
 use super::token_parsing::*;
 use crate::protocol::input_source2::{InputSource2 as InputSource, InputSpan, ParseError};
 use crate::collections::*;
 /// Parses all the imports in the module tokens. Is applied after the
 /// definitions and name of modules are resolved. Hence we should be able to
 /// resolve all symbols to their appropriate module/definition.
 pub(crate) struct PassImport {
     imports: Vec<ImportId>,
     found_symbols: Vec<(AliasedSymbol, SymbolDefinition)>,
     scoped_symbols: Vec<Symbol>,
+}
 impl PassImport {
     pub(crate) fn new() -> Self {
         Self{
             imports: Vec::with_capacity(32),
             found_symbols: Vec::with_capacity(32),
             scoped_symbols: Vec::with_capacity(32),
+        }
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let module_range = &module.tokens.ranges[0];
         debug_assert!(modules.iter().all(|m| m.phase >= ModuleCompilationPhase::SymbolsScanned));
         debug_assert_eq!(module.phase, ModuleCompilationPhase::SymbolsScanned);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         let mut range_idx = module_range.first_child_idx;
         loop {
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
             if cur_range.range_kind == TokenRangeKind::Import {
                 self.visit_import_range(modules, module_idx, ctx, range_idx_usize)?;
+            }
             match cur_range.next_sibling_idx {
                 Some(idx) => { range_idx = idx; },
                 None => { break; }
+            }
+        }
         let root = &mut ctx.heap[module.root_id];
         root.imports.extend(self.imports.drain(..));
         let module = &mut modules[module_idx];
         module.phase = ModuleCompilationPhase::ImportsResolved;
         Ok(())
+    }
     pub(crate) fn visit_import_range(
         &mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize
     ) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let import_range = &module.tokens.ranges[range_idx];
         debug_assert_eq!(import_range.range_kind, TokenRangeKind::Import);
         let mut iter = module.tokens.iter_range(import_range);
         // Consume "import"
         let (_import_ident, import_span) =
             consume_any_ident(&module.source, &mut iter)?;
         debug_assert_eq!(_import_ident, KW_IMPORT);
         // Consume module name
         let (module_name, module_name_span) = consume_domain_ident(&module.source, &mut iter)?;
         let target_root_id = ctx.symbols.get_module_by_name(module_name);
         if target_root_id.is_none() {
             return Err(ParseError::new_error_at_span(
                 &module.source, module_name_span,
                 format!("could not resolve module '{}'", String::from_utf8_lossy(module_name))
             ));
+        }
         let module_name = ctx.pool.intern(module_name);
         let module_identifier = Identifier{ span: module_name_span, value: module_name };
         let target_root_id = target_root_id.unwrap();
         // Check for subsequent characters (alias, multiple imported symbols)
         let next = iter.next();
         let import_id;
         if has_ident(&module.source, &mut iter, b"as") {
             // Alias for module
             iter.consume();
             let alias_identifier = consume_ident_interned(&module.source, &mut iter, ctx)?;
             let alias_name = alias_identifier.value.clone();
             import_id = ctx.heap.alloc_import(|this| Import::Module(ImportModule{
                 this,
                 span: import_span,
                 module: module_identifier,
                 alias: alias_identifier,
                 module_id: target_root_id
             }));
             ctx.symbols.insert_symbol(SymbolScope::Module(module.root_id), Symbol{
                 name: alias_name,
                 variant: SymbolVariant::Module(SymbolModule{
                     root_id: target_root_id,
                     introduced_at: import_id,
                 }),
             });
         } else if Some(TokenKind::ColonColon) == next {
             iter.consume();
             // Helper function to consume symbols, their alias, and the
             // definition the symbol is pointing to.
             fn consume_symbol_and_maybe_alias<'a>(
                 source: &'a InputSource, iter: &mut TokenIter, ctx: &mut PassCtx,
                 module_name: &StringRef<'static>, module_root_id: RootId,
             ) -> Result<(AliasedSymbol, SymbolDefinition), ParseError> {
                 // Consume symbol name and make sure it points to an existing definition
                 let symbol_identifier = consume_ident_interned(source, iter, ctx)?;
                 let target = ctx.symbols.get_symbol_by_name_defined_in_scope(
                     SymbolScope::Module(module_root_id), symbol
                 );
                 if target.is_none() {
                     return Err(ParseError::new_error_at_span(
                         source, symbol_span,
                         format!(
                             "could not find symbol '{}' within module '{}'",
                             symbol_identifier.value.as_str(), module_name.as_str()
+                        )
                     ));
+                }
                 let target = target.unwrap();
                 debug_assert_ne!(target.class(), SymbolClass::Module);
                 let target_definition = target.variant.as_definition();
                 // Consume alias text if specified
                 let alias_identifier = if peek_ident(source, iter) == b"as" {
                     // Consume alias
                     iter.consume();
                     Some(consume_ident_interned(source, iter, ctx)?)
                 } else {
                     None
                 };
                 Ok((
                     AliasedSymbol{
                         name: symbol_identifier,
                         alias: alias_identifier,
                         definition_id: target_definition.definition_id,
                     },
                     target_definition.clone()
                 ))
+            }
             let next = iter.next();
             if Some(TokenKind::Ident) = next {
                 // Importing a single symbol
                 iter.consume();
                 let (imported_symbol, symbol_definition) = consume_symbol_and_maybe_alias(
                     &module.source, &mut iter, ctx, &module_identifier.value, target_root_id
                 )?;
                 let alias_identifier = imported_symbol.alias.unwrap_or_else(|| { imported_symbol.name.clone() });
                 import_id = ctx.heap.alloc_import(|this| Import::Symbols(ImportSymbols{
                     this,
                     span: InputSpan::from_positions(import_span.begin, alias_identifier.span.end),
                     module: module_identifier,
                     module_id: target_root_id,
                     symbols: vec![imported_symbol],
                 }));
                 if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(
                     SymbolScope::Module(module.root_id),
                     Symbol{
                         name: alias_identifier.value,
                         variant: SymbolVariant::Definition(symbol_definition.into_imported(import_id))
+                    }
                 ) {
                     return Err(construct_symbol_conflict_error(
                         modules, module_idx, ctx, &new_symbol, old_symbol
                     ));
+                }
             } else if Some(TokenKind::OpenCurly) = next {
                 // Importing multiple symbols
                 consume_comma_separated(
                     TokenKind::OpenCurly, TokenKind::CloseCurly, source, &mut iter,
                     |source, iter| consume_symbol_and_maybe_alias(
                         source, iter, ctx, &module_identifier.value, target_root_id
                     ),
                     &mut self.found_symbols, "a symbol", "a list of symbols to import"
                 )?;
                 let end_of_list = iter.last_valid_pos();
                 // Preallocate import
                 import_id = ctx.heap.alloc_import(|this| Import::Symbols(ImportSymbols {
                     this,
                     span: InputSpan::from_positions(import_span.begin, end_of_list),
                     module: module_identifier,
                     module_id: target_root_id,
                     symbols: Vec::with_capacity(self.found_symbols.len()),
                 }));
                 // Fill import symbols while inserting symbols in the
                 // appropriate scope in the symbol table.
                 let import = ctx.heap[import_id].as_symbols_mut();
                 for (imported_symbol, symbol_definition) in self.found_symbols.drain(..) {
                     let import_name = imported_symbol.alias.map_or_else(
                         || imported_symbol.name.value.clone(),
                         |v| v.value.clone()
                     );
                     import.symbols.push(imported_symbol);
                     if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(
                         SymbolScope::Module(module.root_id), Symbol{
                             name: import_name,
                             variant: SymbolVariant::Definition(symbol_definition.into_imported(import_id))
+                        }
                     ) {
                         return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, old_symbol));
+                    }
+                }
             } else if Some(TokenKind::Star) = next {
                 // Import all symbols from the module
                 let star_span = iter.next_span();
                 iter.consume();
                 self.scoped_symbols.clear();
                 let _found = ctx.symbols.get_all_symbols_defined_in_scope(
                     SymbolScope::Module(target_root_id),
                     &mut self.scoped_symbols
                 );
                 debug_assert!(_found); // even modules without symbols should have a scope
                 // Preallocate import
                 import_id = ctx.heap.alloc_import(|this| Import::Symbols(ImportSymbols{
                     this,
                     span: InputSpan::from_positions(import_span.begin, star_span.end),
                     module: module_identifier,
                     module_id: target_root_id,
                     symbols: Vec::with_capacity(self.scoped_symbols.len())
                 }));
                 // Fill import AST node and symbol table
                 let import = ctx.heap[import_id].as_symbols_mut();
                 for symbol in self.scoped_symbols.drain(..) {
                     let symbol_name = symbol.name;
                     match symbol.variant {
                         SymbolVariant::Definition(symbol_definition) => {
                             import.symbols.push(AliasedSymbol{
                                 name: Identifier{ span: star_span, value: symbol.name.clone() },
                                 alias: None,
                                 definition_id: symbol_definition.definition_id,
                             });
                             if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(
                                 SymbolScope::Module(module.root_id),
                                 Symbol{
                                     name: symbol_name,
                                     variant: SymbolVariant::Definition(symbol_definition.into_imported(import_id))
+                                }
                             ) {
                                 return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, old_symbol));
+                            }
                         },
                         _ => unreachable!(),
+                    }
+                }
             } else {
                 return Err(ParseError::new_error_str_at_pos(
                     &module.source, iter.last_valid_pos(), "expected symbol name, '{' or '*'"
                 ));
+            }
         } else {
             // Assume implicit alias
             let module_name_str = module_identifier.value.clone();
             let last_ident_start = module_name_str.rfind('.').map_or(0, |v| v + 1);
             let alias_text = &module_name_str.as_bytes()[last_ident_start..];
             let alias = ctx.pool.intern(alias_text);
             let alias_span = InputSpan::from_positions(
                 module_name_span.begin.with_offset(last_ident_start as u32),
                 module_name_span.end
             );
             let alias_identifier = Identifier{ span: alias_span, value: alias.clone() };
             import_id = ctx.heap.alloc_import(|this| Import::Module(ImportModule{
                 this,
                 span: InputSpan::from_positions(import_span.begin, module_identifier.span.end),
                 module: module_identifier,
                 alias: alias_identifier,
                 module_id: target_root_id,
             }));
             ctx.symbols.insert_symbol(SymbolScope::Module(module.root_id), Symbol{
                 name: alias,
                 variant: SymbolVariant::Module(SymbolModule{
                     root_id: target_root_id,
                     introduced_at: import_id
                 })
             });
+        }
         // By now the `import_id` is set, just need to make sure that the import
         // properly ends with a semicolon
         consume_token(&module.source, &mut iter, TokenKind::SemiColon)?;
         self.imports.push(import_id);
         Ok(())
+    }
+}

src/protocol/parser/pass_symbols.rs

➞

Show inline comments

@@ new file 100644 @@
 use crate::protocol::ast::*;
 use super::symbol_table2::*;
 use crate::protocol::input_source2::{ParseError, InputSpan};
 use super::tokens::*;
 use super::token_parsing::*;
 use super::{Module, ModuleCompilationPhase, PassCtx};
 /// Scans the module and finds all module-level type definitions. These will be
 /// added to the symbol table such that during AST-construction we know which
 /// identifiers point to types. Will also parse all pragmas to determine module
 /// names.
 pub(crate) struct PassSymbols {
     symbols: Vec<Symbol>,
     pragmas: Vec<PragmaId>,
     imports: Vec<ImportId>,
     definitions: Vec<DefinitionId>,
     buffer: String,
     has_pragma_version: bool,
     has_pragma_module: bool,
+}
 impl PassSymbols {
     pub(crate) fn new() -> Self {
         Self{
             symbols: Vec::with_capacity(128),
             pragmas: Vec::with_capacity(8),
             imports: Vec::with_capacity(32),
             definitions: Vec::with_capacity(128),
             buffer: String::with_capacity(128),
             has_pragma_version: false,
             has_pragma_module: false,
+        }
+    }
     fn reset(&mut self) {
         self.symbols.clear();
         self.pragmas.clear();
         self.imports.clear();
         self.definitions.clear();
         self.has_pragma_version = false;
         self.has_pragma_module = false;
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         self.reset();
         let module = &mut modules[module_idx];
         let module_range = &module.tokens.ranges[0];
         debug_assert_eq!(module.phase, ModuleCompilationPhase::Tokenized);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         debug_assert!(module.root_id.is_invalid()); // not set yet,
         // Preallocate root in the heap
         let root_id = ctx.heap.alloc_protocol_description(|this| {
             Root{
                 this,
                 pragmas: Vec::new(),
                 imports: Vec::new(),
                 definitions: Vec::new(),
+            }
         });
         module.root_id = root_id;
         // Visit token ranges to detect definitions and pragmas
         let mut range_idx = module_range.first_child_idx;
         loop {
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
             // Parse if it is a definition or a pragma
             if cur_range.range_kind == TokenRangeKind::Definition {
                 self.visit_definition_range(modules, module_idx, ctx, range_idx_usize)?;
             } else if cur_range.range_kind == TokenRangeKind::Pragma {
                 self.visit_pragma_range(modules, module_idx, ctx, range_idx_usize)?;
+            }
             match cur_range.next_sibling_idx {
                 Some(idx) => { range_idx = idx; },
                 None => { break; },
+            }
+        }
         // Add the module's symbol scope and the symbols we just parsed
         let module_scope = SymbolScope::Module(root_id);
         ctx.symbols.insert_scope(None, module_scope);
         for symbol in self.symbols.drain(..) {
             if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(module_scope, symbol) {
                 return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, old_symbol))
+            }
+        }
         // Modify the preallocated root
         let root = &mut ctx.heap[root_id];
         root.pragmas.extend(self.pragmas.drain(..));
         root.definitions.extend(self.definitions.drain(..));
         module.phase = ModuleCompilationPhase::SymbolsScanned;
         Ok(())
+    }
     fn visit_pragma_range(&mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let range = &module.tokens.ranges[range_idx];
         let mut iter = module.tokens.iter_range(range);
         // Consume pragma name
         let (pragma_section, pragma_start, _) = consume_pragma(&self.source, &mut iter)?;
         // Consume pragma values
         if pragma_section == "#module" {
             // Check if name is defined twice within the same file
             if self.has_pragma_module {
                 return Err(ParseError::new_error(&module.source, pragma_start, "module name is defined twice"));
+            }
             // Consume the domain-name
             let (module_name, module_span) = consume_domain_ident(&module.source, &mut iter)?;
             if iter.next().is_some() {
                 return Err(ParseError::new_error(&module.source, iter.last_valid_pos(), "expected end of #module pragma after module name"));
+            }
             // Add to heap and symbol table
             let pragma_span = InputSpan::from_positions(pragma_start, module_span.end);
             let module_name = ctx.pool.intern(module_name);
             let pragma_id = ctx.heap.alloc_pragma(|this| Pragma::Module(PragmaModule{
                 this,
                 span: pragma_span,
                 value: Identifier{ span: module_span, value: module_name.clone() },
             }));
             self.pragmas.push(pragma_id);
             if let Err(other_module_root_id) = ctx.symbols.insert_module(module_name, module.root_id) {
                 // Naming conflict
                 let this_module = &modules[module_idx];
                 let other_module = seek_module(modules, other_module_root_id).unwrap();
                 let (other_module_pragma_id, _) = other_module.name.unwrap();
                 let other_pragma = ctx.heap[other_module_pragma_id].as_module();
                 return Err(ParseError::new_error_str_at_span(
                     &this_module.source, pragma_span, "conflict in module name"
                 ).with_info_str_at_span(
                     &other_module.source, other_pragma.span, "other module is defined here"
                 ));
+            }
             self.has_pragma_module = true;
         } else if pragma_section == "#version" {
             // Check if version is defined twice within the same file
             if self.has_pragma_version {
                 return Err(ParseError::new_error(&module.source, pragma_start, "module version is defined twice"));
+            }
             // Consume the version pragma
             let (version, version_span) = consume_integer_literal(&module.source, &mut iter, &mut self.buffer)?;
             let pragma_id = ctx.heap.alloc_pragma(|this| Pragma::Version(PragmaVersion{
                 this,
                 span: InputSpan::from_positions(pragma_start, version_span.end),
                 version,
             }));
             self.pragmas.push(pragma_id);
             self.has_pragma_version = true;
         } else {
             // Custom pragma, maybe we support this in the future, but for now
             // we don't.
             return Err(ParseError::new_error(&module.source, pragma_start, "illegal pragma name"));
+        }
         Ok(())
+    }
     fn visit_definition_range(&mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let range = &module.tokens.ranges[range_idx];
         let definition_span = InputSpan::from_positions(
             module.tokens.start_pos(range),
             module.tokens.end_pos(range)
         );
         let mut iter = module.tokens.iter_range(range);
         // First ident must be type of symbol
         let (kw_text, _) = consume_any_ident(&module.source, &mut iter).unwrap();
         // Retrieve identifier of definition
         let identifier = consume_ident_interned(&module.source, &mut iter, ctx)?;
         let ident_text = identifier.value.clone(); // because we need it later
         // Reserve space in AST for definition and add it to the symbol table
         let definition_class;
         let ast_definition_id;
         match kw_text {
             KW_STRUCT => {
                 let struct_def_id = ctx.heap.alloc_struct_definition(|this| {
                     StructDefinition::new_empty(this, definition_span, identifier)
                 });
                 definition_class = DefinitionClass::Struct;
                 ast_definition_id = struct_def_id.upcast();
             },
             KW_ENUM => {
                 let enum_def_id = ctx.heap.alloc_enum_definition(|this| {
                     EnumDefinition::new_empty(this, definition_span, identifier)
                 });
                 definition_class = DefinitionClass::Enum;
                 ast_definition_id = enum_def_id.upcast();
             },
             KW_UNION => {
                 let union_def_id = ctx.heap.alloc_union_definition(|this| {
                     UnionDefinition::new_empty(this, definition_span, identifier)
                 });
                 definition_class = DefinitionClass::Union;
                 ast_definition_id = union_def_id.upcast()
             },
             KW_FUNCTION => {
                 let func_def_id = ctx.heap.alloc_function_definition(|this| {
                     FunctionDefinition::new_empty(this, definition_span, identifier)
                 });
                 definition_class = DefinitionClass::Function;
                 ast_definition_id = func_def_id.upcast();
             },
             KW_PRIMITIVE | KW_COMPOSITE => {
                 let component_variant = if kw_text == KW_PRIMITIVE {
                     ComponentVariant::Primitive
                 } else {
                     ComponentVariant::Composite
                 };
                 let comp_def_id = ctx.heap.alloc_component_definition(|this| {
                     ComponentDefinition::new_empty(this, definition_span, component_variant, identifier)
                 });
                 definition_class = DefinitionClass::Component;
                 ast_definition_id = comp_def_id.upcast();
             },
             _ => unreachable!("encountered keyword '{}' in definition range", String::from_utf8_lossy(kw_text)),
+        }
         let symbol = Symbol{
             name: ident_text,
             variant: SymbolVariant::Definition(SymbolDefinition{
                 defined_in_module: module.root_id,
                 defined_in_scope: SymbolScope::Module(module.root_id),
                 definition_span,
                 identifier_span,
                 imported_at: None,
                 class: definition_class,
                 definition_id: ast_definition_id,
             }),
         };
         self.symbols.push(symbol);
         self.definitions.push(ast_definition_id);
         Ok(())
+    }
+}
@@ \ No newline at end of file @@

src/protocol/parser/pass_tokenizer.rs

➞

Show inline comments

 use crate::protocol::input_source2::{
     InputSource2 as InputSource,
     ParseError,
     InputPosition2 as InputPosition,
     InputSpan
 };
 pub(crate) const KW_STRUCT:    &'static [u8] = b"struct";
 pub(crate) const KW_ENUM:      &'static [u8] = b"enum";
 pub(crate) const KW_UNION:     &'static [u8] = b"union";
 pub(crate) const KW_FUNCTION:  &'static [u8] = b"function";
 pub(crate) const KW_PRIMITIVE: &'static [u8] = b"primitive";
 pub(crate) const KW_COMPOSITE: &'static [u8] = b"composite";
 pub(crate) const KW_IMPORT:    &'static [u8] = b"import";
 #[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)]
 pub(crate) enum TokenKind {
     // Variable-character tokens, followed by a SpanEnd token
     Ident,          // regular identifier
     Pragma,         // identifier with prefixed `#`, range includes `#`
     Integer,        // integer literal
     String,         // string literal, range includes `"`
     Character,      // character literal, range includes `'`
     LineComment,    // line comment, range includes leading `//`, but not newline
     BlockComment,   // block comment, range includes leading `/*` and trailing `*/`
     // Punctuation (single character)
     Exclamation,    // !
     Question,       // ?
     Pound,          // #
     OpenAngle,      // <
     OpenCurly,      // {
     OpenParen,      // (
     OpenSquare,     // [
     CloseAngle,     // >
     CloseCurly,     // }
     CloseParen,     // )
     CloseSquare,    // ]
     Colon,          // :
     Comma,          // ,
     Dot,            // .
     SemiColon,      // ;
     Quote,          // '
     DoubleQuote,    // "
     // Operator-like (single character)
     At,             // @
     Plus,           // +
     Minus,          // -
     Star,           // *
     Slash,          // /
     Percent,        // %
     Caret,          // ^
     And,            // &
     Or,             // |
     Tilde,          // ~
     Equal,          // =
     // Punctuation (two characters)
     ColonColon,     // ::
     DotDot,         // ..
     ArrowRight,     // ->
     // Operator-like (two characters)
     PlusPlus,       // ++
     PlusEquals,     // +=
     MinusMinus,     // --
     MinusEquals,    // -=
     StarEquals,     // *=
     SlashEquals,    // /=
     PercentEquals,  // %=
     CaretEquals,    // ^=
     AndAnd,         // &&
     AndEquals,      // &=
     OrOr,           // ||
     OrEquals,       // |=
     EqualEqual,     // ==
     NotEqual,       // !=
     ShiftLeft,      // <<
     ShiftRight,     // >>
     // Operator-like (three characters)
     ShiftLeftEquals,// <<=
     ShiftRightEquals, // >>=
     // Special marker token to indicate end of variable-character tokens
     SpanEnd,
+}
 impl TokenKind {
     /// Returns true if the next expected token is the special `TokenKind::SpanEnd` token. This is
     /// the case for tokens of variable length (e.g. an identifier).
     fn has_span_end(&self) -> bool {
         return *self <= TokenKind::BlockComment
+    }
     /// Returns the number of characters associated with the token. May only be called on tokens
     /// that do not have a variable length.
     fn num_characters(&self) -> u32 {
         debug_assert!(!self.has_span_end() && *self != TokenKind::SpanEnd);
         if *self <= TokenKind::Equal {
         } else if *self <= TokenKind::ShiftRight {
         } else {
+        }
+    }
     /// Returns the characters that are represented by the token, may only be called on tokens that
     /// do not have a variable length.
     pub fn token_chars(&self) -> &'static str {
         debug_assert!(!self.has_span_end() && *self != TokenKind::SpanEnd);
         use TokenKind as TK;
         match self {
             TK::Exclamation => "!",
             TK::Question => "?",
             TK::Pound => "#",
             TK::OpenAngle => "<",
             TK::OpenCurly => "{",
             TK::OpenParen => "(",
             TK::OpenSquare => "[",
             TK::CloseAngle => ">",
             TK::CloseCurly => "}",
             TK::CloseParen => ")",
             TK::CloseSquare => "]",
             TK::Colon => ":",
             TK::Comma => ",",
             TK::Dot => ".",
             TK::SemiColon => ";",
             TK::Quote => "'",
             TK::DoubleQuote => "\"",
             TK::At => "@",
             TK::Plus => "+",
             TK::Minus => "-",
             TK::Star => "*",
             TK::Slash => "/",
             TK::Percent => "%",
             TK::Caret => "^",
             TK::And => "&",
             TK::Or => "|",
             TK::Tilde => "~",
             TK::Equal => "=",
             TK::ColonColon => "::",
             TK::DotDot => "..",
             TK::ArrowRight => "->",
             TK::PlusPlus => "++",
             TK::PlusEquals => "+=",
             TK::MinusMinus => "--",
             TK::MinusEquals => "-=",
             TK::StarEquals => "*=",
             TK::SlashEquals => "/=",
             TK::PercentEquals => "%=",
             TK::CaretEquals => "^=",
             TK::AndAnd => "&&",
             TK::AndEquals => "&=",
             TK::OrOr => "||",
             TK::OrEquals => "|=",
             TK::EqualEqual => "==",
             TK::NotEqual => "!=",
             TK::ShiftLeft => "<<",
             TK::ShiftRight => ">>",
             TK::ShiftLeftEquals => "<<=",
             TK::ShiftRightEquals => ">>=",
             // Lets keep these in explicitly for now, in case we want to add more symbols
             TK::Ident | TK::Pragma | TK::Integer | TK::String | TK::Character |
             TK::LineComment | TK::BlockComment | TK::SpanEnd => unreachable!(),
+        }
+    }
+}
 pub(crate) struct Token {
     pub kind: TokenKind,
     pub pos: InputPosition,
+}
 impl Token {
     fn new(kind: TokenKind, pos: InputPosition) -> Self {
         Self{ kind, pos }
+    }
+}
 #[derive(Debug, PartialEq, Eq)]
 pub(crate) enum TokenRangeKind {
     Module,
     Pragma,
     Import,
     Definition,
     Code,
+}
 /// TODO: Add first_child and next_sibling indices for slightly faster traversal
 #[derive(Debug)]
 pub(crate) struct TokenRange {
     // Index of parent in `TokenBuffer.ranges`, does not have a parent if the
     // range kind is Module, in that case the parent index points to itself.
     pub parent_idx: usize,
     pub range_kind: TokenRangeKind,
     pub curly_depth: u32,
     // InputPosition offset is limited to u32, so token ranges can be as well.
     pub start: u32,             // first token (inclusive index)
     pub end: u32,               // last token (exclusive index)
     // Subranges and where they can be found
     pub subranges: u32,         // Number of subranges
     pub first_child_idx: u32,   // First subrange (or points to self if no subranges)
     pub last_child_idx: u32,    // Last subrange (or points to self if no subranges)
     pub next_sibling_idx: Option<u32>,
+}
 pub(crate) struct TokenBuffer {
     pub tokens: Vec<Token>,
     pub ranges: Vec<TokenRange>,
+}
 impl TokenBuffer {
     pub(crate) fn new() -> Self {
         Self{ tokens: Vec::new(), ranges: Vec::new() }
+    }
     pub(crate) fn iter_range<'a>(&'a self, range: &TokenRange) -> TokenIter<'a> {
         TokenIter::new(self, range.start as usize, range.end as usize)
+    }
     pub(crate) fn start_pos(&self, range: &TokenRange) -> InputPosition {
         self.tokens[range.start].pos
+    }
     pub(crate) fn end_pos(&self, range: &TokenRange) -> InputPosition {
         let last_token = &self.tokens[range.end - 1];
         if last_token.kind == TokenKind::SpanEnd {
             return last_token.pos
         } else {
             debug_assert!(!last_token.kind.has_span_end());
             return last_token.pos.with_offset(last_token.kind.num_characters());
+        }
+    }
+}
 pub(crate) struct TokenIter<'a> {
     tokens: &'a Vec<Token>,
     cur: usize,
     end: usize,
+}
 impl<'a> TokenIter<'a> {
     fn new(buffer: &'a TokenBuffer, start: usize, end: usize) -> Self {
         Self{ tokens: &buffer.tokens, cur: start, end }
+    }
     /// Returns the next token (may include comments), or `None` if at the end
     /// of the range.
     pub(crate) fn next_including_comments(&self) -> Option<TokenKind> {
         if self.cur >= self.end {
             return None;
+        }
         let token = &self.tokens[self.cur];
         Some(token.kind)
+    }
     /// Returns the next token (but skips over comments), or `None` if at the
     /// end of the range
     pub(crate) fn next(&mut self) -> Option<TokenKind> {
         while let Some(token_kind) = self.next() {
             if token_kind != TokenKind::LineComment && token_kind != TokenKind::BlockComment {
                 return Some(token_kind);
+            }
             self.consume();
+        }
         return None
+    }
     /// Returns the start position belonging to the token returned by `next`. If
     /// there is not a next token, then we return the end position of the
     /// previous token.
     pub(crate) fn last_valid_pos(&self) -> InputPosition {
         if self.cur < self.end {
             // Return token position
             return self.tokens[self.cur].pos
+        }
         // Return previous token end
         let token = &self.tokens[self.cur - 1];
         return if token.kind == TokenKind::SpanEnd {
             token.pos
         } else {
             token.pos.with_offset(token.kind.num_characters());
         };
+    }
     /// Returns the token range belonging to the token returned by `next`. This
     /// assumes that we're not at the end of the range we're iterating over.
     pub(crate) fn next_range(&self) -> (InputPosition, InputPosition) {
         debug_assert!(self.cur < self.end);
         let token = &self.tokens[self.cur];
         if token.kind.has_span_end() {
             let span_end = &self.tokens[self.cur + 1];
             debug_assert_eq!(span_end.kind, TokenKind::SpanEnd);
             (token.pos, span_end.pos)
         } else {
             let offset = token.kind.num_characters();
             (token.pos, token.pos.with_offset(offset))
+        }
+    }
     pub(crate) fn consume(&mut self) {
         if let Some(kind) = self.next() {
             if kind.has_span_end() {
                 self.cur += 2;
             } else {
                 self.cur += 1;
+            }
+        }
+    }
+}
 use crate::protocol::parser::tokens::*;
 /// Tokenizer is a reusable parser to tokenize multiple source files using the
 /// same allocated buffers. In a well-formed program, we produce a consistent
 /// tree of token ranges such that we may identify tokens that represent a
 /// defintion or an import before producing the entire AST.
 ///
 /// If the program is not well-formed then the tree may be inconsistent, but we
 /// will detect this once we transform the tokens into the AST. To ensure a
 /// consistent AST-producing phase we will require the import to have balanced
 /// curly braces
-pub(crate) struct Tokenizer {
+pub(crate) struct PassTokenizer {
     // Stack of input positions of opening curly braces, used to detect
     // unmatched opening braces, unmatched closing braces are detected
     // immediately.
     curly_stack: Vec<InputPosition>,
     // Points to an element in the `TokenBuffer.ranges` variable.
     stack_idx: usize,
+}
-impl Tokenizer {
+impl PassTokenizer {
     pub(crate) fn new() -> Self {
         Self{ curly_stack: Vec::with_capacity(32), stack_idx: 0 }
+    }
     pub(crate) fn tokenize(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         // Assert source and buffer are at start
         debug_assert_eq!(source.pos().offset, 0);
         debug_assert!(target.tokens.is_empty());
         debug_assert!(target.ranges.is_empty());
         // Set up for tokenization by pushing the first range onto the stack.
         // This range may get transformed into the appropriate range kind later,
         // see `push_range` and `pop_range`.
         self.curly_depth = 0;
         self.stack_idx = 0;
         target.ranges.push(TokenRange{
             parent_idx: 0,
             range_kind: TokenRangeKind::Module,
             curly_depth: 0,
             start: 0,
             end: 0,
-            subranges: 0,
+            num_child_ranges: 0,
             first_child_idx: 0,
             last_child_idx: 0,
             next_sibling_idx: None,
         });
         // Main tokenization loop
         while let Some(c) = source.next() {
             let token_index = target.tokens.len() as u32;
             if is_char_literal_start(c) {
                 self.consume_char_literal(source, target)?;
             } else if is_string_literal_start(c) {
                 self.consume_string_literal(source, target)?;
             } else if is_identifier_start(c) {
                 let ident = self.consume_identifier(source, target)?;
                 if demarks_definition(ident) {
                     self.push_range(target, TokenRangeKind::Definition, token_index);
                 } else if demarks_import(ident) {
                     self.push_range(target, TokenRangeKind::Import, token_index);
+                }
             } else if is_integer_literal_start(c) {
                 self.consume_number(source, target)?;
             } else if is_pragma_start_or_pound(c) {
                 let was_pragma = self.consume_pragma_or_pound(c, source, target)?;
                 if was_pragma {
                     self.push_range(target, TokenRangeKind::Pragma, token_index);
+                }
             } else if self.is_line_comment_start(c, source) {
                 self.consume_line_comment(source, target)?;
             } else if self.is_block_comment_start(c, source) {
                 self.consume_block_comment(source, target)?;
             } else if is_whitespace(c) {
                 let contained_newline = self.consume_whitespace(source);
                 if contained_newline {
                     let range = &target.ranges[self.stack_idx];
                     if range.range_kind == TokenRangeKind::Pragma {
                         self.pop_range(target, target.tokens.len() as u32);
+                    }
+                }
             } else {
                 let was_punctuation = self.maybe_parse_punctuation(c, source, target)?;
                 if let Some((token, token_pos)) = was_punctuation {
                     if token == TokenKind::OpenCurly {
                         self.curly_stack.push(token_pos);
                     } else if token == TokenKind::CloseCurly {
                         // Check if this marks the end of a range we're
                         // currently processing
                         if self.curly_stack.is_empty() {
                             return Err(ParseError::new_error(
                                 source, token_pos, "unmatched closing curly brace '}'"
                             ));
+                        }
                         self.curly_stack.pop();
                         let range = &target.ranges[self.stack_idx];
                         if range.range_kind == TokenRangeKind::Definition && range.curly_depth == self.curly_depth {
                             self.pop_range(target, target.tokens.len() as u32);
+                        }
                         // Exit early if we have more closing curly braces than
                         // opening curly braces
                     } else if token == TokenKind::SemiColon {
                         // Check if this marks the end of an import
                         let range = &target.ranges[self.stack_idx];
                         if range.range_kind == TokenRangeKind::Import {
                             self.pop_range(target, target.tokens.len() as u32);
+                        }
+                    }
                 } else {
                     return Err(ParseError::new_error(source, source.pos(), "unexpected character"));
+                }
+            }
+        }
         // End of file, check if our state is correct
         if let Some(error) = source.had_error.take() {
             return Err(error);
+        }
         if !self.curly_stack.is_empty() {
             // Let's not add a lot of heuristics and just tell the programmer
             // that something is wrong
             let last_unmatched_open = self.curly_stack.pop().unwrap();
             return Err(ParseError::new_error(
                 source, last_unmatched_open, "unmatched opening curly brace '{'"
             ));
+        }
         // TODO: @remove once I'm sure the algorithm works. For now it is better
         //  if the debugging is a little more expedient
         if cfg!(debug_assertions) {
             // For each range make sure its children make sense
             for parent_idx in 0..target.ranges.len() {
                 let cur_range = &target.ranges[parent_idx];
-                if cur_range.subranges == 0 {
+                if cur_range.num_child_ranges == 0 {
                     assert_eq!(cur_range.first_child_idx, parent_idx as u32);
                     assert_eq!(cur_range.last_child_idx, parent_idx as u32);
                 } else {
                     assert_ne!(cur_range.first_child_idx, parent_idx as u32);
                     assert_ne!(cur_range.last_child_idx, parent_idx as u32);
                     let mut child_counter = 0u32;
                     let mut last_child_idx = cur_range.first_child_idx;
                     let mut child_idx = Some(cur_range.first_child_idx);
                     while let Some(cur_child_idx) = child_idx {
                         let child_range = &target.ranges[cur_child_idx as usize];
                         assert_eq!(child_range.parent_idx, parent_idx);
                         child_idx = child_range.next_sibling_idx;
                         child_counter += 1;
+                    }
                     assert_eq!(cur_range.last_child_idx, last_child_idx);
-                    assert_eq!(cur_range.subranges, child_counter);
+                    assert_eq!(cur_range.num_child_ranges, child_counter);
+                }
+            }
+        }
         Ok(())
+    }
     fn is_line_comment_start(&self, first_char: u8, source: &InputSource) -> bool {
         return first_char == b'/' && Some(b'/') == source.lookahead(1);
+    }
     fn is_block_comment_start(&self, first_char: u8, source: &InputSource) -> bool {
         return first_char == b'/' && Some(b'*') == source.lookahead(1);
+    }
     fn maybe_parse_punctuation(
         &mut self, first_char: u8, source: &mut InputSource, target: &mut TokenBuffer
     ) -> Result<Option<(TokenKind, InputPosition)>, ParseError> {
         debug_assert!(first_char != b'#', "'#' needs special handling");
         debug_assert!(first_char != b'\'', "'\'' needs special handling");
         debug_assert!(first_char != b'"', "'\"' needs special handling");
         let pos = source.pos();
         let token_kind;
         if first_char == b'!' {
             source.consume();
             if Some(b'=') == source.next() {
                 source.consume();
                 token_kind = TokenKind::NotEqual;
             } else {
                 token_kind = TokenKind::Exclamation;
+            }
         } else if first_char == b'%' {
             source.consume();
             if Some(b'=') == source.next() {
                 source.consume();
                 token_kind = TokenKind::PercentEquals;
             } else {
                 token_kind = TokenKind::Percent;
+            }
         } else if first_char == b'&' {
             source.consume();
             let next = source.next();
             if Some(b'&') == next {
                 source.consume();
                 token_kind = TokenKind::AndAnd;
             } else if Some(b'=') == next {
                 source.consume();
                 token_kind = TokenKind::AndEquals;
             } else {
                 token_kind = TokenKind::And;
+            }
         } else if first_char == b'(' {
             source.consume();
             token_kind = TokenKind::OpenParen;
         } else if first_char == b')' {
             source.consume();
             token_kind = TokenKind::CloseParen;
         } else if first_char == b'*' {
             source.consume();
             if let Some(b'=') = source.next() {
                 source.consume();
                 token_kind = TokenKind::StarEquals;
             } else {
                 token_kind = TokenKind::Star;
+            }
         } else if first_char == b'+' {
             source.consume();
             let next = source.next();
             if Some(b'+') == next {
                 source.consume();
                 token_kind = TokenKind::PlusPlus;
             } else if Some(b'=') == next {
                 source.consume();
                 token_kind = TokenKind::PlusEquals;
             } else {
                 token_kind = TokenKind::Plus;
+            }
         } else if first_char == b',' {
             source.consume();
             token_kind = TokenKind::Comma;
         } else if first_char == b'-' {
             source.consume();
             let next = source.next();
             if Some(b'-') == next {
                 source.consume();
                 token_kind = TokenKind::MinusMinus;
             } else if Some(b'>') == next {
                 source.consume();
                 token_kind = TokenKind::ArrowRight;
             } else if Some(b'=') == next {
                 source.consume();
                 token_kind = TokenKind::MinusEquals;
             } else {
                 token_kind = TokenKind::Minus;
+            }
         } else if first_char == b'.' {
             source.consume();
             if let Some(b'.') = source.next() {
                 source.consume();
                 token_kind = TokenKind::DotDot;
             } else {
                 token_kind = TokenKind::Dot
+            }
         } else if first_char == b'/' {
             source.consume();
             debug_assert_ne!(Some(b'/'), source.next());
             debug_assert_ne!(Some(b'*'), source.next());
             if let Some(b'=') = source.next() {
                 source.consume();
                 token_kind = TokenKind::SlashEquals;
             } else {
                 token_kind = TokenKind::Slash;
+            }
         } else if first_char == b':' {
             source.consume();
             if let Some(b':') = source.next() {
                 source.consume();
                 token_kind = TokenKind::ColonColon;
             } else {
                 token_kind = TokenKind::Colon;
+            }
         } else if first_char == b';' {
             source.consume();
             token_kind = TokenKind::SemiColon;
         } else if first_char == b'<' {
             source.consume();
             if let Some(b'<') = source.next() {
                 source.consume();
                 if let Some(b'=') = source.next() {
                     source.consume();
                     token_kind = TokenKind::ShiftLeftEquals;
                 } else {
                     token_kind = TokenKind::ShiftLeft;
+                }
             } else {
                 token_kind = TokenKind::OpenAngle;
+            }
         } else if first_char == b'=' {
             source.consume();
             if let Some(b'=') = source.next() {
                 source.consume();
                 token_kind = TokenKind::EqualEqual;
             } else {
                 token_kind = TokenKind::Equal;
+            }
         } else if first_char == b'>' {
             source.consume();
             if let Some(b'>') = source.next() {
                 source.consume();
                 if let Some(b'=') = source.next() {
                     source.consume();
                     token_kind = TokenKind::ShiftRightEquals;
                 } else {
                     token_kind = TokenKind::ShiftRight;
+                }
             } else {
                 token_kind = TokenKind::CloseAngle;
+            }
         } else if first_char == b'?' {
             source.consume();
             token_kind = TokenKind::Question;
         } else if first_char == b'@' {
             source.consume();
             token_kind = TokenKind::At;
         } else if first_char == b'[' {
             source.consume();
             token_kind = TokenKind::OpenSquare;
         } else if first_char == b']' {
             source.consume();
             token_kind = TokenKind::CloseSquare;
         } else if first_char == b'^' {
             source.consume();
             if let Some(b'=') = source.next() {
                 source.consume();
                 token_kind = TokenKind::CaretEquals;
             } else {
                 token_kind = TokenKind::Caret;
+            }
         } else if first_char == b'{' {
             source.consume();
             token_kind = TokenKind::OpenCurly;
         } else if first_char == b'|' {
             source.consume();
             let next = source.next();
             if Some(b'|') == next {
                 source.consume();
                 token_kind = TokenKind::OrOr;
             } else if Some(b'=') == next {
                 source.consume();
                 token_kind = TokenKind::OrEquals;
             } else {
@@ @@ -676,456 +374,456 @@ impl Tokenizer { @@
         // Consume the leading quote
         debug_assert!(source.next().unwrap() == b'\'');
         source.consume();
         let mut prev_char = b'\'';
         while let Some(c) = source.next() {
             if !c.is_ascii() {
                 return Err(ParseError::new_error(source, source.pos(), "non-ASCII character in char literal"));
+            }
             source.consume();
             // Make sure ending quote was not escaped
             if c == b'\'' && prev_char != b'\\' {
                 prev_char = c;
                 break;
+            }
             prev_char = c;
+        }
         if prev_char != b'\'' {
             // Unterminated character literal, reached end of file.
             return Err(ParseError::new_error(source, begin_pos, "encountered unterminated character literal"));
+        }
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::Character, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     fn consume_string_literal(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         // Consume the leading double quotes
         debug_assert!(source.next().unwrap() == b'"');
         source.consume();
         let mut prev_char = b'"';
         while let Some(c) = source.next() {
             if !c.is_ascii() {
                 return Err(ParseError::new_error(source, source.pos(), "non-ASCII character in string literal"));
+            }
             source.consume();
             if c == b'"' && prev_char != b'\\' {
                 prev_char = c;
                 break;
+            }
             prev_char = c;
+        }
         if prev_char != b'"' {
             // Unterminated string literal
             return Err(ParseError::new_error(source, begin_pos, "encountered unterminated string literal"));
+        }
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::String, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     fn consume_pragma_or_pound(&mut self, first_char: u8, source: &mut InputSource, target: &mut TokenBuffer) -> Result<bool, ParseError> {
         let start_pos = source.pos();
         debug_assert_eq!(first_char, b'#');
         source.consume();
         let next = source.next();
         if next.is_none() || !is_identifier_start(next.unwrap()) {
             // Just a pound sign
             target.tokens.push(Token::new(TokenKind::Pound, start_pos));
             Ok(false)
         } else {
             // Pound sign followed by identifier
             source.consume();
             while let Some(c) = source.next() {
                 if !is_identifier_remaining(c) {
                     break;
+                }
                 source.consume();
+            }
             self.check_ascii(source)?;
             let end_pos = source.pos();
             target.tokens.push(Token::new(TokenKind::Pragma, start_pos));
             target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
             Ok(true)
+        }
+    }
     fn consume_line_comment(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         // Consume the leading "//"
         debug_assert!(source.next().unwrap() == b'/' && source.lookahead(1).unwrap() == b'/');
         source.consume();
         source.consume();
         let mut prev_char = b'/';
         let mut cur_char = b'/';
         while let Some(c) = source.next() {
             prev_char = cur_char;
             cur_char = c;
             if c == b'\n' {
                 // End of line, note that the newline is not consumed
                 break;
+            }
             source.consume();
+        }
         let mut end_pos = source.pos();
         debug_assert_eq!(begin_pos.line, end_pos.line);
         // Modify offset to not include the newline characters
         if cur_char == b'\n' {
             if prev_char == b'\r' {
                 end_pos.offset -= 2;
             } else {
                 end_pos.offset -= 1;
+            }
             // Consume final newline
             source.consume();
         } else {
             // End of comment was due to EOF
             debug_assert!(source.next().is_none())
+        }
         target.tokens.push(Token::new(TokenKind::LineComment, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     fn consume_block_comment(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         // Consume the leading "/*"
         debug_assert!(source.next().unwrap() == b'/' && source.lookahead(1).unwrap() == b'*');
         source.consume();
         source.consume();
         // Explicitly do not put prev_char at "*", because then "/*/" would
         // represent a valid and closed block comment
         let mut prev_char = b' ';
         let mut is_closed = false;
         while let Some(c) = source.next() {
             source.consume();
             if prev_char == b'*' && c == b'/' {
                 // End of block comment
                 is_closed = true;
                 break;
+            }
             prev_char = c;
+        }
         if !is_closed {
             return Err(ParseError::new_error(source, source.pos(), "encountered unterminated block comment"));
+        }
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::BlockComment, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     fn consume_identifier<'a>(&mut self, source: &'a mut InputSource, target: &mut TokenBuffer) -> Result<&'a [u8], ParseError> {
         let begin_pos = source.pos();
         debug_assert!(is_identifier_start(source.next().unwrap()));
         source.consume();
         // Keep reading until no more identifier
         while let Some(c) = source.next() {
             if !is_identifier_remaining(c) {
                 break;
+            }
             source.consume();
+        }
         self.check_ascii(source)?;
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::Ident, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
-        Ok(source.section(begin_pos, end_pos))
+        Ok(source.section_at_pos(begin_pos, end_pos))
+    }
     fn consume_number(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         debug_assert!(is_integer_literal_start(source.next().unwrap()));
         source.consume();
         // Keep reading until it doesn't look like a number anymore
         while let Some(c) = source.next() {
             if !maybe_number_remaining(c) {
                 break;
+            }
             source.consume();
+        }
         self.check_ascii(source)?;
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::Integer, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     // Consumes whitespace and returns whether or not the whitespace contained
     // a newline.
     fn consume_whitespace(&self, source: &mut InputSource) -> bool {
         debug_assert!(is_whitespace(source.next().unwrap()));
         let mut has_newline = false;
         while let Some(c) = source.next() {
             if !is_whitespace(c) {
                 break;
+            }
             if c == b'\n' {
                 has_newline = true;
+            }
             source.consume();
+        }
         has_newline
+    }
     /// Pushes a new token range onto the stack in the buffers.
     fn push_range(&mut self, target: &mut TokenBuffer, range_kind: TokenRangeKind, first_token: u32) {
         let cur_range = &mut target.ranges[self.stack_idx];
         // If we have just popped a range and then push a new range, then the
         // first token is equal to the last token registered on the current
         // range. If not, then we had some intermediate tokens that did not
         // belong to a particular kind of token range: hence we insert an
         // intermediate "code" range.
         if cur_range.end != first_token {
             let code_start = cur_range.end;
             let code_range_idx = target.ranges.len() as u32;
             if cur_range.first_child_idx == self.stack_idx as u32 {
                 // The parent of the new "code" range we're going to push does
                 // not have any registered children yet.
                 cur_range.first_child_idx = code_range_idx;
+            }
             cur_range.last_child_idx = code_range_idx + 1;
             cur_range.end = first_token;
-            cur_range.subranges += 1;
+            cur_range.num_child_ranges += 1;
             target.ranges.push(TokenRange{
                 parent_idx: self.stack_idx,
                 range_kind: TokenRangeKind::Code,
                 curly_depth: self.curly_depth,
                 start: code_start,
                 end: first_token,
-                subranges: 0,
+                num_child_ranges: 0,
                 first_child_idx: code_range_idx,
                 last_child_idx: code_range_idx,
                 next_sibling_idx: Some(code_range_idx + 1), // we're going to push this thing next
             });
         } else {
             // We're going to push the range in the code below, but while we
             // have the `cur_range` borrowed mutably, we fix up its children
             // indices.
             let new_range_idx = target.ranges.len() as u32;
             if cur_range.first_child_idx == self.stack_idx as u32 {
                 cur_range.first_child_idx = new_range_idx;
+            }
             cur_range.last_child_idx = new_range_idx;
+        }
         // Insert a new range
         let parent_idx = self.stack_idx;
         self.stack_idx = target.ranges.len();
         let new_range_idx = self.stack_idx as u32;
         target.ranges.push(TokenRange{
             parent_idx,
             range_kind,
             curly_depth: self.curly_depth,
             start: first_token,
             end: first_token,
-            subranges: 0,
+            num_child_ranges: 0,
             first_child_idx: new_range_idx,
             last_child_idx: new_range_idx,
             next_sibling_idx: None,
         });
+    }
     fn pop_range(&mut self, target: &mut TokenBuffer, end_index: u32) {
         let last = &mut target.ranges[self.stack_idx];
         debug_assert!(self.stack_idx != last.parent_idx, "attempting to pop top-level range");
         // Fix up the current range before going back to parent
         last.end = end_index;
         debug_assert_ne!(last.start, end_index);
         // Go back to parent
         self.stack_idx = last.parent_idx;
         let parent = &mut target.ranges[self.stack_idx];
         parent.end = end_index;
-        parent.subranges += 1;
+        parent.num_child_ranges += 1;
+    }
     fn check_ascii(&self, source: &InputSource) -> Result<(), ParseError> {
         match source.next() {
             Some(c) if !c.is_ascii() => {
                 Err(ParseError::new_error(source, source.pos(), "encountered a non-ASCII character"))
             },
             _else => {
                 Ok(())
             },
+        }
+    }
+}
 // Helpers for characters
 fn demarks_definition(ident: &[u8]) -> bool {
     return
         ident == KW_STRUCT ||
             ident == KW_ENUM ||
             ident == KW_UNION ||
             ident == KW_FUNCTION ||
             ident == KW_PRIMITIVE ||
             ident == KW_COMPOSITE
+}
 fn demarks_import(ident: &[u8]) -> bool {
     return ident == KW_IMPORT;
+}
 fn is_whitespace(c: u8) -> bool {
     c.is_ascii_whitespace()
+}
 fn is_char_literal_start(c: u8) -> bool {
     return c == b'\'';
+}
 fn is_string_literal_start(c: u8) -> bool {
     return c == b'"';
+}
 fn is_pragma_start_or_pound(c: u8) -> bool {
     return c == b'#';
+}
 fn is_identifier_start(c: u8) -> bool {
     return
         (c >= b'a' && c <= b'z') ||
             (c >= b'A' && c <= b'Z') ||
             c == b'_'
+}
 fn is_identifier_remaining(c: u8) -> bool {
     return
         (c >= b'0' && c <= b'9') ||
             (c >= b'a' && c <= b'z') ||
             (c >= b'A' && c <= b'Z') ||
             c == b'_'
+}
 fn is_integer_literal_start(c: u8) -> bool {
     return c >= b'0' && c <= b'9';
+}
 fn maybe_number_remaining(c: u8) -> bool {
     return
         (c == b'b' || c == b'B' || c == b'o' || c == b'O' || c == b'x' || c == b'X') ||
             (c >= b'0' && c <= b'9') ||
             c == b'_';
+}
 #[cfg(test)]
 mod tests {
     use super::*;
     // TODO: Remove at some point
     #[test]
     fn test_tokenizer() {
         let mut source = InputSource::new_test("
         #version 500
         # hello 2
         import std.reo::*;
         struct Thing {
             int a: 5,
+        }
         enum Hello {
             A,
+            B
+        }
         // Hello hello, is it me you are looking for?
         // I can seee it in your eeeyes
         func something(int a, int b, int c) -> byte {
             int a = 5;
             struct Inner {
                 int a
+            }
             struct City {
                 int b
+            }
             /* Waza
             How are you doing
             Things in here yo
             /* */ */
             a = a + 5 * 2;
             struct Pressure {
                 int d
+            }
+        }
         ");
-        let mut t = Tokenizer::new();
+        let mut t = PassTokenizer::new();
         let mut buffer = TokenBuffer::new();
         t.tokenize(&mut source, &mut buffer).expect("tokenize");
         println!("Ranges:\n");
         for (idx, range) in buffer.ranges.iter().enumerate() {
             println!("[{}] {:?}", idx, range)
+        }
         println!("Tokens:\n");
         let mut iter = buffer.tokens.iter().enumerate();
         while let Some((idx, token)) = iter.next() {
             match token.kind {
                 TokenKind::Ident | TokenKind::Pragma | TokenKind::Integer |
                 TokenKind::String | TokenKind::Character | TokenKind::LineComment |
                 TokenKind::BlockComment => {
                     let (_, end) = iter.next().unwrap();
                     println!("[{}] {:?} ......", idx, token.kind);
                     assert_eq!(end.kind, TokenKind::SpanEnd);
-                    let text = source.section(token.pos, end.pos);
+                    let text = source.section_at_pos(token.pos, end.pos);
                     println!("{}", String::from_utf8_lossy(text));
                 },
                 _ => {
                     println!("[{}] {:?}", idx, token.kind);
+                }
+            }
+        }
+    }
+}
@@ \ No newline at end of file @@

src/protocol/parser/symbol_table.rs

➞

Show inline comments

 // TODO: Maybe allow namespaced-aliased imports. It is currently not possible
 //  to express the following:
 //      import Module.Submodule as SubMod
 //      import SubMod::{Symbol}
 //  And it is especially not possible to express the following:
 //      import SubMod::{Symbol}
 //      import Module.Submodule as SubMod
 use crate::protocol::ast::*;
 use crate::protocol::inputsource::*;
 use std::collections::{HashMap, hash_map::Entry};
 use crate::protocol::parser::LexedModule;
 #[derive(PartialEq, Eq, Hash)]
 struct SymbolKey {
     module_id: RootId,
     symbol_name: Vec<u8>,
+}
 impl SymbolKey {
     fn from_identifier(module_id: RootId, symbol: &Identifier) -> Self {
         Self{ module_id, symbol_name: symbol.value.clone() }
+    }
     fn from_namespaced_identifier(module_id: RootId, symbol: &NamespacedIdentifier) -> Self {
         Self{ module_id, symbol_name: symbol.strip_poly_args() }
+    }
+}
 pub(crate) enum Symbol {
     Namespace(RootId),
     Definition((RootId, DefinitionId)),
+}
 pub(crate) struct SymbolValue {
     // Position is the place where the symbol is introduced to a module (this
     // position always corresponds to the module whose RootId is stored in the
     // `SymbolKey` associated with this `SymbolValue`). For a definition this
     // is the position where the symbol is defined, for an import this is the
     // position of the import statement.
     pub(crate) position: InputPosition,
     pub(crate) symbol: Symbol,
+}
 impl SymbolValue {
     pub(crate) fn is_namespace(&self) -> bool {
         match &self.symbol {
             Symbol::Namespace(_) => true,
             _ => false
+        }
+    }
     pub(crate) fn as_namespace(&self) -> Option<RootId> {
         match &self.symbol {
             Symbol::Namespace(root_id) => Some(*root_id),
             _ => None,
+        }
+    }
     pub(crate) fn as_definition(&self) -> Option<(RootId, DefinitionId)> {
         match &self.symbol {
             Symbol::Definition((root_id, definition_id)) => Some((*root_id, *definition_id)),
             _ => None,
+        }
+    }
+}
 /// `SymbolTable` is responsible for two parts of the parsing process: firstly
 /// it ensures that there are no clashing symbol definitions within each file,
 /// and secondly it will resolve all symbols within a module to their
 /// appropriate definitions (in case of enums, functions, etc.) and namespaces
 /// (currently only external modules can act as namespaces). If a symbol clashes
 /// or if a symbol cannot be resolved this will be an error.
 ///
 /// Within the compilation process the symbol table is responsible for resolving
 /// namespaced identifiers (e.g. Module::Enum::EnumVariant) to the appropriate
 /// definition (i.e. not namespaces; as the language has no way to use
 /// namespaces except for using them in namespaced identifiers).
 pub(crate) struct SymbolTable {
     // Lookup from module name (not any aliases) to the root id
     module_lookup: HashMap<Vec<u8>, RootId>,
     // Lookup from within a module, to a particular imported (potentially
     // aliased) or defined symbol. Basically speaking: if the source code of a
     // module contains correctly imported/defined symbols, then this lookup
     // will always return the corresponding definition
     symbol_lookup: HashMap<SymbolKey, SymbolValue>,
+}
 impl SymbolTable {
     pub(crate) fn new() -> Self {
         Self{ module_lookup: HashMap::new(), symbol_lookup: HashMap::new() }
+    }
     pub(crate) fn build(&mut self, heap: &Heap, modules: &[LexedModule]) -> Result<(), ParseError> {
         // Sanity checks
         debug_assert!(self.module_lookup.is_empty());
         debug_assert!(self.symbol_lookup.is_empty());
         if cfg!(debug_assertions) {
             for (index, module) in modules.iter().enumerate() {
                 debug_assert_eq!(
                     index, module.root_id.index as usize,
                     "module RootId does not correspond to LexedModule index"
+                )
+            }
+        }
         // Preparation: create a lookup from module name to root id. This does
         // not take aliasing into account.
         self.module_lookup.reserve(modules.len());
         for module in modules {
             // TODO: Maybe put duplicate module name checking here?
             // TODO: @string
             self.module_lookup.insert(module.module_name.clone(), module.root_id);
+        }
         // Preparation: determine total number of imports we will be inserting
         // into the lookup table. We could just iterate over the arena, but then
         // we don't know the source file the import belongs to.
         let mut lookup_reserve_size = 0;
         for module in modules {
             let module_root = &heap[module.root_id];
             for import_id in &module_root.imports {
                 match &heap[*import_id] {
                     Import::Module(_) => lookup_reserve_size += 1,
                     Import::Symbols(import) => {
                         if import.symbols.is_empty() {
                             // Add all symbols from the other module
-                            match self.module_lookup.get(&import.module_name) {
+                            match self.module_lookup.get(&import.module) {
                                 Some(target_module_id) => {
                                     lookup_reserve_size += heap[*target_module_id].definitions.len()
                                 },
                                 None => {
                                     return Err(
                                         ParseError::new_error(&module.source, import.position, "Cannot resolve module")
                                     );
+                                }
+                            }
                         } else {
                             lookup_reserve_size += import.symbols.len();
+                        }
+                    }
+                }
+            }
             lookup_reserve_size += module_root.definitions.len();
+        }
         self.symbol_lookup.reserve(lookup_reserve_size);
         // First pass: we go through all of the modules and add lookups to
         // symbols that are defined within that module. Cross-module imports are
         // not yet resolved
         for module in modules {
             let root = &heap[module.root_id];
             for definition_id in &root.definitions {
                 let definition = &heap[*definition_id];
                 let identifier = definition.identifier();
                 if let Err(previous_position) = self.add_definition_symbol(
                     identifier.position, SymbolKey::from_identifier(module.root_id, &identifier),
                     module.root_id, *definition_id
                 ) {
                     return Err(
                         ParseError::new_error(&module.source, definition.position(), "Symbol is multiply defined")
                             .with_postfixed_info(&module.source, previous_position, "Previous definition was here")
+                    )
+                }
+            }
+        }
         // Second pass: now that we can find symbols in modules, we can resolve
         // all imports (if they're correct, that is)
         for module in modules {
             let root = &heap[module.root_id];
             for import_id in &root.imports {
                 let import = &heap[*import_id];
                 match import {
                     Import::Module(import) => {
                         // Find the module using its name
-                        let target_root_id = self.resolve_module(&import.module_name);
+                        let target_root_id = self.resolve_module(&import.module);
                         if target_root_id.is_none() {
                             return Err(ParseError::new_error(&module.source, import.position, "Could not resolve module"));
+                        }
                         let target_root_id = target_root_id.unwrap();
                         if target_root_id == module.root_id {
                             return Err(ParseError::new_error(&module.source, import.position, "Illegal import of self"));
+                        }
                         // Add the target module under its alias
                         if let Err(previous_position) = self.add_namespace_symbol(
                             import.position, SymbolKey::from_identifier(module.root_id, &import.alias),
                             target_root_id
                         ) {
                             return Err(
                                 ParseError::new_error(&module.source, import.position, "Symbol is multiply defined")
                                     .with_postfixed_info(&module.source, previous_position, "Previous definition was here")
                             );
+                        }
                     },
                     Import::Symbols(import) => {
                         // Find the target module using its name
-                        let target_root_id = self.resolve_module(&import.module_name);
+                        let target_root_id = self.resolve_module(&import.module);
                         if target_root_id.is_none() {
                             return Err(ParseError::new_error(&module.source, import.position, "Could not resolve module of symbol imports"));
+                        }
                         let target_root_id = target_root_id.unwrap();
                         if target_root_id == module.root_id {
                             return Err(ParseError::new_error(&module.source, import.position, "Illegal import of self"));
+                        }
                         // Determine which symbols to import
                         if import.symbols.is_empty() {
                             // Import of all symbols, not using any aliases
                             for definition_id in &heap[target_root_id].definitions {
                                 let definition = &heap[*definition_id];
                                 let identifier = definition.identifier();
                                 if let Err(previous_position) = self.add_definition_symbol(
                                     import.position, SymbolKey::from_identifier(module.root_id, identifier),
                                     target_root_id, *definition_id
                                 ) {
                                     return Err(
                                         ParseError::new_error(
                                             &module.source, import.position,
                                             &format!("Imported symbol '{}' is already defined", String::from_utf8_lossy(&identifier.value))
+                                        )
                                         .with_postfixed_info(
                                             &modules[target_root_id.index as usize].source,
                                             definition.position(),
                                             "The imported symbol is defined here"
+                                        )
                                         .with_postfixed_info(
                                             &module.source, previous_position, "And is previously defined here"
+                                        )
+                                    )
+                                }
+                            }
                         } else {
                             // Import of specific symbols, optionally using aliases
                             for symbol in &import.symbols {
                                 // Because we have already added per-module definitions, we can use
                                 // the table to lookup this particular symbol. Note: within a single
                                 // module a namespace-import and a symbol-import may not collide.
                                 // Hence per-module symbols are unique.
                                 // However: if we import a symbol from another module, we don't want
                                 // to "import a module's imported symbol". And so if we do find
                                 // a symbol match, we need to make sure it is a definition from
                                 // within that module by checking `source_root_id == target_root_id`
                                 let key = SymbolKey::from_identifier(target_root_id, &symbol.name);
                                 let target_symbol = self.symbol_lookup.get(&key);
                                 let symbol_definition_id = match target_symbol {
                                     Some(target_symbol) => {
                                         match target_symbol.symbol {
                                             Symbol::Definition((symbol_root_id, symbol_definition_id)) => {
                                                 if symbol_root_id == target_root_id {
                                                     Some(symbol_definition_id)
                                                 } else {
                                                     // This is imported within the target module, and not
                                                     // defined within the target module
                                                     None
+                                                }
                                             },
                                             Symbol::Namespace(_) => {
                                                 // We don't import a module's "module import"
                                                 None
+                                            }
+                                        }
                                     },
                                     None => None
                                 };
                                 if symbol_definition_id.is_none() {
                                     return Err(
                                         ParseError::new_error(&module.source, symbol.position, "Could not resolve symbol")
+                                    )
+                                }
                                 let symbol_definition_id = symbol_definition_id.unwrap();
                                 if let Err(previous_position) = self.add_definition_symbol(
                                     symbol.position, SymbolKey::from_identifier(module.root_id, &symbol.alias),
                                     target_root_id, symbol_definition_id
                                 ) {
                                     return Err(
                                         ParseError::new_error(&module.source, symbol.position, "Symbol is multiply defined")
                                             .with_postfixed_info(&module.source, previous_position, "Previous definition was here")
+                                    )
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        }
         fn find_name(heap: &Heap, root_id: RootId) -> String {
             let root = &heap[root_id];
             for pragma_id in &root.pragmas {
                 match &heap[*pragma_id] {
                     Pragma::Module(module) => {
                         return String::from_utf8_lossy(&module.value).to_string()
                     },
                     _ => {},
+                }
+            }
             return String::from("Unknown")
+        }
         debug_assert_eq!(
             self.symbol_lookup.len(), lookup_reserve_size,
             "miscalculated reserved size for symbol lookup table"
         );
         Ok(())
+    }
     /// Resolves a module by its defined name
     pub(crate) fn resolve_module(&self, identifier: &Vec<u8>) -> Option<RootId> {
         self.module_lookup.get(identifier).map(|v| *v)
+    }
     pub(crate) fn resolve_symbol<'t>(
         &'t self, root_module_id: RootId, identifier: &[u8]
     ) -> Option<&'t SymbolValue> {
         let lookup_key = SymbolKey{ module_id: root_module_id, symbol_name: Vec::from(identifier) };
         self.symbol_lookup.get(&lookup_key)
+    }
     pub(crate) fn resolve_identifier<'t>(
         &'t self, root_module_id: RootId, identifier: &Identifier
     ) -> Option<&'t SymbolValue> {
         let lookup_key = SymbolKey::from_identifier(root_module_id, identifier);
         self.symbol_lookup.get(&lookup_key)
+    }
     /// Resolves a namespaced symbol. This method will go as far as possible in
     /// going to the right symbol. It will halt the search when:
     /// 1. Polymorphic arguments are encountered on the identifier.
     /// 2. A non-namespace symbol is encountered.
     /// 3. A part of the identifier couldn't be resolved to anything
     /// The returned iterator will always point to the next symbol (even if
     /// nothing was found)
     pub(crate) fn resolve_namespaced_identifier<'t, 'i>(
         &'t self, root_module_id: RootId, identifier: &'i NamespacedIdentifier
     ) -> (Option<&'t SymbolValue>, NamespacedIdentifierIter<'i>) {
         let mut iter = identifier.iter();
         let mut symbol: Option<&SymbolValue> = None;
         let mut within_module_id = root_module_id;
         while let Some((partial, poly_args)) = iter.next() {
             // Lookup the symbol within the currently iterated upon module
             let lookup_key = SymbolKey{ module_id: within_module_id, symbol_name: Vec::from(partial) };
             let new_symbol = self.symbol_lookup.get(&lookup_key);
             match new_symbol {
                 None => {
                     // Can't find anything
                     symbol = None;
                     break;
                 },
                 Some(new_symbol) => {
                     // Found something, but if we already moved to another
                     // module then we don't want to keep jumping across modules,
                     // we're only interested in symbols defined within that
                     // module.
                     match &new_symbol.symbol {
                         Symbol::Namespace(new_root_id) => {
                             if root_module_id != within_module_id {
                                 // This new symbol is imported by a foreign
                                 // module, so this is an error
                                 debug_assert!(symbol.is_some());
                                 debug_assert!(symbol.unwrap().is_namespace());
                                 debug_assert!(iter.num_returned() > 1);
                                 symbol = None;
                                 break;
+                            }
                             within_module_id = *new_root_id;
                             symbol = Some(new_symbol);
                         },
                         Symbol::Definition((definition_root_id, _)) => {
                             // Found a definition, but if we already jumped
                             // modules, then this must be defined within that
                             // module.
                             if root_module_id != within_module_id && within_module_id != *definition_root_id {
                                 // This is an imported definition within the module
                                 // So keep the old
                                 debug_assert!(symbol.is_some());
                                 debug_assert!(symbol.unwrap().is_namespace());
                                 debug_assert!(iter.num_returned() > 1);
                                 symbol = None;
                                 break;
+                            }
                             symbol = Some(new_symbol);
                             break;
+                        }
+                    }
+                }

src/protocol/parser/symbol_table2.rs

➞

Show inline comments

 /// symbol_table.rs
 ///
 /// The datastructure used to lookup symbols within particular scopes. Scopes
 /// may be module-level or definition level, although imports and definitions
 /// within definitions are currently not allowed.
 ///
 /// TODO: Once the compiler has matured, find out ways to optimize to prevent
 ///     the repeated HashMap lookup.
 use std::collections::HashMap;
 use std::collections::hash_map::Entry;
 use crate::protocol::input_source2::*;
 use crate::protocol::ast::*;
 use crate::collections::*;
 const RESERVED_SYMBOLS: usize = 32;
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum SymbolScope {
     Module(RootId),
     Definition(DefinitionId),
+}
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub enum SymbolClass {
     Module,
     Struct,
     Enum,
     Union,
     Function,
     Component
+}
 #[derive(Clone, Copy, PartialEq, Eq)]
 pub enum DefinitionClass {
     Struct,
     Enum,
     Union,
     Function,
     Component,
+}
 impl DefinitionClass {
     fn as_symbol_class(&self) -> SymbolClass {
         match self {
             DefinitionClass::Struct => SymbolClass::Struct,
             DefinitionClass::Enum => SymbolClass::Enum,
             DefinitionClass::Union => SymbolClass::Union,
             DefinitionClass::Function => SymbolClass::Function,
             DefinitionClass::Component => SymbolClass::Component,
+        }
+    }
+}
 struct ScopedSymbols {
     scope: SymbolScope,
     parent_scope: Option<SymbolScope>,
     child_scopes: Vec<SymbolScope>,
     symbols: Vec<Symbol>,
+}
 impl ScopedSymbols {
     fn get_symbol<'a>(&'a self, name: &StringRef) -> Option<&'a Symbol> {
         for symbol in self.symbols.iter() {
             if symbol.name == *name {
                 return Some(symbol);
+            }
+        }
         None
+    }
+}
 impl SymbolDefinition {
     pub fn symbol_class(&self) -> SymbolClass {
         use SymbolDefinition as SD;
         use SymbolClass as SC;
         match self {
             SD::Module(_) => SC::Module,
             SD::Struct(_) => SC::Struct,
             SD::Enum(_) => SC::Enum,
             SD::Union(_) => SC::Union,
             SD::Function(_) => SC::Function,
             SD::Component(_) => SC::Component,
+        }
+    }
+}
 #[derive(Debug)]
 pub struct SymbolModule {
     pub root_id: RootId,
     pub introduced_at: ImportId,
+}
 #[derive(Debug, Clone)]
 pub struct SymbolDefinition {
     // Definition location (not necessarily the place where the symbol
     // is introduced, as it may be imported)
     pub defined_in_module: RootId,
     pub defined_in_scope: SymbolScope,
     pub definition_span: InputSpan, // full span of definition
     pub identifier_span: InputSpan, // span of just the identifier
     // Location where the symbol is introduced in its scope
     pub imported_at: Option<ImportId>,
     // Definition in the heap, with a utility enum to determine its
     // class if the ID is not needed.
     pub class: DefinitionClass,
     pub definition_id: DefinitionId,
+}
 impl SymbolDefinition {
     /// Clones the entire data structure, but replaces the `imported_at` field
     /// with the supplied `ImportId`.
     pub(crate) fn into_imported(mut self, imported_at: ImportId) -> Self {
         self.imported_at = Some(imported_at);
         self
+    }
+}
 #[derive(Debug)]
 pub enum SymbolVariant {
     Module(SymbolModule),
     Definition(SymbolDefinition),
+}
 impl SymbolVariant {
     pub(crate) fn as_module(&self) -> &SymbolModule {
         match self {
             SymbolVariant::Module(v) => v,
             SymbolVariant::Definition(_) => unreachable!("called 'as_module' on {:?}", self),
+        }
+    }
     pub(crate) fn as_definition(&self) -> &SymbolDefinition {
         match self {
             SymbolVariant::Module(v) => unreachable!("called 'as_definition' on {:?}", self),
             SymbolVariant::Definition(v) => v,
+        }
+    }
     pub(crate) fn as_definition_mut(&mut self) -> &mut SymbolDefinition {
         match self {
             SymbolVariant::Module(v) => unreachable!("called 'as_definition_mut' on {:?}", self),
             SymbolVariant::Definition(v) => v,
+        }
+    }
+}
 #[derive(Clone)]
 pub struct Symbol {
     pub name: StringRef<'static>,
-    pub data: SymbolVariant,
+    pub variant: SymbolVariant,
+}
 impl Symbol {
     fn class(&self) -> SymbolClass {
         match &self.data {
     pub(crate) fn class(&self) -> SymbolClass {
         match &self.variant {
             SymbolVariant::Module(_) => SymbolClass::Module,
             SymbolVariant::Definition(data) => data.class.as_symbol_class(),
+        }
+    }
+}
 pub struct SymbolTable {
     module_lookup: HashMap<StringRef<'static>, RootId>,
     scope_lookup: HashMap<SymbolScope, ScopedSymbols>,
+}
 impl SymbolTable {
     /// Inserts a new module by its name. Upon module naming conflict the
     /// previously associated `RootId` will be returned.
     pub(crate) fn insert_module(&mut self, module_name: StringRef<'static>, root_id: RootId) -> Result<(), RootId> {
         match self.module_lookup.entry(module_name) {
             Entry::Occupied(v) => {
                 Err(*v.get())
             },
             Entry::Vacant(v) => {
                 v.insert(root_id);
                 Ok(())
+            }
+        }
+    }
     /// Retrieves module `RootId` by name
     pub(crate) fn get_module_by_name(&mut self, name: &[u8]) -> Option<RootId> {
         let string_ref = StringRef::new(name);
         self.module_lookup.get(&string_ref).map(|v| *v)
+    }
     /// Inserts a new symbol scope. The parent must have been added to the
     /// symbol table before.
     pub(crate) fn insert_scope(&mut self, parent_scope: Option<SymbolScope>, new_scope: SymbolScope) {
         debug_assert!(
             parent_scope.is_none() || self.scope_lookup.contains_key(parent_scope.as_ref().unwrap()),
             "inserting scope {:?} but parent {:?} does not exist", new_scope, parent_scope
         );
         debug_assert!(!self.scope_lookup.contains_key(&new_scope), "inserting scope {:?}, but it already exists", new_scope);
         if let Some(parent_scope) = parent_scope {
             let parent = self.scope_lookup.get_mut(&parent_scope).unwrap();
             parent.child_scopes.push(new_scope);
+        }
         let scope = ScopedSymbols {
             scope: new_scope,
             parent_scope,
             child_scopes: Vec::with_capacity(RESERVED_SYMBOLS),
             symbols: Vec::with_capacity(RESERVED_SYMBOLS)
         };
         self.scope_lookup.insert(new_scope, scope);
+    }
     /// Inserts a symbol into a particular scope. The symbol's name may not
     /// exist in the scope or any of its parents. If it does collide then the
     /// symbol will be returned, together with the symbol that has the same
     /// name.
     pub(crate) fn insert_symbol(&mut self, in_scope: SymbolScope, symbol: Symbol) -> Result<(), (Symbol, &Symbol)> {
         debug_assert!(self.scope_lookup.contains_key(&in_scope), "inserting symbol {}, but scope {:?} does not exist", symbol.name.as_str(), in_scope);
         let mut seek_scope = in_scope;
         loop {
             let scoped_symbols = self.scope_lookup.get(&seek_scope).unwrap();
             for existing_symbol in scoped_symbols.symbols.iter() {
                 if symbol.name == existing_symbol.name {
                     return Err((symbol, existing_symbol))
+                }
+            }
             match scoped_symbols.parent_scope {
                 Some(parent_scope) => { seek_scope = parent_scope; },
                 None => { break; }
+            }
+        }
         // If here, then there is no collision
         let scoped_symbols = self.scope_lookup.get_mut(&in_scope).unwrap();
         scoped_symbols.symbols.push(symbol);
         Ok(())
+    }
     /// Retrieves a symbol by name by searching in a particular scope and that scope's parents. The
     /// returned symbol may both be imported as defined within any of the searched scopes.
     pub(crate) fn get_symbol_by_name(
         &self, mut in_scope: SymbolScope, name: &[u8]
     ) -> Option<&Symbol> {
         let string_ref = StringRef::new(name);
         loop {
             let scope = self.scope_lookup.get(&in_scope);
             if scope.is_none() {
                 return None;
+            }
             let scope = scope.unwrap();
             if let Some(symbol) = scope.get_symbol(&string_ref) {
                 return Some(symbol);
             } else {
                 // Could not find symbol in current scope, seek in the parent scope if it exists
                 match &scope.parent_scope {
                     Some(parent_scope) => { in_scope = *parent_scope; },
                     None => return None,
+                }
+            }
+        }
+    }
     /// Retrieves a symbol by name by searching in a particular scope and that scope's parents. The
     /// returned symbol must be defined within any of the searched scopes and may not be imported.
     /// In case such an imported symbol exists then this function still returns `None`.
     pub(crate) fn get_symbol_by_name_defined_in_scope(
         &self, in_scope: SymbolScope, name: &[u8]
     ) -> Option<&Symbol> {
         match self.get_symbol_by_name(in_scope, name) {
             Some(symbol) => {
-                match &symbol.data {
+                match &symbol.variant {
                     SymbolVariant::Module(_) => {
                         None // in-scope modules are always imported
                     },
                     SymbolVariant::Definition(variant) => {
                         if variant.imported_at.is_some() {
                             None
                         } else {
                             Some(symbol)
+                        }
+                    }
+                }
             },
             None => None,
+        }
+    }
     /// Retrieves all symbols that are defined within a particular scope. Imported symbols are
     /// ignored. Returns `true` if the scope was found (which may contain 0 defined symbols) and
     /// `false` if the scope was not found.
     pub(crate) fn get_all_symbols_defined_in_scope(&self, in_scope: SymbolScope, target: &mut Vec<Symbol>) -> bool {
         match self.scope_lookup.get(&in_scope) {
             Some(scope) => {
                 for symbol in &scope.symbols {
                     if let SymbolVariant::Definition(definition) = &symbol.variant {
                         if definition.imported_at.is_some() {
                             continue;
+                        }
                         // Defined in scope, so push onto target
                         target.push(symbol.clone());
+                    }
+                }
                 true
             },
             None => false,
+        }
+    }
+}
@@ \ No newline at end of file @@

Changeset was too big and was cut off... Show full diff anyway

0 comments (0 inline, 0 general)