CSY/reowolf Changeset - 113e4349a706 · Centrum Wiskunde & Informatica (CWI)

Changeset - 113e4349a706

Parent rev.

Child rev.

[Not reviewed]

1 32 12

Max Henger - 3 years ago 2022-04-13 14:57:03
henger@cwi.nl

feat: Builtin internet component

16 files changed:

Cargo.toml

bin-compiler/src/main.rs

src/collections/scoped_buffer.rs

src/protocol/ast.rs

src/protocol/ast_writer.rs

src/protocol/eval/executor.rs

src/protocol/eval/value.rs

src/protocol/input_source.rs

src/protocol/mod.rs

src/protocol/parser/mod.rs

125

148

src/protocol/parser/pass_definitions.rs

146

src/protocol/parser/pass_definitions_types.rs

src/protocol/parser/pass_imports.rs

src/protocol/parser/pass_rewriting.rs

src/protocol/parser/pass_symbols.rs

src/protocol/parser/pass_tokenizer.rs

183

Changeset was too big and was cut off... Show full diff anyway

0 comments (0 inline, 0 general)

Cargo.toml

➞

Show inline comments

 [package]
 name = "reowolf_rs"
 version = "1.2.0"
 authors = [
 	"Max Henger <henger@cwi.nl>",
 	"Christopher Esterhuyse <esterhuy@cwi.nl>",
 	"Hans-Dieter Hiep <hdh@cwi.nl>"
+]
 edition = "2021"
 [dependencies]
 # convenience macros
 maplit = "1.0.2"
 derive_more = "0.99.2"
 # runtime
 bincode = "1.3.1"
 serde = { version = "1.0.114", features = ["derive"] }
 getrandom = "0.1.14" # tiny crate. used to guess controller-id
 # network
 mio = { version = "0.7.0", package = "mio", features = ["udp", "tcp", "os-poll"] }
 socket2 = { version = "0.3.12", optional = true }
 [features]
 default=["internet"]
 internet=["libc"]
 # protocol
 backtrace = "0.3"
 lazy_static = "1.4.0"
 # ffi
 [dependencies]
 # socket ffi
 libc = { version = "^0.2", optional = true }
 os_socketaddr = { version = "0.1.0", optional = true }
 libc = { version = "^0.2", optional = true } # raw sockets
 mio = { version = "0.8", features = ["os-poll"] } # cross-platform IO notification queue
 # randomness
 rand = "0.8.4"
 rand_pcg = "0.3.1"
 [lib]
 crate-type = [
 	"rlib", # for use as a Rust dependency.
+]
@@ \ No newline at end of file @@

bin-compiler/src/main.rs

➞

Show inline comments

 use std::fs::File;
 use std::io::Read;
 use clap::{App, Arg};
 use reowolf_rs as rw;
 fn main() {
     let app = App::new("rwc")
         .author("Henger, M.")
         .version(env!("CARGO_PKG_VERSION"))
         .about("Reowolf compiler")
         .arg(
             Arg::new("input")
                 .long("input")
                 .short('i')
                 .help("input files")
                 .required(true)
                 .takes_value(true)
                 .multiple_occurrences(true)
+        )
         .arg(
             Arg::new("threads")
                 .long("threads")
                 .short('t')
                 .help("number of runtime threads")
                 .default_value("1")
                 .takes_value(true)
+        )
         .arg(
             Arg::new("debug")
                 .long("debug")
                 .short('d')
                 .help("enable debug logging")
+        )
         .arg(
             Arg::new("stdlib")
                 .long("stdlib")
                 .short('s')
                 .help("standard library directory (overrides default)")
                 .takes_value(true)
         );
     // Retrieve arguments and convert
     let app = app.get_matches();
     let input_files = app.values_of("input");
     if input_files.is_none() {
         println!("ERROR: Expected at least one input file");
         return;
+    }
     let num_threads = app.value_of("threads").unwrap();
     let num_threads = match num_threads.parse::<i32>() {
         Ok(num_threads) => {
             if num_threads < 0 || num_threads > 255 {
                 println!("ERROR: Number of threads must be a number between 0 and 256");
                 return;
+            }
             num_threads as u32
         },
         Err(err) => {
             println!("ERROR: Failed to parse number of threads\nbecause: {}", err);
             return;
+        }
     };
     let debug_enabled = app.is_present("debug");
     let standard_library_dir = app.value_of("stdlib")
         .map(|v| v.to_string());
     // Add input files to file buffer
     let input_files = input_files.unwrap();
     assert!(input_files.len() > 0); // because arg is required
     let mut builder = rw::ProtocolDescriptionBuilder::new();
     let mut builder = rw::ProtocolDescriptionBuilder::new(standard_library_dir)
         .expect("create protocol description builder");
     let mut file_buffer = Vec::with_capacity(4096);
     for input_file in input_files {
         print!("Adding file: {} ... ", input_file);
         let mut file = match File::open(input_file) {
             Ok(file) => file,
             Err(err) => {
                 println!("FAILED (to open file)\nbecause:\n{}", err);
                 return;
+            }
         };
         file_buffer.clear();
         if let Err(err) = file.read_to_end(&mut file_buffer) {
             println!("FAILED (to read file)\nbecause:\n{}", err);
             return;
+        }
         if let Err(err) = builder.add(input_file.to_string(), file_buffer.clone()) {
             println!("FAILED (to tokenize file)\nbecause:\n{}", err);
+        }
         println!("Success");
+    }
     // Compile the program
     print!("Compiling program ... ");
     let protocol_description = match builder.compile() {
         Ok(pd) => pd,
         Err(err) => {
             println!("FAILED\nbecause:\n{}", err);
             return;
+        }
     };
     println!("Success");
     // Start runtime
     print!("Startup of runtime ... ");
     let runtime = rw::runtime2::Runtime::new(num_threads, debug_enabled, protocol_description);
     if let Err(err) = &runtime {
         println!("FAILED\nbecause:\n{}", err);
+    }
     println!("Success");
     // Make sure there is a nameless module with a main component
     print!("Creating main component ... ");
-    let runtime = rw::runtime2::Runtime::new(num_threads, debug_enabled, protocol_description);
+    let runtime = runtime.unwrap();
     if let Err(err) = runtime.create_component(b"", b"main") {
         use rw::ComponentCreationError as CCE;
         let reason = match err {
             CCE::ModuleDoesntExist => "Input files did not contain a nameless module (that should contain the 'main' component)",
             CCE::DefinitionDoesntExist => "Input files did not contain a component called 'main'",
             CCE::DefinitionNotComponent => "Input file contained a 'main' function, but not a 'main' component",
             _ => "Unexpected error"
         };
         println!("FAILED\nbecause:\n{} (raw error: {:?})", reason, err);
         return;
+    }
     println!("Success");
     println!("Now running until all components have exited");
     println!("--------------------------------------------\n\n");
+}
@@ \ No newline at end of file @@

src/collections/scoped_buffer.rs

➞

Show inline comments

@@ @@ -131,81 +131,84 @@ impl<T: Sized + PartialEq> ScopedSection<T> { @@
                 return;
+            }
+        }
         vec.push(value);
         hide!(self.cur_size += 1);
+    }
     #[inline]
     pub(crate) fn contains(&self, value: &T) -> bool {
         self.check_length();
         let vec = unsafe{&*self.inner};
         for index in self.start_size as usize..vec.len() {
             if &vec[index] == value {
                 return true;
+            }
+        }
         return false;
+    }
+}
 impl<T: Copy> ScopedSection<T> {
     pub(crate) fn iter_copied(&self) -> ScopedIter<T> {
         return ScopedIter{
             inner: self.inner,
             cur_index: self.start_size,
             last_index: unsafe{ (*self.inner).len() as u32 },
+        }
+    }
+}
 impl<T> std::ops::Index<usize> for ScopedSection<T> {
     type Output = T;
     fn index(&self, index: usize) -> &Self::Output {
         let vec = unsafe{&*self.inner};
         return &vec[self.start_size as usize + index]
+    }
+}
 impl<T> std::ops::IndexMut<usize> for ScopedSection<T> {
     fn index_mut(&mut self, index: usize) -> &mut Self::Output {
         let vec = unsafe{&mut *self.inner};
         return &mut vec[self.start_size as usize + index]
+    }
+}
 #[cfg(debug_assertions)]
 // note: this `Drop` impl used to be debug-only, requiring the programmer to
 // call `into_vec` or `forget`. But this is rather error prone. So we'll check
 // in debug mode, but always truncate in release mode (even though this is a
 // noop in most cases).
 impl<T: Sized> Drop for ScopedSection<T> {
     fn drop(&mut self) {
         let vec = unsafe{&mut *self.inner};
         hide!(debug_assert_eq!(vec.len(), self.cur_size as usize));
         vec.truncate(self.start_size as usize);
+    }
+}
 /// Small utility for iterating over a section of the buffer. Same conditions as
 /// the buffer apply: each time we retrieve an element the buffer must have the
 /// same size as the moment of creation.
 pub(crate) struct ScopedIter<T: Copy> {
     inner: *mut Vec<T>,
     cur_index: u32,
     last_index: u32,
+}
 impl<T: Copy> Iterator for ScopedIter<T> {
     type Item = T;
     fn next(&mut self) -> Option<Self::Item> {
         hide!(debug_assert_eq!(self.last_index as usize, unsafe { (*self.inner).len() }));
         if self.cur_index >= self.last_index {
             return None;
+        }
         let vec = unsafe{ &*self.inner };
         let index = self.cur_index as usize;
         self.cur_index += 1;
         return Some(vec[index]);
+    }
+}
@@ \ No newline at end of file @@

src/protocol/ast.rs

➞

Show inline comments

@@ @@ -197,97 +197,97 @@ impl Heap { @@
             Statement::Local(LocalStatement::Memory(
                 f(MemoryStatementId(LocalStatementId(id)))
             ))
         })))
+    }
     pub fn alloc_channel_statement(
         &mut self,
         f: impl FnOnce(ChannelStatementId) -> ChannelStatement,
     ) -> ChannelStatementId {
         ChannelStatementId(LocalStatementId(self.statements.alloc_with_id(|id| {
             Statement::Local(LocalStatement::Channel(
                 f(ChannelStatementId(LocalStatementId(id)))
             ))
         })))
+    }
+}
 impl Index<MemoryStatementId> for Heap {
     type Output = MemoryStatement;
     fn index(&self, index: MemoryStatementId) -> &Self::Output {
         match &self.statements[index.0.0] {
             Statement::Local(LocalStatement::Memory(v)) => v,
             _ => unreachable!(),
+        }
+    }
+}
 impl Index<ChannelStatementId> for Heap {
     type Output = ChannelStatement;
     fn index(&self, index: ChannelStatementId) -> &Self::Output {
         match &self.statements[index.0.0] {
             Statement::Local(LocalStatement::Channel(v)) => v,
             _ => unreachable!(),
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct Root {
     pub this: RootId,
     // Phase 1: parser
     // pub position: InputPosition,
     pub pragmas: Vec<PragmaId>,
     pub imports: Vec<ImportId>,
     pub definitions: Vec<DefinitionId>,
+}
 impl Root {
-    pub fn get_definition_ident(&self, h: &Heap, id: &[u8]) -> Option<DefinitionId> {
+    pub fn get_definition_by_ident(&self, h: &Heap, id: &[u8]) -> Option<DefinitionId> {
         for &def in self.definitions.iter() {
             if h[def].identifier().value.as_bytes() == id {
                 return Some(def);
+            }
+        }
         None
+    }
+}
 #[derive(Debug, Clone)]
 pub enum Pragma {
     Version(PragmaVersion),
     Module(PragmaModule),
+}
 impl Pragma {
     pub(crate) fn as_module(&self) -> &PragmaModule {
         match self {
             Pragma::Module(pragma) => pragma,
             _ => unreachable!("Tried to obtain {:?} as PragmaModule", self),
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct PragmaVersion {
     pub this: PragmaId,
     // Phase 1: parser
     pub span: InputSpan, // of full pragma
     pub version: u64,
+}
 #[derive(Debug, Clone)]
 pub struct PragmaModule {
     pub this: PragmaId,
     // Phase 1: parser
     pub span: InputSpan, // of full pragma
     pub value: Identifier,
+}
 #[derive(Debug, Clone)]
 pub enum Import {
     Module(ImportModule),
     Symbols(ImportSymbols)
+}
 impl Import {
     pub(crate) fn span(&self) -> InputSpan {
@@ @@ -887,268 +887,294 @@ impl Definition { @@
+    }
     pub(crate) fn as_procedure_mut(&mut self) -> &mut ProcedureDefinition {
         match self {
             Definition::Procedure(result) => result,
             _ => panic!("Unable to cast `Definition` to `Function`"),
+        }
+    }
     pub fn defined_in(&self) -> RootId {
         match self {
             Definition::Struct(def) => def.defined_in,
             Definition::Enum(def) => def.defined_in,
             Definition::Union(def) => def.defined_in,
             Definition::Procedure(def) => def.defined_in,
+        }
+    }
     pub fn identifier(&self) -> &Identifier {
         match self {
             Definition::Struct(def) => &def.identifier,
             Definition::Enum(def) => &def.identifier,
             Definition::Union(def) => &def.identifier,
             Definition::Procedure(def) => &def.identifier,
+        }
+    }
     pub fn poly_vars(&self) -> &Vec<Identifier> {
         match self {
             Definition::Struct(def) => &def.poly_vars,
             Definition::Enum(def) => &def.poly_vars,
             Definition::Union(def) => &def.poly_vars,
             Definition::Procedure(def) => &def.poly_vars,
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct StructFieldDefinition {
     pub span: InputSpan,
     pub field: Identifier,
     pub parser_type: ParserType,
+}
 #[derive(Debug, Clone)]
 pub struct StructDefinition {
     pub this: StructDefinitionId,
     pub defined_in: RootId,
     // Symbol scanning
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Parsing
     pub fields: Vec<StructFieldDefinition>
+}
 impl StructDefinition {
     pub(crate) fn new_empty(
-        this: StructDefinitionId, defined_in: RootId, span: InputSpan,
         this: StructDefinitionId, defined_in: RootId,
         identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
-        Self{ this, defined_in, span, identifier, poly_vars, fields: Vec::new() }
         Self{ this, defined_in, identifier, poly_vars, fields: Vec::new() }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub enum EnumVariantValue {
     None,
     Integer(i64),
+}
 #[derive(Debug, Clone)]
 pub struct EnumVariantDefinition {
     pub identifier: Identifier,
     pub value: EnumVariantValue,
+}
 #[derive(Debug, Clone)]
 pub struct EnumDefinition {
     pub this: EnumDefinitionId,
     pub defined_in: RootId,
     // Symbol scanning
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Parsing
     pub variants: Vec<EnumVariantDefinition>,
+}
 impl EnumDefinition {
     pub(crate) fn new_empty(
-        this: EnumDefinitionId, defined_in: RootId, span: InputSpan,
         this: EnumDefinitionId, defined_in: RootId,
         identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
-        Self{ this, defined_in, span, identifier, poly_vars, variants: Vec::new() }
         Self{ this, defined_in, identifier, poly_vars, variants: Vec::new() }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct UnionVariantDefinition {
     pub span: InputSpan,
     pub identifier: Identifier,
     pub value: Vec<ParserType>, // if empty, then union variant does not contain any embedded types
+}
 #[derive(Debug, Clone)]
 pub struct UnionDefinition {
     pub this: UnionDefinitionId,
     pub defined_in: RootId,
     // Phase 1: symbol scanning
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Phase 2: parsing
     pub variants: Vec<UnionVariantDefinition>,
+}
 impl UnionDefinition {
     pub(crate) fn new_empty(
-        this: UnionDefinitionId, defined_in: RootId, span: InputSpan,
         this: UnionDefinitionId, defined_in: RootId,
         identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
-        Self{ this, defined_in, span, identifier, poly_vars, variants: Vec::new() }
         Self{ this, defined_in, identifier, poly_vars, variants: Vec::new() }
+    }
+}
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ProcedureKind {
     Function, // with return type
     Primitive, // without return type
     Composite,
+}
 /// Monomorphed instantiation of a procedure (or the sole instantiation of a
 /// non-polymorphic procedure).
 #[derive(Debug)]
 pub struct ProcedureDefinitionMonomorph {
     pub argument_types: Vec<TypeId>,
     pub expr_info: Vec<ExpressionInfo>
+}
 impl ProcedureDefinitionMonomorph {
     pub(crate) fn new_invalid() -> Self {
         return Self{
             argument_types: Vec::new(),
             expr_info: Vec::new(),
+        }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub struct ExpressionInfo {
     pub type_id: TypeId,
     pub variant: ExpressionInfoVariant,
+}
 impl ExpressionInfo {
     pub(crate) fn new_invalid() -> Self {
         return Self{
             type_id: TypeId::new_invalid(),
             variant: ExpressionInfoVariant::Generic,
+        }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub enum ExpressionInfoVariant {
     Generic,
     Procedure(TypeId, u32), // procedure TypeID and its monomorph index
     Select(i32), // index
+}
 impl ExpressionInfoVariant {
     pub(crate) fn as_select(&self) -> i32 {
         match self {
             ExpressionInfoVariant::Select(v) => *v,
             _ => unreachable!(),
+        }
+    }
     pub(crate) fn as_procedure(&self) -> (TypeId, u32) {
         match self {
             ExpressionInfoVariant::Procedure(type_id, monomorph_index) => (*type_id, *monomorph_index),
             _ => unreachable!(),
+        }
+    }
+}
 #[derive(Debug)]
 pub enum ProcedureSource {
     FuncUserDefined,
     CompUserDefined,
     // Builtin functions, available to user
     FuncGet,
     FuncPut,
     FuncFires,
     FuncCreate,
     FuncLength,
     FuncAssert,
     FuncPrint,
     // Buitlin functions, not available to user
     FuncSelectStart,
     FuncSelectRegisterCasePort,
     FuncSelectWait,
     // Builtin components, available to user
     CompRandomU32, // TODO: Remove, temporary thing
     CompTcpClient,
+}
 impl ProcedureSource {
     pub(crate) fn is_builtin(&self) -> bool {
         match self {
             ProcedureSource::FuncUserDefined | ProcedureSource::CompUserDefined => false,
             _ => true,
+        }
+    }
+}
 /// Generic storage for functions, primitive components and composite
 /// components.
 // Note that we will have function definitions for builtin functions as well. In
 // that case the span, the identifier span and the body are all invalid.
 #[derive(Debug)]
 pub struct ProcedureDefinition {
     pub this: ProcedureDefinitionId,
     pub defined_in: RootId,
     // Symbol scanning
     pub builtin: bool,
     pub kind: ProcedureKind,
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Parser
     pub source: ProcedureSource,
     pub return_type: Option<ParserType>, // present on functions, not components
     pub parameters: Vec<VariableId>,
     pub scope: ScopeId,
     pub body: BlockStatementId,
     // Monomorphization of typed procedures
     pub monomorphs: Vec<ProcedureDefinitionMonomorph>,
+}
 impl ProcedureDefinition {
     pub(crate) fn new_empty(
-        this: ProcedureDefinitionId, defined_in: RootId, span: InputSpan,
         this: ProcedureDefinitionId, defined_in: RootId,
         kind: ProcedureKind, identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
         Self {
             this, defined_in,
             builtin: false,
             span,
             kind, identifier, poly_vars,
             source: ProcedureSource::FuncUserDefined,
             return_type: None,
             parameters: Vec::new(),
             scope: ScopeId::new_invalid(),
             body: BlockStatementId::new_invalid(),
             monomorphs: Vec::new(),
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub enum Statement {
     Block(BlockStatement),
     EndBlock(EndBlockStatement),
     Local(LocalStatement),
     Labeled(LabeledStatement),
     If(IfStatement),
     EndIf(EndIfStatement),
     While(WhileStatement),
     EndWhile(EndWhileStatement),
     Break(BreakStatement),
     Continue(ContinueStatement),
     Synchronous(SynchronousStatement),
     EndSynchronous(EndSynchronousStatement),
     Fork(ForkStatement),
     EndFork(EndForkStatement),
     Select(SelectStatement),
     EndSelect(EndSelectStatement),
     Return(ReturnStatement),
     Goto(GotoStatement),
     New(NewStatement),
     Expression(ExpressionStatement),
+}
 impl Statement {
     pub fn as_new(&self) -> &NewStatement {
         match self {
             Statement::New(result) => result,
             _ => panic!("Unable to cast `Statement` to `NewStatement`"),
+        }
+    }
     pub fn span(&self) -> InputSpan {
         match self {
             Statement::Block(v) => v.span,
             Statement::Local(v) => v.span(),
             Statement::Labeled(v) => v.label.span,
             Statement::If(v) => v.span,
             Statement::While(v) => v.span,
@@ @@ -1768,149 +1794,154 @@ pub enum SelectKind { @@
+}
 #[derive(Debug, Clone)]
 pub struct SelectExpression {
     pub this: SelectExpressionId,
     // Parsing
     pub operator_span: InputSpan, // of the '.'
     pub full_span: InputSpan, // includes subject and field
     pub subject: ExpressionId,
     pub kind: SelectKind,
     // Validator/Linker
     pub parent: ExpressionParent,
     // Typing
     pub type_index: i32,
+}
 #[derive(Debug, Clone)]
 pub struct CastExpression {
     pub this: CastExpressionId,
     // Parsing
     pub cast_span: InputSpan, // of the "cast" keyword,
     pub full_span: InputSpan, // includes the cast subject
     pub to_type: ParserType,
     pub subject: ExpressionId,
     // Validator/linker
     pub parent: ExpressionParent,
     // Typing
     pub type_index: i32,
+}
 #[derive(Debug, Clone)]
 pub struct CallExpression {
     pub this: CallExpressionId,
     // Parsing
     pub func_span: InputSpan, // of the function name
     pub full_span: InputSpan, // includes the arguments and parentheses
     pub parser_type: ParserType, // of the function call, not the return type
     pub method: Method,
     pub arguments: Vec<ExpressionId>,
     pub procedure: ProcedureDefinitionId,
     // Validator/Linker
     pub parent: ExpressionParent,
     // Typing
     pub type_index: i32,
+}
 #[derive(Debug, Clone, PartialEq, Eq)]
 pub enum Method {
     // Builtin, accessible by programmer
+    // Builtin function, accessible by programmer
     Get,
     Put,
     Fires,
     Create,
     Length,
     Assert,
     Print,
     // Builtin, not accessible by programmer
+    // Builtin function, not accessible by programmer
     SelectStart, // SelectStart(total_num_cases, total_num_ports)
     SelectRegisterCasePort, // SelectRegisterCasePort(case_index, port_index, port_id)
     SelectWait, // SelectWait() -> u32
     // Builtin component,
     ComponentRandomU32,
     ComponentTcpClient,
     // User-defined
     UserFunction,
     UserComponent,
+}
 impl Method {
     pub(crate) fn is_public_builtin(&self) -> bool {
         use Method::*;
         match self {
             Get | Put | Fires | Create | Length | Assert | Print => true,
             ComponentRandomU32 | ComponentTcpClient => true,
             _ => false,
+        }
+    }
     pub(crate) fn is_user_defined(&self) -> bool {
         use Method::*;
         match self {
             UserFunction | UserComponent => true,
             _ => false,
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct LiteralExpression {
     pub this: LiteralExpressionId,
     // Parsing
     pub span: InputSpan,
     pub value: Literal,
     // Validator/Linker
     pub parent: ExpressionParent,
     // Typing
     pub type_index: i32,
+}
 #[derive(Debug, Clone)]
 pub enum Literal {
     Null, // message
     True,
     False,
     Character(char),
     Bytestring(Vec<u8>),
     String(StringRef<'static>),
     Integer(LiteralInteger),
     Struct(LiteralStruct),
     Enum(LiteralEnum),
     Union(LiteralUnion),
     Array(Vec<ExpressionId>),
     Tuple(Vec<ExpressionId>),
+}
 impl Literal {
     pub(crate) fn as_struct(&self) -> &LiteralStruct {
         if let Literal::Struct(literal) = self{
             literal
         } else {
             unreachable!("Attempted to obtain {:?} as Literal::Struct", self)
+        }
+    }
     pub(crate) fn as_enum(&self) -> &LiteralEnum {
         if let Literal::Enum(literal) = self {
             literal
         } else {
             unreachable!("Attempted to obtain {:?} as Literal::Enum", self)
+        }
+    }
     pub(crate) fn as_union(&self) -> &LiteralUnion {
         if let Literal::Union(literal) = self {
             literal
         } else {
             unreachable!("Attempted to obtain {:?} as Literal::Union", self)
+        }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct LiteralInteger {
     pub(crate) unsigned_value: u64,
     pub(crate) negated: bool, // for constant expression evaluation, TODO: @Int
+}
 #[derive(Debug, Clone)]
 pub struct LiteralStructField {
     // Phase 1: parser
     pub(crate) identifier: Identifier,
     pub(crate) value: ExpressionId,
     // Phase 2: linker
     pub(crate) field_idx: usize, // in struct definition

src/protocol/ast_writer.rs

➞

Show inline comments

@@ file renamed from src/protocol/ast_printer.rs to src/protocol/ast_writer.rs @@
@@ @@ -320,98 +320,102 @@ impl ASTWriter { @@
+                }
             },
             Definition::Union(def) => {
                 self.kv(indent).with_id(PREFIX_UNION_ID, def.this.0.index)
                     .with_s_key("DefinitionUnion");
                 self.kv(indent2).with_s_key("Name").with_identifier_val(&def.identifier);
                 for poly_var_id in &def.poly_vars {
                     self.kv(indent3).with_s_key("PolyVar").with_identifier_val(&poly_var_id);
+                }
                 self.kv(indent2).with_s_key("Variants");
                 for variant in &def.variants {
                     self.kv(indent3).with_s_key("Variant");
                     self.kv(indent4).with_s_key("Name")
                         .with_identifier_val(&variant.identifier);
                     if variant.value.is_empty() {
                         self.kv(indent4).with_s_key("Value").with_s_val("None");
                     } else {
                         self.kv(indent4).with_s_key("Values");
                         for embedded in &variant.value {
                             self.kv(indent4+1).with_s_key("Value")
                                 .with_custom_val(|v| write_parser_type(v, heap, embedded));
+                        }
+                    }
+                }
+            }
             Definition::Procedure(def) => {
                 self.kv(indent).with_id(PREFIX_FUNCTION_ID, def.this.0.index)
                     .with_s_key("DefinitionFunction");
                 self.kv(indent2).with_s_key("Name").with_identifier_val(&def.identifier);
                 for poly_var_id in &def.poly_vars {
                     self.kv(indent3).with_s_key("PolyVar").with_identifier_val(&poly_var_id);
+                }
                 self.kv(indent2).with_s_key("Kind").with_debug_val(&def.kind);
                 if let Some(parser_type) = &def.return_type {
                     self.kv(indent2).with_s_key("ReturnParserType")
                         .with_custom_val(|s| write_parser_type(s, heap, parser_type));
+                }
                 self.kv(indent2).with_s_key("Parameters");
                 for variable_id in &def.parameters {
                     self.write_variable(heap, *variable_id, indent3);
+                }
                 self.kv(indent2).with_s_key("Body");
                 self.write_stmt(heap, def.body.upcast(), indent3);
                 if def.source.is_builtin() {
                     self.kv(indent2).with_s_key("Body").with_s_val("Builtin");
                 } else {
                     self.kv(indent2).with_s_key("Body");
                     self.write_stmt(heap, def.body.upcast(), indent3);
+                }
             },
+        }
+    }
     fn write_stmt(&mut self, heap: &Heap, stmt_id: StatementId, indent: usize) {
         let stmt = &heap[stmt_id];
         let indent2 = indent + 1;
         let indent3 = indent2 + 1;
         match stmt {
             Statement::Block(stmt) => {
                 self.kv(indent).with_id(PREFIX_BLOCK_STMT_ID, stmt.this.0.index)
                     .with_s_key("Block");
                 self.kv(indent2).with_s_key("EndBlockID").with_disp_val(&stmt.end_block.0.index);
                 self.kv(indent2).with_s_key("ScopeID").with_disp_val(&stmt.scope.index);
                 self.kv(indent2).with_s_key("Statements");
                 for stmt_id in &stmt.statements {
                     self.write_stmt(heap, *stmt_id, indent3);
+                }
             },
             Statement::EndBlock(stmt) => {
                 self.kv(indent).with_id(PREFIX_ENDBLOCK_STMT_ID, stmt.this.0.index)
                     .with_s_key("EndBlock");
                 self.kv(indent2).with_s_key("StartBlockID").with_disp_val(&stmt.start_block.0.index);
+            }
             Statement::Local(stmt) => {
                 match stmt {
                     LocalStatement::Channel(stmt) => {
                         self.kv(indent).with_id(PREFIX_CHANNEL_STMT_ID, stmt.this.0.0.index)
                             .with_s_key("LocalChannel");
                         self.kv(indent2).with_s_key("From");
                         self.write_variable(heap, stmt.from, indent3);
                         self.kv(indent2).with_s_key("To");
                         self.write_variable(heap, stmt.to, indent3);
                         self.kv(indent2).with_s_key("Next").with_disp_val(&stmt.next.index);
                     },
                     LocalStatement::Memory(stmt) => {
                         self.kv(indent).with_id(PREFIX_MEM_STMT_ID, stmt.this.0.0.index)
                             .with_s_key("LocalMemory");
                         self.kv(indent2).with_s_key("Variable");
                         self.write_variable(heap, stmt.variable, indent3);
                         self.kv(indent2).with_s_key("InitialValue");
                         self.write_expr(heap, stmt.initial_expr.upcast(), indent3);
                         self.kv(indent2).with_s_key("Next").with_disp_val(&stmt.next.index);
+                    }
@@ @@ -640,182 +644,187 @@ impl ASTWriter { @@
                 self.kv(indent2).with_s_key("Index");
                 self.write_expr(heap, expr.index, indent3);
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
             },
             Expression::Slicing(expr) => {
                 self.kv(indent).with_id(PREFIX_SLICING_EXPR_ID, expr.this.0.index)
                     .with_s_key("SlicingExpr");
                 self.kv(indent2).with_s_key("TypeIndex").with_disp_val(&expr.type_index);
                 self.kv(indent2).with_s_key("Subject");
                 self.write_expr(heap, expr.subject, indent3);
                 self.kv(indent2).with_s_key("FromIndex");
                 self.write_expr(heap, expr.from_index, indent3);
                 self.kv(indent2).with_s_key("ToIndex");
                 self.write_expr(heap, expr.to_index, indent3);
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
             },
             Expression::Select(expr) => {
                 self.kv(indent).with_id(PREFIX_SELECT_EXPR_ID, expr.this.0.index)
                     .with_s_key("SelectExpr");
                 self.kv(indent2).with_s_key("TypeIndex").with_disp_val(&expr.type_index);
                 self.kv(indent2).with_s_key("Subject");
                 self.write_expr(heap, expr.subject, indent3);
                 match &expr.kind {
                     SelectKind::StructField(field_name) => {
                         self.kv(indent2).with_s_key("StructField").with_identifier_val(field_name);
                     },
                     SelectKind::TupleMember(member_index) => {
                         self.kv(indent2).with_s_key("TupleMember").with_disp_val(member_index);
                     },
+                }
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
             },
             Expression::Literal(expr) => {
                 self.kv(indent).with_id(PREFIX_LITERAL_EXPR_ID, expr.this.0.index)
                     .with_s_key("LiteralExpr");
                 self.kv(indent2).with_s_key("TypeIndex").with_disp_val(&expr.type_index);
                 let val = self.kv(indent2).with_s_key("Value");
                 match &expr.value {
                     Literal::Null => { val.with_s_val("null"); },
                     Literal::True => { val.with_s_val("true"); },
                     Literal::False => { val.with_s_val("false"); },
                     Literal::Character(data) => { val.with_disp_val(data); },
                     Literal::Bytestring(bytes) => {
                         // Bytestrings are ASCII, so just convert back
                         let string = String::from_utf8_lossy(bytes.as_slice());
                         val.with_disp_val(&string);
                     },
                     Literal::String(data) => {
                         // Stupid hack
                         let string = String::from(data.as_str());
                         val.with_disp_val(&string);
                     },
                     Literal::Integer(data) => { val.with_debug_val(data); },
                     Literal::Struct(data) => {
                         val.with_s_val("Struct");
                         let indent4 = indent3 + 1;
                         self.kv(indent3).with_s_key("ParserType")
                             .with_custom_val(|t| write_parser_type(t, heap, &data.parser_type));
                         self.kv(indent3).with_s_key("Definition").with_disp_val(&data.definition.index);
                         for field in &data.fields {
                             self.kv(indent3).with_s_key("Field");
                             self.kv(indent4).with_s_key("Name").with_identifier_val(&field.identifier);
                             self.kv(indent4).with_s_key("Index").with_disp_val(&field.field_idx);
                             self.kv(indent4).with_s_key("ParserType");
                             self.write_expr(heap, field.value, indent4 + 1);
+                        }
                     },
                     Literal::Enum(data) => {
                         val.with_s_val("Enum");
                         self.kv(indent3).with_s_key("ParserType")
                             .with_custom_val(|t| write_parser_type(t, heap, &data.parser_type));
                         self.kv(indent3).with_s_key("Definition").with_disp_val(&data.definition.index);
                         self.kv(indent3).with_s_key("VariantIdx").with_disp_val(&data.variant_idx);
                     },
                     Literal::Union(data) => {
                         val.with_s_val("Union");
                         let indent4 = indent3 + 1;
                         self.kv(indent3).with_s_key("ParserType")
                             .with_custom_val(|t| write_parser_type(t, heap, &data.parser_type));
                         self.kv(indent3).with_s_key("Definition").with_disp_val(&data.definition.index);
                         self.kv(indent3).with_s_key("VariantIdx").with_disp_val(&data.variant_idx);
                         for value in &data.values {
                             self.kv(indent3).with_s_key("Value");
                             self.write_expr(heap, *value, indent4);
+                        }
                     },
                     Literal::Array(data) => {
                         val.with_s_val("Array");
                         let indent4 = indent3 + 1;
                         self.kv(indent3).with_s_key("Elements");
                         for expr_id in data {
                             self.write_expr(heap, *expr_id, indent4);
+                        }
                     },
                     Literal::Tuple(data) => {
                         val.with_s_val("Tuple");
                         let indent4 = indent3 + 1;
                         self.kv(indent3).with_s_key("Elements");
                         for expr_id in data {
                             self.write_expr(heap, *expr_id, indent4);
+                        }
+                    }
+                }
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
             },
             Expression::Cast(expr) => {
                 self.kv(indent).with_id(PREFIX_CAST_EXPR_ID, expr.this.0.index)
                     .with_s_key("CallExpr");
                 self.kv(indent2).with_s_key("TypeIndex").with_disp_val(&expr.type_index);
                 self.kv(indent2).with_s_key("ToType")
                     .with_custom_val(|t| write_parser_type(t, heap, &expr.to_type));
                 self.kv(indent2).with_s_key("Subject");
                 self.write_expr(heap, expr.subject, indent3);
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
+            }
             Expression::Call(expr) => {
                 self.kv(indent).with_id(PREFIX_CALL_EXPR_ID, expr.this.0.index)
                     .with_s_key("CallExpr");
                 self.kv(indent2).with_s_key("TypeIndex").with_disp_val(&expr.type_index);
                 self.kv(indent2).with_s_key("Method").with_debug_val(&expr.method);
                 if !expr.procedure.is_invalid() {
                     let definition = &heap[expr.procedure];
-                    self.kv(indent2).with_s_key("BuiltIn").with_disp_val(&definition.builtin);
+                    self.kv(indent2).with_s_key("Source").with_debug_val(&definition.source);
                     self.kv(indent2).with_s_key("Variant").with_debug_val(&definition.kind);
                     self.kv(indent2).with_s_key("MethodName").with_identifier_val(&definition.identifier);
                     self.kv(indent2).with_s_key("ParserType")
                         .with_custom_val(|t| write_parser_type(t, heap, &expr.parser_type));
+                }
                 // Arguments
                 self.kv(indent2).with_s_key("Arguments");
                 for arg_id in &expr.arguments {
                     self.write_expr(heap, *arg_id, indent3);
+                }
                 // Parent
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
             },
             Expression::Variable(expr) => {
                 self.kv(indent).with_id(PREFIX_VARIABLE_EXPR_ID, expr.this.0.index)
                     .with_s_key("VariableExpr");
                 self.kv(indent2).with_s_key("TypeIndex").with_disp_val(&expr.type_index);
                 self.kv(indent2).with_s_key("Name").with_identifier_val(&expr.identifier);
                 self.kv(indent2).with_s_key("Definition")
                     .with_opt_disp_val(expr.declaration.as_ref().map(|v| &v.index));
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
+            }
+        }
+    }
     fn write_variable(&mut self, heap: &Heap, variable_id: VariableId, indent: usize) {
         let var = &heap[variable_id];
         let indent2 = indent + 1;
         self.kv(indent).with_id(PREFIX_VARIABLE_ID, variable_id.index)
             .with_s_key("Variable");
         self.kv(indent2).with_s_key("Name").with_identifier_val(&var.identifier);
         self.kv(indent2).with_s_key("Kind").with_debug_val(&var.kind);
         self.kv(indent2).with_s_key("ParserType")
             .with_custom_val(|w| write_parser_type(w, heap, &var.parser_type));
         self.kv(indent2).with_s_key("RelativePos").with_disp_val(&var.relative_pos_in_parent);
         self.kv(indent2).with_s_key("UniqueScopeID").with_disp_val(&var.unique_id_in_scope);
+    }
     //--------------------------------------------------------------------------
     // Printing Utilities
     //--------------------------------------------------------------------------

src/protocol/eval/executor.rs

➞

Show inline comments

@@ @@ -90,97 +90,97 @@ impl Frame { @@
         self.expr_values.clear();
         for expr_id in expr_ids {
             self.expr_stack.push_back(ExprInstruction::PushValToFront);
             self.serialize_expression(heap, *expr_id);
+        }
+    }
     /// Performs depth-first serialization of expression tree. Let's not care
     /// about performance for a temporary runtime implementation
     fn serialize_expression(&mut self, heap: &Heap, id: ExpressionId) {
         self.expr_stack.push_back(ExprInstruction::EvalExpr(id));
         match &heap[id] {
             Expression::Assignment(expr) => {
                 self.serialize_expression(heap, expr.left);
                 self.serialize_expression(heap, expr.right);
             },
             Expression::Binding(expr) => {
                 self.serialize_expression(heap, expr.bound_to);
                 self.serialize_expression(heap, expr.bound_from);
             },
             Expression::Conditional(expr) => {
                 self.serialize_expression(heap, expr.test);
             },
             Expression::Binary(expr) => {
                 self.serialize_expression(heap, expr.left);
                 self.serialize_expression(heap, expr.right);
             },
             Expression::Unary(expr) => {
                 self.serialize_expression(heap, expr.expression);
             },
             Expression::Indexing(expr) => {
                 self.serialize_expression(heap, expr.index);
                 self.serialize_expression(heap, expr.subject);
             },
             Expression::Slicing(expr) => {
                 self.serialize_expression(heap, expr.from_index);
                 self.serialize_expression(heap, expr.to_index);
                 self.serialize_expression(heap, expr.subject);
             },
             Expression::Select(expr) => {
                 self.serialize_expression(heap, expr.subject);
             },
             Expression::Literal(expr) => {
                 // Here we only care about literals that have subexpressions
                 match &expr.value {
                     Literal::Null | Literal::True | Literal::False |
                     Literal::Character(_) | Literal::String(_) |
+                    Literal::Character(_) | Literal::Bytestring(_) | Literal::String(_) |
                     Literal::Integer(_) | Literal::Enum(_) => {
                         // No subexpressions
                     },
                     Literal::Struct(literal) => {
                         // Note: fields expressions are evaluated in programmer-
                         // specified order. But struct construction expects them
                         // in type-defined order. I might want to come back to
                         // this.
                         let mut _num_pushed = 0;
                         for want_field_idx in 0..literal.fields.len() {
                             for field in &literal.fields {
                                 if field.field_idx == want_field_idx {
                                     _num_pushed += 1;
                                     self.expr_stack.push_back(ExprInstruction::PushValToFront);
                                     self.serialize_expression(heap, field.value);
+                                }
+                            }
+                        }
                         debug_assert_eq!(_num_pushed, literal.fields.len())
                     },
                     Literal::Union(literal) => {
                         for value_expr_id in &literal.values {
                             self.expr_stack.push_back(ExprInstruction::PushValToFront);
                             self.serialize_expression(heap, *value_expr_id);
+                        }
                     },
                     Literal::Array(value_expr_ids) => {
                         for value_expr_id in value_expr_ids {
                             self.expr_stack.push_back(ExprInstruction::PushValToFront);
                             self.serialize_expression(heap, *value_expr_id);
+                        }
                     },
                     Literal::Tuple(value_expr_ids) => {
                         for value_expr_id in value_expr_ids {
                             self.expr_stack.push_back(ExprInstruction::PushValToFront);
                             self.serialize_expression(heap, *value_expr_id);
+                        }
+                    }
+                }
             },
             Expression::Cast(expr) => {
                 self.serialize_expression(heap, expr.subject);
+            }
             Expression::Call(expr) => {
                 for arg_expr_id in &expr.arguments {
                     self.expr_stack.push_back(ExprInstruction::PushValToFront);
                     self.serialize_expression(heap, *arg_expr_id);
+                }
@@ @@ -469,96 +469,106 @@ impl Prompt { @@
                             } // else: empty range
                             cur_frame.expr_values.push_back(match value_kind {
                                 ValueKind::Array => Value::Array(new_heap_pos),
                                 ValueKind::String => Value::String(new_heap_pos),
                                 ValueKind::Message => Value::Message(new_heap_pos),
                             });
                             // Dropping the original subject, because we don't
                             // want to drop something on the stack
                             self.store.drop_value(subject.get_heap_pos());
                         },
                         Expression::Select(expr) => {
                             let subject= cur_frame.expr_values.pop_back().unwrap();
                             let mono_data = &heap[cur_frame.definition].monomorphs[cur_frame.monomorph_index];
                             let field_idx = mono_data.expr_info[expr.type_index as usize].variant.as_select() as u32;
                             // Note: same as above: clone if value lives on expr stack, simply
                             // refer to it if it already lives on the stack/heap.
                             let (deallocate_heap_pos, value_to_push) = match subject {
                                 Value::Ref(value_ref) => {
                                     let subject = self.store.read_ref(value_ref);
                                     let subject_heap_pos = match expr.kind {
                                         SelectKind::StructField(_) => subject.as_struct(),
                                         SelectKind::TupleMember(_) => subject.as_tuple(),
                                     };
                                     (None, Value::Ref(ValueId::Heap(subject_heap_pos, field_idx)))
                                 },
                                 _ => {
                                     let subject_heap_pos = match expr.kind {
                                         SelectKind::StructField(_) => subject.as_struct(),
                                         SelectKind::TupleMember(_) => subject.as_tuple(),
                                     };
                                     let subject_indexed = Value::Ref(ValueId::Heap(subject_heap_pos, field_idx));
                                     (Some(subject_heap_pos), self.store.clone_value(subject_indexed))
                                 },
                             };
                             cur_frame.expr_values.push_back(value_to_push);
                             self.store.drop_value(deallocate_heap_pos);
                         },
                         Expression::Literal(expr) => {
                             let value = match &expr.value {
                                 Literal::Null => Value::Null,
                                 Literal::True => Value::Bool(true),
                                 Literal::False => Value::Bool(false),
                                 Literal::Character(lit_value) => Value::Char(*lit_value),
                                 Literal::Bytestring(lit_value) => {
                                     let heap_pos = self.store.alloc_heap();
                                     let values = &mut self.store.heap_regions[heap_pos as usize].values;
                                     debug_assert!(values.is_empty());
                                     values.reserve(lit_value.len());
                                     for byte in lit_value {
                                         values.push(Value::UInt8(*byte));
+                                    }
                                     Value::Array(heap_pos)
+                                }
                                 Literal::String(lit_value) => {
                                     let heap_pos = self.store.alloc_heap();
                                     let values = &mut self.store.heap_regions[heap_pos as usize].values;
                                     let value = lit_value.as_str();
                                     debug_assert!(values.is_empty());
                                     values.reserve(value.len());
                                     for character in value.as_bytes() {
                                         debug_assert!(character.is_ascii());
                                         values.push(Value::Char(*character as char));
+                                    }
                                     Value::String(heap_pos)
+                                }
                                 Literal::Integer(lit_value) => {
                                     use ConcreteTypePart as CTP;
                                     let mono_data = &heap[cur_frame.definition].monomorphs[cur_frame.monomorph_index];
                                     let type_id = mono_data.expr_info[expr.type_index as usize].type_id;
                                     let concrete_type = &types.get_monomorph(type_id).concrete_type;
                                     debug_assert_eq!(concrete_type.parts.len(), 1);
                                     match concrete_type.parts[0] {
                                         CTP::UInt8  => Value::UInt8(lit_value.unsigned_value as u8),
                                         CTP::UInt16 => Value::UInt16(lit_value.unsigned_value as u16),
                                         CTP::UInt32 => Value::UInt32(lit_value.unsigned_value as u32),
                                         CTP::UInt64 => Value::UInt64(lit_value.unsigned_value as u64),
                                         CTP::SInt8  => Value::SInt8(lit_value.unsigned_value as i8),
                                         CTP::SInt16 => Value::SInt16(lit_value.unsigned_value as i16),
                                         CTP::SInt32 => Value::SInt32(lit_value.unsigned_value as i32),
                                         CTP::SInt64 => Value::SInt64(lit_value.unsigned_value as i64),
                                         _ => unreachable!("got concrete type {:?} for integer literal at expr {:?}", concrete_type, expr_id),
+                                    }
+                                }
                                 Literal::Struct(lit_value) => {
                                     let heap_pos = transfer_expression_values_front_into_heap(
                                         cur_frame, &mut self.store, lit_value.fields.len()
                                     );
                                     Value::Struct(heap_pos)
+                                }
                                 Literal::Enum(lit_value) => {
                                     Value::Enum(lit_value.variant_idx as i64)
+                                }
                                 Literal::Union(lit_value) => {
                                     let heap_pos = transfer_expression_values_front_into_heap(
                                         cur_frame, &mut self.store, lit_value.values.len()
                                     );
                                     Value::Union(lit_value.variant_idx as i64, heap_pos)
+                                }
                                 Literal::Array(lit_value) => {
                                     let heap_pos = transfer_expression_values_front_into_heap(
@@ @@ -673,138 +683,147 @@ impl Prompt { @@
                                         let length_value = length_value.as_signed_integer();
                                         if length_value < 0 {
                                             return Err(EvalError::new_error_at_expr(
                                                 self, modules, heap, expr_id,
                                                 format!("got length '{}', can only create a message with a non-negative length", length_value)
                                             ));
+                                        }
                                         length_value as u64
                                     } else {
                                         debug_assert!(length_value.is_unsigned_integer());
                                         length_value.as_unsigned_integer()
                                     };
                                     let heap_pos = self.store.alloc_heap();
                                     let values = &mut self.store.heap_regions[heap_pos as usize].values;
                                     debug_assert!(values.is_empty());
                                     values.resize(length as usize, Value::UInt8(0));
                                     cur_frame.expr_values.push_back(Value::Message(heap_pos));
                                 },
                                 Method::Length => {
                                     let value = cur_frame.expr_values.pop_front().unwrap();
                                     let value_heap_pos = value.get_heap_pos();
                                     let value = self.store.maybe_read_ref(&value);
                                     let heap_pos = match value {
                                         Value::Array(pos) => *pos,
                                         Value::String(pos) => *pos,
                                         _ => unreachable!("length(...) on {:?}", value),
                                     };
                                     let len = self.store.heap_regions[heap_pos as usize].values.len();
                                     // TODO: @PtrInt
                                     cur_frame.expr_values.push_back(Value::UInt32(len as u32));
                                     self.store.drop_value(value_heap_pos);
                                 },
                                 Method::Assert => {
                                     let value = cur_frame.expr_values.pop_front().unwrap();
                                     let value = self.store.maybe_read_ref(&value).clone();
                                     if !value.as_bool() {
                                         return Ok(EvalContinuation::BranchInconsistent)
+                                    }
                                 },
                                 Method::Print => {
                                     // Convert the runtime-variant of a string
                                     // into an actual string.
                                     let value = cur_frame.expr_values.pop_front().unwrap();
                                     let mut is_literal_string = value.get_heap_pos().is_some();
                                     let value = self.store.maybe_read_ref(&value);
                                     let value_heap_pos = value.as_string();
                                     let elements = &self.store.heap_regions[value_heap_pos as usize].values;
                                     let mut message = String::with_capacity(elements.len());
                                     for element in elements {
                                         message.push(element.as_char());
+                                    }
                                     // Drop the heap-allocated value from the
                                     // store
                                     self.store.drop_heap_pos(value_heap_pos);
                                     if is_literal_string {
                                         self.store.drop_heap_pos(value_heap_pos);
+                                    }
                                     println!("{}", message);
                                 },
                                 Method::SelectStart => {
                                     let num_cases = self.store.maybe_read_ref(&cur_frame.expr_values.pop_front().unwrap()).as_uint32();
                                     let num_ports = self.store.maybe_read_ref(&cur_frame.expr_values.pop_front().unwrap()).as_uint32();
                                     return Ok(EvalContinuation::SelectStart(num_cases, num_ports));
                                 },
                                 Method::SelectRegisterCasePort => {
                                     let case_index = self.store.maybe_read_ref(&cur_frame.expr_values.pop_front().unwrap()).as_uint32();
                                     let port_index = self.store.maybe_read_ref(&cur_frame.expr_values.pop_front().unwrap()).as_uint32();
                                     let port_value = self.store.maybe_read_ref(&cur_frame.expr_values.pop_front().unwrap()).as_port_id();
                                     return Ok(EvalContinuation::SelectRegisterPort(case_index, port_index, port_value));
                                 },
                                 Method::SelectWait => {
                                     match ctx.performed_select_wait() {
                                         Some(select_index) => {
                                             cur_frame.expr_values.push_back(Value::UInt32(select_index));
                                         },
                                         None => {
                                             cur_frame.expr_stack.push_back(ExprInstruction::EvalExpr(expr.this.upcast()));
                                             return Ok(EvalContinuation::SelectWait)
                                         },
+                                    }
                                 },
                                 Method::ComponentRandomU32 | Method::ComponentTcpClient => {
                                     debug_assert_eq!(heap[expr.procedure].parameters.len(), cur_frame.expr_values.len());
                                     debug_assert_eq!(heap[cur_frame.position].as_new().expression, expr.this);
                                 },
                                 Method::UserComponent => {
                                     // This is actually handled by the evaluation
                                     // of the statement.
                                     debug_assert_eq!(heap[expr.procedure].parameters.len(), cur_frame.expr_values.len());
                                     debug_assert_eq!(heap[cur_frame.position].as_new().expression, expr.this)
+                                    debug_assert_eq!(heap[cur_frame.position].as_new().expression, expr.this);
                                 },
                                 Method::UserFunction => {
                                     // Push a new frame. Note that all expressions have
                                     // been pushed to the front, so they're in the order
                                     // of the definition.
                                     let num_args = expr.arguments.len();
                                     // Determine stack boundaries
                                     let cur_stack_boundary = self.store.cur_stack_boundary;
                                     let new_stack_boundary = self.store.stack.len();
                                     // Push new boundary and function arguments for new frame
                                     self.store.stack.push(Value::PrevStackBoundary(cur_stack_boundary as isize));
                                     for _ in 0..num_args {
                                         let argument = self.store.read_take_ownership(cur_frame.expr_values.pop_front().unwrap());
                                         self.store.stack.push(argument);
+                                    }
                                     // Determine the monomorph index of the function we're calling
                                     let mono_data = &heap[cur_frame.definition].monomorphs[cur_frame.monomorph_index];
                                     let (type_id, monomorph_index) = mono_data.expr_info[expr.type_index as usize].variant.as_procedure();
                                     // Push the new frame and reserve its stack size
                                     let new_frame = Frame::new(heap, expr.procedure, type_id, monomorph_index);
                                     let new_stack_size = new_frame.max_stack_size;
                                     self.frames.push(new_frame);
                                     self.store.cur_stack_boundary = new_stack_boundary;
                                     self.store.reserve_stack(new_stack_size);
                                     // To simplify the logic a little bit we will now
                                     // return and ask our caller to call us again
                                     return Ok(EvalContinuation::Stepping);
+                                }
+                            }
                         },
                         Expression::Variable(expr) => {
                             let variable = &heap[expr.declaration.unwrap()];
                             let ref_value = if expr.used_as_binding_target {
                                 Value::Binding(variable.unique_id_in_scope as StackPos)
                             } else {
                                 Value::Ref(ValueId::Stack(variable.unique_id_in_scope as StackPos))
                             };
                             cur_frame.expr_values.push_back(ref_value);
+                        }
+                    }
+                }
+            }
+        }

src/protocol/eval/value.rs

➞

Show inline comments

@@ @@ -138,110 +138,120 @@ impl Value { @@
         match self {
             Value::SInt8(v)  => *v as i64,
             Value::SInt16(v) => *v as i64,
             Value::SInt32(v) => *v as i64,
             Value::SInt64(v) => *v as i64,
             _ => unreachable!("called as_signed_integer on {:?}", self)
+        }
+    }
     /// Returns the heap position associated with the value. If the value
     /// doesn't store anything in the heap then we return `None`.
     pub(crate) fn get_heap_pos(&self) -> Option<HeapPos> {
         match self {
             Value::Message(v) => Some(*v),
             Value::String(v) => Some(*v),
             Value::Array(v) => Some(*v),
             Value::Tuple(v) => Some(*v),
             Value::Union(_, v) => Some(*v),
             Value::Struct(v) => Some(*v),
             _ => None
+        }
+    }
+}
 /// When providing arguments to a new component, or when transferring values
 /// from one component's store to a newly instantiated component, one has to
 /// transfer stack and heap values. This `ValueGroup` represents such a
 /// temporary group of values with potential heap allocations.
 ///
 /// Constructing such a ValueGroup manually requires some extra care to make
 /// sure all elements of `values` point to valid elements of `regions`.
 ///
 /// Again: this is a temporary thing, hopefully removed once we move to a
 /// bytecode interpreter.
 #[derive(Clone, Debug)]
 pub struct ValueGroup {
     pub(crate) values: Vec<Value>,
     pub(crate) regions: Vec<Vec<Value>>
+}
 impl ValueGroup {
     pub(crate) fn new_stack(values: Vec<Value>) -> Self {
         debug_assert!(values.iter().all(|v| v.get_heap_pos().is_none()));
         Self{
             values,
             regions: Vec::new(),
+        }
+    }
     pub(crate) fn from_store(store: &Store, values: &[Value]) -> Self {
         let mut group = ValueGroup{
             values: Vec::with_capacity(values.len()),
             regions: Vec::with_capacity(values.len()), // estimation
         };
         for value in values {
             let transferred = group.retrieve_value(value, store);
             group.values.push(transferred);
+        }
         group
+    }
     /// Creates a clone of the value group, but leaves the memory inside of the
     /// ValueGroup vectors allocated.
     pub(crate) fn take(&mut self) -> ValueGroup {
         let cloned = self.clone();
         self.values.clear();
         self.regions.clear();
         return cloned;
+    }
     /// Transfers a provided value from a store into a local value with its
     /// heap allocations (if any) stored in the ValueGroup. Calling this
     /// function will not store the returned value in the `values` member.
     fn retrieve_value(&mut self, value: &Value, from_store: &Store) -> Value {
         let value = from_store.maybe_read_ref(value);
         if let Some(heap_pos) = value.get_heap_pos() {
             // Value points to a heap allocation, so transfer the heap values
             // internally.
             let from_region = &from_store.heap_regions[heap_pos as usize].values;
             let mut new_region = Vec::with_capacity(from_region.len());
             for value in from_region {
                 let transferred = self.retrieve_value(value, from_store);
                 new_region.push(transferred);
+            }
             // Region is constructed, store internally and return the new value.
             let new_region_idx = self.regions.len() as HeapPos;
             self.regions.push(new_region);
             return match value {
                 Value::Message(_)    => Value::Message(new_region_idx),
                 Value::String(_)     => Value::String(new_region_idx),
                 Value::Array(_)      => Value::Array(new_region_idx),
                 Value::Tuple(_)      => Value::Tuple(new_region_idx),
                 Value::Union(tag, _) => Value::Union(*tag, new_region_idx),
                 Value::Struct(_)     => Value::Struct(new_region_idx),
                 _ => unreachable!(),
             };
         } else {
             return value.clone();
+        }
+    }
     /// Transfers the heap values and the stack values into the store. Stack
     /// values are pushed onto the Store's stack in the order in which they
     /// appear in the value group.
     pub(crate) fn into_store(self, store: &mut Store) {
         for value in &self.values {
             let transferred = self.provide_value(value, store);
             store.stack.push(transferred);
+        }
+    }
     /// Transfers the heap values into the store, but will put the stack values
     /// into the provided `VecDeque`. This is mainly used to merge `ValueGroup`
     /// instances retrieved by the code by `get` calls into the expression
     /// stack.
     pub(crate) fn into_stack(self, stack: &mut VecDeque<Value>, store: &mut Store) {

src/protocol/input_source.rs

➞

Show inline comments

@@ @@ -125,96 +125,103 @@ impl InputSource { @@
+    }
     fn set_error(&mut self, msg: &str) {
         if self.had_error.is_none() {
             self.had_error = Some(ParseError::new_error_str_at_pos(self, self.pos(), msg));
+        }
+    }
     fn get_lookup(&self) -> RwLockReadGuard<Vec<u32>> {
         // Once constructed the lookup always contains one element. We use this
         // to see if it is constructed already.
+        {
             let lookup = self.offset_lookup.read().unwrap();
             if !lookup.is_empty() {
                 return lookup;
+            }
+        }
         // Lookup was not constructed yet
         let mut lookup = self.offset_lookup.write().unwrap();
         if !lookup.is_empty() {
             // Somebody created it before we had the chance
             drop(lookup);
             let lookup = self.offset_lookup.read().unwrap();
             return lookup;
+        }
         // Build the line number (!) to offset lookup, so offset by 1. We
         // assume the entire source file is scanned (most common case) for
         // preallocation.
         lookup.reserve(self.line as usize + 2);
         lookup.push(0); // line 0: never used
         lookup.push(0); // first line: first character
         for char_idx in 0..self.input.len() {
             if self.input[char_idx] == b'\n' {
                 lookup.push(char_idx as u32 + 1);
+            }
+        }
         lookup.push(self.input.len() as u32 + 1); // for lookup_line_end, intentionally adding one character
         // Return created lookup
         drop(lookup);
         let lookup = self.offset_lookup.read().unwrap();
         return lookup;
+    }
     /// Retrieves the column associated with a line. Calling this incurs a read
     /// lock, so don't spam it in happy-path compiler code.
     pub(crate) fn get_column(&self, pos: InputPosition) -> u32 {
         let line_start = self.lookup_line_start_offset(pos.line);
         return pos.offset - line_start + 1;
+    }
     /// Retrieves offset at which line starts (right after newline)
     fn lookup_line_start_offset(&self, line_number: u32) -> u32 {
         let lookup = self.get_lookup();
         lookup[line_number as usize]
+    }
     /// Retrieves offset at which line ends (at the newline character or the
     /// preceding carriage feed for \r\n-encoded newlines)
     fn lookup_line_end_offset(&self, line_number: u32) -> u32 {
         let lookup = self.get_lookup();
         let offset = lookup[(line_number + 1) as usize] - 1;
         let offset_usize = offset as usize;
         // Compensate for newlines and a potential carriage feed. Note that the
         // end position is exclusive. So we only need to compensate for a
         // "\r\n"
         if offset_usize > 0 && offset_usize < self.input.len() && self.input[offset_usize] == b'\n' && self.input[offset_usize - 1] == b'\r' {
             offset - 1
         } else {
             offset
+        }
+    }
+}
 #[derive(Debug)]
 pub enum StatementKind {
     Info,
     Error
+}
 #[derive(Debug)]
 pub enum ContextKind {
     SingleLine,
     MultiLine,
+}
 #[derive(Debug)]
 pub struct ErrorStatement {
     pub(crate) statement_kind: StatementKind,
     pub(crate) context_kind: ContextKind,
     pub(crate) start_line: u32,
     pub(crate) start_column: u32,
     pub(crate) end_line: u32,
     pub(crate) end_column: u32,
     pub(crate) filename: String,
     pub(crate) context: String,
     pub(crate) message: String,
+}

src/protocol/mod.rs

➞

Show inline comments

 mod arena;
 pub(crate) mod eval;
 pub(crate) mod input_source;
 mod parser;
 #[cfg(test)] mod tests;
 pub(crate) mod ast;
 pub(crate) mod ast_printer;
 pub(crate) mod ast_writer;
 mod token_writer;
 use std::sync::Mutex;
 use crate::collections::{StringPool, StringRef};
 use crate::protocol::ast::*;
+pub use crate::protocol::ast::*;
 use crate::protocol::eval::*;
 use crate::protocol::input_source::*;
 use crate::protocol::parser::*;
 use crate::protocol::type_table::*;
 pub use parser::type_table::TypeId;
 /// A protocol description module
 pub struct Module {
     pub(crate) source: InputSource,
     pub(crate) root_id: RootId,
     pub(crate) name: Option<StringRef<'static>>,
+}
 /// Description of a protocol object, used to configure new connectors.
 #[repr(C)]
 pub struct ProtocolDescription {
     pub(crate) modules: Vec<Module>,
     pub(crate) heap: Heap,
     pub(crate) types: TypeTable,
     pub(crate) pool: Mutex<StringPool>,
+}
 #[derive(Debug, Clone)]
 pub(crate) struct ComponentState {
     pub(crate) prompt: Prompt,
+}
 #[derive(Debug)]
 pub enum ComponentCreationError {
     ModuleDoesntExist,
     DefinitionDoesntExist,
     DefinitionNotComponent,
     InvalidNumArguments,
     InvalidArgumentType(usize),
     UnownedPort,
     InSync,
+}
 impl ProtocolDescription {
     pub fn parse(buffer: &[u8]) -> Result<Self, String> {
         let source = InputSource::new(String::new(), Vec::from(buffer));
         let mut parser = Parser::new();
+        let mut parser = Parser::new(None)?;
         parser.feed(source).expect("failed to feed source");
         if let Err(err) = parser.parse() {
             println!("ERROR:\n{}", err);
             return Err(format!("{}", err))
+        }
         debug_assert_eq!(parser.modules.len(), 1, "only supporting one module here for now");
         let modules: Vec<Module> = parser.modules.into_iter()
             .map(|module| Module{
                 source: module.source,
                 root_id: module.root_id,
                 name: module.name.map(|(_, name)| name)
             })
             .collect();
         return Ok(ProtocolDescription {
             modules,
             heap: parser.heap,
             types: parser.type_table,
             pool: Mutex::new(parser.string_pool),
         });
+    }
     pub(crate) fn new_component(
         &self, module_name: &[u8], identifier: &[u8], arguments: ValueGroup
     ) -> Result<Prompt, ComponentCreationError> {
         // Find the module in which the definition can be found
         let module_root = self.lookup_module_root(module_name);
         if module_root.is_none() {
             return Err(ComponentCreationError::ModuleDoesntExist);
+        }
         let module_root = module_root.unwrap();
         let root = &self.heap[module_root];
-        let definition_id = root.get_definition_ident(&self.heap, identifier);
+        let definition_id = root.get_definition_by_ident(&self.heap, identifier);
         if definition_id.is_none() {
             return Err(ComponentCreationError::DefinitionDoesntExist);
+        }
         let definition_id = definition_id.unwrap();
         let ast_definition = &self.heap[definition_id];
         if !ast_definition.is_procedure() {
             return Err(ComponentCreationError::DefinitionNotComponent);
+        }
         // Make sure that the types of the provided value group matches that of
         // the expected types.
         let ast_definition = ast_definition.as_procedure();
         if !ast_definition.poly_vars.is_empty() || ast_definition.kind == ProcedureKind::Function {
             return Err(ComponentCreationError::DefinitionNotComponent);
+        }
         // - check number of arguments by retrieving the one instantiated
         //   monomorph
         let concrete_type = ConcreteType{ parts: vec![ConcreteTypePart::Component(ast_definition.this, 0)] };
-        let procedure_type_id = self.types.get_procedure_monomorph_type_id(&definition_id, &concrete_type.parts).unwrap();
+        let procedure_type_id = self.types.get_monomorph_type_id(&definition_id, &concrete_type.parts).unwrap();
         let procedure_monomorph_index = self.types.get_monomorph(procedure_type_id).variant.as_procedure().monomorph_index;
         let monomorph_info = &ast_definition.monomorphs[procedure_monomorph_index as usize];
         if monomorph_info.argument_types.len() != arguments.values.len() {
             return Err(ComponentCreationError::InvalidNumArguments);
+        }
         // - for each argument try to make sure the types match
         for arg_idx in 0..arguments.values.len() {
             let expected_type_id = monomorph_info.argument_types[arg_idx];
             let expected_type = &self.types.get_monomorph(expected_type_id).concrete_type;
             let provided_value = &arguments.values[arg_idx];
             if !self.verify_same_type(expected_type, 0, &arguments, provided_value) {
                 return Err(ComponentCreationError::InvalidArgumentType(arg_idx));
+            }
+        }
         // By now we're sure that all of the arguments are correct. So create
         // the connector.
         return Ok(Prompt::new(&self.types, &self.heap, ast_definition.this, procedure_type_id, arguments));
+    }
     /// A somewhat temporary method. Can be used by components to lookup type
     /// definitions by their name (to have their implementation somewhat
     /// resistant to changes in the standard library)
     pub(crate) fn find_type<'a>(&'a self, module_name: &[u8], type_name: &[u8]) -> Option<TypeInspector<'a>> {
         // Lookup type definition in module
         let root_id = self.lookup_module_root(module_name)?;
         let module = &self.heap[root_id];
         let definition_id = module.get_definition_by_ident(&self.heap, type_name)?;
         let definition = &self.heap[definition_id];
         // Make sure type is not polymorphic and is not a procedure
         if !definition.poly_vars().is_empty() {
             return None;
+        }
         if definition.is_procedure() {
             return None;
+        }
         // Lookup type in type table
         let type_parts = [ConcreteTypePart::Instance(definition_id, 0)];
         let type_id = self.types.get_monomorph_type_id(&definition_id, &type_parts)
             .expect("type ID for non-polymorphic type");
         let type_monomorph = self.types.get_monomorph(type_id);
         return Some(TypeInspector{
             heap: definition,
             type_table: type_monomorph
         });
+    }
     fn lookup_module_root(&self, module_name: &[u8]) -> Option<RootId> {
         for module in self.modules.iter() {
             match &module.name {
                 Some(name) => if name.as_bytes() == module_name {
                     return Some(module.root_id);
                 },
                 None => if module_name.is_empty() {
                     return Some(module.root_id);
+                }
+            }
+        }
         return None;
+    }
     fn verify_same_type(&self, expected: &ConcreteType, expected_idx: usize, arguments: &ValueGroup, argument: &Value) -> bool {
         use ConcreteTypePart as CTP;
         match &expected.parts[expected_idx] {
             CTP::Void | CTP::Message | CTP::Slice | CTP::Pointer | CTP::Function(_, _) | CTP::Component(_, _) => unreachable!(),
             CTP::Bool => if let Value::Bool(_) = argument { true } else { false },
             CTP::UInt8 => if let Value::UInt8(_) = argument { true } else { false },
             CTP::UInt16 => if let Value::UInt16(_) = argument { true } else { false },
             CTP::UInt32 => if let Value::UInt32(_) = argument { true } else { false },
             CTP::UInt64 => if let Value::UInt64(_) = argument { true } else { false },
             CTP::SInt8 => if let Value::SInt8(_) = argument { true } else { false },
             CTP::SInt16 => if let Value::SInt16(_) = argument { true } else { false },
             CTP::SInt32 => if let Value::SInt32(_) = argument { true } else { false },
             CTP::SInt64 => if let Value::SInt64(_) = argument { true } else { false },
             CTP::Character => if let Value::Char(_) = argument { true } else { false },
             CTP::String => {
                 // Match outer string type and embedded character types
                 if let Value::String(heap_pos) = argument {
                     for element in &arguments.regions[*heap_pos as usize] {
                         if let Value::Char(_) = element {} else {
                             return false;
+                        }
+                    }
                 } else {
                     return false;
+                }
                 return true;
             },
             CTP::Array => {
                 if let Value::Array(heap_pos) = argument {
                     let heap_pos = *heap_pos;
                     for element in &arguments.regions[heap_pos as usize] {
                         if !self.verify_same_type(expected, expected_idx + 1, arguments, element) {
                             return false;
+                        }
+                    }
                     return true;
                 } else {
                     return false;
+                }
             },
             CTP::Input => if let Value::Input(_) = argument { true } else { false },
             CTP::Output => if let Value::Output(_) = argument { true } else { false },
             CTP::Tuple(_) => todo!("implement full type checking on user-supplied arguments"),
             CTP::Instance(definition_id, _num_embedded) => {
                 let definition = self.types.get_base_definition(definition_id).unwrap();
                 match &definition.definition {
                     DefinedTypeVariant::Enum(definition) => {
                         if let Value::Enum(variant_value) = argument {
                             let is_valid = definition.variants.iter()
                                 .any(|v| v.value == *variant_value);
                             return is_valid;
+                        }
                     },
                     _ => todo!("implement full type checking on user-supplied arguments"),
+                }
                 return false;
             },
+        }
+    }
+}
 pub trait RunContext {
     fn performed_put(&mut self, port: PortId) -> bool;
     fn performed_get(&mut self, port: PortId) -> Option<ValueGroup>; // None if still waiting on message
     fn fires(&mut self, port: PortId) -> Option<Value>; // None if not yet branched
     fn performed_fork(&mut self) -> Option<bool>; // None if not yet forked
     fn created_channel(&mut self) -> Option<(Value, Value)>; // None if not yet prepared
     fn performed_select_wait(&mut self) -> Option<u32>; // None if not yet notified runtime of select blocker
+}
 pub struct ProtocolDescriptionBuilder {
     parser: Parser,
+}
 impl ProtocolDescriptionBuilder {
     pub fn new() -> Self {
         return Self{
             parser: Parser::new(),
+        }
     pub fn new(std_lib_dir: Option<String>) -> Result<Self, String> {
         let mut parser = Parser::new(std_lib_dir)?;
         return Ok(Self{ parser })
+    }
     pub fn add(&mut self, filename: String, buffer: Vec<u8>) -> Result<(), ParseError> {
         let input = InputSource::new(filename, buffer);
         self.parser.feed(input)?;
         return Ok(())
+    }
     pub fn compile(mut self) -> Result<ProtocolDescription, ParseError> {
         self.parser.parse()?;
         let modules: Vec<Module> = self.parser.modules.into_iter()
             .map(|module| Module{
                 source: module.source,
                 root_id: module.root_id,
                 name: module.name.map(|(_, name)| name)
             })
             .collect();
         return Ok(ProtocolDescription {
             modules,
             heap: self.parser.heap,
             types: self.parser.type_table,
             pool: Mutex::new(self.parser.string_pool),
         });
+    }
+}
 pub struct TypeInspector<'a> {
     heap: &'a Definition,
     type_table: &'a MonoType,
+}
 impl<'a> TypeInspector<'a> {
     pub fn as_union(&'a self) -> UnionTypeInspector<'a> {
         let heap = self.heap.as_union();
         let type_table = self.type_table.variant.as_union();
         return UnionTypeInspector{ heap, type_table };
+    }
+}
 pub struct UnionTypeInspector<'a> {
     heap: &'a UnionDefinition,
     type_table: &'a UnionMonomorph,
+}
 impl UnionTypeInspector<'_> {
     /// Retrieves union variant tag value.
     pub fn get_variant_tag_value(&self, variant_name: &[u8]) -> Option<i64> {
         let variant_index = self.heap.variants.iter()
             .position(|v| v.identifier.value.as_bytes() == variant_name)?;
         return Some(variant_index as i64);
+    }
+}
@@ \ No newline at end of file @@

src/protocol/parser/mod.rs

➞

Show inline comments

 #[macro_use] mod visitor;
 pub(crate) mod symbol_table;
 pub(crate) mod type_table;
 pub(crate) mod tokens;
 pub(crate) mod token_parsing;
 pub(crate) mod pass_tokenizer;
 pub(crate) mod pass_symbols;
 pub(crate) mod pass_imports;
 pub(crate) mod pass_definitions;
 pub(crate) mod pass_definitions_types;
 pub(crate) mod pass_validation_linking;
 pub(crate) mod pass_rewriting;
 pub(crate) mod pass_typing;
 pub(crate) mod pass_stack_size;
 use tokens::*;
 use crate::collections::*;
 use visitor::Visitor;
 use pass_tokenizer::PassTokenizer;
 use pass_symbols::PassSymbols;
 use pass_imports::PassImport;
 use pass_definitions::PassDefinitions;
 use pass_validation_linking::PassValidationLinking;
 use pass_typing::{PassTyping, ResolveQueue};
 use pass_rewriting::PassRewriting;
 use pass_stack_size::PassStackSize;
 use symbol_table::*;
 use type_table::*;
 use crate::protocol::ast::*;
 use crate::protocol::input_source::*;
-use crate::protocol::ast_printer::ASTWriter;
+use crate::protocol::ast_writer::ASTWriter;
 use crate::protocol::parser::type_table::PolymorphicVariable;
 use crate::protocol::token_writer::TokenWriter;
 const REOWOLF_PATH_ENV: &'static str = "REOWOLF_ROOT"; // first lookup reowolf path
 const REOWOLF_PATH_DIR: &'static str = "std"; // then try folder in current working directory
 #[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]
 pub enum ModuleCompilationPhase {
     Tokenized,              // source is tokenized
     SymbolsScanned,         // all definitions are linked to their type class
     ImportsResolved,        // all imports are added to the symbol table
     DefinitionsParsed,      // produced the AST for the entire module
     TypesAddedToTable,      // added all definitions to the type table
     ValidatedAndLinked,     // AST is traversed and has linked the required AST nodes
     Typed,                  // Type inference and checking has been performed
     Rewritten,              // Special AST nodes are rewritten into regular AST nodes
     // When we continue with the compiler:
     // StackSize
+}
 pub struct Module {
     // Buffers
     pub source: InputSource,
     pub tokens: TokenBuffer,
     // Identifiers
     pub is_compiler_file: bool, // TODO: @Hack for custom compiler-only types
     pub add_to_global_namespace: bool,
     pub root_id: RootId,
     pub name: Option<(PragmaId, StringRef<'static>)>,
     pub version: Option<(PragmaId, i64)>,
     pub phase: ModuleCompilationPhase,
+}
 pub struct TargetArch {
     pub void_type_id: TypeId,
     pub message_type_id: TypeId,
     pub bool_type_id: TypeId,
     pub uint8_type_id: TypeId,
     pub uint16_type_id: TypeId,
     pub uint32_type_id: TypeId,
     pub uint64_type_id: TypeId,
     pub sint8_type_id: TypeId,
     pub sint16_type_id: TypeId,
     pub sint32_type_id: TypeId,
     pub sint64_type_id: TypeId,
     pub char_type_id: TypeId,
     pub string_type_id: TypeId,
     pub array_type_id: TypeId,
     pub slice_type_id: TypeId,
     pub input_type_id: TypeId,
     pub output_type_id: TypeId,
     pub pointer_type_id: TypeId,
+}
 impl TargetArch {
     fn new() -> Self {
         return Self{
             void_type_id: TypeId::new_invalid(),
             bool_type_id: TypeId::new_invalid(),
             message_type_id: TypeId::new_invalid(),
             uint8_type_id: TypeId::new_invalid(),
             uint16_type_id: TypeId::new_invalid(),
             uint32_type_id: TypeId::new_invalid(),
             uint64_type_id: TypeId::new_invalid(),
             sint8_type_id: TypeId::new_invalid(),
             sint16_type_id: TypeId::new_invalid(),
             sint32_type_id: TypeId::new_invalid(),
             sint64_type_id: TypeId::new_invalid(),
             char_type_id: TypeId::new_invalid(),
             string_type_id: TypeId::new_invalid(),
             array_type_id: TypeId::new_invalid(),
             slice_type_id: TypeId::new_invalid(),
             input_type_id: TypeId::new_invalid(),
             output_type_id: TypeId::new_invalid(),
             pointer_type_id: TypeId::new_invalid(),
+        }
+    }
+}
 pub struct PassCtx<'a> {
     heap: &'a mut Heap,
     symbols: &'a mut SymbolTable,
     pool: &'a mut StringPool,
     arch: &'a TargetArch,
+}
 pub struct Parser {
     // Storage of all information created/gathered during compilation.
     pub(crate) heap: Heap,
     pub(crate) string_pool: StringPool, // Do not deallocate, holds all strings
     pub(crate) modules: Vec<Module>,
     pub(crate) symbol_table: SymbolTable,
     pub(crate) type_table: TypeTable,
     pub(crate) global_module_index: usize, // contains globals, implicitly imported everywhere
     // Compiler passes, used as little state machine that keep their memory
     // around.
     pass_tokenizer: PassTokenizer,
     pass_symbols: PassSymbols,
     pass_import: PassImport,
     pass_definitions: PassDefinitions,
     pass_validation: PassValidationLinking,
     pass_typing: PassTyping,
     pass_rewriting: PassRewriting,
     pass_stack_size: PassStackSize,
     // Compiler options
     pub write_tokens_to: Option<String>,
     pub write_ast_to: Option<String>,
     pub std_lib_dir: Option<String>,
     pub(crate) arch: TargetArch,
+}
 impl Parser {
     pub fn new() -> Self {
+    pub fn new(std_lib_dir: Option<String>) -> Result<Self, String> {
         let mut parser = Parser{
             heap: Heap::new(),
             string_pool: StringPool::new(),
             modules: Vec::new(),
             symbol_table: SymbolTable::new(),
             type_table: TypeTable::new(),
             global_module_index: 0,
             pass_tokenizer: PassTokenizer::new(),
             pass_symbols: PassSymbols::new(),
             pass_import: PassImport::new(),
             pass_definitions: PassDefinitions::new(),
             pass_validation: PassValidationLinking::new(),
             pass_typing: PassTyping::new(),
             pass_rewriting: PassRewriting::new(),
             pass_stack_size: PassStackSize::new(),
             write_tokens_to: None,
             write_ast_to: None,
             std_lib_dir,
             arch: TargetArch::new(),
         };
         parser.symbol_table.insert_scope(None, SymbolScope::Global);
         // Insert builtin types
         // TODO: At some point use correct values for size/alignment
         parser.arch.void_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Void], false, 0, 1);
         parser.arch.message_type_id = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Message], false, 24, 8);
         parser.arch.bool_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Bool], false, 1, 1);
         parser.arch.uint8_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt8], false, 1, 1);
         parser.arch.uint16_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt16], false, 2, 2);
         parser.arch.uint32_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt32], false, 4, 4);
         parser.arch.uint64_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt64], false, 8, 8);
         parser.arch.sint8_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt8], false, 1, 1);
         parser.arch.sint16_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt16], false, 2, 2);
         parser.arch.sint32_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt32], false, 4, 4);
         parser.arch.sint64_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt64], false, 8, 8);
         parser.arch.char_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Character], false, 4, 4);
         parser.arch.string_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::String], false, 24, 8);
         parser.arch.array_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Array, ConcreteTypePart::Void], true, 24, 8);
         parser.arch.slice_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Slice, ConcreteTypePart::Void], true, 16, 4);
         parser.arch.input_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Input, ConcreteTypePart::Void], true, 8, 8);
         parser.arch.output_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Output, ConcreteTypePart::Void], true, 8, 8);
         parser.arch.pointer_type_id = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Pointer, ConcreteTypePart::Void], true, 8, 8);
         // Insert builtin functions
         fn quick_type(variants: &[ParserTypeVariant]) -> ParserType {
             let mut t = ParserType{ elements: Vec::with_capacity(variants.len()), full_span: InputSpan::new() };
             for variant in variants {
                 t.elements.push(ParserTypeElement{ element_span: InputSpan::new(), variant: variant.clone() });
+            }
+            t
+        }
         // Parse standard library
         parser.feed_standard_library()?;
         use ParserTypeVariant as PTV;
         insert_builtin_function(&mut parser, "get", &["T"], |id| (
             vec![
                 ("input", quick_type(&[PTV::Input, PTV::PolymorphicArgument(id.upcast(), 0)]))
             ],
             quick_type(&[PTV::PolymorphicArgument(id.upcast(), 0)])
         ));
         insert_builtin_function(&mut parser, "put", &["T"], |id| (
             vec![
                 ("output", quick_type(&[PTV::Output, PTV::PolymorphicArgument(id.upcast(), 0)])),
                 ("value", quick_type(&[PTV::PolymorphicArgument(id.upcast(), 0)])),
             ],
             quick_type(&[PTV::Void])
         ));
         insert_builtin_function(&mut parser, "fires", &["T"], |id| (
             vec![
                 ("port", quick_type(&[PTV::InputOrOutput, PTV::PolymorphicArgument(id.upcast(), 0)]))
             ],
             quick_type(&[PTV::Bool])
         ));
         insert_builtin_function(&mut parser, "create", &["T"], |id| (
             vec![
                 ("length", quick_type(&[PTV::IntegerLike]))
             ],
             quick_type(&[PTV::ArrayLike, PTV::PolymorphicArgument(id.upcast(), 0)])
         ));
         insert_builtin_function(&mut parser, "length", &["T"], |id| (
             vec![
                 ("array", quick_type(&[PTV::ArrayLike, PTV::PolymorphicArgument(id.upcast(), 0)]))
             ],
             quick_type(&[PTV::UInt32]) // TODO: @PtrInt
         ));
         insert_builtin_function(&mut parser, "assert", &[], |_id| (
             vec![
                 ("condition", quick_type(&[PTV::Bool])),
             ],
             quick_type(&[PTV::Void])
         ));
         insert_builtin_function(&mut parser, "print", &[], |_id| (
             vec![
                 ("message", quick_type(&[PTV::String])),
             ],
             quick_type(&[PTV::Void])
         ));
         parser
         return Ok(parser)
+    }
     pub fn feed(&mut self, mut source: InputSource) -> Result<(), ParseError> {
         let mut token_buffer = TokenBuffer::new();
         self.pass_tokenizer.tokenize(&mut source, &mut token_buffer)?;
         let module = Module{
             source,
             tokens: token_buffer,
             root_id: RootId::new_invalid(),
             name: None,
             version: None,
             phase: ModuleCompilationPhase::Tokenized,
         };
         self.modules.push(module);
         Ok(())
     /// Feeds a new InputSource to the parser, which will tokenize it and store
     /// it internally for later parsing (when all modules are present). Returns
     /// the index of the new module.
     pub fn feed(&mut self, mut source: InputSource) -> Result<usize, ParseError> {
         return self.feed_internal(source, false, false);
+    }
     pub fn parse(&mut self) -> Result<(), ParseError> {
         let mut pass_ctx = PassCtx{
             heap: &mut self.heap,
             symbols: &mut self.symbol_table,
             pool: &mut self.string_pool,
             arch: &self.arch,
         };
         // Advance all modules to the phase where all symbols are scanned
         for module_idx in 0..self.modules.len() {
             self.pass_symbols.parse(&mut self.modules, module_idx, &mut pass_ctx)?;
+        }
         // With all symbols scanned, perform further compilation until we can
         // add all base types to the type table.
         for module_idx in 0..self.modules.len() {
             self.pass_import.parse(&mut self.modules, module_idx, &mut pass_ctx)?;
             self.pass_definitions.parse(&mut self.modules, module_idx, &mut pass_ctx)?;
+        }
         if let Some(filename) = &self.write_tokens_to {
             let mut writer = TokenWriter::new();
             let mut file = std::fs::File::create(std::path::Path::new(filename)).unwrap();
             writer.write(&mut file, &self.modules);
+        }
         // Add every known type to the type table
         self.type_table.build_base_types(&mut self.modules, &mut pass_ctx)?;
         // Continue compilation with the remaining phases now that the types
         // are all in the type table
         for module_idx in 0..self.modules.len() {
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_validation.visit_module(&mut ctx)?;
+        }
         // Perform typechecking on all modules
         let mut queue = ResolveQueue::new();
         for module_idx in 0..self.modules.len() {
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_typing.queue_module_definitions(&mut ctx, &mut queue);
         };
         while !queue.is_empty() {
             let top = queue.pop_front().unwrap();
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx: top.root_id.index as usize,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_typing.handle_module_definition(&mut ctx, &mut queue, top)?;
+        }
         // Rewrite nodes in tree, then prepare for execution of code
         for module_idx in 0..self.modules.len() {
             self.modules[module_idx].phase = ModuleCompilationPhase::Typed;
             let mut ctx = visitor::Ctx{
                 heap: &mut self.heap,
                 modules: &mut self.modules,
                 module_idx,
                 symbols: &mut self.symbol_table,
                 types: &mut self.type_table,
                 arch: &self.arch,
             };
             self.pass_rewriting.visit_module(&mut ctx)?;
             self.pass_stack_size.visit_module(&mut ctx)?;
+        }
         // Write out desired information
         if let Some(filename) = &self.write_ast_to {
             let mut writer = ASTWriter::new();
             let mut file = std::fs::File::create(std::path::Path::new(filename)).unwrap();
             writer.write_ast(&mut file, &self.heap);
+        }
         Ok(())
+    }
     /// Tries to find the standard library and add the files for parsing.
     fn feed_standard_library(&mut self) -> Result<(), String> {
         use std::env;
         use std::path::{Path, PathBuf};
         use std::fs;
         // Pair is (name, add_to_global_namespace)
         const FILES: [(&'static str, bool); 3] = [
             ("std.global.pdl", true),
             ("std.internet.pdl", false),
             ("std.random.pdl", false),
         ];
         // Determine base directory
         let (base_path, from_env) = if let Ok(path) = env::var(REOWOLF_PATH_ENV) {
             // Path variable is set
             (path, true)
         } else {
             let path = match self.std_lib_dir.take() {
                 Some(path) => path,
                 None => {
                     let mut path = String::with_capacity(REOWOLF_PATH_DIR.len() + 2);
                     path.push_str("./");
                     path.push_str(REOWOLF_PATH_DIR);
                     path
+                }
             };
             (path, false)
         };
         // Make sure directory exists
         let path = Path::new(&base_path);
         if !path.exists() {
             return Err(format!("std lib root directory '{}' does not exist", base_path));
+        }
         // Try to load all standard library files. We might need a more unified
         // way to do this in the future (i.e. a "std" package, containing all
         // of the modules)
         let mut file_path = PathBuf::new();
         let mut first_file = true;
         for (file, add_to_global_namespace) in FILES {
             file_path.clear();
             file_path.push(path);
             file_path.push(file);
             let source = fs::read(file_path.as_path());
             if let Err(err) = source {
                 return Err(format!(
                     "failed to read std lib file '{}' in root directory '{}', because: {}",
                     file, base_path, err
                 ));
+            }
             let source = source.unwrap();
             let input_source = InputSource::new(file.to_string(), source);
             let module_index = self.feed_internal(input_source, true, add_to_global_namespace);
             if let Err(err) = module_index {
                 // A bit of a hack, but shouldn't really happen anyway: the
                 // compiler should ship with a decent standard library (at some
                 // point)
                 return Err(format!("{}", err));
+            }
             let module_index = module_index.unwrap();
             if first_file {
                 self.global_module_index = module_index;
                 first_file = false;
+            }
+        }
         return Ok(())
+    }
     fn feed_internal(&mut self, mut source: InputSource, is_compiler_file: bool, add_to_global_namespace: bool) -> Result<usize, ParseError> {
         let mut token_buffer = TokenBuffer::new();
         self.pass_tokenizer.tokenize(&mut source, &mut token_buffer)?;
         let module = Module{
             source,
             tokens: token_buffer,
             is_compiler_file,
             add_to_global_namespace,
             root_id: RootId::new_invalid(),
             name: None,
             version: None,
             phase: ModuleCompilationPhase::Tokenized,
         };
         let module_index = self.modules.len();
         self.modules.push(module);
         return Ok(module_index);
+    }
+}
 fn insert_builtin_type(type_table: &mut TypeTable, parts: Vec<ConcreteTypePart>, has_poly_var: bool, size: usize, alignment: usize) -> TypeId {
     const POLY_VARS: [PolymorphicVariable; 1] = [PolymorphicVariable{
         identifier: Identifier::new_empty(InputSpan::new()),
         is_in_use: false,
     }];
     let concrete_type = ConcreteType{ parts };
     let poly_var = if has_poly_var {
         POLY_VARS.as_slice()
     } else {
         &[]
     };
     return type_table.add_builtin_data_type(concrete_type, poly_var, size, alignment);
+}
 // Note: args and return type need to be a function because we need to know the function ID.
 fn insert_builtin_function<T: Fn(ProcedureDefinitionId) -> (Vec<(&'static str, ParserType)>, ParserType)> (
     p: &mut Parser, func_name: &str, polymorphic: &[&str], arg_and_return_fn: T
 ) {
     // Insert into AST (to get an ID), also prepare the polymorphic variables
     // we need later for the type table
     let mut ast_poly_vars = Vec::with_capacity(polymorphic.len());
     let mut type_poly_vars = Vec::with_capacity(polymorphic.len());
     for poly_var in polymorphic {
         let identifier = Identifier{ span: InputSpan::new(), value: p.string_pool.intern(poly_var.as_bytes()) } ;
         ast_poly_vars.push(identifier.clone());
         type_poly_vars.push(PolymorphicVariable{ identifier, is_in_use: false });
+    }
     let func_ident_ref = p.string_pool.intern(func_name.as_bytes());
     let procedure_id = p.heap.alloc_procedure_definition(|this| ProcedureDefinition {
         this,
         defined_in: RootId::new_invalid(),
         builtin: true,
         kind: ProcedureKind::Function,
         span: InputSpan::new(),
         identifier: Identifier{ span: InputSpan::new(), value: func_ident_ref.clone() },
         poly_vars: ast_poly_vars,
         return_type: None,
         parameters: Vec::new(),
         scope: ScopeId::new_invalid(),
         body: BlockStatementId::new_invalid(),
         monomorphs: Vec::new(),
     });
     // Modify AST with more information about the procedure
     let (arguments, return_type) = arg_and_return_fn(procedure_id);
     let mut parameters = Vec::with_capacity(arguments.len());
     for (arg_name, arg_type) in arguments {
         let identifier = Identifier{ span: InputSpan::new(), value: p.string_pool.intern(arg_name.as_bytes()) };
         let param_id = p.heap.alloc_variable(|this| Variable{
             this,
             kind: VariableKind::Parameter,
             parser_type: arg_type.clone(),
             identifier,
             relative_pos_in_parent: 0,
             unique_id_in_scope: 0
         });
         parameters.push(param_id);
+    }
     let func = &mut p.heap[procedure_id];
     func.parameters = parameters;
     func.return_type = Some(return_type);
     // Insert into symbol table
     p.symbol_table.insert_symbol(SymbolScope::Global, Symbol{
         name: func_ident_ref,
         variant: SymbolVariant::Definition(SymbolDefinition{
             defined_in_module: RootId::new_invalid(),
             defined_in_scope: SymbolScope::Global,
             definition_span: InputSpan::new(),
             identifier_span: InputSpan::new(),
             imported_at: None,
             class: DefinitionClass::Function,
             definition_id: procedure_id.upcast(),
         })
     }).unwrap();
     // Insert into type table
     // let mut concrete_type = ConcreteType::default();
     // concrete_type.parts.push(ConcreteTypePart::Function(procedure_id, type_poly_vars.len() as u32));
     //
     // for _ in 0..type_poly_vars.len() {
     //     concrete_type.parts.push(ConcreteTypePart::Void); // doesn't matter (I hope...)
     // }
     // p.type_table.add_builtin_procedure_type(concrete_type, &type_poly_vars);
+}
@@ \ No newline at end of file @@

src/protocol/parser/pass_definitions.rs

➞

Show inline comments

 use crate::protocol::ast::*;
 use super::symbol_table::*;
 use super::{Module, ModuleCompilationPhase, PassCtx};
 use super::tokens::*;
 use super::token_parsing::*;
 use super::pass_definitions_types::*;
 use crate::protocol::input_source::{InputSource, InputPosition, InputSpan, ParseError};
 use crate::collections::*;
 /// Parses all the tokenized definitions into actual AST nodes.
 pub(crate) struct PassDefinitions {
     // State associated with the definition currently being processed
     cur_definition: DefinitionId,
     // Itty bitty parsing machines
     type_parser: ParserTypeParser,
     // Temporary buffers of various kinds
     buffer: String,
     struct_fields: ScopedBuffer<StructFieldDefinition>,
     enum_variants: ScopedBuffer<EnumVariantDefinition>,
     union_variants: ScopedBuffer<UnionVariantDefinition>,
     variables: ScopedBuffer<VariableId>,
     expressions: ScopedBuffer<ExpressionId>,
     statements: ScopedBuffer<StatementId>,
     parser_types: ScopedBuffer<ParserType>,
+}
 impl PassDefinitions {
     pub(crate) fn new() -> Self {
         Self{
             cur_definition: DefinitionId::new_invalid(),
             type_parser: ParserTypeParser::new(),
             buffer: String::with_capacity(128),
             struct_fields: ScopedBuffer::with_capacity(128),
             enum_variants: ScopedBuffer::with_capacity(128),
             union_variants: ScopedBuffer::with_capacity(128),
             variables: ScopedBuffer::with_capacity(128),
             expressions: ScopedBuffer::with_capacity(128),
             statements: ScopedBuffer::with_capacity(128),
             parser_types: ScopedBuffer::with_capacity(128),
+        }
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let module_range = &module.tokens.ranges[0];
         debug_assert_eq!(module.phase, ModuleCompilationPhase::ImportsResolved);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         // Although we only need to parse the definitions, we want to go through
         // code ranges as well such that we can throw errors if we get
         // unexpected tokens at the module level of the source.
         let mut range_idx = module_range.first_child_idx;
         loop {
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
             match cur_range.range_kind {
                 TokenRangeKind::Module => unreachable!(), // should not be reachable
                 TokenRangeKind::Pragma | TokenRangeKind::Import => {
                     // Already fully parsed, fall through and go to next range
                 },
                 TokenRangeKind::Definition | TokenRangeKind::Code => {
                     // Visit range even if it is a "code" range to provide
                     // proper error messages.
                     self.visit_range(modules, module_idx, ctx, range_idx_usize)?;
                 },
         // We iterate through the entire document. If we find a marker that has
         // been handled then we skip over it. It is important that we properly
         // parse all other tokens in the document to ensure that we throw the
         // correct kind of errors.
         let num_tokens = module.tokens.tokens.len() as u32;
         let num_markers = module.tokens.markers.len();
         let mut marker_index = 0;
         let mut first_token_index = 0;
         while first_token_index < num_tokens {
             // Seek ahead to the next marker that was already handled.
             let mut last_token_index = num_tokens;
             let mut new_first_token_index = num_tokens;
             while marker_index < num_markers {
                 let marker = &module.tokens.markers[marker_index];
                 marker_index += 1;
                 if marker.handled {
                     last_token_index = marker.first_token;
                     new_first_token_index = marker.last_token;
                     break;
+                }
+            }
             if cur_range.next_sibling_idx == NO_SIBLING {
                 break;
             } else {
                 range_idx = cur_range.next_sibling_idx;
+            }
             self.visit_token_range(modules, module_idx, ctx, first_token_index, last_token_index)?;
             first_token_index = new_first_token_index;
+        }
         modules[module_idx].phase = ModuleCompilationPhase::DefinitionsParsed;
         Ok(())
+    }
     fn visit_range(
         &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize
     fn visit_token_range(
         &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx,
         token_range_begin: u32, token_range_end: u32,
     ) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let cur_range = &module.tokens.ranges[range_idx];
         debug_assert!(cur_range.range_kind == TokenRangeKind::Definition || cur_range.range_kind == TokenRangeKind::Code);
         // Detect which definition we're parsing
-        let mut iter = module.tokens.iter_range(cur_range);
+        let mut iter = module.tokens.iter_range(token_range_begin, Some(token_range_end));
         loop {
             let next = iter.next();
             if next.is_none() {
                 return Ok(())
+            }
             // Token was not None, so peek_ident returns None if not an ident
             let ident = peek_ident(&module.source, &mut iter);
             match ident {
                 Some(KW_STRUCT) => self.visit_struct_definition(module, &mut iter, ctx)?,
                 Some(KW_ENUM) => self.visit_enum_definition(module, &mut iter, ctx)?,
                 Some(KW_UNION) => self.visit_union_definition(module, &mut iter, ctx)?,
                 Some(KW_FUNCTION) => self.visit_function_definition(module, &mut iter, ctx)?,
                 Some(KW_PRIMITIVE) | Some(KW_COMPOSITE) => self.visit_component_definition(module, &mut iter, ctx)?,
                 _ => return Err(ParseError::new_error_str_at_pos(
                     &module.source, iter.last_valid_pos(),
                     "unexpected symbol, expected a keyword marking the start of a definition"
                 )),
+            }
+        }
+    }
     fn visit_struct_definition(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<(), ParseError> {
         consume_exact_ident(&module.source, iter, KW_STRUCT)?;
         let (ident_text, _) = consume_ident(&module.source, iter)?;
         // Retrieve preallocated DefinitionId
         let module_scope = SymbolScope::Module(module.root_id);
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text)
             .unwrap().variant.as_definition().definition_id;
         self.cur_definition = definition_id;
         // Parse struct definition
         consume_polymorphic_vars_spilled(&module.source, iter, ctx)?;
         let mut fields_section = self.struct_fields.start_section();
         consume_comma_separated(
             TokenKind::OpenCurly, TokenKind::CloseCurly, &module.source, iter, ctx,
             |source, iter, ctx| {
                 let poly_vars = ctx.heap[definition_id].poly_vars();
                 let start_pos = iter.last_valid_pos();
                 let parser_type = self.type_parser.consume_parser_type(
                     iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id,
                     module_scope, false, None
+                    module_scope, false, false, None
                 )?;
                 let field = consume_ident_interned(source, iter, ctx)?;
                 Ok(StructFieldDefinition{
                     span: InputSpan::from_positions(start_pos, field.span.end),
                     field, parser_type
                 })
             },
             &mut fields_section, "a struct field", "a list of struct fields", None
         )?;
         // Transfer to preallocated definition
         let struct_def = ctx.heap[definition_id].as_struct_mut();
         struct_def.fields = fields_section.into_vec();
         Ok(())
+    }
     fn visit_enum_definition(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<(), ParseError> {
         consume_exact_ident(&module.source, iter, KW_ENUM)?;
         let (ident_text, _) = consume_ident(&module.source, iter)?;
         // Retrieve preallocated DefinitionId
         let module_scope = SymbolScope::Module(module.root_id);
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text)
             .unwrap().variant.as_definition().definition_id;
         self.cur_definition = definition_id;
         // Parse enum definition
         consume_polymorphic_vars_spilled(&module.source, iter, ctx)?;
         let mut enum_section = self.enum_variants.start_section();
         consume_comma_separated(
             TokenKind::OpenCurly, TokenKind::CloseCurly, &module.source, iter, ctx,
             |source, iter, ctx| {
                 let identifier = consume_ident_interned(source, iter, ctx)?;
                 let value = if iter.next() == Some(TokenKind::Equal) {
                     iter.consume();
                     let (variant_number, _) = consume_integer_literal(source, iter, &mut self.buffer)?;
                     EnumVariantValue::Integer(variant_number as i64) // TODO: @int
                 } else {
                     EnumVariantValue::None
                 };
                 Ok(EnumVariantDefinition{ identifier, value })
             },
             &mut enum_section, "an enum variant", "a list of enum variants", None
         )?;
         // Transfer to definition
         let enum_def = ctx.heap[definition_id].as_enum_mut();
         enum_def.variants = enum_section.into_vec();
         Ok(())
+    }
     fn visit_union_definition(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<(), ParseError> {
         consume_exact_ident(&module.source, iter, KW_UNION)?;
         let (ident_text, _) = consume_ident(&module.source, iter)?;
         // Retrieve preallocated DefinitionId
         let module_scope = SymbolScope::Module(module.root_id);
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text)
             .unwrap().variant.as_definition().definition_id;
         self.cur_definition = definition_id;
         // Parse union definition
         consume_polymorphic_vars_spilled(&module.source, iter, ctx)?;
         let mut variants_section = self.union_variants.start_section();
         consume_comma_separated(
             TokenKind::OpenCurly, TokenKind::CloseCurly, &module.source, iter, ctx,
             |source, iter, ctx| {
                 let identifier = consume_ident_interned(source, iter, ctx)?;
                 let mut close_pos = identifier.span.end;
                 let mut types_section = self.parser_types.start_section();
                 let has_embedded = maybe_consume_comma_separated(
                     TokenKind::OpenParen, TokenKind::CloseParen, source, iter, ctx,
                     |source, iter, ctx| {
                         let poly_vars = ctx.heap[definition_id].poly_vars();
                         self.type_parser.consume_parser_type(
                             iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id,
                             module_scope, false, None
+                            module_scope, false, false, None
+                        )
                     },
                     &mut types_section, "an embedded type", Some(&mut close_pos)
                 )?;
                 let value = if has_embedded {
                     types_section.into_vec()
                 } else {
                     types_section.forget();
                     Vec::new()
                 };
                 Ok(UnionVariantDefinition{
                     span: InputSpan::from_positions(identifier.span.begin, close_pos),
                     identifier,
                     value
                 })
             },
             &mut variants_section, "a union variant", "a list of union variants", None
         )?;
         // Transfer to AST
         let union_def = ctx.heap[definition_id].as_union_mut();
         union_def.variants = variants_section.into_vec();
         Ok(())
+    }
     fn visit_function_definition(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<(), ParseError> {
         // Retrieve function name
         consume_exact_ident(&module.source, iter, KW_FUNCTION)?;
         let (ident_text, _) = consume_ident(&module.source, iter)?;
         // Retrieve preallocated DefinitionId
         let module_scope = SymbolScope::Module(module.root_id);
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text)
             .unwrap().variant.as_definition().definition_id;
         self.cur_definition = definition_id;
         let allow_compiler_types = module.is_compiler_file;
         consume_polymorphic_vars_spilled(&module.source, iter, ctx)?;
         // Parse function's argument list
         let mut parameter_section = self.variables.start_section();
         consume_parameter_list(
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, module_scope, definition_id
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section,
             module_scope, definition_id, allow_compiler_types
         )?;
         let parameters = parameter_section.into_vec();
         // Consume return types
         consume_token(&module.source, iter, TokenKind::ArrowRight)?;
         let poly_vars = ctx.heap[definition_id].poly_vars();
         let parser_type = self.type_parser.consume_parser_type(
             iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, definition_id,
             module_scope, false, None
+            module_scope, false, allow_compiler_types, None
         )?;
         // Consume block and the definition's scope
         let body_id = self.consume_block_statement(module, iter, ctx)?;
         // Consume body
         let (body_id, source) = self.consume_procedure_body(module, iter, ctx, definition_id, ProcedureKind::Function)?;
         let scope_id = ctx.heap.alloc_scope(|this| Scope::new(this, ScopeAssociation::Definition(definition_id)));
         // Assign everything in the preallocated AST node
         let function = ctx.heap[definition_id].as_procedure_mut();
         function.source = source;
         function.return_type = Some(parser_type);
         function.parameters = parameters;
         function.scope = scope_id;
         function.body = body_id;
         Ok(())
+    }
     fn visit_component_definition(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<(), ParseError> {
         // Consume component variant and name
         let (_variant_text, _) = consume_any_ident(&module.source, iter)?;
         debug_assert!(_variant_text == KW_PRIMITIVE || _variant_text == KW_COMPOSITE);
         let (ident_text, _) = consume_ident(&module.source, iter)?;
         // Retrieve preallocated definition
         let module_scope = SymbolScope::Module(module.root_id);
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text)
             .unwrap().variant.as_definition().definition_id;
         self.cur_definition = definition_id;
         let allow_compiler_types = module.is_compiler_file;
         consume_polymorphic_vars_spilled(&module.source, iter, ctx)?;
         // Parse component's argument list
         let mut parameter_section = self.variables.start_section();
         consume_parameter_list(
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, module_scope, definition_id
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section,
             module_scope, definition_id, allow_compiler_types
         )?;
         let parameters = parameter_section.into_vec();
         // Consume block
         let body_id = self.consume_block_statement(module, iter, ctx)?;
         // Consume body
         let procedure_kind = ctx.heap[definition_id].as_procedure().kind;
         let (body_id, source) = self.consume_procedure_body(module, iter, ctx, definition_id, procedure_kind)?;
         let scope_id = ctx.heap.alloc_scope(|this| Scope::new(this, ScopeAssociation::Definition(definition_id)));
         // Assign everything in the AST node
         let component = ctx.heap[definition_id].as_procedure_mut();
         debug_assert!(component.return_type.is_none());
         component.source = source;
         component.parameters = parameters;
         component.scope = scope_id;
         component.body = body_id;
         Ok(())
+    }
     /// Consumes a procedure's body: either a user-defined procedure, which we
     /// parse as normal, or a builtin function, where we'll make sure we expect
     /// the particular builtin.
     ///
     /// We expect that the procedure's name is already stored in the
     /// preallocated AST node.
     fn consume_procedure_body(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx, definition_id: DefinitionId, kind: ProcedureKind
     ) -> Result<(BlockStatementId, ProcedureSource), ParseError> {
         if iter.next() == Some(TokenKind::OpenCurly) && iter.peek() == Some(TokenKind::Pragma) {
             // Consume the placeholder "{ #builtin }" tokens
             iter.consume(); // opening curly brace
             let (pragma, pragma_span) = consume_pragma(&module.source, iter)?;
             if pragma != b"#builtin" {
                 return Err(ParseError::new_error_str_at_span(
                     &module.source, pragma_span,
                     "expected a '#builtin' pragma, or a function body"
                 ));
+            }
             if iter.next() != Some(TokenKind::CloseCurly) {
                 // Just to keep the compiler writers in line ;)
                 panic!("compiler error: when using the #builtin pragma, wrap it in curly braces");
+            }
             iter.consume();
             // Retrieve module and procedure name
             assert!(module.name.is_some(), "compiler error: builtin procedure body in unnamed module");
             let (_, module_name) = module.name.as_ref().unwrap();
             let module_name = module_name.as_str();
             let definition = ctx.heap[definition_id].as_procedure();
             let procedure_name = definition.identifier.value.as_str();
             let source = match (module_name, procedure_name) {
                 ("std.global", "get") => ProcedureSource::FuncGet,
                 ("std.global", "put") => ProcedureSource::FuncPut,
                 ("std.global", "fires") => ProcedureSource::FuncFires,
                 ("std.global", "create") => ProcedureSource::FuncCreate,
                 ("std.global", "length") => ProcedureSource::FuncLength,
                 ("std.global", "assert") => ProcedureSource::FuncAssert,
                 ("std.global", "print") => ProcedureSource::FuncPrint,
                 ("std.random", "random_u32") => ProcedureSource::CompRandomU32,
                 ("std.internet", "tcp_client") => ProcedureSource::CompTcpClient,
                 _ => panic!(
                     "compiler error: unknown builtin procedure '{}' in module '{}'",
                     procedure_name, module_name
                 ),
             };
             return Ok((BlockStatementId::new_invalid(), source));
         } else {
             let body_id = self.consume_block_statement(module, iter, ctx)?;
             let source = match kind {
                 ProcedureKind::Function =>
                     ProcedureSource::FuncUserDefined,
                 ProcedureKind::Primitive | ProcedureKind::Composite =>
                     ProcedureSource::CompUserDefined,
             };
             return Ok((body_id, source))
+        }
+    }
     /// Consumes a statement and returns a boolean indicating whether it was a
     /// block or not.
     fn consume_statement(&mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx) -> Result<StatementId, ParseError> {
         let next = iter.next().expect("consume_statement has a next token");
         if next == TokenKind::OpenCurly {
             let id = self.consume_block_statement(module, iter, ctx)?;
             return Ok(id.upcast());
         } else if next == TokenKind::Ident {
             let ident = peek_ident(&module.source, iter).unwrap();
             if ident == KW_STMT_IF {
                 // Consume if statement and place end-if statement directly
                 // after it.
                 let id = self.consume_if_statement(module, iter, ctx)?;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_WHILE {
                 let id = self.consume_while_statement(module, iter, ctx)?;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_BREAK {
                 let id = self.consume_break_statement(module, iter, ctx)?;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_CONTINUE {
                 let id = self.consume_continue_statement(module, iter, ctx)?;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_SYNC {
                 let id = self.consume_synchronous_statement(module, iter, ctx)?;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_FORK {
                 let id = self.consume_fork_statement(module, iter, ctx)?;
                 let end_fork = ctx.heap.alloc_end_fork_statement(|this| EndForkStatement {
                     this,
                     start_fork: id,
                     next: StatementId::new_invalid(),
                 });
                 let fork_stmt = &mut ctx.heap[id];
                 fork_stmt.end_fork = end_fork;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_SELECT {
                 let id = self.consume_select_statement(module, iter, ctx)?;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_RETURN {
                 let id = self.consume_return_statement(module, iter, ctx)?;
                 return Ok(id.upcast());
             } else if ident == KW_STMT_GOTO {
                 let id = self.consume_goto_statement(module, iter, ctx)?;
@@ @@ -714,231 +782,230 @@ impl PassDefinitions { @@
         consume_comma_separated_until(
             TokenKind::SemiColon, &module.source, iter, ctx,
             |_source, iter, ctx| self.consume_expression(module, iter, ctx),
             &mut scoped_section, "an expression", None
         )?;
         let expressions = scoped_section.into_vec();
         if expressions.is_empty() {
             return Err(ParseError::new_error_str_at_span(&module.source, return_span, "expected at least one return value"));
         } else if expressions.len() > 1 {
             return Err(ParseError::new_error_str_at_span(&module.source, return_span, "multiple return values are not (yet) supported"))
+        }
         Ok(ctx.heap.alloc_return_statement(|this| ReturnStatement{
             this,
             span: return_span,
             expressions
         }))
+    }
     fn consume_goto_statement(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<GotoStatementId, ParseError> {
         let goto_span = consume_exact_ident(&module.source, iter, KW_STMT_GOTO)?;
         let label = consume_ident_interned(&module.source, iter, ctx)?;
         consume_token(&module.source, iter, TokenKind::SemiColon)?;
         Ok(ctx.heap.alloc_goto_statement(|this| GotoStatement{
             this,
             span: goto_span,
             label,
             target: LabeledStatementId::new_invalid(),
         }))
+    }
     fn consume_new_statement(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<NewStatementId, ParseError> {
         let new_span = consume_exact_ident(&module.source, iter, KW_STMT_NEW)?;
         let start_pos = iter.last_valid_pos();
         let expression_id = self.consume_primary_expression(module, iter, ctx)?;
         let expression = &ctx.heap[expression_id];
         let mut valid = false;
         let mut call_id = CallExpressionId::new_invalid();
         if let Expression::Call(expression) = expression {
             // Allow both components and functions, as it makes more sense to
             // check their correct use in the validation and linking pass
             if expression.method == Method::UserComponent || expression.method == Method::UserFunction {
                 call_id = expression.this;
                 valid = true;
+            }
             call_id = expression.this;
             valid = true;
+        }
         if !valid {
             return Err(ParseError::new_error_str_at_span(
                 &module.source, InputSpan::from_positions(start_pos, iter.last_valid_pos()), "expected a call expression"
             ));
+        }
         consume_token(&module.source, iter, TokenKind::SemiColon)?;
         debug_assert!(!call_id.is_invalid());
         Ok(ctx.heap.alloc_new_statement(|this| NewStatement{
             this,
             span: new_span,
             expression: call_id,
             next: StatementId::new_invalid(),
         }))
+    }
     fn consume_channel_statement(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<ChannelStatementId, ParseError> {
         // Consume channel specification
         let channel_span = consume_exact_ident(&module.source, iter, KW_STMT_CHANNEL)?;
         let (inner_port_type, end_pos) = if Some(TokenKind::OpenAngle) == iter.next() {
             // Retrieve the type of the channel, we're cheating a bit here by
             // consuming the first '<' and setting the initial angle depth to 1
             // such that our final '>' will be consumed as well.
             let angle_start_pos = iter.next_start_position();
             iter.consume();
             let definition_id = self.cur_definition;
             let poly_vars = ctx.heap[definition_id].poly_vars();
             let parser_type = self.type_parser.consume_parser_type(
                 iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars,
                 definition_id, SymbolScope::Module(module.root_id),
                 true, Some(angle_start_pos)
+                true, false, Some(angle_start_pos)
             )?;
             (parser_type.elements, parser_type.full_span.end)
         } else {
             // Assume inferred
+            (
                 vec![ParserTypeElement{
                     element_span: channel_span,
                     variant: ParserTypeVariant::Inferred
                 }],
                 channel_span.end
+            )
         };
         let from_identifier = consume_ident_interned(&module.source, iter, ctx)?;
         consume_token(&module.source, iter, TokenKind::ArrowRight)?;
         let to_identifier = consume_ident_interned(&module.source, iter, ctx)?;
         consume_token(&module.source, iter, TokenKind::SemiColon)?;
         // Construct ports
         let port_type_span = InputSpan::from_positions(channel_span.begin, end_pos);
         let port_type_len = inner_port_type.len() + 1;
         let mut from_port_type = ParserType{ elements: Vec::with_capacity(port_type_len), full_span: port_type_span };
         from_port_type.elements.push(ParserTypeElement{
             element_span: channel_span,
             variant: ParserTypeVariant::Output,
         });
         from_port_type.elements.extend_from_slice(&inner_port_type);
         let from = ctx.heap.alloc_variable(|this| Variable{
             this,
             kind: VariableKind::Local,
             identifier: from_identifier,
             parser_type: from_port_type,
             relative_pos_in_parent: 0,
             unique_id_in_scope: -1,
         });
         let mut to_port_type = ParserType{ elements: Vec::with_capacity(port_type_len), full_span: port_type_span };
         to_port_type.elements.push(ParserTypeElement{
             element_span: channel_span,
             variant: ParserTypeVariant::Input
         });
         to_port_type.elements.extend_from_slice(&inner_port_type);
         let to = ctx.heap.alloc_variable(|this|Variable{
             this,
             kind: VariableKind::Local,
             identifier: to_identifier,
             parser_type: to_port_type,
             relative_pos_in_parent: 0,
             unique_id_in_scope: -1,
         });
         // Construct the channel
         Ok(ctx.heap.alloc_channel_statement(|this| ChannelStatement{
             this,
             span: channel_span,
             from, to,
             relative_pos_in_parent: 0,
             next: StatementId::new_invalid(),
         }))
+    }
     fn consume_labeled_statement(&mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx) -> Result<LabeledStatementId, ParseError> {
         let label = consume_ident_interned(&module.source, iter, ctx)?;
         consume_token(&module.source, iter, TokenKind::Colon)?;
         let inner_stmt_id = self.consume_statement(module, iter, ctx)?;
         let stmt_id = ctx.heap.alloc_labeled_statement(|this| LabeledStatement {
             this,
             label,
             body: inner_stmt_id,
             relative_pos_in_parent: 0,
             in_sync: SynchronousStatementId::new_invalid(),
         });
         return Ok(stmt_id);
+    }
     /// Attempts to consume a memory statement (a statement along the lines of
     /// `type var_name = initial_expr`). Will return `Ok(None)` if it didn't
     /// seem like there was a memory statement, `Ok(Some(...))` if there was
     /// one, and `Err(...)` if its reasonable to assume that there was a memory
     /// statement, but we failed to parse it.
     fn maybe_consume_memory_statement_without_semicolon(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<Option<MemoryStatementId>, ParseError> {
         // This is a bit ugly. It would be nicer if we could somehow
         // consume the expression with a type hint if we do get a valid
         // type, but we don't get an identifier following it
         let iter_state = iter.save();
         let definition_id = self.cur_definition;
         let poly_vars = ctx.heap[definition_id].poly_vars();
         let parser_type = self.type_parser.consume_parser_type(
             iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars,
             definition_id, SymbolScope::Definition(definition_id), true, None
             definition_id, SymbolScope::Definition(definition_id),
             true, false, None
         );
         if let Ok(parser_type) = parser_type {
             if Some(TokenKind::Ident) == iter.next() {
                 // Assume this is a proper memory statement
                 let identifier = consume_ident_interned(&module.source, iter, ctx)?;
                 let memory_span = InputSpan::from_positions(parser_type.full_span.begin, identifier.span.end);
                 let assign_span = consume_token(&module.source, iter, TokenKind::Equal)?;
                 let initial_expr_id = self.consume_expression(module, iter, ctx)?;
                 let initial_expr_end_pos = iter.last_valid_pos();
                 // Create the AST variable
                 let local_id = ctx.heap.alloc_variable(|this| Variable{
                     this,
                     kind: VariableKind::Local,
                     identifier: identifier.clone(),
                     parser_type,
                     relative_pos_in_parent: 0,
                     unique_id_in_scope: -1,
                 });
                 // Create the initial assignment expression
                 // Note: we set the initial variable declaration here
                 let variable_expr_id = ctx.heap.alloc_variable_expression(|this| VariableExpression{
                     this,
                     identifier,
                     declaration: Some(local_id),
                     used_as_binding_target: false,
                     parent: ExpressionParent::None,
                     type_index: -1,
                 });
                 let assignment_expr_id = ctx.heap.alloc_assignment_expression(|this| AssignmentExpression{
                     this,
                     operator_span: assign_span,
                     full_span: InputSpan::from_positions(memory_span.begin, initial_expr_end_pos),
                     left: variable_expr_id.upcast(),
                     operation: AssignmentOperator::Set,
                     right: initial_expr_id,
                     parent: ExpressionParent::None,
                     type_index: -1,
                 });
                 // Put both together in the memory statement
                 let memory_stmt_id = ctx.heap.alloc_memory_statement(|this| MemoryStatement{
                     this,
                     span: memory_span,
                     variable: local_id,
@@ @@ -1411,309 +1478,322 @@ impl PassDefinitions { @@
                     scoped_section.push(expr_id);
                     let mut close_paren_pos = open_paren_pos;
                     consume_comma_separated_until(
                         TokenKind::CloseParen, &module.source, iter, ctx,
                         |_source, iter, ctx| self.consume_expression(module, iter, ctx),
                         &mut scoped_section, "an expression", Some(&mut close_paren_pos)
                     )?;
                     debug_assert!(scoped_section.len() > 1); // peeked token wasn't CloseParen, must be expression
                     let literal_id = ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                         this,
                         span: InputSpan::from_positions(open_paren_pos, close_paren_pos),
                         value: Literal::Tuple(scoped_section.into_vec()),
                         parent: ExpressionParent::None,
                         type_index: -1,
                     });
                     literal_id.upcast()
                 } else {
                     // Assume we're dealing with a normal expression
                     consume_token(&module.source, iter, TokenKind::CloseParen)?;
                     expr_id
+                }
             };
             result
         } else if next == Some(TokenKind::OpenCurly) {
             // Array literal
             let (start_pos, mut end_pos) = iter.next_positions();
             let mut scoped_section = self.expressions.start_section();
             consume_comma_separated(
                 TokenKind::OpenCurly, TokenKind::CloseCurly, &module.source, iter, ctx,
                 |_source, iter, ctx| self.consume_expression(module, iter, ctx),
                 &mut scoped_section, "an expression", "a list of expressions", Some(&mut end_pos)
             )?;
             ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                 this,
                 span: InputSpan::from_positions(start_pos, end_pos),
                 value: Literal::Array(scoped_section.into_vec()),
                 parent: ExpressionParent::None,
                 type_index: -1,
             }).upcast()
         } else if next == Some(TokenKind::Integer) {
             let (literal, span) = consume_integer_literal(&module.source, iter, &mut self.buffer)?;
             ctx.heap.alloc_literal_expression(|this| LiteralExpression {
                 this,
                 span,
                 value: Literal::Integer(LiteralInteger { unsigned_value: literal, negated: false }),
                 parent: ExpressionParent::None,
                 type_index: -1,
             }).upcast()
         } else if next == Some(TokenKind::Bytestring) {
             let span = consume_bytestring_literal(&module.source, iter, &mut self.buffer)?;
             let mut bytes = Vec::with_capacity(self.buffer.len());
             for byte in self.buffer.as_bytes().iter().copied() {
                 bytes.push(byte);
+            }
             ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                 this, span,
-                value: Literal::Integer(LiteralInteger{ unsigned_value: literal, negated: false }),
+                value: Literal::Bytestring(bytes),
                 parent: ExpressionParent::None,
-                type_index: -1,
                 type_index: -1
             }).upcast()
         } else if next == Some(TokenKind::String) {
             let span = consume_string_literal(&module.source, iter, &mut self.buffer)?;
             let interned = ctx.pool.intern(self.buffer.as_bytes());
             ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                 this, span,
                 value: Literal::String(interned),
                 parent: ExpressionParent::None,
                 type_index: -1,
             }).upcast()
         } else if next == Some(TokenKind::Character) {
             let (character, span) = consume_character_literal(&module.source, iter)?;
             ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                 this, span,
                 value: Literal::Character(character),
                 parent: ExpressionParent::None,
                 type_index: -1,
             }).upcast()
         } else if next == Some(TokenKind::Ident) {
             // May be a variable, a type instantiation or a function call. If we
             // have a single identifier that we cannot find in the type table
             // then we're going to assume that we're dealing with a variable.
             let ident_span = iter.next_span();
             let ident_text = module.source.section_at_span(ident_span);
             let symbol = ctx.symbols.get_symbol_by_name(SymbolScope::Module(module.root_id), ident_text);
             if symbol.is_some() {
                 // The first bit looked like a symbol, so we're going to follow
                 // that all the way through, assume we arrive at some kind of
                 // function call or type instantiation
                 use ParserTypeVariant as PTV;
                 let symbol_scope = SymbolScope::Definition(self.cur_definition);
                 let poly_vars = ctx.heap[self.cur_definition].poly_vars();
                 let parser_type = self.type_parser.consume_parser_type(
                     iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, self.cur_definition,
                     symbol_scope, true, None
+                    symbol_scope, true, false, None
                 )?;
                 debug_assert!(!parser_type.elements.is_empty());
                 match parser_type.elements[0].variant {
                     PTV::Definition(target_definition_id, _) => {
                         let definition = &ctx.heap[target_definition_id];
                         match definition {
                             Definition::Struct(_) => {
                                 // Struct literal
                                 let mut last_token = iter.last_valid_pos();
                                 let mut struct_fields = Vec::new();
                                 consume_comma_separated(
                                     TokenKind::OpenCurly, TokenKind::CloseCurly, &module.source, iter, ctx,
                                     |source, iter, ctx| {
                                         let identifier = consume_ident_interned(source, iter, ctx)?;
                                         consume_token(source, iter, TokenKind::Colon)?;
                                         let value = self.consume_expression(module, iter, ctx)?;
                                         Ok(LiteralStructField{ identifier, value, field_idx: 0 })
                                     },
                                     &mut struct_fields, "a struct field", "a list of struct fields", Some(&mut last_token)
                                 )?;
                                 ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                                     this,
                                     span: InputSpan::from_positions(ident_span.begin, last_token),
                                     value: Literal::Struct(LiteralStruct{
                                         parser_type,
                                         fields: struct_fields,
                                         definition: target_definition_id,
                                     }),
                                     parent: ExpressionParent::None,
                                     type_index: -1,
                                 }).upcast()
                             },
                             Definition::Enum(_) => {
                                 // Enum literal: consume the variant
                                 consume_token(&module.source, iter, TokenKind::ColonColon)?;
                                 let variant = consume_ident_interned(&module.source, iter, ctx)?;
                                 ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                                     this,
                                     span: InputSpan::from_positions(ident_span.begin, variant.span.end),
                                     value: Literal::Enum(LiteralEnum{
                                         parser_type,
                                         variant,
                                         definition: target_definition_id,
                                         variant_idx: 0
                                     }),
                                     parent: ExpressionParent::None,
                                     type_index: -1,
                                 }).upcast()
                             },
                             Definition::Union(_) => {
                                 // Union literal: consume the variant
                                 consume_token(&module.source, iter, TokenKind::ColonColon)?;
                                 let variant = consume_ident_interned(&module.source, iter, ctx)?;
                                 // Consume any possible embedded values
                                 let mut end_pos = variant.span.end;
                                 let values = if Some(TokenKind::OpenParen) == iter.next() {
                                     self.consume_expression_list(module, iter, ctx, Some(&mut end_pos))?
                                 } else {
                                     Vec::new()
                                 };
                                 ctx.heap.alloc_literal_expression(|this| LiteralExpression{
                                     this,
                                     span: InputSpan::from_positions(ident_span.begin, end_pos),
                                     value: Literal::Union(LiteralUnion{
                                         parser_type, variant, values,
                                         definition: target_definition_id,
                                         variant_idx: 0,
                                     }),
                                     parent: ExpressionParent::None,
                                     type_index: -1,
                                 }).upcast()
                             },
                             Definition::Procedure(proc_def) => {
                                 // Check whether it is a builtin function
                                 // TODO: Once we start generating bytecode this is unnecessary
                                 let procedure_id = proc_def.this;
                                 let method = if proc_def.builtin {
                                     match proc_def.identifier.value.as_bytes() {
                                         KW_FUNC_GET => Method::Get,
                                         KW_FUNC_PUT => Method::Put,
                                         KW_FUNC_FIRES => Method::Fires,
                                         KW_FUNC_CREATE => Method::Create,
                                         KW_FUNC_LENGTH => Method::Length,
                                         KW_FUNC_ASSERT => Method::Assert,
                                         KW_FUNC_PRINT => Method::Print,
                                         _ => unreachable!(),
+                                    }
                                 } else if proc_def.kind == ProcedureKind::Function {
                                     Method::UserFunction
                                 } else {
                                     Method::UserComponent
                                 let method = match proc_def.source {
                                     ProcedureSource::FuncUserDefined => Method::UserFunction,
                                     ProcedureSource::CompUserDefined => Method::UserComponent,
                                     ProcedureSource::FuncGet => Method::Get,
                                     ProcedureSource::FuncPut => Method::Put,
                                     ProcedureSource::FuncFires => Method::Fires,
                                     ProcedureSource::FuncCreate => Method::Create,
                                     ProcedureSource::FuncLength => Method::Length,
                                     ProcedureSource::FuncAssert => Method::Assert,
                                     ProcedureSource::FuncPrint => Method::Print,
                                     ProcedureSource::CompRandomU32 => Method::ComponentRandomU32,
                                     ProcedureSource::CompTcpClient => Method::ComponentTcpClient,
                                     _ => todo!("other procedure sources"),
                                 };
                                 // Function call: consume the arguments
                                 let func_span = parser_type.full_span;
                                 let mut full_span = func_span;
                                 let arguments = self.consume_expression_list(
                                     module, iter, ctx, Some(&mut full_span.end)
                                 )?;
                                 ctx.heap.alloc_call_expression(|this| CallExpression{
                                     this, func_span, full_span, parser_type, method, arguments,
                                     procedure: procedure_id,
                                     parent: ExpressionParent::None,
                                     type_index: -1,
                                 }).upcast()
+                            }
+                        }
                     },
                     _ => {
                         return Err(ParseError::new_error_str_at_span(
                             &module.source, parser_type.full_span, "unexpected type in expression"
                         ))
+                    }
+                }
             } else {
                 // Check for builtin keywords or builtin functions
                 if ident_text == KW_LIT_NULL || ident_text == KW_LIT_TRUE || ident_text == KW_LIT_FALSE {
                     iter.consume();
                     // Parse builtin literal
                     let value = match ident_text {
                         KW_LIT_NULL => Literal::Null,
                         KW_LIT_TRUE => Literal::True,
                         KW_LIT_FALSE => Literal::False,
                         _ => unreachable!(),
                     };
                     ctx.heap.alloc_literal_expression(|this| LiteralExpression {
                         this,
                         span: ident_span,
                         value,
                         parent: ExpressionParent::None,
                         type_index: -1,
                     }).upcast()
                 } else if ident_text == KW_LET {
                     // Binding expression
                     let operator_span = iter.next_span();
                     iter.consume();
                     let bound_to = self.consume_prefix_expression(module, iter, ctx)?;
                     consume_token(&module.source, iter, TokenKind::Equal)?;
                     let bound_from = self.consume_prefix_expression(module, iter, ctx)?;
                     let full_span = InputSpan::from_positions(
                         operator_span.begin, ctx.heap[bound_from].full_span().end,
                     );
                     ctx.heap.alloc_binding_expression(|this| BindingExpression{
                         this, operator_span, full_span, bound_to, bound_from,
                         parent: ExpressionParent::None,
                         type_index: -1,
                     }).upcast()
                 } else if ident_text == KW_CAST {
                     // Casting expression
                     iter.consume();
                     let to_type = if Some(TokenKind::OpenAngle) == iter.next() {
                         let angle_start_pos = iter.next_start_position();
                         iter.consume();
                         let definition_id = self.cur_definition;
                         let poly_vars = ctx.heap[definition_id].poly_vars();
                         self.type_parser.consume_parser_type(
                             iter, &ctx.heap, &module.source, &ctx.symbols,
                             poly_vars, definition_id, SymbolScope::Module(module.root_id),
                             true, Some(angle_start_pos)
+                            true, false, Some(angle_start_pos)
                         )?
                     } else {
                         // Automatic casting with inferred target type
                         ParserType{
                             elements: vec![ParserTypeElement{
                                 element_span: ident_span,
                                 variant: ParserTypeVariant::Inferred,
                             }],
                             full_span: ident_span
+                        }
                     };
                     consume_token(&module.source, iter, TokenKind::OpenParen)?;
                     let subject = self.consume_expression(module, iter, ctx)?;
                     let mut full_span = iter.next_span();
                     full_span.begin = to_type.full_span.begin;
                     consume_token(&module.source, iter, TokenKind::CloseParen)?;
                     ctx.heap.alloc_cast_expression(|this| CastExpression{
                         this,
                         cast_span: to_type.full_span,
                         full_span, to_type, subject,
                         parent: ExpressionParent::None,
                         type_index: -1,
                     }).upcast()
                 } else {
                     // Not a builtin literal, but also not a known type. So we
                     // assume it is a variable expression. Although if we do,
                     // then if a programmer mistyped a struct/function name the
                     // error messages will be rather cryptic. For polymorphic
                     // arguments we can't really do anything at all (because it
                     // uses the '<' token). In the other cases we try to provide
                     // a better error message.
                     iter.consume();
                     let next = iter.next();
                     if Some(TokenKind::ColonColon) == next {
                         return Err(ParseError::new_error_str_at_span(&module.source, ident_span, "unknown identifier"));
                     } else if Some(TokenKind::OpenParen) == next {
                         return Err(ParseError::new_error_str_at_span(
                             &module.source, ident_span,
                             "unknown identifier, did you mistype a union variant's, component's, or function's name?"
                         ));
                     } else if Some(TokenKind::OpenCurly) == next {
                         return Err(ParseError::new_error_str_at_span(
                             &module.source, ident_span,
                             "unknown identifier, did you mistype a struct type's name?"
                         ))
+                    }
@@ @@ -1759,72 +1839,72 @@ impl PassDefinitions { @@
             let left = result;
             let right = higher_precedence_fn(self, module, iter, ctx)?;
             let full_span = InputSpan::from_positions(
                 ctx.heap[left].full_span().begin,
                 ctx.heap[right].full_span().end,
             );
             result = ctx.heap.alloc_binary_expression(|this| BinaryExpression{
                 this, operator_span, full_span, left, operation, right,
                 parent: ExpressionParent::None,
                 type_index: -1,
             }).upcast();
+        }
         Ok(result)
+    }
     #[inline]
     fn consume_expression_list(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx, end_pos: Option<&mut InputPosition>
     ) -> Result<Vec<ExpressionId>, ParseError> {
         let mut section = self.expressions.start_section();
         consume_comma_separated(
             TokenKind::OpenParen, TokenKind::CloseParen, &module.source, iter, ctx,
             |_source, iter, ctx| self.consume_expression(module, iter, ctx),
             &mut section, "an expression", "a list of expressions", end_pos
         )?;
         Ok(section.into_vec())
+    }
+}
 /// Consumes polymorphic variables and throws them on the floor.
 fn consume_polymorphic_vars_spilled(source: &InputSource, iter: &mut TokenIter, _ctx: &mut PassCtx) -> Result<(), ParseError> {
     maybe_consume_comma_separated_spilled(
         TokenKind::OpenAngle, TokenKind::CloseAngle, source, iter, _ctx,
         |source, iter, _ctx| {
             consume_ident(source, iter)?;
             Ok(())
         }, "a polymorphic variable"
     )?;
     Ok(())
+}
 /// Consumes the parameter list to functions/components
 fn consume_parameter_list(
     parser: &mut ParserTypeParser, source: &InputSource, iter: &mut TokenIter,
     ctx: &mut PassCtx, target: &mut ScopedSection<VariableId>,
     scope: SymbolScope, definition_id: DefinitionId
+    scope: SymbolScope, definition_id: DefinitionId, allow_compiler_types: bool
 ) -> Result<(), ParseError> {
     consume_comma_separated(
         TokenKind::OpenParen, TokenKind::CloseParen, source, iter, ctx,
         |source, iter, ctx| {
             let poly_vars = ctx.heap[definition_id].poly_vars(); // Rust being rust, multiple lookups
             let parser_type = parser.consume_parser_type(
                 iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id,
                 scope, false, None
+                scope, false, allow_compiler_types, None
             )?;
             let identifier = consume_ident_interned(source, iter, ctx)?;
             let parameter_id = ctx.heap.alloc_variable(|this| Variable{
                 this,
                 kind: VariableKind::Parameter,
                 parser_type,
                 identifier,
                 relative_pos_in_parent: 0,
                 unique_id_in_scope: -1,
             });
             Ok(parameter_id)
         },
         target, "a parameter", "a parameter list", None
+    )
+}
@@ \ No newline at end of file @@

src/protocol/parser/pass_definitions_types.rs

➞

Show inline comments

@@ @@ -14,220 +14,225 @@ enum DepthKind { @@
+}
 #[derive(Debug)]
 struct DepthElement {
     kind: DepthKind,
     entry_index: u32, // in `entries` array of parser
     pos: InputPosition,
+}
 /// Current parsing state, for documentation's sake: types may be named, or may
 /// be tuples. Named types may have polymorphic arguments (if the type
 /// declaration allows) and a type may be turned into an array of that type by
 /// postfixing a "[]".
 #[derive(Debug)]
 enum ParseState {
     TypeMaybePolyArgs,  // just parsed a type, might have poly arguments
     TypeNeverPolyArgs,  // just parsed a type that cannot have poly arguments
     PolyArgStart,       // just opened a polymorphic argument list
     TupleStart,         // just opened a tuple list
     ParsedComma,        // just had a comma
+}
 /// Parsers tokens into `ParserType` instances (yes, the name of the struct is
 /// silly). Essentially a little state machine with its own temporary storage.
 #[derive(Debug)]
 pub(crate) struct ParserTypeParser {
     entries: Vec<Entry>,
     depths: Vec<DepthElement>,
     parse_state: ParseState,
     first_pos: InputPosition,
     last_pos: InputPosition,
+}
 impl ParserTypeParser {
     pub(crate) fn new() -> Self {
         return Self{
             entries: Vec::with_capacity(16),
             depths: Vec::with_capacity(16),
             parse_state: ParseState::TypeMaybePolyArgs,
             first_pos: InputPosition{ line: 0, offset: 0 },
             last_pos: InputPosition{ line: 0, offset: 0 }
+        }
+    }
     pub(crate) fn consume_parser_type(
         &mut self, iter: &mut TokenIter, heap: &Heap, source: &InputSource,
         symbols: &SymbolTable, poly_vars: &[Identifier],
         wrapping_definition: DefinitionId, cur_scope: SymbolScope,
         allow_inference: bool, inside_angular_bracket: Option<InputPosition>,
         allow_inference: bool, allow_compiler_types: bool,
         inside_angular_bracket: Option<InputPosition>,
     ) -> Result<ParserType, ParseError> {
         // Prepare
         self.entries.clear();
         self.depths.clear();
         // Setup processing
         if let Some(bracket_pos) = inside_angular_bracket {
             self.push_depth(DepthKind::PolyArgs, u32::MAX, bracket_pos);
+        }
         let initial_state = match iter.next() {
             Some(TokenKind::Ident) => {
+            Some(TokenKind::Ident) | Some(TokenKind::Pragma) => {
                 let element = Self::consume_parser_type_element(
                     iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference
                     iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                     allow_inference, allow_compiler_types
                 )?;
                 self.first_pos = element.element_span.begin;
                 self.last_pos = element.element_span.end;
                 self.entries.push(Entry{
                     element,
                     depth: self.cur_depth(),
                 });
                 // Due to the nature of the subsequent type parsing algorithm,
                 // we check the opening polymorphic argument list paren here.
                 if let Some(TokenKind::OpenAngle) = iter.next() {
                     self.consume_open_angle(iter);
                     ParseState::PolyArgStart
                 } else {
                     ParseState::TypeMaybePolyArgs
+                }
             },
             Some(TokenKind::OpenParen) => {
                 let tuple_start_pos = iter.next_start_position();
                 self.first_pos = tuple_start_pos; // last pos will be set later, this is a tuple
                 let tuple_entry_index = self.entries.len() as u32;
                 let tuple_depth = self.cur_depth();
                 self.push_depth(DepthKind::Tuple, tuple_entry_index, tuple_start_pos);
                 self.entries.push(Entry{
                     element: ParserTypeElement{
                         element_span: InputSpan::from_positions(tuple_start_pos, tuple_start_pos),
                         variant: ParserTypeVariant::Tuple(0),
                     },
                     depth: tuple_depth,
                 });
                 iter.consume();
                 ParseState::TupleStart
             },
             _ => return Err(ParseError::new_error_str_at_pos(source, iter.last_valid_pos(), "expected a type")),
         };
         self.parse_state = initial_state;
         // Depth stack and entries are initialized, continue until depth stack
         // is empty, or until an unexpected set of tokens is encountered
         while !self.depths.is_empty() {
             let next = iter.next();
             match self.parse_state {
                 ParseState::TypeMaybePolyArgs => {
                     // Allowed tokens: , < > >> ) [
                     match next {
                         Some(TokenKind::Comma) => self.consume_comma(iter),
                         Some(TokenKind::OpenAngle) => self.consume_open_angle(iter),
                         Some(TokenKind::CloseAngle) => self.consume_close_angle(source, iter)?,
                         Some(TokenKind::ShiftRight) => self.consume_double_close_angle(source, iter)?,
                         Some(TokenKind::CloseParen) => self.consume_close_paren(source, iter)?,
                         Some(TokenKind::OpenSquare) => self.consume_square_parens(source, iter)?,
                         _ => return Err(ParseError::new_error_str_at_pos(
                             source, iter.last_valid_pos(),
                             "unexpected token: expected ',', '<', '>', '<<', ')' or '['"
                         )),
+                    }
                 },
                 ParseState::TypeNeverPolyArgs => {
                     // Allowed tokens: , > >> ) [
                     match next {
                         Some(TokenKind::Comma) => self.consume_comma(iter),
                         Some(TokenKind::CloseAngle) => self.consume_close_angle(source, iter)?,
                         Some(TokenKind::ShiftRight) => self.consume_double_close_angle(source, iter)?,
                         Some(TokenKind::CloseParen) => self.consume_close_paren(source, iter)?,
                         Some(TokenKind::OpenSquare) => self.consume_square_parens(source, iter)?,
                         _ => return Err(ParseError::new_error_str_at_pos(
                             source, iter.last_valid_pos(),
                             "unexpected token: expected ',', '>', '>>', ')' or '['"
                         )),
+                    }
                 },
                 ParseState::PolyArgStart => {
                     // Allowed tokens: ident (
                     match next {
                         Some(TokenKind::Ident) => self.consume_type_idents(
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference, iter
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                             allow_inference, allow_compiler_types, iter
                         )?,
                         Some(TokenKind::OpenParen) => self.consume_open_paren(iter),
                         _ => return Err(ParseError::new_error_str_at_pos(
                             source, iter.last_valid_pos(),
                             "unexpected token: expected typename or '('"
                         )),
+                    }
                 },
                 ParseState::TupleStart => {
                     // Allowed tokens: ident ( )
                     // We'll strip the nested tuple later in this function
                     match next {
                         Some(TokenKind::Ident) => self.consume_type_idents(
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference, iter
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                             allow_inference, allow_compiler_types, iter
                         )?,
                         Some(TokenKind::OpenParen) => self.consume_open_paren(iter),
                         Some(TokenKind::CloseParen) => self.consume_close_paren(source, iter)?,
                         _ => return Err(ParseError::new_error_str_at_pos(
                             source, iter.last_valid_pos(),
                             "unexpected token: expected typename or ')'"
                         )),
+                    }
                 },
                 ParseState::ParsedComma => {
                     // Allowed tokens: ident ( > >> )
                     match next {
                         Some(TokenKind::Ident) => self.consume_type_idents(
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference, iter
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                             allow_inference, allow_compiler_types, iter
                         )?,
                         Some(TokenKind::OpenParen) => self.consume_open_paren(iter),
                         Some(TokenKind::CloseAngle) => self.consume_close_angle(source, iter)?,
                         Some(TokenKind::ShiftRight) => self.consume_double_close_angle(source, iter)?,
                         Some(TokenKind::CloseParen) => self.consume_close_paren(source, iter)?,
                         _ => return Err(ParseError::new_error_str_at_pos(
                             source, iter.last_valid_pos(),
                             "unexpected token: expected typename, '(', '>', '>>' or ')'"
                         ))
+                    }
+                }
+            }
+        }
         // If here then we have found the correct number of closing braces.
         // However we might still have any number of array postfixed
         if inside_angular_bracket.is_none() {
             while Some(TokenKind::OpenSquare) == iter.next() {
                 self.consume_square_parens(source, iter)?;
+            }
+        }
         // Type should be completed. But we still need to check the polymorphic
         // arguments and strip tuples with just one embedded type.
         let num_entries = self.entries.len();
         for el_index in 0..num_entries {
             let cur_element = &self.entries[el_index];
             // Peek ahead to see how many embedded types we have
             let mut encountered_embedded = 0;
             for peek_index in el_index + 1..num_entries {
                 let peek_element = &self.entries[peek_index];
                 if peek_element.depth == cur_element.depth + 1 {
                     encountered_embedded += 1;
                 } else if peek_element.depth <= cur_element.depth {
                     break;
+                }
+            }
             // If we're dealing with a tuple then we don't need to determine if
             // the number of embedded types is correct, we simply need to set it
             // to whatever what was encountered.
             if let ParserTypeVariant::Tuple(_) = cur_element.element.variant {
                 self.entries[el_index].element.variant = ParserTypeVariant::Tuple(encountered_embedded);
             } else {
                 let expected_embedded = cur_element.element.variant.num_embedded() as u32;
                 if expected_embedded != encountered_embedded {
@@ @@ -243,100 +248,102 @@ impl ParserTypeParser { @@
                         // Insert missing types
                         let inserted_span = cur_element.element.element_span;
                         let inserted_depth = cur_element.depth + 1;
                         self.entries.reserve(expected_embedded as usize);
                         for _ in 0..expected_embedded {
                             self.entries.insert(el_index + 1, Entry {
                                 element: ParserTypeElement {
                                     element_span: inserted_span,
                                     variant: ParserTypeVariant::Inferred,
                                 },
                                 depth: inserted_depth,
                             });
+                        }
                     } else {
                         // Mismatch in number of embedded types
                         return Err(Self::construct_poly_arg_mismatch_error(
                             source, cur_element.element.element_span, allow_inference,
                             expected_embedded, encountered_embedded
                         ));
+                    }
+                }
+            }
+        }
         // Convert the results from parsing into the `ParserType`
         let mut elements = Vec::with_capacity(self.entries.len());
         debug_assert!(!self.entries.is_empty());
         for entry in self.entries.drain(..) {
             if ParserTypeVariant::Tuple(1) == entry.element.variant {
                 // We strip these ones
             } else {
                 elements.push(entry.element);
+            }
+        }
         return Ok(ParserType{
             elements,
             full_span: InputSpan::from_positions(self.first_pos, self.last_pos),
         });
+    }
     // --- Parsing Utilities
     #[inline]
     fn consume_type_idents(
         &mut self, source: &InputSource, heap: &Heap, symbols: &SymbolTable,
         wrapping_definition: DefinitionId, poly_vars: &[Identifier],
         cur_scope: SymbolScope, allow_inference: bool, iter: &mut TokenIter
         cur_scope: SymbolScope, allow_inference: bool, allow_compiler_types: bool,
         iter: &mut TokenIter
     ) -> Result<(), ParseError> {
         let element = Self::consume_parser_type_element(
             iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference
             iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
             allow_inference, allow_compiler_types
         )?;
         let depth = self.cur_depth();
         self.last_pos = element.element_span.end;
         self.entries.push(Entry{ element, depth });
         self.parse_state = ParseState::TypeMaybePolyArgs;
         return Ok(());
+    }
     #[inline]
     fn consume_open_angle(&mut self, iter: &mut TokenIter) {
         // Note: open angular bracket is only consumed when we just parsed an
         //  ident-based type. So the last element of the `entries` array is the
         //  one that this angular bracket starts the polymorphic arguments for.
         let angle_start_pos = iter.next_start_position();
         let entry_index = (self.entries.len() - 1) as u32;
         self.push_depth(DepthKind::PolyArgs, entry_index, angle_start_pos);
         self.parse_state = ParseState::PolyArgStart;
         iter.consume();
+    }
     #[inline]
     fn consume_close_angle(&mut self, source: &InputSource, iter: &mut TokenIter) -> Result<(), ParseError> {
         let (angle_start_pos, angle_end_pos) = iter.next_positions();
         self.last_pos = angle_end_pos;
         self.pop_depth(source, DepthKind::PolyArgs, angle_start_pos)?;
         self.parse_state = ParseState::TypeNeverPolyArgs;
         iter.consume();
         return Ok(())
+    }
     #[inline]
     fn consume_double_close_angle(&mut self, source: &InputSource, iter: &mut TokenIter) -> Result<(), ParseError> {
         let (angle_start_pos, angle_end_pos) = iter.next_positions();
         self.last_pos = angle_end_pos;
         self.pop_depth(source, DepthKind::PolyArgs, angle_start_pos)?; // first '>' in '>>'.
         self.pop_depth(source, DepthKind::PolyArgs, angle_start_pos.with_offset(1))?; // second '>' in '>>'
         self.parse_state = ParseState::TypeNeverPolyArgs;
         iter.consume(); // consume once, '>>' is one token
         return Ok(())
+    }
     #[inline]
     fn consume_open_paren(&mut self, iter: &mut TokenIter) {
@@ @@ -383,101 +390,125 @@ impl ParserTypeParser { @@
         if iter.next() != Some(TokenKind::CloseSquare) {
             return Err(ParseError::new_error_str_at_pos(
                 source, iter.last_valid_pos(),
                 "unexpected token: expected ']'"
             ));
+        }
         let (_, array_end_pos) = iter.next_positions();
         let array_span = InputSpan::from_positions(array_start_pos, array_end_pos);
         self.last_pos = array_end_pos;
         iter.consume();
         // In the language we put the array specification after a type, in the
         // type tree we need to make the array type the parent, so:
         let insert_depth = self.cur_depth();
         let insert_at = self.entries.iter().rposition(|e| e.depth == insert_depth).unwrap();
         let num_embedded = self.entries[insert_at].element.variant.num_embedded();
         self.entries.insert(insert_at, Entry{
             element: ParserTypeElement{
                 element_span: array_span,
                 variant: ParserTypeVariant::Array,
             },
             depth: insert_depth
         });
         // Need to increment the depth of the child types
         self.entries[insert_at + 1].depth += 1; // element we applied the array type to
         if num_embedded != 0 {
             for index in insert_at + 2..self.entries.len() {
                 let element = &mut self.entries[index];
                 if element.depth >= insert_depth + 1 {
                     element.depth += 1;
                 } else {
                     break;
+                }
+            }
+        }
         return Ok(())
+    }
     /// Consumes a namespaced identifier that should resolve to some kind of
     /// type. There may be commas or polymorphic arguments remaining after this
     /// function has finished.
     fn consume_parser_type_element(
         iter: &mut TokenIter, source: &InputSource, heap: &Heap, symbols: &SymbolTable,
         wrapping_definition: DefinitionId, poly_vars: &[Identifier],
         mut scope: SymbolScope, allow_inference: bool,
+        mut scope: SymbolScope, allow_inference: bool, allow_compiler_types: bool,
     ) -> Result<ParserTypeElement, ParseError> {
         use ParserTypeVariant as PTV;
         let (mut type_text, mut type_span) = consume_any_ident(source, iter)?;
         // Early check for special builtin types available to the compiler
         if iter.next() == Some(TokenKind::Pragma) {
             let (type_text, pragma_span) = consume_pragma(source, iter)?;
             let variant = match type_text {
                 PRAGMA_TYPE_VOID => Some(PTV::Void),
                 PRAGMA_TYPE_PORTLIKE => Some(PTV::InputOrOutput),
                 PRAGMA_TYPE_INTEGERLIKE => Some(PTV::IntegerLike),
                 PRAGMA_TYPE_ARRAYLIKE => Some(PTV::ArrayLike),
                 _ => None,
             };
             if !allow_compiler_types || variant.is_none() {
                 return Err(ParseError::new_error_str_at_span(
                     source, pragma_span, "unexpected pragma in type"
                 ));
+            }
             return Ok(ParserTypeElement{
                 variant: variant.unwrap(),
                 element_span: pragma_span,
             });
+        }
         // No special type, parse as normal
         let (mut type_text, mut type_span) = consume_any_ident(source, iter)?;
         let variant = match type_text {
             KW_TYPE_MESSAGE => PTV::Message,
             KW_TYPE_BOOL => PTV::Bool,
             KW_TYPE_UINT8 => PTV::UInt8,
             KW_TYPE_UINT16 => PTV::UInt16,
             KW_TYPE_UINT32 => PTV::UInt32,
             KW_TYPE_UINT64 => PTV::UInt64,
             KW_TYPE_SINT8 => PTV::SInt8,
             KW_TYPE_SINT16 => PTV::SInt16,
             KW_TYPE_SINT32 => PTV::SInt32,
             KW_TYPE_SINT64 => PTV::SInt64,
             KW_TYPE_IN_PORT => PTV::Input,
             KW_TYPE_OUT_PORT => PTV::Output,
             KW_TYPE_CHAR => PTV::Character,
             KW_TYPE_STRING => PTV::String,
             KW_TYPE_INFERRED => {
                 if !allow_inference {
                     return Err(ParseError::new_error_str_at_span(
                         source, type_span, "type inference is not allowed here"
                     ));
+                }
                 PTV::Inferred
             },
             _ => {
                 // Must be some kind of symbolic type
                 let mut type_kind = None;
                 for (poly_idx, poly_var) in poly_vars.iter().enumerate() {
                     if poly_var.value.as_bytes() == type_text {
                         type_kind = Some(PTV::PolymorphicArgument(wrapping_definition, poly_idx as u32));
+                    }
+                }
                 if type_kind.is_none() {
                     // Check symbol table for definition. To be fair, the language
                     // only allows a single namespace for now. That said:
                     let last_symbol = symbols.get_symbol_by_name(scope, type_text);
                     if last_symbol.is_none() {
                         return Err(ParseError::new_error_str_at_span(
                             source, type_span, "unknown type"
                         ));
+                    }
                     let mut last_symbol = last_symbol.unwrap();
                     // Resolving scopes until we reach the intended type
                     loop {
                         match &last_symbol.variant {
                             SymbolVariant::Module(symbol_module) => {

src/protocol/parser/pass_imports.rs

➞

Show inline comments

 use crate::protocol::ast::*;
 use super::symbol_table::*;
 use super::{Module, ModuleCompilationPhase, PassCtx};
 use super::tokens::*;
 use super::token_parsing::*;
 use crate::protocol::input_source::{InputSource as InputSource, InputSpan, ParseError};
 use crate::collections::*;
 /// Parses all the imports in the module tokens. Is applied after the
 /// definitions and name of modules are resolved. Hence we should be able to
 /// resolve all symbols to their appropriate module/definition.
 pub(crate) struct PassImport {
     imports: Vec<ImportId>,
     found_symbols: Vec<(AliasedSymbol, SymbolDefinition)>,
     scoped_symbols: Vec<Symbol>,
+}
 impl PassImport {
     pub(crate) fn new() -> Self {
         Self{
             imports: Vec::with_capacity(32),
             found_symbols: Vec::with_capacity(32),
             scoped_symbols: Vec::with_capacity(32),
+        }
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let module_range = &module.tokens.ranges[0];
         debug_assert!(modules.iter().all(|m| m.phase >= ModuleCompilationPhase::SymbolsScanned));
         debug_assert_eq!(module.phase, ModuleCompilationPhase::SymbolsScanned);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         let mut range_idx = module_range.first_child_idx;
         loop {
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
         let module_root_id = module.root_id;
         let num_markers = module.tokens.markers.len();
             if cur_range.range_kind == TokenRangeKind::Import {
                 self.visit_import_range(modules, module_idx, ctx, range_idx_usize)?;
+            }
             if cur_range.next_sibling_idx == NO_SIBLING {
                 break;
             } else {
                 range_idx = cur_range.next_sibling_idx;
         for marker_index in 0..num_markers {
             let marker = &modules[module_idx].tokens.markers[marker_index];
             match marker.kind {
                 TokenMarkerKind::Import => {
                     self.visit_import_marker(modules, module_idx, ctx, marker_index)?;
                 },
                 TokenMarkerKind::Definition | TokenMarkerKind::Pragma => {},
+            }
+        }
-        let root = &mut ctx.heap[module.root_id];
+        let root = &mut ctx.heap[module_root_id];
         root.imports.extend(self.imports.drain(..));
         let module = &mut modules[module_idx];
         module.phase = ModuleCompilationPhase::ImportsResolved;
         Ok(())
+    }
     pub(crate) fn visit_import_range(
         &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize
     pub(crate) fn visit_import_marker(
         &mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, marker_index: usize
     ) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let import_range = &module.tokens.ranges[range_idx];
         debug_assert_eq!(import_range.range_kind, TokenRangeKind::Import);
         let marker = &module.tokens.markers[marker_index];
-        let mut iter = module.tokens.iter_range(import_range);
+        let mut iter = module.tokens.iter_range(marker.first_token, None);
         // Consume "import"
         let (_import_ident, import_span) =
             consume_any_ident(&module.source, &mut iter)?;
         debug_assert_eq!(_import_ident, KW_IMPORT);
         // Consume module name
         let (module_name, module_name_span) = consume_domain_ident(&module.source, &mut iter)?;
         let target_root_id = ctx.symbols.get_module_by_name(module_name);
         if target_root_id.is_none() {
             return Err(ParseError::new_error_at_span(
                 &module.source, module_name_span,
                 format!("could not resolve module '{}'", String::from_utf8_lossy(module_name))
             ));
+        }
         let module_name = ctx.pool.intern(module_name);
         let module_identifier = Identifier{ span: module_name_span, value: module_name };
         let target_root_id = target_root_id.unwrap();
         // Check for subsequent characters (alias, multiple imported symbols)
         let next = iter.next();
         let import_id;
         if has_ident(&module.source, &mut iter, b"as") {
             // Alias for module
             iter.consume();
             let alias_identifier = consume_ident_interned(&module.source, &mut iter, ctx)?;
             let alias_name = alias_identifier.value.clone();
             import_id = ctx.heap.alloc_import(|this| Import::Module(ImportModule{
                 this,
                 span: import_span,
                 module: module_identifier,
                 alias: alias_identifier,
                 module_id: target_root_id
             }));
             if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(SymbolScope::Module(module.root_id), Symbol{
                 name: alias_name,
                 variant: SymbolVariant::Module(SymbolModule{
                     root_id: target_root_id,
                     introduced_at: import_id,
                 }),
             }) {
                 return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, &old_symbol));
+            }
         } else if Some(TokenKind::ColonColon) == next {
             iter.consume();
@@ @@ -270,51 +264,57 @@ impl PassImport { @@
+                                }
                             ) {
                                 return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, &old_symbol));
+                            }
                         },
                         _ => unreachable!(),
+                    }
+                }
             } else {
                 return Err(ParseError::new_error_str_at_pos(
                     &module.source, iter.last_valid_pos(), "expected symbol name, '{' or '*'"
                 ));
+            }
         } else {
             // Assume implicit alias
             let module_name_str = module_identifier.value.clone();
             let last_ident_start = module_name_str.as_str().rfind('.').map_or(0, |v| v + 1);
             let alias_text = &module_name_str.as_bytes()[last_ident_start..];
             let alias = ctx.pool.intern(alias_text);
             let alias_span = InputSpan::from_positions(
                 module_name_span.begin.with_offset(last_ident_start as u32),
                 module_name_span.end
             );
             let alias_identifier = Identifier{ span: alias_span, value: alias.clone() };
             import_id = ctx.heap.alloc_import(|this| Import::Module(ImportModule{
                 this,
                 span: InputSpan::from_positions(import_span.begin, module_identifier.span.end),
                 module: module_identifier,
                 alias: alias_identifier,
                 module_id: target_root_id,
             }));
             if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(SymbolScope::Module(module.root_id), Symbol{
                 name: alias,
                 variant: SymbolVariant::Module(SymbolModule{
                     root_id: target_root_id,
                     introduced_at: import_id
                 })
             }) {
                 return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, &old_symbol));
+            }
+        }
         // By now the `import_id` is set, just need to make sure that the import
         // properly ends with a semicolon
         consume_token(&module.source, &mut iter, TokenKind::SemiColon)?;
         self.imports.push(import_id);
         // Update the marker
         let marker_last_token = iter.token_index();
         let marker = &mut modules[module_idx].tokens.markers[marker_index];
         marker.last_token = marker_last_token;
         marker.handled = true;
         Ok(())
+    }
+}

src/protocol/parser/pass_rewriting.rs

➞

Show inline comments

@@ @@ -4,96 +4,100 @@ use crate::protocol::*; @@
 use super::visitor::*;
 pub(crate) struct PassRewriting {
     current_scope: ScopeId,
     current_procedure_id: ProcedureDefinitionId,
     definition_buffer: ScopedBuffer<DefinitionId>,
     statement_buffer: ScopedBuffer<StatementId>,
     call_expr_buffer: ScopedBuffer<CallExpressionId>,
     expression_buffer: ScopedBuffer<ExpressionId>,
     scope_buffer: ScopedBuffer<ScopeId>,
+}
 impl PassRewriting {
     pub(crate) fn new() -> Self {
         Self{
             current_scope: ScopeId::new_invalid(),
             current_procedure_id: ProcedureDefinitionId::new_invalid(),
             definition_buffer: ScopedBuffer::with_capacity(BUFFER_INIT_CAP_LARGE),
             statement_buffer: ScopedBuffer::with_capacity(BUFFER_INIT_CAP_SMALL),
             call_expr_buffer: ScopedBuffer::with_capacity(BUFFER_INIT_CAP_SMALL),
             expression_buffer: ScopedBuffer::with_capacity(BUFFER_INIT_CAP_SMALL),
             scope_buffer: ScopedBuffer::with_capacity(BUFFER_INIT_CAP_SMALL),
+        }
+    }
+}
 impl Visitor for PassRewriting {
     fn visit_module(&mut self, ctx: &mut Ctx) -> VisitorResult {
         let module = ctx.module();
         debug_assert_eq!(module.phase, ModuleCompilationPhase::Typed);
         let root_id = module.root_id;
         let root = &ctx.heap[root_id];
         let definition_section = self.definition_buffer.start_section_initialized(&root.definitions);
         for definition_index in 0..definition_section.len() {
             let definition_id = definition_section[definition_index];
             self.visit_definition(ctx, definition_id)?;
+        }
         definition_section.forget();
         ctx.module_mut().phase = ModuleCompilationPhase::Rewritten;
         return Ok(())
+    }
     // --- Visiting procedures
     fn visit_procedure_definition(&mut self, ctx: &mut Ctx, id: ProcedureDefinitionId) -> VisitorResult {
         let definition = &ctx.heap[id];
         if definition.source.is_builtin() {
             return Ok(());
+        }
         let body_id = definition.body;
         self.current_scope = definition.scope;
         self.current_procedure_id = id;
         return self.visit_block_stmt(ctx, body_id);
+    }
     // --- Visiting statements (that are not the select statement)
     fn visit_block_stmt(&mut self, ctx: &mut Ctx, id: BlockStatementId) -> VisitorResult {
         let block_stmt = &ctx.heap[id];
         let stmt_section = self.statement_buffer.start_section_initialized(&block_stmt.statements);
         self.current_scope = block_stmt.scope;
         for stmt_idx in 0..stmt_section.len() {
             self.visit_stmt(ctx, stmt_section[stmt_idx])?;
+        }
         stmt_section.forget();
         return Ok(())
+    }
     fn visit_labeled_stmt(&mut self, ctx: &mut Ctx, id: LabeledStatementId) -> VisitorResult {
         let labeled_stmt = &ctx.heap[id];
         let body_id = labeled_stmt.body;
         return self.visit_stmt(ctx, body_id);
+    }
     fn visit_if_stmt(&mut self, ctx: &mut Ctx, id: IfStatementId) -> VisitorResult {
         let if_stmt = &ctx.heap[id];
         let true_case = if_stmt.true_case;
         let false_case = if_stmt.false_case;
         self.current_scope = true_case.scope;
         self.visit_stmt(ctx, true_case.body)?;
         if let Some(false_case) = false_case {
             self.current_scope = false_case.scope;
             self.visit_stmt(ctx, false_case.body)?;
+        }
         return Ok(())
+    }
     fn visit_while_stmt(&mut self, ctx: &mut Ctx, id: WhileStatementId) -> VisitorResult {
         let while_stmt = &ctx.heap[id];
         let body_id = while_stmt.body;
         self.current_scope = while_stmt.scope;
         return self.visit_stmt(ctx, body_id);
+    }

src/protocol/parser/pass_symbols.rs

➞

Show inline comments

 use crate::protocol::ast::*;
 use super::symbol_table::*;
 use crate::protocol::input_source::{ParseError, InputSpan};
 use super::tokens::*;
 use super::token_parsing::*;
 use super::{Module, ModuleCompilationPhase, PassCtx};
 /// Scans the module and finds all module-level type definitions. These will be
 /// added to the symbol table such that during AST-construction we know which
 /// identifiers point to types. Will also parse all pragmas to determine module
 /// names.
 pub(crate) struct PassSymbols {
     symbols: Vec<Symbol>,
     pragmas: Vec<PragmaId>,
     imports: Vec<ImportId>,
     definitions: Vec<DefinitionId>,
     buffer: String,
     has_pragma_version: bool,
     has_pragma_module: bool,
+}
 impl PassSymbols {
     pub(crate) fn new() -> Self {
         Self{
             symbols: Vec::with_capacity(128),
             pragmas: Vec::with_capacity(8),
             imports: Vec::with_capacity(32),
             definitions: Vec::with_capacity(128),
             buffer: String::with_capacity(128),
             has_pragma_version: false,
             has_pragma_module: false,
+        }
+    }
     fn reset(&mut self) {
         self.symbols.clear();
         self.pragmas.clear();
         self.imports.clear();
         self.definitions.clear();
         self.has_pragma_version = false;
         self.has_pragma_module = false;
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         self.reset();
         let module = &mut modules[module_idx];
-        let module_range = &module.tokens.ranges[0];
+        let add_to_global_namespace = module.add_to_global_namespace;
         debug_assert_eq!(module.phase, ModuleCompilationPhase::Tokenized);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         debug_assert!(module.root_id.is_invalid()); // not set yet,
         debug_assert!(module.root_id.is_invalid()); // not set yet
         // Preallocate root in the heap
         let root_id = ctx.heap.alloc_protocol_description(|this| {
             Root{
                 this,
                 pragmas: Vec::new(),
                 imports: Vec::new(),
                 definitions: Vec::new(),
+            }
         });
         module.root_id = root_id;
         // Retrieve first range index, then make immutable borrow
         let mut range_idx = module_range.first_child_idx;
         // Visit token ranges to detect definitions and pragmas
         loop {
         // Use pragma token markers to detects symbol definitions and pragmas
         let num_markers = module.tokens.markers.len();
         for marker_index in 0..num_markers {
             let module = &modules[module_idx];
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
             let next_sibling_idx = cur_range.next_sibling_idx;
             let range_kind = cur_range.range_kind;
             let marker = &module.tokens.markers[marker_index];
             // Parse if it is a definition or a pragma
             if range_kind == TokenRangeKind::Definition {
                 self.visit_definition_range(modules, module_idx, ctx, range_idx_usize)?;
             } else if range_kind == TokenRangeKind::Pragma {
                 self.visit_pragma_range(modules, module_idx, ctx, range_idx_usize)?;
+            }
             if next_sibling_idx == NO_SIBLING {
                 break;
             } else {
                 range_idx = next_sibling_idx;
             match marker.kind {
                 TokenMarkerKind::Pragma => {
                     self.visit_pragma_marker(modules, module_idx, ctx, marker_index)?;
                 },
                 TokenMarkerKind::Definition => {
                     self.visit_definition_marker(modules, module_idx, ctx, marker_index)?;
+                }
                 TokenMarkerKind::Import => {}, // we don't care yet
+            }
+        }
         // Add the module's symbol scope and the symbols we just parsed
         let module_scope = SymbolScope::Module(root_id);
         ctx.symbols.insert_scope(Some(SymbolScope::Global), module_scope);
         for symbol in self.symbols.drain(..) {
             ctx.symbols.insert_scope(Some(module_scope), SymbolScope::Definition(symbol.variant.as_definition().definition_id));
             if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(module_scope, symbol) {
                 return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, &old_symbol))
+            }
+        }
         if add_to_global_namespace {
             debug_assert!(self.symbols.is_empty());
             ctx.symbols.get_all_symbols_defined_in_scope(module_scope, &mut self.symbols);
             for symbol in self.symbols.drain(..) {
                 ctx.symbols.insert_symbol_in_global_scope(symbol);
+            }
+        }
         // Modify the preallocated root
         let root = &mut ctx.heap[root_id];
         root.pragmas.extend(self.pragmas.drain(..));
         root.definitions.extend(self.definitions.drain(..));
         // Modify module
         let module = &mut modules[module_idx];
         module.phase = ModuleCompilationPhase::SymbolsScanned;
         Ok(())
+    }
-    fn visit_pragma_range(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize) -> Result<(), ParseError> {
+    fn visit_pragma_marker(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, marker_index: usize) -> Result<(), ParseError> {
         let module = &mut modules[module_idx];
         let range = &module.tokens.ranges[range_idx];
         let mut iter = module.tokens.iter_range(range);
         let marker = &module.tokens.markers[marker_index];
         let mut iter = module.tokens.iter_range(marker.first_token, None);
         // Consume pragma name
-        let (pragma_section, pragma_start, _) = consume_pragma(&module.source, &mut iter)?;
+        let (pragma_section, mut pragma_span) = consume_pragma(&module.source, &mut iter)?;
         // Consume pragma values
         if pragma_section == b"#module" {
             // Check if name is defined twice within the same file
             if self.has_pragma_module {
-                return Err(ParseError::new_error_str_at_pos(&module.source, pragma_start, "module name is defined twice"));
+                return Err(ParseError::new_error_str_at_span(&module.source, pragma_span, "module name is defined twice"));
+            }
             // Consume the domain-name
+            // Consume the domain-name, then record end of pragma
             let (module_name, module_span) = consume_domain_ident(&module.source, &mut iter)?;
             if iter.next().is_some() {
                 return Err(ParseError::new_error_str_at_pos(&module.source, iter.last_valid_pos(), "expected end of #module pragma after module name"));
+            }
             let marker_last_token = iter.token_index();
             // Add to heap and symbol table
-            let pragma_span = InputSpan::from_positions(pragma_start, module_span.end);
+            pragma_span.end = module_span.end;
             let module_name = ctx.pool.intern(module_name);
             let pragma_id = ctx.heap.alloc_pragma(|this| Pragma::Module(PragmaModule{
                 this,
                 span: pragma_span,
                 value: Identifier{ span: module_span, value: module_name.clone() },
             }));
             self.pragmas.push(pragma_id);
             if let Err(other_module_root_id) = ctx.symbols.insert_module(module_name.clone(), module.root_id) {
                 // Naming conflict
                 let this_module = &modules[module_idx];
                 let other_module = seek_module(modules, other_module_root_id).unwrap();
                 let other_module_pragma_id = other_module.name.as_ref().map(|v| (*v).0).unwrap();
                 let other_pragma = ctx.heap[other_module_pragma_id].as_module();
                 return Err(ParseError::new_error_str_at_span(
                     &this_module.source, pragma_span, "conflict in module name"
                 ).with_info_str_at_span(
                     &other_module.source, other_pragma.span, "other module is defined here"
                 ));
+            }
             let marker = &mut module.tokens.markers[marker_index];
             marker.last_token = marker_last_token;
             marker.handled = true;
             module.name = Some((pragma_id, module_name));
             self.has_pragma_module = true;
         } else if pragma_section == b"#version" {
             // Check if version is defined twice within the same file
             if self.has_pragma_version {
-                return Err(ParseError::new_error_str_at_pos(&module.source, pragma_start, "module version is defined twice"));
+                return Err(ParseError::new_error_str_at_span(&module.source, pragma_span, "module version is defined twice"));
+            }
             // Consume the version pragma
             let (version, version_span) = consume_integer_literal(&module.source, &mut iter, &mut self.buffer)?;
             let marker_last_token = iter.token_index();
             pragma_span.end = version_span.end;
             let pragma_id = ctx.heap.alloc_pragma(|this| Pragma::Version(PragmaVersion{
                 this,
-                span: InputSpan::from_positions(pragma_start, version_span.end),
+                span: pragma_span,
                 version,
             }));
             self.pragmas.push(pragma_id);
             let marker = &mut module.tokens.markers[marker_index];
             marker.last_token = marker_last_token;
             marker.handled = true;
             module.version = Some((pragma_id, version as i64));
             self.has_pragma_version = true;
         } else {
             // Custom pragma, maybe we support this in the future, but for now
             // we don't.
             return Err(ParseError::new_error_str_at_pos(&module.source, pragma_start, "illegal pragma name"));
+        }
         } // else: custom pragma used for something else, will be handled later (or rejected with an error)
         Ok(())
+    }
-    fn visit_definition_range(&mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize) -> Result<(), ParseError> {
+    fn visit_definition_marker(&mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, marker_index: usize) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let range = &module.tokens.ranges[range_idx];
         let definition_span = InputSpan::from_positions(
             module.tokens.start_pos(range),
             module.tokens.end_pos(range)
         );
         let mut iter = module.tokens.iter_range(range);
         let marker = &module.tokens.markers[marker_index];
         let mut iter = module.tokens.iter_range(marker.first_token, None);
         // First ident must be type of symbol
         let (kw_text, _) = consume_any_ident(&module.source, &mut iter).unwrap();
         // Retrieve identifier of definition
         let identifier = consume_ident_interned(&module.source, &mut iter, ctx)?;
         let mut poly_vars = Vec::new();
         maybe_consume_comma_separated(
             TokenKind::OpenAngle, TokenKind::CloseAngle, &module.source, &mut iter, ctx,
             |source, iter, ctx| consume_ident_interned(source, iter, ctx),
             &mut poly_vars, "a polymorphic variable", None
         )?;
         let ident_text = identifier.value.clone(); // because we need it later
         let ident_span = identifier.span.clone();
         // Reserve space in AST for definition and add it to the symbol table
         let definition_class;
         let ast_definition_id;
         match kw_text {
             KW_STRUCT => {
                 let struct_def_id = ctx.heap.alloc_struct_definition(|this| {
-                    StructDefinition::new_empty(this, module.root_id, definition_span, identifier, poly_vars)
                     StructDefinition::new_empty(this, module.root_id, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Struct;
                 ast_definition_id = struct_def_id.upcast();
             },
             KW_ENUM => {
                 let enum_def_id = ctx.heap.alloc_enum_definition(|this| {
-                    EnumDefinition::new_empty(this, module.root_id, definition_span, identifier, poly_vars)
                     EnumDefinition::new_empty(this, module.root_id, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Enum;
                 ast_definition_id = enum_def_id.upcast();
             },
             KW_UNION => {
                 let union_def_id = ctx.heap.alloc_union_definition(|this| {
-                    UnionDefinition::new_empty(this, module.root_id, definition_span, identifier, poly_vars)
                     UnionDefinition::new_empty(this, module.root_id, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Union;
                 ast_definition_id = union_def_id.upcast()
             },
             KW_FUNCTION => {
                 let proc_def_id = ctx.heap.alloc_procedure_definition(|this| {
-                    ProcedureDefinition::new_empty(this, module.root_id, definition_span, ProcedureKind::Function, identifier, poly_vars)
                     ProcedureDefinition::new_empty(this, module.root_id, ProcedureKind::Function, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Function;
                 ast_definition_id = proc_def_id.upcast();
             },
             KW_PRIMITIVE | KW_COMPOSITE => {
                 let procedure_kind = if kw_text == KW_PRIMITIVE {
                     ProcedureKind::Primitive
                 } else {
                     ProcedureKind::Composite
                 };
                 let proc_def_id = ctx.heap.alloc_procedure_definition(|this| {
-                    ProcedureDefinition::new_empty(this, module.root_id, definition_span, procedure_kind, identifier, poly_vars)
                     ProcedureDefinition::new_empty(this, module.root_id, procedure_kind, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Component;
                 ast_definition_id = proc_def_id.upcast();
             },
             _ => unreachable!("encountered keyword '{}' in definition range", String::from_utf8_lossy(kw_text)),
+        }
         let symbol = Symbol{
             name: ident_text,
             variant: SymbolVariant::Definition(SymbolDefinition{
                 defined_in_module: module.root_id,
                 defined_in_scope: SymbolScope::Module(module.root_id),
                 definition_span,
                 identifier_span: ident_span,
                 imported_at: None,
                 class: definition_class,
                 definition_id: ast_definition_id,
             }),
         };
         self.symbols.push(symbol);
         self.definitions.push(ast_definition_id);
         Ok(())
+    }
+}
@@ \ No newline at end of file @@

src/protocol/parser/pass_tokenizer.rs

➞

Show inline comments

 use crate::protocol::input_source::{
     InputSource as InputSource,
     ParseError,
     InputPosition as InputPosition,
 };
 use super::tokens::*;
 use super::token_parsing::*;
 /// Tokenizer is a reusable parser to tokenize multiple source files using the
 /// same allocated buffers. In a well-formed program, we produce a consistent
 /// tree of token ranges such that we may identify tokens that represent a
 /// defintion or an import before producing the entire AST.
 ///
 /// If the program is not well-formed then the tree may be inconsistent, but we
 /// will detect this once we transform the tokens into the AST. To ensure a
 /// consistent AST-producing phase we will require the import to have balanced
 /// curly braces
 pub(crate) struct PassTokenizer {
     // Stack of input positions of opening curly braces, used to detect
     // unmatched opening braces, unmatched closing braces are detected
     // immediately.
     curly_stack: Vec<InputPosition>,
     // Points to an element in the `TokenBuffer.ranges` variable.
     stack_idx: usize,
+}
 impl PassTokenizer {
     pub(crate) fn new() -> Self {
         Self{
             curly_stack: Vec::with_capacity(32),
             stack_idx: 0
+        }
+    }
     pub(crate) fn tokenize(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         // Assert source and buffer are at start
         debug_assert_eq!(source.pos().offset, 0);
         debug_assert!(target.tokens.is_empty());
         debug_assert!(target.ranges.is_empty());
         // Set up for tokenization by pushing the first range onto the stack.
         // This range may get transformed into the appropriate range kind later,
         // see `push_range` and `pop_range`.
         self.stack_idx = 0;
         target.ranges.push(TokenRange{
             parent_idx: NO_RELATION,
             range_kind: TokenRangeKind::Module,
             curly_depth: 0,
             start: 0,
             end: 0,
             num_child_ranges: 0,
             first_child_idx: NO_RELATION,
             last_child_idx: NO_RELATION,
             next_sibling_idx: NO_RELATION,
         });
         // Main tokenization loop
         while let Some(c) = source.next() {
             let token_index = target.tokens.len() as u32;
             if is_char_literal_start(c) {
                 self.consume_char_literal(source, target)?;
             } else if is_bytestring_literal_start(c, source) {
                 self.consume_bytestring_literal(source, target)?;
             } else if is_string_literal_start(c) {
                 self.consume_string_literal(source, target)?;
             } else if is_identifier_start(c) {
                 let ident = self.consume_identifier(source, target)?;
                 if demarks_definition(ident) {
                     self.push_range(target, TokenRangeKind::Definition, token_index);
                 if demarks_symbol(ident) {
                     self.emit_marker(target, TokenMarkerKind::Definition, token_index);
                 } else if demarks_import(ident) {
-                    self.push_range(target, TokenRangeKind::Import, token_index);
+                    self.emit_marker(target, TokenMarkerKind::Import, token_index);
+                }
             } else if is_integer_literal_start(c) {
                 self.consume_number(source, target)?;
             } else if is_pragma_start_or_pound(c) {
                 let was_pragma = self.consume_pragma_or_pound(c, source, target)?;
                 if was_pragma {
-                    self.push_range(target, TokenRangeKind::Pragma, token_index);
+                    self.emit_marker(target, TokenMarkerKind::Pragma, token_index);
+                }
             } else if self.is_line_comment_start(c, source) {
                 self.consume_line_comment(source, target)?;
             } else if self.is_block_comment_start(c, source) {
                 self.consume_block_comment(source, target)?;
             } else if is_whitespace(c) {
                 let contained_newline = self.consume_whitespace(source);
                 if contained_newline {
                     let range = &target.ranges[self.stack_idx];
                     if range.range_kind == TokenRangeKind::Pragma {
                         self.pop_range(target, target.tokens.len() as u32);
+                    }
+                }
                 self.consume_whitespace(source);
             } else {
                 let was_punctuation = self.maybe_parse_punctuation(c, source, target)?;
                 if let Some((token, token_pos)) = was_punctuation {
                     if token == TokenKind::OpenCurly {
                         self.curly_stack.push(token_pos);
                     } else if token == TokenKind::CloseCurly {
                         // Check if this marks the end of a range we're
                         // currently processing
                         if self.curly_stack.is_empty() {
                             return Err(ParseError::new_error_str_at_pos(
                                 source, token_pos, "unmatched closing curly brace '}'"
                             ));
+                        }
                         self.curly_stack.pop();
                         let range = &target.ranges[self.stack_idx];
                         if range.range_kind == TokenRangeKind::Definition && range.curly_depth == self.curly_stack.len() as u32 {
                             self.pop_range(target, target.tokens.len() as u32);
+                        }
                         // Exit early if we have more closing curly braces than
                         // opening curly braces
                     } else if token == TokenKind::SemiColon {
                         // Check if this marks the end of an import
                         let range = &target.ranges[self.stack_idx];
                         if range.range_kind == TokenRangeKind::Import {
                             self.pop_range(target, target.tokens.len() as u32);
+                        }
+                    }
                 } else {
                     return Err(ParseError::new_error_str_at_pos(
                         source, source.pos(), "unexpected character"
                     ));
+                }
+            }
+        }
         // End of file, check if our state is correct
         if let Some(error) = source.had_error.take() {
             return Err(error);
+        }
         if !self.curly_stack.is_empty() {
             // Let's not add a lot of heuristics and just tell the programmer
             // that something is wrong
             let last_unmatched_open = self.curly_stack.pop().unwrap();
             return Err(ParseError::new_error_str_at_pos(
                 source, last_unmatched_open, "unmatched opening curly brace '{'"
             ));
+        }
         // Ranges that did not depend on curly braces may have missing tokens.
         // So close all of the active tokens
         while self.stack_idx != 0 {
             self.pop_range(target, target.tokens.len() as u32);
+        }
         // And finally, we may have a token range at the end that doesn't belong
         // to a range yet, so insert a "code" range if this is the case.
         debug_assert_eq!(self.stack_idx, 0);
         let last_registered_idx = target.ranges[0].end;
         let last_token_idx = target.tokens.len() as u32;
         if last_registered_idx != last_token_idx {
             self.add_code_range(target, 0, last_registered_idx, last_token_idx, NO_RELATION);
+        }
         Ok(())
+    }
     fn is_line_comment_start(&self, first_char: u8, source: &InputSource) -> bool {
         return first_char == b'/' && Some(b'/') == source.lookahead(1);
+    }
     fn is_block_comment_start(&self, first_char: u8, source: &InputSource) -> bool {
         return first_char == b'/' && Some(b'*') == source.lookahead(1);
+    }
     fn maybe_parse_punctuation(
         &mut self, first_char: u8, source: &mut InputSource, target: &mut TokenBuffer
     ) -> Result<Option<(TokenKind, InputPosition)>, ParseError> {
         debug_assert!(first_char != b'#', "'#' needs special handling");
         debug_assert!(first_char != b'\'', "'\'' needs special handling");
         debug_assert!(first_char != b'"', "'\"' needs special handling");
         let pos = source.pos();
         let token_kind;
         if first_char == b'!' {
             source.consume();
             if Some(b'=') == source.next() {
                 source.consume();
                 token_kind = TokenKind::NotEqual;
             } else {
                 token_kind = TokenKind::Exclamation;
+            }
         } else if first_char == b'%' {
             source.consume();
             if Some(b'=') == source.next() {
                 source.consume();
                 token_kind = TokenKind::PercentEquals;
             } else {
                 token_kind = TokenKind::Percent;
+            }
         } else if first_char == b'&' {
             source.consume();
             let next = source.next();
             if Some(b'&') == next {
                 source.consume();
                 token_kind = TokenKind::AndAnd;
             } else if Some(b'=') == next {
                 source.consume();
                 token_kind = TokenKind::AndEquals;
             } else {
                 token_kind = TokenKind::And;
+            }
@@ @@ -366,423 +313,374 @@ impl PassTokenizer { @@
         } else if first_char == b'~' {
             source.consume();
             token_kind = TokenKind::Tilde;
         } else {
             self.check_ascii(source)?;
             return Ok(None);
+        }
         target.tokens.push(Token::new(token_kind, pos));
         Ok(Some((token_kind, pos)))
+    }
     fn consume_char_literal(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         // Consume the leading quote
         debug_assert!(source.next().unwrap() == b'\'');
         source.consume();
         let mut prev_char = b'\'';
         while let Some(c) = source.next() {
             if !c.is_ascii() {
                 return Err(ParseError::new_error_str_at_pos(source, source.pos(), "non-ASCII character in char literal"));
+            }
             source.consume();
             // Make sure ending quote was not escaped
             if c == b'\'' && prev_char != b'\\' {
                 prev_char = c;
                 break;
+            }
             prev_char = c;
+        }
         if prev_char != b'\'' {
             // Unterminated character literal, reached end of file.
             return Err(ParseError::new_error_str_at_pos(source, begin_pos, "encountered unterminated character literal"));
+        }
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::Character, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
-    fn consume_string_literal(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
+    fn consume_bytestring_literal(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         // Consume the leading double quotes
         debug_assert!(source.next().unwrap() == b'"');
         debug_assert!(source.next().unwrap() == b'b');
         source.consume();
         let mut prev_char = b'"';
         while let Some(c) = source.next() {
             if !c.is_ascii() {
                 return Err(ParseError::new_error_str_at_pos(source, source.pos(), "non-ASCII character in string literal"));
+            }
             source.consume();
             if c == b'"' && prev_char != b'\\' {
                 // Unescaped string terminator
                 prev_char = c;
                 break;
+            }
             if prev_char == b'\\' && c == b'\\' {
                 // Escaped backslash, set prev_char to bogus to not conflict
                 // with escaped-" and unterminated string literal detection.
                 prev_char = b'\0';
             } else {
                 prev_char = c;
+            }
+        }
         let end_pos = self.consume_ascii_string(begin_pos, source)?;
         target.tokens.push(Token::new(TokenKind::Bytestring, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         if prev_char != b'"' {
             // Unterminated string literal
             return Err(ParseError::new_error_str_at_pos(source, begin_pos, "encountered unterminated string literal"));
+        }
         Ok(())
+    }
         let end_pos = source.pos();
     fn consume_string_literal(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         let end_pos = self.consume_ascii_string(begin_pos, source)?;
         target.tokens.push(Token::new(TokenKind::String, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     fn consume_pragma_or_pound(&mut self, first_char: u8, source: &mut InputSource, target: &mut TokenBuffer) -> Result<bool, ParseError> {
         let start_pos = source.pos();
         debug_assert_eq!(first_char, b'#');
         source.consume();
         let next = source.next();
         if next.is_none() || !is_identifier_start(next.unwrap()) {
             // Just a pound sign
             target.tokens.push(Token::new(TokenKind::Pound, start_pos));
             Ok(false)
         } else {
             // Pound sign followed by identifier
             source.consume();
             while let Some(c) = source.next() {
                 if !is_identifier_remaining(c) {
                     break;
+                }
                 source.consume();
+            }
             self.check_ascii(source)?;
             let end_pos = source.pos();
             target.tokens.push(Token::new(TokenKind::Pragma, start_pos));
             target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
             Ok(true)
+        }
+    }
     fn consume_line_comment(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         // Consume the leading "//"
         debug_assert!(source.next().unwrap() == b'/' && source.lookahead(1).unwrap() == b'/');
         source.consume();
         source.consume();
         let mut prev_char = b'/';
         let mut cur_char = b'/';
         while let Some(c) = source.next() {
             prev_char = cur_char;
             cur_char = c;
             if c == b'\n' {
                 // End of line, note that the newline is not consumed
                 break;
+            }
             source.consume();
+        }
         let mut end_pos = source.pos();
         debug_assert_eq!(begin_pos.line, end_pos.line);
         // Modify offset to not include the newline characters
         if cur_char == b'\n' {
             if prev_char == b'\r' {
                 end_pos.offset -= 2;
             } else {
                 end_pos.offset -= 1;
+            }
             // Consume final newline
             source.consume();
         } else {
             // End of comment was due to EOF
             debug_assert!(source.next().is_none())
+        }
         target.tokens.push(Token::new(TokenKind::LineComment, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     fn consume_block_comment(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         // Consume the leading "/*"
         debug_assert!(source.next().unwrap() == b'/' && source.lookahead(1).unwrap() == b'*');
         source.consume();
         source.consume();
         // Explicitly do not put prev_char at "*", because then "/*/" would
         // represent a valid and closed block comment
         let mut prev_char = b' ';
         let mut is_closed = false;
         while let Some(c) = source.next() {
             source.consume();
             if prev_char == b'*' && c == b'/' {
                 // End of block comment
                 is_closed = true;
                 break;
+            }
             prev_char = c;
+        }
         if !is_closed {
             return Err(ParseError::new_error_str_at_pos(
                 source, source.pos(), "encountered unterminated block comment")
             );
+        }
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::BlockComment, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     fn consume_identifier<'a>(&mut self, source: &'a mut InputSource, target: &mut TokenBuffer) -> Result<&'a [u8], ParseError> {
         let begin_pos = source.pos();
         debug_assert!(is_identifier_start(source.next().unwrap()));
         source.consume();
         // Keep reading until no more identifier
         while let Some(c) = source.next() {
             if !is_identifier_remaining(c) {
                 break;
+            }
             source.consume();
+        }
         self.check_ascii(source)?;
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::Ident, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(source.section_at_pos(begin_pos, end_pos))
+    }
     fn consume_number(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {
         let begin_pos = source.pos();
         debug_assert!(is_integer_literal_start(source.next().unwrap()));
         source.consume();
         // Keep reading until it doesn't look like a number anymore
         while let Some(c) = source.next() {
             if !maybe_number_remaining(c) {
                 break;
+            }
             source.consume();
+        }
         self.check_ascii(source)?;
         let end_pos = source.pos();
         target.tokens.push(Token::new(TokenKind::Integer, begin_pos));
         target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));
         Ok(())
+    }
     // Consumes the ascii string (including leading and trailing quotation
     // marks) and returns the input position *after* the last quotation mark (or
     // an error, if something went wrong).
     fn consume_ascii_string(&self, begin_pos: InputPosition, source: &mut InputSource) -> Result<InputPosition, ParseError> {
         debug_assert!(source.next().unwrap() == b'"');
         source.consume();
         let mut prev_char = b'"';
         while let Some(c) = source.next() {
             if !c.is_ascii() {
                 return Err(ParseError::new_error_str_at_pos(source, source.pos(), "non-ASCII character in string literal"));
+            }
             source.consume();
             if c == b'"' && prev_char != b'\\' {
                 // Unescaped string terminator
                 prev_char = c;
                 break;
+            }
             if prev_char == b'\\' && c == b'\\' {
                 // Escaped backslash, set prev_char to bogus to not conflict
                 // with escaped-" and unterminated string literal detection.
                 prev_char = b'\0';
             } else {
                 prev_char = c;
+            }
+        }
         if prev_char != b'"' {
             // Unterminated string literal
             return Err(ParseError::new_error_str_at_pos(source, begin_pos, "encountered unterminated string literal"));
+        }
         let end_pos = source.pos();
         return Ok(end_pos)
+    }
     // Consumes whitespace and returns whether or not the whitespace contained
     // a newline.
     fn consume_whitespace(&self, source: &mut InputSource) -> bool {
         debug_assert!(is_whitespace(source.next().unwrap()));
         let mut has_newline = false;
         while let Some(c) = source.next() {
             if !is_whitespace(c) {
                 break;
+            }
             if c == b'\n' {
                 has_newline = true;
+            }
             source.consume();
+        }
         has_newline
+    }
     fn add_code_range(
         &mut self, target: &mut TokenBuffer, parent_idx: i32,
         code_start_idx: u32, code_end_idx: u32, next_sibling_idx: i32
     ) {
         let new_range_idx = target.ranges.len() as i32;
         let parent_range = &mut target.ranges[parent_idx as usize];
         debug_assert_ne!(parent_range.end, code_end_idx, "called push_code_range without a need to do so");
         let sibling_idx = parent_range.last_child_idx;
         parent_range.last_child_idx = new_range_idx;
         parent_range.end = code_end_idx;
         parent_range.num_child_ranges += 1;
         let curly_depth = self.curly_stack.len() as u32;
         target.ranges.push(TokenRange{
             parent_idx,
             range_kind: TokenRangeKind::Code,
             curly_depth,
             start: code_start_idx,
             end: code_end_idx,
             num_child_ranges: 0,
             first_child_idx: NO_RELATION,
             last_child_idx: NO_RELATION,
             next_sibling_idx,
     fn emit_marker(&mut self, target: &mut TokenBuffer, kind: TokenMarkerKind, first_token: u32) {
         debug_assert!(
             target.markers
                 .last().map(|v| v.first_token < first_token)
                 .unwrap_or(true)
         );
         target.markers.push(TokenMarker{
             kind,
             curly_depth: self.curly_stack.len() as u32,
             first_token,
             last_token: u32::MAX,
             handled: false,
         });
         // Fix up the sibling indices
         if sibling_idx != NO_RELATION {
             let sibling_range = &mut target.ranges[sibling_idx as usize];
             sibling_range.next_sibling_idx = new_range_idx;
+        }
+    }
     fn push_range(&mut self, target: &mut TokenBuffer, range_kind: TokenRangeKind, first_token_idx: u32) {
         let new_range_idx = target.ranges.len() as i32;
         let parent_idx = self.stack_idx as i32;
         let parent_range = &mut target.ranges[self.stack_idx];
         if parent_range.first_child_idx == NO_RELATION {
             parent_range.first_child_idx = new_range_idx;
+        }
         let last_registered_idx = parent_range.end;
         if last_registered_idx != first_token_idx {
             self.add_code_range(target, parent_idx, last_registered_idx, first_token_idx, new_range_idx + 1);
+        }
         // Push the new range
         self.stack_idx = target.ranges.len();
         let curly_depth = self.curly_stack.len() as u32;
         target.ranges.push(TokenRange{
             parent_idx,
             range_kind,
             curly_depth,
             start: first_token_idx,
             end: first_token_idx, // modified when popped
             num_child_ranges: 0,
             first_child_idx: NO_RELATION,
             last_child_idx: NO_RELATION,
             next_sibling_idx: NO_RELATION
         })
+    }
     fn pop_range(&mut self, target: &mut TokenBuffer, end_token_idx: u32) {
         let popped_idx = self.stack_idx as i32;
         let popped_range = &mut target.ranges[self.stack_idx];
         debug_assert!(self.stack_idx != 0, "attempting to pop top-level range");
         // Fix up the current range before going back to parent
         popped_range.end = end_token_idx;
         debug_assert_ne!(popped_range.start, end_token_idx);
         // Go back to parent and fix up its child pointers, but remember the
         // last child, so we can link it to the newly popped range.
         self.stack_idx = popped_range.parent_idx as usize;
         let parent = &mut target.ranges[self.stack_idx];
         if parent.first_child_idx == NO_RELATION {
             parent.first_child_idx = popped_idx;
+        }
         let prev_sibling_idx = parent.last_child_idx;
         parent.last_child_idx = popped_idx;
         parent.end = end_token_idx;
         parent.num_child_ranges += 1;
         // Fix up the sibling (if it exists)
         if prev_sibling_idx != NO_RELATION {
             let sibling = &mut target.ranges[prev_sibling_idx as usize];
             sibling.next_sibling_idx = popped_idx;
+        }
+    }
     fn check_ascii(&self, source: &InputSource) -> Result<(), ParseError> {
         match source.next() {
             Some(c) if !c.is_ascii() => {
                 Err(ParseError::new_error_str_at_pos(source, source.pos(), "encountered a non-ASCII character"))
             },
             _else => {
                 Ok(())
             },
+        }
+    }
+}
 // Helpers for characters
-fn demarks_definition(ident: &[u8]) -> bool {
+fn demarks_symbol(ident: &[u8]) -> bool {
     return
         ident == KW_STRUCT ||
             ident == KW_ENUM ||
             ident == KW_UNION ||
             ident == KW_FUNCTION ||
             ident == KW_PRIMITIVE ||
             ident == KW_COMPOSITE
+}
 #[inline]
 fn demarks_import(ident: &[u8]) -> bool {
     return ident == KW_IMPORT;
+}
 #[inline]
 fn is_whitespace(c: u8) -> bool {
     c.is_ascii_whitespace()
+}
 #[inline]
 fn is_char_literal_start(c: u8) -> bool {
     return c == b'\'';
+}
 #[inline]
 fn is_bytestring_literal_start(c: u8, source: &InputSource) -> bool {
     return c == b'b' && source.lookahead(1) == Some(b'"');
+}
 #[inline]
 fn is_string_literal_start(c: u8) -> bool {
     return c == b'"';
+}
 #[inline]
 fn is_pragma_start_or_pound(c: u8) -> bool {
     return c == b'#';
+}
 fn is_identifier_start(c: u8) -> bool {
     return
         (c >= b'a' && c <= b'z') ||
             (c >= b'A' && c <= b'Z') ||
             c == b'_'
+}
 fn is_identifier_remaining(c: u8) -> bool {
     return
         (c >= b'0' && c <= b'9') ||
             (c >= b'a' && c <= b'z') ||
             (c >= b'A' && c <= b'Z') ||
             c == b'_'
+}
 #[inline]
 fn is_integer_literal_start(c: u8) -> bool {
     return c >= b'0' && c <= b'9';
+}
 fn maybe_number_remaining(c: u8) -> bool {
     // Note: hex range includes the possible binary indicator 'b' and 'B';
     return
         (c == b'o' || c == b'O' || c == b'x' || c == b'X') ||
             (c >= b'0' && c <= b'9') || (c >= b'A' && c <= b'F') || (c >= b'a' && c <= b'f') ||
             c == b'_';
+}

Changeset was too big and was cut off... Show full diff anyway

0 comments (0 inline, 0 general)