CSY/reowolf Changeset - c9800c8f19d7 · Centrum Wiskunde & Informatica (CWI)

Changeset - c9800c8f19d7

Parent rev.

Child rev.

[Not reviewed]

0 17 0

mh - 3 years ago 2022-03-29 16:28:33
contact@maxhenger.nl

Rewrite tokenizer to emit markers iso ranges

17 files changed with 282 insertions and 190 deletions:

src/protocol/ast.rs

src/protocol/mod.rs

src/protocol/parser/mod.rs

src/protocol/parser/pass_definitions.rs

src/protocol/parser/pass_definitions_types.rs

src/protocol/parser/pass_imports.rs

src/protocol/parser/pass_rewriting.rs

src/protocol/parser/pass_symbols.rs

src/protocol/parser/pass_tokenizer.rs

src/protocol/parser/pass_typing.rs

src/protocol/parser/pass_validation_linking.rs

src/protocol/parser/symbol_table.rs

src/protocol/parser/token_parsing.rs

src/protocol/parser/tokens.rs

src/protocol/tests/utils.rs

src/protocol/token_writer.rs

std/std.global.pdl

0 comments (0 inline, 0 general)

src/protocol/ast.rs

➞

Show inline comments

@@ @@ -929,25 +929,24 @@ pub struct StructFieldDefinition { @@
 #[derive(Debug, Clone)]
 pub struct StructDefinition {
     pub this: StructDefinitionId,
     pub defined_in: RootId,
     // Symbol scanning
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Parsing
     pub fields: Vec<StructFieldDefinition>
+}
 impl StructDefinition {
     pub(crate) fn new_empty(
-        this: StructDefinitionId, defined_in: RootId, span: InputSpan,
         this: StructDefinitionId, defined_in: RootId,
         identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
-        Self{ this, defined_in, span, identifier, poly_vars, fields: Vec::new() }
         Self{ this, defined_in, identifier, poly_vars, fields: Vec::new() }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub enum EnumVariantValue {
     None,
@@ @@ -962,25 +961,24 @@ pub struct EnumVariantDefinition { @@
 #[derive(Debug, Clone)]
 pub struct EnumDefinition {
     pub this: EnumDefinitionId,
     pub defined_in: RootId,
     // Symbol scanning
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Parsing
     pub variants: Vec<EnumVariantDefinition>,
+}
 impl EnumDefinition {
     pub(crate) fn new_empty(
-        this: EnumDefinitionId, defined_in: RootId, span: InputSpan,
         this: EnumDefinitionId, defined_in: RootId,
         identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
-        Self{ this, defined_in, span, identifier, poly_vars, variants: Vec::new() }
         Self{ this, defined_in, identifier, poly_vars, variants: Vec::new() }
+    }
+}
 #[derive(Debug, Clone)]
 pub struct UnionVariantDefinition {
     pub span: InputSpan,
@@ @@ -990,25 +988,24 @@ pub struct UnionVariantDefinition { @@
 #[derive(Debug, Clone)]
 pub struct UnionDefinition {
     pub this: UnionDefinitionId,
     pub defined_in: RootId,
     // Phase 1: symbol scanning
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Phase 2: parsing
     pub variants: Vec<UnionVariantDefinition>,
+}
 impl UnionDefinition {
     pub(crate) fn new_empty(
-        this: UnionDefinitionId, defined_in: RootId, span: InputSpan,
         this: UnionDefinitionId, defined_in: RootId,
         identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
-        Self{ this, defined_in, span, identifier, poly_vars, variants: Vec::new() }
         Self{ this, defined_in, identifier, poly_vars, variants: Vec::new() }
+    }
+}
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum ProcedureKind {
     Function, // with return type
@@ @@ -1108,13 +1105,12 @@ impl ProcedureSource { @@
 #[derive(Debug)]
 pub struct ProcedureDefinition {
     pub this: ProcedureDefinitionId,
     pub defined_in: RootId,
     // Symbol scanning
     pub kind: ProcedureKind,
     pub span: InputSpan,
     pub identifier: Identifier,
     pub poly_vars: Vec<Identifier>,
     // Parser
     pub source: ProcedureSource,
     pub return_type: Option<ParserType>, // present on functions, not components
     pub parameters: Vec<VariableId>,
@@ @@ -1123,18 +1119,17 @@ pub struct ProcedureDefinition { @@
     // Monomorphization of typed procedures
     pub monomorphs: Vec<ProcedureDefinitionMonomorph>,
+}
 impl ProcedureDefinition {
     pub(crate) fn new_empty(
-        this: ProcedureDefinitionId, defined_in: RootId, span: InputSpan,
         this: ProcedureDefinitionId, defined_in: RootId,
         kind: ProcedureKind, identifier: Identifier, poly_vars: Vec<Identifier>
     ) -> Self {
         Self {
             this, defined_in,
             span,
             kind, identifier, poly_vars,
             source: ProcedureSource::FuncUserDefined,
             return_type: None,
             parameters: Vec::new(),
             scope: ScopeId::new_invalid(),
             body: BlockStatementId::new_invalid(),

src/protocol/mod.rs

➞

Show inline comments

@@ @@ -57,13 +57,12 @@ impl ProtocolDescription { @@
         if let Err(err) = parser.parse() {
             println!("ERROR:\n{}", err);
             return Err(format!("{}", err))
+        }
         debug_assert_eq!(parser.modules.len(), 1, "only supporting one module here for now");
         let modules: Vec<Module> = parser.modules.into_iter()
             .map(|module| Module{
                 source: module.source,
                 root_id: module.root_id,
                 name: module.name.map(|(_, name)| name)
             })

src/protocol/parser/mod.rs

➞

Show inline comments

@@ @@ -49,16 +49,15 @@ pub enum ModuleCompilationPhase { @@
     Rewritten,              // Special AST nodes are rewritten into regular AST nodes
     // When we continue with the compiler:
     // StackSize
+}
 pub struct Module {
     // Buffers
     pub source: InputSource,
     pub tokens: TokenBuffer,
-    // Identifiers
+    pub is_compiler_file: bool, // TODO: @Hack
     pub root_id: RootId,
     pub name: Option<(PragmaId, StringRef<'static>)>,
     pub version: Option<(PragmaId, i64)>,
     pub phase: ModuleCompilationPhase,
+}
@@ @@ -153,13 +152,13 @@ impl Parser { @@
             pass_import: PassImport::new(),
             pass_definitions: PassDefinitions::new(),
             pass_validation: PassValidationLinking::new(),
             pass_typing: PassTyping::new(),
             pass_rewriting: PassRewriting::new(),
             pass_stack_size: PassStackSize::new(),
-            write_tokens_to: Some("tokens.txt".to_string()),
+            write_tokens_to: None,
             write_ast_to: None,
             arch: TargetArch::new(),
         };
         parser.symbol_table.insert_scope(None, SymbolScope::Global);
@@ @@ -191,27 +190,13 @@ impl Parser { @@
+    }
     /// Feeds a new InputSource to the parser, which will tokenize it and store
     /// it internally for later parsing (when all modules are present). Returns
     /// the index of the new module.
     pub fn feed(&mut self, mut source: InputSource) -> Result<usize, ParseError> {
         let mut token_buffer = TokenBuffer::new();
         self.pass_tokenizer.tokenize(&mut source, &mut token_buffer)?;
         let module = Module{
             source,
             tokens: token_buffer,
             root_id: RootId::new_invalid(),
             name: None,
             version: None,
             phase: ModuleCompilationPhase::Tokenized,
         };
         let module_index = self.modules.len();
         self.modules.push(module);
         return Ok(module_index);
         return self.feed_internal(source, false);
+    }
     pub fn parse(&mut self) -> Result<(), ParseError> {
         let mut pass_ctx = PassCtx{
             heap: &mut self.heap,
             symbols: &mut self.symbol_table,
@@ @@ -350,13 +335,13 @@ impl Parser { @@
                 ));
+            }
             let source = source.unwrap();
             let input_source = InputSource::new(file.to_string(), source);
-            let module_index = self.feed(input_source);
+            let module_index = self.feed_internal(input_source, true);
             if let Err(err) = module_index {
                 // A bit of a hack, but shouldn't really happen anyway: the
                 // compiler should ship with a decent standard library (at some
                 // point)
                 return Err(format!("{}", err));
+            }
@@ @@ -367,12 +352,31 @@ impl Parser { @@
                 first_file = false;
+            }
+        }
         return Ok(())
+    }
     fn feed_internal(&mut self, mut source: InputSource, is_compiler_file: bool) -> Result<usize, ParseError> {
         let mut token_buffer = TokenBuffer::new();
         self.pass_tokenizer.tokenize(&mut source, &mut token_buffer)?;
         let module = Module{
             source,
             tokens: token_buffer,
             is_compiler_file,
             root_id: RootId::new_invalid(),
             name: None,
             version: None,
             phase: ModuleCompilationPhase::Tokenized,
         };
         let module_index = self.modules.len();
         self.modules.push(module);
         return Ok(module_index);
+    }
+}
 fn insert_builtin_type(type_table: &mut TypeTable, parts: Vec<ConcreteTypePart>, has_poly_var: bool, size: usize, alignment: usize) -> TypeId {
     const POLY_VARS: [PolymorphicVariable; 1] = [PolymorphicVariable{
         identifier: Identifier::new_empty(InputSpan::new()),
         is_in_use: false,

src/protocol/parser/pass_definitions.rs

➞

Show inline comments

@@ @@ -40,57 +40,54 @@ impl PassDefinitions { @@
             parser_types: ScopedBuffer::with_capacity(128),
+        }
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let module_range = &module.tokens.ranges[0];
         debug_assert_eq!(module.phase, ModuleCompilationPhase::ImportsResolved);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         // Although we only need to parse the definitions, we want to go through
         // code ranges as well such that we can throw errors if we get
         // unexpected tokens at the module level of the source.
         let mut range_idx = module_range.first_child_idx;
         loop {
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
             match cur_range.range_kind {
                 TokenRangeKind::Module => unreachable!(), // should not be reachable
                 TokenRangeKind::Pragma | TokenRangeKind::Import => {
                     // Already fully parsed, fall through and go to next range
                 },
                 TokenRangeKind::Definition | TokenRangeKind::Code => {
                     // Visit range even if it is a "code" range to provide
                     // proper error messages.
                     self.visit_range(modules, module_idx, ctx, range_idx_usize)?;
                 },
+            }
             if cur_range.next_sibling_idx == NO_SIBLING {
         // We iterate through the entire document. If we find a marker that has
         // been handled then we skip over it. It is important that we properly
         // parse all other tokens in the document to ensure that we throw the
         // correct kind of errors.
         let num_tokens = module.tokens.tokens.len() as u32;
         let num_markers = module.tokens.markers.len();
         let mut marker_index = 0;
         let mut first_token_index = 0;
         while first_token_index < num_tokens {
             // Seek ahead to the next marker that was already handled.
             let mut last_token_index = num_tokens;
             let mut new_first_token_index = num_tokens;
             while marker_index < num_markers {
                 let marker = &module.tokens.markers[marker_index];
                 marker_index += 1;
                 if marker.handled {
                     last_token_index = marker.first_token;
                     new_first_token_index = marker.last_token;
                     break;
             } else {
                 range_idx = cur_range.next_sibling_idx;
+                }
+            }
             self.visit_token_range(modules, module_idx, ctx, first_token_index, last_token_index)?;
             first_token_index = new_first_token_index;
+        }
         modules[module_idx].phase = ModuleCompilationPhase::DefinitionsParsed;
         Ok(())
+    }
     fn visit_range(
         &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize
     fn visit_token_range(
         &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx,
         token_range_begin: u32, token_range_end: u32,
     ) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let cur_range = &module.tokens.ranges[range_idx];
         debug_assert!(cur_range.range_kind == TokenRangeKind::Definition || cur_range.range_kind == TokenRangeKind::Code);
         // Detect which definition we're parsing
-        let mut iter = module.tokens.iter_range(cur_range.start, cur_range.end);
+        let mut iter = module.tokens.iter_range(token_range_begin, Some(token_range_end));
         loop {
             let next = iter.next();
             if next.is_none() {
                 return Ok(())
+            }
@@ @@ -131,13 +128,13 @@ impl PassDefinitions { @@
             |source, iter, ctx| {
                 let poly_vars = ctx.heap[definition_id].poly_vars();
                 let start_pos = iter.last_valid_pos();
                 let parser_type = self.type_parser.consume_parser_type(
                     iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id,
                     module_scope, false, None
+                    module_scope, false, false, None
                 )?;
                 let field = consume_ident_interned(source, iter, ctx)?;
                 Ok(StructFieldDefinition{
                     span: InputSpan::from_positions(start_pos, field.span.end),
                     field, parser_type
                 })
@@ @@ -218,13 +215,13 @@ impl PassDefinitions { @@
                 let has_embedded = maybe_consume_comma_separated(
                     TokenKind::OpenParen, TokenKind::CloseParen, source, iter, ctx,
                     |source, iter, ctx| {
                         let poly_vars = ctx.heap[definition_id].poly_vars();
                         self.type_parser.consume_parser_type(
                             iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id,
                             module_scope, false, None
+                            module_scope, false, false, None
+                        )
                     },
                     &mut types_section, "an embedded type", Some(&mut close_pos)
                 )?;
                 let value = if has_embedded {
                     types_section.into_vec()
@@ @@ -252,35 +249,36 @@ impl PassDefinitions { @@
     fn visit_function_definition(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx
     ) -> Result<(), ParseError> {
         // Retrieve function name
         consume_exact_ident(&module.source, iter, KW_FUNCTION)?;
         let (ident_text, _) = consume_ident(&module.source, iter)?;
         let stringy = String::from_utf8_lossy(ident_text).to_string();
         // Retrieve preallocated DefinitionId
         let module_scope = SymbolScope::Module(module.root_id);
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text)
             .unwrap().variant.as_definition().definition_id;
         self.cur_definition = definition_id;
         let allow_compiler_types = module.is_compiler_file;
         consume_polymorphic_vars_spilled(&module.source, iter, ctx)?;
         // Parse function's argument list
         let mut parameter_section = self.variables.start_section();
         consume_parameter_list(
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, module_scope, definition_id
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section,
             module_scope, definition_id, allow_compiler_types
         )?;
         let parameters = parameter_section.into_vec();
         // Consume return types
         consume_token(&module.source, iter, TokenKind::ArrowRight)?;
         let poly_vars = ctx.heap[definition_id].poly_vars();
         let parser_type = self.type_parser.consume_parser_type(
             iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, definition_id,
             module_scope, false, None
+            module_scope, false, allow_compiler_types, None
         )?;
         // Consume body
         let (body_id, source) = self.consume_procedure_body(module, iter, ctx, definition_id, ProcedureKind::Function)?;
         let scope_id = ctx.heap.alloc_scope(|this| Scope::new(this, ScopeAssociation::Definition(definition_id)));
@@ @@ -305,19 +303,21 @@ impl PassDefinitions { @@
         // Retrieve preallocated definition
         let module_scope = SymbolScope::Module(module.root_id);
         let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text)
             .unwrap().variant.as_definition().definition_id;
         self.cur_definition = definition_id;
         let allow_compiler_types = module.is_compiler_file;
         consume_polymorphic_vars_spilled(&module.source, iter, ctx)?;
         // Parse component's argument list
         let mut parameter_section = self.variables.start_section();
         consume_parameter_list(
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, module_scope, definition_id
             &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section,
             module_scope, definition_id, allow_compiler_types
         )?;
         let parameters = parameter_section.into_vec();
         // Consume body
         let procedure_kind = ctx.heap[definition_id].as_procedure().kind;
         let (body_id, source) = self.consume_procedure_body(module, iter, ctx, definition_id, procedure_kind)?;
@@ @@ -343,16 +343,16 @@ impl PassDefinitions { @@
     fn consume_procedure_body(
         &mut self, module: &Module, iter: &mut TokenIter, ctx: &mut PassCtx, definition_id: DefinitionId, kind: ProcedureKind
     ) -> Result<(BlockStatementId, ProcedureSource), ParseError> {
         if iter.next() == Some(TokenKind::OpenCurly) && iter.peek() == Some(TokenKind::Pragma) {
             // Consume the placeholder "{ #builtin }" tokens
             iter.consume(); // opening curly brace
-            let (pragma, pragma_start, pragma_end) = consume_pragma(&module.source, iter)?;
+            let (pragma, pragma_span) = consume_pragma(&module.source, iter)?;
             if pragma != b"#builtin" {
                 return Err(ParseError::new_error_str_at_span(
-                    &module.source, InputSpan::from_positions(pragma_start, pragma_end),
+                    &module.source, pragma_span,
                     "expected a '#builtin' pragma, or a function body"
                 ));
+            }
             if iter.next() != Some(TokenKind::CloseCurly) {
                 // Just to keep the compiler writers in line ;)
@@ @@ -860,13 +860,13 @@ impl PassDefinitions { @@
             iter.consume();
             let definition_id = self.cur_definition;
             let poly_vars = ctx.heap[definition_id].poly_vars();
             let parser_type = self.type_parser.consume_parser_type(
                 iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars,
                 definition_id, SymbolScope::Module(module.root_id),
                 true, Some(angle_start_pos)
+                true, false, Some(angle_start_pos)
             )?;
             (parser_type.elements, parser_type.full_span.end)
         } else {
             // Assume inferred
+            (
@@ @@ -956,13 +956,14 @@ impl PassDefinitions { @@
         let iter_state = iter.save();
         let definition_id = self.cur_definition;
         let poly_vars = ctx.heap[definition_id].poly_vars();
         let parser_type = self.type_parser.consume_parser_type(
             iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars,
             definition_id, SymbolScope::Definition(definition_id), true, None
             definition_id, SymbolScope::Definition(definition_id),
             true, false, None
         );
         if let Ok(parser_type) = parser_type {
             if Some(TokenKind::Ident) == iter.next() {
                 // Assume this is a proper memory statement
                 let identifier = consume_ident_interned(&module.source, iter, ctx)?;
@@ @@ -1563,13 +1564,13 @@ impl PassDefinitions { @@
                 use ParserTypeVariant as PTV;
                 let symbol_scope = SymbolScope::Definition(self.cur_definition);
                 let poly_vars = ctx.heap[self.cur_definition].poly_vars();
                 let parser_type = self.type_parser.consume_parser_type(
                     iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, self.cur_definition,
                     symbol_scope, true, None
+                    symbol_scope, true, false, None
                 )?;
                 debug_assert!(!parser_type.elements.is_empty());
                 match parser_type.elements[0].variant {
                     PTV::Definition(target_definition_id, _) => {
                         let definition = &ctx.heap[target_definition_id];
                         match definition {
@@ @@ -1728,13 +1729,13 @@ impl PassDefinitions { @@
                         iter.consume();
                         let definition_id = self.cur_definition;
                         let poly_vars = ctx.heap[definition_id].poly_vars();
                         self.type_parser.consume_parser_type(
                             iter, &ctx.heap, &module.source, &ctx.symbols,
                             poly_vars, definition_id, SymbolScope::Module(module.root_id),
                             true, Some(angle_start_pos)
+                            true, false, Some(angle_start_pos)
                         )?
                     } else {
                         // Automatic casting with inferred target type
                         ParserType{
                             elements: vec![ParserTypeElement{
                                 element_span: ident_span,
+}
 /// Consumes the parameter list to functions/components
 fn consume_parameter_list(
     parser: &mut ParserTypeParser, source: &InputSource, iter: &mut TokenIter,
     ctx: &mut PassCtx, target: &mut ScopedSection<VariableId>,
     scope: SymbolScope, definition_id: DefinitionId
+    scope: SymbolScope, definition_id: DefinitionId, allow_compiler_types: bool
 ) -> Result<(), ParseError> {
     consume_comma_separated(
         TokenKind::OpenParen, TokenKind::CloseParen, source, iter, ctx,
         |source, iter, ctx| {
             let poly_vars = ctx.heap[definition_id].poly_vars(); // Rust being rust, multiple lookups
             let parser_type = parser.consume_parser_type(
                 iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id,
                 scope, false, None
+                scope, false, allow_compiler_types, None
             )?;
             let identifier = consume_ident_interned(source, iter, ctx)?;
             let parameter_id = ctx.heap.alloc_variable(|this| Variable{
                 this,
                 kind: VariableKind::Parameter,
                 parser_type,

src/protocol/parser/pass_definitions_types.rs

➞

Show inline comments

@@ @@ -56,27 +56,29 @@ impl ParserTypeParser { @@
+    }
     pub(crate) fn consume_parser_type(
         &mut self, iter: &mut TokenIter, heap: &Heap, source: &InputSource,
         symbols: &SymbolTable, poly_vars: &[Identifier],
         wrapping_definition: DefinitionId, cur_scope: SymbolScope,
         allow_inference: bool, inside_angular_bracket: Option<InputPosition>,
         allow_inference: bool, allow_compiler_types: bool,
         inside_angular_bracket: Option<InputPosition>,
     ) -> Result<ParserType, ParseError> {
         // Prepare
         self.entries.clear();
         self.depths.clear();
         // Setup processing
         if let Some(bracket_pos) = inside_angular_bracket {
             self.push_depth(DepthKind::PolyArgs, u32::MAX, bracket_pos);
+        }
         let initial_state = match iter.next() {
             Some(TokenKind::Ident) => {
+            Some(TokenKind::Ident) | Some(TokenKind::Pragma) => {
                 let element = Self::consume_parser_type_element(
                     iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference
                     iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                     allow_inference, allow_compiler_types
                 )?;
                 self.first_pos = element.element_span.begin;
                 self.last_pos = element.element_span.end;
                 self.entries.push(Entry{
                     element,
@@ @@ -151,13 +153,14 @@ impl ParserTypeParser { @@
+                    }
                 },
                 ParseState::PolyArgStart => {
                     // Allowed tokens: ident (
                     match next {
                         Some(TokenKind::Ident) => self.consume_type_idents(
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference, iter
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                             allow_inference, allow_compiler_types, iter
                         )?,
                         Some(TokenKind::OpenParen) => self.consume_open_paren(iter),
                         _ => return Err(ParseError::new_error_str_at_pos(
                             source, iter.last_valid_pos(),
                             "unexpected token: expected typename or '('"
                         )),
@@ @@ -165,13 +168,14 @@ impl ParserTypeParser { @@
                 },
                 ParseState::TupleStart => {
                     // Allowed tokens: ident ( )
                     // We'll strip the nested tuple later in this function
                     match next {
                         Some(TokenKind::Ident) => self.consume_type_idents(
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference, iter
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                             allow_inference, allow_compiler_types, iter
                         )?,
                         Some(TokenKind::OpenParen) => self.consume_open_paren(iter),
                         Some(TokenKind::CloseParen) => self.consume_close_paren(source, iter)?,
                         _ => return Err(ParseError::new_error_str_at_pos(
                             source, iter.last_valid_pos(),
                             "unexpected token: expected typename or ')'"
@@ @@ -179,13 +183,14 @@ impl ParserTypeParser { @@
+                    }
                 },
                 ParseState::ParsedComma => {
                     // Allowed tokens: ident ( > >> )
                     match next {
                         Some(TokenKind::Ident) => self.consume_type_idents(
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference, iter
                             source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
                             allow_inference, allow_compiler_types, iter
                         )?,
                         Some(TokenKind::OpenParen) => self.consume_open_paren(iter),
                         Some(TokenKind::CloseAngle) => self.consume_close_angle(source, iter)?,
                         Some(TokenKind::ShiftRight) => self.consume_double_close_angle(source, iter)?,
                         Some(TokenKind::CloseParen) => self.consume_close_paren(source, iter)?,
                         _ => return Err(ParseError::new_error_str_at_pos(
@@ @@ -285,16 +290,18 @@ impl ParserTypeParser { @@
     // --- Parsing Utilities
     #[inline]
     fn consume_type_idents(
         &mut self, source: &InputSource, heap: &Heap, symbols: &SymbolTable,
         wrapping_definition: DefinitionId, poly_vars: &[Identifier],
         cur_scope: SymbolScope, allow_inference: bool, iter: &mut TokenIter
         cur_scope: SymbolScope, allow_inference: bool, allow_compiler_types: bool,
         iter: &mut TokenIter
     ) -> Result<(), ParseError> {
         let element = Self::consume_parser_type_element(
             iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope, allow_inference
             iter, source, heap, symbols, wrapping_definition, poly_vars, cur_scope,
             allow_inference, allow_compiler_types
         )?;
         let depth = self.cur_depth();
         self.last_pos = element.element_span.end;
         self.entries.push(Entry{ element, depth });
         self.parse_state = ParseState::TypeMaybePolyArgs;
@@ @@ -425,17 +432,41 @@ impl ParserTypeParser { @@
     /// Consumes a namespaced identifier that should resolve to some kind of
     /// type. There may be commas or polymorphic arguments remaining after this
     /// function has finished.
     fn consume_parser_type_element(
         iter: &mut TokenIter, source: &InputSource, heap: &Heap, symbols: &SymbolTable,
         wrapping_definition: DefinitionId, poly_vars: &[Identifier],
         mut scope: SymbolScope, allow_inference: bool,
+        mut scope: SymbolScope, allow_inference: bool, allow_compiler_types: bool,
     ) -> Result<ParserTypeElement, ParseError> {
         use ParserTypeVariant as PTV;
         let (mut type_text, mut type_span) = consume_any_ident(source, iter)?;
         // Early check for special builtin types available to the compiler
         if iter.next() == Some(TokenKind::Pragma) {
             let (type_text, pragma_span) = consume_pragma(source, iter)?;
             let variant = match type_text {
                 PRAGMA_TYPE_VOID => Some(PTV::Void),
                 PRAGMA_TYPE_PORTLIKE => Some(PTV::InputOrOutput),
                 PRAGMA_TYPE_INTEGERLIKE => Some(PTV::IntegerLike),
                 PRAGMA_TYPE_ARRAYLIKE => Some(PTV::ArrayLike),
                 _ => None,
             };
             if !allow_compiler_types || variant.is_none() {
                 return Err(ParseError::new_error_str_at_span(
                     source, pragma_span, "unexpected pragma in type"
                 ));
+            }
             return Ok(ParserTypeElement{
                 variant: variant.unwrap(),
                 element_span: pragma_span,
             });
+        }
         // No special type, parse as normal
         let (mut type_text, mut type_span) = consume_any_ident(source, iter)?;
         let variant = match type_text {
             KW_TYPE_MESSAGE => PTV::Message,
             KW_TYPE_BOOL => PTV::Bool,
             KW_TYPE_UINT8 => PTV::UInt8,
             KW_TYPE_UINT16 => PTV::UInt16,
             KW_TYPE_UINT32 => PTV::UInt32,

src/protocol/parser/pass_imports.rs

➞

Show inline comments

@@ @@ -22,50 +22,44 @@ impl PassImport { @@
             found_symbols: Vec::with_capacity(32),
             scoped_symbols: Vec::with_capacity(32),
+        }
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let module_range = &module.tokens.ranges[0];
         debug_assert!(modules.iter().all(|m| m.phase >= ModuleCompilationPhase::SymbolsScanned));
         debug_assert_eq!(module.phase, ModuleCompilationPhase::SymbolsScanned);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         let mut range_idx = module_range.first_child_idx;
         loop {
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
         let module_root_id = module.root_id;
         let num_markers = module.tokens.markers.len();
             if cur_range.range_kind == TokenRangeKind::Import {
                 self.visit_import_range(modules, module_idx, ctx, range_idx_usize)?;
+            }
             if cur_range.next_sibling_idx == NO_SIBLING {
                 break;
             } else {
                 range_idx = cur_range.next_sibling_idx;
         for marker_index in 0..num_markers {
             let marker = &modules[module_idx].tokens.markers[marker_index];
             match marker.kind {
                 TokenMarkerKind::Import => {
                     self.visit_import_marker(modules, module_idx, ctx, marker_index)?;
                 },
                 TokenMarkerKind::Definition | TokenMarkerKind::Pragma => {},
+            }
+        }
-        let root = &mut ctx.heap[module.root_id];
+        let root = &mut ctx.heap[module_root_id];
         root.imports.extend(self.imports.drain(..));
         let module = &mut modules[module_idx];
         module.phase = ModuleCompilationPhase::ImportsResolved;
         Ok(())
+    }
     pub(crate) fn visit_import_range(
         &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize
     pub(crate) fn visit_import_marker(
         &mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, marker_index: usize
     ) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let import_range = &module.tokens.ranges[range_idx];
         debug_assert_eq!(import_range.range_kind, TokenRangeKind::Import);
         let marker = &module.tokens.markers[marker_index];
-        let mut iter = module.tokens.iter_range(import_range.start, import_range.end);
+        let mut iter = module.tokens.iter_range(marker.first_token, None);
         // Consume "import"
         let (_import_ident, import_span) =
             consume_any_ident(&module.source, &mut iter)?;
         debug_assert_eq!(_import_ident, KW_IMPORT);
@@ @@ -312,9 +306,15 @@ impl PassImport { @@
         // By now the `import_id` is set, just need to make sure that the import
         // properly ends with a semicolon
         consume_token(&module.source, &mut iter, TokenKind::SemiColon)?;
         self.imports.push(import_id);
         // Update the marker
         let marker_last_token = iter.token_index();
         let marker = &mut modules[module_idx].tokens.markers[marker_index];
         marker.last_token = marker_last_token;
         marker.handled = true;
         Ok(())
+    }
+}

src/protocol/parser/pass_rewriting.rs

➞

Show inline comments

@@ @@ -46,12 +46,16 @@ impl Visitor for PassRewriting { @@
+    }
     // --- Visiting procedures
     fn visit_procedure_definition(&mut self, ctx: &mut Ctx, id: ProcedureDefinitionId) -> VisitorResult {
         let definition = &ctx.heap[id];
         if definition.source.is_builtin() {
             return Ok(());
+        }
         let body_id = definition.body;
         self.current_scope = definition.scope;
         self.current_procedure_id = id;
         return self.visit_block_stmt(ctx, body_id);
+    }

src/protocol/parser/pass_symbols.rs

➞

Show inline comments

@@ @@ -42,51 +42,43 @@ impl PassSymbols { @@
+    }
     pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {
         self.reset();
         let module = &mut modules[module_idx];
-        let module_range = &module.tokens.ranges[0];
+        let module_is_compiler_file = module.is_compiler_file;
         debug_assert_eq!(module.phase, ModuleCompilationPhase::Tokenized);
         debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module);
         debug_assert!(module.root_id.is_invalid()); // not set yet,
         debug_assert!(module.root_id.is_invalid()); // not set yet
         // Preallocate root in the heap
         let root_id = ctx.heap.alloc_protocol_description(|this| {
             Root{
                 this,
                 pragmas: Vec::new(),
                 imports: Vec::new(),
                 definitions: Vec::new(),
+            }
         });
         module.root_id = root_id;
         // Retrieve first range index, then make immutable borrow
         let mut range_idx = module_range.first_child_idx;
         // Visit token ranges to detect definitions and pragmas
         loop {
         // Use pragma token markers to detects symbol definitions and pragmas
         let num_markers = module.tokens.markers.len();
         for marker_index in 0..num_markers {
             let module = &modules[module_idx];
             let range_idx_usize = range_idx as usize;
             let cur_range = &module.tokens.ranges[range_idx_usize];
             let next_sibling_idx = cur_range.next_sibling_idx;
             let range_kind = cur_range.range_kind;
             let marker = &module.tokens.markers[marker_index];
             // Parse if it is a definition or a pragma
             if range_kind == TokenRangeKind::Definition {
                 self.visit_definition_range(modules, module_idx, ctx, range_idx_usize)?;
             } else if range_kind == TokenRangeKind::Pragma {
                 self.visit_pragma_range(modules, module_idx, ctx, range_idx_usize)?;
             match marker.kind {
                 TokenMarkerKind::Pragma => {
                     self.visit_pragma_marker(modules, module_idx, ctx, marker_index)?;
                 },
                 TokenMarkerKind::Definition => {
                     self.visit_definition_marker(modules, module_idx, ctx, marker_index)?;
+                }
             if next_sibling_idx == NO_SIBLING {
                 break;
             } else {
                 range_idx = next_sibling_idx;
                 TokenMarkerKind::Import => {}, // we don't care yet
+            }
+        }
         // Add the module's symbol scope and the symbols we just parsed
         let module_scope = SymbolScope::Module(root_id);
         ctx.symbols.insert_scope(Some(SymbolScope::Global), module_scope);
@@ @@ -94,50 +86,53 @@ impl PassSymbols { @@
             ctx.symbols.insert_scope(Some(module_scope), SymbolScope::Definition(symbol.variant.as_definition().definition_id));
             if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(module_scope, symbol) {
                 return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, &old_symbol))
+            }
+        }
         if module_is_compiler_file {
             debug_assert!(self.symbols.is_empty());
             ctx.symbols.get_all_symbols_defined_in_scope(module_scope, &mut self.symbols);
             for symbol in self.symbols.drain(..) {
                 ctx.symbols.insert_symbol_in_global_scope(symbol);
+            }
+        }
         // Modify the preallocated root
         let root = &mut ctx.heap[root_id];
         root.pragmas.extend(self.pragmas.drain(..));
         root.definitions.extend(self.definitions.drain(..));
         // Modify module
         let module = &mut modules[module_idx];
         module.phase = ModuleCompilationPhase::SymbolsScanned;
         Ok(())
+    }
-    fn visit_pragma_range(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize) -> Result<(), ParseError> {
+    fn visit_pragma_marker(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, marker_index: usize) -> Result<(), ParseError> {
         let module = &mut modules[module_idx];
         let range = &module.tokens.ranges[range_idx];
         let mut iter = module.tokens.iter_range(range.start, module.tokens.tokens.len() as u32);
         let marker = &module.tokens.markers[marker_index];
         let mut iter = module.tokens.iter_range(marker.first_token, None);
         // Consume pragma name
-        let (pragma_section, pragma_start, _) = consume_pragma(&module.source, &mut iter)?;
+        let (pragma_section, mut pragma_span) = consume_pragma(&module.source, &mut iter)?;
         // Consume pragma values
         if pragma_section == b"#module" {
             // Check if name is defined twice within the same file
             if self.has_pragma_module {
-                return Err(ParseError::new_error_str_at_pos(&module.source, pragma_start, "module name is defined twice"));
+                return Err(ParseError::new_error_str_at_span(&module.source, pragma_span, "module name is defined twice"));
+            }
             // Consume the domain-name
+            // Consume the domain-name, then record end of pragma
             let (module_name, module_span) = consume_domain_ident(&module.source, &mut iter)?;
             // TODO: Fix with newer token range parsing
             module.tokens.ranges[range_idx as usize].end = iter.token_index();
             // if iter.next().is_some() {
             //     return Err(ParseError::new_error_str_at_pos(&module.source, iter.last_valid_pos(), "expected end of #module pragma after module name"));
             // }
             let marker_last_token = iter.token_index();
             // Add to heap and symbol table
-            let pragma_span = InputSpan::from_positions(pragma_start, module_span.end);
+            pragma_span.end = module_span.end;
             let module_name = ctx.pool.intern(module_name);
             let pragma_id = ctx.heap.alloc_pragma(|this| Pragma::Module(PragmaModule{
                 this,
                 span: pragma_span,
                 value: Identifier{ span: module_span, value: module_name.clone() },
             }));
@@ @@ -153,55 +148,57 @@ impl PassSymbols { @@
                     &this_module.source, pragma_span, "conflict in module name"
                 ).with_info_str_at_span(
                     &other_module.source, other_pragma.span, "other module is defined here"
                 ));
+            }
             let marker = &mut module.tokens.markers[marker_index];
             marker.last_token = marker_last_token;
             marker.handled = true;
             module.name = Some((pragma_id, module_name));
             self.has_pragma_module = true;
         } else if pragma_section == b"#version" {
             // Check if version is defined twice within the same file
             if self.has_pragma_version {
-                return Err(ParseError::new_error_str_at_pos(&module.source, pragma_start, "module version is defined twice"));
+                return Err(ParseError::new_error_str_at_span(&module.source, pragma_span, "module version is defined twice"));
+            }
             // Consume the version pragma
             let (version, version_span) = consume_integer_literal(&module.source, &mut iter, &mut self.buffer)?;
             let marker_last_token = iter.token_index();
             pragma_span.end = version_span.end;
             let pragma_id = ctx.heap.alloc_pragma(|this| Pragma::Version(PragmaVersion{
                 this,
-                span: InputSpan::from_positions(pragma_start, version_span.end),
+                span: pragma_span,
                 version,
             }));
             self.pragmas.push(pragma_id);
             let marker = &mut module.tokens.markers[marker_index];
             marker.last_token = marker_last_token;
             marker.handled = true;
             module.version = Some((pragma_id, version as i64));
             self.has_pragma_version = true;
         } else {
             // Custom pragma, maybe we support this in the future, but for now
             // we don't.
             return Err(ParseError::new_error_str_at_pos(&module.source, pragma_start, "illegal pragma name"));
+        }
         } // else: custom pragma used for something else, will be handled later (or rejected with an error)
         Ok(())
+    }
-    fn visit_definition_range(&mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize) -> Result<(), ParseError> {
+    fn visit_definition_marker(&mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, marker_index: usize) -> Result<(), ParseError> {
         let module = &modules[module_idx];
         let range = &module.tokens.ranges[range_idx];
         let definition_span = InputSpan::from_positions(
             module.tokens.start_pos(range),
             module.tokens.end_pos(range)
         );
         let mut iter = module.tokens.iter_range(range.start, range.end);
         let marker = &module.tokens.markers[marker_index];
         let mut iter = module.tokens.iter_range(marker.first_token, None);
         // First ident must be type of symbol
         let (kw_text, _) = consume_any_ident(&module.source, &mut iter).unwrap();
         // Retrieve identifier of definition
         let identifier = consume_ident_interned(&module.source, &mut iter, ctx)?;
         println!("DEBUG: Parsing {} --- {}", String::from_utf8_lossy(kw_text).to_string(), identifier.value.as_str());
         let mut poly_vars = Vec::new();
         maybe_consume_comma_separated(
             TokenKind::OpenAngle, TokenKind::CloseAngle, &module.source, &mut iter, ctx,
             |source, iter, ctx| consume_ident_interned(source, iter, ctx),
             &mut poly_vars, "a polymorphic variable", None
         )?;
@@ @@ -211,59 +208,58 @@ impl PassSymbols { @@
         // Reserve space in AST for definition and add it to the symbol table
         let definition_class;
         let ast_definition_id;
         match kw_text {
             KW_STRUCT => {
                 let struct_def_id = ctx.heap.alloc_struct_definition(|this| {
-                    StructDefinition::new_empty(this, module.root_id, definition_span, identifier, poly_vars)
                     StructDefinition::new_empty(this, module.root_id, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Struct;
                 ast_definition_id = struct_def_id.upcast();
             },
             KW_ENUM => {
                 let enum_def_id = ctx.heap.alloc_enum_definition(|this| {
-                    EnumDefinition::new_empty(this, module.root_id, definition_span, identifier, poly_vars)
                     EnumDefinition::new_empty(this, module.root_id, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Enum;
                 ast_definition_id = enum_def_id.upcast();
             },
             KW_UNION => {
                 let union_def_id = ctx.heap.alloc_union_definition(|this| {
-                    UnionDefinition::new_empty(this, module.root_id, definition_span, identifier, poly_vars)
                     UnionDefinition::new_empty(this, module.root_id, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Union;
                 ast_definition_id = union_def_id.upcast()
             },
             KW_FUNCTION => {
                 let proc_def_id = ctx.heap.alloc_procedure_definition(|this| {
-                    ProcedureDefinition::new_empty(this, module.root_id, definition_span, ProcedureKind::Function, identifier, poly_vars)
                     ProcedureDefinition::new_empty(this, module.root_id, ProcedureKind::Function, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Function;
                 ast_definition_id = proc_def_id.upcast();
             },
             KW_PRIMITIVE | KW_COMPOSITE => {
                 let procedure_kind = if kw_text == KW_PRIMITIVE {
                     ProcedureKind::Primitive
                 } else {
                     ProcedureKind::Composite
                 };
                 let proc_def_id = ctx.heap.alloc_procedure_definition(|this| {
-                    ProcedureDefinition::new_empty(this, module.root_id, definition_span, procedure_kind, identifier, poly_vars)
                     ProcedureDefinition::new_empty(this, module.root_id, procedure_kind, identifier, poly_vars)
                 });
                 definition_class = DefinitionClass::Component;
                 ast_definition_id = proc_def_id.upcast();
             },
             _ => unreachable!("encountered keyword '{}' in definition range", String::from_utf8_lossy(kw_text)),
+        }
         let symbol = Symbol{
             name: ident_text,
             variant: SymbolVariant::Definition(SymbolDefinition{
                 defined_in_module: module.root_id,
                 defined_in_scope: SymbolScope::Module(module.root_id),
                 definition_span,
                 identifier_span: ident_span,
                 imported_at: None,
                 class: definition_class,
                 definition_id: ast_definition_id,
             }),
         };

src/protocol/parser/pass_tokenizer.rs

➞

Show inline comments

@@ @@ -63,22 +63,25 @@ impl PassTokenizer { @@
                 self.consume_char_literal(source, target)?;
             } else if is_string_literal_start(c) {
                 self.consume_string_literal(source, target)?;
             } else if is_identifier_start(c) {
                 let ident = self.consume_identifier(source, target)?;
                 if demarks_definition(ident) {
                 if demarks_symbol(ident) {
                     self.emit_marker(target, TokenMarkerKind::Definition, token_index);
                     self.push_range(target, TokenRangeKind::Definition, token_index);
                 } else if demarks_import(ident) {
                     self.emit_marker(target, TokenMarkerKind::Import, token_index);
                     self.push_range(target, TokenRangeKind::Import, token_index);
+                }
             } else if is_integer_literal_start(c) {
                 self.consume_number(source, target)?;
             } else if is_pragma_start_or_pound(c) {
                 let was_pragma = self.consume_pragma_or_pound(c, source, target)?;
                 if was_pragma {
                     self.emit_marker(target, TokenMarkerKind::Pragma, token_index);
                     self.push_range(target, TokenRangeKind::Pragma, token_index);
+                }
             } else if self.is_line_comment_start(c, source) {
                 self.consume_line_comment(source, target)?;
             } else if self.is_block_comment_start(c, source) {
                 self.consume_block_comment(source, target)?;
@@ @@ -652,12 +655,28 @@ impl PassTokenizer { @@
         if sibling_idx != NO_RELATION {
             let sibling_range = &mut target.ranges[sibling_idx as usize];
             sibling_range.next_sibling_idx = new_range_idx;
+        }
+    }
     fn emit_marker(&mut self, target: &mut TokenBuffer, kind: TokenMarkerKind, first_token: u32) {
         debug_assert!(
             target.markers
                 .last().map(|v| v.first_token < first_token)
                 .unwrap_or(true)
         );
         target.markers.push(TokenMarker{
             kind,
             curly_depth: self.curly_stack.len() as u32,
             first_token,
             last_token: u32::MAX,
             handled: false,
         });
+    }
     fn push_range(&mut self, target: &mut TokenBuffer, range_kind: TokenRangeKind, first_token_idx: u32) {
         let new_range_idx = target.ranges.len() as i32;
         let parent_idx = self.stack_idx as i32;
         let parent_range = &mut target.ranges[self.stack_idx];
         if parent_range.first_child_idx == NO_RELATION {
@@ @@ -724,13 +743,13 @@ impl PassTokenizer { @@
             },
+        }
+    }
+}
 // Helpers for characters
-fn demarks_definition(ident: &[u8]) -> bool {
+fn demarks_symbol(ident: &[u8]) -> bool {
     return
         ident == KW_STRUCT ||
             ident == KW_ENUM ||
             ident == KW_UNION ||
             ident == KW_FUNCTION ||
             ident == KW_PRIMITIVE ||

src/protocol/parser/pass_typing.rs

➞

Show inline comments

@@ @@ -1220,12 +1220,13 @@ impl PassTyping { @@
     fn visit_procedure_definition(&mut self, ctx: &mut Ctx, id: ProcedureDefinitionId) -> VisitorResult {
         let procedure_def = &ctx.heap[id];
         self.procedure_id = id;
         self.procedure_kind = procedure_def.kind;
         let body_id = procedure_def.body;
         let procedure_is_builtin = procedure_def.source.is_builtin();
         debug_log!("{}", "-".repeat(50));
         debug_log!("Visiting procedure: '{}' (id: {}, kind: {:?})", procedure_def.identifier.value.as_str(), id.0.index, procedure_def.kind);
         debug_log!("{}", "-".repeat(50));
         // Visit parameters
@@ @@ -1242,13 +1243,17 @@ impl PassTyping { @@
             })
+        }
         section.forget();
         // Visit all of the expressions within the body
         self.parent_index = None;
         if !procedure_is_builtin {
             return self.visit_block_stmt(ctx, body_id);
         } else {
             return Ok(());
+        }
+    }
     // Statements
     fn visit_stmt(&mut self, ctx: &mut Ctx, id: StatementId) -> VisitorResult {
         return visitor_recursive_statement_impl!(self, &ctx.heap[id], ctx, Ok(()));

src/protocol/parser/pass_validation_linking.rs

➞

Show inline comments

@@ @@ -197,21 +197,25 @@ impl Visitor for PassValidationLinking { @@
         // Visit parameters
         let scope_id = definition.scope;
         let old_scope = self.push_scope(ctx, true, scope_id);
         let definition = &ctx.heap[id];
         let body_id = definition.body;
         let definition_is_builtin = definition.source.is_builtin();
         let section = self.variable_buffer.start_section_initialized(&definition.parameters);
         for variable_idx in 0..section.len() {
             let variable_id = section[variable_idx];
             self.checked_at_single_scope_add_local(ctx, self.cur_scope, -1, variable_id)?;
+        }
         section.forget();
         // Visit statements in function body
         // Visit statements in function body, if present at all
         if !definition_is_builtin {
             self.visit_block_stmt(ctx, body_id)?;
+        }
         self.pop_scope(old_scope);
         self.resolve_pending_control_flow_targets(ctx)?;
         Ok(())
+    }

src/protocol/parser/symbol_table.rs

➞

Show inline comments

@@ @@ -82,13 +82,12 @@ pub struct SymbolModule { @@
 pub struct SymbolDefinition {
     // Definition location (not necessarily the place where the symbol
     // is introduced, as it may be imported). Builtin symbols will have invalid
     // spans and module IDs
     pub defined_in_module: RootId,
     pub defined_in_scope: SymbolScope,
     pub definition_span: InputSpan, // full span of definition
     pub identifier_span: InputSpan, // span of just the identifier
     // Location where the symbol is introduced in its scope
     pub imported_at: Option<ImportId>,
     // Definition in the heap, with a utility enum to determine its
     // class if the ID is not needed.
     pub class: DefinitionClass,
@@ @@ -228,12 +227,20 @@ impl SymbolTable { @@
         // If here, then there is no collision
         let scoped_symbols = self.scope_lookup.get_mut(&in_scope).unwrap();
         scoped_symbols.symbols.push(symbol);
         Ok(())
+    }
     /// Insert a symbol in the global scope. Naturally there will be a
     /// collision (as the symbol originates from a module), so we do *not* check
     /// for this.
     pub(crate) fn insert_symbol_in_global_scope(&mut self, symbol: Symbol) {
         let scoped_symbols = self.scope_lookup.get_mut(&SymbolScope::Global).unwrap();
         scoped_symbols.symbols.push(symbol);
+    }
     /// Retrieves a symbol by name by searching in a particular scope and that scope's parents. The
     /// returned symbol may both be imported as defined within any of the searched scopes.
     pub(crate) fn get_symbol_by_name(
         &self, mut in_scope: SymbolScope, name: &[u8]
     ) -> Option<&Symbol> {
         let string_ref = StringRef::new(name);

src/protocol/parser/token_parsing.rs

➞

Show inline comments

 pub(crate) const KW_TYPE_SINT32:   &'static [u8] = KW_TYPE_SINT32_STR.as_bytes();
 pub(crate) const KW_TYPE_SINT64:   &'static [u8] = KW_TYPE_SINT64_STR.as_bytes();
 pub(crate) const KW_TYPE_CHAR:     &'static [u8] = KW_TYPE_CHAR_STR.as_bytes();
 pub(crate) const KW_TYPE_STRING:   &'static [u8] = KW_TYPE_STRING_STR.as_bytes();
 pub(crate) const KW_TYPE_INFERRED: &'static [u8] = KW_TYPE_INFERRED_STR.as_bytes();
 // Builtin pragma types
 // Not usable by the programmer, but usable in the standard library. These hint
 // at the fact that we need a different system (e.g. function overloading)
 pub(crate) const PRAGMA_TYPE_VOID: &'static [u8] = b"#type_void";
 pub(crate) const PRAGMA_TYPE_PORTLIKE: &'static [u8] = b"#type_portlike";
 pub(crate) const PRAGMA_TYPE_INTEGERLIKE: &'static [u8] = b"#type_integerlike";
 pub(crate) const PRAGMA_TYPE_ARRAYLIKE: &'static [u8] = b"#type_arraylike";
 /// A special trait for when consuming comma-separated things such that we can
 /// push them onto a `Vec` and onto a `ScopedSection`. As we monomorph for
 /// very specific comma-separated cases I don't expect polymorph bloat.
 /// Also, I really don't like this solution.
 pub(crate) trait Extendable {
     type Value;
             return Err(ParseError::new_error_at_span(source, literal_span, msg))
         },
     };
     Ok(result)
+}
-pub(crate) fn consume_pragma<'a>(source: &'a InputSource, iter: &mut TokenIter) -> Result<(&'a [u8], InputPosition, InputPosition), ParseError> {
+pub(crate) fn consume_pragma<'a>(source: &'a InputSource, iter: &mut TokenIter) -> Result<(&'a [u8], InputSpan), ParseError> {
     if Some(TokenKind::Pragma) != iter.next() {
         return Err(ParseError::new_error_str_at_pos(source, iter.last_valid_pos(), "expected a pragma"));
+    }
-    let (pragma_start, pragma_end) = iter.next_positions();
+    let pragma_span = iter.next_span();
     iter.consume();
-    Ok((source.section_at_pos(pragma_start, pragma_end), pragma_start, pragma_end))
+    Ok((source.section_at_span(pragma_span), pragma_span))
+}
 pub(crate) fn has_ident(source: &InputSource, iter: &mut TokenIter, expected: &[u8]) -> bool {
     peek_ident(source, iter).map_or(false, |section| section == expected)
+}

src/protocol/parser/tokens.rs

➞

Show inline comments

@@ @@ -167,12 +167,35 @@ pub struct Token { @@
 impl Token {
     pub(crate) fn new(kind: TokenKind, pos: InputPosition) -> Self {
         Self{ kind, pos }
+    }
+}
 #[derive(Debug, Clone, Copy)]
 pub enum TokenMarkerKind {
     Pragma,
     Import,
     Definition,
+}
 /// A marker for a specific token. These are stored separately from the array of
 /// tokens. These are used for initial symbol, module name, and import
 /// discovery.
 #[derive(Debug)]
 pub struct TokenMarker {
     pub kind: TokenMarkerKind,
     pub curly_depth: u32,
     // Indices into token buffer. The first token is inclusive and set upon
     // tokenization, the last token is set at a later stage in parsing (e.g.
     // at symbol discovery we may parse some of the `Pragma` tokens and set the
     // last parsed token)
     pub first_token: u32,
     pub last_token: u32,
     pub handled: bool,
+}
 /// The kind of token ranges that are specially parsed by the tokenizer.
 #[derive(Debug, Clone, Copy, PartialEq, Eq)]
 pub enum TokenRangeKind {
     Module,
     Pragma,
     Import,
@@ @@ -201,37 +224,31 @@ pub struct TokenRange { @@
     pub last_child_idx: i32,    // Last subrange (or -1 if no subranges)
     pub next_sibling_idx: i32,  // Next subrange (or -1 if no next subrange)
+}
 pub struct TokenBuffer {
     pub tokens: Vec<Token>,
     pub markers: Vec<TokenMarker>,
     pub ranges: Vec<TokenRange>,
+}
 impl TokenBuffer {
     pub(crate) fn new() -> Self {
         Self{ tokens: Vec::new(), ranges: Vec::new() }
+    }
     pub(crate) fn iter_range<'a>(&'a self, inclusive_start: u32, exclusive_end: u32) -> TokenIter<'a> {
         debug_assert!(exclusive_end as usize <= self.tokens.len());
         TokenIter::new(self, inclusive_start as usize, exclusive_end as usize)
+    }
     pub(crate) fn start_pos(&self, range: &TokenRange) -> InputPosition {
         self.tokens[range.start as usize].pos
         return Self{
             tokens: Vec::new(),
             markers: Vec::new(),
             ranges: Vec::new()
         };
+    }
     pub(crate) fn end_pos(&self, range: &TokenRange) -> InputPosition {
         let last_token = &self.tokens[range.end as usize - 1];
         if last_token.kind == TokenKind::SpanEnd {
             return last_token.pos
         } else {
             debug_assert!(!last_token.kind.has_span_end());
             return last_token.pos.with_offset(last_token.kind.num_characters());
+        }
     pub(crate) fn iter_range(
         &self, inclusive_start: u32, exclusive_end: Option<u32>
     ) -> TokenIter {
         let exclusive_end = exclusive_end.unwrap_or(self.tokens.len() as u32) as usize;
         debug_assert!(exclusive_end <= self.tokens.len());
         TokenIter::new(self, inclusive_start as usize, exclusive_end)
+    }
+}
 /// Iterator over tokens within a specific `TokenRange`.
 pub(crate) struct TokenIter<'a> {
     tokens: &'a Vec<Token>,

src/protocol/tests/utils.rs

➞

Show inline comments

@@ @@ -597,13 +597,14 @@ impl<'a> FunctionTester<'a> { @@
         let module = seek_def_in_modules(
             &self.ctx.heap, &self.ctx.modules, self.def.this.upcast()
         ).unwrap();
         // Find the first occurrence of the expression after the definition of
         // the function, we'll check that it is included in the body later.
         let mut outer_match_idx = self.def.span.begin.offset as usize;
         let body = &self.ctx.heap[self.def.body];
         let mut outer_match_idx = body.span.begin.offset as usize;
         while outer_match_idx < module.source.input.len() {
             if module.source.input[outer_match_idx..].starts_with(outer_match.as_bytes()) {
                 break;
+            }
             outer_match_idx += 1
+        }

src/protocol/token_writer.rs

➞

Show inline comments

@@ @@ -2,13 +2,13 @@ @@
 use std::fmt::Write;
 use std::io::Write as IOWrite;
 use crate::protocol::input_source::{InputSource, InputSpan};
 use crate::protocol::parser::Module;
-use crate::protocol::tokens::{Token, TokenBuffer, TokenKind, TokenRange, TokenIter, TokenRangeKind};
 use crate::protocol::tokens::{Token, TokenKind, TokenRange};
 pub(crate) struct TokenWriter {
     buffer: String,
+}
 impl TokenWriter {

std/std.global.pdl

➞

Show inline comments

@@ @@ -4,11 +4,11 @@ @@
 // spaces to work with the current system. Needs to be a system where the
 // pragmas, "func" keywords (and similar keywords) indicate initial points to
 // start parsing.
 func get<T>(in<T> input) -> T { #builtin }
 func put<T>(out<T> output, T value) -> #type_void { #builtin }
 func fires<T>(#type_portlike <T>) -> bool { #builtin }
 func create<T>(#type_integerlike length) -> T[] { #builtin }
 func fires<T>(#type_portlike<T> port) -> bool { #builtin }
 func create<T>(#type_integerlike len) -> T[] { #builtin }
 func length<T>(#type_arraylike<T> array) -> u32 { #builtin }
 func assert(bool condition) -> #type_void { #builtin }
 func print(string message) -> #type_void { #builtin }
@@ \ No newline at end of file @@

0 comments (0 inline, 0 general)