diff --git a/src/protocol/parser/pass_definitions.rs b/src/protocol/parser/pass_definitions.rs index 607f8caacc21fb71b60c8bca23e60ec45a27fa5d..7f9817270909d65c1cfc3c9fa702aef204e1506e 100644 --- a/src/protocol/parser/pass_definitions.rs +++ b/src/protocol/parser/pass_definitions.rs @@ -43,35 +43,33 @@ impl PassDefinitions { pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> { let module = &modules[module_idx]; - let module_range = &module.tokens.ranges[0]; debug_assert_eq!(module.phase, ModuleCompilationPhase::ImportsResolved); - debug_assert_eq!(module_range.range_kind, TokenRangeKind::Module); - // Although we only need to parse the definitions, we want to go through - // code ranges as well such that we can throw errors if we get - // unexpected tokens at the module level of the source. - let mut range_idx = module_range.first_child_idx; - loop { - let range_idx_usize = range_idx as usize; - let cur_range = &module.tokens.ranges[range_idx_usize]; - - match cur_range.range_kind { - TokenRangeKind::Module => unreachable!(), // should not be reachable - TokenRangeKind::Pragma | TokenRangeKind::Import => { - // Already fully parsed, fall through and go to next range - }, - TokenRangeKind::Definition | TokenRangeKind::Code => { - // Visit range even if it is a "code" range to provide - // proper error messages. - self.visit_range(modules, module_idx, ctx, range_idx_usize)?; - }, + // We iterate through the entire document. If we find a marker that has + // been handled then we skip over it. It is important that we properly + // parse all other tokens in the document to ensure that we throw the + // correct kind of errors. + let num_tokens = module.tokens.tokens.len() as u32; + let num_markers = module.tokens.markers.len(); + + let mut marker_index = 0; + let mut first_token_index = 0; + while first_token_index < num_tokens { + // Seek ahead to the next marker that was already handled. + let mut last_token_index = num_tokens; + let mut new_first_token_index = num_tokens; + while marker_index < num_markers { + let marker = &module.tokens.markers[marker_index]; + marker_index += 1; + if marker.handled { + last_token_index = marker.first_token; + new_first_token_index = marker.last_token; + break; + } } - if cur_range.next_sibling_idx == NO_SIBLING { - break; - } else { - range_idx = cur_range.next_sibling_idx; - } + self.visit_token_range(modules, module_idx, ctx, first_token_index, last_token_index)?; + first_token_index = new_first_token_index; } modules[module_idx].phase = ModuleCompilationPhase::DefinitionsParsed; @@ -79,15 +77,14 @@ impl PassDefinitions { Ok(()) } - fn visit_range( - &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize + fn visit_token_range( + &mut self, modules: &[Module], module_idx: usize, ctx: &mut PassCtx, + token_range_begin: u32, token_range_end: u32, ) -> Result<(), ParseError> { let module = &modules[module_idx]; - let cur_range = &module.tokens.ranges[range_idx]; - debug_assert!(cur_range.range_kind == TokenRangeKind::Definition || cur_range.range_kind == TokenRangeKind::Code); // Detect which definition we're parsing - let mut iter = module.tokens.iter_range(cur_range.start, cur_range.end); + let mut iter = module.tokens.iter_range(token_range_begin, Some(token_range_end)); loop { let next = iter.next(); if next.is_none() { @@ -134,7 +131,7 @@ impl PassDefinitions { let start_pos = iter.last_valid_pos(); let parser_type = self.type_parser.consume_parser_type( iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id, - module_scope, false, None + module_scope, false, false, None )?; let field = consume_ident_interned(source, iter, ctx)?; Ok(StructFieldDefinition{ @@ -221,7 +218,7 @@ impl PassDefinitions { let poly_vars = ctx.heap[definition_id].poly_vars(); self.type_parser.consume_parser_type( iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id, - module_scope, false, None + module_scope, false, false, None ) }, &mut types_section, "an embedded type", Some(&mut close_pos) @@ -255,20 +252,21 @@ impl PassDefinitions { // Retrieve function name consume_exact_ident(&module.source, iter, KW_FUNCTION)?; let (ident_text, _) = consume_ident(&module.source, iter)?; - let stringy = String::from_utf8_lossy(ident_text).to_string(); // Retrieve preallocated DefinitionId let module_scope = SymbolScope::Module(module.root_id); let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text) .unwrap().variant.as_definition().definition_id; self.cur_definition = definition_id; + let allow_compiler_types = module.is_compiler_file; consume_polymorphic_vars_spilled(&module.source, iter, ctx)?; // Parse function's argument list let mut parameter_section = self.variables.start_section(); consume_parameter_list( - &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, module_scope, definition_id + &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, + module_scope, definition_id, allow_compiler_types )?; let parameters = parameter_section.into_vec(); @@ -277,7 +275,7 @@ impl PassDefinitions { let poly_vars = ctx.heap[definition_id].poly_vars(); let parser_type = self.type_parser.consume_parser_type( iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, definition_id, - module_scope, false, None + module_scope, false, allow_compiler_types, None )?; // Consume body @@ -308,13 +306,15 @@ impl PassDefinitions { let definition_id = ctx.symbols.get_symbol_by_name_defined_in_scope(module_scope, ident_text) .unwrap().variant.as_definition().definition_id; self.cur_definition = definition_id; + let allow_compiler_types = module.is_compiler_file; consume_polymorphic_vars_spilled(&module.source, iter, ctx)?; // Parse component's argument list let mut parameter_section = self.variables.start_section(); consume_parameter_list( - &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, module_scope, definition_id + &mut self.type_parser, &module.source, iter, ctx, &mut parameter_section, + module_scope, definition_id, allow_compiler_types )?; let parameters = parameter_section.into_vec(); @@ -346,10 +346,10 @@ impl PassDefinitions { if iter.next() == Some(TokenKind::OpenCurly) && iter.peek() == Some(TokenKind::Pragma) { // Consume the placeholder "{ #builtin }" tokens iter.consume(); // opening curly brace - let (pragma, pragma_start, pragma_end) = consume_pragma(&module.source, iter)?; + let (pragma, pragma_span) = consume_pragma(&module.source, iter)?; if pragma != b"#builtin" { return Err(ParseError::new_error_str_at_span( - &module.source, InputSpan::from_positions(pragma_start, pragma_end), + &module.source, pragma_span, "expected a '#builtin' pragma, or a function body" )); } @@ -863,7 +863,7 @@ impl PassDefinitions { let parser_type = self.type_parser.consume_parser_type( iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, definition_id, SymbolScope::Module(module.root_id), - true, Some(angle_start_pos) + true, false, Some(angle_start_pos) )?; (parser_type.elements, parser_type.full_span.end) @@ -959,7 +959,8 @@ impl PassDefinitions { let parser_type = self.type_parser.consume_parser_type( iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, - definition_id, SymbolScope::Definition(definition_id), true, None + definition_id, SymbolScope::Definition(definition_id), + true, false, None ); if let Ok(parser_type) = parser_type { @@ -1566,7 +1567,7 @@ impl PassDefinitions { let poly_vars = ctx.heap[self.cur_definition].poly_vars(); let parser_type = self.type_parser.consume_parser_type( iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, self.cur_definition, - symbol_scope, true, None + symbol_scope, true, false, None )?; debug_assert!(!parser_type.elements.is_empty()); match parser_type.elements[0].variant { @@ -1731,7 +1732,7 @@ impl PassDefinitions { self.type_parser.consume_parser_type( iter, &ctx.heap, &module.source, &ctx.symbols, poly_vars, definition_id, SymbolScope::Module(module.root_id), - true, Some(angle_start_pos) + true, false, Some(angle_start_pos) )? } else { // Automatic casting with inferred target type @@ -1867,7 +1868,7 @@ fn consume_polymorphic_vars_spilled(source: &InputSource, iter: &mut TokenIter, fn consume_parameter_list( parser: &mut ParserTypeParser, source: &InputSource, iter: &mut TokenIter, ctx: &mut PassCtx, target: &mut ScopedSection, - scope: SymbolScope, definition_id: DefinitionId + scope: SymbolScope, definition_id: DefinitionId, allow_compiler_types: bool ) -> Result<(), ParseError> { consume_comma_separated( TokenKind::OpenParen, TokenKind::CloseParen, source, iter, ctx, @@ -1875,7 +1876,7 @@ fn consume_parameter_list( let poly_vars = ctx.heap[definition_id].poly_vars(); // Rust being rust, multiple lookups let parser_type = parser.consume_parser_type( iter, &ctx.heap, source, &ctx.symbols, poly_vars, definition_id, - scope, false, None + scope, false, allow_compiler_types, None )?; let identifier = consume_ident_interned(source, iter, ctx)?; let parameter_id = ctx.heap.alloc_variable(|this| Variable{