CSY/reowolf Changeset - 6d6c5b5f07ae · Centrum Wiskunde & Informatica (CWI)

Changeset - 6d6c5b5f07ae

Parent rev.

Child rev.

[Not reviewed]

1 6 0

MH - 3 years ago 2022-03-28 22:18:20
contact@maxhenger.nl

Attempting to fix token tree construction

7 files changed with 24 insertions and 514 deletions:

src/protocol/parser/pass_definitions.rs

src/protocol/parser/pass_imports.rs

src/protocol/parser/pass_symbols.rs

src/protocol/parser/pass_tokenizer.rs

src/protocol/parser/tokens.rs

std/std.global.pdl

tokens.txt

500

0 comments (0 inline, 0 general)

src/protocol/parser/pass_definitions.rs

➞

Show inline comments

@@ @@ -87,7 +87,7 @@ impl PassDefinitions { @@
         debug_assert!(cur_range.range_kind == TokenRangeKind::Definition || cur_range.range_kind == TokenRangeKind::Code);
         // Detect which definition we're parsing
         let mut iter = module.tokens.iter_range(cur_range);
+        let mut iter = module.tokens.iter_range(cur_range.start, cur_range.end);
         loop {
             let next = iter.next();
             if next.is_none() {

src/protocol/parser/pass_imports.rs

➞

Show inline comments

@@ @@ -62,7 +62,7 @@ impl PassImport { @@
         let import_range = &module.tokens.ranges[range_idx];
         debug_assert_eq!(import_range.range_kind, TokenRangeKind::Import);
         let mut iter = module.tokens.iter_range(import_range);
+        let mut iter = module.tokens.iter_range(import_range.start, import_range.end);
         // Consume "import"
         let (_import_ident, import_span) =

src/protocol/parser/pass_symbols.rs

➞

Show inline comments

@@ @@ -112,7 +112,7 @@ impl PassSymbols { @@
     fn visit_pragma_range(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, range_idx: usize) -> Result<(), ParseError> {
         let module = &mut modules[module_idx];
         let range = &module.tokens.ranges[range_idx];
         let mut iter = module.tokens.iter_range(range);
+        let mut iter = module.tokens.iter_range(range.start, module.tokens.tokens.len() as u32);
         // Consume pragma name
         let (pragma_section, pragma_start, _) = consume_pragma(&module.source, &mut iter)?;
@@ @@ -126,9 +126,12 @@ impl PassSymbols { @@
             // Consume the domain-name
             let (module_name, module_span) = consume_domain_ident(&module.source, &mut iter)?;
             if iter.next().is_some() {
                 return Err(ParseError::new_error_str_at_pos(&module.source, iter.last_valid_pos(), "expected end of #module pragma after module name"));
+            }
             // TODO: Fix with newer token range parsing
             module.tokens.ranges[range_idx as usize].end = iter.token_index();
             // if iter.next().is_some() {
             //     return Err(ParseError::new_error_str_at_pos(&module.source, iter.last_valid_pos(), "expected end of #module pragma after module name"));
             // }
             // Add to heap and symbol table
             let pragma_span = InputSpan::from_positions(pragma_start, module_span.end);
@@ @@ -188,7 +191,7 @@ impl PassSymbols { @@
             module.tokens.start_pos(range),
             module.tokens.end_pos(range)
         );
         let mut iter = module.tokens.iter_range(range);
+        let mut iter = module.tokens.iter_range(range.start, range.end);
         // First ident must be type of symbol
         let (kw_text, _) = consume_any_ident(&module.source, &mut iter).unwrap();

src/protocol/parser/pass_tokenizer.rs

➞

Show inline comments

@@ @@ -83,13 +83,11 @@ impl PassTokenizer { @@
             } else if self.is_block_comment_start(c, source) {
                 self.consume_block_comment(source, target)?;
             } else if is_whitespace(c) {
                 let contained_newline = self.consume_whitespace(source);
                 if contained_newline {
                 self.consume_whitespace(source);
                 let range = &target.ranges[self.stack_idx];
                 if range.range_kind == TokenRangeKind::Pragma {
                     self.pop_range(target, target.tokens.len() as u32);
+                }
+                }
             } else {
                 let was_punctuation = self.maybe_parse_punctuation(c, source, target)?;
                 if let Some((token, token_pos)) = was_punctuation {
@@ @@ -509,10 +507,9 @@ impl PassTokenizer { @@
         // Modify offset to not include the newline characters
         if cur_char == b'\n' {
             if prev_char == b'\r' {
                 end_pos.offset -= 2;
             } else {
                 end_pos.offset -= 1;
+            }
             // Consume final newline
             source.consume();
         } else {

src/protocol/parser/tokens.rs

➞

Show inline comments

@@ @@ -212,8 +212,9 @@ impl TokenBuffer { @@
         Self{ tokens: Vec::new(), ranges: Vec::new() }
+    }
     pub(crate) fn iter_range<'a>(&'a self, range: &TokenRange) -> TokenIter<'a> {
         TokenIter::new(self, range.start as usize, range.end as usize)
     pub(crate) fn iter_range<'a>(&'a self, inclusive_start: u32, exclusive_end: u32) -> TokenIter<'a> {
         debug_assert!(exclusive_end as usize <= self.tokens.len());
         TokenIter::new(self, inclusive_start as usize, exclusive_end as usize)
+    }
     pub(crate) fn start_pos(&self, range: &TokenRange) -> InputPosition {
@@ @@ -337,6 +338,10 @@ impl<'a> TokenIter<'a> { @@
+        }
+    }
     pub(crate) fn token_index(&self) -> u32 {
         return self.cur as u32;
+    }
     /// Saves the current iteration position, may be passed to `load` to return
     /// the iterator to a previous position.
     pub(crate) fn save(&self) -> (usize, usize) {

std/std.global.pdl

➞

Show inline comments

 #module std.global
 // Note: parsing of token ranges and pragma needs to change. For now we insert
 // spaces to work with the current system. Needs to be a system where the
 // pragmas, "func" keywords (and similar keywords) indicate initial points to
 // start parsing.
 func get<T>(in<T> input) -> T { #builtin }
 func put<T>(out<T> output, T value) -> #type_void { #builtin }
 func fires<T>(#type_portlike <T>) -> bool { #builtin }

tokens.txt

➞

Show inline comments

deleted file

0 comments (0 inline, 0 general)