diff --git a/src/protocol/parser/pass_tokenizer.rs b/src/protocol/parser/pass_tokenizer.rs index 6cb83d224693328d20a184c3c0bb26eb8c9b2482..e00569be823498ec4cb3e1b2258be896d29a78c0 100644 --- a/src/protocol/parser/pass_tokenizer.rs +++ b/src/protocol/parser/pass_tokenizer.rs @@ -66,9 +66,11 @@ impl PassTokenizer { } else if is_identifier_start(c) { let ident = self.consume_identifier(source, target)?; - if demarks_definition(ident) { + if demarks_symbol(ident) { + self.emit_marker(target, TokenMarkerKind::Definition, token_index); self.push_range(target, TokenRangeKind::Definition, token_index); } else if demarks_import(ident) { + self.emit_marker(target, TokenMarkerKind::Import, token_index); self.push_range(target, TokenRangeKind::Import, token_index); } } else if is_integer_literal_start(c) { @@ -76,6 +78,7 @@ impl PassTokenizer { } else if is_pragma_start_or_pound(c) { let was_pragma = self.consume_pragma_or_pound(c, source, target)?; if was_pragma { + self.emit_marker(target, TokenMarkerKind::Pragma, token_index); self.push_range(target, TokenRangeKind::Pragma, token_index); } } else if self.is_line_comment_start(c, source) { @@ -655,6 +658,22 @@ impl PassTokenizer { } } + fn emit_marker(&mut self, target: &mut TokenBuffer, kind: TokenMarkerKind, first_token: u32) { + debug_assert!( + target.markers + .last().map(|v| v.first_token < first_token) + .unwrap_or(true) + ); + + target.markers.push(TokenMarker{ + kind, + curly_depth: self.curly_stack.len() as u32, + first_token, + last_token: u32::MAX, + handled: false, + }); + } + fn push_range(&mut self, target: &mut TokenBuffer, range_kind: TokenRangeKind, first_token_idx: u32) { let new_range_idx = target.ranges.len() as i32; let parent_idx = self.stack_idx as i32; @@ -727,7 +746,7 @@ impl PassTokenizer { } // Helpers for characters -fn demarks_definition(ident: &[u8]) -> bool { +fn demarks_symbol(ident: &[u8]) -> bool { return ident == KW_STRUCT || ident == KW_ENUM ||