diff --git a/src/protocol/parser/tokens.rs b/src/protocol/parser/tokens.rs index d8c6b7d25b01b89aea578664be187660c0a025bb..72c019ae259a40e7a4d0e92e0e6b4ea93e8183f8 100644 --- a/src/protocol/parser/tokens.rs +++ b/src/protocol/parser/tokens.rs @@ -170,6 +170,29 @@ impl Token { } } +#[derive(Debug, Clone, Copy)] +pub enum TokenMarkerKind { + Pragma, + Import, + Definition, +} + +/// A marker for a specific token. These are stored separately from the array of +/// tokens. These are used for initial symbol, module name, and import +/// discovery. +#[derive(Debug)] +pub struct TokenMarker { + pub kind: TokenMarkerKind, + pub curly_depth: u32, + // Indices into token buffer. The first token is inclusive and set upon + // tokenization, the last token is set at a later stage in parsing (e.g. + // at symbol discovery we may parse some of the `Pragma` tokens and set the + // last parsed token) + pub first_token: u32, + pub last_token: u32, + pub handled: bool, +} + /// The kind of token ranges that are specially parsed by the tokenizer. #[derive(Debug, Clone, Copy, PartialEq, Eq)] pub enum TokenRangeKind { @@ -204,31 +227,25 @@ pub struct TokenRange { pub struct TokenBuffer { pub tokens: Vec, + pub markers: Vec, pub ranges: Vec, } impl TokenBuffer { pub(crate) fn new() -> Self { - Self{ tokens: Vec::new(), ranges: Vec::new() } - } - - pub(crate) fn iter_range<'a>(&'a self, inclusive_start: u32, exclusive_end: u32) -> TokenIter<'a> { - debug_assert!(exclusive_end as usize <= self.tokens.len()); - TokenIter::new(self, inclusive_start as usize, exclusive_end as usize) - } - - pub(crate) fn start_pos(&self, range: &TokenRange) -> InputPosition { - self.tokens[range.start as usize].pos + return Self{ + tokens: Vec::new(), + markers: Vec::new(), + ranges: Vec::new() + }; } - pub(crate) fn end_pos(&self, range: &TokenRange) -> InputPosition { - let last_token = &self.tokens[range.end as usize - 1]; - if last_token.kind == TokenKind::SpanEnd { - return last_token.pos - } else { - debug_assert!(!last_token.kind.has_span_end()); - return last_token.pos.with_offset(last_token.kind.num_characters()); - } + pub(crate) fn iter_range( + &self, inclusive_start: u32, exclusive_end: Option + ) -> TokenIter { + let exclusive_end = exclusive_end.unwrap_or(self.tokens.len() as u32) as usize; + debug_assert!(exclusive_end <= self.tokens.len()); + TokenIter::new(self, inclusive_start as usize, exclusive_end) } }