diff --git a/src/protocol/tokenizer/mod.rs b/src/protocol/tokenizer/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..685cac45f4a39d478de1a9d19352935b10ba16ac --- /dev/null +++ b/src/protocol/tokenizer/mod.rs @@ -0,0 +1,21 @@ +enum Token { + +} + +enum TokenRangeType { + Pragma, + Import, + Defintion, +} + +struct TokenRange { + range_type: TokenRangeType, + start: usize, + end: usize, +} + +struct TokenBuffer { + tokens: Vec, + ranges: Vec, +} +