From 68010622a220b699777e0a9427050a72606531fa 2021-04-08 15:50:25 From: MH Date: 2021-04-08 15:50:25 Subject: [PATCH] prepare tokenization to implement binding expressions --- diff --git a/src/protocol/ast.rs b/src/protocol/ast.rs index 95cbc2805eee0963ce98b29bbbdb366651dce657..366d6ae9754350dcf51fcd71f0d21fe614a6dc25 100644 --- a/src/protocol/ast.rs +++ b/src/protocol/ast.rs @@ -2073,7 +2073,7 @@ pub struct BindingExpression { pub this: BindingExpressionId, // Phase 1: parser pub position: InputPosition, - pub left: ExpressionId, + pub left: LiteralExpressionId, pub right: ExpressionId, // Phase 2: linker pub parent: ExpressionParent, diff --git a/src/protocol/ast_printer.rs b/src/protocol/ast_printer.rs index 0818cfbac700900528d25955f2820909bcaa9307..dd338abccd1d9a1dcaddf15254327f9f8320b7b9 100644 --- a/src/protocol/ast_printer.rs +++ b/src/protocol/ast_printer.rs @@ -599,7 +599,7 @@ impl ASTWriter { self.kv(indent).with_id(PREFIX_BINARY_EXPR_ID, expr.this.0.index) .with_s_key("BindingExpr"); self.kv(indent2).with_s_key("LeftExpression"); - self.write_expr(heap, expr.left, indent3); + self.write_expr(heap, expr.left.upcast(), indent3); self.kv(indent2).with_s_key("RightExpression"); self.write_expr(heap, expr.right, indent3); self.kv(indent2).with_s_key("Parent") diff --git a/src/protocol/lexer.rs b/src/protocol/lexer.rs index 899c94b56269f342729fbef52b089ce4fe8f8145..29ed1c92942d206de98b701953917367d665bd00 100644 --- a/src/protocol/lexer.rs +++ b/src/protocol/lexer.rs @@ -1261,19 +1261,6 @@ impl Lexer<'_> { } Ok(result) } - fn consume_binding_expression(&mut self, h: &mut Heap) -> Result { - if self.has_string("let") { - let position = self.source.pos(); - self.consume_whitespace(true)?; - let left_expr = self.consume_expression(h)?; - self.consume_whitespace(false)?; - self.consume_string(b"=")?; - self.consume_whitespace(false)?; - let right_expr = self.consume_expression(h)?; - } else { - self.consume_prefix_expression(h) - } - } fn consume_prefix_expression(&mut self, h: &mut Heap) -> Result { if self.has_string(b"+") || self.has_string(b"-") diff --git a/src/protocol/mod.rs b/src/protocol/mod.rs index 6825431d551c25f2eab6ab6329d3c1621c5aa011..bd0b669313f8225717076c7356ca92bb4ab53483 100644 --- a/src/protocol/mod.rs +++ b/src/protocol/mod.rs @@ -3,7 +3,9 @@ mod arena; mod eval; pub(crate) mod inputsource; // mod lexer; +mod tokenizer; mod parser; +mod pools; #[cfg(test)] mod tests; // TODO: Remove when not benchmarking diff --git a/src/protocol/parser/depth_visitor.rs b/src/protocol/parser/depth_visitor.rs index 8189cf0cfdf0fb1b5611e57c97a33e51985b3500..71dce7197addfb878f454c4b608e1b2f7ff3ca67 100644 --- a/src/protocol/parser/depth_visitor.rs +++ b/src/protocol/parser/depth_visitor.rs @@ -139,6 +139,13 @@ pub(crate) trait Visitor: Sized { ) -> VisitorResult { recursive_assignment_expression(self, h, expr) } + fn visit_binding_expression( + &mut self, + h: &mut Heap, + expr: BindingExpressionId + ) -> VisitorResult { + recursive_binding_expression(self, h, expr) + } fn visit_conditional_expression( &mut self, h: &mut Heap, @@ -431,6 +438,7 @@ fn recursive_expression( ) -> VisitorResult { match h[expr].clone() { Expression::Assignment(expr) => this.visit_assignment_expression(h, expr.this), + Expression::Binding(expr) => this.visit_binding_expression(h, expr.this), Expression::Conditional(expr) => this.visit_conditional_expression(h, expr.this), Expression::Binary(expr) => this.visit_binary_expression(h, expr.this), Expression::Unary(expr) => this.visit_unary_expression(h, expr.this), @@ -453,6 +461,15 @@ fn recursive_assignment_expression( this.visit_expression(h, h[expr].right) } +fn recursive_binding_expression( + this: &mut T, + h: &mut Heap, + expr: BindingExpressionId, +) -> VisitorResult { + this.visit_expression(h, h[expr].left.upcast())?; + this.visit_expression(h, h[expr].right) +} + fn recursive_conditional_expression( this: &mut T, h: &mut Heap, diff --git a/src/protocol/parser/type_resolver.rs b/src/protocol/parser/type_resolver.rs index e6061c05796a07c9957aeb6484bca766f96a3ede..4ef731dd9ce10c76f5d9595e1a7d960170519b31 100644 --- a/src/protocol/parser/type_resolver.rs +++ b/src/protocol/parser/type_resolver.rs @@ -1447,6 +1447,9 @@ impl TypeResolvingVisitor { let id = expr.this; self.progress_assignment_expr(ctx, id) }, + Expression::Binding(expr) => { + unimplemented!("progress binding expression"); + }, Expression::Conditional(expr) => { let id = expr.this; self.progress_conditional_expr(ctx, id) diff --git a/src/protocol/parser/visitor.rs b/src/protocol/parser/visitor.rs index bcfb755fdbd0acbfaeeb24c4e3f221e3e2fbfd5c..51a5d0073e90148c48eebac83b0e89a1bafae199 100644 --- a/src/protocol/parser/visitor.rs +++ b/src/protocol/parser/visitor.rs @@ -184,6 +184,10 @@ pub(crate) trait Visitor2 { let this = expr.this; self.visit_assignment_expr(ctx, this) }, + Expression::Binding(expr) => { + let this = expr.this; + self.visit_binding_expr(ctx, this) + } Expression::Conditional(expr) => { let this = expr.this; self.visit_conditional_expr(ctx, this) @@ -228,6 +232,7 @@ pub(crate) trait Visitor2 { } fn visit_assignment_expr(&mut self, _ctx: &mut Ctx, _id: AssignmentExpressionId) -> VisitorResult { Ok(()) } + fn visit_binding_expr(&mut self, _ctx: &mut Ctx, _id: BindingExpressionId) -> VisitorResult { Ok(()) } fn visit_conditional_expr(&mut self, _ctx: &mut Ctx, _id: ConditionalExpressionId) -> VisitorResult { Ok(()) } fn visit_binary_expr(&mut self, _ctx: &mut Ctx, _id: BinaryExpressionId) -> VisitorResult { Ok(()) } fn visit_unary_expr(&mut self, _ctx: &mut Ctx, _id: UnaryExpressionId) -> VisitorResult { Ok(()) } diff --git a/src/protocol/pools.rs b/src/protocol/pools.rs new file mode 100644 index 0000000000000000000000000000000000000000..5850d9c547e8361df2b42c2d09bd2eff435c91cf --- /dev/null +++ b/src/protocol/pools.rs @@ -0,0 +1,5 @@ + + +struct StringPool { + +} \ No newline at end of file diff --git a/src/protocol/tests/utils.rs b/src/protocol/tests/utils.rs index a2f6b37fba643e77166b334ae94a11a2a18eed41..34bba8faaac0ac6023bdd85dad38e5dc1e67f7ab 100644 --- a/src/protocol/tests/utils.rs +++ b/src/protocol/tests/utils.rs @@ -988,7 +988,7 @@ fn seek_expr_in_expr bool>(heap: &Heap, start: ExpressionI }, Expression::Binding(expr) => { None - .or_else(|| seek_expr_in_expr(heap, expr.left, f)) + .or_else(|| seek_expr_in_expr(heap, expr.left.upcast(), f)) .or_else(|| seek_expr_in_expr(heap, expr.right, f)) } Expression::Conditional(expr) => { diff --git a/src/protocol/tokenizer/mod.rs b/src/protocol/tokenizer/mod.rs new file mode 100644 index 0000000000000000000000000000000000000000..685cac45f4a39d478de1a9d19352935b10ba16ac --- /dev/null +++ b/src/protocol/tokenizer/mod.rs @@ -0,0 +1,21 @@ +enum Token { + +} + +enum TokenRangeType { + Pragma, + Import, + Defintion, +} + +struct TokenRange { + range_type: TokenRangeType, + start: usize, + end: usize, +} + +struct TokenBuffer { + tokens: Vec, + ranges: Vec, +} +