CSY/reowolf Changeset - 68010622a220 · Centrum Wiskunde & Informatica (CWI)

Changeset - 68010622a220

Parent rev.

Child rev.

[Not reviewed]

0 8 2

MH - 4 years ago 2021-04-08 15:50:25
contact@maxhenger.nl

prepare tokenization to implement binding expressions

10 files changed with 56 insertions and 16 deletions:

src/protocol/ast.rs

src/protocol/ast_printer.rs

src/protocol/lexer.rs

src/protocol/mod.rs

src/protocol/parser/depth_visitor.rs

src/protocol/parser/type_resolver.rs

src/protocol/parser/visitor.rs

src/protocol/pools.rs

src/protocol/tests/utils.rs

src/protocol/tokenizer/mod.rs

0 comments (0 inline, 0 general)

src/protocol/ast.rs

➞

Show inline comments

@@ @@ -2052,49 +2052,49 @@ pub enum AssignmentOperator { @@
 pub struct AssignmentExpression {
     pub this: AssignmentExpressionId,
     // Phase 1: parser
     pub position: InputPosition,
     pub left: ExpressionId,
     pub operation: AssignmentOperator,
     pub right: ExpressionId,
     // Phase 2: linker
     pub parent: ExpressionParent,
     // Phase 3: type checking
     pub concrete_type: ConcreteType,
+}
 impl SyntaxElement for AssignmentExpression {
     fn position(&self) -> InputPosition {
         self.position
+    }
+}
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct BindingExpression {
     pub this: BindingExpressionId,
     // Phase 1: parser
     pub position: InputPosition,
-    pub left: ExpressionId,
+    pub left: LiteralExpressionId,
     pub right: ExpressionId,
     // Phase 2: linker
     pub parent: ExpressionParent,
     // Phase 3: type checking
     pub concrete_type: ConcreteType,
+}
 #[derive(Debug, Clone, serde::Serialize, serde::Deserialize)]
 pub struct ConditionalExpression {
     pub this: ConditionalExpressionId,
     // Phase 1: parser
     pub position: InputPosition,
     pub test: ExpressionId,
     pub true_expression: ExpressionId,
     pub false_expression: ExpressionId,
     // Phase 2: linker
     pub parent: ExpressionParent,
     // Phase 3: type checking
     pub concrete_type: ConcreteType,
+}
 impl SyntaxElement for ConditionalExpression {
     fn position(&self) -> InputPosition {
         self.position

src/protocol/ast_printer.rs

➞

Show inline comments

@@ @@ -578,49 +578,49 @@ impl ASTWriter { @@
     fn write_expr(&mut self, heap: &Heap, expr_id: ExpressionId, indent: usize) {
         let expr = &heap[expr_id];
         let indent2 = indent + 1;
         let indent3 = indent2 + 1;
         let def_id = self.cur_definition.unwrap();
         match expr {
             Expression::Assignment(expr) => {
                 self.kv(indent).with_id(PREFIX_ASSIGNMENT_EXPR_ID, expr.this.0.index)
                     .with_s_key("AssignmentExpr");
                 self.kv(indent2).with_s_key("Operation").with_debug_val(&expr.operation);
                 self.kv(indent2).with_s_key("Left");
                 self.write_expr(heap, expr.left, indent3);
                 self.kv(indent2).with_s_key("Right");
                 self.write_expr(heap, expr.right, indent3);
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
                 self.kv(indent2).with_s_key("ConcreteType")
                     .with_custom_val(|v| write_concrete_type(v, heap, def_id, &expr.concrete_type));
             },
             Expression::Binding(expr) => {
                 self.kv(indent).with_id(PREFIX_BINARY_EXPR_ID, expr.this.0.index)
                     .with_s_key("BindingExpr");
                 self.kv(indent2).with_s_key("LeftExpression");
                 self.write_expr(heap, expr.left, indent3);
+                self.write_expr(heap, expr.left.upcast(), indent3);
                 self.kv(indent2).with_s_key("RightExpression");
                 self.write_expr(heap, expr.right, indent3);
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
                 self.kv(indent2).with_s_key("ConcreteType")
                     .with_custom_val(|v| write_concrete_type(v, heap, def_id, &expr.concrete_type));
             },
             Expression::Conditional(expr) => {
                 self.kv(indent).with_id(PREFIX_CONDITIONAL_EXPR_ID, expr.this.0.index)
                     .with_s_key("ConditionalExpr");
                 self.kv(indent2).with_s_key("Condition");
                 self.write_expr(heap, expr.test, indent3);
                 self.kv(indent2).with_s_key("TrueExpression");
                 self.write_expr(heap, expr.true_expression, indent3);
                 self.kv(indent2).with_s_key("FalseExpression");
                 self.write_expr(heap, expr.false_expression, indent3);
                 self.kv(indent2).with_s_key("Parent")
                     .with_custom_val(|v| write_expression_parent(v, &expr.parent));
                 self.kv(indent2).with_s_key("ConcreteType")
                     .with_custom_val(|v| write_concrete_type(v, heap, def_id, &expr.concrete_type));
             },
             Expression::Binary(expr) => {
                 self.kv(indent).with_id(PREFIX_BINARY_EXPR_ID, expr.this.0.index)
                     .with_s_key("BinaryExpr");

src/protocol/lexer.rs

➞

Show inline comments

@@ @@ -1240,61 +1240,48 @@ impl Lexer<'_> { @@
             } else if self.has_string(b"/") {
                 self.consume_string(b"/")?;
                 operation = BinaryOperator::Divide;
             } else {
                 self.consume_string(b"%")?;
                 operation = BinaryOperator::Remainder;
+            }
             self.consume_whitespace(false)?;
             let right = self.consume_prefix_expression(h)?;
             self.consume_whitespace(false)?;
             result = h
                 .alloc_binary_expression(|this| BinaryExpression {
                     this,
                     position,
                     left,
                     operation,
                     right,
                     parent: ExpressionParent::None,
                     concrete_type: ConcreteType::default(),
                 })
                 .upcast();
+        }
         Ok(result)
+    }
     fn consume_binding_expression(&mut self, h: &mut Heap) -> Result<ExpressionId, ParseError> {
         if self.has_string("let") {
             let position = self.source.pos();
             self.consume_whitespace(true)?;
             let left_expr = self.consume_expression(h)?;
             self.consume_whitespace(false)?;
             self.consume_string(b"=")?;
             self.consume_whitespace(false)?;
             let right_expr = self.consume_expression(h)?;
         } else {
             self.consume_prefix_expression(h)
+        }
+    }
     fn consume_prefix_expression(&mut self, h: &mut Heap) -> Result<ExpressionId, ParseError> {
         if self.has_string(b"+")
             || self.has_string(b"-")
             || self.has_string(b"~")
             || self.has_string(b"!")
+        {
             let position = self.source.pos();
             let operation;
             if self.has_string(b"+") {
                 self.consume_string(b"+")?;
                 if self.has_string(b"+") {
                     self.consume_string(b"+")?;
                     operation = UnaryOperation::PreIncrement;
                 } else {
                     operation = UnaryOperation::Positive;
+                }
             } else if self.has_string(b"-") {
                 self.consume_string(b"-")?;
                 if self.has_string(b"-") {
                     self.consume_string(b"-")?;
                     operation = UnaryOperation::PreDecrement;
                 } else {
                     operation = UnaryOperation::Negative;
+                }

src/protocol/mod.rs

➞

Show inline comments

 mod arena;
 // mod ast;
 mod eval;
 pub(crate) mod inputsource;
 // mod lexer;
 mod tokenizer;
 mod parser;
 mod pools;
 #[cfg(test)] mod tests;
 // TODO: Remove when not benchmarking
 pub(crate) mod ast;
 pub(crate) mod ast_printer;
 pub(crate) mod lexer;
 lazy_static::lazy_static! {
     /// Conveniently-provided protocol description initialized with a zero-length PDL string.
     /// Exposed to minimize repeated initializations of this common protocol description.
     pub static ref TRIVIAL_PD: std::sync::Arc<ProtocolDescription> = {
         std::sync::Arc::new(ProtocolDescription::parse(b"").unwrap())
     };
+}
 use crate::common::*;
 use crate::protocol::ast::*;
 use crate::protocol::eval::*;
 use crate::protocol::inputsource::*;
 use crate::protocol::parser::*;
 /// Description of a protocol object, used to configure new connectors.
 /// (De)serializable.
 #[derive(serde::Serialize, serde::Deserialize)]

src/protocol/parser/depth_visitor.rs

➞

Show inline comments

@@ @@ -118,48 +118,55 @@ pub(crate) trait Visitor: Sized { @@
     fn visit_goto_statement(&mut self, _h: &mut Heap, _stmt: GotoStatementId) -> VisitorResult {
         Ok(())
+    }
     fn visit_new_statement(&mut self, h: &mut Heap, stmt: NewStatementId) -> VisitorResult {
         recursive_new_statement(self, h, stmt)
+    }
     fn visit_expression_statement(
         &mut self,
         h: &mut Heap,
         stmt: ExpressionStatementId,
     ) -> VisitorResult {
         recursive_expression_statement(self, h, stmt)
+    }
     fn visit_expression(&mut self, h: &mut Heap, expr: ExpressionId) -> VisitorResult {
         recursive_expression(self, h, expr)
+    }
     fn visit_assignment_expression(
         &mut self,
         h: &mut Heap,
         expr: AssignmentExpressionId,
     ) -> VisitorResult {
         recursive_assignment_expression(self, h, expr)
+    }
     fn visit_binding_expression(
         &mut self,
         h: &mut Heap,
         expr: BindingExpressionId
     ) -> VisitorResult {
         recursive_binding_expression(self, h, expr)
+    }
     fn visit_conditional_expression(
         &mut self,
         h: &mut Heap,
         expr: ConditionalExpressionId,
     ) -> VisitorResult {
         recursive_conditional_expression(self, h, expr)
+    }
     fn visit_binary_expression(&mut self, h: &mut Heap, expr: BinaryExpressionId) -> VisitorResult {
         recursive_binary_expression(self, h, expr)
+    }
     fn visit_unary_expression(&mut self, h: &mut Heap, expr: UnaryExpressionId) -> VisitorResult {
         recursive_unary_expression(self, h, expr)
+    }
     fn visit_indexing_expression(
         &mut self,
         h: &mut Heap,
         expr: IndexingExpressionId,
     ) -> VisitorResult {
         recursive_indexing_expression(self, h, expr)
+    }
     fn visit_slicing_expression(
         &mut self,
         h: &mut Heap,
         expr: SlicingExpressionId,
@@ @@ -410,70 +417,80 @@ fn recursive_assert_statement<T: Visitor>( @@
 fn recursive_new_statement<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     stmt: NewStatementId,
 ) -> VisitorResult {
     recursive_call_expression_as_expression(this, h, h[stmt].expression)
+}
 fn recursive_expression_statement<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     stmt: ExpressionStatementId,
 ) -> VisitorResult {
     this.visit_expression(h, h[stmt].expression)
+}
 fn recursive_expression<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     expr: ExpressionId,
 ) -> VisitorResult {
     match h[expr].clone() {
         Expression::Assignment(expr) => this.visit_assignment_expression(h, expr.this),
         Expression::Binding(expr) => this.visit_binding_expression(h, expr.this),
         Expression::Conditional(expr) => this.visit_conditional_expression(h, expr.this),
         Expression::Binary(expr) => this.visit_binary_expression(h, expr.this),
         Expression::Unary(expr) => this.visit_unary_expression(h, expr.this),
         Expression::Indexing(expr) => this.visit_indexing_expression(h, expr.this),
         Expression::Slicing(expr) => this.visit_slicing_expression(h, expr.this),
         Expression::Select(expr) => this.visit_select_expression(h, expr.this),
         Expression::Array(expr) => this.visit_array_expression(h, expr.this),
         Expression::Literal(expr) => this.visit_constant_expression(h, expr.this),
         Expression::Call(expr) => this.visit_call_expression(h, expr.this),
         Expression::Variable(expr) => this.visit_variable_expression(h, expr.this),
+    }
+}
 fn recursive_assignment_expression<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     expr: AssignmentExpressionId,
 ) -> VisitorResult {
     this.visit_expression(h, h[expr].left)?;
     this.visit_expression(h, h[expr].right)
+}
 fn recursive_binding_expression<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     expr: BindingExpressionId,
 ) -> VisitorResult {
     this.visit_expression(h, h[expr].left.upcast())?;
     this.visit_expression(h, h[expr].right)
+}
 fn recursive_conditional_expression<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     expr: ConditionalExpressionId,
 ) -> VisitorResult {
     this.visit_expression(h, h[expr].test)?;
     this.visit_expression(h, h[expr].true_expression)?;
     this.visit_expression(h, h[expr].false_expression)
+}
 fn recursive_binary_expression<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     expr: BinaryExpressionId,
 ) -> VisitorResult {
     this.visit_expression(h, h[expr].left)?;
     this.visit_expression(h, h[expr].right)
+}
 fn recursive_unary_expression<T: Visitor>(
     this: &mut T,
     h: &mut Heap,
     expr: UnaryExpressionId,
 ) -> VisitorResult {

src/protocol/parser/type_resolver.rs

➞

Show inline comments

@@ @@ -1426,48 +1426,51 @@ impl TypeResolvingVisitor { @@
                         let definition_id = match &lit_expr.value {
                             Literal::Struct(literal) => literal.definition.as_ref().unwrap(),
                             Literal::Enum(literal) => literal.definition.as_ref().unwrap(),
                             Literal::Union(literal) => literal.definition.as_ref().unwrap(),
                             _ => unreachable!("post-inference monomorph for non-struct, non-enum literal")
                         };
                         if !ctx.types.has_monomorph(definition_id, &monomorph_types) {
                             ctx.types.add_monomorph(definition_id, monomorph_types);
+                        }
                     },
                     _ => unreachable!("needs fully inference, but not a struct literal or call expression")
+                }
             } // else: was just a helper structure...
+        }
         Ok(())
+    }
     fn progress_expr(&mut self, ctx: &mut Ctx, id: ExpressionId) -> Result<(), ParseError> {
         match &ctx.heap[id] {
             Expression::Assignment(expr) => {
                 let id = expr.this;
                 self.progress_assignment_expr(ctx, id)
             },
             Expression::Binding(expr) => {
                 unimplemented!("progress binding expression");
             },
             Expression::Conditional(expr) => {
                 let id = expr.this;
                 self.progress_conditional_expr(ctx, id)
             },
             Expression::Binary(expr) => {
                 let id = expr.this;
                 self.progress_binary_expr(ctx, id)
             },
             Expression::Unary(expr) => {
                 let id = expr.this;
                 self.progress_unary_expr(ctx, id)
             },
             Expression::Indexing(expr) => {
                 let id = expr.this;
                 self.progress_indexing_expr(ctx, id)
             },
             Expression::Slicing(expr) => {
                 let id = expr.this;
                 self.progress_slicing_expr(ctx, id)
             },
             Expression::Select(expr) => {
                 let id = expr.this;
                 self.progress_select_expr(ctx, id)
             },

src/protocol/parser/visitor.rs

➞

Show inline comments

@@ @@ -163,82 +163,87 @@ pub(crate) trait Visitor2 { @@
     fn visit_block_stmt(&mut self, _ctx: &mut Ctx, _id: BlockStatementId) -> VisitorResult { Ok(()) }
     fn visit_local_memory_stmt(&mut self, _ctx: &mut Ctx, _id: MemoryStatementId) -> VisitorResult { Ok(()) }
     fn visit_local_channel_stmt(&mut self, _ctx: &mut Ctx, _id: ChannelStatementId) -> VisitorResult { Ok(()) }
     fn visit_skip_stmt(&mut self, _ctx: &mut Ctx, _id: SkipStatementId) -> VisitorResult { Ok(()) }
     fn visit_labeled_stmt(&mut self, _ctx: &mut Ctx, _id: LabeledStatementId) -> VisitorResult { Ok(()) }
     fn visit_if_stmt(&mut self, _ctx: &mut Ctx, _id: IfStatementId) -> VisitorResult { Ok(()) }
     fn visit_while_stmt(&mut self, _ctx: &mut Ctx, _id: WhileStatementId) -> VisitorResult { Ok(()) }
     fn visit_break_stmt(&mut self, _ctx: &mut Ctx, _id: BreakStatementId) -> VisitorResult { Ok(()) }
     fn visit_continue_stmt(&mut self, _ctx: &mut Ctx, _id: ContinueStatementId) -> VisitorResult { Ok(()) }
     fn visit_synchronous_stmt(&mut self, _ctx: &mut Ctx, _id: SynchronousStatementId) -> VisitorResult { Ok(()) }
     fn visit_return_stmt(&mut self, _ctx: &mut Ctx, _id: ReturnStatementId) -> VisitorResult { Ok(()) }
     fn visit_assert_stmt(&mut self, _ctx: &mut Ctx, _id: AssertStatementId) -> VisitorResult { Ok(()) }
     fn visit_goto_stmt(&mut self, _ctx: &mut Ctx, _id: GotoStatementId) -> VisitorResult { Ok(()) }
     fn visit_new_stmt(&mut self, _ctx: &mut Ctx, _id: NewStatementId) -> VisitorResult { Ok(()) }
     fn visit_expr_stmt(&mut self, _ctx: &mut Ctx, _id: ExpressionStatementId) -> VisitorResult { Ok(()) }
     // Expressions
     // --- enum matching
     fn visit_expr(&mut self, ctx: &mut Ctx, id: ExpressionId) -> VisitorResult {
         match &ctx.heap[id] {
             Expression::Assignment(expr) => {
                 let this = expr.this;
                 self.visit_assignment_expr(ctx, this)
             },
             Expression::Binding(expr) => {
                 let this = expr.this;
                 self.visit_binding_expr(ctx, this)
+            }
             Expression::Conditional(expr) => {
                 let this = expr.this;
                 self.visit_conditional_expr(ctx, this)
+            }
             Expression::Binary(expr) => {
                 let this = expr.this;
                 self.visit_binary_expr(ctx, this)
+            }
             Expression::Unary(expr) => {
                 let this = expr.this;
                 self.visit_unary_expr(ctx, this)
+            }
             Expression::Indexing(expr) => {
                 let this = expr.this;
                 self.visit_indexing_expr(ctx, this)
+            }
             Expression::Slicing(expr) => {
                 let this = expr.this;
                 self.visit_slicing_expr(ctx, this)
+            }
             Expression::Select(expr) => {
                 let this = expr.this;
                 self.visit_select_expr(ctx, this)
+            }
             Expression::Array(expr) => {
                 let this = expr.this;
                 self.visit_array_expr(ctx, this)
+            }
             Expression::Literal(expr) => {
                 let this = expr.this;
                 self.visit_literal_expr(ctx, this)
+            }
             Expression::Call(expr) => {
                 let this = expr.this;
                 self.visit_call_expr(ctx, this)
+            }
             Expression::Variable(expr) => {
                 let this = expr.this;
                 self.visit_variable_expr(ctx, this)
+            }
+        }
+    }
     fn visit_assignment_expr(&mut self, _ctx: &mut Ctx, _id: AssignmentExpressionId) -> VisitorResult { Ok(()) }
     fn visit_binding_expr(&mut self, _ctx: &mut Ctx, _id: BindingExpressionId) -> VisitorResult { Ok(()) }
     fn visit_conditional_expr(&mut self, _ctx: &mut Ctx, _id: ConditionalExpressionId) -> VisitorResult { Ok(()) }
     fn visit_binary_expr(&mut self, _ctx: &mut Ctx, _id: BinaryExpressionId) -> VisitorResult { Ok(()) }
     fn visit_unary_expr(&mut self, _ctx: &mut Ctx, _id: UnaryExpressionId) -> VisitorResult { Ok(()) }
     fn visit_indexing_expr(&mut self, _ctx: &mut Ctx, _id: IndexingExpressionId) -> VisitorResult { Ok(()) }
     fn visit_slicing_expr(&mut self, _ctx: &mut Ctx, _id: SlicingExpressionId) -> VisitorResult { Ok(()) }
     fn visit_select_expr(&mut self, _ctx: &mut Ctx, _id: SelectExpressionId) -> VisitorResult { Ok(()) }
     fn visit_array_expr(&mut self, _ctx: &mut Ctx, _id: ArrayExpressionId) -> VisitorResult { Ok(()) }
     fn visit_literal_expr(&mut self, _ctx: &mut Ctx, _id: LiteralExpressionId) -> VisitorResult { Ok(()) }
     fn visit_call_expr(&mut self, _ctx: &mut Ctx, _id: CallExpressionId) -> VisitorResult { Ok(()) }
     fn visit_variable_expr(&mut self, _ctx: &mut Ctx, _id: VariableExpressionId) -> VisitorResult { Ok(()) }
     // Types
     fn visit_parser_type(&mut self, _ctx: &mut Ctx, _id: ParserTypeId) -> VisitorResult { Ok(()) }
+}
@@ \ No newline at end of file @@

src/protocol/pools.rs

➞

Show inline comments

		new file 100644


		struct StringPool {

		}
		\ No newline at end of file

src/protocol/tests/utils.rs

➞

Show inline comments

                 return Some(id);
+            }
             None
         },
         Statement::While(stmt) => seek_stmt(heap, stmt.body, f),
         Statement::Synchronous(stmt) => seek_stmt(heap, stmt.body, f),
         _ => None
     };
     matched
+}
 fn seek_expr_in_expr<F: Fn(&Expression) -> bool>(heap: &Heap, start: ExpressionId, f: &F) -> Option<ExpressionId> {
     let expr = &heap[start];
     if f(expr) { return Some(start); }
     match expr {
         Expression::Assignment(expr) => {
             None
             .or_else(|| seek_expr_in_expr(heap, expr.left, f))
             .or_else(|| seek_expr_in_expr(heap, expr.right, f))
         },
         Expression::Binding(expr) => {
             None
             .or_else(|| seek_expr_in_expr(heap, expr.left, f))
+            .or_else(|| seek_expr_in_expr(heap, expr.left.upcast(), f))
             .or_else(|| seek_expr_in_expr(heap, expr.right, f))
+        }
         Expression::Conditional(expr) => {
             None
             .or_else(|| seek_expr_in_expr(heap, expr.test, f))
             .or_else(|| seek_expr_in_expr(heap, expr.true_expression, f))
             .or_else(|| seek_expr_in_expr(heap, expr.false_expression, f))
         },
         Expression::Binary(expr) => {
             None
             .or_else(|| seek_expr_in_expr(heap, expr.left, f))
             .or_else(|| seek_expr_in_expr(heap, expr.right, f))
         },
         Expression::Unary(expr) => {
             seek_expr_in_expr(heap, expr.expression, f)
         },
         Expression::Indexing(expr) => {
             None
             .or_else(|| seek_expr_in_expr(heap, expr.subject, f))
             .or_else(|| seek_expr_in_expr(heap, expr.index, f))
         },
         Expression::Slicing(expr) => {
             None
             .or_else(|| seek_expr_in_expr(heap, expr.subject, f))

src/protocol/tokenizer/mod.rs

➞

Show inline comments

@@ new file 100644 @@
 enum Token {
+}
 enum TokenRangeType {
     Pragma,
     Import,
     Defintion,
+}
 struct TokenRange {
     range_type: TokenRangeType,
     start: usize,
     end: usize,
+}
 struct TokenBuffer {
     tokens: Vec<Token>,
     ranges: Vec<TokenRange>,
+}

0 comments (0 inline, 0 general)