[package]

name = "reowolf_rs"

version = "1.2.0"

authors = [

	"Max Henger <henger@cwi.nl>",

	"Christopher Esterhuyse <esterhuy@cwi.nl>",

	"Hans-Dieter Hiep <hdh@cwi.nl>"

]

edition = "2021"

[dependencies]

# convenience macros

maplit = "1.0.2"

derive_more = "0.99.2"

# runtime

bincode = "1.3.1"

serde = { version = "1.0.114", features = ["derive"] }

getrandom = "0.1.14" # tiny crate. used to guess controller-id

# network

mio = { version = "0.7.0", package = "mio", features = ["udp", "tcp", "os-poll"] }

socket2 = { version = "0.3.12", optional = true }

# protocol

backtrace = "0.3"

lazy_static = "1.4.0"

# socket ffi

libc = { version = "^0.2", optional = true }

os_socketaddr = { version = "0.1.0", optional = true }

# randomness

rand = "0.8.4"

rand_pcg = "0.3.1"

[lib]

crate-type = [

	"rlib", # for use as a Rust dependency.

]

pub(crate) mod pass_definitions;

pub(crate) mod pass_definitions_types;

pub(crate) mod pass_validation_linking;

pub(crate) mod pass_rewriting;

pub(crate) mod pass_typing;

pub(crate) mod pass_stack_size;

use tokens::*;

use crate::collections::*;

use visitor::Visitor;

use pass_tokenizer::PassTokenizer;

use pass_symbols::PassSymbols;

use pass_imports::PassImport;

use pass_definitions::PassDefinitions;

use pass_validation_linking::PassValidationLinking;

use pass_typing::{PassTyping, ResolveQueue};

use pass_rewriting::PassRewriting;

use pass_stack_size::PassStackSize;

use symbol_table::*;

use type_table::*;

use crate::protocol::ast::*;

use crate::protocol::input_source::*;

use crate::protocol::ast_writer::ASTWriter;

use crate::protocol::parser::type_table::PolymorphicVariable;

use crate::protocol::token_writer::TokenWriter;

const REOWOLF_PATH_ENV: &'static str = "REOWOLF_ROOT"; // first lookup reowolf path

const REOWOLF_PATH_DIR: &'static str = "std"; // then try folder in current working directory

#[derive(Debug, PartialEq, Eq, PartialOrd, Ord)]

pub enum ModuleCompilationPhase {

    Tokenized,              // source is tokenized

    SymbolsScanned,         // all definitions are linked to their type class

    ImportsResolved,        // all imports are added to the symbol table

    DefinitionsParsed,      // produced the AST for the entire module

    TypesAddedToTable,      // added all definitions to the type table

    ValidatedAndLinked,     // AST is traversed and has linked the required AST nodes

    Typed,                  // Type inference and checking has been performed

    Rewritten,              // Special AST nodes are rewritten into regular AST nodes

    // When we continue with the compiler:

    // StackSize

}

pub struct Module {

    pub source: InputSource,

    pub tokens: TokenBuffer,

    pub root_id: RootId,

    pub name: Option<(PragmaId, StringRef<'static>)>,

    pub version: Option<(PragmaId, i64)>,

    pub phase: ModuleCompilationPhase,

}

pub struct TargetArch {

    pub void_type_id: TypeId,

    pub message_type_id: TypeId,

    pub bool_type_id: TypeId,

    pub uint8_type_id: TypeId,

    pub uint16_type_id: TypeId,

    pub uint32_type_id: TypeId,

    pub uint64_type_id: TypeId,

    pub sint8_type_id: TypeId,

    pub sint16_type_id: TypeId,

    pub sint32_type_id: TypeId,

    pub sint64_type_id: TypeId,

    pub char_type_id: TypeId,

    pub string_type_id: TypeId,

    pub array_type_id: TypeId,

    pub slice_type_id: TypeId,

    pub input_type_id: TypeId,

    pub output_type_id: TypeId,

    pub pointer_type_id: TypeId,

}

impl TargetArch {

    fn new() -> Self {

        return Self{

            void_type_id: TypeId::new_invalid(),

            bool_type_id: TypeId::new_invalid(),

            message_type_id: TypeId::new_invalid(),

            uint8_type_id: TypeId::new_invalid(),

            uint16_type_id: TypeId::new_invalid(),

            uint32_type_id: TypeId::new_invalid(),

            uint64_type_id: TypeId::new_invalid(),

            sint8_type_id: TypeId::new_invalid(),

            sint16_type_id: TypeId::new_invalid(),

            sint32_type_id: TypeId::new_invalid(),

            sint64_type_id: TypeId::new_invalid(),

            char_type_id: TypeId::new_invalid(),

            string_type_id: TypeId::new_invalid(),

            array_type_id: TypeId::new_invalid(),

            slice_type_id: TypeId::new_invalid(),

            input_type_id: TypeId::new_invalid(),

            output_type_id: TypeId::new_invalid(),

            pointer_type_id: TypeId::new_invalid(),

            type_table: TypeTable::new(),

            global_module_index: 0,

            pass_tokenizer: PassTokenizer::new(),

            pass_symbols: PassSymbols::new(),

            pass_import: PassImport::new(),

            pass_definitions: PassDefinitions::new(),

            pass_validation: PassValidationLinking::new(),

            pass_typing: PassTyping::new(),

            pass_rewriting: PassRewriting::new(),

            pass_stack_size: PassStackSize::new(),

            write_tokens_to: None,

            write_ast_to: None,

            arch: TargetArch::new(),

        };

        parser.symbol_table.insert_scope(None, SymbolScope::Global);

        // Insert builtin types

        // TODO: At some point use correct values for size/alignment

        parser.arch.void_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Void], false, 0, 1);

        parser.arch.message_type_id = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Message], false, 24, 8);

        parser.arch.bool_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Bool], false, 1, 1);

        parser.arch.uint8_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt8], false, 1, 1);

        parser.arch.uint16_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt16], false, 2, 2);

        parser.arch.uint32_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt32], false, 4, 4);

        parser.arch.uint64_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::UInt64], false, 8, 8);

        parser.arch.sint8_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt8], false, 1, 1);

        parser.arch.sint16_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt16], false, 2, 2);

        parser.arch.sint32_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt32], false, 4, 4);

        parser.arch.sint64_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::SInt64], false, 8, 8);

        parser.arch.char_type_id    = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Character], false, 4, 4);

        parser.arch.string_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::String], false, 24, 8);

        parser.arch.array_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Array, ConcreteTypePart::Void], true, 24, 8);

        parser.arch.slice_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Slice, ConcreteTypePart::Void], true, 16, 4);

        parser.arch.input_type_id   = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Input, ConcreteTypePart::Void], true, 8, 8);

        parser.arch.output_type_id  = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Output, ConcreteTypePart::Void], true, 8, 8);

        parser.arch.pointer_type_id = insert_builtin_type(&mut parser.type_table, vec![ConcreteTypePart::Pointer, ConcreteTypePart::Void], true, 8, 8);

        // Parse standard library

        parser.feed_standard_library()?;

        return Ok(parser)

    }

    /// Feeds a new InputSource to the parser, which will tokenize it and store

    /// it internally for later parsing (when all modules are present). Returns

    /// the index of the new module.

    pub fn feed(&mut self, mut source: InputSource) -> Result<usize, ParseError> {

    }

    pub fn parse(&mut self) -> Result<(), ParseError> {

        let mut pass_ctx = PassCtx{

            heap: &mut self.heap,

            symbols: &mut self.symbol_table,

            pool: &mut self.string_pool,

            arch: &self.arch,

        };

        // Advance all modules to the phase where all symbols are scanned

        for module_idx in 0..self.modules.len() {

            self.pass_symbols.parse(&mut self.modules, module_idx, &mut pass_ctx)?;

        }

        // With all symbols scanned, perform further compilation until we can

        // add all base types to the type table.

        for module_idx in 0..self.modules.len() {

            self.pass_import.parse(&mut self.modules, module_idx, &mut pass_ctx)?;

            self.pass_definitions.parse(&mut self.modules, module_idx, &mut pass_ctx)?;

        }

        if let Some(filename) = &self.write_tokens_to {

            let mut writer = TokenWriter::new();

            let mut file = std::fs::File::create(std::path::Path::new(filename)).unwrap();

            writer.write(&mut file, &self.modules);

        }

        // Add every known type to the type table

        self.type_table.build_base_types(&mut self.modules, &mut pass_ctx)?;

        // Continue compilation with the remaining phases now that the types

        // are all in the type table

        for module_idx in 0..self.modules.len() {

            let mut ctx = visitor::Ctx{

                heap: &mut self.heap,

                modules: &mut self.modules,

                module_idx,

                symbols: &mut self.symbol_table,

                types: &mut self.type_table,

                arch: &self.arch,

            };

            self.pass_validation.visit_module(&mut ctx)?;

        }

        // Perform typechecking on all modules

        let mut queue = ResolveQueue::new();

        for module_idx in 0..self.modules.len() {

                arch: &self.arch,

            };

            self.pass_typing.queue_module_definitions(&mut ctx, &mut queue);

        };

        while !queue.is_empty() {

            let top = queue.pop_front().unwrap();

            let mut ctx = visitor::Ctx{

                heap: &mut self.heap,

                modules: &mut self.modules,

                module_idx: top.root_id.index as usize,

                symbols: &mut self.symbol_table,

                types: &mut self.type_table,

                arch: &self.arch,

            };

            self.pass_typing.handle_module_definition(&mut ctx, &mut queue, top)?;

        }

        // Rewrite nodes in tree, then prepare for execution of code

        for module_idx in 0..self.modules.len() {

            self.modules[module_idx].phase = ModuleCompilationPhase::Typed;

            let mut ctx = visitor::Ctx{

                heap: &mut self.heap,

                modules: &mut self.modules,

                module_idx,

                symbols: &mut self.symbol_table,

                types: &mut self.type_table,

                arch: &self.arch,

            };

            self.pass_rewriting.visit_module(&mut ctx)?;

            self.pass_stack_size.visit_module(&mut ctx)?;

        }

        // Write out desired information

        if let Some(filename) = &self.write_ast_to {

            let mut writer = ASTWriter::new();

            let mut file = std::fs::File::create(std::path::Path::new(filename)).unwrap();

            writer.write_ast(&mut file, &self.heap);

        }

        Ok(())

    }

    /// Tries to find the standard library and add the files for parsing.

    fn feed_standard_library(&mut self) -> Result<(), String> {

        use std::env;

        use std::path::{Path, PathBuf};

        use std::fs;

        ];

        // Determine base directory

        let (base_path, from_env) = if let Ok(path) = env::var(REOWOLF_PATH_ENV) {

            // Path variable is set

            (path, true)

        } else {

            let mut path = String::with_capacity(REOWOLF_PATH_DIR.len() + 2);

            path.push_str("./");

            path.push_str(REOWOLF_PATH_DIR);

            (path, false)

        };

        // Make sure directory exists

        let path = Path::new(&base_path);

        if !path.exists() {

            return Err(format!("std lib root directory '{}' does not exist", base_path));

        }

        // Try to load all standard library files. We might need a more unified

        // way to do this in the future (i.e. a "std" package, containing all

        // of the modules)

        let mut file_path = PathBuf::new();

        let mut first_file = true;

            file_path.clear();

            file_path.push(path);

            file_path.push(file);

            let source = fs::read(file_path.as_path());

            if let Err(err) = source {

                return Err(format!(

                    "failed to read std lib file '{}' in root directory '{}', because: {}",

                    file, base_path, err

                ));

            }

            let source = source.unwrap();

            let input_source = InputSource::new(file.to_string(), source);

            if let Err(err) = module_index {

                // A bit of a hack, but shouldn't really happen anyway: the

                // compiler should ship with a decent standard library (at some

                // point)

                return Err(format!("{}", err));

            }

            let module_index = module_index.unwrap();

            if first_file {

                self.global_module_index = module_index;

                first_file = false;

            }

        }

        return Ok(())

    }

        let mut token_buffer = TokenBuffer::new();

        self.pass_tokenizer.tokenize(&mut source, &mut token_buffer)?;

        let module = Module{

            source,

            tokens: token_buffer,

            is_compiler_file,

            root_id: RootId::new_invalid(),

            name: None,

            version: None,

            phase: ModuleCompilationPhase::Tokenized,

        };

        let module_index = self.modules.len();

        self.modules.push(module);

        return Ok(module_index);

    }

}

fn insert_builtin_type(type_table: &mut TypeTable, parts: Vec<ConcreteTypePart>, has_poly_var: bool, size: usize, alignment: usize) -> TypeId {

    const POLY_VARS: [PolymorphicVariable; 1] = [PolymorphicVariable{

        identifier: Identifier::new_empty(InputSpan::new()),

        is_in_use: false,

    }];

    let concrete_type = ConcreteType{ parts };

    let poly_var = if has_poly_var {

        POLY_VARS.as_slice()

    } else {

        &[]

    };

    return type_table.add_builtin_data_type(concrete_type, poly_var, size, alignment);

}

use crate::protocol::ast::*;

use super::symbol_table::*;

use crate::protocol::input_source::{ParseError, InputSpan};

use super::tokens::*;

use super::token_parsing::*;

use super::{Module, ModuleCompilationPhase, PassCtx};

/// Scans the module and finds all module-level type definitions. These will be

/// added to the symbol table such that during AST-construction we know which

/// identifiers point to types. Will also parse all pragmas to determine module

/// names.

pub(crate) struct PassSymbols {

    symbols: Vec<Symbol>,

    pragmas: Vec<PragmaId>,

    imports: Vec<ImportId>,

    definitions: Vec<DefinitionId>,

    buffer: String,

    has_pragma_version: bool,

    has_pragma_module: bool,

}

impl PassSymbols {

    pub(crate) fn new() -> Self {

        Self{

            symbols: Vec::with_capacity(128),

            pragmas: Vec::with_capacity(8),

            imports: Vec::with_capacity(32),

            definitions: Vec::with_capacity(128),

            buffer: String::with_capacity(128),

            has_pragma_version: false,

            has_pragma_module: false,

        }

    }

    fn reset(&mut self) {

        self.symbols.clear();

        self.pragmas.clear();

        self.imports.clear();

        self.definitions.clear();

        self.has_pragma_version = false;

        self.has_pragma_module = false;

    }

    pub(crate) fn parse(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx) -> Result<(), ParseError> {

        self.reset();

        let module = &mut modules[module_idx];

        debug_assert_eq!(module.phase, ModuleCompilationPhase::Tokenized);

        debug_assert!(module.root_id.is_invalid()); // not set yet

        // Preallocate root in the heap

        let root_id = ctx.heap.alloc_protocol_description(|this| {

            Root{

                this,

                pragmas: Vec::new(),

                imports: Vec::new(),

                definitions: Vec::new(),

            }

        });

        module.root_id = root_id;

        // Use pragma token markers to detects symbol definitions and pragmas

        let num_markers = module.tokens.markers.len();

        for marker_index in 0..num_markers {

            let module = &modules[module_idx];

            let marker = &module.tokens.markers[marker_index];

            // Parse if it is a definition or a pragma

            match marker.kind {

                TokenMarkerKind::Pragma => {

                    self.visit_pragma_marker(modules, module_idx, ctx, marker_index)?;

                },

                TokenMarkerKind::Definition => {

                    self.visit_definition_marker(modules, module_idx, ctx, marker_index)?;

                }

                TokenMarkerKind::Import => {}, // we don't care yet

            }

        }

        // Add the module's symbol scope and the symbols we just parsed

        let module_scope = SymbolScope::Module(root_id);

        ctx.symbols.insert_scope(Some(SymbolScope::Global), module_scope);

        for symbol in self.symbols.drain(..) {

            ctx.symbols.insert_scope(Some(module_scope), SymbolScope::Definition(symbol.variant.as_definition().definition_id));

            if let Err((new_symbol, old_symbol)) = ctx.symbols.insert_symbol(module_scope, symbol) {

                return Err(construct_symbol_conflict_error(modules, module_idx, ctx, &new_symbol, &old_symbol))

            }

        }

            debug_assert!(self.symbols.is_empty());

            ctx.symbols.get_all_symbols_defined_in_scope(module_scope, &mut self.symbols);

            for symbol in self.symbols.drain(..) {

                ctx.symbols.insert_symbol_in_global_scope(symbol);

            }

        }

        // Modify the preallocated root

        let root = &mut ctx.heap[root_id];

        root.pragmas.extend(self.pragmas.drain(..));

        root.definitions.extend(self.definitions.drain(..));

        // Modify module

        let module = &mut modules[module_idx];

        module.phase = ModuleCompilationPhase::SymbolsScanned;

        Ok(())

    }

    fn visit_pragma_marker(&mut self, modules: &mut [Module], module_idx: usize, ctx: &mut PassCtx, marker_index: usize) -> Result<(), ParseError> {

        let module = &mut modules[module_idx];

        let marker = &module.tokens.markers[marker_index];

        let mut iter = module.tokens.iter_range(marker.first_token, None);

        // Consume pragma name

        let (pragma_section, mut pragma_span) = consume_pragma(&module.source, &mut iter)?;

        // Consume pragma values

        if pragma_section == b"#module" {

            // Check if name is defined twice within the same file

            if self.has_pragma_module {

                return Err(ParseError::new_error_str_at_span(&module.source, pragma_span, "module name is defined twice"));

            }

            // Consume the domain-name, then record end of pragma

            let (module_name, module_span) = consume_domain_ident(&module.source, &mut iter)?;

            let marker_last_token = iter.token_index();

            // Add to heap and symbol table

            pragma_span.end = module_span.end;

            let module_name = ctx.pool.intern(module_name);

            let pragma_id = ctx.heap.alloc_pragma(|this| Pragma::Module(PragmaModule{

                this,

                span: pragma_span,

                value: Identifier{ span: module_span, value: module_name.clone() },

            }));

            self.pragmas.push(pragma_id);

mod store;

mod runtime;

mod component;

mod communication;

mod scheduler;

#[cfg(test)] mod tests;

pub use runtime::Runtime;

pub(crate) use scheduler::SchedulerCtx;

pub(crate) use communication::{

    PortId, PortKind, PortState,

    Message, ControlMessage, SyncMessage, DataMessage,

    SyncRoundDecision

};

        new sender(tx_b, num_sends);

    }

    ").expect("compilation");

    let rt = Runtime::new(3, false, pd);

    create_component(&rt, "", "constructor", no_args());

}

#[test]

fn test_unguarded_select() {

    let pd = ProtocolDescription::parse(b"

    primitive constructor_outside_select() {

        u32 index = 0;

        while (index < 5) {

            sync select { auto v = () -> print(\"hello\"); }

            index += 1;

        }

    }

    primitive constructor_inside_select() {

        u32 index = 0;

        while (index < 5) {

            sync select { auto v = () -> index += 1; }

        }

    }

    ").expect("compilation");

    let rt = Runtime::new(3, false, pd);

    create_component(&rt, "", "constructor_outside_select", no_args());

    create_component(&rt, "", "constructor_inside_select", no_args());

}

#[test]

fn test_empty_select() {

    let pd = ProtocolDescription::parse(b"

    primitive constructor() {

        u32 index = 0;

        while (index < 5) {

            sync select {}

            index += 1;

        }

    }

    ").expect("compilation");

    let rt = Runtime::new(3, false, pd);

    create_component(&rt, "", "constructor", no_args());

}

#[test]

fn test_random_u32_temporary_thingo() {

    let pd = ProtocolDescription::parse(b"

    primitive random_taker(in<u32> generator, u32 num_values) {

        auto i = 0;

        while (i < num_values) {

            sync {

                auto a = get(generator);

            }

            i += 1;

        }

    }

    composite constructor() {

        channel tx -> rx;

        auto num_values = 25;

        new random_u32(tx, 1, 100, num_values);

        new random_taker(rx, num_values);

    }

    ").expect("compilation");

    let rt = Runtime::new(1, true, pd);

    create_component(&rt, "", "constructor", no_args());

}