pub(crate) had_error: Option<ParseError>,

    #[cfg(test)]

    pub fn new_test(input: &str) -> Self {

        let bytes = Vec::from(input.as_bytes());

        return Self::new(String::from("test"), bytes)

    }

        let mut lookup = self.offset_lookup.borrow_mut();

    pub line: usize,

    pub column: usize,

    pub offset: usize,

use crate::protocol::Heap;

use crate::protocol::tokenizer::{TokenBuffer, Token};

use crate::protocol::input_source2::{InputSource2 as InputSource, ParseError};

struct Ctx<'a> {

    heap: &'a mut Heap,

    source: &'a InputSource,

    tokens: &'a TokenBuffer,

}

// Lexes definitions. Should be the first pass over each of the module files 

// after tokenization. Only once all definitions are parsed can we do the full

// AST creation pass.

struct LexerDefinitions {

}

impl LexerDefinitions {

    pub(crate) fn parse(ctx: &mut Ctx) -> Result<(), ParseError> {

        debug_assert!(ctx.tokens.ranges.len() > 0);

    }

    pub(crate) fn parse_definition(heap: &mut Heap, source: &InputSource, range: &TokenRang)

        let col = self.error.statements[idx].position.column;

#[derive(Debug, Clone, Copy, PartialEq, Eq)]

pub(crate) enum TokenKind {

    Ident,          // regular identifier

    Pragma,         // identifier with prefixed `#`, range includes `#`

    Integer,        // integer literal

    String,         // string literal, range includes `"`

    Character,      // character literal, range includes `'`

    LineComment,    // line comment, range includes leading `//`, but not newline

    BlockComment,   // block comment, range includes leading `/*` and trailing `*/`

pub(crate) struct Token {

    pub kind: TokenKind,

    pub pos: InputPosition,

pub(crate) enum TokenRangeKind {

#[derive(Debug)]

    curly_depth: i32,

pub(crate) struct TokenBuffer {

    pub tokens: Vec<Token>,

    pub ranges: Vec<TokenRange>,

impl TokenBuffer {

    pub(crate) fn new() -> Self {

        Self{ tokens: Vec::new(), ranges: Vec::new() }

    }

// will detect this once we transform the tokens into the AST. Maybe we want to

// detect a mismatch in opening/closing curly braces in the future?

    // Signed because programmer might have placed too many closing curly braces

    curly_depth: i32,

    // Points to an element in the `TokenBuffer.ranges` variable.

    pub(crate) fn new() -> Self {

        Self{ curly_depth: 0, stack_idx: 0 }

    }

        self.stack_idx = 0;

        // Main tokenization loop

            } else if is_pragma_start_or_pound(c) {

                let was_pragma = self.consume_pragma_or_pound(c, source, target)?;

                if was_pragma {

                    self.push_range(target, TokenRangeKind::Pragma, token_index);

                }

                if contained_newline {

                    let range = &target.ranges[self.stack_idx];

                    if range.range_kind == TokenRangeKind::Pragma {

                        self.pop_range(target, target.tokens.len());

                    }

                }

        if let Some(error) = source.had_error.take() {

            return Err(error);

        }

    fn maybe_parse_punctuation(&mut self, first_char: u8, source: &mut InputSource, target: &mut TokenBuffer) -> Result<Option<TokenKind>, ParseError> {

    fn consume_char_literal(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {

            if !c.is_ascii() {

                return Err(ParseError::new_error(source, source.pos(), "non-ASCII character in char literal"));

            }

            // Make sure ending quote was not escaped

            // Unterminated character literal, reached end of file.

    fn consume_string_literal(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {

            if !c.is_ascii() {

                return Err(ParseError::new_error(source, source.pos(), "non-ASCII character in string literal"));

            }

    fn consume_pragma_or_pound(&mut self, first_char: u8, source: &mut InputSource, target: &mut TokenBuffer) -> Result<bool, ParseError> {

        let start_pos = source.pos();

        let next = source.next();

        if next.is_none() || !is_identifier_start(next.unwrap()) {

            // Just a pound sign

            target.tokens.push(Token::new(TokenKind::Pound, start_pos));

            Ok(false)

        } else {

            // Pound sign followed by identifier

            source.consume();

            while let Some(c) = source.next() {

                if !is_identifier_remaining(c) {

                    break;

                }

                source.consume();

            }

            self.check_ascii(source)?;

            let end_pos = source.pos();

            target.tokens.push(Token::new(TokenKind::Pragma, start_pos));

            target.tokens.push(Token::new(TokenKind::SpanEnd, end_pos));

            Ok(true)

        }

    fn consume_line_comment(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {

            prev_char = cur_char;

                // End of line, note that the newline is not consumed

            source.consume();

            // Consume final newline

            source.consume();

            // End of comment was due to EOF

    fn consume_block_comment(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {

    fn consume_identifier<'a>(&mut self, source: &'a mut InputSource, target: &mut TokenBuffer) -> Result<&'a [u8], ParseError> {

    fn consume_number(&mut self, source: &mut InputSource, target: &mut TokenBuffer) -> Result<(), ParseError> {

            source.consume();

    fn push_range(&mut self, target: &mut TokenBuffer, range_kind: TokenRangeKind, first_token: usize) {

        let cur_range = &mut target.ranges[self.stack_idx];

        println!(

            "DEBUG: push_range [1] | stack_idx: {}, range_end: {}, first_token: {}", 

            self.stack_idx, cur_range.end, first_token

        );

        // If we have just popped a range and then push a new range, then the

        // first token is equal to the last token registered on the current 

        // range. If not, then we had some intermediate tokens that did not 

        // belong to a particular kind of token range: hence we insert an 

        // intermediate "code" range.

        if cur_range.end != first_token {

            println!("DEBUG: push_range [2] | inserting code range");

            let code_start = cur_range.end;

            cur_range.end = first_token;

            cur_range.subranges += 1;

                parent_idx: self.stack_idx,

                curly_depth: self.curly_depth,

                start: code_start,

        println!(

            "DEBUG: push_range [3] | kind: {:?}, parent_idx: {}, stack_idx: {}", 

            range_kind, self.stack_idx, target.ranges.len()

        );

    fn pop_range(&mut self, target: &mut TokenBuffer, end_index: usize) {

        // Fix up the current range before going back to parent

        println!(

            "DEBUG: pop_range  [1] | stack_idx: {}, kind: {:?}, start: {}, old_end: {}, new_end: {}",

            self.stack_idx, last.range_kind, last.start, last.end, end_index

        );

        // Go back to parent

        self.stack_idx = last.parent_idx;

        println!(

            "DEBUG: pop_range  [2] | returning to kind: {:?}, idx: {}, new_end: {}",

            parent.range_kind, self.stack_idx, end_index

        );

fn is_pragma_start_or_pound(c: u8) -> bool {

}

#[cfg(test)]

mod tests {

    use super::*;

    // TODO: Remove at some point

    #[test]

    fn test_tokenizer() {

        let mut source = InputSource::new_test("

        #version 500

        # hello 2

        import std.reo::*;

        struct Thing {

            int a: 5,

        }

        enum Hello {

            A,

            B

        }

        // Hello hello, is it me you are looking for?

        // I can seee it in your eeeyes

        func something(int a, int b, int c) -> byte {

            int a = 5;

            struct Inner {

                int a

            }

            struct City {

                int b

            }

            /* Waza

            How are you doing

            Things in here yo

            /* */ */

            a = a + 5 * 2;

            struct Pressure {

                int d

            }

        }

        ");

        let mut t = Tokenizer::new();

        let mut buffer = TokenBuffer::new();

        t.tokenize(&mut source, &mut buffer).expect("tokenize");

        println!("Ranges:\n");

        for (idx, range) in buffer.ranges.iter().enumerate() {

            println!("[{}] {:?}", idx, range)

        }

        println!("Tokens:\n");

        let mut iter = buffer.tokens.iter().enumerate();

        while let Some((idx, token)) = iter.next() {

            match token.kind {

                TokenKind::Ident | TokenKind::Pragma | TokenKind::Integer |

                TokenKind::String | TokenKind::Character | TokenKind::LineComment |

                TokenKind::BlockComment => {

                    let (_, end) = iter.next().unwrap();

                    println!("[{}] {:?} ......", idx, token.kind);

                    assert_eq!(end.kind, TokenKind::SpanEnd);

                    let text = source.section(token.pos.offset, end.pos.offset);

                    println!("{}", String::from_utf8_lossy(text));

                },

                _ => {

                    println!("[{}] {:?}", idx, token.kind);

                }

            }

        }

    }