vn/code/vn_tokenizer.h

/* date = June 18th 2023 5:12 pm */

#ifndef VN_TOKENIZER_H
#define VN_TOKENIZER_H

enum token_type
{
    Token_EndOfFile,
    Token_Identifier,
    Token_IntegerValue,
    //Token_RealValue,
    Token_CurlyOpen,
    Token_CurlyClose,
    Token_Semicolon,
    Token_Equals,
};

struct tokenizer
{
    string Input;
    s64 At;
    s64 Index;
};

struct token
{
    token_type Type;
    string String;
};

inline tokenizer Tokenizer_BeginTokenization(string Input)
{
    tokenizer Result = {};
    Result.Input = Input;
    
    return(Result);
}

static token Tokenizer_GetNextToken(tokenizer *Tokenizer)
{
    token Token = {};
    
    string Input = Tokenizer->Input;
    u8 *Base = Input.Data;
    
    // sixten: Consume whitespace
    while(IsWhitespace(Base[Tokenizer->Index]))
    {
        ++Tokenizer->Index;
    }
    
    // sixten(NOTE): Assume single char token.
    Token.String.Data = Base + Tokenizer->Index;
    Token.String.Count = 1;
    
    if(Tokenizer->Index < Input.Count)
    {
        switch(Base[Tokenizer->Index])
        {
            case '{': { Token.Type = Token_CurlyOpen;  } break;
            case '}': { Token.Type = Token_CurlyClose; } break;
            case ';': { Token.Type = Token_Semicolon;  } break;
            case '=': { Token.Type = Token_Equals;  } break;
            
            default:
            {
                if(IsDigit(Base[Tokenizer->Index]) || Base[Tokenizer->Index] == '-')
                {
                    // sixten: Parse integer number
                    Token.Type = Token_IntegerValue;
                    
                    Token.String.Data = Base + Tokenizer->Index;
                    Token.String.Count = 0;
                    
                    while(IsDigit(Token.String.Data[Token.String.Count]) || Token.String.Data[Token.String.Count] == '-')
                    {
                        ++Token.String.Count;
                    }
                }
                else
                {
                    // sixten: Parse tokenizer
                    Token.Type = Token_Identifier;
                    
                    Token.String.Data = Base + Tokenizer->Index;
                    Token.String.Count = 0;
                    
                    while(IsDigit(Token.String.Data[Token.String.Count]) ||
                          IsLetter(Token.String.Data[Token.String.Count]))
                    {
                        ++Token.String.Count;
                    }
                }
            } break;
        }
    }
    else
    {
        Token.Type = Token_EndOfFile;
    }
    
    Tokenizer->Index += Token.String.Count;
    
    return(Token);
}

inline token Tokenizer_PeekNextToken(tokenizer Tokenizer)
{
    // sixten(NOTE): Yup, we just make a copy of the tokenizer and read the next token.
    token Result = Tokenizer_GetNextToken(&Tokenizer);
    return(Result);
}

inline b32 Tokenizer_RequireToken(tokenizer *Tokenizer, token_type Type)
{
    token Token = Tokenizer_GetNextToken(Tokenizer);
    
    b32 Result = (Token.Type == Type);
    return(Result);
}

#endif //VN_TOKENIZER_H