123 lines
3.0 KiB
C
123 lines
3.0 KiB
C
/* date = June 18th 2023 5:12 pm */
|
|
|
|
#ifndef VN_TOKENIZER_H
|
|
#define VN_TOKENIZER_H
|
|
|
|
enum token_type
|
|
{
|
|
Token_EndOfFile,
|
|
Token_Identifier,
|
|
Token_IntegerValue,
|
|
//Token_RealValue,
|
|
Token_CurlyOpen,
|
|
Token_CurlyClose,
|
|
Token_Semicolon,
|
|
Token_Equals,
|
|
};
|
|
|
|
struct tokenizer
|
|
{
|
|
string Input;
|
|
s64 At;
|
|
s64 Index;
|
|
};
|
|
|
|
struct token
|
|
{
|
|
token_type Type;
|
|
string String;
|
|
};
|
|
|
|
inline tokenizer Tokenizer_BeginTokenization(string Input)
|
|
{
|
|
tokenizer Result = {};
|
|
Result.Input = Input;
|
|
|
|
return(Result);
|
|
}
|
|
|
|
static token Tokenizer_GetNextToken(tokenizer *Tokenizer)
|
|
{
|
|
token Token = {};
|
|
|
|
string Input = Tokenizer->Input;
|
|
u8 *Base = Input.Data;
|
|
|
|
// sixten: Consume whitespace
|
|
while(IsWhitespace(Base[Tokenizer->Index]))
|
|
{
|
|
++Tokenizer->Index;
|
|
}
|
|
|
|
// sixten(NOTE): Assume single char token.
|
|
Token.String.Data = Base + Tokenizer->Index;
|
|
Token.String.Count = 1;
|
|
|
|
if(Tokenizer->Index < Input.Count)
|
|
{
|
|
switch(Base[Tokenizer->Index])
|
|
{
|
|
case '{': { Token.Type = Token_CurlyOpen; } break;
|
|
case '}': { Token.Type = Token_CurlyClose; } break;
|
|
case ';': { Token.Type = Token_Semicolon; } break;
|
|
case '=': { Token.Type = Token_Equals; } break;
|
|
|
|
default:
|
|
{
|
|
if(IsDigit(Base[Tokenizer->Index]) || Base[Tokenizer->Index] == '-')
|
|
{
|
|
// sixten: Parse integer number
|
|
Token.Type = Token_IntegerValue;
|
|
|
|
Token.String.Data = Base + Tokenizer->Index;
|
|
Token.String.Count = 0;
|
|
|
|
while(IsDigit(Token.String.Data[Token.String.Count]) || Token.String.Data[Token.String.Count] == '-')
|
|
{
|
|
++Token.String.Count;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
// sixten: Parse tokenizer
|
|
Token.Type = Token_Identifier;
|
|
|
|
Token.String.Data = Base + Tokenizer->Index;
|
|
Token.String.Count = 0;
|
|
|
|
while(IsDigit(Token.String.Data[Token.String.Count]) ||
|
|
IsLetter(Token.String.Data[Token.String.Count]))
|
|
{
|
|
++Token.String.Count;
|
|
}
|
|
}
|
|
} break;
|
|
}
|
|
}
|
|
else
|
|
{
|
|
Token.Type = Token_EndOfFile;
|
|
}
|
|
|
|
Tokenizer->Index += Token.String.Count;
|
|
|
|
return(Token);
|
|
}
|
|
|
|
inline token Tokenizer_PeekNextToken(tokenizer Tokenizer)
|
|
{
|
|
// sixten(NOTE): Yup, we just make a copy of the tokenizer and read the next token.
|
|
token Result = Tokenizer_GetNextToken(&Tokenizer);
|
|
return(Result);
|
|
}
|
|
|
|
inline b32 Tokenizer_RequireToken(tokenizer *Tokenizer, token_type Type)
|
|
{
|
|
token Token = Tokenizer_GetNextToken(Tokenizer);
|
|
|
|
b32 Result = (Token.Type == Type);
|
|
return(Result);
|
|
}
|
|
|
|
#endif //VN_TOKENIZER_H
|