114 lines
2 KiB
C++
114 lines
2 KiB
C++
|
|
|
|
#ifndef TTOKENIZER_INCLUDED
|
|
#define TTOKENIZER_INCLUDED
|
|
|
|
#include "tcommon.h"
|
|
|
|
#ifdef _WIN32
|
|
#pragma warning(push)
|
|
#pragma warning(disable : 4251)
|
|
#endif
|
|
|
|
#undef DVAPI
|
|
#undef DVVAR
|
|
#ifdef TNZBASE_EXPORTS
|
|
#define DVAPI DV_EXPORT_API
|
|
#define DVVAR DV_EXPORT_VAR
|
|
#else
|
|
#define DVAPI DV_IMPORT_API
|
|
#define DVVAR DV_IMPORT_VAR
|
|
#endif
|
|
|
|
namespace TSyntax
|
|
{
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
class DVAPI Token
|
|
{
|
|
public:
|
|
enum Type { None,
|
|
Space,
|
|
Ident,
|
|
Number,
|
|
Punct,
|
|
Eos };
|
|
|
|
Token(int pos = 0)
|
|
: m_text(""), m_type(None), m_pos(pos)
|
|
{
|
|
}
|
|
Token(int p0, int p1)
|
|
: m_text(p1 - p0 + 1, ' '), m_type(Space), m_pos(p0)
|
|
{
|
|
}
|
|
|
|
Token(string text, Type type, int pos)
|
|
: m_text(text), m_type(type), m_pos(pos)
|
|
{
|
|
}
|
|
|
|
Type getType() const { return m_type; }
|
|
|
|
string getText() const { return m_text; }
|
|
int getIntValue() const;
|
|
double getDoubleValue() const;
|
|
|
|
int getPos() const { return m_pos; }
|
|
int getPos1() const { return m_pos + m_text.length() - 1; }
|
|
|
|
private:
|
|
string m_text;
|
|
int m_pos;
|
|
Type m_type;
|
|
};
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
class DVAPI Tokenizer
|
|
{
|
|
|
|
string m_buffer;
|
|
std::vector<Token> m_tokens;
|
|
int m_index;
|
|
|
|
public:
|
|
Tokenizer();
|
|
Tokenizer(string buffer);
|
|
~Tokenizer();
|
|
|
|
void setBuffer(string buffer);
|
|
string getBuffer() const { return m_buffer; }
|
|
|
|
int getTokenCount() const;
|
|
const Token &getToken(int index) const;
|
|
|
|
//! get the token containing the pos-th character in the input string
|
|
Token getTokenFromPos(int pos) const;
|
|
|
|
//! reset the token index. (set it to 0)
|
|
void reset();
|
|
|
|
//! return the current token (possibly Eos)
|
|
const Token &getToken();
|
|
|
|
//! same as getToken(), but post-increment the token index (if !eos())
|
|
Token nextToken();
|
|
|
|
//! return true if the current token is the last one
|
|
bool eos() const;
|
|
|
|
//! to read all the sequence:
|
|
//! while(!tokenizer.eos()) {token = tokenizer.nextToken();...}
|
|
};
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
} // namespace TSyntax
|
|
|
|
#ifdef _WIN32
|
|
#pragma warning(pop)
|
|
#endif
|
|
|
|
#endif
|