169 lines
4.5 KiB
C++
169 lines
4.5 KiB
C++
|
|
|
|
#include "ttokenizer.h"
|
|
#include <QString>
|
|
|
|
namespace TSyntax {
|
|
|
|
int Token::getIntValue() const {
|
|
return QString::fromStdString(getText()).toInt();
|
|
}
|
|
|
|
double Token::getDoubleValue() const {
|
|
return QString::fromStdString(getText()).toDouble();
|
|
}
|
|
|
|
//===================================================================
|
|
|
|
Tokenizer::Tokenizer() : m_buffer(), m_index(0) {}
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
Tokenizer::Tokenizer(std::string buffer) : m_buffer(), m_index(0) {
|
|
setBuffer(buffer);
|
|
}
|
|
|
|
//===================================================================
|
|
|
|
Tokenizer::~Tokenizer() {}
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
void Tokenizer::setBuffer(std::string buffer) {
|
|
m_buffer = buffer + '\0';
|
|
m_index = 0;
|
|
m_tokens.clear();
|
|
|
|
bool stringBlock = false;
|
|
|
|
int i = 0;
|
|
const char *s = &m_buffer[0];
|
|
for (;;) {
|
|
while (isascii(s[i]) && isspace(s[i])) i++;
|
|
|
|
int j = i;
|
|
|
|
if (s[i] == '\0') {
|
|
m_tokens.push_back(Token("", Token::Eos, j));
|
|
break;
|
|
}
|
|
|
|
if (s[i] == '"') {
|
|
stringBlock = !stringBlock;
|
|
m_tokens.push_back(Token("\"", Token::Punct, j));
|
|
|
|
++i;
|
|
continue;
|
|
}
|
|
|
|
std::string token;
|
|
|
|
if (stringBlock) {
|
|
// string block - read mercilessly until either another '"' or EOS
|
|
token = std::string(1, s[i++]);
|
|
|
|
while (s[i] != '"' && s[i] != '\0') token.append(1, s[i++]);
|
|
|
|
m_tokens.push_back(Token(token, Token::Ident, j));
|
|
} else if ((isascii(s[i]) && isalpha(s[i])) || s[i] == '_') {
|
|
// ident
|
|
token = std::string(1, s[i++]);
|
|
|
|
while (isascii(s[i]) && (isalpha(s[i]) || s[i] == '_' || isdigit(s[i])))
|
|
token.append(1, s[i++]);
|
|
|
|
m_tokens.push_back(Token(token, Token::Ident, j));
|
|
} else if ((isascii(s[i]) && isdigit(s[i])) || s[i] == '.') {
|
|
// number
|
|
while (isascii(s[i]) && isdigit(s[i])) token.append(1, s[i++]);
|
|
|
|
if (s[i] == '.') {
|
|
token.append(1, s[i++]);
|
|
|
|
while (isascii(s[i]) && isdigit(s[i])) token.append(1, s[i++]);
|
|
|
|
if ((s[i] == 'e' || s[i] == 'E') &&
|
|
(((isascii(s[i + 1]) && isdigit(s[i + 1])) ||
|
|
s[i + 1] == '-' || s[i + 1] == '+') && isascii(s[i + 2]) &&
|
|
isdigit(s[i + 2]))) {
|
|
token.append(1, s[i++]);
|
|
|
|
if (s[i] == '-' || s[i] == '+') token.append(1, s[i++]);
|
|
|
|
while (isascii(s[i]) && isdigit(s[i])) token.append(1, s[i++]);
|
|
}
|
|
}
|
|
m_tokens.push_back(Token(token, Token::Number, j));
|
|
} else {
|
|
// punct.
|
|
if (s[i + 1] != '\0') {
|
|
token = std::string(s + i, 2);
|
|
|
|
const std::string ss[] = {"==", "!=", ">=", "<=", "||", "&&"};
|
|
|
|
if (std::find(std::begin(ss), std::end(ss), token) != std::end(ss))
|
|
i += 2;
|
|
else
|
|
token = std::string(1, s[i++]);
|
|
} else
|
|
token = std::string(1, s[i++]);
|
|
|
|
m_tokens.push_back(Token(token, Token::Punct, j));
|
|
}
|
|
}
|
|
}
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
int Tokenizer::getTokenCount() const { return m_tokens.size(); }
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
const Token &Tokenizer::getToken(int index) const {
|
|
assert(0 <= index && index < getTokenCount());
|
|
return m_tokens[index];
|
|
}
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
void Tokenizer::reset() { m_index = 0; }
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
const Token &Tokenizer::getToken() { return getToken(m_index); }
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
Token Tokenizer::nextToken() {
|
|
Token token = getToken();
|
|
if (m_index + 1 < getTokenCount()) m_index++;
|
|
return token;
|
|
}
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
bool Tokenizer::eos() const { return m_index + 1 == getTokenCount(); }
|
|
|
|
//-------------------------------------------------------------------
|
|
|
|
Token Tokenizer::getTokenFromPos(int pos) const {
|
|
int len = m_buffer.length();
|
|
if (pos < 0 || pos >= len) return Token(pos);
|
|
int x = 0;
|
|
for (int i = 0; i < getTokenCount(); i++) {
|
|
const Token &token = getToken(i);
|
|
int y = token.getPos();
|
|
if (pos < y) {
|
|
assert(x < y);
|
|
return Token(x, y - 1);
|
|
}
|
|
x = y + (int)token.getText().length();
|
|
if (pos < x) return token;
|
|
}
|
|
assert(x < len);
|
|
return Token(x, len - 1);
|
|
}
|
|
|
|
//===================================================================
|
|
|
|
} // namespace TSyntax
|