/****************************************************************************

**

** Copyright (C) 2008-2009 Nokia Corporation and/or its subsidiary(-ies).

** All rights reserved.

** Contact: Nokia Corporation (qt-info@nokia.com)

**

** This file is part of the Qt Script Generator project on Qt Labs.

**

** $QT_BEGIN_LICENSE:LGPL$

** No Commercial Usage

** This file contains pre-release code and may not be distributed.

** You may use this file in accordance with the terms and conditions

** contained in the Technology Preview License Agreement accompanying

** this package.

**

** GNU Lesser General Public License Usage

** Alternatively, this file may be used under the terms of the GNU Lesser

** General Public License version 2.1 as published by the Free Software

** Foundation and appearing in the file LICENSE.LGPL included in the

** packaging of this file. Please review the following information to

** ensure the GNU Lesser General Public License version 2.1 requirements

** will be met: http://www.gnu.org/licenses/old-licenses/lgpl-2.1.html.

**

** In addition, as a special exception, Nokia gives you certain additional

** rights. These rights are described in the Nokia Qt LGPL Exception

** version 1.1, included in the file LGPL_EXCEPTION.txt in this package.

**

** If you have questions regarding the use of this file, please contact

** Nokia at qt-info@nokia.com.

**

**

**

**

**

**

**

**

** $QT_END_LICENSE$

**

****************************************************************************/

#include "lexer.h"

#include "tokens.h"

#include "control.h"

#include <cctype>

#include <iostream>

scan_fun_ptr Lexer::s_scan_keyword_table[] = {

&Lexer::scanKeyword0, &Lexer::scanKeyword0,

&Lexer::scanKeyword2, &Lexer::scanKeyword3,

&Lexer::scanKeyword4, &Lexer::scanKeyword5,

&Lexer::scanKeyword6, &Lexer::scanKeyword7,

&Lexer::scanKeyword8, &Lexer::scanKeyword9,

&Lexer::scanKeyword10, &Lexer::scanKeyword11,

&Lexer::scanKeyword12, &Lexer::scanKeyword13,

&Lexer::scanKeyword14, &Lexer::scanKeyword0,

&Lexer::scanKeyword16

};

void LocationManager::extract_line(int offset, int *line, QString *filename) const

{

*line = 0;

if (token_stream.size () < 1)

return;

const unsigned char *begin_buffer = reinterpret_cast<const unsigned char *>(token_stream[0].text);

const unsigned char *cursor = begin_buffer + offset;

++cursor; // skip '#'

if (std::isspace(*cursor) && std::isdigit(*(cursor + 1)))

{

++cursor;

char buffer[1024], *cp = buffer;

do {

*cp++ = *cursor++;

} while (std::isdigit(*cursor));

*cp = '\0';

int l = strtol(buffer, 0, 0);

Q_ASSERT(std::isspace(*cursor));

++cursor;

Q_ASSERT(*cursor == '"');

++cursor;

cp = buffer;

while (*cursor && *cursor != '"') {

*cp++ = *cursor++;

}

*cp = '\0';

Q_ASSERT(*cursor == '"');

++cursor;

*filename = buffer;

*line = l;

// printf("filename: %s line: %d\n", buffer, line);

}

}

void LocationManager::positionAt(std::size_t offset, int *line, int *column,

QString *filename) const

{

int ppline, ppcolumn;

line_table.positionAt(offset, &ppline, &ppcolumn);

int base_line;

extract_line((int) line_table[ppline-1], &base_line, filename);

int line2, column2;

location_table.positionAt((int) line_table[ppline-1], &line2, &column2);

location_table.positionAt(offset, line, column);

*line = base_line + *line - line2 - 1;

}

scan_fun_ptr Lexer::s_scan_table[256];

bool Lexer::s_initialized = false;

void Lexer::tokenize(const char *contents, std::size_t size)

{

if (!s_initialized)

initialize_scan_table();

token_stream.resize(1024);

token_stream[0].kind = Token_EOF;

token_stream[0].text = contents;

index = 1;

cursor = (const unsigned char *) contents;

begin_buffer = (const unsigned char *) contents;

end_buffer = cursor + size;

location_table.resize(1024);

location_table[0] = 0;

location_table.current_line = 1;

line_table.resize(1024);

line_table[0] = 0;

line_table.current_line = 1;

do {

if (index >= token_stream.size()-4) // increase size of token_stream early, in case more than one token is written in one go

token_stream.resize(token_stream.size() * 2);

Token *current_token = &token_stream[(int) index];

current_token->text = reinterpret_cast<const char*>(begin_buffer);

current_token->position = cursor - begin_buffer;

(this->*s_scan_table[*cursor])();

current_token->size = cursor - begin_buffer - current_token->position;

} while (cursor < end_buffer);

if (index == token_stream.size())

token_stream.resize(token_stream.size() * 2);

Q_ASSERT(index < token_stream.size());

token_stream[(int) index].position = cursor - begin_buffer;

token_stream[(int) index].kind = Token_EOF;

}

void Lexer::reportError(const QString& msg)

{

int line, column;

QString fileName;

std::size_t tok = token_stream.cursor();

_M_location.positionAt(token_stream.position(tok),

&line, &column, &fileName);

Control::ErrorMessage errmsg;

errmsg.setLine(line + 1);

errmsg.setColumn(column);

errmsg.setFileName(fileName);

errmsg.setMessage(QLatin1String("** LEXER ERROR ") + msg);

control->reportError(errmsg);

}

void Lexer::initialize_scan_table()

{

s_initialized = true;

for (int i=0; i<256; ++i)

{

if (isspace(i))

s_scan_table[i] = &Lexer::scan_white_spaces;

else if (isalpha(i) || i == '_')

s_scan_table[i] = &Lexer::scan_identifier_or_keyword;

else if (isdigit(i))

s_scan_table[i] = &Lexer::scan_int_constant;

else

s_scan_table[i] = &Lexer::scan_invalid_input;

}

s_scan_table[int('L')] = &Lexer::scan_identifier_or_literal;

s_scan_table[int('\n')] = &Lexer::scan_newline;

s_scan_table[int('#')] = &Lexer::scan_preprocessor;

s_scan_table[int('\'')] = &Lexer::scan_char_constant;

s_scan_table[int('"')] = &Lexer::scan_string_constant;

s_scan_table[int('.')] = &Lexer::scan_int_constant;

s_scan_table[int('!')] = &Lexer::scan_not;

s_scan_table[int('%')] = &Lexer::scan_remainder;

s_scan_table[int('&')] = &Lexer::scan_and;

s_scan_table[int('(')] = &Lexer::scan_left_paren;

s_scan_table[int(')')] = &Lexer::scan_right_paren;

s_scan_table[int('*')] = &Lexer::scan_star;

s_scan_table[int('+')] = &Lexer::scan_plus;

s_scan_table[int(',')] = &Lexer::scan_comma;

s_scan_table[int('-')] = &Lexer::scan_minus;

s_scan_table[int('/')] = &Lexer::scan_divide;

s_scan_table[int(':')] = &Lexer::scan_colon;

s_scan_table[int(';')] = &Lexer::scan_semicolon;

s_scan_table[int('<')] = &Lexer::scan_less;

s_scan_table[int('=')] = &Lexer::scan_equal;

s_scan_table[int('>')] = &Lexer::scan_greater;

s_scan_table[int('?')] = &Lexer::scan_question;

s_scan_table[int('[')] = &Lexer::scan_left_bracket;

s_scan_table[int(']')] = &Lexer::scan_right_bracket;

s_scan_table[int('^')] = &Lexer::scan_xor;

s_scan_table[int('{')] = &Lexer::scan_left_brace;

s_scan_table[int('|')] = &Lexer::scan_or;

s_scan_table[int('}')] = &Lexer::scan_right_brace;

s_scan_table[int('~')] = &Lexer::scan_tilde;

s_scan_table[0] = &Lexer::scan_EOF;

}

void Lexer::scan_preprocessor()

{

if (line_table.current_line == line_table.size())

line_table.resize(line_table.current_line * 2);

line_table[(int) line_table.current_line++] = (cursor - begin_buffer);

while (*cursor && *cursor != '\n')

++cursor;

if (*cursor != '\n')

reportError("expected newline");

}

void Lexer::scan_char_constant_with_prefix(const unsigned char* prefix)

{

const unsigned char *begin = prefix ? prefix : cursor;

++cursor;

while (*cursor && *cursor != '\'')

{

if (*cursor == '\n')

reportError("did not expect newline");

if (*cursor == '\\')

++cursor;

++cursor;

}

if (*cursor != '\'')

reportError("expected \'");

++cursor;

token_stream[(int) index].extra.symbol =

control->findOrInsertName((const char*) begin, cursor - begin);

token_stream[(int) index++].kind = Token_char_literal;

}

void Lexer::scan_string_constant_with_prefix(const unsigned char* prefix)

{

const unsigned char *begin = prefix ? prefix : cursor;

++cursor;

while (*cursor && *cursor != '"')

{

if (*cursor == '\n')

reportError("did not expect newline");

if (*cursor == '\\')

++cursor;

++cursor;

}

if (*cursor != '"')

reportError("expected \"");

++cursor;

token_stream[(int) index].extra.symbol =

control->findOrInsertName((const char*) begin, cursor - begin);

token_stream[(int) index++].kind = Token_string_literal;

}

void Lexer::scan_raw_string_constant_with_prefix(const unsigned char* prefix)

{

// always starts with "

const unsigned char* begin = prefix ? prefix : cursor;

int delimiterLength = 0;

int endSequenceLength = 0;

bool stillValidDelimiter = true;

++cursor;

while (*cursor)

{

if (!delimiterLength)

{

if (*cursor == '"')

{

break;

}

else if (*cursor == '\n')

{

// this would probably not be a valid delimiter

stillValidDelimiter = false;

}

else if (stillValidDelimiter && *cursor == '(' && (cursor - begin) < 16)

{

// delimiter sequence identified (see https://en.cppreference.com/w/cpp/language/string_literal)

delimiterLength = cursor - begin;

}

}

else if (endSequenceLength)

{

// possible end delimiter sequence

if (endSequenceLength == delimiterLength && *cursor == '"')

{

break;

}

else if (endSequenceLength < delimiterLength && *cursor == begin[endSequenceLength])

{

endSequenceLength++;

}

else

{

// this is not the end of the string, go back to

// after the starting ')' and try again

cursor -= endSequenceLength;

endSequenceLength = 0;

}

}

else if (*cursor == ')')

{

// this might be the start of the end delimiter sequence

endSequenceLength = 1;

}

++cursor;

}

if (*cursor != '"')

reportError("expected \"");

++cursor;

token_stream[(int)index].extra.symbol =

control->findOrInsertName((const char*)begin, cursor - begin);

token_stream[(int)index++].kind = Token_string_literal;

}

void Lexer::scan_newline()

{

if (location_table.current_line == location_table.size())

location_table.resize(location_table.current_line * 2);

location_table[(int) location_table.current_line++] = (cursor - begin_buffer);

++cursor;

}

void Lexer::scan_white_spaces()

{

while (isspace(*cursor))

{

if (*cursor == '\n')

scan_newline();

else

++cursor;

}

}

void Lexer::scan_identifier_or_literal()

{

switch (*(cursor + 1))

{

case '\'':

++cursor;

scan_char_constant();

break;

case '\"':

++cursor;

scan_string_constant();

break;

default:

scan_identifier_or_keyword();

break;

}

}

void Lexer::scan_identifier_or_keyword()

{

const unsigned char *skip = cursor;

const unsigned char* start = cursor;

while (isalnum(*skip) || *skip== '_')

++skip;

int n = skip - cursor;

if (*skip == '"' && n <= 3)

{

cursor = skip;

// this should be a unicode and/or raw string -

// we pass through anything that does not follow the standard, though

if (skip[-1] == 'R')

{

scan_raw_string_constant_with_prefix(start);

}

else

{

scan_string_constant_with_prefix(start);

}

}

else if (*skip == '\'' && n <= 2)

{

// probably some special encoding

cursor = skip;

scan_char_constant_with_prefix(start);

}

else

{

Token* current_token = &token_stream[(int)index];

(this->*s_scan_keyword_table[n < 17 ? n : 0])();

if (current_token->kind == Token_identifier)

{

current_token->extra.symbol =

control->findOrInsertName((const char*)cursor, n);

}

cursor = skip;

}

}

void Lexer::scan_int_constant()

{

if (*cursor == '.' && !std::isdigit(*(cursor + 1)))

{

scan_dot();

return;

}

const unsigned char *begin = cursor;

while (isalnum(*cursor) || *cursor == '.')

++cursor;

token_stream[(int) index].extra.symbol =

control->findOrInsertName((const char*) begin, cursor - begin);

token_stream[(int) index++].kind = Token_number_literal;

}

void Lexer::scan_not()

{

/*

'!' ::= not

'!=' ::= not_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_not_eq;

}

else

{

token_stream[(int) index++].kind = '!';

}

}

void Lexer::scan_remainder()

{

/*

'%' ::= remainder

'%=' ::= remainder_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else

{

token_stream[(int) index++].kind = '%';

}

}

void Lexer::scan_and()

{

/*

'&&' ::= and_and

'&' ::= and

'&=' ::= and_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else if (*cursor == '&')

{

++cursor;

token_stream[(int) index++].kind = Token_and;

}

else

{

token_stream[(int) index++].kind = '&';

}

}

void Lexer::scan_left_paren()

{

++cursor;

token_stream[(int) index++].kind = '(';

}

void Lexer::scan_right_paren()

{

++cursor;

token_stream[(int) index++].kind = ')';

}

void Lexer::scan_star()

{

/*

'*' ::= star

'*=' ::= star_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else

{

token_stream[(int) index++].kind = '*';

}

}

void Lexer::scan_plus()

{

/*

'+' ::= plus

'++' ::= incr

'+=' ::= plus_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else if (*cursor == '+')

{

++cursor;

token_stream[(int) index++].kind = Token_incr;

}

else

{

token_stream[(int) index++].kind = '+';

}

}

void Lexer::scan_comma()

{

++cursor;

token_stream[(int) index++].kind = ',';

}

void Lexer::scan_minus()

{

/*

'-' ::= minus

'--' ::= decr

'-=' ::= minus_equal

'->' ::= left_arrow

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else if (*cursor == '-')

{

++cursor;

token_stream[(int) index++].kind = Token_decr;

}

else if (*cursor == '>')

{

++cursor;

token_stream[(int) index++].kind = Token_arrow;

if (*cursor == '*')

{

++cursor;

token_stream[(int) index++].kind = Token_ptrmem;

}

}

else

{

token_stream[(int) index++].kind = '-';

}

}

void Lexer::scan_dot()

{

/*

'.' ::= dot

'...' ::= ellipsis

*/

++cursor;

if (*cursor == '.' && *(cursor + 1) == '.')

{

cursor += 2;

token_stream[(int) index++].kind = Token_ellipsis;

}

else if (*cursor == '.' && *(cursor + 1) == '*')

{

cursor += 2;

token_stream[(int) index++].kind = Token_ptrmem;

}

else

token_stream[(int) index++].kind = '.';

}

void Lexer::scan_divide()

{

/*

'/' ::= divide

'/=' ::= divide_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else

{

token_stream[(int) index++].kind = '/';

}

}

void Lexer::scan_colon()

{

++cursor;

if (*cursor == ':')

{

++cursor;

token_stream[(int) index++].kind = Token_scope;

}

else

{

token_stream[(int) index++].kind = ':';

}

}

void Lexer::scan_semicolon()

{

++cursor;

token_stream[(int) index++].kind = ';';

}

void Lexer::scan_less()

{

/*

'<' ::= less

'<<' ::= left_shift

'<<=' ::= left_shift_equal

'<=' ::= less_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_leq;

}

else if (*cursor == '<')

{

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else

{

token_stream[(int) index++].kind = Token_shift_left;

}

}

else

{

token_stream[(int) index++].kind = '<';

}

}

void Lexer::scan_equal()

{

/*

'=' ::= equal

'==' ::= equal_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_eq;

}

else

{

token_stream[(int) index++].kind = '=';

}

}

void Lexer::scan_greater()

{

/*

'>' ::= greater

'>=' ::= greater_equal

'>>' ::= right_shift

'>>=' ::= right_shift_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_geq;

}

else if (*cursor == '>')

{

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else

{

// may be replaced by two ">" during parsing

token_stream[(int) index++].kind = Token_shift_right;

token_stream[(int) index++].kind = Token_placeholder;

}

}

else

{

token_stream[(int) index++].kind = '>';

}

}

void Lexer::scan_question()

{

++cursor;

token_stream[(int) index++].kind = '?';

}

void Lexer::scan_left_bracket()

{

++cursor;

token_stream[(int) index++].kind = '[';

}

void Lexer::scan_right_bracket()

{

++cursor;

token_stream[(int) index++].kind = ']';

}

void Lexer::scan_xor()

{

/*

'^' ::= xor

'^=' ::= xor_equal

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else

{

token_stream[(int) index++].kind = '^';

}

}

void Lexer::scan_left_brace()

{

++cursor;

token_stream[(int) index++].kind = '{';

}

void Lexer::scan_or()

{

/*

'|' ::= or

'|=' ::= or_equal

'||' ::= or_or

*/

++cursor;

if (*cursor == '=')

{

++cursor;

token_stream[(int) index++].kind = Token_assign;

}

else if (*cursor == '|')

{

++cursor;

token_stream[(int) index++].kind = Token_or;

}

else

{

token_stream[(int) index++].kind = '|';

}

}

void Lexer::scan_right_brace()

{

++cursor;

token_stream[(int) index++].kind = '}';

}

void Lexer::scan_tilde()

{

++cursor;

token_stream[(int) index++].kind = '~';

}

void Lexer::scan_EOF()

{

++cursor;

token_stream[(int) index++].kind = Token_EOF;

}

void Lexer::scan_invalid_input()

{

QString errmsg("invalid input: %1");

reportError(errmsg.arg(int(*cursor)));

++cursor;

}

void LocationTable::positionAt(std::size_t offset, int max_line,

int *line, int *column) const

{

if (!(line && column && max_line != 0))

return;

int first = 0;

int len = max_line;

int half;

int middle;

while (len > 0)

{

half = len >> 1;

middle = first;

middle += half;

if (lines[middle] < offset)

{

first = middle;

++first;

len = len - half - 1;

}

else

len = half;

}

*line = std::max(first, 1);

*column = (int) (offset - lines[*line - 1] - 1);

if (*column < 0)

{

*column = 0;

}

}

void Lexer::scanKeyword0()

{

token_stream[(int) index++].kind = Token_identifier;

}

void Lexer::scanKeyword2()

{

switch (*cursor)

{

case 'i':

if (*(cursor + 1) == 'f')

{

token_stream[(int) index++].kind = Token_if;

return;

}

break;

case 'd':

if (*(cursor + 1) == 'o')

{

token_stream[(int) index++].kind = Token_do;

return;

}

break;

case 'o':

if (*(cursor + 1) == 'r')

{

token_stream[(int) index++].kind = Token_or;

return;

}

break;

}

token_stream[(int) index++].kind = Token_identifier;

}

void Lexer::scanKeyword3()

{

switch (*cursor)

{

case 'a':

if (*(cursor + 1) == 'n' &&

*(cursor + 2) == 'd')

{

token_stream[(int) index++].kind = Token_and;

return;

}

if (*(cursor + 1) == 's' &&

*(cursor + 2) == 'm')

{

token_stream[(int) index++].kind = Token_asm;

return;

}

break;

case 'f':

if (*(cursor + 1) == 'o' &&

*(cursor + 2) == 'r')

{

token_stream[(int) index++].kind = Token_for;

return;

}

break;

case 'i':

if (*(cursor + 1) == 'n' &&

*(cursor + 2) == 't')

{

token_stream[(int) index++].kind = Token_int;

return;

}

break;

case 'n':

if (*(cursor + 1) == 'e' &&

*(cursor + 2) == 'w')

{

token_stream[(int) index++].kind = Token_new;

return;

}

if (*(cursor + 1) == 'o' &&

*(cursor + 2) == 't')

{

token_stream[(int) index++].kind = Token_not;

return;

}

break;

case 't':

if (*(cursor + 1) == 'r' &&

*(cursor + 2) == 'y')