function get_tokens(str)
local string = string
local table = table
local tokens = {}
local stringbuffer = ""
local numberbuffer = ""
local previous_is_number = false
local previous_is_string = false
local previous_is_quote = false
local has_process = false
local is_X16mode = false
for i = 1, #str do
local char = string.sub(str, i, i)
has_process = false
if previous_is_number == false and (string.find(char, "%a") or string.find(char, "_")
or (string.find(char, "%d") and previous_is_string) )
then
stringbuffer = string.format("%s%s", stringbuffer, char)
previous_is_string = true
has_process = true
end
if not (previous_is_number and char == ".") and (has_process == false and string.find(char, "%p")) then
if stringbuffer ~= "" then
table.insert(tokens, stringbuffer)
stringbuffer = ""
previous_is_string = false
end
if numberbuffer ~= "" then
table.insert(tokens, numberbuffer)
numberbuffer = ""
previous_is_number = false
is_X16mode = false
end
table.insert(tokens, char)
previous_is_quote = true
has_process = true
end
if has_process == false and ((string.find(char, "%d") or (numberbuffer ~= "" and string.find(char, "%.")))
or (is_X16mode and string.find(char, "%x")) ) then
if (previous_is_number == false) then
local nextchar = string.sub(str, i+1, i+1)
if nextchar and (nextchar == "x" or nextchar == "X") then
is_X16mode = true
i = i+1
char = char .. nextchar
end
end
numberbuffer = string.format("%s%s", numberbuffer, char)
previous_is_number = true
has_process = true
end
end
print_table(tokens)
return tokens
end
--local str = [[dequeOf.use("1155.321"). STRINGI.push_back(CComVariant("{"));]]
--local str = [[if ((vtVal.vt != VT_I4) || (0x0000FFFF<(DWORD)vtVal.lVal))]]
--local str = [[333, 0xB3667A2E, 0xC4614AB8, 0x5D681B02, 0x2A6F2B94, 0xB40BBE37, 0xC30C8EA1, 0x5A05DF1B, 0x2D02EF8D};]]
--get_tokens(str)
