-
Notifications
You must be signed in to change notification settings - Fork 0
Expand file tree
/
Copy pathTokenizer.java
More file actions
147 lines (129 loc) · 4.83 KB
/
Tokenizer.java
File metadata and controls
147 lines (129 loc) · 4.83 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
package token;
import java.io.IOException;
import java.util.ArrayList;
import java.nio.file.Files;
import java.nio.file.Paths;
public class Tokenizer
{
public static void main(String[]args)
{
String filePath="Input.txt";
String testt="";
try{
testt=new String(Files.readAllBytes(Paths.get(filePath)));
}catch(IOException e){
e.printStackTrace();
}
//Integer x equals 6
//Character b equals 5
//String[] tokens=testt.split(" ");
//for (String token : tokens)
//{System.out.print(token + ", ");}
//System.out.print("\n\n\n");
Tokenizer tokenizer=new Tokenizer();
try{
ArrayList<Token> listoftoken=tokenizer.tokenize(testt);
for(Token token:listoftoken)
{System.out.println(token.getValue()+":"+token.getType()+"\n");}
}
catch (IOException e)
{
System.out.println("ERROR");
}
//ExpressionTree tree=tokenizer.buildExpressionTree(listoftoken);
//Calculator calculator=new Calculator();
//double result=calculator.evaluate(tree.root);
//tree.traverseInOrder(tree.root);
//System.out.println("\nresult="+result);
//Integer Integer x equals 5
}
public static boolean isNumeric(String str) {
for (char c : str.toCharArray()) {
if (!Character.isDigit(c)) {return false;}
}
return true;
}
public boolean isDataType(TokenType input)
{
if (input!=TokenType.NUMBER&&input!=TokenType.OPERATOR&&input!=TokenType.CHARACTER)return true;
return false;
}
public boolean isOperator(String input)
{
return (input.equals("equals")||input.equals('-')||input.equals('+')||input.equals('*')||input.equals('/')
||input.equals("plus")||input.equals("minus")||input.equals("times")||input.equals('=')||input.equals("by"));
}
public TokenType nextTokenType(String input)
{
//Character.isAlphabetic(input.charAt(0))
if(input.equals("Integer"))return TokenType.INTEGER;
else if(input.equals("String"))return TokenType.STRING;
else if(input.equals("Character"))return TokenType.CHAR;
else if(input.equals("Double"))return TokenType.DOUBLE;
else if(input.equals("Long"))return TokenType.LONG;
else if(isNumeric(input))return TokenType.NUMBER;
else if(input.equals("if"))return TokenType.IF;
else if(input.equals("Print")||input.equals("Print Line"))return TokenType.PRINT;
else if(input.length()==1)return TokenType.CHARACTER;
// else if(Character.isWhitespace(nextChar))
// {
// return TokenType.WHITESPACE;
// }
else if (isOperator(input))
{
return TokenType.OPERATOR;
}
else
{
return TokenType.INVALID;
}
}
public ArrayList<Token> tokenize(String input) throws IOException{
ArrayList<Token> result = new ArrayList<Token>();
String[] tokens=input.split(" ");
String[] tokens2=input.split("GO");
int stop=0;
// for(String token:tokens2){
// System.out.println(token);}
for(String token:tokens2){
System.out.println(token+"\n");
String[] GO=token.split(" ");
try {
for (int i = 0; i < GO.length; i++) {
String token2 = GO[i];
TokenType k;LineType k2=LineType.ASSIGNMENT;
if (nextTokenType(GO[0])==TokenType.IF)
k2=LineType.CONDITION;
else if (isDataType(nextTokenType(GO[0])))
k2=LineType.ASSIGNMENT;
else if (nextTokenType(GO[0])==TokenType.PRINT)
k2=LineType.PRINT;
else if(nextTokenType(GO[0])==TokenType.INVALID)k2=LineType.INVALID;
if(k2==LineType.ASSIGNMENT)
{
if (i == 0 && !isDataType(nextTokenType(token2))) {
throw new Exception("Invalid data type declaration.");
}
//Character k equals kasfgjasdg
else if (nextTokenType(GO[0])==TokenType.CHAR&&nextTokenType(GO[3])!=TokenType.CHARACTER)
{throw new Exception("ERROR! Declared token type should be CHARACTER");}
result.add(new Token(token2, nextTokenType(token2)));
}
}
} catch (Exception e) {
e.printStackTrace();System.exit(1);
}
}
// for(Token token4:result)
// {
// System.out.println(token4.tokenValue+" "+token4.tokenType);
// }
return result;
}
// public ExpressionTree buildExpressionTree(ArrayList<Token> tokens)
// {
// ExpressionTree tree = new ExpressionTree();
// tree.buildTree(tokens);
// return tree;
// }
}