To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/lenasys/trunk
« back to all changes in this revision
Viewing changes to Code Tokenizer and Parser/Parser_Test.js
- browse files at revision 21
- compare with another revision
- download diff
- download tarball
- view history from revision 21
- Committer: Gustav Hartvigsson
- Date: 2013-04-08 08:05:58 UTC
- mfrom: (19.4.4 lenasys)
- Revision ID: gustav.hartvigsson@gmail.com-20130408080558-0o7itutyu7jrn48b
Note to self: "bzr resolve --all" works like magic.
Merged implementation group 1's branch.
20130408.
Merged implementation group 1's branch.
20130408.
- files modified:
- Code Tokenizer and Parser/Parser_Test.css
added
removed
Code Tokenizer and Parser/Parser_Test.js
1
// Array to hold the tokens
2
var tokens = [];
1
3
/*
2
4
* Token class and storage definition
3
5
*/
4
5
6
function token(kind, val, fromchar, tochar, row) {
6
7
this.kind = kind;
7
8
this.val = val;
8
9
this.from = fromchar;
9
10
this.to = tochar;
10
11
this.row = row;
11
}
12
13
// Array to hold the tokens
14
var tokens = [];
12
}
15
13
/*
16
14
* Store token in tokens array
17
15
* Creates a new token object using the constructor
18
16
*/
19
20
17
function makeToken(kind, val, from, to, rowno) {
21
18
newtoken = new token(kind, val, from, to, rowno);
22
19
tokens.push(newtoken);
23
20
}
24
25
21
/*
26
22
* Writes error from tokenizer
27
23
*/
28
29
24
function error(str, val, row) {
30
25
alert("Tokenizer Error: " + str + val + " at row " + row);
31
26
}
32
33
27
/*
34
28
* Tokenize function
35
29
* Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
36
30
* The tokenizer is passed a string, and a string of prefix and suffix terminators
37
*/
38
31
*/
39
32
function tokenize(instring, inprefix, insuffix) {
40
33
// index of the start of the token.
41
34
var from;
77
70
break;
78
71
}
79
72
}
80
maketoken('name', str, from, i, row);
73
makeToken('name', str, from, i, row);
81
74
// Number token
82
} else if (c >= '0' && c <= '9') {
83
str = c;
84
i++;
85
for(;;) {
86
c = instring.charAt(i);
87
if(c < '0' || c > '9') {
88
break;
89
}
90
i++;
91
str += c;
92
}
93
if(c == '.') {
94
i++;
95
str += c;
96
for(;;){
97
c=instring.charAt(i);
98
if(c < '0' || c > '9'){
99
break;
100
}
101
i++;
102
str += c;
103
}
104
}
105
if(c == 'e' || c =='E') {
106
i++;
107
str += c;
108
c=instring.charAt(i);
109
if(c == '-' || c == '+') {
110
i++;
111
str += c;
112
c=instring.charAt(i);
113
}
114
if(c < '0' || c > '9') {
115
error('Bad Exponent in Number: ',str,row);
116
}
117
do {
118
i++;
119
str += c;
120
c = instring.charAt(i);
121
} while(c >= '0' && c <= '9');
122
}
123
if(c >= 'a' && c <= 'z') {
124
str += c;
125
i++;
126
error('Bad Number: ',str,row);
127
}
128
n =+ str;
129
if(isFinite(n)) {
130
maketoken('number', n, from, i, row);
131
} else {
132
error('Bad Number: ', str, row);
75
} else if (c >= '0' && c <= '9') {
76
str = c;
77
i++;
78
for(;;) {
79
c = instring.charAt(i);
80
if(c < '0' || c > '9') {
81
break;
82
}
83
i++;
84
str += c;
85
}
86
if(c == '.') {
87
i++;
88
str += c;
89
for(;;){
90
c=instring.charAt(i);
91
if(c < '0' || c > '9'){
92
break;
93
}
94
i++;
95
str += c;
96
}
97
}
98
if(c == 'e' || c =='E') {
99
i++;
100
str += c;
101
c=instring.charAt(i);
102
if(c == '-' || c == '+') {
103
i++;
104
str += c;
105
c=instring.charAt(i);
133
106
}
134
// String .. handles c style breaking codes
135
} else if (c == '\'' || c == '"') {
136
str = '';
137
q = c;
138
i++;
139
for(;;) {
140
c = instring.charAt(i);
141
if(c < ' ') {
142
if((c == '\n') || (c == '\r') || (c == '')) {
143
// Add row if this white space is a row terminator
144
row++;
145
} else {
146
error('Unterminated String: ', str, row);
147
}
148
}
149
if (c == q) {
150
break;
151
}
152
if(c == '\\') {
153
i++;
154
if(i >= length) {
155
error('Unterminated String: ', str, row);
156
}
157
c = instring.charAt(i);
158
if(c == 'b') {
159
c = '\b';
160
break;
161
}
162
if(c == 'f') {
163
c = '\f';
164
break;
165
}
166
if(c == 'n') {
167
c = '\n';
168
break;
169
}
170
if(c == 'r') {
171
c = '\r';
172
break;
173
}
174
if(c == 't') {
175
c = '\t';
176
break;
177
}
178
if(c == 'u') {
179
if(i >= length) {
180
error('Unterminated String: ', str, row);
181
}
182
c = parseInt(this.substr(i + 1, 4), 16);
183
if(!isFinite(c) || c < 0) {
184
error('Unterminated String: ', str, row);
185
}
186
c = String.fromCharCode(c);
187
i+=4;
188
break;
189
}
190
}
191
str += c;
192
i++;
193
}
194
i++;
195
maketoken('string', str, from, i, row);
196
c = instring.charAt(i);
197
// Comment of // type ... does not cover block comments
198
} else if (c == '/' && instring.charAt(i+1) == '/') {
199
i++;
200
for(;;) {
201
c=instring.charAt(i);
202
if(c == '\n' || c == '\r' || c == '') {
203
row++;
204
break;
205
}
206
i++;
207
}
107
if(c < '0' || c > '9') {
108
error('Bad Exponent in Number: ',str,row);
109
}
110
do {
111
i++;
112
str += c;
113
c = instring.charAt(i);
114
} while(c >= '0' && c <= '9');
115
}
116
if(c >= 'a' && c <= 'z') {
117
str += c;
118
i++;
119
error('Bad Number: ',str,row);
120
}
121
n =+ str;
122
if(isFinite(n)) {
123
makeToken('number', n, from, i, row);
124
} else {
125
error('Bad Number: ', str, row);
126
}
127
// String .. handles c style breaking codes
128
} else if (c == '\'' || c == '"') {
129
str = '';
130
q = c;
131
i++;
132
for(;;) {
133
c = instring.charAt(i);
134
if(c < ' ') {
135
if((c == '\n') || (c == '\r') || (c == '')) {
136
// Add row if this white space is a row terminator
137
row++;
138
} else {
139
error('Unterminated String: ', str, row);
140
}
141
}
142
if (c == q) {
143
break;
144
}
145
if(c == '\\') {
146
i++;
147
if(i >= length) {
148
error('Unterminated String: ', str, row);
149
}
150
c = instring.charAt(i);
151
if(c == 'b') {
152
c = '\b';
153
break;
154
}
155
if(c == 'f') {
156
c = '\f';
157
break;
158
}
159
if(c == 'n') {
160
c = '\n';
161
break;
162
}
163
if(c == 'r') {
164
c = '\r';
165
break;
166
}
167
if(c == 't') {
168
c = '\t';
169
break;
170
}
171
if(c == 'u') {
172
if(i >= length) {
173
error('Unterminated String: ', str, row);
174
}
175
c = parseInt(this.substr(i + 1, 4), 16);
176
if(!isFinite(c) || c < 0) {
177
error('Unterminated String: ', str, row);
178
}
179
c = String.fromCharCode(c);
180
i+=4;
181
break;
182
}
183
}
184
str += c;
185
i++;
186
}
187
i++;
188
makeToken('string', str, from, i, row);
189
c = instring.charAt(i);
190
// Comment of // type ... does not cover block comments
191
} else if (c == '/' && instring.charAt(i+1) == '/') {
192
i++;
193
for(;;) {
194
c=instring.charAt(i);
195
if(c == '\n' || c == '\r' || c == '') {
196
row++;
197
break;
198
}
199
i++;
200
}
208
201
// Block comment of /* type
209
} else if (c == '/' && instring.charAt(i+1) == '*') {
210
i++;
211
for(;;) {
212
c=instring.charAt(i);
213
if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {
214
i+=2;
215
c=instring.charAt(i);
216
break;
217
}
218
if(c == '\n' || c == '\r' || c == '') {
219
row++;
220
}
221
i++;
222
}
223
// Multi-character Operators
224
} else if (inprefix.indexOf(c) >= 0) {
225
str = c;
226
i++;
227
while(true) {
228
c=instring.charAt(i);
229
if(i >= length || insuffix.indexOf(c) < 0) {
230
break;
231
}
232
str += c;
233
i++;
234
}
235
// Single-character Operators
236
maketoken('operator', str, from, i, row);
237
} else {
238
i++;
239
maketoken('operator', c, from, i, row);
240
c = instring.charAt(i);
241
}
242
}
243
}
244
function newButton()
245
{
246
var funclist;
247
var oplist;
248
oplist = document.getElementById('infobox');
249
string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
250
tokenize(string_tokenize, "<>+-&","=>&:");
251
// Iterate over token objects and print kind of each token and token type in window
252
printout = document.getElementById('infobox');
253
for(i = 0;i < tokens.length;i++) {
254
printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";
255
}
256
}
202
} else if (c == '/' && instring.charAt(i+1) == '*') {
203
i++;
204
for(;;) {
205
c=instring.charAt(i);
206
if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {
207
i+=2;
208
c=instring.charAt(i);
209
break;
210
}
211
if(c == '\n' || c == '\r' || c == '') {
212
row++;
213
}
214
i++;
215
}
216
// Multi-character Operators
217
} else if (inprefix.indexOf(c) >= 0) {
218
str = c;
219
i++;
220
while(true) {
221
c=instring.charAt(i);
222
if(i >= length || insuffix.indexOf(c) < 0) {
223
break;
224
}
225
str += c;
226
i++;
227
}
228
// Single-character Operators
229
makeToken('operator', str, from, i, row);
230
} else {
231
i++;
232
makeToken('operator', c, from, i, row);
233
c = instring.charAt(i);
234
}
235
}
236
}
237
238
function newButton() {
239
var funclist;
240
var oplist;
241
oplist = document.getElementById('infobox');
242
string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
243
tokenize(string_tokenize, "<>+-&","=>&:");
244
// Iterate over token objects and print kind of each token and token type in window
245
printout = document.getElementById('infobox');
246
for(i = 0;i < tokens.length;i++) {
247
printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";
248
}
249
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0