To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/lenasys/0.1
« back to all changes in this revision
Viewing changes to trunk/Code Tokenizer and Parser/Parser_Test.js
- browse files at revision 5
- compare with another revision
- download diff
- download tarball
- view history from revision 5
- Committer: gustav.hartvigsson at gmail
- Date: 2013-04-02 12:13:01 UTC
- mfrom: (4.2.7 hitlerhorabajs)
- Revision ID: gustav.hartvigsson@gmail.com-20130402121301-ytfzuo7y6cku9s3o
Merge from implemenation group 2:s branch.
- files added:
- trunk/Code Tokenizer and Parser/Parser_Test.js
- trunk/Code Viewer Alt/css
- files removed:
- trunk/Code Viewer Alt/.DS_Store
- files modified:
- trunk/Code Tokenizer and Parser/Parser_Test.html
added
removed
trunk/Code Tokenizer and Parser/Parser_Test.js
1
/*
2
* Token class and storage definition
3
*/
4
5
function token(kind, val, fromchar, tochar, row) {
6
this.kind = kind;
7
this.val = val;
8
this.from = fromchar;
9
this.to = tochar;
10
this.row = row;
11
}
12
13
// Array to hold the tokens
14
var tokens = [];
15
/*
16
* Store token in tokens array
17
* Creates a new token object using the constructor
18
*/
19
20
function maketoken(kind, val, from, to, rowno) {
21
newtoken = new token(kind, val, from, to, rowno);
22
tokens.push(newtoken);
23
}
24
25
/*
26
* Writes error from tokenizer
27
*/
28
29
function error(str, val, row) {
30
alert("Tokenizer Error: " + str + val + " at row " + row);
31
}
32
33
/*
34
* Tokenize function
35
* Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
36
* The tokenizer is passed a string, and a string of prefix and suffix terminators
37
*/
38
39
function tokenize(instring, inprefix, insuffix) {
40
// index of the start of the token.
41
var from;
42
// index of the current character.
43
var i = 0;
44
// length of the string
45
var length = instring.length;
46
// current character.
47
var c;
48
// current numerical value
49
var n;
50
// current quote character
51
var q;
52
// current string value.
53
var str;
54
// current row value
55
var row=1;
56
c = instring.charAt(i);
57
while(c) {
58
from = i;
59
// White space
60
if(c <= ' ') {
61
// Add row if this white space is a row terminator
62
if((c == '\n') || (c == '\r') || (c == '')) {
63
row++;
64
}
65
i++;
66
c = instring.charAt(i);
67
// Names i.e. Text
68
} else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
69
str = c;
70
i++;
71
for(;;) {
72
c = instring.charAt(i);
73
if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')|| (c == '_')) {
74
str += c;
75
i++;
76
} else {
77
break;
78
}
79
}
80
maketoken('name', str, from, i, row);
81
// Number token
82
} else if (c >= '0' && c <= '9') {
83
str = c;
84
i++;
85
for(;;) {
86
c = instring.charAt(i);
87
if(c < '0' || c > '9') {
88
break;
89
}
90
i++;
91
str += c;
92
}
93
if(c == '.') {
94
i++;
95
str += c;
96
for(;;){
97
c=instring.charAt(i);
98
if(c < '0' || c > '9'){
99
break;
100
}
101
i++;
102
str += c;
103
}
104
}
105
if(c == 'e' || c =='E') {
106
i++;
107
str += c;
108
c=instring.charAt(i);
109
if(c == '-' || c == '+') {
110
i++;
111
str += c;
112
c=instring.charAt(i);
113
}
114
if(c < '0' || c > '9') {
115
error('Bad Exponent in Number: ',str,row);
116
}
117
do {
118
i++;
119
str += c;
120
c = instring.charAt(i);
121
} while(c >= '0' && c <= '9');
122
}
123
if(c >= 'a' && c <= 'z') {
124
str += c;
125
i++;
126
error('Bad Number: ',str,row);
127
}
128
n =+ str;
129
if(isFinite(n)) {
130
maketoken('number', n, from, i, row);
131
} else {
132
error('Bad Number: ', str, row);
133
}
134
} else if (c == '\'' || c == '"') { // String .. handles c style breaking codes
135
str = '';
136
q = c;
137
i++;
138
for(;;) {
139
c = instring.charAt(i);
140
if(c < ' ') {
141
if((c == '\n') || (c == '\r') || (c == '')) {
142
// Add row if this white space is a row terminator
143
row++;
144
} else {
145
error('Unterminated String: ', str, row);
146
}
147
}
148
if (c == q) {
149
break;
150
}
151
if(c == '\\') {
152
i++;
153
if(i >= length) {
154
error('Unterminated String: ', str, row);
155
}
156
c = instring.charAt(i);
157
if(c == 'b') {
158
c = '\b';
159
break;
160
}
161
if(c == 'f') {
162
c = '\f';
163
break;
164
}
165
if(c == 'n') {
166
c = '\n';
167
break;
168
}
169
if(c == 'r') {
170
c = '\r';
171
break;
172
}
173
if(c == 't') {
174
c = '\t';
175
break;
176
}
177
if(c == 'u') {
178
if(i >= length) {
179
error('Unterminated String: ', str, row);
180
}
181
c = parseInt(this.substr(i + 1, 4), 16);
182
if(!isFinite(c) || c < 0) {
183
error('Unterminated String: ', str, row);
184
}
185
c = String.fromCharCode(c);
186
i+=4;
187
break;
188
}
189
}
190
str += c;
191
i++;
192
}
193
i++;
194
maketoken('string', str, from, i, row);
195
c = instring.charAt(i);
196
// Comment of // type ... does not cover block comments
197
} else if (c == '/' && instring.charAt(i+1) == '/') {
198
i++;
199
for(;;) {
200
c=instring.charAt(i);
201
if(c == '\n' || c == '\r' || c == '') {
202
row++;
203
break;
204
}
205
i++;
206
}
207
// Block comment of /* type
208
} else if (c == '/' && instring.charAt(i+1) == '*') {
209
i++;
210
for(;;) {
211
c=instring.charAt(i);
212
if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {
213
i+=2;
214
c=instring.charAt(i);
215
break;
216
}
217
if(c == '\n' || c == '\r' || c == '') {
218
row++;
219
}
220
i++;
221
}
222
// Multi-character Operators
223
} else if (inprefix.indexOf(c) >= 0) {
224
str = c;
225
i++;
226
while(true) {
227
c=instring.charAt(i);
228
if(i >= length || insuffix.indexOf(c) < 0) {
229
break;
230
}
231
str += c;
232
i++;
233
}
234
// Single-character Operators
235
maketoken('operator', str, from, i, row);
236
} else {
237
i++;
238
maketoken('operator', c, from, i, row);
239
c = instring.charAt(i);
240
}
241
}
242
}
243
function newbutton()
244
{
245
var funclist;
246
var oplist;
247
oplist = document.getElementById('infobox');
248
string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
249
tokenize(string_tokenize, "<>+-&","=>&:");
250
// Iterate over token objects and print kind of each token and token type in window
251
printout = document.getElementById('infobox');
252
for(i = 0;i < tokens.length;i++) {
253
printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";
254
}
255
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0