To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/lenasys/0.1
« back to all changes in this revision
Viewing changes to trunk/Code Tokenizer and Parser/Parser_Test.js
- browse files at revision 4.2.5
- compare with another revision
- download diff
- download tarball
- view history from revision 4.2.5
- Committer: mattman-03
- Date: 2013-04-02 07:21:23 UTC
- mfrom: (4.1.3 lenasys)
- mto: (9.2.2 lenasys)
- mto: This revision was merged to the branch mainline in revision 5.
- Revision ID: mattman-03-20130402072123-p01939ufqlcksh2f
c11emian klar med städa
- files modified:
- trunk/Code Tokenizer and Parser/Parser_Test.js
added
removed
trunk/Code Tokenizer and Parser/Parser_Test.js
1
// Token class and storage definition
2
function token (kind,val,fromchar,tochar,row) {
3
this.kind = kind;
4
this.val = val;
5
this.from = fromchar;
6
this.to = tochar;
7
this.row = row;
8
}
9
10
var tokens = []; // Array to hold the tokens.
11
12
//----------------------------------------------------------
13
// Store token in tokens array
14
// Creates a new token object using the constructor
15
//----------------------------------------------------------
16
17
function maketoken(kind,val,from,to,rowno)
18
{
19
newtoken=new token(kind,val,from,to,rowno);
20
tokens.push(newtoken);
21
}
22
23
//----------------------------------------------------------
24
// Writes error from tokenizer
25
//----------------------------------------------------------
26
27
function error(str,val,row)
28
{
29
alert("Tokenizer Error: "+str+val+" at row "+row);
30
}
31
32
//----------------------------------------------------------
33
// Tokenize function
34
// Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
35
// The tokenizer is passed a string, and a string of prefix and suffix terminators
36
//----------------------------------------------------------
37
38
function tokenize(instring,inprefix,insuffix){
39
40
var from; // index of the start of the token.
41
var i = 0; // index of the current character.
42
var length=instring.length; // length of the string
43
44
var c; // current character.
45
var n; // current numerical value
46
var q; // current quote character
47
var str; // current string value.
48
var row=1; // current row value
49
50
51
c = instring.charAt(i);
52
while (c) {
53
54
from = i;
55
56
if (c <= ' '){ // White space
57
if((c=='\n')||(c=='\r')||(c == '')) row++; // Add row if this white space is a row terminator
58
i++;
59
c = instring.charAt(i);
60
}else if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')) { // Names i.e. Text
61
str = c;
62
i++;
63
for (;;) {
64
c = instring.charAt(i);
65
if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')||(c>='0'&&c<='9')||c=='_'){
66
str += c;
67
i++;
68
}else{
69
break;
70
}
71
}
72
maketoken('name',str,from,i,row);
73
} else if (c >= '0' && c <= '9') { // Number token
74
str = c;
75
i++;
76
for (;;) {
77
c = instring.charAt(i);
78
if (c < '0' || c > '9') break;
79
i++;
80
str+=c;
81
}
82
if(c=='.'){
83
i++;
84
str+=c;
85
for(;;){
86
c=instring.charAt(i);
87
if (c < '0' || c > '9') break;
88
i++;
89
str+=c;
90
}
91
}
92
if (c=='e'||c=='E') {
93
i++;
94
str+=c;
95
c=instring.charAt(i);
96
if(c=='-'||c=='+') {
97
i+=1;
98
str+=c;
99
c=instring.charAt(i);
100
}
101
if (c < '0' || c > '9') error('Bad Exponent in Number: ',str,row);
102
do {
103
i++;
104
str+=c;
105
c=instring.charAt(i);
106
}while(c>='0'&&c<='9');
107
}
108
if (c>='a'&&c<='z'){
109
str += c;
110
i += 1;
111
error('Bad Number: ',str,row);
112
}
113
n=+str;
114
if (isFinite(n)) {
115
maketoken('number',n,from,i,row);
116
} else {
117
error('Bad Number: ',str,row);
118
}
119
120
} else if(c=='\''||c=='"'){ // String .. handles c style breaking codes
121
str='';
122
q=c;
123
i++;
124
for (;;) {
125
c=instring.charAt(i);
126
127
if (c<' '){
128
if((c=='\n')||(c=='\r')||(c == '')) row++; // Add row if this white space is a row terminator
129
error('Unterminated String: ',str,row);
130
}
131
132
if (c==q) break;
133
134
if (c=='\\'){
135
i += 1;
136
if (i >= length) {
137
error('Unterminated String: ',str,row);
1
/*
2
* Token class and storage definition
3
*/
4
5
function token(kind, val, fromchar, tochar, row) {
6
this.kind = kind;
7
this.val = val;
8
this.from = fromchar;
9
this.to = tochar;
10
this.row = row;
11
}
12
13
// Array to hold the tokens
14
var tokens = [];
15
/*
16
* Store token in tokens array
17
* Creates a new token object using the constructor
18
*/
19
20
function maketoken(kind, val, from, to, rowno) {
21
newtoken = new token(kind, val, from, to, rowno);
22
tokens.push(newtoken);
23
}
24
25
/*
26
* Writes error from tokenizer
27
*/
28
29
function error(str, val, row) {
30
alert("Tokenizer Error: " + str + val + " at row " + row);
31
}
32
33
/*
34
* Tokenize function
35
* Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
36
* The tokenizer is passed a string, and a string of prefix and suffix terminators
37
*/
38
39
function tokenize(instring, inprefix, insuffix) {
40
// index of the start of the token.
41
var from;
42
// index of the current character.
43
var i = 0;
44
// length of the string
45
var length = instring.length;
46
// current character.
47
var c;
48
// current numerical value
49
var n;
50
// current quote character
51
var q;
52
// current string value.
53
var str;
54
// current row value
55
var row=1;
56
c = instring.charAt(i);
57
while(c) {
58
from = i;
59
// White space
60
if(c <= ' ') {
61
// Add row if this white space is a row terminator
62
if((c == '\n') || (c == '\r') || (c == '')) {
63
row++;
64
}
65
i++;
66
c = instring.charAt(i);
67
// Names i.e. Text
68
} else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {
69
str = c;
70
i++;
71
for(;;) {
72
c = instring.charAt(i);
73
if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')|| (c == '_')) {
74
str += c;
75
i++;
76
} else {
77
break;
78
}
79
}
80
maketoken('name', str, from, i, row);
81
// Number token
82
} else if (c >= '0' && c <= '9') {
83
str = c;
84
i++;
85
for(;;) {
86
c = instring.charAt(i);
87
if(c < '0' || c > '9') {
88
break;
89
}
90
i++;
91
str += c;
92
}
93
if(c == '.') {
94
i++;
95
str += c;
96
for(;;){
97
c=instring.charAt(i);
98
if(c < '0' || c > '9'){
99
break;
100
}
101
i++;
102
str += c;
103
}
104
}
105
if(c == 'e' || c =='E') {
106
i++;
107
str += c;
108
c=instring.charAt(i);
109
if(c == '-' || c == '+') {
110
i++;
111
str += c;
112
c=instring.charAt(i);
113
}
114
if(c < '0' || c > '9') {
115
error('Bad Exponent in Number: ',str,row);
116
}
117
do {
118
i++;
119
str += c;
120
c = instring.charAt(i);
121
} while(c >= '0' && c <= '9');
122
}
123
if(c >= 'a' && c <= 'z') {
124
str += c;
125
i++;
126
error('Bad Number: ',str,row);
127
}
128
n =+ str;
129
if(isFinite(n)) {
130
maketoken('number', n, from, i, row);
131
} else {
132
error('Bad Number: ', str, row);
133
}
134
} else if (c == '\'' || c == '"') { // String .. handles c style breaking codes
135
str = '';
136
q = c;
137
i++;
138
for(;;) {
139
c = instring.charAt(i);
140
if(c < ' ') {
141
if((c == '\n') || (c == '\r') || (c == '')) {
142
// Add row if this white space is a row terminator
143
row++;
144
} else {
145
error('Unterminated String: ', str, row);
146
}
147
}
148
if (c == q) {
149
break;
150
}
151
if(c == '\\') {
152
i++;
153
if(i >= length) {
154
error('Unterminated String: ', str, row);
138
155
}
139
c=instring.charAt(i);
140
141
if(c=='b'){ c='\b'; break; }
142
if(c=='f'){ c='\f'; break; }
143
if(c=='n'){ c='\n'; break; }
144
if(c=='r'){ c='\r'; break; }
145
if(c=='t'){ c='\t'; break; }
146
if(c=='u'){
147
if (i >= length) {
148
error('Unterminated String: ',str,row);
156
c = instring.charAt(i);
157
if(c == 'b') {
158
c = '\b';
159
break;
160
}
161
if(c == 'f') {
162
c = '\f';
163
break;
164
}
165
if(c == 'n') {
166
c = '\n';
167
break;
168
}
169
if(c == 'r') {
170
c = '\r';
171
break;
172
}
173
if(c == 't') {
174
c = '\t';
175
break;
176
}
177
if(c == 'u') {
178
if(i >= length) {
179
error('Unterminated String: ', str, row);
149
180
}
150
181
c = parseInt(this.substr(i + 1, 4), 16);
151
if (!isFinite(c) || c < 0) {
152
error('Unterminated String: ',str,row);
182
if(!isFinite(c) || c < 0) {
183
error('Unterminated String: ', str, row);
153
184
}
154
185
c = String.fromCharCode(c);
155
186
i+=4;
160
191
i++;
161
192
}
162
193
i++;
163
maketoken('string',str,from,i,row);
164
c=instring.charAt(i);
165
166
} else if (c=='/'&&instring.charAt(i+1)=='/'){ // Comment of // type ... does not cover block comments
194
maketoken('string', str, from, i, row);
195
c = instring.charAt(i);
196
// Comment of // type ... does not cover block comments
197
} else if (c == '/' && instring.charAt(i+1) == '/') {
167
198
i++;
168
for (;;){
169
c=instring.charAt(i);
170
if (c=='\n'||c=='\r'||c=='') {
199
for(;;) {
200
c=instring.charAt(i);
201
if(c == '\n' || c == '\r' || c == '') {
171
202
row++;
172
203
break;
173
204
}
174
205
i++;
175
206
}
176
} else if (c=='/'&&instring.charAt(i+1)=='*'){ // Block comment of /* type
207
// Block comment of /* type
208
} else if (c == '/' && instring.charAt(i+1) == '*') {
177
209
i++;
178
for (;;){
179
c=instring.charAt(i);
180
if ((c=='*'&&instring.charAt(i+1)=='/')||(i==length)) {
210
for(;;) {
211
c=instring.charAt(i);
212
if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {
181
213
i+=2;
182
c=instring.charAt(i);
214
c=instring.charAt(i);
183
215
break;
184
216
}
185
if (c=='\n'||c=='\r'||c=='') {
217
if(c == '\n' || c == '\r' || c == '') {
186
218
row++;
187
219
}
188
220
i++;
189
221
}
190
}else if(inprefix.indexOf(c) >= 0) { // Multi-character Operators
222
// Multi-character Operators
223
} else if (inprefix.indexOf(c) >= 0) {
191
224
str = c;
192
225
i++;
193
while (true){
194
c=instring.charAt(i);
195
if (i >= length || insuffix.indexOf(c) < 0) {
226
while(true) {
227
c=instring.charAt(i);
228
if(i >= length || insuffix.indexOf(c) < 0) {
196
229
break;
197
230
}
198
231
str += c;
199
232
i++;
200
233
}
201
maketoken('operator',str,from,i,row);
202
} else { // Single-character Operators
234
// Single-character Operators
235
maketoken('operator', str, from, i, row);
236
} else {
203
237
i++;
204
maketoken('operator',c,from,i,row);
238
maketoken('operator', c, from, i, row);
205
239
c = instring.charAt(i);
206
240
}
207
208
}
241
}
209
242
}
210
211
243
function newbutton()
212
244
{
213
var funclist;
214
var oplist;
215
216
oplist=document.getElementById('infobox');
217
218
string_tokenize="foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
219
220
tokenize(string_tokenize,"<>+-&","=>&:");
221
222
223
// Iterate over token objects and print kind of each token and token type in window
224
printout=document.getElementById('infobox');
225
for(i=0;i<tokens.length;i++){
226
printout.innerHTML+=tokens[i].kind+" "+tokens[i].val+" ("+tokens[i].from+"-"+tokens[i].to+") at: "+tokens[i].row+"<br>";
227
}
228
245
var funclist;
246
var oplist;
247
oplist = document.getElementById('infobox');
248
string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
249
tokenize(string_tokenize, "<>+-&","=>&:");
250
// Iterate over token objects and print kind of each token and token type in window
251
printout = document.getElementById('infobox');
252
for(i = 0;i < tokens.length;i++) {
253
printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";
254
}
229
255
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0