To get this branch, use:
bzr branch
http://gegoxaren.bato24.eu/bzr/lenasys/0.1
« back to all changes in this revision
Viewing changes to trunk/Code Tokenizer and Parser/Parser_Test.js
- browse files at revision 4.2.1
- compare with another revision
- download diff
- download tarball
- view history from revision 4.2.1
- Committer: mattman-03
- Date: 2013-03-28 14:58:37 UTC
- mfrom: (4.1.1 lenasys)
- mto: (9.2.2 lenasys)
- mto: This revision was merged to the branch mainline in revision 5.
- Revision ID: mattman-03-20130328145837-xeeerqyw02oeconp
Merged c11emian.
- files added:
- trunk/Code Tokenizer and Parser/Parser_Test.js
- files modified:
- trunk/Code Tokenizer and Parser/Parser_Test.html
added
removed
trunk/Code Tokenizer and Parser/Parser_Test.js
1
// Token class and storage definition
2
function token (kind,val,fromchar,tochar,row) {
3
this.kind = kind;
4
this.val = val;
5
this.from = fromchar;
6
this.to = tochar;
7
this.row = row;
8
}
9
10
var tokens = []; // Array to hold the tokens.
11
12
//----------------------------------------------------------
13
// Store token in tokens array
14
// Creates a new token object using the constructor
15
//----------------------------------------------------------
16
17
function maketoken(kind,val,from,to,rowno)
18
{
19
newtoken=new token(kind,val,from,to,rowno);
20
tokens.push(newtoken);
21
}
22
23
//----------------------------------------------------------
24
// Writes error from tokenizer
25
//----------------------------------------------------------
26
27
function error(str,val,row)
28
{
29
alert("Tokenizer Error: "+str+val+" at row "+row);
30
}
31
32
//----------------------------------------------------------
33
// Tokenize function
34
// Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
35
// The tokenizer is passed a string, and a string of prefix and suffix terminators
36
//----------------------------------------------------------
37
38
function tokenize(instring,inprefix,insuffix){
39
40
var from; // index of the start of the token.
41
var i = 0; // index of the current character.
42
var length=instring.length; // length of the string
43
44
var c; // current character.
45
var n; // current numerical value
46
var q; // current quote character
47
var str; // current string value.
48
var row=1; // current row value
49
50
51
c = instring.charAt(i);
52
while (c) {
53
54
from = i;
55
56
if (c <= ' '){ // White space
57
if((c=='\n')||(c=='\r')||(c == '')) row++; // Add row if this white space is a row terminator
58
i++;
59
c = instring.charAt(i);
60
}else if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')) { // Names i.e. Text
61
str = c;
62
i++;
63
for (;;) {
64
c = instring.charAt(i);
65
if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')||(c>='0'&&c<='9')||c=='_'){
66
str += c;
67
i++;
68
}else{
69
break;
70
}
71
}
72
maketoken('name',str,from,i,row);
73
} else if (c >= '0' && c <= '9') { // Number token
74
str = c;
75
i++;
76
for (;;) {
77
c = instring.charAt(i);
78
if (c < '0' || c > '9') break;
79
i++;
80
str+=c;
81
}
82
if(c=='.'){
83
i++;
84
str+=c;
85
for(;;){
86
c=instring.charAt(i);
87
if (c < '0' || c > '9') break;
88
i++;
89
str+=c;
90
}
91
}
92
if (c=='e'||c=='E') {
93
i++;
94
str+=c;
95
c=instring.charAt(i);
96
if(c=='-'||c=='+') {
97
i+=1;
98
str+=c;
99
c=instring.charAt(i);
100
}
101
if (c < '0' || c > '9') error('Bad Exponent in Number: ',str,row);
102
do {
103
i++;
104
str+=c;
105
c=instring.charAt(i);
106
}while(c>='0'&&c<='9');
107
}
108
if (c>='a'&&c<='z'){
109
str += c;
110
i += 1;
111
error('Bad Number: ',str,row);
112
}
113
n=+str;
114
if (isFinite(n)) {
115
maketoken('number',n,from,i,row);
116
} else {
117
error('Bad Number: ',str,row);
118
}
119
120
} else if(c=='\''||c=='"'){ // String .. handles c style breaking codes
121
str='';
122
q=c;
123
i++;
124
for (;;) {
125
c=instring.charAt(i);
126
127
if (c<' '){
128
if((c=='\n')||(c=='\r')||(c == '')) row++; // Add row if this white space is a row terminator
129
error('Unterminated String: ',str,row);
130
}
131
132
if (c==q) break;
133
134
if (c=='\\'){
135
i += 1;
136
if (i >= length) {
137
error('Unterminated String: ',str,row);
138
}
139
c=instring.charAt(i);
140
141
if(c=='b'){ c='\b'; break; }
142
if(c=='f'){ c='\f'; break; }
143
if(c=='n'){ c='\n'; break; }
144
if(c=='r'){ c='\r'; break; }
145
if(c=='t'){ c='\t'; break; }
146
if(c=='u'){
147
if (i >= length) {
148
error('Unterminated String: ',str,row);
149
}
150
c = parseInt(this.substr(i + 1, 4), 16);
151
if (!isFinite(c) || c < 0) {
152
error('Unterminated String: ',str,row);
153
}
154
c = String.fromCharCode(c);
155
i+=4;
156
break;
157
}
158
}
159
str += c;
160
i++;
161
}
162
i++;
163
maketoken('string',str,from,i,row);
164
c=instring.charAt(i);
165
166
} else if (c=='/'&&instring.charAt(i+1)=='/'){ // Comment of // type ... does not cover block comments
167
i++;
168
for (;;){
169
c=instring.charAt(i);
170
if (c=='\n'||c=='\r'||c=='') {
171
row++;
172
break;
173
}
174
i++;
175
}
176
} else if (c=='/'&&instring.charAt(i+1)=='*'){ // Block comment of /* type
177
i++;
178
for (;;){
179
c=instring.charAt(i);
180
if ((c=='*'&&instring.charAt(i+1)=='/')||(i==length)) {
181
i+=2;
182
c=instring.charAt(i);
183
break;
184
}
185
if (c=='\n'||c=='\r'||c=='') {
186
row++;
187
}
188
i++;
189
}
190
}else if(inprefix.indexOf(c) >= 0) { // Multi-character Operators
191
str = c;
192
i++;
193
while (true){
194
c=instring.charAt(i);
195
if (i >= length || insuffix.indexOf(c) < 0) {
196
break;
197
}
198
str += c;
199
i++;
200
}
201
maketoken('operator',str,from,i,row);
202
} else { // Single-character Operators
203
i++;
204
maketoken('operator',c,from,i,row);
205
c = instring.charAt(i);
206
}
207
208
}
209
}
210
211
function newbutton()
212
{
213
var funclist;
214
var oplist;
215
216
oplist=document.getElementById('infobox');
217
218
string_tokenize="foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
219
220
tokenize(string_tokenize,"<>+-&","=>&:");
221
222
223
// Iterate over token objects and print kind of each token and token type in window
224
printout=document.getElementById('infobox');
225
for(i=0;i<tokens.length;i++){
226
printout.innerHTML+=tokens[i].kind+" "+tokens[i].val+" ("+tokens[i].from+"-"+tokens[i].to+") at: "+tokens[i].row+"<br>";
227
}
228
229
}
Loggerhead is a web-based interface for Breezy
Version: 2.0.2.dev0