/lenasys/trunk

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/lenasys/trunk

« back to all changes in this revision

Viewing changes to Code Tokenizer and Parser/Parser_Test.js

  • Committer: Erik Wikström
  • Date: 2013-04-09 09:09:31 UTC
  • mfrom: (21 lenasys)
  • mto: (21.1.1 lenasys)
  • mto: This revision was merged to the branch mainline in revision 22.
  • Revision ID: wikxen@gmail.com-20130409090931-9jcs9kzg8et0912g
Remerged! Changed password hashes to VARCHAR(32)

Show diffs side-by-side

added added

removed removed

Lines of Context:
Code Tokenizer and Parser/Parser_Test.js
 
1
// Array to hold the tokens                             
 
2
var tokens = [];  
1
3
/* 
2
4
 * Token class and storage definition                                                                   
3
5
 */
4
 
 
5
6
function token(kind, val, fromchar, tochar, row) {
6
7
    this.kind = kind;
7
8
    this.val = val;
8
9
    this.from = fromchar;
9
10
    this.to = tochar;
10
11
    this.row = row;
11
 
}
12
 
 
13
 
// Array to hold the tokens                             
14
 
var tokens = [];            
 
12
}          
15
13
/*
16
14
 * Store token in tokens array
17
15
 * Creates a new token object using the constructor
18
16
 */
19
 
 
20
17
function makeToken(kind, val, from, to, rowno) {
21
18
        newtoken = new token(kind, val, from, to, rowno);
22
19
        tokens.push(newtoken);
23
20
}
24
 
 
25
21
/*
26
22
 * Writes error from tokenizer
27
23
 */             
28
 
 
29
24
function error(str, val, row) {
30
25
        alert("Tokenizer Error: " + str + val + " at row " + row);
31
26
}
32
 
 
33
27
/*
34
28
 * Tokenize function 
35
29
 * Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
36
30
 * The tokenizer is passed a string, and a string of prefix and suffix terminators
37
 
 */
38
 
                
 
31
 */             
39
32
function tokenize(instring, inprefix, insuffix) {           
40
33
        // index of the start of the token.
41
34
        var from;                       
77
70
                    break;
78
71
                }
79
72
            }
80
 
            maketoken('name', str, from, i, row);
 
73
            makeToken('name', str, from, i, row);
81
74
                        // Number token
82
 
                    } else if (c >= '0' && c <= '9') {                                                                                          
83
 
                        str = c;
84
 
                        i++;
85
 
                        for(;;) {
86
 
                                c = instring.charAt(i);
87
 
                            if(c < '0' || c > '9') {
88
 
                                                break;
89
 
                                        }
90
 
                            i++;
91
 
                            str += c;
92
 
                            }
93
 
                        if(c == '.') {
94
 
                            i++;
95
 
                            str += c;
96
 
                            for(;;){
97
 
                                        c=instring.charAt(i);
98
 
                                if(c < '0' || c > '9'){ 
99
 
                                                        break;
100
 
                                                }
101
 
                                i++;
102
 
                                str += c;
103
 
                                }
104
 
                            }
105
 
                        if(c == 'e' || c =='E') {
106
 
                            i++;
107
 
                            str += c;
108
 
                                c=instring.charAt(i);
109
 
                            if(c == '-' || c == '+') {
110
 
                                i++;
111
 
                                str += c;
112
 
                                        c=instring.charAt(i);
113
 
                            }
114
 
                            if(c < '0' || c > '9') {
115
 
                                                error('Bad Exponent in Number: ',str,row);
116
 
                                        }
117
 
                            do {
118
 
                                i++;
119
 
                                str += c;
120
 
                                        c = instring.charAt(i);
121
 
                                } while(c >= '0' && c <= '9');
122
 
                            }
123
 
                        if(c >= 'a' && c <= 'z') {
124
 
                                str += c;
125
 
                                i++;
126
 
                                error('Bad Number: ',str,row);
127
 
                            }
128
 
                        n =+ str;
129
 
                        if(isFinite(n)) {
130
 
                                        maketoken('number', n, from, i, row);                                   
131
 
                        } else {
132
 
                            error('Bad Number: ', str, row);
 
75
                } else if (c >= '0' && c <= '9') {                                                                                              
 
76
                    str = c;
 
77
                    i++;
 
78
                    for(;;) {
 
79
                                c = instring.charAt(i);
 
80
                        if(c < '0' || c > '9') {
 
81
                                break;
 
82
                                }
 
83
                        i++;
 
84
                        str += c;
 
85
                    }
 
86
                    if(c == '.') {
 
87
                        i++;
 
88
                        str += c;
 
89
                        for(;;){
 
90
                                c=instring.charAt(i);
 
91
                            if(c < '0' || c > '9'){ 
 
92
                                        break;
 
93
                                        }
 
94
                            i++;
 
95
                            str += c;
 
96
                            }
 
97
                    }
 
98
                    if(c == 'e' || c =='E') {
 
99
                        i++;
 
100
                        str += c;
 
101
                                c=instring.charAt(i);
 
102
                        if(c == '-' || c == '+') {
 
103
                            i++;
 
104
                            str += c;
 
105
                                        c=instring.charAt(i);
133
106
                        }
134
 
                                // String .. handles c style breaking codes
135
 
                        } else if (c == '\'' || c == '"') {                                                                                                                                                             
136
 
                            str = '';
137
 
                            q = c;
138
 
                            i++;
139
 
                            for(;;) {
140
 
                                    c = instring.charAt(i);
141
 
                                if(c < ' ') {
142
 
                                                        if((c == '\n') || (c == '\r') || (c == '')) {
143
 
                                                                // Add row if this white space is a row terminator
144
 
                                                                row++;
145
 
                                                        } else {
146
 
                                                                error('Unterminated String: ', str, row);                                               
147
 
                                                        }
148
 
                                }
149
 
                                if (c == q) {
150
 
                                                        break;
151
 
                                                }
152
 
                                if(c == '\\') {
153
 
                                    i++;
154
 
                                    if(i >= length) {
155
 
                                                error('Unterminated String: ', str, row);                                               
156
 
                                    }
157
 
                                                c = instring.charAt(i);
158
 
                                    if(c == 'b') { 
159
 
                                                                c = '\b'; 
160
 
                                                                break; 
161
 
                                                        }
162
 
                                    if(c == 'f') { 
163
 
                                                                c = '\f'; 
164
 
                                                                break; 
165
 
                                                        }
166
 
                                    if(c == 'n') { 
167
 
                                                                c = '\n'; 
168
 
                                                                break; 
169
 
                                                        }
170
 
                                    if(c == 'r') { 
171
 
                                                                c = '\r'; 
172
 
                                                                break; 
173
 
                                                        }
174
 
                                    if(c == 't') { 
175
 
                                                                c = '\t'; 
176
 
                                                                break; 
177
 
                                                        }
178
 
                                    if(c == 'u') {
179
 
                                        if(i >= length) {
180
 
                                                                error('Unterminated String: ', str, row);                                               
181
 
                                        }
182
 
                                        c = parseInt(this.substr(i + 1, 4), 16);
183
 
                                        if(!isFinite(c) || c < 0) {
184
 
                                                                error('Unterminated String: ', str, row);                                               
185
 
                                        }
186
 
                                        c = String.fromCharCode(c);
187
 
                                        i+=4;
188
 
                                        break;                              
189
 
                                    }
190
 
                                }
191
 
                                str += c;
192
 
                                i++;
193
 
                            }
194
 
                            i++;
195
 
                            maketoken('string', str, from, i, row);
196
 
                        c = instring.charAt(i);
197
 
                                // Comment of // type ... does not cover block comments
198
 
                        } else if (c == '/' && instring.charAt(i+1) == '/') {                                                           
199
 
                            i++;
200
 
                            for(;;) {
201
 
                                    c=instring.charAt(i);
202
 
                                if(c == '\n' || c == '\r' || c == '') {
203
 
                                    row++;
204
 
                                    break;
205
 
                                }
206
 
                                i++;
207
 
                            }
 
107
                        if(c < '0' || c > '9') {
 
108
                                        error('Bad Exponent in Number: ',str,row);
 
109
                                }
 
110
                        do {
 
111
                            i++;
 
112
                            str += c;
 
113
                                        c = instring.charAt(i);
 
114
                        } while(c >= '0' && c <= '9');
 
115
                    }
 
116
                    if(c >= 'a' && c <= 'z') {
 
117
                        str += c;
 
118
                        i++;
 
119
                        error('Bad Number: ',str,row);
 
120
                    }
 
121
                    n =+ str;
 
122
                    if(isFinite(n)) {
 
123
                                makeToken('number', n, from, i, row);                                   
 
124
                    } else {
 
125
                        error('Bad Number: ', str, row);
 
126
                    }
 
127
                        // String .. handles c style breaking codes
 
128
                } else if (c == '\'' || c == '"') {                                                                                                                                                             
 
129
                    str = '';
 
130
                    q = c;
 
131
                    i++;
 
132
                    for(;;) {
 
133
                                c = instring.charAt(i);
 
134
                                if(c < ' ') {
 
135
                                        if((c == '\n') || (c == '\r') || (c == '')) {
 
136
                                                // Add row if this white space is a row terminator
 
137
                                                row++;
 
138
                                        } else {
 
139
                                                error('Unterminated String: ', str, row);                                               
 
140
                                        }
 
141
                                }
 
142
                                if (c == q) {
 
143
                                        break;
 
144
                                }
 
145
                                if(c == '\\') {
 
146
                                        i++;
 
147
                                        if(i >= length) {
 
148
                                                error('Unterminated String: ', str, row);                                               
 
149
                                        }
 
150
                                        c = instring.charAt(i);
 
151
                                        if(c == 'b') { 
 
152
                                                c = '\b'; 
 
153
                                                break; 
 
154
                                        }
 
155
                                        if(c == 'f') { 
 
156
                                                c = '\f'; 
 
157
                                                break; 
 
158
                                        }
 
159
                                        if(c == 'n') { 
 
160
                                                c = '\n'; 
 
161
                                                break; 
 
162
                                        }
 
163
                                        if(c == 'r') { 
 
164
                                                c = '\r'; 
 
165
                                                break; 
 
166
                                        }
 
167
                                        if(c == 't') { 
 
168
                                                c = '\t'; 
 
169
                                                break; 
 
170
                                        }
 
171
                                        if(c == 'u') {
 
172
                                                if(i >= length) {
 
173
                                                        error('Unterminated String: ', str, row);                                               
 
174
                                                }
 
175
                                                c = parseInt(this.substr(i + 1, 4), 16);
 
176
                                                if(!isFinite(c) || c < 0) {
 
177
                                                        error('Unterminated String: ', str, row);                                               
 
178
                                                }
 
179
                                                c = String.fromCharCode(c);
 
180
                                                i+=4;
 
181
                                                break;                              
 
182
                                        }
 
183
                                }
 
184
                                str += c;
 
185
                                i++;
 
186
                    }
 
187
                    i++;
 
188
                    makeToken('string', str, from, i, row);
 
189
                c = instring.charAt(i);
 
190
                        // Comment of // type ... does not cover block comments
 
191
                } else if (c == '/' && instring.charAt(i+1) == '/') {                                                           
 
192
                    i++;
 
193
                    for(;;) {
 
194
                                c=instring.charAt(i);
 
195
                        if(c == '\n' || c == '\r' || c == '') {
 
196
                            row++;
 
197
                            break;
 
198
                                }
 
199
                        i++;
 
200
                    }
208
201
                                // Block comment of /* type
209
 
                        } else if (c == '/' && instring.charAt(i+1) == '*') {                                                           
210
 
                            i++;
211
 
                            for(;;) {
212
 
                                    c=instring.charAt(i);
213
 
                                if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {
214
 
                                    i+=2;
215
 
                                                c=instring.charAt(i);
216
 
                                    break;
217
 
                                }
218
 
                                if(c == '\n' || c == '\r' || c == '') {
219
 
                                    row++;
220
 
                                }
221
 
                                i++;
222
 
                            }
223
 
                                // Multi-character Operators
224
 
                        } else if (inprefix.indexOf(c) >= 0) {                                                                                  
225
 
                        str = c;
226
 
                        i++;
227
 
                        while(true) {
228
 
                                    c=instring.charAt(i);
229
 
                                if(i >= length || insuffix.indexOf(c) < 0) {
230
 
                                break;
231
 
                                }
232
 
                                str += c;
233
 
                                i++;
234
 
                        }
235
 
                                        // Single-character Operators
236
 
                        maketoken('operator', str, from, i, row);
237
 
                        } else {                                                                                                                                                                                                        
238
 
                        i++;
239
 
                        maketoken('operator', c, from, i, row);
240
 
                        c = instring.charAt(i);
241
 
                        }
242
 
                        }
243
 
                }
244
 
                function newButton()
245
 
                {
246
 
                        var funclist;
247
 
                        var oplist;
248
 
                        oplist = document.getElementById('infobox');
249
 
                        string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
250
 
                        tokenize(string_tokenize, "<>+-&","=>&:");                              
251
 
                        // Iterate over token objects and print kind of each token and token type in window 
252
 
                        printout = document.getElementById('infobox');
253
 
                        for(i = 0;i < tokens.length;i++) {
254
 
                                printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";
255
 
                        }       
256
 
                }               
 
202
                } else if (c == '/' && instring.charAt(i+1) == '*') {                                                           
 
203
                    i++;
 
204
                    for(;;) {
 
205
                                c=instring.charAt(i);
 
206
                        if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {
 
207
                            i+=2;
 
208
                                        c=instring.charAt(i);
 
209
                            break;
 
210
                                }
 
211
                        if(c == '\n' || c == '\r' || c == '') {
 
212
                            row++;
 
213
                                }
 
214
                        i++;
 
215
                    }
 
216
                        // Multi-character Operators
 
217
                } else if (inprefix.indexOf(c) >= 0) {                                                                                  
 
218
                        str = c;
 
219
            i++;
 
220
            while(true) {
 
221
                        c=instring.charAt(i);
 
222
                if(i >= length || insuffix.indexOf(c) < 0) {
 
223
                        break;
 
224
                }
 
225
                str += c;
 
226
                i++;
 
227
            }
 
228
                        // Single-character Operators
 
229
                makeToken('operator', str, from, i, row);
 
230
        } else {                                                                                                                                                                                                        
 
231
                i++;
 
232
                makeToken('operator', c, from, i, row);
 
233
                c = instring.charAt(i);
 
234
        }
 
235
        }
 
236
}
 
237
 
 
238
function newButton() {
 
239
        var funclist;
 
240
        var oplist;
 
241
        oplist = document.getElementById('infobox');
 
242
        string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
 
243
        tokenize(string_tokenize, "<>+-&","=>&:");                              
 
244
        // Iterate over token objects and print kind of each token and token type in window 
 
245
        printout = document.getElementById('infobox');
 
246
        for(i = 0;i < tokens.length;i++) {
 
247
                printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";
 
248
        }       
 
249
}