/lenasys/0.1

To get this branch, use:
bzr branch http://gegoxaren.bato24.eu/bzr/lenasys/0.1

« back to all changes in this revision

Viewing changes to trunk/Code Tokenizer and Parser/Parser_Test.html

  • Committer: Henrik G.
  • Date: 2013-03-26 23:22:55 UTC
  • Revision ID: henrik.gustavsson@his.se-20130326232255-ik6snyatlbkf3zs1
First seed of Lenasys ... Needs to be Organized Further

Show diffs side-by-side

added added

removed removed

Lines of Context:
trunk/Code Tokenizer and Parser/Parser_Test.html
 
1
<html>
 
2
        <body onload="newbutton();">
 
3
        
 
4
        <table>
 
5
                <tr>
 
6
                        <td valign="top">
 
7
                                <div style="border:2px solid black;background-color:#fed;width:500;height:1200;">
 
8
                                        <div id="infobox" style="padding:4px;">
 
9
                                        </div>
 
10
                                </div>
 
11
                        </td>                   
 
12
                </tr>
 
13
        </table>        
 
14
        
 
15
        </div>
 
16
 
 
17
        <script lang='Javascript'>
 
18
                
 
19
                // Token class and storage definition                                                                   
 
20
                function token (kind,val,fromchar,tochar,row) {
 
21
                    this.kind = kind;
 
22
                    this.val = val;
 
23
                    this.from = fromchar;
 
24
                    this.to = tochar;
 
25
                    this.row = row;
 
26
                }
 
27
                                
 
28
    var tokens = [];            // Array to hold the tokens.
 
29
 
 
30
                //----------------------------------------------------------
 
31
                // Store token in tokens array
 
32
                // Creates a new token object using the constructor
 
33
          //----------------------------------------------------------                                          
 
34
 
 
35
                function maketoken(kind,val,from,to,rowno)
 
36
                {
 
37
                                newtoken=new token(kind,val,from,to,rowno);
 
38
                                tokens.push(newtoken);
 
39
                }
 
40
 
 
41
                //----------------------------------------------------------
 
42
                // Writes error from tokenizer
 
43
          //----------------------------------------------------------                                          
 
44
                
 
45
                function error(str,val,row)
 
46
                {
 
47
                                alert("Tokenizer Error: "+str+val+" at row "+row);
 
48
                }
 
49
 
 
50
                //----------------------------------------------------------
 
51
                // Tokenize function 
 
52
                // Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
 
53
                // The tokenizer is passed a string, and a string of prefix and suffix terminators
 
54
          //----------------------------------------------------------                                          
 
55
                
 
56
                function tokenize(instring,inprefix,insuffix){
 
57
                    
 
58
                    var from;                           // index of the start of the token.
 
59
                    var i = 0;                          // index of the current character.
 
60
                    var length=instring.length;         // length of the string
 
61
 
 
62
                    var c;                              // current character.
 
63
                    var n;                              // current numerical value
 
64
                    var q;                              // current quote character
 
65
                    var str;                            // current string value.
 
66
                                var row=1;                                                                              // current row value
 
67
                                
 
68
 
 
69
                    c = instring.charAt(i);
 
70
                while (c) {
 
71
                                
 
72
                        from = i;
 
73
 
 
74
                        if (c <= ' '){                                                                                                                                                                          // White space
 
75
                                        if((c=='\n')||(c=='\r')||(c == '')) row++;                                              // Add row if this white space is a row terminator                                                                                                                                                                                                              
 
76
                            i++;
 
77
                        c = instring.charAt(i);
 
78
                        }else if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')) {                                        // Names i.e. Text
 
79
                        str = c;
 
80
                        i++;
 
81
                        for (;;) {
 
82
                                c = instring.charAt(i);
 
83
                                if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')||(c>='0'&&c<='9')||c=='_'){
 
84
                                str += c;
 
85
                                i++;
 
86
                                }else{
 
87
                                break;
 
88
                                }
 
89
                        }
 
90
                        maketoken('name',str,from,i,row);
 
91
                        } else if (c >= '0' && c <= '9') {                                                                                              // Number token
 
92
                            str = c;
 
93
                            i++;
 
94
                            for (;;) {
 
95
                                        c = instring.charAt(i);
 
96
                                if (c < '0' || c > '9') break;
 
97
                                i++;
 
98
                                str+=c;
 
99
                            }
 
100
                            if(c=='.'){
 
101
                                i++;
 
102
                                str+=c;
 
103
                                for(;;){
 
104
                                                        c=instring.charAt(i);
 
105
                                    if (c < '0' || c > '9') break;
 
106
                                    i++;
 
107
                                    str+=c;
 
108
                                }
 
109
                            }
 
110
                            if (c=='e'||c=='E') {
 
111
                                i++;
 
112
                                str+=c;
 
113
                                        c=instring.charAt(i);
 
114
                                if(c=='-'||c=='+') {
 
115
                                    i+=1;
 
116
                                    str+=c;
 
117
                                                        c=instring.charAt(i);
 
118
                                }
 
119
                                if (c < '0' || c > '9') error('Bad Exponent in Number: ',str,row);
 
120
                                do {
 
121
                                    i++;
 
122
                                    str+=c;
 
123
                                                        c=instring.charAt(i);
 
124
                                }while(c>='0'&&c<='9');
 
125
                            }
 
126
                            if (c>='a'&&c<='z'){
 
127
                                str += c;
 
128
                                i += 1;
 
129
                                error('Bad Number: ',str,row);
 
130
                            }
 
131
                            n=+str;
 
132
                            if (isFinite(n)) {
 
133
                                                                                maketoken('number',n,from,i,row);                                       
 
134
                            } else {
 
135
                                error('Bad Number: ',str,row);
 
136
                            }
 
137
 
 
138
                        } else if(c=='\''||c=='"'){                                                                                                                                                             // String .. handles c style breaking codes
 
139
                            str='';
 
140
                            q=c;
 
141
                            i++;
 
142
                            for (;;) {
 
143
                                        c=instring.charAt(i);
 
144
 
 
145
                                if (c<' '){
 
146
                                                if((c=='\n')||(c=='\r')||(c == '')) row++;                                              // Add row if this white space is a row terminator                                                                                                                                                                                                              
 
147
                                                error('Unterminated String: ',str,row);                                         
 
148
                                }
 
149
 
 
150
                                if (c==q) break;
 
151
 
 
152
                                if (c=='\\'){
 
153
                                    i += 1;
 
154
                                    if (i >= length) {
 
155
                                                                error('Unterminated String: ',str,row);                                         
 
156
                                    }
 
157
                                                        c=instring.charAt(i);
 
158
                                    
 
159
                                    if(c=='b'){ c='\b'; break; }
 
160
                                    if(c=='f'){ c='\f'; break; }
 
161
                                    if(c=='n'){ c='\n'; break; }
 
162
                                    if(c=='r'){ c='\r'; break; }
 
163
                                    if(c=='t'){ c='\t'; break; }
 
164
                                    if(c=='u'){
 
165
                                        if (i >= length) {
 
166
                                                                                error('Unterminated String: ',str,row);                                         
 
167
                                        }
 
168
                                        c = parseInt(this.substr(i + 1, 4), 16);
 
169
                                        if (!isFinite(c) || c < 0) {
 
170
                                                                                error('Unterminated String: ',str,row);                                         
 
171
                                        }
 
172
                                        c = String.fromCharCode(c);
 
173
                                        i+=4;
 
174
                                        break;                              
 
175
                                    }
 
176
                                }
 
177
                                str += c;
 
178
                                i++;
 
179
                            }
 
180
                            i++;
 
181
                            maketoken('string',str,from,i,row);
 
182
                        c=instring.charAt(i);
 
183
 
 
184
                        } else if (c=='/'&&instring.charAt(i+1)=='/'){                                                          // Comment of // type ... does not cover block comments
 
185
                            i++;
 
186
                            for (;;){
 
187
                                        c=instring.charAt(i);
 
188
                                if (c=='\n'||c=='\r'||c=='') {
 
189
                                    row++;
 
190
                                    break;
 
191
                                }
 
192
                                i++;
 
193
                            }
 
194
                        } else if (c=='/'&&instring.charAt(i+1)=='*'){                                                          // Block comment of /* type
 
195
                            i++;
 
196
                            for (;;){
 
197
                                        c=instring.charAt(i);
 
198
                                if ((c=='*'&&instring.charAt(i+1)=='/')||(i==length)) {
 
199
                                    i+=2;
 
200
                                                        c=instring.charAt(i);
 
201
                                    break;
 
202
                                }
 
203
                                if (c=='\n'||c=='\r'||c=='') {
 
204
                                    row++;
 
205
                                }
 
206
                                i++;
 
207
                            }
 
208
                        }else if(inprefix.indexOf(c) >= 0) {                                                                                    // Multi-character Operators
 
209
                        str = c;
 
210
                        i++;
 
211
                        while (true){
 
212
                                        c=instring.charAt(i);
 
213
                                if (i >= length || insuffix.indexOf(c) < 0) {
 
214
                                break;
 
215
                                }
 
216
                                str += c;
 
217
                                i++;
 
218
                        }
 
219
                        maketoken('operator',str,from,i,row);
 
220
                        } else {                                                                                                                                                                                                        // Single-character Operators
 
221
                        i++;
 
222
                        maketoken('operator',c,from,i,row);
 
223
                        c = instring.charAt(i);
 
224
                        }
 
225
                                                
 
226
                                }
 
227
                }
 
228
                
 
229
                function newbutton()
 
230
                {
 
231
                                var funclist;
 
232
                                var oplist;
 
233
                                
 
234
                                oplist=document.getElementById('infobox');
 
235
                                                        
 
236
                                string_tokenize="foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
 
237
 
 
238
                                tokenize(string_tokenize,"<>+-&","=>&:");
 
239
 
 
240
                                
 
241
                                // Iterate over token objects and print kind of each token and token type in window 
 
242
                                printout=document.getElementById('infobox');
 
243
                                for(i=0;i<tokens.length;i++){
 
244
                                                printout.innerHTML+=tokens[i].kind+" "+tokens[i].val+" ("+tokens[i].from+"-"+tokens[i].to+") at: "+tokens[i].row+"<br>";
 
245
                                }
 
246
                                
 
247
                }               
 
248
                
 
249
</script>
 
250
 
 
251
</body>
 
252
</html>
 
253