/lenasys/0.1 : contents of trunk/Code Tokenizer and Parser/Parser

: (revision 2)

To get this branch, use:

bzr branch
http://gegoxaren.bato24.eu/bzr/lenasys/0.1

<html>
	<body onload="newbutton();">
	
	<table>
		<tr>
			<td valign="top">
				<div style="border:2px solid black;background-color:#fed;width:500;height:1200;">
					<div id="infobox" style="padding:4px;">
					</div>
				</div>
			</td>			
		</tr>
	</table>	
	
	</div>

	<script lang='Javascript'>
		
		// Token class and storage definition									
		function token (kind,val,fromchar,tochar,row) {
		    this.kind = kind;
		    this.val = val;
		    this.from = fromchar;
		    this.to = tochar;
		    this.row = row;
		}
				
    var tokens = [];            // Array to hold the tokens.

		//----------------------------------------------------------
		// Store token in tokens array
		// Creates a new token object using the constructor
	  //----------------------------------------------------------						

		function maketoken(kind,val,from,to,rowno)
		{
				newtoken=new token(kind,val,from,to,rowno);
				tokens.push(newtoken);
		}

		//----------------------------------------------------------
		// Writes error from tokenizer
	  //----------------------------------------------------------						
		
		function error(str,val,row)
		{
				alert("Tokenizer Error: "+str+val+" at row "+row);
		}

		//----------------------------------------------------------
		// Tokenize function 
		// Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
		// The tokenizer is passed a string, and a string of prefix and suffix terminators
	  //----------------------------------------------------------						
		
		function tokenize(instring,inprefix,insuffix){
		    
		    var from;                   	// index of the start of the token.
		    var i = 0;                  	// index of the current character.
		    var length=instring.length;		// length of the string

		    var c;                      	// current character.
		    var n;                      	// current numerical value
		    var q;                      	// current quote character
		    var str;                    	// current string value.
				var row=1;										// current row value
				

		    c = instring.charAt(i);
    		while (c) {
 				
        		from = i;

        		if (c <= ' '){																						// White space
        				if((c=='\n')||(c=='\r')||(c == '')) row++;						// Add row if this white space is a row terminator				 																						
		            i++;
    		        c = instring.charAt(i);
        		}else if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')) {					// Names i.e. Text
            		str = c;
            		i++;
            		for (;;) {
                		c = instring.charAt(i);
                		if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')||(c>='0'&&c<='9')||c=='_'){
                    		str += c;
                    		i++;
                		}else{
                    		break;
                		}
            		}
            		maketoken('name',str,from,i,row);
		        } else if (c >= '0' && c <= '9') {												// Number token
		            str = c;
		            i++;
		            for (;;) {
		    		        c = instring.charAt(i);
		                if (c < '0' || c > '9') break;
		                i++;
		                str+=c;
		            }
		            if(c=='.'){
		                i++;
		                str+=c;
		                for(;;){
				    		        c=instring.charAt(i);
		                    if (c < '0' || c > '9') break;
		                    i++;
		                    str+=c;
		                }
		            }
		            if (c=='e'||c=='E') {
		                i++;
		                str+=c;
		    		        c=instring.charAt(i);
		                if(c=='-'||c=='+') {
		                    i+=1;
		                    str+=c;
				    		        c=instring.charAt(i);
		                }
		                if (c < '0' || c > '9') error('Bad Exponent in Number: ',str,row);
		                do {
		                    i++;
		                    str+=c;
				    		        c=instring.charAt(i);
		                }while(c>='0'&&c<='9');
		            }
		            if (c>='a'&&c<='z'){
		                str += c;
		                i += 1;
		                error('Bad Number: ',str,row);
		            }
		            n=+str;
		            if (isFinite(n)) {
 										maketoken('number',n,from,i,row);		            		
		            } else {
		                error('Bad Number: ',str,row);
		            }

		        } else if(c=='\''||c=='"'){																				// String .. handles c style breaking codes
		            str='';
		            q=c;
		            i++;
		            for (;;) {
		    		        c=instring.charAt(i);

		                if (c<' '){
       	        				if((c=='\n')||(c=='\r')||(c == '')) row++;						// Add row if this white space is a row terminator				 																						
		                		error('Unterminated String: ',str,row);		                		
		                }

		                if (c==q) break;

		                if (c=='\\'){
		                    i += 1;
		                    if (i >= length) {
				                		error('Unterminated String: ',str,row);		                		
		                    }
				    		        c=instring.charAt(i);
		                    
		                    if(c=='b'){ c='\b'; break; }
		                    if(c=='f'){ c='\f'; break; }
		                    if(c=='n'){ c='\n'; break; }
		                    if(c=='r'){ c='\r'; break; }
		                    if(c=='t'){ c='\t'; break; }
		                    if(c=='u'){
		                        if (i >= length) {
						                		error('Unterminated String: ',str,row);		                		
		                        }
		                        c = parseInt(this.substr(i + 1, 4), 16);
		                        if (!isFinite(c) || c < 0) {
						                		error('Unterminated String: ',str,row);		                		
		                        }
		                        c = String.fromCharCode(c);
		                        i+=4;
		                        break;		                    
		                    }
		                }
		                str += c;
		                i++;
		            }
		            i++;
		            maketoken('string',str,from,i,row);
    		        c=instring.charAt(i);

		        } else if (c=='/'&&instring.charAt(i+1)=='/'){								// Comment of // type ... does not cover block comments
		            i++;
		            for (;;){
		    		        c=instring.charAt(i);
		                if (c=='\n'||c=='\r'||c=='') {
		                    row++;
		                    break;
		                }
		                i++;
		            }
		        } else if (c=='/'&&instring.charAt(i+1)=='*'){								// Block comment of /* type
		            i++;
		            for (;;){
		    		        c=instring.charAt(i);
		                if ((c=='*'&&instring.charAt(i+1)=='/')||(i==length)) {
		                    i+=2;
				    		        c=instring.charAt(i);
		                    break;
		                }
		                if (c=='\n'||c=='\r'||c=='') {
		                    row++;
		                }
		                i++;
		            }
        		}else if(inprefix.indexOf(c) >= 0) {											// Multi-character Operators
            		str = c;
            		i++;
            		while (true){
		    		        c=instring.charAt(i);
                		if (i >= length || insuffix.indexOf(c) < 0) {
                    		break;
                		}
                		str += c;
                		i++;
            		}
            		maketoken('operator',str,from,i,row);
        		} else {																									// Single-character Operators
            		i++;
            		maketoken('operator',c,from,i,row);
            		c = instring.charAt(i);
        		}
						
				}
		}
		
		function newbutton()
		{
				var funclist;
				var oplist;
				
				oplist=document.getElementById('infobox');
							
				string_tokenize="foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";

				tokenize(string_tokenize,"<>+-&","=>&:");

				
				// Iterate over token objects and print kind of each token and token type in window 
				printout=document.getElementById('infobox');
				for(i=0;i<tokens.length;i++){
						printout.innerHTML+=tokens[i].kind+" "+tokens[i].val+" ("+tokens[i].from+"-"+tokens[i].to+") at: "+tokens[i].row+"<br>";
				}
				
		}		
		
</script>

</body>
</html>


1 by Henrik G. First seed of Lenasys ... Needs to be Organized Further	1	<html>
	2	<body onload="newbutton();">
	3
	4	<table>
	5	<tr>
	6	<td valign="top">
	7	<div style="border:2px solid black;background-color:#fed;width:500;height:1200;">
	8	<div id="infobox" style="padding:4px;">
	9	</div>
	10	</div>
	11	</td>
	12	</tr>
	13	</table>
	14
	15	</div>
	16
	17	<script lang='Javascript'>
	18
	19	// Token class and storage definition
	20	function token (kind,val,fromchar,tochar,row) {
	21	this.kind = kind;
	22	this.val = val;
	23	this.from = fromchar;
	24	this.to = tochar;
	25	this.row = row;
	26	}
	27
	28	var tokens = []; // Array to hold the tokens.
	29
	30	//----------------------------------------------------------
	31	// Store token in tokens array
	32	// Creates a new token object using the constructor
	33	//----------------------------------------------------------
	34
	35	function maketoken(kind,val,from,to,rowno)
	36	{
	37	newtoken=new token(kind,val,from,to,rowno);
	38	tokens.push(newtoken);
	39	}
	40
	41	//----------------------------------------------------------
	42	// Writes error from tokenizer
	43	//----------------------------------------------------------
	44
	45	function error(str,val,row)
	46	{
	47	alert("Tokenizer Error: "+str+val+" at row "+row);
	48	}
	49
	50	//----------------------------------------------------------
	51	// Tokenize function
	52	// Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
	53	// The tokenizer is passed a string, and a string of prefix and suffix terminators
	54	//----------------------------------------------------------
	55
	56	function tokenize(instring,inprefix,insuffix){
	57
	58	var from; // index of the start of the token.
	59	var i = 0; // index of the current character.
	60	var length=instring.length; // length of the string
	61
	62	var c; // current character.
	63	var n; // current numerical value
	64	var q; // current quote character
65	var str; // current string value.
66	var row=1; // current row value
67
68
69	c = instring.charAt(i);
70	while (c) {
71
72	from = i;
73
74	if (c <= ' '){ // White space
75	if((c=='\n')\|\|(c=='\r')\|\|(c == '')) row++; // Add row if this white space is a row terminator
76	i++;
77	c = instring.charAt(i);
78	}else if ((c >='a'&&c<='z')\|\|(c>='A'&&c<='Z')) { // Names i.e. Text
79	str = c;
80	i++;
81	for (;;) {
82	c = instring.charAt(i);
83	if ((c >='a'&&c<='z')\|\|(c>='A'&&c<='Z')\|\|(c>='0'&&c<='9')\|\|c=='_'){
84	str += c;
85	i++;
86	}else{
87	break;
88	}
89	}
90	maketoken('name',str,from,i,row);
91	} else if (c >= '0' && c <= '9') { // Number token
92	str = c;
93	i++;
94	for (;;) {
95	c = instring.charAt(i);
96	if (c < '0' \|\| c > '9') break;
97	i++;
98	str+=c;
99	}
100	if(c=='.'){
101	i++;
102	str+=c;
103	for(;;){
104	c=instring.charAt(i);
105	if (c < '0' \|\| c > '9') break;
106	i++;
107	str+=c;
108	}
109	}
110	if (c=='e'\|\|c=='E') {
111	i++;
112	str+=c;
113	c=instring.charAt(i);
114	if(c=='-'\|\|c=='+') {
115	i+=1;
116	str+=c;
117	c=instring.charAt(i);
118	}
119	if (c < '0' \|\| c > '9') error('Bad Exponent in Number: ',str,row);
120	do {
121	i++;
122	str+=c;
123	c=instring.charAt(i);
124	}while(c>='0'&&c<='9');
125	}
126	if (c>='a'&&c<='z'){
127	str += c;
128	i += 1;
129	error('Bad Number: ',str,row);
130	}
131	n=+str;
132	if (isFinite(n)) {
133	maketoken('number',n,from,i,row);
134	} else {
135	error('Bad Number: ',str,row);
136	}
137
138	} else if(c=='\''\|\|c=='"'){ // String .. handles c style breaking codes
139	str='';
140	q=c;
141	i++;
142	for (;;) {
143	c=instring.charAt(i);
144
145	if (c<' '){
146	if((c=='\n')\|\|(c=='\r')\|\|(c == '')) row++; // Add row if this white space is a row terminator
147	error('Unterminated String: ',str,row);
148	}
149
150	if (c==q) break;
151
152	if (c=='\\'){
153	i += 1;
154	if (i >= length) {
155	error('Unterminated String: ',str,row);
156	}
157	c=instring.charAt(i);
158
159	if(c=='b'){ c='\b'; break; }
160	if(c=='f'){ c='\f'; break; }
161	if(c=='n'){ c='\n'; break; }
162	if(c=='r'){ c='\r'; break; }
163	if(c=='t'){ c='\t'; break; }
164	if(c=='u'){
165	if (i >= length) {
166	error('Unterminated String: ',str,row);
167	}
168	c = parseInt(this.substr(i + 1, 4), 16);
169	if (!isFinite(c) \|\| c < 0) {
170	error('Unterminated String: ',str,row);
171	}
172	c = String.fromCharCode(c);
173	i+=4;
174	break;
175	}
176	}
177	str += c;
178	i++;
179	}
180	i++;
181	maketoken('string',str,from,i,row);
182	c=instring.charAt(i);
183
184	} else if (c=='/'&&instring.charAt(i+1)=='/'){ // Comment of // type ... does not cover block comments
185	i++;
186	for (;;){
187	c=instring.charAt(i);
188	if (c=='\n'\|\|c=='\r'\|\|c=='') {
189	row++;
190	break;
191	}
192	i++;
193	}
194	} else if (c=='/'&&instring.charAt(i+1)==''){ // Block comment of / type
195	i++;
196	for (;;){
197	c=instring.charAt(i);
198	if ((c=='*'&&instring.charAt(i+1)=='/')\|\|(i==length)) {
199	i+=2;
200	c=instring.charAt(i);
201	break;
202	}
203	if (c=='\n'\|\|c=='\r'\|\|c=='') {
204	row++;
205	}
206	i++;
207	}
208	}else if(inprefix.indexOf(c) >= 0) { // Multi-character Operators
209	str = c;
210	i++;
211	while (true){
212	c=instring.charAt(i);
213	if (i >= length \|\| insuffix.indexOf(c) < 0) {
214	break;
215	}
216	str += c;
217	i++;
218	}
219	maketoken('operator',str,from,i,row);
220	} else { // Single-character Operators
221	i++;
222	maketoken('operator',c,from,i,row);
223	c = instring.charAt(i);
224	}
225
226	}
227	}
228
229	function newbutton()
230	{
231	var funclist;
232	var oplist;
233
234	oplist=document.getElementById('infobox');
235
236	string_tokenize="foo // Lederhosen\nsin'foo'(200.0r)+(-21.4/1.51e-6)+/Plodder/(Feeeb+400)ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
237
238	tokenize(string_tokenize,"<>+-&","=>&:");
239
240
241	// Iterate over token objects and print kind of each token and token type in window
242	printout=document.getElementById('infobox');
243	for(i=0;i<tokens.length;i++){
244	printout.innerHTML+=tokens[i].kind+" "+tokens[i].val+" ("+tokens[i].from+"-"+tokens[i].to+") at: "+tokens[i].row+"<br>";
245	}
246
247	}
248
249	</script>
250
251	</body>
252	</html>
253