/lenasys/trunk

// Array to hold the tokens				

var tokens = [];  

/* 

 * Token class and storage definition									

 */

function token(kind, val, fromchar, tochar, row) {

    this.kind = kind;

    this.val = val;

    this.from = fromchar;

    this.to = tochar;

    this.row = row;

}          

/*

 * Store token in tokens array

 * Creates a new token object using the constructor

 */

function makeToken(kind, val, from, to, rowno) {

	newtoken = new token(kind, val, from, to, rowno);

	tokens.push(newtoken);

}

/*

 * Writes error from tokenizer

 */		

function error(str, val, row) {

	alert("Tokenizer Error: " + str + val + " at row " + row);

}

/*

 * Tokenize function 

 * Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford

 * The tokenizer is passed a string, and a string of prefix and suffix terminators

 */		

function tokenize(instring, inprefix, insuffix) {	    

	// index of the start of the token.

	var from;                   	

	// index of the current character.

	var i = 0;                  	

	// length of the string

	var length = instring.length;		

    // current character.

	var c;                      	

    // current numerical value

	var n;                      	

    // current quote character

	var q;                      	

    // current string value.

	var str;                    	

	// current row value		

	var row=1;										

	c = instring.charAt(i);

	while(c) {			

        from = i;

		// White space

        if(c <= ' ') {

			// Add row if this white space is a row terminator		

        	if((c == '\n') || (c == '\r') || (c == '')) {

				row++;

			}										 																						

		    i++;

    		c = instring.charAt(i);

			// Names i.e. Text

        } else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {					

			str = c;

            i++;

            for(;;) {

               	c = instring.charAt(i);

               	if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')|| (c == '_')) {

                    str += c;

                    i++;

               	} else {

                    break;

                }

            }

            makeToken('name', str, from, i, row);

			// Number token

		} else if (c >= '0' && c <= '9') {												

		    str = c;

		    i++;

		    for(;;) {

				c = instring.charAt(i);

		        if(c < '0' || c > '9') {

				break;

				}

		        i++;

		        str += c;

		    }

		    if(c == '.') {

		        i++;

		        str += c;

		        for(;;){

			   	c=instring.charAt(i);

		            if(c < '0' || c > '9'){ 

					break;

					}

		            i++;

		            str += c;

		            }

		    }

		    if(c == 'e' || c =='E') {

		        i++;

		        str += c;

				c=instring.charAt(i);

		        if(c == '-' || c == '+') {

		            i++;

		            str += c;

					c=instring.charAt(i);

		        }

		        if(c < '0' || c > '9') {

					error('Bad Exponent in Number: ',str,row);

				}

		        do {

		            i++;

		            str += c;

				  	c = instring.charAt(i);

		        } while(c >= '0' && c <= '9');

		    }

		    if(c >= 'a' && c <= 'z') {

		        str += c;

		        i++;

		        error('Bad Number: ',str,row);

		    }

		    n =+ str;

		    if(isFinite(n)) {

				makeToken('number', n, from, i, row);		            		

		    } else {

		        error('Bad Number: ', str, row);

		    }

			// String .. handles c style breaking codes

		} else if (c == '\'' || c == '"') {																				

		    str = '';

		    q = c;

		    i++;

		    for(;;) {

				c = instring.charAt(i);

				if(c < ' ') {

					if((c == '\n') || (c == '\r') || (c == '')) {

						// Add row if this white space is a row terminator

						row++;

					} else {

						error('Unterminated String: ', str, row);		                		

					}

				}

				if (c == q) {

					break;

				}

				if(c == '\\') {

					i++;

					if(i >= length) {

						error('Unterminated String: ', str, row);		                		

					}

					c = instring.charAt(i);

					if(c == 'b') { 

						c = '\b'; 

						break; 

					}

					if(c == 'f') { 

						c = '\f'; 

						break; 

					}

					if(c == 'n') { 

						c = '\n'; 

						break; 

					}

					if(c == 'r') { 

						c = '\r'; 

						break; 

					}

					if(c == 't') { 

						c = '\t'; 

						break; 

					}

					if(c == 'u') {

						if(i >= length) {

							error('Unterminated String: ', str, row);		                		

						}

						c = parseInt(this.substr(i + 1, 4), 16);

						if(!isFinite(c) || c < 0) {

							error('Unterminated String: ', str, row);		                		

						}

						c = String.fromCharCode(c);

						i+=4;

						break;		                    

					}

				}

				str += c;

				i++;

		    }

		    i++;

		    makeToken('string', str, from, i, row);

    		c = instring.charAt(i);

			// Comment of // type ... does not cover block comments

		} else if (c == '/' && instring.charAt(i+1) == '/') {								

		    i++;

		    for(;;) {

				c=instring.charAt(i);

		        if(c == '\n' || c == '\r' || c == '') {

		            row++;

		            break;

				}

		        i++;

		    }

				// Block comment of /* type

		} else if (c == '/' && instring.charAt(i+1) == '*') {								

		    i++;

		    for(;;) {

				c=instring.charAt(i);

		        if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {

		            i+=2;

					c=instring.charAt(i);

		            break;

				}

		        if(c == '\n' || c == '\r' || c == '') {

		            row++;

				}

		        i++;

		    }

			// Multi-character Operators

		} else if (inprefix.indexOf(c) >= 0) {											

			str = c;

            i++;

            while(true) {

		        c=instring.charAt(i);

               	if(i >= length || insuffix.indexOf(c) < 0) {

               		break;

               	}

               	str += c;

               	i++;

            }

			// Single-character Operators

           	makeToken('operator', str, from, i, row);

        } else {																									

           	i++;

           	makeToken('operator', c, from, i, row);

           	c = instring.charAt(i);

        }

	}

}

function newButton() {

	var funclist;

	var oplist;

	oplist = document.getElementById('infobox');

	string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";

	tokenize(string_tokenize, "<>+-&","=>&:");				

	// Iterate over token objects and print kind of each token and token type in window 

	printout = document.getElementById('infobox');

	for(i = 0;i < tokens.length;i++) {

		printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";

	}	

}