/lenasys/0.1

/* 

 * Token class and storage definition									

 */

function token(kind, val, fromchar, tochar, row) {

    this.kind = kind;

    this.val = val;

    this.from = fromchar;

    this.to = tochar;

    this.row = row;

}

// Array to hold the tokens				

var tokens = [];            

/*

 * Store token in tokens array

 * Creates a new token object using the constructor

 */

function maketoken(kind, val, from, to, rowno) {

	newtoken = new token(kind, val, from, to, rowno);

	tokens.push(newtoken);

}

/*

 * Writes error from tokenizer

 */		

function error(str, val, row) {

	alert("Tokenizer Error: " + str + val + " at row " + row);

}

/*

 * Tokenize function 

 * Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford

 * The tokenizer is passed a string, and a string of prefix and suffix terminators

 */

function tokenize(instring, inprefix, insuffix) {	    

	// index of the start of the token.

	var from;                   	

	// index of the current character.

	var i = 0;                  	

	// length of the string

	var length = instring.length;		

    // current character.

	var c;                      	

    // current numerical value

	var n;                      	

    // current quote character

	var q;                      	

    // current string value.

	var str;                    	

	// current row value		

	var row=1;										

	c = instring.charAt(i);

	while(c) {			

        from = i;

		// White space

        if(c <= ' ') {

			// Add row if this white space is a row terminator		

        	if((c == '\n') || (c == '\r') || (c == '')) {

				row++;

			}										 																						

		    i++;

    		c = instring.charAt(i);

			// Names i.e. Text

        } else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {					

			str = c;

            i++;

            for(;;) {

               	c = instring.charAt(i);

               	if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')|| (c == '_')) {

                    str += c;

                    i++;

               	} else {

                    break;

                }

            }

            maketoken('name', str, from, i, row);

			// Number token

		    } else if (c >= '0' && c <= '9') {												

		        str = c;

		        i++;

		        for(;;) {

		    		c = instring.charAt(i);

		            if(c < '0' || c > '9') {

						break;

					}

		            i++;

		            str += c;

		            }

		        if(c == '.') {

		            i++;

		            str += c;

		            for(;;){

				    	c=instring.charAt(i);

		                if(c < '0' || c > '9'){ 

							break;

						}

		                i++;

		                str += c;

		                }

		            }

		        if(c == 'e' || c =='E') {

		            i++;

		            str += c;

		    		c=instring.charAt(i);

		            if(c == '-' || c == '+') {

		                i++;

		                str += c;

				    	c=instring.charAt(i);

		            }

		            if(c < '0' || c > '9') {

						error('Bad Exponent in Number: ',str,row);

					}

		            do {

		                i++;

		                str += c;

				    	c = instring.charAt(i);

		                } while(c >= '0' && c <= '9');

		            }

		        if(c >= 'a' && c <= 'z') {

		                str += c;

		                i++;

		                error('Bad Number: ',str,row);

		            }

		        n =+ str;

		        if(isFinite(n)) {

 					maketoken('number', n, from, i, row);		            		

		        } else {

		            error('Bad Number: ', str, row);

		        }

		        } else if (c == '\'' || c == '"') {																				// String .. handles c style breaking codes

		            str = '';

		            q = c;

		            i++;

		            for(;;) {

		    		    c = instring.charAt(i);

		                if(c < ' ') {

							if((c == '\n') || (c == '\r') || (c == '')) {

								// Add row if this white space is a row terminator

								row++;

							} else {

								error('Unterminated String: ', str, row);		                		

							}

		                }

		                if (c == q) {

							break;

						}

		                if(c == '\\') {

		                    i++;

		                    if(i >= length) {

				                error('Unterminated String: ', str, row);		                		

		                    }

				    		c = instring.charAt(i);

		                    if(c == 'b') { 

								c = '\b'; 

								break; 

							}

		                    if(c == 'f') { 

								c = '\f'; 

								break; 

							}

		                    if(c == 'n') { 

								c = '\n'; 

								break; 

							}

		                    if(c == 'r') { 

								c = '\r'; 

								break; 

							}

		                    if(c == 't') { 

								c = '\t'; 

								break; 

							}

		                    if(c == 'u') {

		                        if(i >= length) {

						        	error('Unterminated String: ', str, row);		                		

		                        }

		                        c = parseInt(this.substr(i + 1, 4), 16);

		                        if(!isFinite(c) || c < 0) {

						         	error('Unterminated String: ', str, row);		                		

		                        }

		                        c = String.fromCharCode(c);

		                        i+=4;

		                        break;		                    

		                    }

		                }

		                str += c;

		                i++;

		            }

		            i++;

		            maketoken('string', str, from, i, row);

    		        c = instring.charAt(i);

				// Comment of // type ... does not cover block comments

		        } else if (c == '/' && instring.charAt(i+1) == '/') {								

		            i++;

		            for(;;) {

		    		    c=instring.charAt(i);

		                if(c == '\n' || c == '\r' || c == '') {

		                    row++;

		                    break;

		                }

		                i++;

		            }

				// Block comment of /* type

		        } else if (c == '/' && instring.charAt(i+1) == '*') {								

		            i++;

		            for(;;) {

		    		    c=instring.charAt(i);

		                if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {

		                    i+=2;

				    		c=instring.charAt(i);

		                    break;

		                }

		                if(c == '\n' || c == '\r' || c == '') {

		                    row++;

		                }

		                i++;

		            }

				// Multi-character Operators

        		} else if (inprefix.indexOf(c) >= 0) {											

            		str = c;

            		i++;

            		while(true) {

		    		    c=instring.charAt(i);

                		if(i >= length || insuffix.indexOf(c) < 0) {

                    		break;

                		}

                		str += c;

                		i++;

            		}

					// Single-character Operators

            		maketoken('operator', str, from, i, row);

        		} else {																									

            		i++;

            		maketoken('operator', c, from, i, row);

            		c = instring.charAt(i);

        		}

			}

		}

		function newbutton()

		{

			var funclist;

			var oplist;

			oplist = document.getElementById('infobox');

			string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";

			tokenize(string_tokenize, "<>+-&","=>&:");				

			// Iterate over token objects and print kind of each token and token type in window 

			printout = document.getElementById('infobox');

			for(i = 0;i < tokens.length;i++) {

				printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";

			}	

		}