/lenasys/0.1

		// Token class and storage definition									

		function token (kind,val,fromchar,tochar,row) {

		    this.kind = kind;

		    this.val = val;

		    this.from = fromchar;

		    this.to = tochar;

		    this.row = row;

		}

    var tokens = [];            // Array to hold the tokens.

		//----------------------------------------------------------

		// Store token in tokens array

		// Creates a new token object using the constructor

	  //----------------------------------------------------------						

		function maketoken(kind,val,from,to,rowno)

		{

				newtoken=new token(kind,val,from,to,rowno);

				tokens.push(newtoken);

		}

		//----------------------------------------------------------

		// Writes error from tokenizer

	  //----------------------------------------------------------						

		function error(str,val,row)

		{

				alert("Tokenizer Error: "+str+val+" at row "+row);

		}

		//----------------------------------------------------------

		// Tokenize function 

		// Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford

		// The tokenizer is passed a string, and a string of prefix and suffix terminators

	  //----------------------------------------------------------						

		function tokenize(instring,inprefix,insuffix){

		    var from;                   	// index of the start of the token.

		    var i = 0;                  	// index of the current character.

		    var length=instring.length;		// length of the string

		    var c;                      	// current character.

		    var n;                      	// current numerical value

		    var q;                      	// current quote character

		    var str;                    	// current string value.

				var row=1;										// current row value

		    c = instring.charAt(i);

    		while (c) {

        		from = i;

        		if (c <= ' '){																						// White space

        				if((c=='\n')||(c=='\r')||(c == '')) row++;						// Add row if this white space is a row terminator				 																						

		            i++;

    		        c = instring.charAt(i);

        		}else if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')) {					// Names i.e. Text

            		str = c;

            		i++;

            		for (;;) {

                		c = instring.charAt(i);

                		if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')||(c>='0'&&c<='9')||c=='_'){

                    		str += c;

                    		i++;

                		}else{

                    		break;

                		}

            		}

            		maketoken('name',str,from,i,row);

		        } else if (c >= '0' && c <= '9') {												// Number token

		            str = c;

		            i++;

		            for (;;) {

		    		        c = instring.charAt(i);

		                if (c < '0' || c > '9') break;

		                i++;

		                str+=c;

		            }

		            if(c=='.'){

		                i++;

		                str+=c;

		                for(;;){

				    		        c=instring.charAt(i);

		                    if (c < '0' || c > '9') break;

		                    i++;

		                    str+=c;

		                }

		            }

		            if (c=='e'||c=='E') {

		                i++;

		                str+=c;

		    		        c=instring.charAt(i);

		                if(c=='-'||c=='+') {

		                    i+=1;

		                    str+=c;

				    		        c=instring.charAt(i);

		                }

		                if (c < '0' || c > '9') error('Bad Exponent in Number: ',str,row);

		                do {

		                    i++;

		                    str+=c;

				    		        c=instring.charAt(i);

		                }while(c>='0'&&c<='9');

		            }

		            if (c>='a'&&c<='z'){

		                str += c;

		                i += 1;

		                error('Bad Number: ',str,row);

		            }

		            n=+str;

		            if (isFinite(n)) {

 										maketoken('number',n,from,i,row);		            		

		            } else {

		                error('Bad Number: ',str,row);

		            }

		        } else if(c=='\''||c=='"'){																				// String .. handles c style breaking codes

		            str='';

		            q=c;

		            i++;

		            for (;;) {

		    		        c=instring.charAt(i);

		                if (c<' '){

       	        				if((c=='\n')||(c=='\r')||(c == '')) row++;						// Add row if this white space is a row terminator				 																						

		                		error('Unterminated String: ',str,row);		                		

		                }

		                if (c==q) break;

		                if (c=='\\'){

		                    i += 1;

		                    if (i >= length) {

				                		error('Unterminated String: ',str,row);		                		

		                    }

				    		        c=instring.charAt(i);

		                    if(c=='b'){ c='\b'; break; }

		                    if(c=='f'){ c='\f'; break; }

		                    if(c=='n'){ c='\n'; break; }

		                    if(c=='r'){ c='\r'; break; }

		                    if(c=='t'){ c='\t'; break; }

		                    if(c=='u'){

		                        if (i >= length) {

						                		error('Unterminated String: ',str,row);		                		

		                        }

		                        c = parseInt(this.substr(i + 1, 4), 16);

		                        if (!isFinite(c) || c < 0) {

						                		error('Unterminated String: ',str,row);		                		

		                        }

		                        c = String.fromCharCode(c);

		                        i+=4;

		                        break;		                    

		                    }

		                }

		                str += c;

		                i++;

		            }

		            i++;

		            maketoken('string',str,from,i,row);

    		        c=instring.charAt(i);

		        } else if (c=='/'&&instring.charAt(i+1)=='/'){								// Comment of // type ... does not cover block comments

		            i++;

		            for (;;){

		    		        c=instring.charAt(i);

		                if (c=='\n'||c=='\r'||c=='') {

		                    row++;

		                    break;

		                }

		                i++;

		            }

		        } else if (c=='/'&&instring.charAt(i+1)=='*'){								// Block comment of /* type

		            i++;

		            for (;;){

		    		        c=instring.charAt(i);

		                if ((c=='*'&&instring.charAt(i+1)=='/')||(i==length)) {

		                    i+=2;

				    		        c=instring.charAt(i);

		                    break;

		                }

		                if (c=='\n'||c=='\r'||c=='') {

		                    row++;

		                }

		                i++;

		            }

        		}else if(inprefix.indexOf(c) >= 0) {											// Multi-character Operators

            		str = c;

            		i++;

            		while (true){

		    		        c=instring.charAt(i);

                		if (i >= length || insuffix.indexOf(c) < 0) {

                    		break;

                		}

                		str += c;

                		i++;

            		}

            		maketoken('operator',str,from,i,row);

        		} else {																									// Single-character Operators

            		i++;

            		maketoken('operator',c,from,i,row);

            		c = instring.charAt(i);

        		}

				}

		}

		function newbutton()

		{

				var funclist;

				var oplist;

				oplist=document.getElementById('infobox');

				string_tokenize="foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";

				tokenize(string_tokenize,"<>+-&","=>&:");

				// Iterate over token objects and print kind of each token and token type in window 

				printout=document.getElementById('infobox');

				for(i=0;i<tokens.length;i++){

						printout.innerHTML+=tokens[i].kind+" "+tokens[i].val+" ("+tokens[i].from+"-"+tokens[i].to+") at: "+tokens[i].row+"<br>";

				}

		}