/lenasys/trunk

/* 

 * Token class and storage definition                                                                   

 */

 

function token(kind, val, fromchar, tochar, row) {

    this.kind = kind;

    this.val = val;

    this.from = fromchar;

    this.to = tochar;

    this.row = row;

}

 

// Array to hold the tokens                             

var tokens = [];            

/*

 * Store token in tokens array

 * Creates a new token object using the constructor

 */

 

function makeToken(kind, val, from, to, rowno) {

        newtoken = new token(kind, val, from, to, rowno);

        tokens.push(newtoken);

}

 

/*

 * Writes error from tokenizer

 */             

 

function error(str, val, row) {

        alert("Tokenizer Error: " + str + val + " at row " + row);

}

 

/*

 * Tokenize function 

 * Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford

 * The tokenizer is passed a string, and a string of prefix and suffix terminators

 */

                

function tokenize(instring, inprefix, insuffix) {           

        // index of the start of the token.

        var from;                       

        // index of the current character.

        var i = 0;                      

        // length of the string

        var length = instring.length;           

    // current character.

        var c;                          

    // current numerical value

        var n;                          

    // current quote character

        var q;                          

    // current string value.

        var str;                        

        // current row value            

        var row=1;                                                                              

        c = instring.charAt(i);

        while(c) {                      

        from = i;

                // White space

        if(c <= ' ') {

                        // Add row if this white space is a row terminator              

                if((c == '\n') || (c == '\r') || (c == '')) {

                                row++;

                        }                                                                                                                                                                                                                                                               

                    i++;

                c = instring.charAt(i);

                        // Names i.e. Text

        } else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {                                  

                        str = c;

            i++;

            for(;;) {

                c = instring.charAt(i);

                if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')|| (c == '_')) {

                    str += c;

                    i++;

                } else {

                    break;

                }

            }

            maketoken('name', str, from, i, row);

                        // Number token

                    } else if (c >= '0' && c <= '9') {                                                                                          

                        str = c;

                        i++;

                        for(;;) {

                                c = instring.charAt(i);

                            if(c < '0' || c > '9') {

                                                break;

                                        }

                            i++;

                            str += c;

                            }

                        if(c == '.') {

                            i++;

                            str += c;

                            for(;;){

                                        c=instring.charAt(i);

                                if(c < '0' || c > '9'){ 

                                                        break;

                                                }

                                i++;

                                str += c;

                                }

                            }

                        if(c == 'e' || c =='E') {

                            i++;

                            str += c;

                                c=instring.charAt(i);

                            if(c == '-' || c == '+') {

                                i++;

                                str += c;

                                        c=instring.charAt(i);

                            }

                            if(c < '0' || c > '9') {

                                                error('Bad Exponent in Number: ',str,row);

                                        }

                            do {

                                i++;

                                str += c;

                                        c = instring.charAt(i);

                                } while(c >= '0' && c <= '9');

                            }

                        if(c >= 'a' && c <= 'z') {

                                str += c;

                                i++;

                                error('Bad Number: ',str,row);

                            }

                        n =+ str;

                        if(isFinite(n)) {

                                        maketoken('number', n, from, i, row);                                   

                        } else {

                            error('Bad Number: ', str, row);

                        }

                                // String .. handles c style breaking codes

                        } else if (c == '\'' || c == '"') {                                                                                                                                                             

                            str = '';

                            q = c;

                            i++;

                            for(;;) {

                                    c = instring.charAt(i);

                                if(c < ' ') {

                                                        if((c == '\n') || (c == '\r') || (c == '')) {

                                                                // Add row if this white space is a row terminator

                                                                row++;

                                                        } else {

                                                                error('Unterminated String: ', str, row);                                               

                                                        }

                                }

                                if (c == q) {

                                                        break;

                                                }

                                if(c == '\\') {

                                    i++;

                                    if(i >= length) {

                                                error('Unterminated String: ', str, row);                                               

                                    }

                                                c = instring.charAt(i);

                                    if(c == 'b') { 

                                                                c = '\b'; 

                                                                break; 

                                                        }

                                    if(c == 'f') { 

                                                                c = '\f'; 

                                                                break; 

                                                        }

                                    if(c == 'n') { 

                                                                c = '\n'; 

                                                                break; 

                                                        }

                                    if(c == 'r') { 

                                                                c = '\r'; 

                                                                break; 

                                                        }

                                    if(c == 't') { 

                                                                c = '\t'; 

                                                                break; 

                                                        }

                                    if(c == 'u') {

                                        if(i >= length) {

                                                                error('Unterminated String: ', str, row);                                               

                                        }

                                        c = parseInt(this.substr(i + 1, 4), 16);

                                        if(!isFinite(c) || c < 0) {

                                                                error('Unterminated String: ', str, row);                                               

                                        }

                                        c = String.fromCharCode(c);

                                        i+=4;

                                        break;                              

                                    }

                                }

                                str += c;

                                i++;

                            }

                            i++;

                            maketoken('string', str, from, i, row);

                        c = instring.charAt(i);

                                // Comment of // type ... does not cover block comments

                        } else if (c == '/' && instring.charAt(i+1) == '/') {                                                           

                            i++;

                            for(;;) {

                                    c=instring.charAt(i);

                                if(c == '\n' || c == '\r' || c == '') {

                                    row++;

                                    break;

                                }

                                i++;

                            }

                                // Block comment of /* type

                        } else if (c == '/' && instring.charAt(i+1) == '*') {                                                           

                            i++;

                            for(;;) {

                                    c=instring.charAt(i);

                                if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {

                                    i+=2;

                                                c=instring.charAt(i);

                                    break;

                                }

                                if(c == '\n' || c == '\r' || c == '') {

                                    row++;

                                }

                                i++;

                            }

                                // Multi-character Operators

                        } else if (inprefix.indexOf(c) >= 0) {                                                                                  

                        str = c;

                        i++;

                        while(true) {

                                    c=instring.charAt(i);

                                if(i >= length || insuffix.indexOf(c) < 0) {

                                break;

                                }

                                str += c;

                                i++;

                        }

                                        // Single-character Operators

                        maketoken('operator', str, from, i, row);

                        } else {                                                                                                                                                                                                        

                        i++;

                        maketoken('operator', c, from, i, row);

                        c = instring.charAt(i);

                        }

                        }

                }

                function newButton()

                {

                        var funclist;

                        var oplist;

                        oplist = document.getElementById('infobox');

                        string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";

                        tokenize(string_tokenize, "<>+-&","=>&:");                              

                        // Iterate over token objects and print kind of each token and token type in window 

                        printout = document.getElementById('infobox');

                        for(i = 0;i < tokens.length;i++) {

                                printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";

                        }       

                }