/lenasys/trunk

// Array to hold the tokens                             

var tokens = [];  

/* 

 * Token class and storage definition                                                                   

 */

function token(kind, val, fromchar, tochar, row) {

    this.kind = kind;

    this.val = val;

    this.from = fromchar;

    this.to = tochar;

    this.row = row;

}          

/*

 * Store token in tokens array

 * Creates a new token object using the constructor

 */

function makeToken(kind, val, from, to, rowno) {

        newtoken = new token(kind, val, from, to, rowno);

        tokens.push(newtoken);

}

/*

 * Writes error from tokenizer

 */             

function error(str, val, row) {

        alert("Tokenizer Error: " + str + val + " at row " + row);

}

/*

 * Tokenize function 

 * Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford

 * The tokenizer is passed a string, and a string of prefix and suffix terminators

 */             

function tokenize(instring, inprefix, insuffix) {           

        // index of the start of the token.

        var from;                       

        // index of the current character.

        var i = 0;                      

        // length of the string

        var length = instring.length;           

    // current character.

        var c;                          

    // current numerical value

        var n;                          

    // current quote character

        var q;                          

    // current string value.

        var str;                        

        // current row value            

        var row=1;                                                                              

        c = instring.charAt(i);

        while(c) {                      

        from = i;

                // White space

        if(c <= ' ') {

                        // Add row if this white space is a row terminator              

                if((c == '\n') || (c == '\r') || (c == '')) {

                                row++;

                        }                                                                                                                                                                                                                                                               

                    i++;

                c = instring.charAt(i);

                        // Names i.e. Text

        } else if ((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z')) {                                  

                        str = c;

            i++;

            for(;;) {

                c = instring.charAt(i);

                if((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9')|| (c == '_')) {

                    str += c;

                    i++;

                } else {

                    break;

                }

            }

            makeToken('name', str, from, i, row);

                        // Number token

                } else if (c >= '0' && c <= '9') {                                                                                              

                    str = c;

                    i++;

                    for(;;) {

                                c = instring.charAt(i);

                        if(c < '0' || c > '9') {

                                break;

                                }

                        i++;

                        str += c;

                    }

                    if(c == '.') {

                        i++;

                        str += c;

                        for(;;){

                                c=instring.charAt(i);

                            if(c < '0' || c > '9'){ 

                                        break;

                                        }

                            i++;

                            str += c;

                            }

                    }

                    if(c == 'e' || c =='E') {

                        i++;

                        str += c;

                                c=instring.charAt(i);

                        if(c == '-' || c == '+') {

                            i++;

                            str += c;

                                        c=instring.charAt(i);

                        }

                        if(c < '0' || c > '9') {

                                        error('Bad Exponent in Number: ',str,row);

                                }

                        do {

                            i++;

                            str += c;

                                        c = instring.charAt(i);

                        } while(c >= '0' && c <= '9');

                    }

                    if(c >= 'a' && c <= 'z') {

                        str += c;

                        i++;

                        error('Bad Number: ',str,row);

                    }

                    n =+ str;

                    if(isFinite(n)) {

                                makeToken('number', n, from, i, row);                                   

                    } else {

                        error('Bad Number: ', str, row);

                    }

                        // String .. handles c style breaking codes

                } else if (c == '\'' || c == '"') {                                                                                                                                                             

                    str = '';

                    q = c;

                    i++;

                    for(;;) {

                                c = instring.charAt(i);

                                if(c < ' ') {

                                        if((c == '\n') || (c == '\r') || (c == '')) {

                                                // Add row if this white space is a row terminator

                                                row++;

                                        } else {

                                                error('Unterminated String: ', str, row);                                               

                                        }

                                }

                                if (c == q) {

                                        break;

                                }

                                if(c == '\\') {

                                        i++;

                                        if(i >= length) {

                                                error('Unterminated String: ', str, row);                                               

                                        }

                                        c = instring.charAt(i);

                                        if(c == 'b') { 

                                                c = '\b'; 

                                                break; 

                                        }

                                        if(c == 'f') { 

                                                c = '\f'; 

                                                break; 

                                        }

                                        if(c == 'n') { 

                                                c = '\n'; 

                                                break; 

                                        }

                                        if(c == 'r') { 

                                                c = '\r'; 

                                                break; 

                                        }

                                        if(c == 't') { 

                                                c = '\t'; 

                                                break; 

                                        }

                                        if(c == 'u') {

                                                if(i >= length) {

                                                        error('Unterminated String: ', str, row);                                               

                                                }

                                                c = parseInt(this.substr(i + 1, 4), 16);

                                                if(!isFinite(c) || c < 0) {

                                                        error('Unterminated String: ', str, row);                                               

                                                }

                                                c = String.fromCharCode(c);

                                                i+=4;

                                                break;                              

                                        }

                                }

                                str += c;

                                i++;

                    }

                    i++;

                    makeToken('string', str, from, i, row);

                c = instring.charAt(i);

                        // Comment of // type ... does not cover block comments

                } else if (c == '/' && instring.charAt(i+1) == '/') {                                                           

                    i++;

                    for(;;) {

                                c=instring.charAt(i);

                        if(c == '\n' || c == '\r' || c == '') {

                            row++;

                            break;

                                }

                        i++;

                    }

                                // Block comment of /* type

                } else if (c == '/' && instring.charAt(i+1) == '*') {                                                           

                    i++;

                    for(;;) {

                                c=instring.charAt(i);

                        if((c == '*' && instring.charAt(i+1) == '/') || (i == length)) {

                            i+=2;

                                        c=instring.charAt(i);

                            break;

                                }

                        if(c == '\n' || c == '\r' || c == '') {

                            row++;

                                }

                        i++;

                    }

                        // Multi-character Operators

                } else if (inprefix.indexOf(c) >= 0) {                                                                                  

                        str = c;

            i++;

            while(true) {

                        c=instring.charAt(i);

                if(i >= length || insuffix.indexOf(c) < 0) {

                        break;

                }

                str += c;

                i++;

            }

                        // Single-character Operators

                makeToken('operator', str, from, i, row);

        } else {                                                                                                                                                                                                        

                i++;

                makeToken('operator', c, from, i, row);

                c = instring.charAt(i);

        }

        }

}

 

function newButton() {

        var funclist;

        var oplist;

        oplist = document.getElementById('infobox');

        string_tokenize = "foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";

        tokenize(string_tokenize, "<>+-&","=>&:");                              

        // Iterate over token objects and print kind of each token and token type in window 

        printout = document.getElementById('infobox');

        for(i = 0;i < tokens.length;i++) {

                printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";

        }       

}