/lenasys/trunk : contents of Code Tokenizer and Parser/Parser

: (revision 8.1.1)

To get this branch, use:

bzr branch
http://gegoxaren.bato24.eu/bzr/lenasys/trunk

4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	1	/*
	2	* Token class and storage definition
	3	*/
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	4
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	5	function token(kind, val, fromchar, tochar, row) {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	6	this.kind = kind;
	7	this.val = val;
	8	this.from = fromchar;
	9	this.to = tochar;
	10	this.row = row;
	11	}
	12
	13	// Array to hold the tokens
	14	var tokens = [];
	15	/*
	16	* Store token in tokens array
	17	* Creates a new token object using the constructor
	18	*/
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	19
	20	function maketoken(kind, val, from, to, rowno) {
	21	newtoken = new token(kind, val, from, to, rowno);
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	22	tokens.push(newtoken);
	23	}
	24
	25	/*
	26	* Writes error from tokenizer
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	27	*/
	28
	29	function error(str, val, row) {
	30	alert("Tokenizer Error: " + str + val + " at row " + row);
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	31	}
	32
	33	/*
	34	* Tokenize function
	35	* Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford
	36	* The tokenizer is passed a string, and a string of prefix and suffix terminators
	37	*/
	38
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	39	function tokenize(instring, inprefix, insuffix) {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	40	// index of the start of the token.
	41	var from;
	42	// index of the current character.
	43	var i = 0;
	44	// length of the string
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	45	var length = instring.length;
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	46	// current character.
	47	var c;
	48	// current numerical value
	49	var n;
	50	// current quote character
	51	var q;
	52	// current string value.
	53	var str;
	54	// current row value
	55	var row=1;
	56	c = instring.charAt(i);
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	57	while(c) {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	58	from = i;
	59	// White space
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	60	if(c <= ' ') {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	61	// Add row if this white space is a row terminator
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	62	if((c == '\n') \|\| (c == '\r') \|\| (c == '')) {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	63	row++;
	64	}
	65	i++;
	66	c = instring.charAt(i);
	67	// Names i.e. Text
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	68	} else if ((c >= 'a' && c <= 'z') \|\| (c >= 'A' && c <= 'Z')) {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	69	str = c;
	70	i++;
	71	for(;;) {
	72	c = instring.charAt(i);
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	73	if((c >= 'a' && c <= 'z') \|\| (c >= 'A' && c <= 'Z') \|\| (c >= '0' && c <= '9')\|\| (c == '_')) {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	74	str += c;
	75	i++;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	76	} else {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	77	break;
	78	}
	79	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	80	maketoken('name', str, from, i, row);
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	81	// Number token
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	82	} else if (c >= '0' && c <= '9') {
4.1.2 by c11emian I have begun to fix the javascript so that it conforms to our codestandard.	83	str = c;
	84	i++;
	85	for(;;) {
	86	c = instring.charAt(i);
	87	if(c < '0' \|\| c > '9') {
	88	break;
	89	}
	90	i++;
	91	str += c;
	92	}
	93	if(c == '.') {
	94	i++;
	95	str += c;
	96	for(;;){
	97	c=instring.charAt(i);
	98	if(c < '0' \|\| c > '9'){
	99	break;
	100	}
	101	i++;
	102	str += c;
	103	}
	104	}
	105	if(c == 'e' \|\| c =='E') {
	106	i++;
	107	str += c;
	108	c=instring.charAt(i);
	109	if(c == '-' \|\| c == '+') {
	110	i++;
	111	str += c;
	112	c=instring.charAt(i);
	113	}
	114	if(c < '0' \|\| c > '9') {
	115	error('Bad Exponent in Number: ',str,row);
	116	}
	117	do {
	118	i++;
	119	str += c;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	120	c = instring.charAt(i);
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	121	} while(c >= '0' && c <= '9');
4.1.1 by c11emian Added <head> and <title> tags	122	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	123	if(c >= 'a' && c <= 'z') {
4.1.1 by c11emian Added <head> and <title> tags	124	str += c;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	125	i++;
	126	error('Bad Number: ',str,row);
	127	}
	128	n =+ str;
	129	if(isFinite(n)) {
	130	maketoken('number', n, from, i, row);
	131	} else {
	132	error('Bad Number: ', str, row);
	133	}
	134	} else if (c == '\'' \|\| c == '"') { // String .. handles c style breaking codes
	135	str = '';
	136	q = c;
4.1.1 by c11emian Added <head> and <title> tags	137	i++;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	138	for(;;) {
	139	c = instring.charAt(i);
	140	if(c < ' ') {
	141	if((c == '\n') \|\| (c == '\r') \|\| (c == '')) {
	142	// Add row if this white space is a row terminator
	143	row++;
	144	} else {
	145	error('Unterminated String: ', str, row);
	146	}
4.1.1 by c11emian Added <head> and <title> tags	147	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	148	if (c == q) {
	149	break;
	150	}
	151	if(c == '\\') {
	152	i++;
	153	if(i >= length) {
	154	error('Unterminated String: ', str, row);
4.1.1 by c11emian Added <head> and <title> tags	155	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	156	c = instring.charAt(i);
	157	if(c == 'b') {
	158	c = '\b';
	159	break;
	160	}
	161	if(c == 'f') {
	162	c = '\f';
	163	break;
	164	}
	165	if(c == 'n') {
	166	c = '\n';
	167	break;
	168	}
	169	if(c == 'r') {
	170	c = '\r';
	171	break;
	172	}
	173	if(c == 't') {
	174	c = '\t';
	175	break;
	176	}
	177	if(c == 'u') {
	178	if(i >= length) {
	179	error('Unterminated String: ', str, row);
4.1.1 by c11emian Added <head> and <title> tags	180	}
4.1.1 by c11emian Added <head> and <title> tags	181	c = parseInt(this.substr(i + 1, 4), 16);
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	182	if(!isFinite(c) \|\| c < 0) {
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	183	error('Unterminated String: ', str, row);
4.1.1 by c11emian Added <head> and <title> tags	184	}
	185	c = String.fromCharCode(c);
	186	i+=4;
	187	break;
	188	}
	189	}
	190	str += c;
	191	i++;
	192	}
	193	i++;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	194	maketoken('string', str, from, i, row);
	195	c = instring.charAt(i);
	196	// Comment of // type ... does not cover block comments
	197	} else if (c == '/' && instring.charAt(i+1) == '/') {
4.1.1 by c11emian Added <head> and <title> tags	198	i++;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	199	for(;;) {
	200	c=instring.charAt(i);
	201	if(c == '\n' \|\| c == '\r' \|\| c == '') {
4.1.1 by c11emian Added <head> and <title> tags	202	row++;
	203	break;
	204	}
	205	i++;
	206	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	207	// Block comment of /* type
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	208	} else if (c == '/' && instring.charAt(i+1) == '*') {
4.1.1 by c11emian Added <head> and <title> tags	209	i++;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	210	for(;;) {
	211	c=instring.charAt(i);
	212	if((c == '*' && instring.charAt(i+1) == '/') \|\| (i == length)) {
4.1.1 by c11emian Added <head> and <title> tags	213	i+=2;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	214	c=instring.charAt(i);
4.1.1 by c11emian Added <head> and <title> tags	215	break;
4.1.1 by c11emian Added <head> and <title> tags	216	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	217	if(c == '\n' \|\| c == '\r' \|\| c == '') {
4.1.1 by c11emian Added <head> and <title> tags	218	row++;
	219	}
	220	i++;
	221	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	222	// Multi-character Operators
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	223	} else if (inprefix.indexOf(c) >= 0) {
4.1.1 by c11emian Added <head> and <title> tags	224	str = c;
4.1.1 by c11emian Added <head> and <title> tags	225	i++;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	226	while(true) {
	227	c=instring.charAt(i);
	228	if(i >= length \|\| insuffix.indexOf(c) < 0) {
4.1.1 by c11emian Added <head> and <title> tags	229	break;
	230	}
	231	str += c;
	232	i++;
	233	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	234	// Single-character Operators
	235	maketoken('operator', str, from, i, row);
	236	} else {
4.1.1 by c11emian Added <head> and <title> tags	237	i++;
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	238	maketoken('operator', c, from, i, row);
4.1.1 by c11emian Added <head> and <title> tags	239	c = instring.charAt(i);
4.1.1 by c11emian Added <head> and <title> tags	240	}
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	241	}
4.1.1 by c11emian Added <head> and <title> tags	242	}
	243	function newbutton()
	244	{
4.1.3 by c11emian Fixed the indentation in Parser_Test.js	245	var funclist;
	246	var oplist;
	247	oplist = document.getElementById('infobox');
	248	string_tokenize = "foo // Lederhosen\nsin'foo'(200.0r)+(-21.4/1.51e-6)+/Plodder/(Feeeb+400)ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";
	249	tokenize(string_tokenize, "<>+-&","=>&:");
	250	// Iterate over token objects and print kind of each token and token type in window
	251	printout = document.getElementById('infobox');
	252	for(i = 0;i < tokens.length;i++) {
	253	printout.innerHTML += tokens[i].kind + " " + tokens[i].val + " (" + tokens[i].from + "-" + tokens[i].to + ") at: " + tokens[i].row + "<br>";
	254	}
4.1.1 by c11emian Added <head> and <title> tags	255	}