/lenasys/0.1 : revision 4.1.1

To get this branch, use:

bzr branch
http://gegoxaren.bato24.eu/bzr/lenasys/0.1

« back to all changes in this revision

Viewing changes to trunk/Code Tokenizer and Parser/Parser_Test.js

Committer: c11emian
Date: 2013-03-28 14:41:09 UTC
mto: (4.2.1 hitlerhorabajs)
mto: This revision was merged to the branch mainline in revision 5.
Revision ID: c11emian@student.his.se-20130328144109-nq0tg9tcc8xxx78w

Added <head> and <title> tags
Extracted the javascript from the HTML code and inserted the javascript into an individual file
Changed the indentation, removed an unneccesary </div> and added a <!DOCTYPE HTML> - declartation

files added:
trunk/Code Tokenizer and Parser/Parser_Test.js

files modified:
trunk/Code Tokenizer and Parser/Parser_Test.html

Show diffs side-by-side

added added

removed removed

trunk/Code Tokenizer and Parser/Parser_Test.js

// Token class and storage definition

function token (kind,val,fromchar,tochar,row) {

this.kind = kind;

this.val = val;

this.from = fromchar;

this.to = tochar;

this.row = row;

}

var tokens = []; // Array to hold the tokens.

//----------------------------------------------------------

// Store token in tokens array

// Creates a new token object using the constructor

//----------------------------------------------------------

function maketoken(kind,val,from,to,rowno)

{

newtoken=new token(kind,val,from,to,rowno);

tokens.push(newtoken);

}

//----------------------------------------------------------

// Writes error from tokenizer

//----------------------------------------------------------

function error(str,val,row)

{

alert("Tokenizer Error: "+str+val+" at row "+row);

}

//----------------------------------------------------------

// Tokenize function

// Tokenizer partly based on ideas from the very clever tokenizer written by Douglas Cockford

// The tokenizer is passed a string, and a string of prefix and suffix terminators

//----------------------------------------------------------

function tokenize(instring,inprefix,insuffix){

var from; // index of the start of the token.

var i = 0; // index of the current character.

var length=instring.length; // length of the string

var c; // current character.

var n; // current numerical value

var q; // current quote character

var str; // current string value.

var row=1; // current row value

c = instring.charAt(i);

while (c) {

from = i;

if (c <= ' '){ // White space

if((c=='\n')||(c=='\r')||(c == '')) row++; // Add row if this white space is a row terminator

i++;

c = instring.charAt(i);

}else if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')) { // Names i.e. Text

str = c;

i++;

for (;;) {

c = instring.charAt(i);

if ((c >='a'&&c<='z')||(c>='A'&&c<='Z')||(c>='0'&&c<='9')||c=='_'){

str += c;

i++;

}else{

break;

}

maketoken('name',str,from,i,row);

} else if (c >= '0' && c <= '9') { // Number token

str = c;

i++;

for (;;) {

c = instring.charAt(i);

if (c < '0' || c > '9') break;

i++;

str+=c;

}

if(c=='.'){

i++;

str+=c;

for(;;){

c=instring.charAt(i);

if (c < '0' || c > '9') break;

i++;

str+=c;

}

if (c=='e'||c=='E') {

i++;

str+=c;

c=instring.charAt(i);

if(c=='-'||c=='+') {

i+=1;

str+=c;

c=instring.charAt(i);

100

}

101

if (c < '0' || c > '9') error('Bad Exponent in Number: ',str,row);

102

do {

103

i++;

104

str+=c;

105

c=instring.charAt(i);

106

}while(c>='0'&&c<='9');

107

}

108

if (c>='a'&&c<='z'){

109

str += c;

110

i += 1;

111

error('Bad Number: ',str,row);

112

}

113

n=+str;

114

if (isFinite(n)) {

115

maketoken('number',n,from,i,row);

116

} else {

117

error('Bad Number: ',str,row);

118

}

119

120

} else if(c=='\''||c=='"'){ // String .. handles c style breaking codes

121

str='';

122

q=c;

123

i++;

124

for (;;) {

125

c=instring.charAt(i);

126

127

if (c<' '){

128

if((c=='\n')||(c=='\r')||(c == '')) row++; // Add row if this white space is a row terminator

129

error('Unterminated String: ',str,row);

130

}

131

132

if (c==q) break;

133

134

if (c=='\\'){

135

i += 1;

136

if (i >= length) {

137

error('Unterminated String: ',str,row);

138

}

139

c=instring.charAt(i);

140

141

if(c=='b'){ c='\b'; break; }

142

if(c=='f'){ c='\f'; break; }

143

if(c=='n'){ c='\n'; break; }

144

if(c=='r'){ c='\r'; break; }

145

if(c=='t'){ c='\t'; break; }

146

if(c=='u'){

147

if (i >= length) {

148

error('Unterminated String: ',str,row);

149

}

150

c = parseInt(this.substr(i + 1, 4), 16);

151

if (!isFinite(c) || c < 0) {

152

error('Unterminated String: ',str,row);

153

}

154

c = String.fromCharCode(c);

155

i+=4;

156

break;

157

}

158

}

159

str += c;

160

i++;

161

}

162

i++;

163

maketoken('string',str,from,i,row);

164

c=instring.charAt(i);

165

166

} else if (c=='/'&&instring.charAt(i+1)=='/'){ // Comment of // type ... does not cover block comments

167

i++;

168

for (;;){

169

c=instring.charAt(i);

170

if (c=='\n'||c=='\r'||c=='') {

171

row++;

172

break;

173

}

174

i++;

175

}

176

} else if (c=='/'&&instring.charAt(i+1)=='*'){ // Block comment of /* type

177

i++;

178

for (;;){

179

c=instring.charAt(i);

180

if ((c=='*'&&instring.charAt(i+1)=='/')||(i==length)) {

181

i+=2;

182

c=instring.charAt(i);

183

break;

184

}

185

if (c=='\n'||c=='\r'||c=='') {

186

row++;

187

}

188

i++;

189

}

190

}else if(inprefix.indexOf(c) >= 0) { // Multi-character Operators

191

str = c;

192

i++;

193

while (true){

194

c=instring.charAt(i);

195

if (i >= length || insuffix.indexOf(c) < 0) {

196

break;

197

}

198

str += c;

199

i++;

200

}

201

maketoken('operator',str,from,i,row);

202

} else { // Single-character Operators

203

i++;

204

maketoken('operator',c,from,i,row);

205

c = instring.charAt(i);

206

}

207

208

}

209

}

210

211

function newbutton()

212

{

213

var funclist;

214

var oplist;

215

216

oplist=document.getElementById('infobox');

217

218

string_tokenize="foo // Lederhosen\nsin'foo'(200.0*r)+(-21.4/1.51e-6)+/*Plodder*/(Feeeb+400)*ln(3000);\ncos(200.1-atan2(41.0,51.0));\nHello+85-4*2.6-1.51e-6;";

219

220

tokenize(string_tokenize,"<>+-&","=>&:");

221

222

223

// Iterate over token objects and print kind of each token and token type in window

224

printout=document.getElementById('infobox');

225

for(i=0;i<tokens.length;i++){

226

printout.innerHTML+=tokens[i].kind+" "+tokens[i].val+" ("+tokens[i].from+"-"+tokens[i].to+") at: "+tokens[i].row+"<br>";

227

}

228

229

}

Older »