rewrite lexer.js in oberon

.gitignore

@@ -1,5 +1,5 @@
 _out/
-snapshot/
+snapshot*/
 test/output/
 *.sublime-project
 *.sublime-workspace

src/context.js

@@ -3,7 +3,6 @@
 var Cast = require("cast.js");
 var Code = require("code.js");
 var Errors = require("errors.js");
-var Lexer = require("lexer.js");
 var Module = require("module.js");
 var op = require("operator.js");
 var Parser = require("parser.js");
@@ -112,7 +111,15 @@ var ChainedContext = Class.extend({
     genTypeName: function(){return this.__parent.genTypeName();},
     genVarName: function(id){return this.__parent.genVarName(id);},
     qualifyScope: function(scope){return this.__parent.qualifyScope(scope);},
-    rtl: function(){return this.__parent.rtl();}
+    rtl: function(){return this.__parent.rtl();},
+    
+    // called from oberon code
+    raiseException: function(s){
+        var result = "";
+        for(var i = 0; i < s.length; ++i)
+            result += String.fromCharCode(s[i]);
+        throw new Errors.Error(result);
+    }
 });
 
 exports.Integer = ChainedContext.extend({
@@ -122,7 +129,7 @@ exports.Integer = ChainedContext.extend({
         this.__isHex = false;
     },
     isLexem: function(){return true;},
-    handleChar: function(c){this.__result += c;},
+    handleChar: function(c){this.__result += String.fromCharCode(c);},
     handleLiteral: function(){this.__isHex = true;},
     toInt: function(s){return parseInt(this.__result, 10);},
     endParse: function(){
@@ -144,7 +151,7 @@ exports.Real = ChainedContext.extend({
         this.__result = "";
     },
     isLexem: function(){return true;},
-    handleChar: function(c){this.__result += c;},
+    handleChar: function(c){this.__result += String.fromCharCode(c);},
     handleLiteral: function(s){
         if (s == "D") // LONGREAL
             s = "E";
@@ -192,7 +199,7 @@ exports.Char = exports.String.extend({
         exports.String.prototype.init.call(this, context);
         this.__result = "";
     },
-    handleChar: function(c){this.__result += c;},
+    handleChar: function(c){this.__result += String.fromCharCode(c);},
     toStr: function(s){return String.fromCharCode(parseInt(s, 16));}
 });
 
@@ -227,7 +234,7 @@ exports.QualifiedIdentificator = ChainedContext.extend({
         this.__id = undefined;
         this.__code = "";
     },
-    setIdent: function(id){
+    handleIdent: function(id){
         this.__id = id;
     },
     handleLiteral: function(){
@@ -262,10 +269,10 @@ exports.Identdef = ChainedContext.extend({
         this.__id = undefined;
         this.__export = false;
     },
-    setIdent: function(id){this.__id = id;},
+    handleIdent: function(id){this.__id = id;},
     handleLiteral: function(){this.__export = true;},
     endParse: function(){
-        this.parent().handleIdentef(new Identdef(this.__id, this.__export));
+        this.parent().handleIdentdef(new Identdef(this.__id, this.__export));
     }
 });
 
@@ -291,7 +298,7 @@ exports.Designator = ChainedContext.extend({
         this.__info = info;
         this.__code.write(code);
     },
-    setIdent: function(id){
+    handleIdent: function(id){
         var t = this.__currentType;
         var pointerType;
         var isReadOnly = this.__info instanceof Type.Variable 
@@ -514,12 +521,12 @@ exports.ProcDecl = ChainedContext.extend({
         this.__returnParsed = false;
         this.__outerScope = this.parent().currentScope();
     },
-    handleIdentef: function(id){
+    handleIdentdef: function(id){
         this.__id = id;
         this.codeGenerator().write("\nfunction " + id.id() + "(");
         this.parent().pushScope(new Scope.Procedure());
     },
-    setIdent: function(id){
+    handleIdent: function(id){
         if (this.__id.id() != id)
             throw new Errors.Error("mismatched procedure names: '" + this.__id.id()
                                  + "' at the begining and '" + id + "' at the end");
@@ -605,7 +612,7 @@ exports.ProcParams = HandleSymbolAsType.extend({
         if (s == "VAR")
             this.__isVar = true;
     },
-    setIdent: function(id){ this.__argNamesForType.push(id);},
+    handleIdent: function(id){ this.__argNamesForType.push(id);},
     setType: function(type){
         var names = this.__argNamesForType;
         for(var i = 0; i < names.length; ++i){
@@ -1293,7 +1300,7 @@ exports.CaseRange = ChainedContext.extend({
         }
         this.handleLabel(type, value);
     },
-    setIdent: function(id){
+    handleIdent: function(id){
         var s = getSymbol(this.parent(), id);
         if (!s.isConst())
             throw new Errors.Error("'" + id + "' is not a constant");
@@ -1358,7 +1365,7 @@ exports.For = ChainedContext.extend({
         this.__by_parsed = false;
         this.__by = undefined;
     },
-    setIdent: function(id){
+    handleIdent: function(id){
         var s = getSymbol(this.parent(), id);
         if (!s.isVariable())
             throw new Errors.Error("'" + s.id() + "' is not a variable");
@@ -1449,7 +1456,7 @@ exports.ConstDecl = ChainedContext.extend({
         this.__type = undefined;
         this.__value = undefined;
     },
-    handleIdentef: function(id){
+    handleIdentdef: function(id){
         this.__id = id;
         this.codeGenerator().write("var " + id.id() + " = ");
     },
@@ -1483,7 +1490,7 @@ exports.VariableDeclaration = HandleSymbolAsType.extend({
         this.__idents = [];
         this.__type = undefined;
     },
-    handleIdentef: function(id){this.__idents.push(id);},
+    handleIdentdef: function(id){this.__idents.push(id);},
     exportField: function(name){
         checkIfFieldCanBeExported(name, this.__idents, "variable");
     },
@@ -1517,7 +1524,7 @@ exports.FieldListDeclaration = HandleSymbolAsType.extend({
         this.__type = undefined;
     },
     typeName: function(){return undefined;},
-    handleIdentef: function(id) {this.__idents.push(id);},
+    handleIdentdef: function(id) {this.__idents.push(id);},
     exportField: function(name){
         checkIfFieldCanBeExported(name, this.__idents, "field");
     },
@@ -1684,7 +1691,7 @@ exports.TypeDeclaration = ChainedContext.extend({
         this.__id = undefined;
         this.__symbol = undefined;
     },
-    handleIdentef: function(id){
+    handleIdentdef: function(id){
         var typeId = new Type.LazyTypeId();
         var symbol = this.currentScope().addType(typeId, id);
         this.__id = id;
@@ -1741,7 +1748,7 @@ exports.ModuleDeclaration = ChainedContext.extend({
         this.__moduleScope = undefined;
         this.__moduleGen = undefined;
     },
-    setIdent: function(id){
+    handleIdent: function(id){
         var parent = this.parent();
         if (this.__name === undefined ) {
             this.__name = id;
@@ -1792,7 +1799,7 @@ var ModuleImport = ChainedContext.extend({
         this.__currentModule = undefined;
         this.__currentAlias = undefined;
     },
-    setIdent: function(id){
+    handleIdent: function(id){
         this.__currentModule = id;
     },
     handleLiteral: function(s){

src/grammar.js

@@ -1,12 +1,11 @@
 "use strict";
 
 var Context = require("context.js");
-var Lexer = require("lexer.js");
+var Lexer = require("oberon.js/Lexer.js");
 var Parser = require("parser.js");
 var Class = require("rtl.js").Class;
 
-var character = Lexer.character;
-var literal = Lexer.literal;
+var literal = Parser.literal;
 var digit = Lexer.digit;
 var hexDigit = Lexer.hexDigit;
 var ident = Lexer.ident;

src/lexer.js

@@ -1,161 +0,0 @@
-"use strict";
-
-var Errors = require("errors.js");
-var Stream = require("oberon.js/Stream.js");
-
-function isDigit(c) {return c >= '0' && c <= '9';}
-
-exports.digit = function(stream, context){
-    if (Stream.eof(stream))
-        return false;
-
-    var c = Stream.getChar(stream);
-    if (!isDigit(c))
-        return false;
-    context.handleChar(c);
-    return true;
-};
-
-exports.hexDigit = function(stream, context){
-    var c = Stream.getChar(stream);
-    if (!isDigit(c) && (c < 'A' || c > 'F'))
-        return false;
-    context.handleChar(c);
-    return true;
-};
-
-exports.point = function(stream, context){
-    if (Stream.eof(stream) 
-        || Stream.getChar(stream) != '.'
-        || (!Stream.eof(stream) && Stream.peekChar(stream) == '.')) // not a diapason ".."
-        return false;
-    context.handleLiteral(".");
-    return true;
-};
-
-exports.character = function(stream, context){
-    var c = stream.getChar();
-    if (c == '"')
-        return false;
-    context.handleChar(c);
-    return true;
-};
-
-function string(stream, context){
-    if (Stream.eof(stream))
-        return false;
-
-    var c = Stream.getChar(stream);
-    if (c != '"')
-        return false;
-
-    var result = "";
-    var parsed = false;
-    Stream.read(stream, function(c){
-        if (c == '"'){
-            parsed = true;
-            return false;
-        }
-        result += c;
-        return true;
-    });
-    if (!parsed)
-        throw new Errors.Error("unexpected end of string");
-
-    Stream.next(stream, 1);
-    context.handleString(result);
-    return true;
-}
-
-function isLetter(c) { return (c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z');}
-
-var reservedWords
-    = ["ARRAY", "IMPORT", "THEN", "BEGIN", "IN", "TO", "BY", "IS",
-       "TRUE", "CASE", "MOD", "TYPE", "CONST", "MODULE", "UNTIL", "DIV",
-       "NIL", "VAR", "DO", "OF", "WHILE", "ELSE", "OR", "ELSIF", "POINTER",
-       "END", "PROCEDURE", "FALSE", "RECORD", "FOR", "REPEAT", "IF", "RETURN"
-      ];
-var jsReservedWords 
-    = ["break", "case", "catch", "continue", "debugger", "default", "delete",
-       "do", "else", "finally", "for", "function", "if", "in", "instanceof",
-       "new", "return", "switch", "this", "throw", "try", "typeof", "var",
-       "void", "while", "with",
-       "Math" // Math is used in generated code for some functions so it is reserved word from code generator standpoint
-       ];
-
-exports.ident = function(stream, context){
-    if (Stream.eof(stream) || !isLetter(Stream.peekChar(stream)))
-        return false;
-    
-    var savePos = Stream.pos(stream);
-    var result = "";
-    Stream.read(stream, function(c){
-        if (!isLetter(c) && !isDigit(c) /*&& c != '_'*/)
-            return false;
-        result += c;
-        return true;
-    });
-
-    if (reservedWords.indexOf(result) != -1){
-        Stream.setPos(stream, savePos);
-        return false;
-    }
-    if (jsReservedWords.indexOf(result) != -1)
-        result += "$";
-
-    context.setIdent(result);
-    return true;
-};
-
-function skipComment(stream){
-    if (Stream.peekStr(stream, 2) != "(*")
-        return false;
-
-    Stream.next(stream, 2);
-    while (Stream.peekStr(stream, 2) != "*)"){
-        if (Stream.eof(stream))
-            throw new Errors.Error("comment was not closed");
-        if (!skipComment(stream))
-            Stream.next(stream, 1);
-        }
-    Stream.next(stream, 2);
-    return true;
-}
-
-function readSpaces(c){return ' \t\n\r'.indexOf(c) != -1;}
-
-exports.skipSpaces = function(stream, context){
-    if (context && context.isLexem && context.isLexem())
-        return;
-
-    do {
-        Stream.read(stream, readSpaces);
-    }
-    while (skipComment(stream));
-};
-
-exports.separator = function(stream, context){
-    return Stream.eof(stream) || !isLetter(Stream.peekChar(stream));
-};
-
-exports.literal = function(s){
-    return function(stream, context){
-        if (Stream.peekStr(stream, s.length) != s)
-            return false;
-        Stream.next(stream, s.length);
-        
-        if ((!context.isLexem || !context.isLexem())
-            && isLetter(s[s.length - 1])
-            && !Stream.eof(stream)
-           ){
-            var next = Stream.peekChar(stream);
-            if (isLetter(next) || isDigit(next))
-                return false;
-        }
-        
-        var result = context.handleLiteral(s);
-        return result === undefined || result;
-    };
-};
-
-exports.string = string;

src/oberon.js/JsString.js

@@ -5,6 +5,16 @@ var Type = RTL$.extend({
 	}
 });
 
+function make(s/*ARRAY OF CHAR*/){
+	var result = null;
+	var i = 0;
+	result = '';
+	for (i = 0; i <= s.length - 1 | 0; ++i){
+		result += JS.String.fromCharCode(s[i]);
+	}
+	return result;
+}
+
 function len(self/*Type*/){
 	var result = 0;
 	result = self.length;
@@ -13,7 +23,7 @@ function len(self/*Type*/){
 
 function at(self/*Type*/, pos/*INTEGER*/){
 	var result = 0;
-	result = self[pos];
+	result = self.charCodeAt(pos);
 	return result;
 }
 
@@ -34,9 +44,18 @@ function substr(self/*Type*/, pos/*INTEGER*/, len/*INTEGER*/){
 	result = self.substr(pos, len);
 	return result;
 }
+
+function appendChar(self/*Type*/, c/*CHAR*/){
+	var result = null;
+	result = self;
+	result += JS.String.fromCharCode(c);
+	return result;
+}
 exports.Type = Type;
+exports.make = make;
 exports.len = len;
 exports.at = at;
 exports.indexOf = indexOf;
 exports.indexOfFrom = indexOfFrom;
 exports.substr = substr;
+exports.appendChar = appendChar;

src/oberon.js/Lexer.js

@@ -0,0 +1,212 @@
+var RTL$ = require("RTL$.js").RTL$;
+var JS = GLOBAL;
+var JsString = require("JsString.js");
+var Stream = require("Stream.js");
+var quote = "\"";
+var commentBegin = "(*";
+var commentEnd = "*)";
+var Context = RTL$.extend({
+	init: function Context(){
+		this.handleChar = null;
+		this.handleLiteral = null;
+		this.handleString = null;
+		this.handleIdent = null;
+		this.isLexem = null;
+		this.raiseException = null;
+	}
+});
+var Literal = RTL$.extend({
+	init: function Literal(){
+		this.s = RTL$.makeArray(1, 0);
+	}
+});
+var reservedWords = null;
+var jsReservedWords = null;
+
+function isDigit(c/*CHAR*/){
+	return c >= 48 && c <= 57;
+}
+
+function isLetter(c/*CHAR*/){
+	return c >= 97 && c <= 122 || c >= 65 && c <= 90;
+}
+
+function digit(stream/*Type*/, context/*Context*/){
+	var result = false;
+	var c = 0;
+	if (!Stream.eof(stream)){
+		c = Stream.getChar(stream);
+		if (isDigit(c)){
+			context.handleChar(c);
+			result = true;
+		}
+	}
+	return result;
+}
+
+function hexDigit(stream/*Type*/, context/*Context*/){
+	var result = false;
+	var c = 0;
+	c = Stream.getChar(stream);
+	if (isDigit(c) || c >= 65 && c <= 70){
+		context.handleChar(c);
+		result = true;
+	}
+	return result;
+}
+
+function handleLiteral(context/*Context*/, s/*ARRAY OF CHAR*/){
+	var result = false;
+	var r = context.handleLiteral(JsString.make(s)); result = (r === undefined || r);
+	return result;
+}
+
+function point(stream/*Type*/, context/*Context*/){
+	var result = false;
+	if (!Stream.eof(stream) && Stream.getChar(stream) == 46 && (Stream.eof(stream) || Stream.peekChar(stream) != 46)){
+		result = handleLiteral(context, RTL$.strToArray("."));
+	}
+	return result;
+}
+
+function string(stream/*Type*/, context/*Context*/){
+	var result = false;
+	var c = 0;
+	var s = null;
+	if (!Stream.eof(stream)){
+		c = Stream.getChar(stream);
+		if (c == 34){
+			if (!Stream.eof(stream)){
+				s = JsString.make(RTL$.strToArray(""));
+				c = Stream.getChar(stream);
+				while (true){
+					if (c != 34 && !Stream.eof(stream)){
+						if (c != 34){
+							s = JsString.appendChar(s, c);
+						}
+						c = Stream.getChar(stream);
+					} else break;
+				}
+			}
+			if (s == null || c != 34){
+				context.raiseException(RTL$.strToArray("unexpected end of string"));
+			}
+			context.handleString(s);
+			result = true;
+		}
+	}
+	return result;
+}
+
+function isReservedWorld(s/*Type*/, words/*Type*/){
+	var i = 0;var w = 0;
+	while (true){
+		if (w < JsString.len(words) && i < JsString.len(s) && JsString.at(words, w) == JsString.at(s, i) && (i != 0 || w == 0 || JsString.at(words, w - 1 | 0) == 32)){
+			++w;
+			++i;
+		}
+		else if (w < JsString.len(words) && (i < JsString.len(s) || JsString.at(words, w) != 32)){
+			++w;
+			i = 0;
+		} else break;
+	}
+	return i == JsString.len(s);
+}
+
+function ident(stream/*Type*/, context/*Context*/){
+	var result = false;
+	var c = 0;
+	var s = null;
+	if (!Stream.eof(stream)){
+		c = Stream.getChar(stream);
+		if (isLetter(c)){
+			s = JsString.make(RTL$.strToArray(""));
+			while (true){
+				if (!Stream.eof(stream) && (isLetter(c) || isDigit(c))){
+					s = JsString.appendChar(s, c);
+					c = Stream.getChar(stream);
+				} else break;
+			}
+			if (isLetter(c) || isDigit(c)){
+				s = JsString.appendChar(s, c);
+			}
+			else {
+				Stream.next(stream, -1);
+			}
+			if (!isReservedWorld(s, reservedWords)){
+				if (isReservedWorld(s, jsReservedWords)){
+					s = JsString.appendChar(s, 36);
+				}
+				context.handleIdent(s);
+				result = true;
+			}
+		}
+	}
+	return result;
+}
+
+function skipComment(stream/*Type*/, context/*Context*/){
+	var result = false;
+	if (Stream.peekStr(stream, RTL$.strToArray(commentBegin))){
+		Stream.next(stream, commentBegin.length);
+		while (true){
+			if (!Stream.peekStr(stream, RTL$.strToArray(commentEnd))){
+				if (!skipComment(stream, context)){
+					Stream.next(stream, 1);
+					if (Stream.eof(stream)){
+						context.raiseException(RTL$.strToArray("comment was not closed"));
+					}
+				}
+			} else break;
+		}
+		Stream.next(stream, commentEnd.length);
+		result = true;
+	}
+	return result;
+}
+
+function readSpaces(c/*CHAR*/){
+	return c == 32 || c == 8 || c == 9 || c == 10 || c == 13;
+}
+
+function skipSpaces(stream/*Type*/, context/*Context*/){
+	if (context.isLexem == null || !context.isLexem()){
+		while (true){
+			if (Stream.read(stream, readSpaces) && skipComment(stream, context)){
+			} else break;
+		}
+	}
+}
+
+function separator(stream/*Type*/, context/*Context*/){
+	return Stream.eof(stream) || !isLetter(Stream.peekChar(stream));
+}
+
+function makeLiteral(s/*ARRAY OF CHAR*/){
+	var result = null;
+	result = new Literal();
+	result.s = s;
+	return result;
+}
+
+function literal(l/*Literal*/, stream/*Type*/, context/*Context*/){
+	var result = false;
+	if (Stream.peekStr(stream, l.s)){
+		Stream.next(stream, l.s.length);
+		if (context.isLexem != null && context.isLexem() || !isLetter(l.s[l.s.length - 1 | 0]) || Stream.eof(stream) || !isLetter(Stream.peekChar(stream)) && !isDigit(Stream.peekChar(stream))){
+			result = handleLiteral(context, l.s);
+		}
+	}
+	return result;
+}
+reservedWords = JsString.make(RTL$.strToArray("ARRAY IMPORT THEN BEGIN IN TO BY IS TRUE CASE MOD TYPE CONST MODULE UNTIL DIV NIL VAR DO OF WHILE ELSE OR ELSIF POINTER END PROCEDURE FALSE RECORD FOR REPEAT IF RETURN"));
+jsReservedWords = JsString.make(RTL$.strToArray("break case catch continue debugger default delete do else finally for function if in instanceof new return switch this throw try typeof var void while with Math"));
+exports.digit = digit;
+exports.hexDigit = hexDigit;
+exports.point = point;
+exports.string = string;
+exports.ident = ident;
+exports.skipSpaces = skipSpaces;
+exports.separator = separator;
+exports.makeLiteral = makeLiteral;
+exports.literal = literal;

src/oberon.js/RTL$.js

@@ -17,6 +17,30 @@ var RTL$ = {
         if (!condition)
             throw new Error("assertion failed"
                           + ((code !== undefined) ? " with code " + code : ""));
+    },
+    makeArray: function (/*dimensions, initializer*/){
+        var forward = Array.prototype.slice.call(arguments);
+        var result = new Array(forward.shift());
+        var i;
+        if (forward.length == 1){
+            var init = forward[0];
+            if (typeof init == "function")
+                for(i = 0; i < result.length; ++i)
+                    result[i] = init();
+            else
+                for(i = 0; i < result.length; ++i)
+                    result[i] = init;
+        }
+        else
+            for(i = 0; i < result.length; ++i)
+                result[i] = this.makeArray.apply(this, forward);
+        return result;
+    },
+    strToArray: function (s){
+        var result = new Array(s.length);
+        for(var i = 0; i < s.length; ++i)
+            result[i] = s.charCodeAt(i);
+        return result;
     }
 };
 

src/oberon.js/Stream.js

@@ -45,13 +45,18 @@ function getChar(self/*Type*/){
 	return result;
 }
 
-function peekStr(self/*Type*/, len/*INTEGER*/){
-	var max = 0;
-	max = JsString.len(self.s) - self.pos | 0;
-	if (len > max){
-		len = max;
+function peekStr(self/*Type*/, s/*ARRAY OF CHAR*/){
+	var result = false;
+	var i = 0;
+	if (s.length <= (JsString.len(self.s) - self.pos | 0)){
+		while (true){
+			if (i < s.length && s[i] == JsString.at(self.s, self.pos + i | 0)){
+				++i;
+			} else break;
+		}
+		result = i == s.length;
 	}
-	return JsString.substr(self.s, self.pos, len);
+	return result;
 }
 
 function read(self/*Type*/, f/*ReaderProc*/){
@@ -75,6 +80,7 @@ function lineNumber(self/*Type*/){
 	}
 	return line + 1 | 0;
 }
+exports.Type = Type;
 exports.make = make;
 exports.eof = eof;
 exports.pos = pos;

src/oberon/JsString.ob

@@ -4,6 +4,18 @@ IMPORT JS;
 TYPE
     Type* = POINTER TO RECORD END;
 
+PROCEDURE make*(s: ARRAY OF CHAR): Type;
+VAR 
+    result: Type;
+    i: INTEGER;
+BEGIN
+    JS.do("result = ''");
+    FOR i := 0 TO LEN(s) - 1 DO
+        JS.do("result += JS.String.fromCharCode(s[i])")
+    END
+    RETURN result
+END make;
+
 PROCEDURE len*(self: Type): INTEGER;
 VAR result: INTEGER;
 BEGIN
@@ -14,7 +26,7 @@ END len;
 PROCEDURE at*(self: Type; pos: INTEGER): CHAR;
 VAR result: CHAR;
 BEGIN
-    JS.do("result = self[pos]")
+    JS.do("result = self.charCodeAt(pos)")
     RETURN result
 END at;
 
@@ -39,4 +51,12 @@ BEGIN
     RETURN result
 END substr;
 
+PROCEDURE appendChar*(self: Type; c: CHAR): Type;
+VAR result: Type;
+BEGIN
+    result := self;
+    JS.do("result += JS.String.fromCharCode(c)")
+    RETURN result
+END appendChar;
+
 END JsString.

src/oberon/Lexer.ob

@@ -0,0 +1,232 @@
+MODULE Lexer;
+IMPORT JS, JsString, Stream;
+
+CONST
+    quote = 22X; (* " *)
+    commentBegin = "(*";
+    commentEnd = "*)";
+
+TYPE
+    Context = RECORD
+        handleChar: PROCEDURE(c: CHAR);
+        handleLiteral: PROCEDURE(s: JsString.Type): BOOLEAN;
+        handleString: PROCEDURE(s: JsString.Type);
+        handleIdent: PROCEDURE(s: JsString.Type);
+        isLexem: PROCEDURE(): BOOLEAN;
+        raiseException: PROCEDURE(s: ARRAY OF CHAR)
+    END;
+
+    Literal = POINTER TO RECORD
+        s: ARRAY 1 OF CHAR
+    END;
+
+VAR
+    reservedWords: JsString.Type;
+    jsReservedWords: JsString.Type;
+
+PROCEDURE isDigit(c: CHAR): BOOLEAN;
+    RETURN (c >= "0") & (c <= "9")
+END isDigit;
+
+PROCEDURE isLetter(c: CHAR): BOOLEAN;
+    RETURN ((c >= "a") & (c <= "z")) OR ((c >= "A") & (c <= "Z"))
+END isLetter;
+
+PROCEDURE digit*(stream: Stream.Type; context: Context): BOOLEAN;
+VAR
+    result: BOOLEAN;
+    c: CHAR;
+BEGIN
+    IF ~Stream.eof(stream) THEN
+        c := Stream.getChar(stream);
+        IF isDigit(c) THEN
+            context.handleChar(c);
+            result := TRUE;
+        END
+    END
+    RETURN result
+END digit;
+
+PROCEDURE hexDigit*(stream: Stream.Type; context: Context): BOOLEAN;
+VAR
+    result: BOOLEAN;
+    c: CHAR;
+BEGIN
+    c := Stream.getChar(stream);
+    IF isDigit(c) OR ((c >= "A") & (c <= "F")) THEN
+        context.handleChar(c);
+        result := TRUE;
+    END
+    RETURN result
+END hexDigit;
+
+PROCEDURE handleLiteral(context: Context; s: ARRAY OF CHAR): BOOLEAN;
+VAR result: BOOLEAN;
+BEGIN
+    JS.do("var r = context.handleLiteral(JsString.make(s)); result = (r === undefined || r)");
+    RETURN result
+END handleLiteral;
+
+PROCEDURE point*(stream: Stream.Type; context: Context): BOOLEAN;
+VAR result: BOOLEAN;
+BEGIN
+    IF    ~Stream.eof(stream)
+        & (Stream.getChar(stream) = ".")
+        & (    Stream.eof(stream) 
+            OR (Stream.peekChar(stream) # ".")) THEN (*not a diapason ".."*)        
+        result := handleLiteral(context, ".");
+    END
+    RETURN result
+END point;
+
+PROCEDURE string*(stream: Stream.Type; context: Context): BOOLEAN;
+VAR
+    result: BOOLEAN;
+    c: CHAR;
+    s: JsString.Type;
+BEGIN
+    IF ~Stream.eof(stream) THEN
+        c := Stream.getChar(stream);
+        IF c = quote THEN
+            IF ~Stream.eof(stream) THEN
+                s := JsString.make("");
+                c := Stream.getChar(stream);
+                WHILE (c # quote) & ~Stream.eof(stream) DO
+                    IF c # quote THEN
+                        s := JsString.appendChar(s, c);
+                    END;
+                    c := Stream.getChar(stream);
+                END;
+            END;
+            IF (s = NIL) OR (c # quote) THEN
+                context.raiseException("unexpected end of string");
+            END;
+            context.handleString(s);
+            result := TRUE;
+        END
+    END
+    RETURN result
+END string;
+
+PROCEDURE isReservedWorld(s: JsString.Type; words: JsString.Type): BOOLEAN;
+VAR
+    i, w: INTEGER;
+BEGIN
+    WHILE (w < JsString.len(words))
+        & (i < JsString.len(s))
+        & (JsString.at(words, w) = JsString.at(s, i))
+        & ((i # 0) OR (w = 0) OR (JsString.at(words, w - 1) = " ")) DO
+        INC(w);
+        INC(i);
+    ELSIF (w < JsString.len(words)) 
+        & ((i < JsString.len(s)) OR (JsString.at(words, w) # " ")) DO
+        INC(w);
+        i := 0;
+    END;
+    RETURN i = JsString.len(s)
+END isReservedWorld;
+
+PROCEDURE ident*(stream: Stream.Type; context: Context): BOOLEAN;
+VAR
+    result: BOOLEAN;
+    c: CHAR;
+    s: JsString.Type;
+BEGIN
+    IF ~Stream.eof(stream) THEN
+        c := Stream.getChar(stream);
+        IF isLetter(c) THEN
+            s := JsString.make("");
+            WHILE ~Stream.eof(stream) & (isLetter(c) OR isDigit(c)) DO (* OR c = "_" *)
+                s := JsString.appendChar(s, c);
+                c := Stream.getChar(stream);
+            END;
+            IF isLetter(c) OR isDigit(c) THEN
+                s := JsString.appendChar(s, c);
+            ELSE
+                Stream.next(stream, -1);
+            END;
+
+            IF ~isReservedWorld(s, reservedWords) THEN
+                IF isReservedWorld(s, jsReservedWords) THEN
+                    s := JsString.appendChar(s, "$");
+                END;
+                context.handleIdent(s);
+                result := TRUE;
+            END
+        END
+    END
+    RETURN result
+END ident;
+
+PROCEDURE skipComment(stream: Stream.Type; context: Context): BOOLEAN;
+VAR
+    result: BOOLEAN;
+BEGIN
+    IF Stream.peekStr(stream, commentBegin) THEN
+        Stream.next(stream, LEN(commentBegin));
+        WHILE ~Stream.peekStr(stream, commentEnd) DO
+            IF ~skipComment(stream, context) THEN
+                Stream.next(stream, 1);
+                IF Stream.eof(stream) THEN
+                    context.raiseException("comment was not closed");
+                END
+            END
+        END;
+        Stream.next(stream, LEN(commentEnd));
+        result := TRUE;
+    END
+    RETURN result
+END skipComment;
+
+PROCEDURE readSpaces(c: CHAR): BOOLEAN;
+    RETURN (c = " ") 
+        OR (c = 8X)
+        OR (c = 9X)
+        OR (c = 0AX)
+        OR (c = 0DX)
+END readSpaces;
+
+PROCEDURE skipSpaces*(stream: Stream.Type; context: Context);
+BEGIN
+    IF (context.isLexem = NIL) OR ~context.isLexem() THEN
+        WHILE Stream.read(stream, readSpaces)
+            & skipComment(stream, context) DO END;
+    END
+END skipSpaces;
+
+PROCEDURE separator*(stream: Stream.Type; context: Context): BOOLEAN;
+    RETURN Stream.eof(stream) OR ~isLetter(Stream.peekChar(stream))
+END separator;
+
+PROCEDURE makeLiteral*(s: ARRAY OF CHAR): Literal;
+VAR
+    result: Literal;
+BEGIN
+    NEW(result);
+    JS.do("result.s = s");
+    RETURN result
+END makeLiteral;
+
+PROCEDURE literal*(l: Literal; stream: Stream.Type; context: Context): BOOLEAN;
+VAR
+    result: BOOLEAN;
+BEGIN
+    IF Stream.peekStr(stream, l.s) THEN
+        Stream.next(stream, LEN(l.s));
+        IF ((context.isLexem # NIL) & context.isLexem())
+            OR ~isLetter(l.s[LEN(l.s) - 1])
+            OR Stream.eof(stream)
+            OR (~isLetter(Stream.peekChar(stream)) & ~isDigit(Stream.peekChar(stream)))
+                THEN
+            result := handleLiteral(context, l.s);
+        END;
+    END;
+    RETURN result
+END literal;
+
+BEGIN
+    reservedWords := JsString.make(
+        "ARRAY IMPORT THEN BEGIN IN TO BY IS TRUE CASE MOD TYPE CONST MODULE UNTIL DIV NIL VAR DO OF WHILE ELSE OR ELSIF POINTER END PROCEDURE FALSE RECORD FOR REPEAT IF RETURN");
+    jsReservedWords := JsString.make(
+        "break case catch continue debugger default delete do else finally for function if in instanceof new return switch this throw try typeof var void while with Math"); (* Math is used in generated code for some functions so it is reserved word from code generator standpoint *)
+END Lexer.

src/oberon/Stream.ob

@@ -2,7 +2,7 @@ MODULE Stream;
 IMPORT JsString;
 
 TYPE
-    Type = POINTER TO RECORD
+    Type* = POINTER TO RECORD
         s: JsString.Type;
         pos: INTEGER
     END;
@@ -52,14 +52,18 @@ BEGIN
     RETURN result
 END getChar;
 
-PROCEDURE peekStr*(self: Type; len: INTEGER): JsString.Type;
-VAR max: INTEGER;
+PROCEDURE peekStr*(self: Type; s: ARRAY OF CHAR): BOOLEAN;
+VAR
+    result: BOOLEAN;
+    i: INTEGER;
 BEGIN
-    max := JsString.len(self.s) - self.pos;
-    IF len > max THEN
-        len := max;
+    IF LEN(s) <= JsString.len(self.s) - self.pos THEN
+        WHILE (i < LEN(s)) & (s[i] = JsString.at(self.s, self.pos + i)) DO
+            INC(i)
+        END;
+        result := i = LEN(s);
     END
-    RETURN JsString.substr(self.s, self.pos, len)
+    RETURN result
 END peekStr;
 
 PROCEDURE read*(self: Type; f: ReaderProc): BOOLEAN;

src/oc.js

@@ -4,7 +4,7 @@ var Code = require("code.js");
 var Context = require("context.js");
 var Errors = require("errors.js");
 var Grammar = require("grammar.js");
-var Lexer = require("lexer.js");
+var Lexer = require("oberon.js/lexer.js");
 var ImportRTL = require("rtl.js");
 var Stream = require("oberon.js/Stream.js");
 
@@ -58,7 +58,7 @@ function compileModulesFromText(
         if (!module)
             return;
         handleCompiledModule(module);
-        Lexer.skipSpaces(stream);
+        Lexer.skipSpaces(stream, context);
     }
     while (!Stream.eof(stream));
 }

src/parser.js

@@ -2,11 +2,19 @@
 
 var assert = require("assert.js").ok;
 var Errors = require("errors.js");
-var Lexer = require("lexer.js");
+var Lexer = require("oberon.js/lexer.js");
 var Stream = require("oberon.js/Stream.js");
+var RTL$ = require("oberon.js/RTL$.js").RTL$;
+
+function literal(s){
+	var l = Lexer.makeLiteral(RTL$.strToArray(s));
+	return function(stream, context){
+		return Lexer.literal(l, stream, context);
+	};
+}
 
 function implicitParser(p){
-	return typeof p === "string" ? Lexer.literal(p) : p;
+	return typeof p === "string" ? literal(p) : p;
 }
 
 function argumentsToParsers(args){
@@ -107,3 +115,5 @@ exports.emit = function(parser, action){
 		return true;
 	};
 };
+
+exports.literal = literal;

test/expected/copy.js

@@ -1,46 +0,0 @@
-var RTL$ = {
-    makeArray: function (/*dimensions, initializer*/){
-        var forward = Array.prototype.slice.call(arguments);
-        var result = new Array(forward.shift());
-        var i;
-        if (forward.length == 1){
-            var init = forward[0];
-            if (typeof init == "function")
-                for(i = 0; i < result.length; ++i)
-                    result[i] = init();
-            else
-                for(i = 0; i < result.length; ++i)
-                    result[i] = init;
-        }
-        else
-            for(i = 0; i < result.length; ++i)
-                result[i] = this.makeArray.apply(this, forward);
-        return result;
-    },
-    assignArrayFromString: function (a, s){
-        var i;
-        for(i = 0; i < s.length; ++i)
-            a[i] = s.charCodeAt(i);
-        for(i = s.length; i < a.length; ++i)
-            a[i] = 0;
-    },
-    copy: function (from, to){
-        for(var prop in to){
-            if (to.hasOwnProperty(prop)){
-                var v = from[prop];
-                if (v !== null && typeof v == "object")
-                    this.copy(v, to[prop]);
-                else
-                    to[prop] = v;
-            }
-        }
-    }
-};
-var m = function (){
-var s1 = "\"";
-var s2 = "ABC";
-var ac3 = RTL$.makeArray(3, 0);
-RTL$.assignArrayFromString(ac3, s1);
-RTL$.assignArrayFromString(ac3, s2);
-RTL$.copy(ac3, ac3);
-}();

test/run_nodejs.cmd

@@ -1,2 +1,2 @@
-@SET NODE_PATH=.;%~dp1;%NODE_PATH%
+@SET NODE_PATH=.;%~dp1;%~dp1\oberon.js;%NODE_PATH%
 @"C:\Program Files\nodejs\node.exe" %*

test/test_unit.js

@@ -219,6 +219,7 @@ var testSuite = {
          "22X",
          "0X"),
     fail(["\"", "unexpected end of string"],
+         ["\"abc", "unexpected end of string"],
          ["FFX", "undeclared identifier: 'FFX'"]
         )
     ),
@@ -233,8 +234,8 @@ var testSuite = {
 "identifier": testWithSetup(
     function(){
         var IdentDeclarationContext = Class.extend({
-            init: function(){this.__ident = undefined;},
-            setIdent: function(id){this.__ident = id;},
+            init: function IdentDeclarationContext(){this.__ident = undefined;},
+            handleIdent: function(id){this.__ident = id;},
             ident: function() {return this.__ident;},
             getResult: function() {return this.__ident;}
         });
@@ -243,6 +244,7 @@ var testSuite = {
         return setupParser(Grammar.ident, makeContext);},
     pass("i", "abc1"),
     fail(["", "not parsed"],
+         [";", "not parsed"],
          ["1", "not parsed"]
          )
     ),