Streamline `build.py`, update documentation and `bin/compiled.zip`

IDE/.vscode/tasks.json

@@ -9,12 +9,11 @@
             "command": [
                 "${config:python.pythonPath}",
                 "${workspaceFolder}/build.py",
-                "--file",
-                "${file}",
-                "compile"
+                "compile",
+                "${file}"
             ],
             "problemMatcher": {
-                "fileLocation": ["absolute"],
+                "fileLocation": ["relative", "${workspaceFolder}"],
                 "pattern": {
                   "regexp": "^File\\s+\"(.*)\",\\s+line\\s+(\\d+):\\s+(.*)$",
                   "file": 1,

IDE/Sublime Text/Oberon/Oberon.sublime-build

@@ -1,5 +1,5 @@
 {
-    "cmd": ["build.py", "--file=$file", "compile"],
+    "cmd": ["build.py", "compile", "$file"],
     "file_regex": "^[ ]*File \"(...*?)\", line ([0-9]*)",
     "selector": "source.oberon",
     "working_dir": "${project_path:${folder}}",

build.py

@@ -1,37 +1,43 @@
 #!/usr/bin/python
 from browser.linkjs import link
-import optparse
+import argparse
 import os
 import shutil
 import stat
 import subprocess
 import sys
+import tempfile
 import zipfile
 
-class package( object ):
-    root = os.path.join('bin', 'js')
-    archive = os.path.join('bin', 'compiled.zip')
+root = os.path.dirname(os.path.abspath(__file__))
+src_dir = os.path.join(root, 'src')
+default_out_dir = os.path.join(root, '_out')
+default_out_js = os.path.join(default_out_dir, 'js')
+
+class Package( object ):
+    bin_dir = os.path.join(root, 'bin')
+    js_root = os.path.join(bin_dir, 'js')
+    archive = os.path.join(bin_dir, 'compiled.zip')
 
     @staticmethod
     def pack():
-        file = zipfile.ZipFile(package.archive, 'w')
-        for f in os.listdir(package.root):
-            file.write(os.path.join(package.root, f), f)
+        print('packing current js "%s" -> "%s"...' % (Package.js_root, Package.archive))
+        file = zipfile.ZipFile(Package.archive, 'w')
+        for f in os.listdir(Package.js_root):
+            file.write(os.path.join(Package.js_root, f), f)
 
     @staticmethod
-    def unpack():
-        cleanup(package.root)
-        file = zipfile.ZipFile(package.archive, 'r')
-        file.extractall(package.root)
-
-# http://stackoverflow.com/questions/1889597/deleting-directory-in-python
-def remove_readonly(fn, path, excinfo):
-    if fn is os.rmdir:
-        os.chmod(path, stat.S_IWRITE)
-        os.rmdir(path)
-    elif fn is os.remove:
-        os.chmod(path, stat.S_IWRITE)
-        os.remove(path)
+    def unpacked_bin():
+        if not os.path.exists(Package.js_root):
+            print('unpacking pre-compiled js "%s" -> "%s"...'
+                  % (Package.archive, Package.js_root))
+            file = zipfile.ZipFile(Package.archive, 'r')
+            file.extractall(Package.js_root)
+        return Package.bin_dir
+
+def _remove_readonly(fn, path, excinfo):
+    os.chmod(path, stat.S_IWRITE)
+    fn(path)
 
 def norm_path(path):
     return os.path.normcase(os.path.normpath(os.path.realpath(os.path.abspath(path))))
@@ -54,7 +60,7 @@ def cleanup(dir):
     this_dir = os.path.dirname(__file__)
     if is_parent_for(dir, this_dir):
         raise Exception("cannot delete itself: %s" % this_dir)
-    shutil.rmtree(dir, onerror=remove_readonly)
+    shutil.rmtree(dir, onerror=_remove_readonly)
 
 def copy(src, dst_dir):
     dst = os.path.join(dst_dir, os.path.basename(src))
@@ -86,13 +92,10 @@ def run(cmd, env=None, cwd=None, print_output=False):
         exit(rc)
     return result
 
-root = os.path.dirname(os.path.abspath(__file__))
 snapshot_root = os.path.join(root, 'snapshot')
 
 def make_js_search_dirs(bin):
-    return [os.path.join(root, 'test'), 
-            os.path.join(root, 'src'), 
-            bin];
+    return [os.path.join(root, 'test'), src_dir, bin]
 
 def run_node(args, js_search_dirs, cwd=None):
     node_exe, path_separator = ('node.exe', ';') if os.name == 'nt' else ('node', ':')
@@ -124,10 +127,12 @@ def run_tests(bin, unit_test=None, code_test=None):
         run_node(args, js_search_dirs, cwd=os.path.join(root, 'test'))
     print('<-tests')
 
-def _run_compiler(compiler_dir, sources, js_search_dirs, locale, out_dir, cwd, timing=False):
+def _run_compiler(bin_dir, sources, locale, out_dir, timing=False):
+    if not os.path.exists(out_dir):
+        os.makedirs(out_dir)
     include = ['src/ob', 'src/eberon', 'src/oberon']
     include += [os.path.join(i, locale) for i in include]
-    args = [os.path.join(compiler_dir, 'oc_nodejs.js'), 
+    args = [os.path.join(src_dir, 'oc_nodejs.js'), 
             '--include=' + ';'.join([os.path.join(root, i) for i in include]), 
             '--out-dir=%s' % out_dir, 
             '--import-dir=js'
@@ -135,37 +140,21 @@ def _run_compiler(compiler_dir, sources, js_search_dirs, locale, out_dir, cwd, t
     if timing:
         args.append('--timing=true') 
     args += sources
-    run_node(args, js_search_dirs, cwd=cwd)
-
-def recompile(bin, cwd, locale):
-    print('recompile oberon sources using "%s"...' % bin)
-    sources = ['ContextAssignment.ob', 
-               'EberonSymbols.ob', 
-               'EberonContextCase.ob', 'EberonContextExpression.ob', 'EberonContextIdentdef.ob', 
-               'EberonContextIf.ob', 'EberonContextInPlace.ob', 'EberonContextProcedure', 'EberonContextType.ob', 
-               'EberonContextVar.ob', 'EberonLanguageContext.ob',
-               'OberonContext.ob', 'OberonContextType.ob', 'OberonContextVar.ob',
-               'OberonSymbols.ob', 'Lexer.ob', 'Module.ob']
-    
-    result = os.path.join(root, 'bin.recompile')
-    cleanup(result)
-    os.mkdir(result)
-    out = os.path.join(result, 'js')
-    os.mkdir(out)
-
-    subdirs = ['src/ob', 'src/eberon', 'src/oberon']
-    subdirs += [os.path.join(d, locale) for d in subdirs]
-    _run_compiler(cwd, sources, [bin, cwd], locale, out_dir=out, cwd=cwd, timing=True)
-    return result
-
-def compile_using_snapshot(src, locale):
-    out = os.path.join(root, 'bin', 'js')
-    _run_compiler(snapshot_root, [src], [snapshot_root], locale, out, cwd=root)
+    run_node(args, [bin_dir, src_dir], cwd=root)
+
+all_oberon_sources = [
+    'ContextAssignment.ob', 
+    'EberonSymbols.ob', 
+    'EberonContextCase.ob', 'EberonContextExpression.ob',
+    'EberonContextIdentdef.ob', 'EberonContextIf.ob',
+    'EberonContextInPlace.ob', 'EberonContextProcedure',
+    'EberonContextType.ob', 'EberonContextVar.ob', 'EberonLanguageContext.ob',
+    'OberonContext.ob', 'OberonContextType.ob', 'OberonContextVar.ob',
+    'OberonSymbols.ob', 'Lexer.ob', 'Module.ob']
 
 def build_html(options):
     version = None
     if options.set_version:
-        print(run(['git', 'pull']))
         version = run(['git', 'log', '-1', '--format="%ci%n%H"'])
 
     out = options.out
@@ -181,16 +170,12 @@ def build_html(options):
         print("current html is up to date, do nothing")
         return
 
-    if not options.do_not_unpack_compiled:
-        print('unpacking compiled js to %s...' % package.root)
-        package.unpack()
-
     if not os.path.exists(out):
         os.mkdir(out)
 
     link(['oc.js', 'oberon/oberon_grammar.js', 'eberon/eberon_grammar.js', 'test_unit.js'],
          os.path.join(out, 'oc.js'),
-         ['src', 'bin', 'test'],
+         ['src', Package.unpacked_bin(), 'test'],
          version)
     copy('browser/oberonjs.html', out)
     for d in ['codemirror', 'jslibs']:
@@ -203,65 +188,70 @@ def build_html(options):
         with open(build_version_path, 'w') as f:
             f.write(version)
 
-def recompile_with_replace(bin, cwd, locale, skip_tests=False, out_bin=None):
-    recompiled = recompile(bin, cwd, locale)
-    if not skip_tests:
-        run_tests(recompiled)
-    
-    if out_bin is None:
-        out_bin = bin
-
-    print('%s -> %s' % (recompiled, out_bin))
-    cleanup(out_bin)
-    os.rename(recompiled, out_bin)
+def _recompile_with_replace(locale, skip_tests=False):
+    bin_dir = Package.unpacked_bin()
+    new_bin = tempfile.mkdtemp(dir=bin_dir)
+    try:
+        new_js = os.path.join(new_bin, 'js')
+        print('recompile all oberon sources to "%s"...' % new_js)
+        _run_compiler(bin_dir, all_oberon_sources, locale, new_js)
+        if not skip_tests:
+            run_tests(new_bin)
+        print('replacing: "%s" -> "%s"...' % (new_js, Package.js_root))
+        cleanup(Package.js_root)
+        os.rename(new_js, Package.js_root)
+    finally:
+        shutil.rmtree(new_bin)
+    print('OK!')
 
 def pre_commit_check(options):
-    bin = os.path.join(root, 'bin')
-    run_tests(bin)
-    recompile_with_replace(bin, os.path.join(root, 'src'), options.locale)
-    
-    print('packaging compiled js to %s...' % package.root)
-    package.pack()
+    _recompile_with_replace(options.locale)
+    Package.pack()
 
 class compile_target(object):
     name = 'compile'
-    description = 'compile oberon source file using the snapshot'
+    description = 'compile oberon source file'
 
     @staticmethod
     def setup_options(parser):
-        parser.add_option('--file', help='file to compile')
+        parser.add_argument(
+            'files', nargs='+', metavar='FILE',
+            help='oberon source file(s) to compile')
+        parser.add_argument(
+            '--out',
+            help='output directory, default: "%(default)s"',
+            default=default_out_js)
 
     def __init__(self, options):
-        compile_using_snapshot(options.file, options.locale)
+        _run_compiler(
+            Package.unpacked_bin(), options.files, options.locale, options.out)
 
 class recompile_target(object):
     name = 'recompile'
-    description = 'recompile all oberon source files using the snapshot'
+    description = 'recompile all oberon source files'
 
     @staticmethod
     def setup_options(parser):
-        pass
+        parser.add_argument(
+            '--out',
+            help='output directory, default: "%(default)s"',
+            default=default_out_js)
 
     def __init__(self, options):
-        recompile_with_replace(
-            snapshot_root,
-            snapshot_root,
-            options.locale,
-            skip_tests=True,
-            out_bin=os.path.join(root, 'bin'))
+        _run_compiler(Package.unpacked_bin(), all_oberon_sources,
+                      options.locale, options.out)
 
 class self_recompile_target(object):
     name = 'self-recompile'
-    description = 'compile itself using current sources'
+    description = 'compile and replace itself using current sources'
 
     @staticmethod
     def setup_options(parser):
-        parser.add_option('--skip-tests', help='do not run test after recompile')
-        pass
+        parser.add_argument('--skip-tests', action='store_true',
+                            help='do not run test after recompile')
 
     def __init__(self, options):
-        bin = os.path.join(root, 'bin')
-        recompile_with_replace(bin, os.path.join(root, 'src'), options.locale, options.skip_tests)
+        _recompile_with_replace(options.locale, options.skip_tests)
 
 class html_target(object):
     name = 'html'
@@ -269,9 +259,8 @@ class html_target(object):
 
     @staticmethod
     def setup_options(parser):
-        parser.add_option('--out', help='output directory, default: "%default"', default='_out')
-        parser.add_option('--set-version', action="store_true", help='include version in built html')
-        parser.add_option('--do-not-unpack-compiled', action="store_true", help='do not unpack already compiled "binaries", use current')
+        parser.add_argument('--out', help='output directory, default: "%(default)s"', default='_out')
+        parser.add_argument('--set-version', action="store_true", help='include version in built html')
 
     def __init__(self, options):
         build_html(options)
@@ -282,16 +271,15 @@ class tests_target(object):
 
     @staticmethod
     def setup_options(parser):
-        parser.add_option('--unit', help='run specific unit test, use "*" to run all unit tests')
-        parser.add_option('--code', help='run specific code generator test, use "*" to run all generator tests')
+        parser.add_argument('--unit', help='run specific unit test, use "*" to run all unit tests')
+        parser.add_argument('--code', help='run specific code generator test, use "*" to run all generator tests')
 
     def __init__(self, options):
-        bin = os.path.join(root, 'bin')
-        run_tests(bin, options.unit, options.code)
+        run_tests(Package.unpacked_bin(), options.unit, options.code)
 
 class pre_commit_target(object):
     name = 'pre-commit'
-    description = 'run tests, recompile oberon sources, run tests against just recompiled sources, pack compiled sources and build html'
+    description = 'recompile oberon sources, run tests against just recompiled sources, pack compiled sources'
 
     @staticmethod
     def setup_options(parser):
@@ -300,56 +288,16 @@ class pre_commit_target(object):
     def __init__(self, options):
         pre_commit_check(options)
 
-class snapshot_target(object):
-    name = 'snapshot'
-    description = 'make snapshot - current compiled compiler (set of *.js) to use for compiling oberon sources'
-
-    @staticmethod
-    def setup_options(parser):
-        pass
-        
-    def __init__(self, options):
-        new_dir = snapshot_root + '.new'
-        cleanup(new_dir)
-        shutil.copytree(os.path.join(root, 'src'), new_dir)
-        shutil.copytree(os.path.join(root, 'bin', 'js'), os.path.join(new_dir, 'js'))
-        if os.path.exists(snapshot_root):
-            old_dir = snapshot_root + '.bak'
-            cleanup(old_dir)
-            os.rename(snapshot_root, old_dir)
-        os.rename(new_dir, snapshot_root)
-
-targets = [compile_target, recompile_target, self_recompile_target, html_target, tests_target, pre_commit_target, snapshot_target]
-
-def build(target, options):
-    targets[target](options)
+targets = [compile_target, recompile_target, self_recompile_target, html_target, tests_target, pre_commit_target]
 
 if __name__ == '__main__':
-    description = 'Targets: %s' % '|'.join([t.name for t in targets])
-    parser = optparse.OptionParser(
-        description=description,
-        usage='%prog [options] <target>'
-        )
-    parser.add_option('--locale', help='use specified localization subfolder, default is "%default"', default='en')
+    parser = argparse.ArgumentParser(description='Build tool')
+    parser.add_argument('--locale', help='use specified localization subfolder, default is "%(default)s"', default='en')
+    subparsers = parser.add_subparsers(help='targets') 
     for t in targets:
-        group = optparse.OptionGroup(parser, 'target "%s"' % t.name, t.description)
+        group = subparsers.add_parser(t.name, help=t.description)
         t.setup_options(group)
-        parser.add_option_group(group)
-
-    (options, args) = parser.parse_args()
-    if len(args) != 1:
-        parser.print_help();
-        exit(-1)
-    
-    target_name = args[0]
-    target = None
-    for t in targets:
-        if t.name == target_name:
-            target = t
-            break
+        group.set_defaults(func=t)
 
-    if target is None:
-        print('uknown target: "%s"' % target_name)
-        exit(-1)
-    
-    target(options)
+    args = parser.parse_args()
+    args.func(args)

doc/wiki/home.md

@@ -1,12 +1,25 @@
 ### The goal of the project
 I could formulate the goal as "to have traditional static-typed language on the Web". But more realistically it is: to built my own compiler (never did it before but always wanted). Also I wanted to [[experiment|Eberon]] with my own language.
 
+### Demo
+You can try the compiler online [here](http://oberspace.dyndns.org/oberonjs.html). You can compile more than one module at a time: modules should be separated by spaces or new lines and imported modules should be placed before dependent ones. 
+
 ### How to use
-You can use the project as any other JavaScript library. There is no third-party dependencies. The project is developing using nodejs so you may have some additional operations to accommodate nodejs source modules in your project. All source code is under src/ folder. Compiler entry point is oc.js.
+You can use the project as any other JavaScript library. There is no third-party dependencies. The project has been developing using nodejs, you might need to accommodate nodejs source modules in your project if you want to use nodejs. All source code is under src/ folder. Compiler entry point is `oc.js`.
 
-### Usage examples
-You can try the compiler online [here](http://oberspace.dyndns.org/oberonjs.html). You can compile more than one module at a time: modules should be separated by spaces or new lines and imported modules should be placed before dependent ones. 
-To build a test html page locally and see how it works run build.cmd (Python 2.x or 3.x is required). It will make _out/os.js (glued nodejs modules) and _out/oberonjs.html. Open oberonjs.html in the browser and try the compiler!
+The repository contains pre-compiled JavaScriprt code of the compiler (ready to run) in .zip [[archive|../../bin/compiled.zip]]. This code is needed to self-compile the compiler (from Oberon sources) for the first time.
+
+Python 2.x or 3.x is required. [[build.py|../../build.py]] script is used as a helper for different tasks:
+* build a test html page locally and see how it works
+    ```
+    ./build.py html
+    ```
+    this will create `_out/os.js` (glued nodejs modules) and `_out/oberonjs.html`. Open oberonjs.html in the browser and try the compiler!
+* compile Oberon source file(s)
+    ```
+    ./build.py compile file.ob
+    ```
+See other functions with "./build.py --help".
 
 ### State of development
 * Proof of concept: Oberon modules compile to JavaScript and can be executed in web browser.