#!/usr/bin/env python3 # Copyright 2020 Mattia Giambirtone # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. # A minimalistic build script for JAPL import os import stat import json import shlex import shutil import logging import argparse from time import time from typing import Dict, Optional from subprocess import Popen, PIPE, DEVNULL, run CONFIG_TEMPLATE = '''# Copyright 2020 Mattia Giambirtone # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import strformat const MAP_LOAD_FACTOR* = {map_load_factor} # Load factor for builtin hashmaps const ARRAY_GROW_FACTOR* = {array_grow_factor} # How much extra memory to allocate for dynamic arrays when resizing const FRAMES_MAX* = {frames_max} # The maximum recursion limit const JAPL_VERSION* = "0.3.0" const JAPL_RELEASE* = "alpha" const DEBUG_TRACE_VM* = {debug_vm} # Traces VM execution const SKIP_STDLIB_INIT* = {skip_stdlib_init} # Skips stdlib initialization in debug mode const DEBUG_TRACE_GC* = {debug_gc} # Traces the garbage collector (TODO) const DEBUG_TRACE_ALLOCATION* = {debug_alloc} # Traces memory allocation/deallocation const DEBUG_TRACE_COMPILER* = {debug_compiler} # Traces the compiler const JAPL_VERSION_STRING* = &"JAPL {{JAPL_VERSION}} ({{JAPL_RELEASE}}, {{CompileDate}} {{CompileTime}})" const HELP_MESSAGE* = """The JAPL runtime interface, Copyright (C) 2020 Mattia Giambirtone This program is free software, see the license distributed with this program or check http://www.apache.org/licenses/LICENSE-2.0 for more info. Basic usage ----------- $ jpl -> Starts the REPL $ jpl filename.jpl -> Runs filename.jpl Command-line options -------------------- -h, --help -> Shows this help text and exit -v, --version -> Prints the JAPL version number and exit -s, --string -> Executes the passed string as if it was a file -i, --interactive -> Enables interactive mode, which opens a REPL session after execution of a file or source string """''' def run_command(command: str, mode: str = "Popen", **kwargs): """ Runs a command with subprocess and returns the process' return code, stderr and stdout """ logging.debug(f"Running '{command}'") if mode == "Popen": process = Popen(shlex.split(command, posix=os.name != "nt"), **kwargs) stdout, stderr = process.communicate() else: process = run(command, **kwargs) stdout, stderr = None, None return stdout, stderr, process.returncode def build(path: str, flags: Optional[Dict[str, str]] = {}, options: Optional[Dict[str, bool]] = {}, override: Optional[bool] = False, skip_tests: Optional[bool] = False, install: Optional[bool] = False, ignore_binary: Optional[bool] = False, verbose: Optional[bool] = False, tests_timeout: Optional[int] = 10): """ Builds the JAPL runtime. This function generates the required configuration according to the user's choice, runs tests and performs installation when possible. :param path: The path to JAPL's main source directory :type path: string, optional :param flags: Any extra compiler flags that will be passed to nim. Keys and values should be strings, but any object with a proper __str__ or __repr__ method is probably fine as long as it's a valid nim option for the compiler. Defaults to {} (no flags) :type flags: dict, optional :param options: Compile-time options such as debugging the compiler or the VM, defaults to {} (use defaults) :type options: dict, optional :param override: Wether to ignore an already existing config.nim file from a previous build or not. Setting this to True will overwrite the configuration file and is useful for when build options have to be tweaked, while setting it to False will not touch the file at all. Defaults to False :type override: bool, optional :param skip_tests: Wether to skip running JATS (just another test suite) or not, defaults to False :type skip_tests: bool, optional :param install: Wether to try to install JAPL in PATH so that it can be invoked with "jpl" as a command instead of running it via the binary directly, defaults to False :type install: bool, optional :param ignore_binary: Wether to ignore (and overwrite) a previous JAPL entry in PATH. The build script will complain if there is a file or folder already named "jpl" in ANY entry in PATH so this option allows to overwrite whatever data is there. Note that JAPL right now isn't aware of what it is replacing so make sure you don't lose any sensitive data! :type ignore_binary: bool, optional :param verbose: This parameter tells the test suite to use verbose logs, defaults to False :type verbose: bool, optional :param tests_timeout: The max. amount of seconds each test in the test suite is allowed to run before being killed for taking too long, defaults to 10 seconds :type tests_timeout: int, optional """ config_path = os.path.join(path, "config.nim") main_path = os.path.join(path, "japl.nim") listing = "\n- {} = {}" if not os.path.exists(path): logging.error(f"Input path '{path}' does not exist") return False if os.path.isfile(config_path) and not override: logging.warning(f"A config file exists at '{config_path}', keeping it") else: logging.warning(f"Overriding config file at '{config_path}'") logging.debug(f"Generating config file at '{config_path}'") try: with open(config_path, "w") as build_config: build_config.write(CONFIG_TEMPLATE.format(**options)) except Exception as fatal: logging.error(f"A fatal unhandled exception occurred -> {type(fatal).__name__}: {fatal}") return False else: logging.debug(f"Config file has been generated, compiling with options as follows: {''.join(listing.format(k, v) for k, v in options.items())}") logging.debug(f"Nim compiler options: {''.join(listing.format(k, v) for k, v in flags.items())}") logging.debug(f"Compiling '{main_path}'") nim_flags = " ".join(f"-{name}:{value}" if len(name) == 1 else f"--{name}:{value}" for name, value in flags.items()) command = "nim {flags} compile {path}".format(flags=nim_flags, path=main_path) logging.info("Compiling JAPL") start = time() _, stderr, status = run_command(command, stdout=DEVNULL, stderr=PIPE) if status != 0: logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}") else: logging.debug(f"Compilation completed in {time() - start:.2f} seconds") logging.info("Build completed") if skip_tests: logging.warning("Skipping test suite") else: logging.info("Running tests under tests/") logging.debug("Compiling test suite") start = time() test_runner_path = "./tests/jatr" if os.name != "nt" else ".\\tests\jatr" tests_path = "./tests/jats" if os.name != "nt" else ".\\tests\jats" command = "nim {flags} compile {path}".format(flags=nim_flags, path=test_runner_path) _, stderr, status = run_command(command, stdout=DEVNULL, stderr=PIPE) if status != 0: logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}") return False command = f"nim compile --opt:speed {tests_path}" _, stderr, status = run_command(command, stdout=DEVNULL, stderr=PIPE) if status != 0: logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}") return False logging.debug(f"Test suite compilation completed in {time() - start:.2f} seconds") logging.debug("Running tests") start = time() # TODO: Find a better way of running the test suite process = run_command(f"{tests_path} {'--stdout' if verbose else ''} --timeout={tests_timeout}", mode="run", shell=True, stderr=PIPE) if status != 0: logging.error(f"Command '{command}' exited with non-0 exit code {status}, output below:\n{stderr.decode()}") return False logging.debug(f"Test suite ran in {time() - start:.2f} seconds") logging.info("Test suite completed!") if args.install: if os.name == "nt": logging.warning("Sorry, but automatically installing JAPL is not yet supported on windows") else: # TODO -> Is PATH defined on all linux distros? logging.info(f"Installing JAPL at PATH") if not ignore_binary and any(os.path.exists(os.path.join(path, "jpl")) for path in os.getenv("PATH").split(":")): logging.error("Could not install JAPL because a binary already exists in PATH") return for path in os.getenv("PATH").split(":"): install_path = os.path.join(path, "jpl") logging.debug(f"Attempting to install JAPL at '{install_path}'") try: shutil.move(main_path.strip(".nim"), install_path) except PermissionError: logging.debug(f"Path '{path}' is not writable, attempting next entry in PATH") except Exception as fatal: logging.error(f"A fatal unhandled exception occurred -> {type(fatal).__name__}: {fatal}") else: logging.debug(f"JAPL installed at '{path}', setting executable permissions") # TODO: Use external oschmod library once we support windows! try: os.chmod(install_path, os.stat(install_path).st_mode | stat.S_IXUSR | stat.S_IXGRP | stat.S_IXOTH) except Exception as fatal: logging.error(f"A fatal unhandled exception occurred -> {type(fatal).__name__}: {fatal}") break return True if __name__ == "__main__": try: parser = argparse.ArgumentParser() parser.add_argument("path", help="The path to JAPL's source directory") parser.add_argument("--verbose", help="Prints debug information to stdout", action="store_true", default=os.getenv("JAPL_VERBOSE")) parser.add_argument("--flags", help="Optional flags to be passed to the nim compiler. Must be a comma-separated list of name:value (without spaces)", default=os.getenv("JAPL_FLAGS")) parser.add_argument("--options", help="Set compile-time options and constants, pass a comma-separated list of name:value (without spaces)." " Note that if a config.nim file already exists in the destination directory, that will override any setting defined here unless --override-config is used", default=os.getenv("JAPL_OPTIONS")) parser.add_argument("--override-config", help="Overrides the setting of an already existing config.nim file in the destination directory", action="store_true", default=os.getenv("JAPL_OVERRIDE_CONFIG")) parser.add_argument("--skip-tests", help="Skips running the JAPL test suite, useful for debug builds", action="store_true", default=os.getenv("JAPL_SKIP_TESTS")) parser.add_argument("--install", help="Tries to move the compiled binary to PATH (this is always disabled on windows)", action="store_true", default=os.environ.get("JAPL_INSTALL")) parser.add_argument("--ignore-binary", help="Ignores an already existing 'jpl' binary in any installation directory and overwrites it, use (with care!) with --install", action="store_true", default=os.getenv("JAPL_IGNORE_BINARY")) parser.add_argument("--profile", help="The path to a json file specifying build options and arguments. Overrides ANY other option!", default=os.environ.get("JAPL_PROFILE")) parser.add_argument("--timeout", help="The max. amount of seconds each test in the test suite is allowed to run before being killed for taking too long, defaults to 10 seconds", default=os.environ.get("JAPL_TIMEOUT", 10), type=int) args = parser.parse_args() flags = { "gc": "refc", } options = { "debug_vm": "false", "skip_stdlib_init": "false", "debug_gc": "false", "debug_compiler": "false", "debug_alloc": "false", "map_load_factor": "0.75", "array_grow_factor": "2", "frames_max": "800", } # We support environment variables for build options too! for key, value in options.items(): if var := os.getenv(f"JAPL_{key.upper()}"): options[key] = var logging.basicConfig(format="[JABT - %(levelname)s - %(asctime)s] %(message)s", datefmt="%T", level=logging.DEBUG if args.verbose else logging.INFO ) logging.info("Just Another Build Tool, version 0.3.5") if args.flags: try: for value in args.flags.split(","): k, v = value.split(":", maxsplit=2) flags[k] = v except Exception: logging.error("Invalid parameter for --flags") exit() if args.options: try: for value in args.options.split(","): k, v = value.split(":", maxsplit=2) if k not in options: logging.error(f"Invalid compile-time option '{k}'") exit() options[k] = v except Exception: logging.error("Invalid parameter for --options") exit() if args.profile: try: with open(args.profile) as profile: skip = 0 line = profile.readline() while line.startswith("//"): # We skip comments and keep track of # where we should reset our buffer skip = profile.tell() line = profile.readline() profile.seek(skip - 1) data = json.load(profile) if "options" in data: for option, value in data["options"].items(): options[option] = value if "flags" in data: for flag, value in data["flags"].items(): flags[flag] = value for arg in {"override_config", "skip_tests", "verbose", "install", "ignore_binary"}: setattr(args, arg, data.get(arg, getattr(args, arg))) except Exception as e: logging.error(f"An error occurred while loading profile '{args.profile}' -> {type(e).__name__}: {e}") exit() else: logging.info(f"Using profile '{args.profile}'") if build(args.path, flags, options, args.override_config, args.skip_tests, args.install, args.ignore_binary, args.verbose, args.timeout): logging.debug("Build tool exited successfully") else: logging.debug("Build tool exited with error") exit(1) except KeyboardInterrupt: logging.info("Interrupted by the user")