2021-01-05 13:02:14 +01:00
#!/usr/bin/env python3
2021-02-26 14:01:12 +01:00
2020-11-02 20:06:52 +01:00
# Copyright 2020 Mattia Giambirtone
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# A minimalistic build script for JAPL
import os
2021-01-14 16:03:31 +01:00
import stat
2021-02-03 20:12:00 +01:00
import json
2020-11-02 20:06:52 +01:00
import shlex
2021-01-14 16:03:31 +01:00
import shutil
2020-11-02 20:06:52 +01:00
import logging
import argparse
from time import time
2021-01-31 10:51:29 +01:00
from typing import Dict , Optional
2021-01-09 18:03:47 +01:00
from subprocess import Popen , PIPE , DEVNULL , run
2020-11-02 20:06:52 +01:00
2020-11-02 21:46:33 +01:00
2020-11-02 20:06:52 +01:00
CONFIG_TEMPLATE = ''' # Copyright 2020 Mattia Giambirtone
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import strformat
2021-01-27 15:51:15 +01:00
const MAP_LOAD_FACTOR * = { map_load_factor } # Load factor for builtin hashmaps
2021-01-31 10:51:29 +01:00
const ARRAY_GROW_FACTOR * = { array_grow_factor } # How much extra memory to allocate for dynamic arrays when resizing
2021-01-10 09:49:35 +01:00
const FRAMES_MAX * = { frames_max } # The maximum recursion limit
2020-11-02 20:06:52 +01:00
const JAPL_VERSION * = " 0.3.0 "
const JAPL_RELEASE * = " alpha "
const DEBUG_TRACE_VM * = { debug_vm } # Traces VM execution
2021-01-16 18:14:22 +01:00
const SKIP_STDLIB_INIT * = { skip_stdlib_init } # Skips stdlib initialization in debug mode
2020-11-02 20:06:52 +01:00
const DEBUG_TRACE_GC * = { debug_gc } # Traces the garbage collector (TODO)
2021-01-31 10:51:29 +01:00
const DEBUG_TRACE_ALLOCATION * = { debug_alloc } # Traces memory allocation/deallocation
2020-11-02 20:06:52 +01:00
const DEBUG_TRACE_COMPILER * = { debug_compiler } # Traces the compiler
const JAPL_VERSION_STRING * = & " JAPL {{ JAPL_VERSION}} ( {{ JAPL_RELEASE}}, {{ CompileDate}} {{ CompileTime}}) "
const HELP_MESSAGE * = """ The JAPL runtime interface, Copyright (C) 2020 Mattia Giambirtone
This program is free software , see the license distributed with this program or check
http : / / www . apache . org / licenses / LICENSE - 2.0 for more info .
Basic usage
- - - - - - - - - - -
2021-01-17 16:54:55 +01:00
$ jpl - > Starts the REPL
2020-11-02 20:06:52 +01:00
2021-01-17 16:54:55 +01:00
$ jpl filename . jpl - > Runs filename . jpl
2020-11-02 20:06:52 +01:00
Command - line options
- - - - - - - - - - - - - - - - - - - -
2021-01-17 16:54:55 +01:00
- h , - - help - > Shows this help text and exit
- v , - - version - > Prints the JAPL version number and exit
- s , - - string - > Executes the passed string as if it was a file
- i , - - interactive - > Enables interactive mode , which opens a REPL session after execution of a file or source string
2020-11-02 20:06:52 +01:00
""" ' ' '
2021-01-14 16:03:31 +01:00
def run_command ( command : str , mode : str = " Popen " , * * kwargs ) :
"""
Runs a command with subprocess and returns the process '
return code , stderr and stdout
"""
2021-02-09 18:49:34 +01:00
logging . debug ( f " Running ' { command } ' " )
2021-01-14 16:03:31 +01:00
if mode == " Popen " :
process = Popen ( shlex . split ( command , posix = os . name != " nt " ) , * * kwargs )
stdout , stderr = process . communicate ( )
else :
2021-01-31 10:51:29 +01:00
process = run ( command , * * kwargs )
2021-01-14 16:03:31 +01:00
stdout , stderr = None , None
return stdout , stderr , process . returncode
2021-01-31 10:51:29 +01:00
def build ( path : str , flags : Optional [ Dict [ str , str ] ] = { } , options : Optional [ Dict [ str , bool ] ] = { } ,
override : Optional [ bool ] = False , skip_tests : Optional [ bool ] = False ,
install : Optional [ bool ] = False , ignore_binary : Optional [ bool ] = False ,
2021-04-18 12:04:24 +02:00
verbose : Optional [ bool ] = False , tests_timeout : Optional [ int ] = 10 ) :
2020-11-02 20:06:52 +01:00
"""
2021-01-31 10:51:29 +01:00
Builds the JAPL runtime .
This function generates the required configuration
2021-04-18 12:04:24 +02:00
according to the user ' s choice, runs tests and
2021-01-31 10:51:29 +01:00
performs installation when possible .
2020-11-02 20:06:52 +01:00
2020-11-02 21:37:57 +01:00
: param path : The path to JAPL ' s main source directory
2020-11-02 20:06:52 +01:00
: type path : string , optional
: param flags : Any extra compiler flags that will
be passed to nim . Keys and values should be strings ,
but any object with a proper __str__ or __repr__
method is probably fine as long as it ' s a valid
nim option for the compiler . Defaults to { } ( no flags )
: type flags : dict , optional
: param options : Compile - time options such as debugging the
compiler or the VM , defaults to { } ( use defaults )
: type options : dict , optional
2021-01-30 13:47:05 +01:00
: param override : Wether to ignore an already existing
config . nim file from a previous build or not . Setting this
to True will overwrite the configuration file and is useful
for when build options have to be tweaked , while setting it
to False will not touch the file at all . Defaults to False
: type override : bool , optional
: param skip_tests : Wether to skip running JATS ( just another
test suite ) or not , defaults to False
: type skip_tests : bool , optional
: param install : Wether to try to install JAPL in PATH so that
it can be invoked with " jpl " as a command instead of running it
via the binary directly , defaults to False
: type install : bool , optional
: param ignore_binary : Wether to ignore ( and overwrite ) a previous
JAPL entry in PATH . The build script will complain if there is a file
or folder already named " jpl " in ANY entry in PATH so this option allows
to overwrite whatever data is there . Note that JAPL right now isn ' t aware
of what it is replacing so make sure you don ' t lose any sensitive data!
2021-01-31 10:51:29 +01:00
: type ignore_binary : bool , optional
: param verbose : This parameter tells the test suite to use verbose logs ,
defaults to False
: type verbose : bool , optional
2021-04-18 12:04:24 +02:00
: param tests_timeout : The max . amount of seconds each test in the test
suite is allowed to run before being killed for taking too long , defaults
to 10 seconds
: type tests_timeout : int , optional
2020-11-02 20:06:52 +01:00
"""
2020-11-02 21:37:57 +01:00
config_path = os . path . join ( path , " config.nim " )
main_path = os . path . join ( path , " japl.nim " )
2021-01-16 18:14:22 +01:00
listing = " \n - {} = {} "
2020-11-02 20:06:52 +01:00
if not os . path . exists ( path ) :
logging . error ( f " Input path ' { path } ' does not exist " )
2021-02-26 14:01:12 +01:00
return False
2020-11-02 21:37:57 +01:00
if os . path . isfile ( config_path ) and not override :
logging . warning ( f " A config file exists at ' { config_path } ' , keeping it " )
2020-11-02 20:06:52 +01:00
else :
2020-12-26 17:01:03 +01:00
logging . warning ( f " Overriding config file at ' { config_path } ' " )
2020-11-02 21:37:57 +01:00
logging . debug ( f " Generating config file at ' { config_path } ' " )
try :
with open ( config_path , " w " ) as build_config :
2021-01-09 18:03:47 +01:00
build_config . write ( CONFIG_TEMPLATE . format ( * * options ) )
2020-11-02 21:37:57 +01:00
except Exception as fatal :
logging . error ( f " A fatal unhandled exception occurred -> { type ( fatal ) . __name__ } : { fatal } " )
2021-02-26 14:01:12 +01:00
return False
2020-11-02 21:37:57 +01:00
else :
2021-01-16 18:14:22 +01:00
logging . debug ( f " Config file has been generated, compiling with options as follows: { ' ' . join ( listing . format ( k , v ) for k , v in options . items ( ) ) } " )
2021-02-03 20:12:00 +01:00
logging . debug ( f " Nim compiler options: { ' ' . join ( listing . format ( k , v ) for k , v in flags . items ( ) ) } " )
2020-11-02 21:37:57 +01:00
logging . debug ( f " Compiling ' { main_path } ' " )
2020-11-02 20:06:52 +01:00
nim_flags = " " . join ( f " - { name } : { value } " if len ( name ) == 1 else f " -- { name } : { value } " for name , value in flags . items ( ) )
2021-01-09 18:03:47 +01:00
command = " nim {flags} compile {path} " . format ( flags = nim_flags , path = main_path )
2020-11-02 20:06:52 +01:00
logging . info ( " Compiling JAPL " )
start = time ( )
2021-01-14 16:03:31 +01:00
_ , stderr , status = run_command ( command , stdout = DEVNULL , stderr = PIPE )
if status != 0 :
logging . error ( f " Command ' { command } ' exited with non-0 exit code { status } , output below: \n { stderr . decode ( ) } " )
2020-11-02 20:06:52 +01:00
else :
logging . debug ( f " Compilation completed in { time ( ) - start : .2f } seconds " )
logging . info ( " Build completed " )
2021-01-09 18:03:47 +01:00
if skip_tests :
logging . warning ( " Skipping test suite " )
else :
logging . info ( " Running tests under tests/ " )
logging . debug ( " Compiling test suite " )
start = time ( )
2022-04-03 22:17:13 +02:00
test_runner_path = " ./tests/jatr " if os . name != " nt " else " . \\ tests \ jatr "
2022-04-03 22:21:03 +02:00
tests_path = " ./tests/jats " if os . name != " nt " else " . \\ tests \ jats "
2021-01-29 18:42:45 +01:00
command = " nim {flags} compile {path} " . format ( flags = nim_flags , path = test_runner_path )
_ , stderr , status = run_command ( command , stdout = DEVNULL , stderr = PIPE )
2021-01-14 16:03:31 +01:00
if status != 0 :
logging . error ( f " Command ' { command } ' exited with non-0 exit code { status } , output below: \n { stderr . decode ( ) } " )
2021-02-26 14:01:12 +01:00
return False
command = f " nim compile --opt:speed { tests_path } "
_ , stderr , status = run_command ( command , stdout = DEVNULL , stderr = PIPE )
if status != 0 :
logging . error ( f " Command ' { command } ' exited with non-0 exit code { status } , output below: \n { stderr . decode ( ) } " )
return False
logging . debug ( f " Test suite compilation completed in { time ( ) - start : .2f } seconds " )
logging . debug ( " Running tests " )
start = time ( )
# TODO: Find a better way of running the test suite
2021-04-18 12:04:24 +02:00
process = run_command ( f " { tests_path } { ' --stdout ' if verbose else ' ' } --timeout= { tests_timeout } " , mode = " run " , shell = True , stderr = PIPE )
2021-02-26 14:01:12 +01:00
if status != 0 :
logging . error ( f " Command ' { command } ' exited with non-0 exit code { status } , output below: \n { stderr . decode ( ) } " )
return False
logging . debug ( f " Test suite ran in { time ( ) - start : .2f } seconds " )
logging . info ( " Test suite completed! " )
2021-01-14 22:37:11 +01:00
if args . install :
if os . name == " nt " :
logging . warning ( " Sorry, but automatically installing JAPL is not yet supported on windows " )
else :
# TODO -> Is PATH defined on all linux distros?
logging . info ( f " Installing JAPL at PATH " )
2021-01-16 18:14:22 +01:00
if not ignore_binary and any ( os . path . exists ( os . path . join ( path , " jpl " ) ) for path in os . getenv ( " PATH " ) . split ( " : " ) ) :
2021-01-14 22:37:11 +01:00
logging . error ( " Could not install JAPL because a binary already exists in PATH " )
return
for path in os . getenv ( " PATH " ) . split ( " : " ) :
install_path = os . path . join ( path , " jpl " )
logging . debug ( f " Attempting to install JAPL at ' { install_path } ' " )
try :
shutil . move ( main_path . strip ( " .nim " ) , install_path )
except PermissionError :
logging . debug ( f " Path ' { path } ' is not writable, attempting next entry in PATH " )
except Exception as fatal :
logging . error ( f " A fatal unhandled exception occurred -> { type ( fatal ) . __name__ } : { fatal } " )
2021-02-26 15:33:51 +01:00
else :
logging . debug ( f " JAPL installed at ' { path } ' , setting executable permissions " )
# TODO: Use external oschmod library once we support windows!
try :
os . chmod ( install_path , os . stat ( install_path ) . st_mode | stat . S_IXUSR | stat . S_IXGRP | stat . S_IXOTH )
except Exception as fatal :
logging . error ( f " A fatal unhandled exception occurred -> { type ( fatal ) . __name__ } : { fatal } " )
break
2021-02-26 14:01:12 +01:00
return True
2020-11-02 20:06:52 +01:00
if __name__ == " __main__ " :
2021-01-17 16:54:55 +01:00
try :
parser = argparse . ArgumentParser ( )
parser . add_argument ( " path " , help = " The path to JAPL ' s source directory " )
parser . add_argument ( " --verbose " , help = " Prints debug information to stdout " , action = " store_true " , default = os . getenv ( " JAPL_VERBOSE " ) )
parser . add_argument ( " --flags " , help = " Optional flags to be passed to the nim compiler. Must be a comma-separated list of name:value (without spaces) " , default = os . getenv ( " JAPL_FLAGS " ) )
parser . add_argument ( " --options " , help = " Set compile-time options and constants, pass a comma-separated list of name:value (without spaces). "
2021-02-08 16:40:21 +01:00
" Note that if a config.nim file already exists in the destination directory, that will override any setting defined here unless --override-config is used " , default = os . getenv ( " JAPL_OPTIONS " ) )
2021-01-17 16:54:55 +01:00
parser . add_argument ( " --override-config " , help = " Overrides the setting of an already existing config.nim file in the destination directory " , action = " store_true " , default = os . getenv ( " JAPL_OVERRIDE_CONFIG " ) )
parser . add_argument ( " --skip-tests " , help = " Skips running the JAPL test suite, useful for debug builds " , action = " store_true " , default = os . getenv ( " JAPL_SKIP_TESTS " ) )
parser . add_argument ( " --install " , help = " Tries to move the compiled binary to PATH (this is always disabled on windows) " , action = " store_true " , default = os . environ . get ( " JAPL_INSTALL " ) )
parser . add_argument ( " --ignore-binary " , help = " Ignores an already existing ' jpl ' binary in any installation directory and overwrites it, use (with care!) with --install " , action = " store_true " , default = os . getenv ( " JAPL_IGNORE_BINARY " ) )
2021-02-03 20:12:00 +01:00
parser . add_argument ( " --profile " , help = " The path to a json file specifying build options and arguments. Overrides ANY other option! " , default = os . environ . get ( " JAPL_PROFILE " ) )
2021-04-18 13:49:19 +02:00
parser . add_argument ( " --timeout " , help = " The max. amount of seconds each test in the test suite is allowed to run before being killed for taking too long, defaults to 10 seconds " , default = os . environ . get ( " JAPL_TIMEOUT " , 10 ) , type = int )
2021-01-17 16:54:55 +01:00
args = parser . parse_args ( )
flags = {
" gc " : " refc " ,
}
options = {
" debug_vm " : " false " ,
" skip_stdlib_init " : " false " ,
" debug_gc " : " false " ,
" debug_compiler " : " false " ,
" debug_alloc " : " false " ,
" map_load_factor " : " 0.75 " ,
" array_grow_factor " : " 2 " ,
" frames_max " : " 800 " ,
}
2021-04-18 12:04:24 +02:00
# We support environment variables for build options too!
2021-01-17 16:54:55 +01:00
for key , value in options . items ( ) :
if var := os . getenv ( f " JAPL_ { key . upper ( ) } " ) :
options [ key ] = var
2021-01-30 13:47:05 +01:00
logging . basicConfig ( format = " [JABT - %(levelname)s - %(asctime)s ] %(message)s " ,
2021-01-17 16:54:55 +01:00
datefmt = " % T " ,
level = logging . DEBUG if args . verbose else logging . INFO
)
2021-04-18 12:04:24 +02:00
logging . info ( " Just Another Build Tool, version 0.3.5 " )
2021-01-17 16:54:55 +01:00
if args . flags :
try :
for value in args . flags . split ( " , " ) :
k , v = value . split ( " : " , maxsplit = 2 )
flags [ k ] = v
except Exception :
logging . error ( " Invalid parameter for --flags " )
exit ( )
if args . options :
2021-01-14 16:03:31 +01:00
try :
2021-01-17 16:54:55 +01:00
for value in args . options . split ( " , " ) :
k , v = value . split ( " : " , maxsplit = 2 )
if k not in options :
2021-01-31 10:51:29 +01:00
logging . error ( f " Invalid compile-time option ' { k } ' " )
2021-01-17 16:54:55 +01:00
exit ( )
options [ k ] = v
except Exception :
logging . error ( " Invalid parameter for --options " )
exit ( )
2021-02-03 20:12:00 +01:00
if args . profile :
try :
with open ( args . profile ) as profile :
skip = 0
line = profile . readline ( )
while line . startswith ( " // " ) :
# We skip comments and keep track of
# where we should reset our buffer
skip = profile . tell ( )
line = profile . readline ( )
profile . seek ( skip - 1 )
data = json . load ( profile )
if " options " in data :
for option , value in data [ " options " ] . items ( ) :
options [ option ] = value
if " flags " in data :
for flag , value in data [ " flags " ] . items ( ) :
flags [ flag ] = value
for arg in { " override_config " , " skip_tests " , " verbose " , " install " , " ignore_binary " } :
setattr ( args , arg , data . get ( arg , getattr ( args , arg ) ) )
except Exception as e :
logging . error ( f " An error occurred while loading profile ' { args . profile } ' -> { type ( e ) . __name__ } : { e } " )
exit ( )
else :
logging . info ( f " Using profile ' { args . profile } ' " )
2021-02-26 14:01:12 +01:00
if build ( args . path ,
2021-01-31 10:51:29 +01:00
flags ,
options ,
args . override_config ,
args . skip_tests ,
args . install ,
args . ignore_binary ,
2021-04-18 12:04:24 +02:00
args . verbose ,
args . timeout ) :
2021-02-26 14:01:12 +01:00
logging . debug ( " Build tool exited successfully " )
else :
2021-02-26 14:04:38 +01:00
logging . debug ( " Build tool exited with error " )
exit ( 1 )
2021-01-17 16:54:55 +01:00
except KeyboardInterrupt :
2021-01-29 18:42:45 +01:00
logging . info ( " Interrupted by the user " )