run.py

Go to the documentation of this file.

# Copyright (C) 2023 Ethan Uppal. All rights reserved.
# run.py: Runs tests for libpulsar.
 
from sys import exit, stdout
import subprocess
import os
 
tests_passed = 0
REDO_LINE = "\033[2K\r"
SRCDIR = './src'
LIBDIR = './lib'
REPORTS_DIR = './tests/reports/'
OUT = './tests/bin/a.out'
 
# runs a command and returns the output, error, and exit code
def cmd(args):
    proc = subprocess.Popen(args,
        stdout=subprocess.PIPE,
        stderr=subprocess.PIPE
    )
    out, err = proc.communicate()
    return out, err, proc.returncode
 
# rebuilds the library and clears reports
def rebuild():
    print("+ \033[32;1mclearing reports\033[m")
    for report in os.listdir(REPORTS_DIR):
        if report.endswith(".txt"):
            os.remove(REPORTS_DIR + report)
    stdout.write("* \033[33;1mrebuilding library\033[m")
    stdout.flush()
    out, err, code = cmd(['make', 'clean_code'])
    if code != 0:
        print(err.decode('utf-8')) 
        return False
    out, err, code = cmd(['make', 'static'])
    if code != 0:
        print(err.decode('utf-8')) 
        return False
    print(REDO_LINE + "+ \033[32;1mrebuilt library\033[m")
    return True
 
# indicates that a test is starting
def print_start_test(name):
    stdout.write(f"* \033[33;1mrunning test: {name}\033[m")
    stdout.flush()
 
# compiles the given filename as an executable linked with libpulsar
def create_exec(filename):
    _, err, code = cmd([
        '/usr/bin/gcc', filename,
            '-L', LIBDIR,
            '-l', 'pulsar',
            '-I', SRCDIR,
            '-D', 'MAIN',
            '-o', OUT
    ])
    if code != 0:
        print(f"\n  \033[31;1mcompiling failed with exit code {code}:\033[m")
        print(err.decode('ascii'))
        exit(code)
    return err, code
 
def run_exec(args = []):
    return cmd([
        OUT
    ] + args)
 
# runs an output-comparison test
def run_output_test(case):
    global tests_passed
    TEST_PRGS = './working_programs/test/'
    TEST_DATA = './tests/data/'
    SRCDIR = './src'
    LIBDIR = './lib'
    OUT = './tests/bin/a.out'
 
    # print test name
    filename = case[0]
    casename = case[1]
    note = ''
    if len(case) >= 3:
        note = f" (note: {case[2]})"
    print_start_test(casename)
 
    # run test
    INPUT_FILE = TEST_DATA + casename + '.in'
    OUTPUT_FILE = TEST_DATA + casename + '.out'
    create_exec(TEST_PRGS + filename)    
    out, err, _ = run_exec([INPUT_FILE] + case[3:]);
    
    # compare output and print test status
    with open(OUTPUT_FILE, 'r') as f:
        stdout.write(REDO_LINE)
        act = out.decode('ascii')
        exp = f.read().replace('dev.plsr', INPUT_FILE)
        if act != exp:
            print(f"- \033[31;1mtest '{casename}' failed{note}\033[m")
            print(out.decode('ascii'))
            print(err.decode('ascii'))
        else:
            print(f"+ \033[32;1mtest '{casename}' passed{note}\033[m")
            tests_passed += 1
 
# runs a unit test
def run_unit_test(filename):
    global tests_passed
    TEST_PRGS = './tests/unit/'
 
    # print test name
    name = filename.split('.')[0]
    print_start_test(name)
 
    # run test
    err, code = create_exec(TEST_PRGS + filename)
    out, err, code = run_exec()
 
    # print test status
    stdout.write(REDO_LINE)
    if code != 0:
        print(f"- \033[31;1mtest '{name}' failed\033[m")
        print(out.decode('ascii'))
        print(err.decode('ascii'))
    else:
        print(f"+ \033[32;1mtest '{name}' passed\033[m")
        tests_passed += 1
 
# prints a summary of the tests
def print_summary(count):
    print("\nSummary:")
    if tests_passed == 0:
        print(f"- \033[31;1mNone of the {count} tests passed\033[m")
        exit(1)
    elif tests_passed < count / 2:
        print(f"- \033[31;1mLess than half of the tests ({tests_passed}/{count}) passed\033[m")
        exit(1)
    elif tests_passed < count:
        print(f"~ \033[33;1mMost of the tests ({tests_passed}/{count}) passed\033[m");
        exit(1)
    else:
        print(f"+ \033[32;1mAll {count} tests passed!\033[m");
 
def main():
    # setup for new round of tests
    if not rebuild():
        return
 
    # tests that compare output of file
    output_tests = [
        ['lex.c', 'lexer0', 'empty'],
        ['lex.c', 'lexer1', 'string interp'],
        ['lex.c', 'lexer2', 'string interp'],
        ['lex.c', 'lexer3', 'string interp'],
        ['lex.c', 'everytoken'],
        ['hashmap.c', 'hash1'],
        # ['hashmap2.c', 'hash2'],
        ['parse.c', 'parse1', 'basic expr lits'],
        ['parse.c', 'parse2', 'let statements (no types)'],
        ['parse.c', 'parse3', 'string interp'],
        ['parse.c', 'parse4', 'precedence/associativity'],
        ['parse.c', 'parse5', 'functions/blocks'],
        ['parse.c', 'parse6', 'while/if/return'],
        ['parse.c', 'parse7', 'extern'],
        ['parse.c', 'parse8', 'calls/qualifiers'],
    ]
    # unit tests (think assertions)
    unit_tests = os.listdir('./tests/unit')
    unit_tests.remove('TEMPLATE.c')
    for case in output_tests:
        run_output_test(case)
    for case in unit_tests:
        run_unit_test(case)
    cmd(['rm', OUT])
    print_summary(len(output_tests) + len(unit_tests))
 
if __name__ == "__main__":
    main()