#! /usr/bin/env python3

import os

import subprocess

import sys

import platform

import argparse

import re

from glob import glob

# Tests require at least CPython 3.3. If your default python3 executable

# is of lower version, you can point MICROPY_CPYTHON3 environment var

# to the correct executable.

if os.name == 'nt':

CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3.exe')

MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/windows/micropython.exe')

else:

CPYTHON3 = os.getenv('MICROPY_CPYTHON3', 'python3')

MICROPYTHON = os.getenv('MICROPY_MICROPYTHON', '../ports/unix/micropython')

# mpy-cross is only needed if --via-mpy command-line arg is passed

MPYCROSS = os.getenv('MICROPY_MPYCROSS', '../mpy-cross/mpy-cross')

# Set PYTHONIOENCODING so that CPython will use utf-8 on systems which set another encoding in the locale

os.environ['PYTHONIOENCODING'] = 'utf-8'

def rm_f(fname):

if os.path.exists(fname):

os.remove(fname)

# unescape wanted regex chars and escape unwanted ones

def convert_regex_escapes(line):

cs = []

escape = False

for c in str(line, 'utf8'):

if escape:

escape = False

cs.append(c)

elif c == '\\':

escape = True

elif c in ('(', ')', '[', ']', '{', '}', '.', '*', '+', '^', '$'):

cs.append('\\' + c)

else:

cs.append(c)

# accept carriage-return(s) before final newline

if cs[-1] == '\n':

cs[-1] = '\r*\n'

return bytes(''.join(cs), 'utf8')

def run_micropython(pyb, args, test_file, is_special=False):

special_tests = ('micropython/meminfo.py', 'basics/bytes_compare3.py', 'basics/builtin_help.py', 'thread/thread_exc2.py')

if pyb is None:

# run on PC

if test_file.startswith(('cmdline/', 'feature_check/')) or test_file in special_tests:

# special handling for tests of the unix cmdline program

is_special = True

if is_special:

# check for any cmdline options needed for this test

args = [MICROPYTHON]

with open(test_file, 'rb') as f:

line = f.readline()

if line.startswith(b'# cmdline:'):

# subprocess.check_output on Windows only accepts strings, not bytes

args += [str(c, 'utf-8') for c in line[10:].strip().split()]

# run the test, possibly with redirected input

try:

if 'repl_' in test_file:

# Need to use a PTY to test command line editing

try:

import pty

except ImportError:

# in case pty module is not available, like on Windows

return b'SKIP\n'

import select

def get(required=False):

rv = b''

while True:

ready = select.select([master], [], [], 0.02)

if ready[0] == [master]:

rv += os.read(master, 1024)

else:

if not required or rv:

return rv

def send_get(what):

os.write(master, what)

return get()

with open(test_file, 'rb') as f:

# instead of: output_mupy = subprocess.check_output(args, stdin=f)

master, slave = pty.openpty()

p = subprocess.Popen(args, stdin=slave, stdout=slave,

stderr=subprocess.STDOUT, bufsize=0)

banner = get(True)

output_mupy = banner + b''.join(send_get(line) for line in f)

send_get(b'\x04') # exit the REPL, so coverage info is saved

p.kill()

os.close(master)

os.close(slave)

else:

output_mupy = subprocess.check_output(args + [test_file], stderr=subprocess.STDOUT)

except subprocess.CalledProcessError:

return b'CRASH'

else:

# a standard test run on PC

# create system command

cmdlist = [MICROPYTHON, '-X', 'emit=' + args.emit]

if args.heapsize is not None:

cmdlist.extend(['-X', 'heapsize=' + args.heapsize])

# if running via .mpy, first compile the .py file

if args.via_mpy:

subprocess.check_output([MPYCROSS, '-mcache-lookup-bc', '-o', 'mpytest.mpy', test_file])

cmdlist.extend(['-m', 'mpytest'])

else:

cmdlist.append(test_file)

# run the actual test

e = {"MICROPYPATH": os.getcwd() + ":", "LANG": "en_US.UTF-8"}

p = subprocess.Popen(cmdlist, env=e, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)

output_mupy = b''

while p.poll() is None:

output_mupy += p.stdout.read()

output_mupy += p.stdout.read()

if p.returncode != 0:

output_mupy = b'CRASH'

# clean up if we had an intermediate .mpy file

if args.via_mpy:

rm_f('mpytest.mpy')

else:

# run on pyboard

import pyboard

pyb.enter_raw_repl()

try:

output_mupy = pyb.execfile(test_file)

except pyboard.PyboardError:

output_mupy = b'CRASH'

# canonical form for all ports/platforms is to use \n for end-of-line

output_mupy = output_mupy.replace(b'\r\n', b'\n')

# don't try to convert the output if we should skip this test

if output_mupy in (b'SKIP\n', b'CRASH'):

return output_mupy

if is_special or test_file in special_tests:

# convert parts of the output that are not stable across runs

with open(test_file + '.exp', 'rb') as f:

lines_exp = []

for line in f.readlines():

if line == b'########\n':

line = (line,)

else:

line = (line, re.compile(convert_regex_escapes(line)))

lines_exp.append(line)

lines_mupy = [line + b'\n' for line in output_mupy.split(b'\n')]

if output_mupy.endswith(b'\n'):

lines_mupy = lines_mupy[:-1] # remove erroneous last empty line

i_mupy = 0

for i in range(len(lines_exp)):

if lines_exp[i][0] == b'########\n':

# 8x #'s means match 0 or more whole lines

line_exp = lines_exp[i + 1]

skip = 0

while i_mupy + skip < len(lines_mupy) and not line_exp[1].match(lines_mupy[i_mupy + skip]):

skip += 1

if i_mupy + skip >= len(lines_mupy):

lines_mupy[i_mupy] = b'######## FAIL\n'

break

del lines_mupy[i_mupy:i_mupy + skip]

lines_mupy.insert(i_mupy, b'########\n')

i_mupy += 1

else:

# a regex

if lines_exp[i][1].match(lines_mupy[i_mupy]):

lines_mupy[i_mupy] = lines_exp[i][0]

else:

#print("don't match: %r %s" % (lines_exp[i][1], lines_mupy[i_mupy])) # DEBUG

pass

i_mupy += 1

if i_mupy >= len(lines_mupy):

break

output_mupy = b''.join(lines_mupy)

return output_mupy

def run_feature_check(pyb, args, base_path, test_file):

return run_micropython(pyb, args, base_path + "/feature_check/" + test_file, is_special=True)

def run_tests(pyb, tests, args, base_path="."):

test_count = 0

testcase_count = 0

passed_count = 0

failed_tests = []

skipped_tests = []

skip_tests = set()

skip_native = False

skip_int_big = False

skip_set_type = False

skip_async = False

skip_const = False

skip_revops = False

# Check if micropython.native is supported, and skip such tests if it's not

output = run_feature_check(pyb, args, base_path, 'native_check.py')

if output == b'CRASH':

skip_native = True

# Check if arbitrary-precision integers are supported, and skip such tests if it's not

output = run_feature_check(pyb, args, base_path, 'int_big.py')

if output != b'1000000000000000000000000000000000000000000000\n':

skip_int_big = True

# Check if set type (and set literals) is supported, and skip such tests if it's not

output = run_feature_check(pyb, args, base_path, 'set_check.py')

if output == b'CRASH':

skip_set_type = True

# Check if async/await keywords are supported, and skip such tests if it's not

output = run_feature_check(pyb, args, base_path, 'async_check.py')

if output == b'CRASH':

skip_async = True

# Check if const keyword (MicroPython extension) is supported, and skip such tests if it's not

output = run_feature_check(pyb, args, base_path, 'const.py')

if output == b'CRASH':

skip_const = True

# Check if __rOP__ special methods are supported, and skip such tests if it's not

output = run_feature_check(pyb, args, base_path, 'reverse_ops.py')

if output == b'TypeError\n':

skip_revops = True

# Check if emacs repl is supported, and skip such tests if it's not

t = run_feature_check(pyb, args, base_path, 'repl_emacs_check.py')

if not 'True' in str(t, 'ascii'):

skip_tests.add('cmdline/repl_emacs_keys.py')

upy_byteorder = run_feature_check(pyb, args, base_path, 'byteorder.py')

upy_float_precision = int(run_feature_check(pyb, args, base_path, 'float.py'))

has_complex = run_feature_check(pyb, args, base_path, 'complex.py') == b'complex\n'

has_coverage = run_feature_check(pyb, args, base_path, 'coverage.py') == b'coverage\n'

cpy_byteorder = subprocess.check_output([CPYTHON3, base_path + '/feature_check/byteorder.py'])

skip_endian = (upy_byteorder != cpy_byteorder)

# Some tests shouldn't be run under Travis CI

if os.getenv('TRAVIS') == 'true':

skip_tests.add('basics/memoryerror.py')

skip_tests.add('thread/thread_gc1.py') # has reliability issues

skip_tests.add('thread/thread_lock4.py') # has reliability issues

skip_tests.add('thread/stress_heap.py') # has reliability issues

skip_tests.add('thread/stress_recurse.py') # has reliability issues

if upy_float_precision == 0:

skip_tests.add('extmod/ujson_dumps_float.py')

skip_tests.add('extmod/ujson_loads_float.py')

skip_tests.add('misc/rge_sm.py')

if upy_float_precision < 32:

skip_tests.add('float/float2int_intbig.py') # requires fp32, there's float2int_fp30_intbig.py instead

skip_tests.add('float/string_format.py') # requires fp32, there's string_format_fp30.py instead

skip_tests.add('float/bytes_construct.py') # requires fp32

skip_tests.add('float/bytearray_construct.py') # requires fp32

if upy_float_precision < 64:

skip_tests.add('float/float_divmod.py') # tested by float/float_divmod_relaxed.py instead

skip_tests.add('float/float2int_doubleprec_intbig.py')

if not has_complex:

skip_tests.add('float/complex1.py')

skip_tests.add('float/complex1_intbig.py')

skip_tests.add('float/int_big_float.py')

skip_tests.add('float/true_value.py')

skip_tests.add('float/types.py')

if not has_coverage:

skip_tests.add('cmdline/cmd_parsetree.py')

# Some tests shouldn't be run on a PC

if pyb is None:

# unix build does not have the GIL so can't run thread mutation tests

for t in tests:

if t.startswith('thread/mutate_'):

skip_tests.add(t)

# Some tests shouldn't be run on pyboard

if pyb is not None:

skip_tests.add('basics/exception_chain.py') # warning is not printed

skip_tests.add('micropython/meminfo.py') # output is very different to PC output

skip_tests.add('extmod/machine_mem.py') # raw memory access not supported

if args.target == 'wipy':

skip_tests.add('misc/print_exception.py') # requires error reporting full

skip_tests.add('misc/recursion.py') # requires stack checking enabled

skip_tests.add('misc/recursive_data.py') # requires stack checking enabled

skip_tests.add('misc/recursive_iternext.py') # requires stack checking enabled

skip_tests.update({'extmod/uctypes_%s.py' % t for t in 'bytearray le native_le ptr_le ptr_native_le sizeof sizeof_native array_assign_le array_assign_native_le'.split()}) # requires uctypes

skip_tests.add('extmod/zlibd_decompress.py') # requires zlib

skip_tests.add('extmod/uheapq1.py') # uheapq not supported by WiPy

skip_tests.add('extmod/urandom_basic.py') # requires urandom

skip_tests.add('extmod/urandom_extra.py') # requires urandom

elif args.target == 'esp8266':

skip_tests.add('misc/rge_sm.py') # too large

elif args.target == 'minimal':

skip_tests.add('basics/class_inplace_op.py') # all special methods not supported

skip_tests.add('misc/rge_sm.py') # too large

skip_tests.add('micropython/opt_level.py') # don't assume line numbers are stored

# Some tests are known to fail on 64-bit machines

if pyb is None and platform.architecture()[0] == '64bit':

pass

# Some tests use unsupported features on Windows

if os.name == 'nt':

skip_tests.add('import/import_file.py') # works but CPython prints forward slashes

# Some tests are known to fail with native emitter

# Remove them from the below when they work

if args.emit == 'native':

skip_tests.update({'basics/%s.py' % t for t in 'gen_yield_from gen_yield_from_close gen_yield_from_ducktype gen_yield_from_exc gen_yield_from_iter gen_yield_from_send gen_yield_from_stopped gen_yield_from_throw gen_yield_from_throw2 gen_yield_from_throw3 generator1 generator2 generator_args generator_close generator_closure generator_exc generator_return generator_send'.split()}) # require yield

skip_tests.update({'basics/%s.py' % t for t in 'bytes_gen class_store_class globals_del string_join'.split()}) # require yield

skip_tests.update({'basics/async_%s.py' % t for t in 'def await await2 for for2 with with2'.split()}) # require yield

skip_tests.update({'basics/%s.py' % t for t in 'try_reraise try_reraise2'.split()}) # require raise_varargs

skip_tests.update({'basics/%s.py' % t for t in 'with_break with_continue with_return'.split()}) # require complete with support

skip_tests.add('basics/array_construct2.py') # requires generators

skip_tests.add('basics/bool1.py') # seems to randomly fail

skip_tests.add('basics/builtin_hash_gen.py') # requires yield

skip_tests.add('basics/class_bind_self.py') # requires yield

skip_tests.add('basics/del_deref.py') # requires checking for unbound local

skip_tests.add('basics/del_local.py') # requires checking for unbound local

skip_tests.add('basics/exception_chain.py') # raise from is not supported

skip_tests.add('basics/for_range.py') # requires yield_value

skip_tests.add('basics/try_finally_loops.py') # requires proper try finally code

skip_tests.add('basics/try_finally_return.py') # requires proper try finally code

skip_tests.add('basics/try_finally_return2.py') # requires proper try finally code

skip_tests.add('basics/unboundlocal.py') # requires checking for unbound local

skip_tests.add('import/gen_context.py') # requires yield_value

skip_tests.add('misc/features.py') # requires raise_varargs

skip_tests.add('misc/rge_sm.py') # requires yield

skip_tests.add('misc/print_exception.py') # because native doesn't have proper traceback info

skip_tests.add('misc/sys_exc_info.py') # sys.exc_info() is not supported for native

skip_tests.add('micropython/emg_exc.py') # because native doesn't have proper traceback info

skip_tests.add('micropython/heapalloc_traceback.py') # because native doesn't have proper traceback info

skip_tests.add('micropython/heapalloc_iter.py') # requires generators

skip_tests.add('micropython/schedule.py') # native code doesn't check pending events

for test_file in tests:

test_file = test_file.replace('\\', '/')

test_basename = os.path.basename(test_file)

test_name = os.path.splitext(test_basename)[0]

is_native = test_name.startswith("native_") or test_name.startswith("viper_")

is_endian = test_name.endswith("_endian")

is_int_big = test_name.startswith("int_big") or test_name.endswith("_intbig")

is_set_type = test_name.startswith("set_") or test_name.startswith("frozenset")

is_async = test_name.startswith("async_")

is_const = test_name.startswith("const")

skip_it = test_file in skip_tests

skip_it |= skip_native and is_native

skip_it |= skip_endian and is_endian

skip_it |= skip_int_big and is_int_big

skip_it |= skip_set_type and is_set_type

skip_it |= skip_async and is_async

skip_it |= skip_const and is_const

skip_it |= skip_revops and test_name.startswith("class_reverse_op")

if skip_it:

print("skip ", test_file)

skipped_tests.append(test_name)

continue

# get expected output

test_file_expected = test_file + '.exp'

if os.path.isfile(test_file_expected):

# expected output given by a file, so read that in

with open(test_file_expected, 'rb') as f:

output_expected = f.read()

else:

# run CPython to work out expected output

e = {"PYTHONPATH": os.getcwd(),

"PATH": os.environ["PATH"],

"LANG": "en_US.UTF-8"}

p = subprocess.Popen([CPYTHON3, '-B', test_file], env=e, stdout=subprocess.PIPE)

output_expected = b''

while p.poll() is None:

output_expected += p.stdout.read()

output_expected += p.stdout.read()

if p.returncode != 0:

output_expected = b'CPYTHON3 CRASH'

elif args.write_exp:

with open(test_file_expected, 'wb') as f:

f.write(output_expected)

# canonical form for all host platforms is to use \n for end-of-line

output_expected = output_expected.replace(b'\r\n', b'\n')

if args.write_exp:

continue

# run MicroPython

output_mupy = run_micropython(pyb, args, test_file)

if output_mupy == b'SKIP\n':

print("skip ", test_file)

skipped_tests.append(test_name)

continue

testcase_count += len(output_expected.splitlines())

filename_expected = test_basename + ".exp"

filename_mupy = test_basename + ".out"

if output_expected == output_mupy:

print("pass ", test_file)

passed_count += 1

rm_f(filename_expected)

rm_f(filename_mupy)

else:

with open(filename_expected, "wb") as f:

f.write(output_expected)

with open(filename_mupy, "wb") as f:

f.write(output_mupy)

print("### Expected")

print(output_expected)

print("### Actual")

print(output_mupy)

print("FAIL ", test_file)

failed_tests.append(test_name)

test_count += 1

print("{} tests performed ({} individual testcases)".format(test_count, testcase_count))

print("{} tests passed".format(passed_count))

if len(skipped_tests) > 0:

print("{} tests skipped: {}".format(len(skipped_tests), ' '.join(skipped_tests)))

if len(failed_tests) > 0:

print("{} tests failed: {}".format(len(failed_tests), ' '.join(failed_tests)))

return False

# all tests succeeded

return True

def main():

cmd_parser = argparse.ArgumentParser(description='Run tests for MicroPython.')

cmd_parser.add_argument('--target', default='unix', help='the target platform')

cmd_parser.add_argument('--device', default='/dev/ttyACM0', help='the serial device or the IP address of the pyboard')

cmd_parser.add_argument('-b', '--baudrate', default=115200, help='the baud rate of the serial device')

cmd_parser.add_argument('-u', '--user', default='micro', help='the telnet login username')

cmd_parser.add_argument('-p', '--password', default='python', help='the telnet login password')

cmd_parser.add_argument('-d', '--test-dirs', nargs='*', help='input test directories (if no files given)')

cmd_parser.add_argument('--write-exp', action='store_true', help='save .exp files to run tests w/o CPython')

cmd_parser.add_argument('--emit', default='bytecode', help='MicroPython emitter to use (bytecode or native)')

cmd_parser.add_argument('--heapsize', help='heapsize to use (use default if not specified)')

cmd_parser.add_argument('--via-mpy', action='store_true', help='compile .py files to .mpy first')

cmd_parser.add_argument('--keep-path', action='store_true', help='do not clear MICROPYPATH when running tests')

cmd_parser.add_argument('files', nargs='*', help='input test files')

args = cmd_parser.parse_args()

EXTERNAL_TARGETS = ('pyboard', 'wipy', 'esp8266', 'minimal')

if args.target in EXTERNAL_TARGETS:

import pyboard

pyb = pyboard.Pyboard(args.device, args.baudrate, args.user, args.password)

pyb.enter_raw_repl()

elif args.target == 'unix':

pyb = None

else:

raise ValueError('target must be either %s or unix' % ", ".join(EXTERNAL_TARGETS))

if len(args.files) == 0:

if args.test_dirs is None:

if args.target == 'pyboard':

# run pyboard tests

test_dirs = ('basics', 'micropython', 'float', 'misc', 'stress', 'extmod', 'pyb', 'pybnative', 'inlineasm')

elif args.target in ('esp8266', 'minimal'):

test_dirs = ('basics', 'micropython', 'float', 'misc', 'extmod')

elif args.target == 'wipy':

# run WiPy tests

test_dirs = ('basics', 'micropython', 'misc', 'extmod', 'wipy')

else:

# run PC tests

test_dirs = ('basics', 'micropython', 'float', 'import', 'io', 'misc', 'stress', 'unicode', 'extmod', 'unix', 'cmdline')

else:

# run tests from these directories

test_dirs = args.test_dirs

tests = sorted(test_file for test_files in (glob('{}/*.py'.format(dir)) for dir in test_dirs) for test_file in test_files)

else:

# tests explicitly given

tests = args.files

if not args.keep_path:

# clear search path to make sure tests use only builtin modules

os.environ['MICROPYPATH'] = ''

# Even if we run completely different tests in a different directory,

# we need to access feature_check's from the same directory as the

# run-tests script itself.

base_path = os.path.dirname(sys.argv[0]) or "."

try:

res = run_tests(pyb, tests, args, base_path)

finally:

if pyb:

pyb.close()

if not res:

sys.exit(1)

if __name__ == "__main__":

main()