From 26b467235b7a5dc806ab9fcce6fb30ba05f4acd9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 22 Oct 2014 13:41:48 -0600 Subject: [PATCH 001/909] added IFn and a bit of work on ffi --- pixie/vm/interpreter.py | 2 +- pixie/vm/libs/ffi.py | 22 ++++++++++++++++++++++ pixie/vm/object.py | 7 ++++++- pixie/vm/rt.py | 1 + pixie/vm/stdlib.py | 9 ++++++++- 5 files changed, 38 insertions(+), 3 deletions(-) create mode 100644 pixie/vm/libs/ffi.py diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index ec102630..ee4933e3 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -154,7 +154,7 @@ def interpret(code_obj, args=[]): argc = frame.get_inst() fn = frame.nth(argc - 1) - assert isinstance(fn, code.BaseCode), "Expected callable, got " + str(fn) + #assert isinstance(fn, code.BaseCode), "Expected callable, got " + str(fn) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py new file mode 100644 index 00000000..521e545f --- /dev/null +++ b/pixie/vm/libs/ffi.py @@ -0,0 +1,22 @@ +import rpython.rlib.rdynload as dynload +import pixie.vm.object as object +import pixie.vm.code as code +import pixie.vm.rt as rt +from rpython.rtyper.lltypesystem import rffi, lltype + +class ExternalLib(object.Object): + _type = object.Type("pixie.stdlib.ExternalLib") + + def __init__(self, nm): + self._dyn_lib = dynload.dlopen(nm) + +import ctypes +from rpython.rlib.clibffi import get_libc_name +print get_libc_name() + +s = rffi.str2charp("/usr/lib/libc.dylib") +lib = dynload.dlopen(s) +rffi.free_charp(s) + +#LIBC = ExternalLib(rffi.str2charp("/usr/lib/libc.dynlib")) + diff --git a/pixie/vm/object.py b/pixie/vm/object.py index bc682d22..68547be1 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -1,4 +1,4 @@ - +import rpython.rlib.jit as jit class Object(object): """ Base Object for all VM objects @@ -7,6 +7,11 @@ class Object(object): def type(self): affirm(False, u".type isn't overloaded") + @jit.unroll_safe + def invoke(self, args): + import pixie.vm.stdlib as stdlib + return stdlib.invoke_other(self, args) + class TypeRegistry(object): def __init__(self): self._types = {} diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 3259f9ac..c3191ae4 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -60,6 +60,7 @@ def wrapper(*args): import pixie.vm.compiler as compiler import pixie.vm.map_entry import pixie.vm.reader as reader + #import pixie.vm.libs.ffi diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index fc835e86..8b347084 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -39,6 +39,8 @@ defprotocol("pixie.stdlib", "IStack", ["-push", "-pop"]) +defprotocol("pixie.stdlib", "IFn", ["-invoke"]) + IVector = as_var("pixie.stdlib", "IVector")(Protocol(u"IVector")) IMap = as_var("pixie.stdlib", "IMap")(Protocol(u"IMap")) @@ -383,4 +385,9 @@ def push_binding_frame(): @as_var("pop-binding-frame!") def pop_binding_frame(): code._dynamic_vars.pop_binding_frame() - return nil \ No newline at end of file + return nil + + +def invoke_other(obj, args): + from pixie.vm.array import array + return rt.apply(_invoke, obj, array(args)) \ No newline at end of file From d7a76bd0b287bfecf711b8cfc9037361d979c008 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 23 Oct 2014 16:10:29 -0600 Subject: [PATCH 002/909] some ffi stuff --- pixie/stdlib.lisp | 8 ---- pixie/vm/atom.py | 1 + pixie/vm/code.py | 36 ++++++++++++++- pixie/vm/interpreter.py | 2 +- pixie/vm/keyword.py | 13 +++++- pixie/vm/libs/ffi.py | 100 ++++++++++++++++++++++++++++++++++++---- pixie/vm/numbers.py | 4 ++ pixie/vm/object.py | 3 ++ pixie/vm/stdlib.py | 15 ++++-- pixie/vm/symbol.py | 12 +++++ 10 files changed, 170 insertions(+), 24 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 3bb90944..ccffef47 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -319,14 +319,6 @@ (fn [v] (transduce cat unordered-hash-reducing-fn v))) -(extend -hash Symbol - (fn [v] - (hash [(namespace v) (name v)]))) - -(extend -hash Keyword - (fn [v] - (hash [(namespace v) (name v)]))) - (extend -str Keyword (fn [k] (if (namespace k) diff --git a/pixie/vm/atom.py b/pixie/vm/atom.py index 80a1ab9a..46223da3 100644 --- a/pixie/vm/atom.py +++ b/pixie/vm/atom.py @@ -3,6 +3,7 @@ from pixie.vm.code import extend, as_var from pixie.vm.primitives import nil import pixie.vm.stdlib as proto +import rpython.rlib.jit as jit import pixie.vm.stdlib as proto diff --git a/pixie/vm/code.py b/pixie/vm/code.py index e3f9545f..adb44d88 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -603,6 +603,41 @@ def _invoke(self, args): fn = self.get_fn(a, b, self._rev) return fn.invoke(args) + + +class ElidableFn(object.Object): + _type = object.Type(u"pixie.stdlib.ElidableFn") + __immutable_fields__ = ["_boxed_fn"] + def type(self): + return ElidableFn._type + + def __init__(self, boxed_fn): + self._boxed_fn = boxed_fn + + @elidable + def _elidable_invoke_0(self, fn): + return self._boxed_fn.invoke([]) + + @elidable + def _elidable_invoke_1(self, fn, arg0): + return self._boxed_fn.invoke([arg0]) + + @elidable + def _elidable_invoke_2(self, fn, arg0, arg1): + return self._boxed_fn.invoke([arg0, arg1]) + + + def invoke(self, args): + largs = jit.promote(len(args)) + fn = self._boxed_fn.promote() + if largs == 0: + return self._elidable_invoke_0(fn).promote() + elif largs == 1: + return self._elidable_invoke_1(fn, args[0].promote()).promote() + elif largs == 2: + return self._elidable_invoke_2(fn, args[0].promote(), args[1].promote()).promote() + affirm(False, u"Too many args to Elidable Fn") + def munge(s): return s.replace("-", "_").replace("?", "_QMARK_").replace("!", "_BANG_") @@ -735,6 +770,5 @@ def returns(type): """Tags a var as for unwrapping in rt. When rt imports this var it will be automatically converted to this type""" def with_fn(fn): fn._returns = type - print "registered", fn return fn return with_fn diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index ee4933e3..bd9a01be 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -313,7 +313,7 @@ def interpret(code_obj, args=[]): continue - print "NO DISPATCH FOR: " + code.BYTECODES[inst] + affirm(False, u"NO DISPATCH FOR: " + unicode(code.BYTECODES[inst])) raise Exception() diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index d23cddf8..59df31e6 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -4,14 +4,18 @@ import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var import pixie.vm.rt as rt +import pixie.vm.util as util +from rpython.rlib.rarithmetic import intmask + class Keyword(Object): _type = Type(u"pixie.stdlib.Keyword") - + __immutable_fields__ = ["_hash"] def __init__(self, name): self._str = name self._w_name = None self._w_ns = None + self._hash = 0 def type(self): return Keyword._type @@ -61,6 +65,13 @@ def _namespace(self): self.init_names() return self._w_ns +@extend(proto._hash, Keyword) +def _hash(self): + assert isinstance(self, Keyword) + if self._hash == 0: + self._hash = util.hash_unencoded_chars(self._str) + return rt.wrap(intmask(self._hash)) + @as_var("keyword") def _keyword(s): affirm(isinstance(s, String), u"Symbol name must be a string") diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 521e545f..717dee7d 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -3,20 +3,102 @@ import pixie.vm.code as code import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype +from pixie.vm.numbers import Integer +from rpython.rlib.jit_libffi import CIF_DESCRIPTION +from rpython.rlib import clibffi +from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call class ExternalLib(object.Object): - _type = object.Type("pixie.stdlib.ExternalLib") + _type = object.Type(u"pixie.stdlib.ExternalLib") + + def type(self): + return ExternalLib._type def __init__(self, nm): - self._dyn_lib = dynload.dlopen(nm) + assert isinstance(nm, unicode) + s = rffi.str2charp(str(nm)) + self._dyn_lib = dynload.dlopen(s) + rffi.free_charp(s) + + + def get_fn(self, nm): + assert isinstance(nm, unicode) + s = rffi.str2charp(str(nm)) + sym = dynload.dlsym(self._dyn_lib, s) + rffi.free_charp(s) + return sym + + + +sizes = {Integer._type: rffi.sizeof(rffi.LONG)} + +native_types = {Integer._type: rffi.LONG} +native_types_p = {Integer._type: rffi.LONGP} + +class FFIFn(object.Object): + _type = object.Type(u"pixie.stdlib.FFIFn") + + def type(self): + return FFIFn._type + + def __init__(self, cd, f, arg_types, ret_type, exb_size, arg_offset): + self._cd = cd + self._f_ptr = f + self._arg_types = arg_types + self._ret_type = ret_type + self._exb_size = exb_size + self._arg_offset = arg_offset + + def invoke(self, args): + exb = lltype.malloc(rffi.CCHARP.TO, self._exb_size, flavor="raw") + offset_p = rffi.ptradd(exb, self._arg_offset) + for x in range(len(args)): + pnt = rffi.cast(native_types_p[self._arg_types[x]], offset_p) + pnt[0] = rffi.cast(native_types[self._arg_types[x]], args[x].int_val()) + offset_p = rffi.ptradd(offset_p, sizes[self._arg_types[x]]) + + print "_____________________________\n" + print args[0].int_val() + jit_ffi_call(self._cd, self._f_ptr, exb) + print "_____________________________\n" + + + + +def wrap_dyfn(f, args, ret): + transfer_size = 0 + exchange_result = len(args) * rffi.sizeof(rffi.CCHARP) + for x in args: + exchange_result = transfer_size + transfer_size += sizes[x] + + transfer_size += sizes[ret] + + cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor="raw") + cd.abi = clibffi.FFI_DEFAULT_ABI + cd.nargs = len(args) + cd.rtype = clibffi.cast_type_to_ffitype(native_types[ret]) + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, len(args), flavor="raw") + for x in range(len(args)): + atypes[x] = clibffi.cast_type_to_ffitype(native_types[args[x]]) + + + cd.atypes = atypes + cd.exchange_size = transfer_size + cd.exchange_result = len(args) * rffi.sizeof(rffi.CCHARP) + cd.exchange_result_libffi = len(args) * rffi.sizeof(rffi.CCHARP) + cd.exchange_args[0] = len(args) * rffi.sizeof(rffi.CCHARP) + + jit_ffi_prep_cif(cd) -import ctypes -from rpython.rlib.clibffi import get_libc_name -print get_libc_name() + return FFIFn(cd, f, args, ret, transfer_size, len(args) * rffi.sizeof(rffi.CCHARP)) -s = rffi.str2charp("/usr/lib/libc.dylib") -lib = dynload.dlopen(s) -rffi.free_charp(s) -#LIBC = ExternalLib(rffi.str2charp("/usr/lib/libc.dynlib")) +@code.as_var("doit") +def _doit(): + LIBC = ExternalLib(u"/usr/lib/libc.dylib") + putc = LIBC.get_fn(u"putc") + ffifn = wrap_dyfn(putc, [Integer._type], Integer._type) + ffifn.invoke([Integer(42)]) + return Integer(1) \ No newline at end of file diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index a4bf13c6..4a7cfa5d 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -1,6 +1,7 @@ import pixie.vm.object as object from pixie.vm.primitives import nil, true, false from rpython.rlib.rarithmetic import r_uint +import rpython.rlib.jit as jit from pixie.vm.code import DoublePolymorphicFn, extend, Protocol, as_var, wrap_fn import pixie.vm.rt as rt @@ -22,6 +23,9 @@ def int_val(self): def r_uint_val(self): return r_uint(self._int_val) + def promote(self): + return Integer(jit.promote(self._int_val)) + def type(self): return Integer._type diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 68547be1..51c22acd 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -12,6 +12,9 @@ def invoke(self, args): import pixie.vm.stdlib as stdlib return stdlib.invoke_other(self, args) + def promote(self): + return self + class TypeRegistry(object): def __init__(self): self._types = {} diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 8b347084..6a99477f 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -209,10 +209,10 @@ def apply__args(args): return fn.invoke(out_args) -@as_var("print") -def _print(a): - print rt._str(a)._str - return nil +#@as_var("print") +#def _print(a): +# print rt._str(a)._str +# return nil @returns(bool) @as_var("instance?") @@ -387,6 +387,13 @@ def pop_binding_frame(): code._dynamic_vars.pop_binding_frame() return nil +@as_var("elidable-fn") +def elidable_fn(fn): + return code.ElidableFn(fn) + +@as_var("promote") +def promote(i): + return i.promote() def invoke_other(obj, args): from pixie.vm.array import array diff --git a/pixie/vm/symbol.py b/pixie/vm/symbol.py index 8e58d39a..483f52f9 100644 --- a/pixie/vm/symbol.py +++ b/pixie/vm/symbol.py @@ -5,15 +5,20 @@ from pixie.vm.code import extend, as_var from pixie.vm.string import String import pixie.vm.rt as rt +import pixie.vm.util as util +from rpython.rlib.rarithmetic import intmask + class Symbol(object.Object): _type = object.Type(u"pixie.stdlib.Symbol") + __immutable_fields__ = ["_hash"] def __init__(self, s): #assert isinstance(s, unicode) self._str = s self._w_name = None self._w_ns = None + self._hash = 0 def type(self): return Symbol._type @@ -60,6 +65,13 @@ def _namespace(self): self.init_names() return self._w_ns +@extend(proto._hash, Symbol) +def _hash(self): + assert isinstance(self, Symbol) + if self._hash == 0: + self._hash = util.hash_unencoded_chars(self._str) + return rt.wrap(intmask(self._hash)) + @as_var("symbol") def _symbol(s): affirm(isinstance(s, String), u"Symbol name must be a string") From 6d8b082fd4946aa9cc4079aee56b946a35071f2c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 23 Oct 2014 21:03:47 -0600 Subject: [PATCH 003/909] FFI is working, now we just need to expand supported types and the like --- benchmarks/get_from_hashmap.lisp | 5 + benchmarks/vector_build_and_hash.lisp | 6 + pixie/stdlib.lisp | 4 + pixie/vm/code.py | 13 ++ pixie/vm/libs/ffi.py | 163 +++++++++++++++++++------- pixie/vm/rt.py | 2 +- tests/collections/vector_tests.lisp | 0 7 files changed, 148 insertions(+), 45 deletions(-) create mode 100644 benchmarks/get_from_hashmap.lisp create mode 100644 benchmarks/vector_build_and_hash.lisp create mode 100644 tests/collections/vector_tests.lisp diff --git a/benchmarks/get_from_hashmap.lisp b/benchmarks/get_from_hashmap.lisp new file mode 100644 index 00000000..7eb7583e --- /dev/null +++ b/benchmarks/get_from_hashmap.lisp @@ -0,0 +1,5 @@ +(let [map {:number 1}] + (loop [x 0] + (if (= x 10000) + x + (recur (+ x (get map :number)))))) diff --git a/benchmarks/vector_build_and_hash.lisp b/benchmarks/vector_build_and_hash.lisp new file mode 100644 index 00000000..f09c451e --- /dev/null +++ b/benchmarks/vector_build_and_hash.lisp @@ -0,0 +1,6 @@ +(loop [acc []] + (if (= (count acc) 10000) + (hash acc) + (recur (conj acc (count acc))))) + +:exit-repl diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index ccffef47..60dc8c38 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -397,3 +397,7 @@ `(do ~type-decl ~ctor ~@proto-bodies))) + + +(def libc (ffi-library "/usr/lib/libc.dylib")) +(def exit (ffi-fn libc "exit" [Integer] Integer)) \ No newline at end of file diff --git a/pixie/vm/code.py b/pixie/vm/code.py index adb44d88..dd53a7e3 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -723,6 +723,19 @@ def wrapped_fn(self, args): raise return as_native_fn(wrapped_fn) + if argc == 4: + def wrapped_fn(self, args): + affirm(len(args) == 4, u"Expected 4 arguments to " + fn_name) + + try: + return fn(args[0], args[1], args[2], args[3]) + except object.WrappedException as ex: + ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + + assert False, "implement more" + def extend(pfn, tp1, tp2=None): """Extends a protocol to the given Type (not python type), with the decorated function diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 717dee7d..914d878a 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -1,6 +1,7 @@ import rpython.rlib.rdynload as dynload import pixie.vm.object as object import pixie.vm.code as code +from pixie.vm.code import as_var, affirm import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype from pixie.vm.numbers import Integer @@ -8,6 +9,20 @@ from rpython.rlib import clibffi from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call + +""" +FFI interface for pixie. + +This code gets a bit interesting. We use the RPython rlib module jit_libffi to do the interfacing, you can find +good docs in that module. + +The problem is we can't serialize/translate function pointers (makes sense), so we make use of the _cleanup_ method +hook to clean out all unsupported fields. We then lazy re-load these values as needed. This allows us to specifiy FFI +functions inside of the stdlib (and other places) allowing for fast interpreter boot times. + + +""" + class ExternalLib(object.Object): _type = object.Type(u"pixie.stdlib.ExternalLib") @@ -16,24 +31,42 @@ def type(self): def __init__(self, nm): assert isinstance(nm, unicode) - s = rffi.str2charp(str(nm)) - self._dyn_lib = dynload.dlopen(s) - rffi.free_charp(s) + self._name = nm + self._is_inited = False + self.thaw() + + def thaw(self): + if not self._is_inited: + s = rffi.str2charp(str(self._name)) + self._dyn_lib = dynload.dlopen(s) + self._is_inited = True + rffi.free_charp(s) - def get_fn(self, nm): + def get_fn_ptr(self, nm): assert isinstance(nm, unicode) + self.thaw() s = rffi.str2charp(str(nm)) sym = dynload.dlsym(self._dyn_lib, s) rffi.free_charp(s) return sym + def _cleanup_(self): + self._dyn_lib = lltype.nullptr(rffi.VOIDP.TO) + self._is_inited = False -sizes = {Integer._type: rffi.sizeof(rffi.LONG)} +def get_native_size(tp): + if tp == Integer._type: + return rffi.sizeof(rffi.LONG) + assert False -native_types = {Integer._type: rffi.LONG} -native_types_p = {Integer._type: rffi.LONGP} +def set_native_value(ptr, val, tp): + if tp == Integer._type: + pnt = rffi.cast(rffi.LONGP, ptr) + pnt[0] = rffi.cast(rffi.LONG, val.int_val()) + return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.LONG))) + assert False class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") @@ -41,64 +74,106 @@ class FFIFn(object.Object): def type(self): return FFIFn._type - def __init__(self, cd, f, arg_types, ret_type, exb_size, arg_offset): - self._cd = cd - self._f_ptr = f + def __init__(self, lib, name, arg_types, ret_type): + self._name = name + self._lib = lib self._arg_types = arg_types self._ret_type = ret_type - self._exb_size = exb_size - self._arg_offset = arg_offset + self._is_inited = False + + + + def thaw(self): + if not self._is_inited: + self._f_ptr = self._lib.get_fn_ptr(self._name) + transfer_size = 0 + arg0_offset = len(self._arg_types) * rffi.sizeof(rffi.CCHARP) + exchange_result = arg0_offset + for x in self._arg_types: + exchange_result = transfer_size + transfer_size += get_native_size(x) + + transfer_size += get_native_size(self._ret_type) + + cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor="raw") + cd.abi = clibffi.FFI_DEFAULT_ABI + cd.nargs = len(self._arg_types) + cd.rtype = get_clibffi_type(self._ret_type) + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, len(self._arg_types), flavor="raw") + for x in range(len(self._arg_types)): + atypes[x] = get_clibffi_type(self._arg_types[x]) + + + + cd.atypes = atypes + cd.exchange_size = transfer_size + cd.exchange_result = arg0_offset + cd.exchange_result_libffi = arg0_offset + cd.exchange_args[0] = arg0_offset + + jit_ffi_prep_cif(cd) + self._cd = cd + self._transfer_size = transfer_size + self._arg0_offset = arg0_offset + + return self + + def _cleanup_(self): + self._f_ptr = lltype.nullptr(rffi.VOIDP.TO) + self._cd = lltype.nullptr(CIF_DESCRIPTION) + self._is_inited = False def invoke(self, args): - exb = lltype.malloc(rffi.CCHARP.TO, self._exb_size, flavor="raw") - offset_p = rffi.ptradd(exb, self._arg_offset) + self.thaw() + exb = lltype.malloc(rffi.CCHARP.TO, self._transfer_size, flavor="raw") + offset_p = rffi.ptradd(exb, self._arg0_offset) for x in range(len(args)): - pnt = rffi.cast(native_types_p[self._arg_types[x]], offset_p) - pnt[0] = rffi.cast(native_types[self._arg_types[x]], args[x].int_val()) - offset_p = rffi.ptradd(offset_p, sizes[self._arg_types[x]]) + offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) print "_____________________________\n" - print args[0].int_val() jit_ffi_call(self._cd, self._f_ptr, exb) print "_____________________________\n" + return Integer(42) +def get_clibffi_type(arg): + if arg == Integer._type: + return clibffi.cast_type_to_ffitype(rffi.LONG) + assert False -def wrap_dyfn(f, args, ret): - transfer_size = 0 - exchange_result = len(args) * rffi.sizeof(rffi.CCHARP) - for x in args: - exchange_result = transfer_size - transfer_size += sizes[x] - transfer_size += sizes[ret] - cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor="raw") - cd.abi = clibffi.FFI_DEFAULT_ABI - cd.nargs = len(args) - cd.rtype = clibffi.cast_type_to_ffitype(native_types[ret]) - atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, len(args), flavor="raw") - for x in range(len(args)): - atypes[x] = clibffi.cast_type_to_ffitype(native_types[args[x]]) +LIBC = ExternalLib(u"/usr/lib/libc.dylib") +FN = FFIFn(LIBC, u"exit", [Integer._type], Integer._type) +#putc = LIBC.get_fn(u"exit") +#ffifn = wrap_dyfn(putc, [Integer._type], Integer._type) +@as_var("ffi-library") +def _ffi_library(ns): + nm = rt.name(ns) + return ExternalLib(nm) - cd.atypes = atypes - cd.exchange_size = transfer_size - cd.exchange_result = len(args) * rffi.sizeof(rffi.CCHARP) - cd.exchange_result_libffi = len(args) * rffi.sizeof(rffi.CCHARP) - cd.exchange_args[0] = len(args) * rffi.sizeof(rffi.CCHARP) +@as_var("ffi-fn") +def _ffi_fn(lib, nm, args, ret_type): + affirm(isinstance(lib, ExternalLib), u"First argument must be an ExternalLib") + affirm(isinstance(ret_type, object.Type), u"Ret type must be a type") + affirm(rt.namespace(nm) is None, u"Name must not be namespaced") - jit_ffi_prep_cif(cd) + cnt = rt.count(args).int_val() + new_args = [None] * cnt + for x in range(cnt): + t = rt.nth(args, rt.wrap(x)) + affirm(isinstance(t, object.Type), u"Arg defs must be types") + new_args[x] = t - return FFIFn(cd, f, args, ret, transfer_size, len(args) * rffi.sizeof(rffi.CCHARP)) + f = FFIFn(lib, rt.name(nm), new_args, ret_type) + return f @code.as_var("doit") def _doit(): - LIBC = ExternalLib(u"/usr/lib/libc.dylib") - putc = LIBC.get_fn(u"putc") - ffifn = wrap_dyfn(putc, [Integer._type], Integer._type) + FN.invoke([Integer(11)]) + return Integer(42) + - ffifn.invoke([Integer(42)]) - return Integer(1) \ No newline at end of file diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index c3191ae4..08a2ecda 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -60,7 +60,7 @@ def wrapper(*args): import pixie.vm.compiler as compiler import pixie.vm.map_entry import pixie.vm.reader as reader - #import pixie.vm.libs.ffi + import pixie.vm.libs.ffi diff --git a/tests/collections/vector_tests.lisp b/tests/collections/vector_tests.lisp new file mode 100644 index 00000000..e69de29b From 7e5f4372800830ff62a43050ef1a9b7d7f4c3009 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 23 Oct 2014 21:04:29 -0600 Subject: [PATCH 004/909] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 5c557319..50f349fa 100644 --- a/README.md +++ b/README.md @@ -112,4 +112,4 @@ Although parts of the language may be very close to Clojure (they are both lisps ## Copying -Free use of this software is granted under the terms of the GNU Lesser General Public License (LGPL). For details see the files `COPYING` and `COPYING.LESSER` included with the 0MQ distribution. +Free use of this software is granted under the terms of the GNU Lesser General Public License (LGPL). For details see the files `COPYING` and `COPYING.LESSER` included with the source distribution. From f580c71b39e434f53d3d6269126675c24a090d46 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 23 Oct 2014 21:04:47 -0600 Subject: [PATCH 005/909] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 50f349fa..c71e2269 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Some planned and implemented features: ./checkout-externals ./make-with-jit - ./target-c + ./pixie-vm ## FAQ From e1119dfa4e5b093cdae3f143a9f1007861fe9e43 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 23 Oct 2014 21:05:22 -0600 Subject: [PATCH 006/909] added build file for building without a jit --- make-no-jit | 1 + 1 file changed, 1 insertion(+) create mode 100755 make-no-jit diff --git a/make-no-jit b/make-no-jit new file mode 100755 index 00000000..51decd0f --- /dev/null +++ b/make-no-jit @@ -0,0 +1 @@ +python ../externals/pypy/rpython/bin/rpython --continuation --no-shared target.py From f3bfee82b3a84bc973a8ed35858495f8e1ce7a34 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 24 Oct 2014 08:09:44 -0600 Subject: [PATCH 007/909] build fixes --- pixie/stdlib.lisp | 6 ++--- pixie/vm/code.py | 64 ++++++++++++++++++++++---------------------- pixie/vm/libs/ffi.py | 3 ++- pixie/vm/stdlib.py | 6 ++--- 4 files changed, 40 insertions(+), 39 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 60dc8c38..74a42c54 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -398,6 +398,6 @@ ~ctor ~@proto-bodies))) - -(def libc (ffi-library "/usr/lib/libc.dylib")) -(def exit (ffi-fn libc "exit" [Integer] Integer)) \ No newline at end of file +(comment + (def libc (ffi-library "/usr/lib/libc.dylib")) + (def exit (ffi-fn libc "exit" [Integer] Integer))) \ No newline at end of file diff --git a/pixie/vm/code.py b/pixie/vm/code.py index dd53a7e3..c79f1dc0 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -605,38 +605,38 @@ def _invoke(self, args): -class ElidableFn(object.Object): - _type = object.Type(u"pixie.stdlib.ElidableFn") - __immutable_fields__ = ["_boxed_fn"] - def type(self): - return ElidableFn._type - - def __init__(self, boxed_fn): - self._boxed_fn = boxed_fn - - @elidable - def _elidable_invoke_0(self, fn): - return self._boxed_fn.invoke([]) - - @elidable - def _elidable_invoke_1(self, fn, arg0): - return self._boxed_fn.invoke([arg0]) - - @elidable - def _elidable_invoke_2(self, fn, arg0, arg1): - return self._boxed_fn.invoke([arg0, arg1]) - - - def invoke(self, args): - largs = jit.promote(len(args)) - fn = self._boxed_fn.promote() - if largs == 0: - return self._elidable_invoke_0(fn).promote() - elif largs == 1: - return self._elidable_invoke_1(fn, args[0].promote()).promote() - elif largs == 2: - return self._elidable_invoke_2(fn, args[0].promote(), args[1].promote()).promote() - affirm(False, u"Too many args to Elidable Fn") +# class ElidableFn(object.Object): +# _type = object.Type(u"pixie.stdlib.ElidableFn") +# __immutable_fields__ = ["_boxed_fn"] +# def type(self): +# return ElidableFn._type +# +# def __init__(self, boxed_fn): +# self._boxed_fn = boxed_fn +# +# @elidable +# def _elidable_invoke_0(self, fn): +# return self._boxed_fn.invoke([]) +# +# @elidable +# def _elidable_invoke_1(self, fn, arg0): +# return self._boxed_fn.invoke([arg0]) +# +# @elidable +# def _elidable_invoke_2(self, fn, arg0, arg1): +# return self._boxed_fn.invoke([arg0, arg1]) +# +# +# def invoke(self, args): +# largs = jit.promote(len(args)) +# fn = self._boxed_fn.promote() +# if largs == 0: +# return self._elidable_invoke_0(fn).promote() +# elif largs == 1: +# return self._elidable_invoke_1(fn, args[0].promote()).promote() +# elif largs == 2: +# return self._elidable_invoke_2(fn, args[0].promote(), args[1].promote()).promote() +# affirm(False, u"Too many args to Elidable Fn") def munge(s): return s.replace("-", "_").replace("?", "_QMARK_").replace("!", "_BANG_") diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 914d878a..590b0721 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -18,7 +18,7 @@ The problem is we can't serialize/translate function pointers (makes sense), so we make use of the _cleanup_ method hook to clean out all unsupported fields. We then lazy re-load these values as needed. This allows us to specifiy FFI -functions inside of the stdlib (and other places) allowing for fast interpreter boot times. +functions inside of the stdlib (and other places) allowing for fast interpreter boot times. """ @@ -132,6 +132,7 @@ def invoke(self, args): print "_____________________________\n" jit_ffi_call(self._cd, self._f_ptr, exb) + lltype.free(exb, flavor="raw") print "_____________________________\n" return Integer(42) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 6a99477f..0dc4edaf 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -387,9 +387,9 @@ def pop_binding_frame(): code._dynamic_vars.pop_binding_frame() return nil -@as_var("elidable-fn") -def elidable_fn(fn): - return code.ElidableFn(fn) +# @as_var("elidable-fn") +# def elidable_fn(fn): +# return code.ElidableFn(fn) @as_var("promote") def promote(i): From e3ab9cd575d0204165e5c7f8bccdfb07ced01850 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 24 Oct 2014 11:00:50 -0600 Subject: [PATCH 008/909] deleted some debug code --- pixie/vm/libs/ffi.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 590b0721..154e21f4 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -144,12 +144,6 @@ def get_clibffi_type(arg): assert False - -LIBC = ExternalLib(u"/usr/lib/libc.dylib") -FN = FFIFn(LIBC, u"exit", [Integer._type], Integer._type) -#putc = LIBC.get_fn(u"exit") -#ffifn = wrap_dyfn(putc, [Integer._type], Integer._type) - @as_var("ffi-library") def _ffi_library(ns): nm = rt.name(ns) @@ -172,9 +166,3 @@ def _ffi_fn(lib, nm, args, ret_type): return f -@code.as_var("doit") -def _doit(): - FN.invoke([Integer(11)]) - return Integer(42) - - From 02e25de45a3d76301cfd718463ff0280af23ae29 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 24 Oct 2014 12:07:08 -0600 Subject: [PATCH 009/909] added pixie.platform --- pixie/stdlib.lisp | 6 +++--- pixie/vm/libs/ffi.py | 14 +++++++++----- pixie/vm/libs/platform.py | 11 +++++++++++ pixie/vm/rt.py | 6 ++++-- 4 files changed, 27 insertions(+), 10 deletions(-) create mode 100644 pixie/vm/libs/platform.py diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 74a42c54..1d9f97db 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -398,6 +398,6 @@ ~ctor ~@proto-bodies))) -(comment - (def libc (ffi-library "/usr/lib/libc.dylib")) - (def exit (ffi-fn libc "exit" [Integer] Integer))) \ No newline at end of file + + (def libc (ffi-library (str "libc" "." pixie.platform/so-ext))) + (def exit (ffi-fn libc "exit" [Integer] Integer)) \ No newline at end of file diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 154e21f4..0aeca661 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -38,9 +38,15 @@ def __init__(self, nm): def thaw(self): if not self._is_inited: s = rffi.str2charp(str(self._name)) - self._dyn_lib = dynload.dlopen(s) + + try: + self._dyn_lib = dynload.dlopen(s) + except dynload.DLOpenError as ex: + raise object.WrappedException(object.RuntimeException(rt.wrap(ex.msg))) + finally: + rffi.free_charp(s) + self._is_inited = True - rffi.free_charp(s) def get_fn_ptr(self, nm): @@ -130,11 +136,9 @@ def invoke(self, args): for x in range(len(args)): offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) - print "_____________________________\n" jit_ffi_call(self._cd, self._f_ptr, exb) lltype.free(exb, flavor="raw") - print "_____________________________\n" - return Integer(42) + return Integer(0) diff --git a/pixie/vm/libs/platform.py b/pixie/vm/libs/platform.py new file mode 100644 index 00000000..f81f9858 --- /dev/null +++ b/pixie/vm/libs/platform.py @@ -0,0 +1,11 @@ +from rpython.translator.platform import platform +import pixie.vm.rt as rt +from pixie.vm.string import String +from pixie.vm.code import as_var +import os + + +as_var("pixie.platform", "os")(String(os.name)) +as_var("pixie.platform", "name")(String(platform.name)) +as_var("pixie.platform", "so-ext")(String(platform.so_ext)) + diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 08a2ecda..75b8b6bf 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -1,5 +1,6 @@ __config__ = None py_list = list +py_str = str from rpython.rlib.objectmodel import specialize @@ -60,6 +61,7 @@ def wrapper(*args): import pixie.vm.compiler as compiler import pixie.vm.map_entry import pixie.vm.reader as reader + import pixie.vm.libs.platform import pixie.vm.libs.ffi @@ -72,8 +74,8 @@ def wrap(x): return numbers.Integer(x) if isinstance(x, unicode): return String(x) - #if isinstance(x, str): - # return String(unicode(x)) + if isinstance(x, py_str): + return String(unicode(x)) affirm(False, u"Bad wrap") globals()["wrap"] = wrap From 26579419de2bdadd3f4f17aae71d9fb410533f8d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 24 Oct 2014 16:35:09 -0600 Subject: [PATCH 010/909] working on ffi a bit more --- pixie/stdlib.lisp | 4 +++- pixie/vm/libs/ffi.py | 47 +++++++++++++++++++++++++++++++-------- pixie/vm/libs/platform.py | 6 ++--- pixie/vm/reader.py | 2 +- target.py | 3 +++ 5 files changed, 48 insertions(+), 14 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 1d9f97db..59462348 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -400,4 +400,6 @@ (def libc (ffi-library (str "libc" "." pixie.platform/so-ext))) - (def exit (ffi-fn libc "exit" [Integer] Integer)) \ No newline at end of file + (def exit (ffi-fn libc "exit" [Integer] Integer)) + (def puts (ffi-fn libc "puts" [String] Integer)) + (def printf (ffi-fn libc "printf" [String] Integer)) \ No newline at end of file diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 0aeca661..a48f6a1b 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -5,9 +5,11 @@ import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype from pixie.vm.numbers import Integer +from pixie.vm.string import String from rpython.rlib.jit_libffi import CIF_DESCRIPTION from rpython.rlib import clibffi from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call +import rpython.rlib.jit as jit """ @@ -65,13 +67,27 @@ def _cleanup_(self): def get_native_size(tp): if tp == Integer._type: return rffi.sizeof(rffi.LONG) + if tp == String._type: + return rffi.sizeof(rffi.CCHARP) assert False -def set_native_value(ptr, val, tp): +def get_ret_val(ptr, tp): if tp == Integer._type: + pnt = rffi.cast(rffi.LONGP, ptr) + val = pnt[0] + return Integer(val) + + assert False + +def set_native_value(ptr, val, tp): + if tp is Integer._type: pnt = rffi.cast(rffi.LONGP, ptr) pnt[0] = rffi.cast(rffi.LONG, val.int_val()) return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.LONG))) + if tp is String._type: + pnt = rffi.cast(rffi.CCHARPP, ptr) + pnt[0] = rffi.str2charp(str(rt.name(val))) + return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.CCHARP))) assert False class FFIFn(object.Object): @@ -99,6 +115,7 @@ def thaw(self): exchange_result = transfer_size transfer_size += get_native_size(x) + ret_offset = transfer_size transfer_size += get_native_size(self._ret_type) cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor="raw") @@ -113,14 +130,15 @@ def thaw(self): cd.atypes = atypes cd.exchange_size = transfer_size - cd.exchange_result = arg0_offset - cd.exchange_result_libffi = arg0_offset + cd.exchange_result = ret_offset + cd.exchange_result_libffi = ret_offset cd.exchange_args[0] = arg0_offset jit_ffi_prep_cif(cd) self._cd = cd self._transfer_size = transfer_size self._arg0_offset = arg0_offset + self._ret_offset = ret_offset return self @@ -129,22 +147,33 @@ def _cleanup_(self): self._cd = lltype.nullptr(CIF_DESCRIPTION) self._is_inited = False - def invoke(self, args): - self.thaw() + #@jit.unroll_safe + def pack_args(self, offset_p, args, arg_types): + for x in range(len(arg_types)): + offset_p = set_native_value(offset_p, args[x], arg_types[x]) + + + def _invoke(self, args): exb = lltype.malloc(rffi.CCHARP.TO, self._transfer_size, flavor="raw") offset_p = rffi.ptradd(exb, self._arg0_offset) - for x in range(len(args)): - offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) - + self.pack_args(offset_p, args, self._arg_types) jit_ffi_call(self._cd, self._f_ptr, exb) + offset_p = rffi.ptradd(exb, self._ret_offset) + ret_val = get_ret_val(offset_p, self._ret_type) lltype.free(exb, flavor="raw") - return Integer(0) + return ret_val + + def invoke(self, args): + self.thaw() + return self._invoke(args) def get_clibffi_type(arg): if arg == Integer._type: return clibffi.cast_type_to_ffitype(rffi.LONG) + if arg == String._type: + return clibffi.ffi_type_pointer assert False diff --git a/pixie/vm/libs/platform.py b/pixie/vm/libs/platform.py index f81f9858..9e5f6860 100644 --- a/pixie/vm/libs/platform.py +++ b/pixie/vm/libs/platform.py @@ -5,7 +5,7 @@ import os -as_var("pixie.platform", "os")(String(os.name)) -as_var("pixie.platform", "name")(String(platform.name)) -as_var("pixie.platform", "so-ext")(String(platform.so_ext)) +as_var("pixie.platform", "os")(String(unicode(os.name))) +as_var("pixie.platform", "name")(String(unicode(platform.name))) +as_var("pixie.platform", "so-ext")(String(unicode(platform.so_ext))) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 78644fd8..d9da5fe4 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -58,7 +58,7 @@ def __init__(self): def read(self): if self._string_reader is None: - result = _readline(">>") + result = _readline(str(rt.name(compiler.NS_VAR.deref())) + " => ") if result == u"": raise EOFError() self._string_reader = StringReader(result + u"\n") diff --git a/target.py b/target.py index ea0f70ba..ca8eba5b 100644 --- a/target.py +++ b/target.py @@ -8,6 +8,7 @@ from pixie.vm.stacklet import with_stacklets import pixie.vm.stacklet as stacklet from pixie.vm.object import RuntimeException, WrappedException +from rpython.translator.platform import platform import sys @@ -57,6 +58,8 @@ def repl(): def entry_point(foo=None): print "Pixie 0.1 - Interactive REPL" + print "(" + platform.name + ", " + platform.cc + ")" + print "----------------------------" #try: # code = argv[1] #except IndexError: From ff2fbc167a46b0eab0f765dffcc8479d8c063e5e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 24 Oct 2014 16:40:15 -0600 Subject: [PATCH 011/909] switch to osx builds for now --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 1b9e9cb0..0c2df75d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,6 @@ script: - ./checkout-externals - ./make-with-jit + +os: + - osx From a76a2dbf785a017b260bc766f0ef32b5d5a04291 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 24 Oct 2014 16:51:50 -0600 Subject: [PATCH 012/909] rt.count now unwraps --- pixie/vm/compiler.py | 22 +++++++++++----------- pixie/vm/custom_types.py | 4 ++-- pixie/vm/libs/ffi.py | 2 +- pixie/vm/persistent_vector.py | 6 +++--- pixie/vm/stdlib.py | 5 +++-- 5 files changed, 20 insertions(+), 19 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index bd6e0ad5..0bdc8bb8 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -256,7 +256,7 @@ def is_macro_call(form, ctx): def call_macro(var, form, ctx): form = rt.next(form) - args = [None] * rt.count(form).int_val() + args = [None] * rt.count(form) i = 0 while form is not nil: args[i] = rt.first(form) @@ -278,7 +278,7 @@ def compile_map_literal(form, ctx): rt.reduce(CompileMapRf(ctx), nil, form) - size = rt.count(form).int_val() * 2 + size = rt.count(form) * 2 ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(size) + 1) @@ -321,7 +321,7 @@ def compile_form(form, ctx): if isinstance(form, PersistentVector): vector_var = rt.vector() - size = rt.count(form).int_val() + size = rt.count(form) #assert rt.count(form).int_val() == 0 ctx.push_const(code.intern_var(u"pixie.stdlib", u"vector")) for x in range(size): @@ -359,7 +359,7 @@ def compile_platform_plus(form, ctx): def compile_platform_eq(form, ctx): form = form.next() - affirm(rt.count(form).int_val() == 2, u"TODO: REMOVE") + affirm(rt.count(form) == 2, u"TODO: REMOVE") while form is not nil: compile_form(form.first(), ctx) form = form.next() @@ -371,7 +371,7 @@ def compile_platform_eq(form, ctx): def add_args(args, ctx): required_args = -1 local_idx = 0 - for x in range(rt.count(args).int_val()): + for x in range(rt.count(args)): arg = rt.nth(args, rt.wrap(x)) affirm(isinstance(arg, symbol.Symbol), u"Argument names must be symbols") if arg._str == u"&": @@ -416,7 +416,7 @@ def compile_fn(form, ctx): def compile_fn_body(name, args, body, ctx): - new_ctx = Context(name._str, rt.count(args).int_val(), ctx) + new_ctx = Context(name._str, rt.count(args), ctx) required_args = add_args(args, new_ctx) bc = 0 @@ -455,11 +455,11 @@ def compile_fn_body(name, args, body, ctx): ctx.bytecode.append(code.MAKE_VARIADIC) ctx.bytecode.append(r_uint(required_args)) - return required_args, rt.count(args).int_val() + return required_args, rt.count(args) def compile_if(form, ctx): form = form.next() - affirm(2 <= rt.count(form).int_val() <= 3, u"If must have either 2 or 3 forms") + affirm(2 <= rt.count(form) <= 3, u"If must have either 2 or 3 forms") test = rt.first(form) form = rt.next(form) @@ -547,7 +547,7 @@ def compile_let(form, ctx): ctx.disable_tail_call() binding_count = 0 - for i in range(0, rt.count(bindings).int_val(), 2): + for i in range(0, rt.count(bindings), 2): binding_count += 1 name = rt.nth(bindings, rt.wrap(i)) affirm(isinstance(name, symbol.Symbol), u"Let locals must be symbols") @@ -583,7 +583,7 @@ def compile_loop(form, ctx): ctx.disable_tail_call() binding_count = 0 - for i in range(0, rt.count(bindings).int_val(), 2): + for i in range(0, rt.count(bindings), 2): binding_count += 1 name = rt.nth(bindings, rt.wrap(i)) affirm(isinstance(name, symbol.Symbol), u"Loop must bindings must be symbols") @@ -615,7 +615,7 @@ def compile_comment(form, ctx): ctx.push_const(nil) def compile_ns(form, ctx): - affirm(rt.count(form).int_val() == 2, u"ns only takes one argument, a symbol") + affirm(rt.count(form) == 2, u"ns only takes one argument, a symbol") nm = rt.first(rt.next(form)) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 9575b6b1..debffa3f 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -53,9 +53,9 @@ def set_field_by_idx(self, idx, val): def create_type(type_name, fields): affirm(isinstance(type_name, Keyword), u"Type name must be a keyword") - field_count = rt.count(fields).int_val() + field_count = rt.count(fields) acc = {} - for i in range(rt.count(fields).int_val()): + for i in range(rt.count(fields)): val = rt.nth(fields, rt.wrap(i)) affirm(isinstance(val, Keyword), u"Field names must be keywords") acc[val] = i diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index a48f6a1b..d154052c 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -188,7 +188,7 @@ def _ffi_fn(lib, nm, args, ret_type): affirm(isinstance(ret_type, object.Type), u"Ret type must be a type") affirm(rt.namespace(nm) is None, u"Name must not be namespaced") - cnt = rt.count(args).int_val() + cnt = rt.count(args) new_args = [None] * cnt for x in range(cnt): t = rt.nth(args, rt.wrap(x)) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 2508d9e3..0510f604 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -423,10 +423,10 @@ def _reduce(self, f, init): @as_var("vector") def vector__args(args): - acc = EMPTY + acc = rt._transient(EMPTY) for x in range(len(args)): - acc = acc.conj(args[x]) - return acc + acc = rt._conj_BANG_(acc, args[x]) + return rt._persistent_BANG_(acc) @extend(proto._transient, PersistentVector) def _transient(self): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 0dc4edaf..b496939b 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -142,6 +142,7 @@ def __hash(x): _count_driver = jit.JitDriver(name="pixie.stdlib.count", greens=["tp"], reds="auto") +@returns(r_uint) @as_var("count") def count(x): acc = 0 @@ -199,11 +200,11 @@ def apply__args(args): fn = args[0] argc = r_uint(len(args) - 2) - out_args = [None] * (argc + r_uint(rt.count(last_itm).int_val())) + out_args = [None] * (argc + r_uint(rt.count(last_itm))) list_copy(args, 1, out_args, 0, argc) - for x in range(rt.count(last_itm).int_val()): + for x in range(rt.count(last_itm)): out_args[argc + x] = rt.nth(last_itm, rt.wrap(x)) return fn.invoke(out_args) From d11e7b1b67b17d42c53791ef7c762eb2f0f540c0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 24 Oct 2014 16:58:52 -0600 Subject: [PATCH 013/909] fixes translation on osx --- pixie/vm/compiler.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 0bdc8bb8..69ec2964 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -376,7 +376,7 @@ def add_args(args, ctx): affirm(isinstance(arg, symbol.Symbol), u"Argument names must be symbols") if arg._str == u"&": - required_args = x + required_args = intmask(x) continue ctx.add_local(arg._str, Arg(local_idx)) local_idx += 1 @@ -455,7 +455,7 @@ def compile_fn_body(name, args, body, ctx): ctx.bytecode.append(code.MAKE_VARIADIC) ctx.bytecode.append(r_uint(required_args)) - return required_args, rt.count(args) + return required_args, intmask(rt.count(args)) def compile_if(form, ctx): form = form.next() @@ -534,7 +534,7 @@ def compile_recur(form, ctx): ctx.get_recur_point().emit(ctx, args) if ctc: ctx.enable_tail_call() - ctx.sub_sp(args - 1) + ctx.sub_sp(r_uint(args - 1)) def compile_let(form, ctx): @@ -686,7 +686,7 @@ def compile_cons(form, ctx): ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(cnt) - ctx.sub_sp(cnt - 1) + ctx.sub_sp(r_uint(cnt - 1)) def compile(form): From 24062c2666ce41a68de2bee4c5066da2377f1e46 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 25 Oct 2014 07:36:35 -0600 Subject: [PATCH 014/909] disable ffi inside stdlib --- pixie/stdlib.lisp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 59462348..309f1e38 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -398,8 +398,8 @@ ~ctor ~@proto-bodies))) - +(comment (def libc (ffi-library (str "libc" "." pixie.platform/so-ext))) (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) - (def printf (ffi-fn libc "printf" [String] Integer)) \ No newline at end of file + (def printf (ffi-fn libc "printf" [String] Integer))) \ No newline at end of file From 604746338433bd8f827e0d31dd6ae705af0eb6bc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 25 Oct 2014 17:39:01 -0600 Subject: [PATCH 015/909] start on metadata literals --- checkout-externals | 2 +- pixie/stdlib.lisp | 7 ++++--- pixie/vm/compiler.py | 20 ++++++++++++++++++-- pixie/vm/libs/ffi.py | 23 ++++++++++++++--------- pixie/vm/persistent_hash_map.py | 19 ++++++++++++++++++- pixie/vm/reader.py | 19 +++++++++++++++++-- pixie/vm/rt.py | 3 +++ pixie/vm/stdlib.py | 31 +++++++++++++++++++++++++++++-- pixie/vm/symbol.py | 18 +++++++++++++++++- 9 files changed, 121 insertions(+), 21 deletions(-) diff --git a/checkout-externals b/checkout-externals index 0cb310f8..719a808c 100755 --- a/checkout-externals +++ b/checkout-externals @@ -1,6 +1,6 @@ mkdir ../externals cd ../externals -curl https://bitbucket.org/halgari/pypy/get/e5d9ee0be134.tar.bz2 > pypy.tar.bz2 +curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2 mkdir pypy cd pypy tar -jxf ../pypy.tar.bz2 --strip-components=1 diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 309f1e38..cfa930fa 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -371,7 +371,8 @@ (identical? Symbol (type x))) - +(defmacro lazy-seq [& body] + `(lazy-seq* (fn [] ~@body))) (defmacro deftype [nm fields & body] (let [ctor-name (symbol (str "->" (name nm))) @@ -398,8 +399,8 @@ ~ctor ~@proto-bodies))) -(comment + (def libc (ffi-library (str "libc" "." pixie.platform/so-ext))) (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) - (def printf (ffi-fn libc "printf" [String] Integer))) \ No newline at end of file + (def printf (ffi-fn libc "printf" [String] Integer)) \ No newline at end of file diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 69ec2964..08285ad3 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -1,4 +1,4 @@ -from pixie.vm.object import Object, _type_registry, affirm +from pixie.vm.object import Object, _type_registry, affirm, InterpreterCodeInfo from pixie.vm.primitives import nil, true, false, Bool from pixie.vm.persistent_vector import EMPTY, PersistentVector import pixie.vm.numbers as numbers @@ -281,8 +281,21 @@ def compile_map_literal(form, ctx): size = rt.count(form) * 2 ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(size) + 1) + ctx.sub_sp(size - 1) + compile_meta(rt.meta(form), ctx) +def compile_meta(meta, ctx): + ctx.push_const(code.intern_var(u"pixie.stdlib", u'with-meta')) + ctx.bytecode.append(code.DUP_NTH) + ctx.bytecode.append(r_uint(1)) + ctx.push_const(meta) + ctx.bytecode.append(code.INVOKE) + ctx.bytecode.append(r_uint(3)) + ctx.sub_sp(1) + ctx.bytecode.append(code.POP_UP_N) + ctx.bytecode.append(1) + ctx.sub_sp(1) def compile_form(form, ctx): if form is nil: @@ -330,6 +343,9 @@ def compile_form(form, ctx): ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(size + 1)) ctx.sub_sp(size) + + compile_meta(rt.meta(form), ctx) + return if rt.instance_QMARK_(rt.IMap.deref(), form): @@ -682,7 +698,7 @@ def compile_cons(form, ctx): # ctx.bytecode.append(code.TAIL_CALL) #else: if meta is not nil: - ctx.debug_points[len(ctx.bytecode)] = meta + ctx.debug_points[len(ctx.bytecode)] = rt.interpreter_code_info(meta) ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(cnt) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index d154052c..9ea9cfd5 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -92,11 +92,14 @@ def set_native_value(ptr, val, tp): class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") + __immutable_fields__ = ["_is_inited?", "_lib", "_name", "_arg_types[*]", "_ret_type", \ + "_transfer_size?", "_arg0_offset?", "_ret_offset?", "_cd?"] def type(self): return FFIFn._type def __init__(self, lib, name, arg_types, ret_type): + self._rev = 0 self._name = name self._lib = lib self._arg_types = arg_types @@ -104,7 +107,6 @@ def __init__(self, lib, name, arg_types, ret_type): self._is_inited = False - def thaw(self): if not self._is_inited: self._f_ptr = self._lib.get_fn_ptr(self._name) @@ -140,23 +142,26 @@ def thaw(self): self._arg0_offset = arg0_offset self._ret_offset = ret_offset + self._is_inited = True + return self def _cleanup_(self): + self._rev += 1 self._f_ptr = lltype.nullptr(rffi.VOIDP.TO) self._cd = lltype.nullptr(CIF_DESCRIPTION) self._is_inited = False - #@jit.unroll_safe - def pack_args(self, offset_p, args, arg_types): - for x in range(len(arg_types)): - offset_p = set_native_value(offset_p, args[x], arg_types[x]) - - + @jit.unroll_safe def _invoke(self, args): + if not self._is_inited: + self.thaw() exb = lltype.malloc(rffi.CCHARP.TO, self._transfer_size, flavor="raw") offset_p = rffi.ptradd(exb, self._arg0_offset) - self.pack_args(offset_p, args, self._arg_types) + + for x in range(len(self._arg_types)): + offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) + jit_ffi_call(self._cd, self._f_ptr, exb) offset_p = rffi.ptradd(exb, self._ret_offset) ret_val = get_ret_val(offset_p, self._ret_type) @@ -164,7 +169,7 @@ def _invoke(self, args): return ret_val def invoke(self, args): - self.thaw() + self = jit.promote(self) return self._invoke(args) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index cd118bd7..0349aec1 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -26,6 +26,12 @@ def __init__(self, cnt, root, meta=nil): self._root = root self._meta = meta + def meta(self): + return self._meta + + def with_meta(self, meta): + return PersistentHashMap(self._cnt, self._root, meta) + def assoc(self, key, val): added_leaf = Box() @@ -324,4 +330,15 @@ def _assoc(self, key, val): @extend(proto._count, PersistentHashMap) def _count(self): assert isinstance(self, PersistentHashMap) - return rt.wrap(intmask(self._cnt)) \ No newline at end of file + return rt.wrap(intmask(self._cnt)) + + +@extend(proto._meta, PersistentHashMap) +def _meta(self): + assert isinstance(self, PersistentHashMap) + return self.meta() + +@extend(proto._with_meta, PersistentHashMap) +def _with_meta(self, meta): + assert isinstance(self, PersistentHashMap) + return self.with_meta(meta) \ No newline at end of file diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index d9da5fe4..ff751723 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -138,7 +138,10 @@ def read(self): return ch def get_metadata(self): - return object.InterpreterCodeInfo(self._line, self._line_number, self._column_number, self._filename) + return rt.hashmap(LINE_KW, rt.wrap(self._line), + LINE_NUMBER_KW, rt.wrap(self._line_number), + COLUMN_NUMBER_KW, rt.wrap(self._column_number), + FILE_KW, rt.wrap(self._filename)) def unread(self, ch): @@ -347,6 +350,17 @@ def invoke(self, rdr, ch): form = read(rdr, True) return rt.list(sym, form) +class MetaReader(ReaderHandler): + def invoke(self, rdr, ch): + meta = read(rdr, True) + obj = read(rdr, True) + + if rt.instance_QMARK_(rt.IMeta.deref(), obj): + + return rt._with_meta(obj, meta) + print "not meta" + return obj + handlers = {u"(": ListReader(), u")": UnmachedListReader(), u"[": VectorReader(), @@ -358,7 +372,8 @@ def invoke(self, rdr, ch): u"\"": LiteralStringReader(), u"@": DerefReader(), u"`": SyntaxQuoteReader(), - u"~": UnquoteReader()} + u"~": UnquoteReader(), + u"^": MetaReader()} def read_number(rdr, ch): acc = [ch] diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 75b8b6bf..03628cb5 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -12,6 +12,7 @@ def init(): from rpython.rlib.rarithmetic import r_uint from pixie.vm.primitives import nil, true, false from pixie.vm.string import String + from pixie.vm.object import Object _type_registry.set_registry(code._ns_registry) @@ -76,6 +77,8 @@ def wrap(x): return String(x) if isinstance(x, py_str): return String(unicode(x)) + if isinstance(x, Object): + return x affirm(False, u"Bad wrap") globals()["wrap"] = wrap diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index b496939b..bb5b99d4 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm +from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ resize_list, list_copy, returns, get_var_if_defined import pixie.vm.code as code @@ -398,4 +398,31 @@ def promote(i): def invoke_other(obj, args): from pixie.vm.array import array - return rt.apply(_invoke, obj, array(args)) \ No newline at end of file + return rt.apply(_invoke, obj, array(args)) + +@as_var("interpreter_code_info") +def _ici(meta): + import pixie.vm.reader as reader + line = rt._val_at(meta, reader.LINE_KW, nil) + line_number = rt._val_at(meta, reader.LINE_NUMBER_KW, nil) + col_number = rt._val_at(meta, reader.COLUMN_NUMBER_KW, nil) + file = rt._val_at(meta, reader.FILE_KW, nil) + + return InterpreterCodeInfo(line, + line_number.int_val() if line_number is not nil else 0, + col_number.int_val() if col_number is not nil else 0, + rt.name(file) if file is not nil else u" 0, u"Merge takes at least one arg") + x = 1 + acc = args[0] + for x in range(1, len(args)): + acc = rt._reduce(acc, merge_fn, args[x]) + return acc diff --git a/pixie/vm/symbol.py b/pixie/vm/symbol.py index 483f52f9..c052cf80 100644 --- a/pixie/vm/symbol.py +++ b/pixie/vm/symbol.py @@ -13,12 +13,13 @@ class Symbol(object.Object): _type = object.Type(u"pixie.stdlib.Symbol") __immutable_fields__ = ["_hash"] - def __init__(self, s): + def __init__(self, s, meta=nil): #assert isinstance(s, unicode) self._str = s self._w_name = None self._w_ns = None self._hash = 0 + self._meta = meta def type(self): return Symbol._type @@ -36,6 +37,12 @@ def init_names(self): self._w_ns = rt.wrap(s[0]) self._w_name = rt.wrap(u"/".join(s[1:])) + def with_meta(self, meta): + return Symbol(self._str, meta) + + def meta(self): + return self._meta + def symbol(s): @@ -79,3 +86,12 @@ def _symbol(s): +@extend(proto._meta, Symbol) +def _meta(self): + assert isinstance(self, Symbol) + return self.meta() + +@extend(proto._with_meta, Symbol) +def _with_meta(self, meta): + assert isinstance(self, Symbol) + return self.with_meta(meta) \ No newline at end of file From 9317e9960dab7f7904e2c3847082c4c617594d19 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 27 Oct 2014 06:14:06 -0600 Subject: [PATCH 016/909] work on ffi and linux porting stuff --- pixie/stdlib.lisp | 4 ++-- pixie/vm/persistent_hash_map.py | 10 +++++----- pixie/vm/persistent_vector.py | 2 +- pixie/vm/util.py | 10 +++++----- 4 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index cfa930fa..55ac6155 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -400,7 +400,7 @@ ~@proto-bodies))) - (def libc (ffi-library (str "libc" "." pixie.platform/so-ext))) + (def libc (ffi-library (str "libc.so.6"))) (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) - (def printf (ffi-fn libc "printf" [String] Integer)) \ No newline at end of file + (def printf (ffi-fn libc "printf" [String] Integer)) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 0349aec1..cbbf3db9 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -9,7 +9,7 @@ import rpython.rlib.jit as jit import pixie.vm.rt as rt -MASK_32 = 0xFFFFFFFF +MASK_32 = r_uint(0xFFFFFFFF) class Box(py_object): def __init__(self): @@ -241,9 +241,9 @@ def create_node(shift, key1, val1, key2hash, key2, val2): def bit_count(i): assert isinstance(i, r_uint) - i = i - ((i >> 1) & 0x55555555) - i = (i & 0x33333333) + ((i >> 2) & 0x33333333) - return (((i + (i >> 4) & 0xF0F0F0F) * 0x1010101) & 0xffffffff) >> 24 + i = i - ((i >> 1) & r_uint(0x55555555)) + i = (i & r_uint(0x33333333)) + ((i >> 2) & r_uint(0x33333333)) + return (((i + (i >> 4) & r_uint(0xF0F0F0F)) * r_uint(0x1010101)) & r_uint(0xffffffff)) >> 24 @jit.unroll_safe def list_copy(from_lst, from_loc, to_list, to_loc, count): @@ -341,4 +341,4 @@ def _meta(self): @extend(proto._with_meta, PersistentHashMap) def _with_meta(self, meta): assert isinstance(self, PersistentHashMap) - return self.with_meta(meta) \ No newline at end of file + return self.with_meta(meta) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 0510f604..870bc0fa 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -71,7 +71,7 @@ def nth(self, i, not_found=nil): return not_found def conj(self, val): - assert self._cnt < 0xFFFFFFFF + assert self._cnt < r_uint(0xFFFFFFFF) i = self._cnt if self._cnt - self.tailoff() < 32: diff --git a/pixie/vm/util.py b/pixie/vm/util.py index 00c1264e..4d52a09b 100644 --- a/pixie/vm/util.py +++ b/pixie/vm/util.py @@ -2,8 +2,8 @@ from pixie.vm.object import affirm seed = 0 -C1 = 0xcc9e2d51 -C2 = 0x1b873593 +C1 = r_uint(0xcc9e2d51) +C2 = r_uint(0x1b873593) @@ -30,7 +30,7 @@ def mix_k1(k1): def mix_h1(h1, k1): h1 ^= k1 h1 = rotl(h1, 13) - h1 = h1 * 5 + 0xe6546b64 + h1 = h1 * 5 + r_uint(0xe6546b64) return h1 def hash_unencoded_chars(u): @@ -55,9 +55,9 @@ def hash_unencoded_chars(u): def fmix(h1, length): h1 ^= length h1 ^= h1 >> 16 - h1 *= 0x85ebca6b + h1 *= r_uint(0x85ebca6b) h1 ^= h1 >> 13 - h1 *= 0xc2b2ae35 + h1 *= r_uint(0xc2b2ae35) h1 ^= h1 >> 16 return h1 From ec9aa7d6fdbef35182914ea88674154018d666be Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 27 Oct 2014 06:25:10 -0600 Subject: [PATCH 017/909] add platform/lib-c-name --- pixie/stdlib.lisp | 2 +- pixie/vm/libs/platform.py | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 55ac6155..a09aa581 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -400,7 +400,7 @@ ~@proto-bodies))) - (def libc (ffi-library (str "libc.so.6"))) + (def libc (ffi-library pixie.platform/lib-c-name)) (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) (def printf (ffi-fn libc "printf" [String] Integer)) diff --git a/pixie/vm/libs/platform.py b/pixie/vm/libs/platform.py index 9e5f6860..0b7843a8 100644 --- a/pixie/vm/libs/platform.py +++ b/pixie/vm/libs/platform.py @@ -2,10 +2,11 @@ import pixie.vm.rt as rt from pixie.vm.string import String from pixie.vm.code import as_var +from rpython.rlib.clibffi import get_libc_name import os as_var("pixie.platform", "os")(String(unicode(os.name))) as_var("pixie.platform", "name")(String(unicode(platform.name))) as_var("pixie.platform", "so-ext")(String(unicode(platform.so_ext))) - +as_var("pixie.platform", "lib-c-name")(String(unicode(get_libc_name()))) From cc29ada992555cf19812ffbca03b9cf5f9f67722 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 27 Oct 2014 06:48:36 -0600 Subject: [PATCH 018/909] finished adding metadata support to the reader --- pixie/vm/reader.py | 30 +++++++++++++++++++++++++++--- pixie/vm/stdlib.py | 9 +++++++++ target.py | 1 + 3 files changed, 37 insertions(+), 3 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index ff751723..7c78c2e5 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -34,6 +34,9 @@ def read(self): def unread(self, ch): pass + def reset_line(self): + return self + class StringReader(PlatformReader): def __init__(self, s): @@ -69,6 +72,9 @@ def read(self): self._string_reader = None return self.read() + def reset_line(self): + self._string_reader = None + def unread(self, ch): assert self._string_reader is not None self._string_reader.unread(ch) @@ -137,6 +143,13 @@ def read(self): self._cur_ch = ch return ch + def reset_line(self): + self._line.finalize() + self._line_number += 1 + self._column_number = 0 + self._parent_reader.reset_line() + return self + def get_metadata(self): return rt.hashmap(LINE_KW, rt.wrap(self._line), LINE_NUMBER_KW, rt.wrap(self._line_number), @@ -197,7 +210,7 @@ def invoke(self, rdr, ch): class UnmachedListReader(ReaderHandler): def invoke(self, rdr, ch): - raise SyntaxError() + throw_syntax_error_with_data(rdr, u"Unmatched list close ')'") class VectorReader(ReaderHandler): def invoke(self, rdr, ch): @@ -213,7 +226,7 @@ def invoke(self, rdr, ch): class UnmachedVectorReader(ReaderHandler): def invoke(self, rdr, ch): - raise SyntaxError() + throw_syntax_error_with_data(rdr, u"Unmatched vector close ']'") class MapReader(ReaderHandler): def invoke(self, rdr, ch): @@ -419,6 +432,17 @@ class EOF(object.Object): +def throw_syntax_error_with_data(rdr, txt): + assert isinstance(txt, unicode) + if isinstance(rdr, MetaDataReader): + meta = rdr.get_metadata() + else: + meta = nil + + data = rt.interpreter_code_info(meta) + err = object.RuntimeException(rt.wrap(txt)) + err._trace.append(data) + raise object.WrappedException(err) @@ -459,7 +483,7 @@ def read(rdr, error_on_eof): itm = read_symbol(rdr, ch) if rt.has_meta_QMARK_(itm): - itm = rt.with_meta(itm, meta) + itm = rt.with_meta(itm, rt.merge(meta, rt._meta(itm))) return itm diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index bb5b99d4..10948455 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -109,6 +109,14 @@ def _seq(_): def _count(_): return numbers.zero_int +@extend(_assoc, nil._type) +def __assoc(_, k, v): + return rt.hashmap(k, v) + +@extend(_reduce, nil._type) +def __reudce(self, f, init): + return init + @extend(_name, Var) def __name(self): assert isinstance(self, Var) @@ -419,6 +427,7 @@ def merge_fn(acc, x): @as_var("merge") +@jit.unroll_safe def _merge__args(args): affirm(len(args) > 0, u"Merge takes at least one arg") x = 1 diff --git a/target.py b/target.py index ca8eba5b..def9e19e 100644 --- a/target.py +++ b/target.py @@ -49,6 +49,7 @@ def repl(): val = interpret(compile(val)) except WrappedException as ex: print "Error: ", ex._ex.__repr__() + rdr.reset_line() continue if val is keyword(u"exit-repl"): break From 55c83e39a02e4d78012eb78fb150126f0461756b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 27 Oct 2014 13:32:02 -0600 Subject: [PATCH 019/909] added run-interpreted script --- run-interpreted | 1 + 1 file changed, 1 insertion(+) create mode 100755 run-interpreted diff --git a/run-interpreted b/run-interpreted new file mode 100755 index 00000000..1f62325c --- /dev/null +++ b/run-interpreted @@ -0,0 +1 @@ +PYTHONPATH=$PYTHONPATH:../externals/pypy python target.py From 9ee1ad0c1dc1632c5ae2bf21f03cbcbcb8b13e74 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 27 Oct 2014 22:12:48 +0100 Subject: [PATCH 020/909] support more num literals similar to the ones in clojure (and the regex is very much inspired by the one in tools.reader), e.g. hexadecimal, octal and my personal favourite: radix are all there. i think this breaks when the parsing fails, but it parses different numbers for now and addition still works. --- pixie/vm/reader.py | 33 ++++++++++++++++++++++++++++++++- 1 file changed, 32 insertions(+), 1 deletion(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 7c78c2e5..bb73b536 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -16,6 +16,8 @@ import pixie.vm.stdlib as proto import pixie.vm.compiler as compiler +from rpython.rlib.rsre import rsre_re as re + LINE_NUMBER_KW = keyword(u"line-number") COLUMN_NUMBER_KW = keyword(u"column-number") LINE_KW = keyword(u"line") @@ -388,6 +390,35 @@ def invoke(self, rdr, ch): u"~": UnquoteReader(), u"^": MetaReader()} +# inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 +# sign hex oct radix decimal +# 1 2 3 4 5 6 7 +int_matcher = re.compile('([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))') + +def parse_int(s): + m = int_matcher.match(s) + + sign = 1 + if m.group(1) == '-': + sign = -1 + + radix = 10 + num = None + + if m.group(7): + num = m.group(7) + elif m.group(2): + radix = 16 + num = m.group(3) + elif m.group(4): + radix = 7 + num = m.group(4) + elif m.group(5): + radix = int(m.group(5)) + num = m.group(6) + + return sign * int(num, radix) + def read_number(rdr, ch): acc = [ch] try: @@ -400,7 +431,7 @@ def read_number(rdr, ch): except EOFError: pass - return rt.wrap(int(u"".join(acc))) + return rt.wrap(parse_int(u"".join(acc))) def read_symbol(rdr, ch): acc = [ch] From 655b0c5560af064a1c7df25e27aceaabd8f10f86 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 27 Oct 2014 17:37:42 -0600 Subject: [PATCH 021/909] we have docstrings --- pixie/stdlib.lisp | 8 ++++++- pixie/vm/code.py | 38 +++++++++++++++++++++++++++------ pixie/vm/compiler.py | 6 ++++-- pixie/vm/persistent_hash_map.py | 3 ++- pixie/vm/reader.py | 2 +- pixie/vm/rt.py | 1 + pixie/vm/stdlib.py | 16 ++++++++++++++ 7 files changed, 62 insertions(+), 12 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index a09aa581..dffdefa1 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -2,7 +2,9 @@ (def reset! -reset!) -(def map (fn map [f] + +(def map (fn ^{:doc "map - creates a transducer that applies f to every input element" :added "0.1"} + map [f] (fn [xf] (fn ([] (xf)) @@ -404,3 +406,7 @@ (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) (def printf (ffi-fn libc "printf" [String] Integer)) + + +(defn doc [x] + (get (meta x) :doc)) \ No newline at end of file diff --git a/pixie/vm/code.py b/pixie/vm/code.py index c79f1dc0..01183c7e 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -87,6 +87,13 @@ class BaseCode(object.Object): def __init__(self): assert isinstance(self, BaseCode) self._is_macro = False + self._meta = nil + + def meta(self): + return self._meta + + def with_meta(self, meta): + assert false, "not implemented" def set_macro(self): self._is_macro = True @@ -120,11 +127,15 @@ class MultiArityFn(BaseCode): def type(self): return MultiArityFn._type - def __init__(self, arities, required_arity=0, rest_fn=None): + def __init__(self, arities, required_arity=0, rest_fn=None, meta=nil): BaseCode.__init__(self) self._arities = arities self._required_arity = required_arity self._rest_fn = rest_fn + self._meta = meta + + def with_meta(self, meta): + return MultiArityFn(self._arities, self._required_arity, self._rest_fn, meta) @elidable_promote() def get_fn(self, arity): @@ -169,18 +180,22 @@ def inner_invoke(self, args): class Code(BaseCode): """Interpreted code block. Contains consts and """ _type = object.Type(u"Code") - __immutable_fields__ = ["_consts[*]", "_bytecode", "_stack_size"] + __immutable_fields__ = ["_consts[*]", "_bytecode", "_stack_size", "_meta"] def type(self): return Code._type - def __init__(self, name, bytecode, consts, stack_size, debug_points): + def __init__(self, name, bytecode, consts, stack_size, debug_points, meta=nil): BaseCode.__init__(self) self._bytecode = bytecode self._consts = consts self._name = name self._stack_size = stack_size self._debug_points = debug_points + self._meta = meta + + def with_meta(self, meta): + return Code(self._name, self._bytecode, self._consts, self._stack_size, self._debug_points, meta) def get_debug_points(self): return self._debug_points @@ -211,16 +226,20 @@ def get_base_code(self): class VariadicCode(BaseCode): - __immutable_fields__ = ["_required_arity", "_code"] + __immutable_fields__ = ["_required_arity", "_code", "_meta"] _type = object.Type(u"pixie.stdlib.VariadicCode") def type(self): return VariadicCode._type - def __init__(self, code, required_arity): + def __init__(self, code, required_arity, meta=nil): BaseCode.__init__(self) self._required_arity = r_uint(required_arity) self._code = code + self._meta = meta + + def with_meta(self, meta): + return VariadicCode(self._code, self._required_arity, meta) def _invoke(self, args): from pixie.vm.array import array @@ -240,15 +259,19 @@ def _invoke(self, args): class Closure(BaseCode): _type = object.Type(u"Closure") - __immutable_fields__ = ["_closed_overs[*]", "_code"] + __immutable_fields__ = ["_closed_overs[*]", "_code", "_meta"] def type(self): return Closure._type - def __init__(self, code, closed_overs): + def __init__(self, code, closed_overs, meta=nil): BaseCode.__init__(self) affirm(isinstance(code, Code), u"Code argument to Closure must be an instance of Code") self._code = code self._closed_overs = closed_overs + self._meta = meta + + def with_meta(self, meta): + return Closure(self._code, self._closed_overs, meta) def _invoke(self, args): try: @@ -319,6 +342,7 @@ def __init__(self, ns, name): self._dynamic = False def set_root(self, o): + affirm(o is not None, u"Invalid var set") self._rev += 1 self._root = o return self diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 08285ad3..ab852b7f 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -289,10 +289,11 @@ def compile_meta(meta, ctx): ctx.push_const(code.intern_var(u"pixie.stdlib", u'with-meta')) ctx.bytecode.append(code.DUP_NTH) ctx.bytecode.append(r_uint(1)) + ctx.add_sp(1) ctx.push_const(meta) ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(3)) - ctx.sub_sp(1) + ctx.sub_sp(2) ctx.bytecode.append(code.POP_UP_N) ctx.bytecode.append(1) ctx.sub_sp(1) @@ -428,7 +429,8 @@ def compile_fn(form, ctx): else: compile_fn_body(name, rt.first(form), rt.next(form), ctx) - + if rt.meta(name) is not nil: + compile_meta(rt.meta(name), ctx) def compile_fn_body(name, args, body, ctx): diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index cbbf3db9..d5026a76 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -154,7 +154,7 @@ def reduce_inode(self, f, init): key_or_none = self._array[x] val_or_node = self._array[x + 1] if key_or_none is None and val_or_node is not None: - init = init.reduce_inode(f, init) + init = val_or_node.reduce_inode(f, init) else: init = f.invoke([init, rt.map_entry(key_or_none, val_or_node)]) if rt.reduced_QMARK_(init): @@ -275,6 +275,7 @@ def clone_and_set2(array, i, a, j, b): idx = r_uint(0) while idx < len(array): clone[idx] = array[idx] + idx += 1 clone[i] = a clone[j] = b diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 7c78c2e5..820960ae 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -483,7 +483,7 @@ def read(rdr, error_on_eof): itm = read_symbol(rdr, ch) if rt.has_meta_QMARK_(itm): - itm = rt.with_meta(itm, rt.merge(meta, rt._meta(itm))) + itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) return itm diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 03628cb5..6d903e5e 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -64,6 +64,7 @@ def wrapper(*args): import pixie.vm.reader as reader import pixie.vm.libs.platform import pixie.vm.libs.ffi + import pixie.vm.symbol diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 10948455..d862ef28 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -53,6 +53,22 @@ defprotocol("pixie.stdlib", "ITransientCollection", ["-conj!"]) +def __make_code_overrides(x): + @extend(_meta, x._type) + def __meta(self): + assert isinstance(self, x) + return self.meta() + + @extend(_with_meta, x._type) + def __meta(self, meta): + assert isinstance(self, x) + return self.with_meta(meta) + +for x in (code.Code, code.Closure, code.VariadicCode, code.MultiArityFn): + __make_code_overrides(x) + + + def default_str(x): from pixie.vm.string import String From 5ed419a048d0407c847ebb5491b45c5b2d84a518 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 27 Oct 2014 21:28:21 -0600 Subject: [PATCH 022/909] fixes for enhanced int parsing --- pixie/vm/reader.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index bb73b536..3260513c 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -393,17 +393,16 @@ def invoke(self, rdr, ch): # inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 # sign hex oct radix decimal # 1 2 3 4 5 6 7 -int_matcher = re.compile('([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))') +int_matcher = re.compile(u'([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))') def parse_int(s): m = int_matcher.match(s) sign = 1 - if m.group(1) == '-': + if m.group(1) == u'-': sign = -1 radix = 10 - num = None if m.group(7): num = m.group(7) @@ -416,8 +415,10 @@ def parse_int(s): elif m.group(5): radix = int(m.group(5)) num = m.group(6) + else: + return None - return sign * int(num, radix) + return rt.wrap(sign * int(str(num), radix)) def read_number(rdr, ch): acc = [ch] @@ -431,7 +432,11 @@ def read_number(rdr, ch): except EOFError: pass - return rt.wrap(parse_int(u"".join(acc))) + joined = u"".join(acc) + parsed = parse_int(joined) + if parsed is not None: + return parsed + return Symbol(joined) def read_symbol(rdr, ch): acc = [ch] @@ -506,6 +511,7 @@ def read(rdr, error_on_eof): if is_digit(ch2): rdr.unread(ch2) itm = read_number(rdr, ch) + else: rdr.unread(ch2) itm = read_symbol(rdr, ch) From 555a378d05b6a86e94b0a1ca7be9a1c3ed72d654 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 28 Oct 2014 07:00:10 -0600 Subject: [PATCH 023/909] added more metadata support, dynamic var meta data --- pixie/vm/compiler.py | 17 ++++++++++++++++- pixie/vm/interpreter.py | 12 ++++++++++-- pixie/vm/persistent_hash_map.py | 1 + pixie/vm/reader.py | 13 ++++++++----- 4 files changed, 35 insertions(+), 8 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index ab852b7f..15b801c1 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -5,7 +5,7 @@ from pixie.vm.cons import cons, Cons import pixie.vm.symbol as symbol import pixie.vm.code as code -from pixie.vm.keyword import Keyword +from pixie.vm.keyword import Keyword, keyword from pixie.vm.string import String from pixie.vm.atom import Atom import pixie.vm.stdlib as proto @@ -20,6 +20,8 @@ FN_NAME = code.intern_var(u"pixie.stdlib", u"*fn-name*") FN_NAME.set_dynamic() +DYNAMIC_KW = keyword(u"dynamic") + gensym_id = Atom(numbers.zero_int) @@ -320,6 +322,10 @@ def compile_form(form, ctx): ctx.push_const(var) + meta = rt.meta(form) + if meta is not nil: + ctx.debug_points[len(ctx.bytecode)] = rt.interpreter_code_info(meta) + ctx.bytecode.append(code.DEREF_VAR) return loc.emit(ctx) @@ -515,6 +521,12 @@ def compile_def(form, ctx): affirm(isinstance(name, symbol.Symbol), u"Def'd name must be a symbol") var = NS_VAR.deref().intern_or_make(rt.name(name)) + + if rt._val_at(rt.meta(name), DYNAMIC_KW, nil) is true: + assert isinstance(var, code.Var) + var.set_dynamic() + + ctx.push_const(var) compile_form(val, ctx) ctx.bytecode.append(code.SET_VAR) @@ -536,6 +548,9 @@ def compile_quote(form, ctx): data = rt.first(rt.next(form)) ctx.push_const(data) + if rt.meta(form) is not nil: + compile_meta(rt.meta(form), ctx) + def compile_recur(form, ctx): form = form.next() affirm(ctx.can_tail_call, u"Can't recur in non-tail position") diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index bd9a01be..c09e51e0 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -246,11 +246,19 @@ def interpret(code_obj, args=[]): continue if inst == code.DEREF_VAR: + debug_ip = frame.ip var = frame.pop() if not isinstance(var, code.Var): affirm(False, u"Can't deref " + var.type()._name) - frame.push(var.deref()) - continue + try: + frame.push(var.deref()) + continue + except WrappedException as ex: + dp = frame.debug_points.get(debug_ip - 1, None) + if dp: + ex._ex._trace.append(dp) + raise + if inst == code.RECUR: argc = frame.get_inst() diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index d5026a76..06eae851 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -264,6 +264,7 @@ def clone_and_set(array, i, a): idx = r_uint(0) while idx < len(array): clone[idx] = array[idx] + idx += 1 clone[i] = a return clone diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index de1ed178..37339787 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -6,7 +6,7 @@ import pixie.vm.numbers as numbers from pixie.vm.cons import cons from pixie.vm.symbol import symbol, Symbol -from pixie.vm.keyword import keyword +from pixie.vm.keyword import keyword, Keyword import pixie.vm.rt as rt from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR from pixie.vm.libs.readline import _readline @@ -94,8 +94,9 @@ def add_char(self, ch): self._chrs.append(ch) def finalize(self): - self._str = u"".join(self._chrs) - self._chrs = None + if self._chrs is not None: + self._str = u"".join(self._chrs) + self._chrs = None def is_finalized(self): return self._chrs is None @@ -370,10 +371,12 @@ def invoke(self, rdr, ch): meta = read(rdr, True) obj = read(rdr, True) + if isinstance(meta, Keyword): + meta = rt.hashmap(meta, true) + if rt.instance_QMARK_(rt.IMeta.deref(), obj): + return rt.with_meta(obj, rt.merge(meta, rt.meta(obj))) - return rt._with_meta(obj, meta) - print "not meta" return obj handlers = {u"(": ListReader(), From ce2cfa450427f96d889b6624550ca873d61f4753 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 28 Oct 2014 08:53:25 -0600 Subject: [PATCH 024/909] added a disclaimer bit --- README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c71e2269..d89c10b0 100644 --- a/README.md +++ b/README.md @@ -110,6 +110,9 @@ Currently there isn't a whole lot that newcomers can help out with, since the en Although parts of the language may be very close to Clojure (they are both lisps after all), language parity is not a design goal. We will take the features from Clojure or other languages that are suitable to our needs, and feel free to reject those that aren't. Therefore this should not be considered a "Clojure Dialect", but instead a "Clojure inspired lisp". +## Disclaimer +This project is the personal work of Timothy Baldridge and contributors. It is not supported by any entity, including Timothy's employer, or any employers of any other contributors. + ## Copying -Free use of this software is granted under the terms of the GNU Lesser General Public License (LGPL). For details see the files `COPYING` and `COPYING.LESSER` included with the source distribution. +Free use of this software is granted under the terms of the GNU Lesser General Public License (LGPL). For details see the files `COPYING` and `COPYING.LESSER` included with the source distribution. All copyrights are owned by their respective authors. From 5dc4a28ea631e0cd53c12ac6add5f34173c033df Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 28 Oct 2014 16:44:03 +0100 Subject: [PATCH 025/909] add a makefile i want this because this allows me to specify which python to use, because my system doesn't have a python2 as it's default python. otherwise i'd have to change all the scripts. it also has the advantage of having all the utilities in one place and even a "help" command. --- Makefile | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 Makefile diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..46844a01 --- /dev/null +++ b/Makefile @@ -0,0 +1,35 @@ +all: help + +EXTERNALS=../externals + +PYTHON=python +PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy + +help: + @echo "make help - display this message" + @echo "make run - run the compiled interpreter" + @echo "make run_interactive - run without compiling (slow)" + @echo "make build_with_jit - build with jit enabled" + @echo "make build_no_jit - build without jit" + +build_with_jit: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --continuation --no-shared target.py + +build_no_jit: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --continuation --no-shared target.py + +fetch_externals: $(EXTERNALS)/pypy + +$(EXTERNALS)/pypy: + mkdir $(EXTERNALS) + cd $(EXTERNALS) + curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2 + mkdir pypy + cd pypy + tar -jxf ../pypy.tar.bz2 --strip-components=1 + +run: + ./pixie-vm + +run_interactive: + PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py From 7f5ceca023d418b7776664934a6517aabb34ce1c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 28 Oct 2014 16:48:21 +0100 Subject: [PATCH 026/909] support specifying PYTHON as via the environment e.g. the following works: ``` $ export PYTHON=python2 $ make run_interactive ``` --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 46844a01..7e62d3dc 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=../externals -PYTHON=python +PYTHON ?= python PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy help: From 32fa6a26fb4f4af00c2e86912225e09472308dc5 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 28 Oct 2014 18:00:05 +0100 Subject: [PATCH 027/909] initial float support they are parsed correctly and you can add them directly with -add, but it only works in the interactive mode. compilation fails with a "don't know how to represent" error. --- pixie/vm/compiler.py | 5 ++++- pixie/vm/numbers.py | 45 ++++++++++++++++++++++++++++++++++++++++++++ pixie/vm/reader.py | 26 +++++++++++++++++++------ pixie/vm/rt.py | 2 ++ 4 files changed, 71 insertions(+), 7 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 15b801c1..bb266e2b 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -310,6 +310,9 @@ def compile_form(form, ctx): if isinstance(form, numbers.Integer): ctx.push_const(form) return + if isinstance(form, numbers.Float): + ctx.push_const(form) + return if isinstance(form, symbol.Symbol): name = form._str @@ -726,4 +729,4 @@ def compile(form): ctx = Context(u"main", 0, None) compile_form(form, ctx) ctx.bytecode.append(code.RETURN) - return ctx.to_code() \ No newline at end of file + return ctx.to_code() diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 4a7cfa5d..710759db 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -32,6 +32,19 @@ def type(self): zero_int = Integer(0) one_int = Integer(1) +class Float(Number): + _type = object.Type(u"pixie.stdlib.Float") + _immutable_fields_ = ["_float_val"] + + def __init__(self, f_val): + self._float_val = f_val + + def float_val(self): + return self._float_val + + def type(self): + return Float._type + IMath = as_var("IMath")(Protocol(u"IMath")) _add = as_var("-add")(DoublePolymorphicFn(u"-add", IMath)) _sub = as_var("-sub")(DoublePolymorphicFn(u"-sub", IMath)) @@ -83,6 +96,32 @@ def eq(a, b): raise Exception("Add error") +@extend(_add, Float._type, Float._type) +def _add(a, b): + assert isinstance(a, Float) and isinstance(b, Float) + return Float(a.float_val() + b.float_val()) + +@extend(_sub, Float._type, Float._type) +def _sub(a, b): + assert isinstance(a, Float) and isinstance(b, Float) + return rt.wrap(a.float_val() - b.float_val()) + +@extend(_mul, Float._type, Float._type) +def _mul(a, b): + assert isinstance(a, Float) and isinstance(b, Float) + return rt.wrap(a.float_val() * b.float_val()) + +@extend(_div, Float._type, Float._type) +def _div(a, b): + assert isinstance(a, Float) and isinstance(b, Float) + return rt.wrap(a.float_val() / b.float_val()) + +@extend(_num_eq, Float._type, Float._type) +def _num_eq(a, b): + assert isinstance(a, Float) and isinstance(b, Float) + return true if a.float_val() == b.float_val() else false + + def init(): import pixie.vm.stdlib as proto from pixie.vm.string import String @@ -95,4 +134,10 @@ def _str(i): def _repr(i): return rt.wrap(unicode(str(i.int_val()))) + @extend(proto._str, Float._type) + def _str(i): + return rt.wrap(unicode(str(i.float_val()))) + @extend(proto._repr, Float._type) + def _repr(i): + return rt.wrap(unicode(str(i.float_val()))) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 37339787..7214f242 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -394,13 +394,13 @@ def invoke(self, rdr, ch): u"^": MetaReader()} # inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 -# sign hex oct radix decimal -# 1 2 3 4 5 6 7 -int_matcher = re.compile(u'([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))') +# sign hex oct radix decimal +# 1 2 3 4 5 6 7 +int_matcher = re.compile(u'^([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))$') -def parse_int(s): - m = int_matcher.match(s) +float_matcher = re.compile(u'^([-+]?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+)?)$') +def parse_int(m): sign = 1 if m.group(1) == u'-': sign = -1 @@ -423,6 +423,20 @@ def parse_int(s): return rt.wrap(sign * int(str(num), radix)) +def parse_float(m): + return rt.wrap(float(str(m.string))) + +def parse_number(s): + m = int_matcher.match(s) + if m: + return parse_int(m) + else: + m = float_matcher.match(s) + if m: + return parse_float(m) + else: + return None + def read_number(rdr, ch): acc = [ch] try: @@ -436,7 +450,7 @@ def read_number(rdr, ch): pass joined = u"".join(acc) - parsed = parse_int(joined) + parsed = parse_number(joined) if parsed is not None: return parsed return Symbol(joined) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 6d903e5e..9e6934f2 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -74,6 +74,8 @@ def wrapper(*args): def wrap(x): if isinstance(x, int): return numbers.Integer(x) + if isinstance(x, float): + return numbers.Float(x) if isinstance(x, unicode): return String(x) if isinstance(x, py_str): From f1c906f1b2c54165fe944e77d661eccb012b5fae Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 28 Oct 2014 18:05:52 +0100 Subject: [PATCH 028/909] fix variable names --- pixie/vm/numbers.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 710759db..48e0b1ab 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -135,9 +135,9 @@ def _repr(i): return rt.wrap(unicode(str(i.int_val()))) @extend(proto._str, Float._type) - def _str(i): - return rt.wrap(unicode(str(i.float_val()))) + def _str(f): + return rt.wrap(unicode(str(f.float_val()))) @extend(proto._repr, Float._type) - def _repr(i): - return rt.wrap(unicode(str(i.float_val()))) + def _repr(f): + return rt.wrap(unicode(str(f.float_val()))) From cba0a771da747d4fd1ccebaf7d1e1bd1e3209cc6 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 28 Oct 2014 20:42:47 +0100 Subject: [PATCH 029/909] fix rtyping error somehow rpython can't figure out that m.string has type string, but it can figure out that m.group(0) is a string. are there annotations to tell rpython that something is a string? (or has some specific type in general?) --- pixie/vm/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 7214f242..d1de3ed1 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -424,7 +424,7 @@ def parse_int(m): return rt.wrap(sign * int(str(num), radix)) def parse_float(m): - return rt.wrap(float(str(m.string))) + return rt.wrap(float(str(m.group(0)))) def parse_number(s): m = int_matcher.match(s) From d85ffe13e214364fc3d7ce88273d9aa5a3191ab7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 28 Oct 2014 20:52:44 +0100 Subject: [PATCH 030/909] support adding ints to floats --- pixie/vm/numbers.py | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 48e0b1ab..f68d2085 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -101,6 +101,16 @@ def _add(a, b): assert isinstance(a, Float) and isinstance(b, Float) return Float(a.float_val() + b.float_val()) +@extend(_add, Integer._type, Float._type) +def _add(a, b): + assert isinstance(a, Integer) and isinstance(b, Float) + return rt.wrap(a.int_val() + b.float_val()) + +@extend(_add, Float._type, Integer._type) +def _add(a, b): + assert isinstance(a, Float) and isinstance(b, Integer) + return rt.wrap(a.float_val() + b.int_val()) + @extend(_sub, Float._type, Float._type) def _sub(a, b): assert isinstance(a, Float) and isinstance(b, Float) From 151b9fad943f8a45e60ee607e9b3be77dc3b7760 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 28 Oct 2014 17:15:47 -0600 Subject: [PATCH 031/909] more work on the testing framework --- pixie/stdlib.lisp | 30 ++++++++++++++++++++++++++++-- pixie/vm/compiler.py | 24 +++++++++++++++++++++--- pixie/vm/interpreter.py | 4 ++-- pixie/vm/reader.py | 3 ++- pixie/vm/stdlib.py | 4 ++++ 5 files changed, 57 insertions(+), 8 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index dffdefa1..4f72ca9a 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -351,7 +351,7 @@ (reduce (fn [_ map-entry] (set! (resolve (key map-entry)) (val map-entry))) nil - (apply hashmap ~binds)) + (apply hashmap ~@binds)) (let [ret (do ~@body)] (pop-binding-frame!) ret))) @@ -407,6 +407,32 @@ (def puts (ffi-fn libc "puts" [String] Integer)) (def printf (ffi-fn libc "printf" [String] Integer)) +(defn print [& args] + (puts (apply str args))) + (defn doc [x] - (get (meta x) :doc)) \ No newline at end of file + (get (meta x) :doc)) + +(defn swap! [a f & args] + (reset! a (apply f @a args))) + +(def update-inner-f (fn inner-f + ([m f k] + (assoc m k (f (get m k)))) + ([m f k & ks] + (assoc m k (apply update-inner-f m f ks))))) + +(defn update-in + [m ks f & args] + (let [f (fn [m] (apply f m args))] + (apply update-inner-f m f ks))) + +(defn nil? [x] + (identical? x nil)) + +(defn fnil [f else] + (fn [x & args] + (apply f (if (nil? x) else x) args))) + + diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 15b801c1..c5214d7b 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -391,9 +391,10 @@ def compile_platform_eq(form, ctx): ctx.sub_sp(1) return ctx -def add_args(args, ctx): +def add_args(name, args, ctx): required_args = -1 local_idx = 0 + #ctx.add_local(name, Self()) for x in range(rt.count(args)): arg = rt.nth(args, rt.wrap(x)) affirm(isinstance(arg, symbol.Symbol), u"Argument names must be symbols") @@ -441,7 +442,7 @@ def compile_fn(form, ctx): def compile_fn_body(name, args, body, ctx): new_ctx = Context(name._str, rt.count(args), ctx) - required_args = add_args(args, new_ctx) + required_args = add_args(name, args, new_ctx) bc = 0 if name is not None: @@ -663,6 +664,21 @@ def compile_ns(form, ctx): def compile_this_ns(form, ctx): ctx.push_const(NS_VAR.deref()) +def compile_var(form, ctx): + form = rt.next(form) + name = rt.first(form) + + affirm(isinstance(name, symbol.Symbol), u"var name must be a symbol") + + if rt.namespace(name) is not None: + var = code._ns_registry.find_or_make(rt.namespace(name)) + else: + var = NS_VAR.deref().intern_or_make(rt.name(name)) + + ctx.push_const(var) + +def compile_catch(form, ctx): + affirm(False, u"Catch used outside of try") builtins = {u"fn": compile_fn, u"if": compile_if, @@ -674,7 +690,9 @@ def compile_this_ns(form, ctx): u"let": compile_let, u"loop": compile_loop, u"comment": compile_comment, + u"var": compile_var, u"__ns__": compile_ns, + u"catch": compile_catch, u"this-ns-name": compile_this_ns} def compiler_special(s): @@ -726,4 +744,4 @@ def compile(form): ctx = Context(u"main", 0, None) compile_form(form, ctx) ctx.bytecode.append(code.RETURN) - return ctx.to_code() \ No newline at end of file + return ctx.to_code() diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index c09e51e0..beb94d06 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -82,7 +82,7 @@ def push_nth(self, delta): self.push(self.nth(delta)) def push_arg(self, idx): - assert 0 <= idx < len(self.args) + affirm(0 <= idx < len(self.args), u"Argument doesn't exist") self.push(self.args[r_uint(idx)]) @unroll_safe @@ -132,7 +132,7 @@ def make_multi_arity(frame, argc): return code.MultiArityFn(d, required_arity, rest_fn) -def interpret(code_obj, args=[]): +def interpret(code_obj, args=[], self_obj = None): frame = Frame(code_obj, args) while True: jitdriver.jit_merge_point(bc=frame.bc, diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 37339787..3e2c2002 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -312,7 +312,8 @@ def invoke(self, rdr, ch): def syntax_quote(form): if isinstance(form, Symbol) and compiler.is_compiler_special(form): ret = rt.list(QUOTE, form) - if isinstance(form, Symbol): + + elif isinstance(form, Symbol): if rt.namespace(form) is None and rt.name(form).endswith("#"): gmap = rt.deref(GEN_SYM_ENV) affirm(gmap is not nil, u"Gensym literal used outside a syntax quote") diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index d862ef28..e89765bc 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -133,6 +133,10 @@ def __assoc(_, k, v): def __reudce(self, f, init): return init +@extend(_val_at, nil._type) +def __val_at(x, k, not_found): + return not_found + @extend(_name, Var) def __name(self): assert isinstance(self, Var) From 32f64df1fd86888f36fb231a62a566c804e2ae47 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 06:25:57 -0600 Subject: [PATCH 032/909] added load-paths to stdlib --- pixie/stdlib.lisp | 2 ++ pixie/vm/object.py | 8 ++++++++ pixie/vm/rt.py | 7 +++++++ pixie/vm/stdlib.py | 20 ++++++++++++++++++-- 4 files changed, 35 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 4f72ca9a..775ef306 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -2,6 +2,8 @@ (def reset! -reset!) +(def load-paths (atom ["./"])) + (def map (fn ^{:doc "map - creates a transducer that applies f to every input element" :added "0.1"} map [f] diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 51c22acd..9a53f1c5 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -12,6 +12,14 @@ def invoke(self, args): import pixie.vm.stdlib as stdlib return stdlib.invoke_other(self, args) + def int_val(self): + affirm(False, u"Expected Number") + return 0 + + def r_uint_val(self): + affirm(False, u"Expected Number") + return 0 + def promote(self): return self diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 6d903e5e..34d30881 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -84,6 +84,13 @@ def wrap(x): globals()["wrap"] = wrap + + def int_val(x): + affirm(isinstance(x, numbers.Number), u"Expected number") + return x.int_val() + + globals()["int_val"] = int_val + from pixie.vm.code import _ns_registry, BaseCode, munge for name, var in _ns_registry._registry[u"pixie.stdlib"]._registry.iteritems(): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index e89765bc..f6695bae 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -7,6 +7,7 @@ from pixie.vm.primitives import true, false, nil import pixie.vm.numbers as numbers import rpython.rlib.jit as jit +import os.path as path import rpython.rlib.rstacklet as rstacklet from rpython.rlib.rarithmetic import r_uint @@ -52,6 +53,9 @@ defprotocol("pixie.stdlib", "ITransientCollection", ["-conj!"]) +defprotocol("pixie.stdlib", "IIterable", ["-iterator"]) +defprotocol("pixie.stdlib", "IIterator", ["-current", "-has-next?", "-move-next!"]) + def __make_code_overrides(x): @extend(_meta, x._type) @@ -262,6 +266,7 @@ def load_file(filename): import pixie.vm.symbol as symbol import os.path as path if isinstance(filename, symbol.Symbol): + affirm(rt.namespace(filename) is None, u"load-file takes a un-namespaced symbol") filename_str = rt.name(filename).replace(u".", u"/") + u".lisp" @@ -273,9 +278,20 @@ def load_file(filename): affirm(isinstance(filename, string.String), u"Filename must be string") filename_str = filename._str - affirm(path.isfile(str(filename_str)), u"File does not exist on path") - f = open(str(filename_str)) + paths = rt.deref(rt.deref(rt.load_paths)) + f = None + for x in range(rt.count(paths)): + path_x = rt.nth(paths, rt.wrap(x)) + affirm(isinstance(path_x, string.String), u"Contents of load-paths must be strings") + full_path = path.join(rt.name(path_x), filename_str) + if path.isfile(full_path): + f = open(str(full_path)) + break + + if f is None: + affirm(False, u"File does not exist in any directory found in load-paths") + data = f.read() f.close() rdr = reader.StringReader(unicode(data)) From 9e892a1003ea11ba07a37de5f39254f41fc97cae Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 06:36:17 -0600 Subject: [PATCH 033/909] fix translation --- pixie/vm/stdlib.py | 24 +++++++++++++----------- 1 file changed, 13 insertions(+), 11 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index f6695bae..f212ed02 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -276,7 +276,7 @@ def load_file(filename): else: affirm(isinstance(filename, string.String), u"Filename must be string") - filename_str = filename._str + filename_str = rt.name(filename) paths = rt.deref(rt.deref(rt.load_paths)) @@ -284,24 +284,26 @@ def load_file(filename): for x in range(rt.count(paths)): path_x = rt.nth(paths, rt.wrap(x)) affirm(isinstance(path_x, string.String), u"Contents of load-paths must be strings") - full_path = path.join(rt.name(path_x), filename_str) + full_path = path.join(str(rt.name(path_x)), str(filename_str)) if path.isfile(full_path): f = open(str(full_path)) break if f is None: affirm(False, u"File does not exist in any directory found in load-paths") + else: + data = f.read() + f.close() - data = f.read() - f.close() - rdr = reader.StringReader(unicode(data)) + rdr = reader.StringReader(unicode(data)) - with compiler.with_ns(u"user"): - while True: - form = reader.read(rdr, False) - if form is reader.eof: - return nil - result = compiler.compile(form).invoke([]) + with compiler.with_ns(u"user"): + while True: + form = reader.read(rdr, False) + if form is reader.eof: + return nil + result = compiler.compile(form).invoke([]) + return nil @as_var("the-ns") def the_ns(ns_name): From 13f60d50bab88ff86be9b9a0970cb664e3701ab9 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 14:04:23 +0100 Subject: [PATCH 034/909] generate the num ops programmatically --- pixie/vm/numbers.py | 85 ++++++++++++--------------------------------- 1 file changed, 23 insertions(+), 62 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index f68d2085..3fd46bdc 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -54,33 +54,6 @@ def type(self): _num_eq.set_default_fn(wrap_fn(lambda a, b: false)) - -@extend(_add, Integer._type, Integer._type) -def _add(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return Integer(a.int_val() + b.int_val()) - -@extend(_sub, Integer._type, Integer._type) -def _sub(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return rt.wrap(a.int_val() - b.int_val()) - -@extend(_mul, Integer._type, Integer._type) -def _mul(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return rt.wrap(a.int_val() * b.int_val()) - -@extend(_div, Integer._type, Integer._type) -def _div(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return rt.wrap(a.int_val() / b.int_val()) - -@extend(_num_eq, Integer._type, Integer._type) -def _div(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return true if a.int_val() == b.int_val() else false - - # def add(a, b): # if isinstance(a, Integer): # if isinstance(b, Integer): @@ -95,41 +68,29 @@ def eq(a, b): raise Exception("Add error") - -@extend(_add, Float._type, Float._type) -def _add(a, b): - assert isinstance(a, Float) and isinstance(b, Float) - return Float(a.float_val() + b.float_val()) - -@extend(_add, Integer._type, Float._type) -def _add(a, b): - assert isinstance(a, Integer) and isinstance(b, Float) - return rt.wrap(a.int_val() + b.float_val()) - -@extend(_add, Float._type, Integer._type) -def _add(a, b): - assert isinstance(a, Float) and isinstance(b, Integer) - return rt.wrap(a.float_val() + b.int_val()) - -@extend(_sub, Float._type, Float._type) -def _sub(a, b): - assert isinstance(a, Float) and isinstance(b, Float) - return rt.wrap(a.float_val() - b.float_val()) - -@extend(_mul, Float._type, Float._type) -def _mul(a, b): - assert isinstance(a, Float) and isinstance(b, Float) - return rt.wrap(a.float_val() * b.float_val()) - -@extend(_div, Float._type, Float._type) -def _div(a, b): - assert isinstance(a, Float) and isinstance(b, Float) - return rt.wrap(a.float_val() / b.float_val()) - -@extend(_num_eq, Float._type, Float._type) -def _num_eq(a, b): - assert isinstance(a, Float) and isinstance(b, Float) - return true if a.float_val() == b.float_val() else false +num_op_template = """@extend({pfn}, {ty1}._type, {ty2}._type) +def {pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + return {wrap_start}a.{conv1}() {op} b.{conv2}(){wrap_end} +""" + +def extend_num_op(pfn, ty1, ty2, conv1, op, conv2, wrap_start = "rt.wrap(", wrap_end = ")"): + tp = num_op_template.format(pfn=pfn, ty1=ty1.__name__, ty2=ty2.__name__, + conv1=conv1, op=op, conv2=conv2, + wrap_start=wrap_start, wrap_end=wrap_end) + exec tp + +def def_num_ops(): + # maybe define get_val() instead of using tuples? + num_classes = [(Integer, "int_val"), (Float, "float_val")] + for (c1, conv1) in num_classes: + for (c2, conv2) in num_classes: + for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/")]: + extend_num_op(op, c1, c2, conv1, sym, conv2) + extend_num_op("_num_eq", c1, c2, conv1, "==", conv2, + wrap_start = "true if ", wrap_end = " else false") + +define_num_ops() def init(): From 6d89e3f4c96a89a17241a25f481f7e5ef932ed7e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 14:06:07 +0100 Subject: [PATCH 035/909] move num op generation closer to protocol definition --- pixie/vm/numbers.py | 29 +++++++++++++++-------------- 1 file changed, 15 insertions(+), 14 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 3fd46bdc..81f9f775 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -54,20 +54,6 @@ def type(self): _num_eq.set_default_fn(wrap_fn(lambda a, b: false)) -# def add(a, b): -# if isinstance(a, Integer): -# if isinstance(b, Integer): -# return Integer(a.int_val() + b.int_val()) -# -# raise Exception("Add error") - -def eq(a, b): - if isinstance(a, Integer): - if isinstance(b, Integer): - return true if a.int_val() == b.int_val() else false - - raise Exception("Add error") - num_op_template = """@extend({pfn}, {ty1}._type, {ty2}._type) def {pfn}_{ty1}_{ty2}(a, b): assert isinstance(a, {ty1}) and isinstance(b, {ty2}) @@ -93,6 +79,21 @@ def def_num_ops(): define_num_ops() +# def add(a, b): +# if isinstance(a, Integer): +# if isinstance(b, Integer): +# return Integer(a.int_val() + b.int_val()) +# +# raise Exception("Add error") + +def eq(a, b): + if isinstance(a, Integer): + if isinstance(b, Integer): + return true if a.int_val() == b.int_val() else false + + raise Exception("Add error") + + def init(): import pixie.vm.stdlib as proto from pixie.vm.string import String From 1e21573eb94f034af0c566c8a853e4c2b62bc94f Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 14:06:46 +0100 Subject: [PATCH 036/909] fix variable name --- pixie/vm/numbers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 81f9f775..5dbb2e93 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -66,7 +66,7 @@ def extend_num_op(pfn, ty1, ty2, conv1, op, conv2, wrap_start = "rt.wrap(", wrap wrap_start=wrap_start, wrap_end=wrap_end) exec tp -def def_num_ops(): +def define_num_ops(): # maybe define get_val() instead of using tuples? num_classes = [(Integer, "int_val"), (Float, "float_val")] for (c1, conv1) in num_classes: From eee5d1b6bc5bc719fbc99d53ab5a0f8e9f90fd64 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 07:09:13 -0600 Subject: [PATCH 037/909] started adding pixie.path --- pixie/vm/libs/path.py | 39 +++++++++++++++++++++++++++++++++++++++ pixie/vm/rt.py | 1 + pixie/vm/stdlib.py | 1 - 3 files changed, 40 insertions(+), 1 deletion(-) create mode 100644 pixie/vm/libs/path.py diff --git a/pixie/vm/libs/path.py b/pixie/vm/libs/path.py new file mode 100644 index 00000000..f9f25bcb --- /dev/null +++ b/pixie/vm/libs/path.py @@ -0,0 +1,39 @@ +import pixie.vm.rt as rt +from pixie.vm.string import String +from pixie.vm.code import as_var, extend +from pixie.vm.object import Object, Type +import pixie.vm.stdlib as proto +from pixie.vm.keyword import keyword +from rpython.rlib.clibffi import get_libc_name +import os +import pixie.vm.rt as rt + +class FileList(Object): + _type = Type(u"pixie.path.FileList") + + def type(self): + return FileList._type + + def __init__(self, top): + self._top = rt.name(top) + +KW_DIR = keyword(u"dir") +KW_FILE = keyword(u"file") + +@extend(proto._reduce, FileList) +def _reduce(self, f, init): + assert isinstance(self, FileList) + for dirpath, dirnames, filenames in os.walk(str(self._top)): + for dirname in dirnames: + init = f.invoke([init, rt.vector(rt.wrap(dirpath), KW_DIR, rt.wrap(dirname))]) + if rt.reduced_QMARK_(init): + return rt.deref(init) + + + + return init + + +@as_var("pixie.path", "file-list") +def file_list(path): + return FileList(path) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 34d30881..8e30b363 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -65,6 +65,7 @@ def wrapper(*args): import pixie.vm.libs.platform import pixie.vm.libs.ffi import pixie.vm.symbol + import pixie.vm.libs.path diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index f212ed02..6138af88 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -278,7 +278,6 @@ def load_file(filename): affirm(isinstance(filename, string.String), u"Filename must be string") filename_str = rt.name(filename) - paths = rt.deref(rt.deref(rt.load_paths)) f = None for x in range(rt.count(paths)): From 0b6505ea502bd054fd5769abbce702583d7f0129 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 16:14:00 +0100 Subject: [PATCH 038/909] initial ratio support again (very) directly inspired by clojure's implementation. --- pixie/vm/numbers.py | 55 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 55 insertions(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 5dbb2e93..31aebd3e 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -45,6 +45,23 @@ def float_val(self): def type(self): return Float._type +class Ratio(Number): + _type = object.Type(u"pixie.stdlib.Ratio") + _immutable_fields_ = ["_numerator", "_denominator"] + + def __init__(self, numerator, denominator): + self._numerator = numerator + self._denominator = denominator + + def numerator(self): + return self._numerator + + def denominator(self): + return self._denominator + + def type(self): + return Ratio._type + IMath = as_var("IMath")(Protocol(u"IMath")) _add = as_var("-add")(DoublePolymorphicFn(u"-add", IMath)) _sub = as_var("-sub")(DoublePolymorphicFn(u"-sub", IMath)) @@ -72,6 +89,8 @@ def define_num_ops(): for (c1, conv1) in num_classes: for (c2, conv2) in num_classes: for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/")]: + if op == "_div" and c1 == Integer and c2 == Integer: + continue extend_num_op(op, c1, c2, conv1, sym, conv2) extend_num_op("_num_eq", c1, c2, conv1, "==", conv2, wrap_start = "true if ", wrap_end = " else false") @@ -79,6 +98,34 @@ def define_num_ops(): define_num_ops() +def gcd(u, v): + while v != 0: + r = u % v + u = v + v = r + return u + +@extend(_div, Integer._type, Integer._type) +def _div(n, d): + object.affirm(d != 0, u"Divide by zero") + nv = n.int_val() + dv = d.int_val() + g = gcd(nv, dv) + if g == 0: + return rt.wrap(0) + nv = nv / g + dv = dv / g + if dv == 1: + return rt.wrap(nv) + elif dv == -1: + return rt.wrap(-1 * nv) + else: + if d < 0: + nv = nv * -1 + dv = dv * -1 + return Ratio(nv, dv) + + # def add(a, b): # if isinstance(a, Integer): # if isinstance(b, Integer): @@ -113,3 +160,11 @@ def _str(f): @extend(proto._repr, Float._type) def _repr(f): return rt.wrap(unicode(str(f.float_val()))) + + @extend(proto._repr, Ratio._type) + def _repr(r): + return rt.wrap(unicode(str(r.numerator()) + "/" + str(r.denominator()))) + + @extend(proto._str, Ratio._type) + def _str(r): + return rt.wrap(unicode(str(r.numerator()) + "/" + str(r.denominator()))) From 24e96177806e748b3bf09ef0407c1e8db89d6834 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 16:19:01 +0100 Subject: [PATCH 039/909] add missing assert for integer _div --- pixie/vm/numbers.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 31aebd3e..6240a440 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -107,6 +107,7 @@ def gcd(u, v): @extend(_div, Integer._type, Integer._type) def _div(n, d): + assert isinstance(n, Integer) and isinstance(d, Integer) object.affirm(d != 0, u"Divide by zero") nv = n.int_val() dv = d.int_val() From 02526c5a9d6f451d3ea6e590ca31f5281cabe075 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 16:29:35 +0100 Subject: [PATCH 040/909] implement ratio _add --- pixie/vm/numbers.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 6240a440..8a6e8083 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -126,6 +126,12 @@ def _div(n, d): dv = dv * -1 return Ratio(nv, dv) +@extend(_add, Ratio._type, Ratio._type) +def _add(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt._add(rt.wrap(b.numerator() * a.denominator()), + rt.wrap(a.numerator() * b.denominator())), + rt.wrap(a.denominator() * b.denominator())) # def add(a, b): # if isinstance(a, Integer): From f70681043b4b039227365698d012fbb36d5f651e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 16:40:07 +0100 Subject: [PATCH 041/909] fix divide by zero check --- pixie/vm/numbers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 8a6e8083..eb0e578b 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -108,9 +108,9 @@ def gcd(u, v): @extend(_div, Integer._type, Integer._type) def _div(n, d): assert isinstance(n, Integer) and isinstance(d, Integer) - object.affirm(d != 0, u"Divide by zero") nv = n.int_val() dv = d.int_val() + object.affirm(dv != 0, u"Divide by zero") g = gcd(nv, dv) if g == 0: return rt.wrap(0) From cad403f3f39ca9517db728225e5d455ac0c13082 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 16:57:41 +0100 Subject: [PATCH 042/909] support ratio literals --- pixie/vm/compiler.py | 3 +++ pixie/vm/reader.py | 12 +++++++++++- 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index bb266e2b..ef43deb2 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -313,6 +313,9 @@ def compile_form(form, ctx): if isinstance(form, numbers.Float): ctx.push_const(form) return + if isinstance(form, numbers.Ratio): + ctx.push_const(form) + return if isinstance(form, symbol.Symbol): name = form._str diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index d1de3ed1..50218ccc 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -399,6 +399,7 @@ def invoke(self, rdr, ch): int_matcher = re.compile(u'^([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))$') float_matcher = re.compile(u'^([-+]?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+)?)$') +ratio_matcher = re.compile(u'^([-+]?[0-9]+)/([0-9]+)$') def parse_int(m): sign = 1 @@ -426,6 +427,11 @@ def parse_int(m): def parse_float(m): return rt.wrap(float(str(m.group(0)))) +def parse_ratio(m): + n = rt.wrap(int(str(m.group(1)))) + d = rt.wrap(int(str(m.group(2)))) + return rt._div(n, d) + def parse_number(s): m = int_matcher.match(s) if m: @@ -435,7 +441,11 @@ def parse_number(s): if m: return parse_float(m) else: - return None + m = ratio_matcher.match(s) + if m: + return parse_ratio(m) + else: + return None def read_number(rdr, ch): acc = [ch] From de1f757179b0049e581b2820e386f9d8a5e88155 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 16:57:55 +0100 Subject: [PATCH 043/909] fix compilation --- pixie/vm/numbers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index eb0e578b..3e9d9e36 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -121,7 +121,7 @@ def _div(n, d): elif dv == -1: return rt.wrap(-1 * nv) else: - if d < 0: + if dv < 0: nv = nv * -1 dv = dv * -1 return Ratio(nv, dv) From c4c9b1a9513adbe21265ef669e83d04fb035ff6e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 17:07:31 +0100 Subject: [PATCH 044/909] implement remaining math functions for ratio not yet for ratio with other types, though. it's a bit tricky as we could just convert the arguments to ratios and then use the existing functions, but there are exceptions when floats are involved. maybe that's already it? --- pixie/vm/numbers.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 3e9d9e36..ea3c61e9 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -133,6 +133,25 @@ def _add(a, b): rt.wrap(a.numerator() * b.denominator())), rt.wrap(a.denominator() * b.denominator())) +@extend(_sub, Ratio._type, Ratio._type) +def _sub(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt._add(rt.wrap(-1 * b.numerator() * a.denominator()), + rt.wrap(a.numerator() * b.denominator())), + rt.wrap(a.denominator() * b.denominator())) + +@extend(_mul, Ratio._type, Ratio._type) +def _mul(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt.wrap(b.numerator() * a.numerator()), + rt.wrap(b.denominator() * a.denominator())) + +@extend(_div, Ratio._type, Ratio._type) +def _div(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt.wrap(b.denominator() * a.numerator()), + rt.wrap(b.numerator() * a.denominator())) + # def add(a, b): # if isinstance(a, Integer): # if isinstance(b, Integer): From 1bf17f5af49a211d9f1ada436bb37fefb32d2cbc Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 19:38:43 +0100 Subject: [PATCH 045/909] generate ratio ops with one integer arg --- pixie/vm/numbers.py | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index ea3c61e9..ecb14b51 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -152,6 +152,32 @@ def _div(a, b): return rt._div(rt.wrap(b.denominator() * a.numerator()), rt.wrap(b.numerator() * a.denominator())) +ratio_op_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) +def {pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + return rt.{pfn}({conv1}(a), {conv2}(b)) +""" + +def to_ratio(x): + if isinstance(x, Ratio): + return x + else: + return Ratio(x.int_val(), 1) + +def get_conv(c): + if c == Ratio: + return "" + else: + return "to_ratio" + +def define_ratio_ops(): + for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: + for op in ["_add", "_sub", "_mul", "_div"]: + code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=get_conv(c1), conv2=get_conv(c2)) + exec code + +define_ratio_ops() + # def add(a, b): # if isinstance(a, Integer): # if isinstance(b, Integer): From ba06117041719bfbe5036e49636877ca8e7084ca Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 20:57:12 +0100 Subject: [PATCH 046/909] generate ratio ops with one float arg --- pixie/vm/numbers.py | 21 +++++++++++++++++++-- 1 file changed, 19 insertions(+), 2 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index ecb14b51..f10487c8 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -164,16 +164,33 @@ def to_ratio(x): else: return Ratio(x.int_val(), 1) -def get_conv(c): +def to_ratio_conv(c): if c == Ratio: return "" else: return "to_ratio" +def to_float(x): + if isinstance(x, Float): + return x + else: + return rt.wrap(x.numerator() / float(x.denominator())) + +def to_float_conv(c): + if c == Float: + return "" + else: + return "to_float" + def define_ratio_ops(): for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: for op in ["_add", "_sub", "_mul", "_div"]: - code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=get_conv(c1), conv2=get_conv(c2)) + code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) + exec code + + for (c1, c2) in [(Float, Ratio), (Ratio, Float)]: + for op in ["_add", "_sub", "_mul", "_div"]: + code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) exec code define_ratio_ops() From 026c02756ba9b7fc96e8b56fa2c25b96d03b830b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 21:13:50 +0100 Subject: [PATCH 047/909] add _num_eq for ratios not sure about comparing ratios to floats, but i think it's ok. --- pixie/vm/numbers.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index f10487c8..2b622f52 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -152,6 +152,11 @@ def _div(a, b): return rt._div(rt.wrap(b.denominator() * a.numerator()), rt.wrap(b.numerator() * a.denominator())) +@extend(_num_eq, Ratio._type, Ratio._type) +def _num_eq(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if a.numerator() == b.numerator() and a.denominator() == b.denominator() else false + ratio_op_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) def {pfn}_{ty1}_{ty2}(a, b): assert isinstance(a, {ty1}) and isinstance(b, {ty2}) @@ -184,12 +189,12 @@ def to_float_conv(c): def define_ratio_ops(): for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: - for op in ["_add", "_sub", "_mul", "_div"]: + for op in ["_add", "_sub", "_mul", "_div", "_num_eq"]: code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) exec code for (c1, c2) in [(Float, Ratio), (Ratio, Float)]: - for op in ["_add", "_sub", "_mul", "_div"]: + for op in ["_add", "_sub", "_mul", "_div", "_num_eq"]: code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) exec code From a6c172a16629479ab86a709e0675fce2837a848e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 29 Oct 2014 21:17:55 +0100 Subject: [PATCH 048/909] extend -eq to floats and ratios --- pixie/stdlib.lisp | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index dffdefa1..a51ffe26 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -113,6 +113,8 @@ (extend -hash Integer hash-int) (extend -eq Integer -num-eq) +(extend -eq Float -num-eq) +(extend -eq Ratio -num-eq) (def ordered-hash-reducing-fn (fn ordered-hash-reducing-fn @@ -409,4 +411,4 @@ (defn doc [x] - (get (meta x) :doc)) \ No newline at end of file + (get (meta x) :doc)) From d74a0bbf7bb7362aea7622bb4d7d8ee508456334 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 16:51:54 -0600 Subject: [PATCH 049/909] we have a testing framework --- pixie/test-tests.lisp | 10 ++++++++ pixie/test.lisp | 55 +++++++++++++++++++++++++++++++++++++++++ pixie/vm/libs/string.py | 0 run-tests.lisp | 4 +++ 4 files changed, 69 insertions(+) create mode 100644 pixie/test-tests.lisp create mode 100644 pixie/test.lisp create mode 100644 pixie/vm/libs/string.py create mode 100644 run-tests.lisp diff --git a/pixie/test-tests.lisp b/pixie/test-tests.lisp new file mode 100644 index 00000000..7a65b3dc --- /dev/null +++ b/pixie/test-tests.lisp @@ -0,0 +1,10 @@ +(ns pixie.test-tests + (require pixie.test :as t)) + +(t/deftest foo + (t/assert= 1 2)) + +(t/deftest bar + (t/assert= 1 1)) + +(t/run-tests) \ No newline at end of file diff --git a/pixie/test.lisp b/pixie/test.lisp new file mode 100644 index 00000000..44629a40 --- /dev/null +++ b/pixie/test.lisp @@ -0,0 +1,55 @@ +(ns pixie.test) + +(def tests (atom {})) + + +(def ^:dynamic *stats*) + +(def ^:dynamic *current-test*) + + +(defmacro deftest [nm & body] + `(do (defn ~nm [] + (print "running" ~(name nm)) + (try + ~@body + (swap! *stats* update-in [:pass] (fnil inc 0)) + (catch ex + (swap! *stats* update-in [:fail] (fnil inc 0)) + (swap! *stats* update-in [:errors] (fnil conj []) ex)))) + (swap! tests assoc (symbol (str (namespace (var ~nm)) "/" (name (var ~nm)))) ~nm))) + + + +(defn run-tests [] + (push-binding-frame!) + (set! (var *stats*) (atom {})) + + (reduce + (fn + ([_ f] + ((val f)) + nil) + ([_] nil)) + nil + @tests) + (let [stats @*stats*] + (print stats) + (pop-binding-frame!) + stats)) + + +(defn load-all-tests [] + (foreach [path @load-paths] + (foreach [desc (pixie.path/file-list path)] + (if (= (nth desc 1) :file) + (let [filename (nth desc 2)] + (if (pixie.string/starts-with filename "test-") + (if (pixie.string/ends-with filename ".lisp") + (let [fullpath (str (nth desc 0) "/" filename)] + (print "Loading " fullpath) + (load-file fullpath))))))))) + + +(defn assert= [x y] + (assert (= x y) (str x " != " y))) diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py new file mode 100644 index 00000000..e69de29b diff --git a/run-tests.lisp b/run-tests.lisp new file mode 100644 index 00000000..eb5acd56 --- /dev/null +++ b/run-tests.lisp @@ -0,0 +1,4 @@ +(require pixie.test :as t) +(t/load-all-tests) +(let [result (t/run-tests)] + (exit (get result :fail))) From 46c792ecbbc4c5f9921cd0af24102b89102c30d2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 16:52:33 -0600 Subject: [PATCH 050/909] update travis file, this will break tests --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 0c2df75d..78f27cda 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,7 @@ script: - ./checkout-externals - ./make-with-jit + - make run_built_tests os: - osx From d79213aed70a7410d2f5d6625ca3849a23a1351a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 16:52:53 -0600 Subject: [PATCH 051/909] forgot the makefile --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index 7e62d3dc..961ccd23 100644 --- a/Makefile +++ b/Makefile @@ -33,3 +33,9 @@ run: run_interactive: PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py + +run_built_tests: pixie-vm + cat run-tests.lisp | ./pixie-vm + +run_interpreted_tests: pixie-vm + cat run-tests.lisp | PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py From 92d51f133985a877824baf0549d8a9bd94198359 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 16:53:10 -0600 Subject: [PATCH 052/909] add the rest of the files --- pixie/stdlib.lisp | 8 ++++++++ pixie/vm/libs/path.py | 5 +++++ pixie/vm/libs/string.py | 20 ++++++++++++++++++++ 3 files changed, 33 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 775ef306..6606a4b0 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -438,3 +438,11 @@ (apply f (if (nil? x) else x) args))) +(defmacro foreach [binding & body] + (assert (= 2 (count binding)) "binding and collection required") + `(reduce + (fn [_ ~ (nth binding 0)] + ~@body + nil) + nil + ~(nth binding 1))) diff --git a/pixie/vm/libs/path.py b/pixie/vm/libs/path.py index f9f25bcb..f19644e1 100644 --- a/pixie/vm/libs/path.py +++ b/pixie/vm/libs/path.py @@ -29,6 +29,11 @@ def _reduce(self, f, init): if rt.reduced_QMARK_(init): return rt.deref(init) + for filename in filenames: + init = f.invoke([init, rt.vector(rt.wrap(dirpath), KW_FILE, rt.wrap(filename))]) + if rt.reduced_QMARK_(init): + return rt.deref(init) + return init diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index e69de29b..d2bf3231 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -0,0 +1,20 @@ +import pixie.vm.rt as rt +from pixie.vm.string import String +from pixie.vm.code import as_var, extend +from pixie.vm.object import Object, Type +import pixie.vm.stdlib as proto +from pixie.vm.keyword import keyword +from rpython.rlib.clibffi import get_libc_name +import os +import pixie.vm.rt as rt + + + +@as_var("pixie.string", "starts-with") +def startswith(a, b): + return rt.wrap(rt.name(a).startswith(rt.name(b))) + + +@as_var("pixie.string", "ends-with") +def endswith(a, b): + return rt.wrap(rt.name(a).endswith(rt.name(b))) From 35b1b90079dd99b52ab4f6a400f3e047a3167a0d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 29 Oct 2014 17:01:21 -0600 Subject: [PATCH 053/909] remove failing test --- pixie/test-tests.lisp | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pixie/test-tests.lisp b/pixie/test-tests.lisp index 7a65b3dc..f3c4d784 100644 --- a/pixie/test-tests.lisp +++ b/pixie/test-tests.lisp @@ -1,10 +1,3 @@ (ns pixie.test-tests (require pixie.test :as t)) -(t/deftest foo - (t/assert= 1 2)) - -(t/deftest bar - (t/assert= 1 1)) - -(t/run-tests) \ No newline at end of file From e1022f0088520c3b82f28d017c4a6903be3d024f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Oct 2014 06:28:48 -0600 Subject: [PATCH 054/909] a few fixes to the test running, implmented first vector test --- pixie/stdlib.lisp | 10 ++++++++++ pixie/test.lisp | 6 ++++-- pixie/vm/libs/readline.py | 2 +- pixie/vm/reader.py | 4 ++-- run-tests.lisp | 7 +++++++ tests/collections/test-vectors.lisp | 12 ++++++++++++ tests/collections/vector_tests.lisp | 0 7 files changed, 36 insertions(+), 5 deletions(-) create mode 100644 tests/collections/test-vectors.lisp delete mode 100644 tests/collections/vector_tests.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index b5e58403..c627e3f4 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -447,3 +447,13 @@ nil) nil ~(nth binding 1))) + + +(defmacro dotimes [bind & body] + (let [b (nth bind 0)] + `(let [max# ~(nth bind 1)] + (loop [~b 0] + (if (= ~b max#) + nil + (do ~@body + (recur (inc ~b)))))))) diff --git a/pixie/test.lisp b/pixie/test.lisp index 44629a40..e283b301 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -10,7 +10,7 @@ (defmacro deftest [nm & body] `(do (defn ~nm [] - (print "running" ~(name nm)) + (print ".") (try ~@body (swap! *stats* update-in [:pass] (fnil inc 0)) @@ -23,7 +23,7 @@ (defn run-tests [] (push-binding-frame!) - (set! (var *stats*) (atom {})) + (set! (var *stats*) (atom {:fail 0 :pass 0})) (reduce (fn @@ -40,7 +40,9 @@ (defn load-all-tests [] + (print "Looking for tests...") (foreach [path @load-paths] + (print "Looking for tests in: " path) (foreach [desc (pixie.path/file-list path)] (if (= (nth desc 1) :file) (let [filename (nth desc 2)] diff --git a/pixie/vm/libs/readline.py b/pixie/vm/libs/readline.py index 3aa8ec0b..5299f47f 100644 --- a/pixie/vm/libs/readline.py +++ b/pixie/vm/libs/readline.py @@ -22,4 +22,4 @@ def _readline(prompt): if result == lltype.nullptr(rffi.CCHARP.TO): return u"" else: - return unicode(rffi.charp2str(result)) + return unicode(rffi.charp2str(result)) + u"\n" diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index bc8c9264..944ffb1d 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -66,7 +66,7 @@ def read(self): result = _readline(str(rt.name(compiler.NS_VAR.deref())) + " => ") if result == u"": raise EOFError() - self._string_reader = StringReader(result + u"\n") + self._string_reader = StringReader(result) try: return self._string_reader.read() @@ -321,7 +321,7 @@ def syntax_quote(form): if gs is nil: gs = rt.symbol(rt.str(form, rt.wrap(u"__"), rt.gensym())) GEN_SYM_ENV.set_value(rt.assoc(gmap, form, gs)) - form = gs + form = gs else: var = rt.resolve_in(compiler.NS_VAR.deref(), form) if var is nil: diff --git a/run-tests.lisp b/run-tests.lisp index eb5acd56..05fed604 100644 --- a/run-tests.lisp +++ b/run-tests.lisp @@ -1,4 +1,11 @@ (require pixie.test :as t) + + +(swap! load-paths conj "./tests/") + +(print @load-paths) + (t/load-all-tests) + (let [result (t/run-tests)] (exit (get result :fail))) diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.lisp new file mode 100644 index 00000000..2775047e --- /dev/null +++ b/tests/collections/test-vectors.lisp @@ -0,0 +1,12 @@ +(ns collections.test-vectors + (require pixie.test :as t)) + +(def MAX-SIZE 2000) + +(t/deftest vector-creation + (loop [acc []] + (if (= (count acc) MAX-SIZE) + acc + (do (dotimes [j (count acc)] + (t/assert= j (nth acc j))) + (recur (conj acc (count acc))))))) diff --git a/tests/collections/vector_tests.lisp b/tests/collections/vector_tests.lisp deleted file mode 100644 index e69de29b..00000000 From 5999381a73fd7a207810d9d3a74559b1cdd75cbc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Oct 2014 06:53:43 -0600 Subject: [PATCH 055/909] added and, or, and support for commandline args, also removed cruft in target.py --- pixie/stdlib.lisp | 14 +++++++ target.py | 100 +++++++++++++++++++++------------------------- 2 files changed, 59 insertions(+), 55 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index c627e3f4..ccfcbaec 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -3,6 +3,7 @@ (def reset! -reset!) (def load-paths (atom ["./"])) +(def program-arguments []) (def map (fn ^{:doc "map - creates a transducer that applies f to every input element" :added "0.1"} @@ -457,3 +458,16 @@ nil (do ~@body (recur (inc ~b)))))))) + + +(defmacro and + ([x] x) + ([x y] `(if ~x ~y nil)) + ([x y & more] `(if ~x (and ~y ~@more)))) + +(defmacro or + ([x] x) + ([x y] `(let [r# ~x] + (if r# r# ~y))) + ([x y & more] `(let [r# ~x] + (if r# r# (or ~y ~@more))))) diff --git a/target.py b/target.py index def9e19e..2b6fc9fc 100644 --- a/target.py +++ b/target.py @@ -4,12 +4,12 @@ from rpython.jit.codewriter.policy import JitPolicy from rpython.rlib.jit import JitHookInterface, Counters from rpython.annotator.policy import AnnotatorPolicy -from pixie.vm.code import wrap_fn +from pixie.vm.code import wrap_fn, NativeFn, intern_var from pixie.vm.stacklet import with_stacklets import pixie.vm.stacklet as stacklet from pixie.vm.object import RuntimeException, WrappedException from rpython.translator.platform import platform - +from pixie.vm.primitives import nil import sys @@ -23,65 +23,58 @@ class Policy(JitPolicy, AnnotatorPolicy): def __init__(self): JitPolicy.__init__(self, DebugIFace()) - #def event(pol, bookkeeper, what, x): - # pass - def jitpolicy(driver): return JitPolicy(jithookiface=DebugIFace()) -@wrap_fn -def repl(): - from pixie.vm.keyword import keyword - import pixie.vm.rt as rt - from pixie.vm.string import String +PROGRAM_ARGUMENTS = intern_var(u"pixie.stdlib", u"program-arguments") +PROGRAM_ARGUMENTS.set_root(nil) + + +class ReplFn(NativeFn): + def __init__(self, args): + self._argv = args + + def inner_invoke(self, args): + from pixie.vm.keyword import keyword + import pixie.vm.rt as rt + from pixie.vm.string import String + import pixie.vm.persistent_vector as vector - with with_ns(u"user"): - NS_VAR.deref().include_stdlib() + with with_ns(u"user"): + NS_VAR.deref().include_stdlib() - rdr = MetaDataReader(PromptReader()) - with with_ns(u"user"): - while True: - try: - val = read(rdr, False) - if val is eof: + acc = vector.EMPTY + for x in self._argv: + acc = rt.conj(acc, rt.wrap(x)) + + PROGRAM_ARGUMENTS.set_root(acc) + + + rdr = MetaDataReader(PromptReader()) + with with_ns(u"user"): + while True: + try: + val = read(rdr, False) + if val is eof: + break + val = interpret(compile(val)) + except WrappedException as ex: + print "Error: ", ex._ex.__repr__() + rdr.reset_line() + continue + if val is keyword(u"exit-repl"): break - val = interpret(compile(val)) - except WrappedException as ex: - print "Error: ", ex._ex.__repr__() - rdr.reset_line() - continue - if val is keyword(u"exit-repl"): - break - val = rt.str(val) - assert isinstance(val, String), "str should always return a string" - print val._str - -def entry_point(foo=None): + val = rt.str(val) + assert isinstance(val, String), "str should always return a string" + print val._str + +def entry_point(args): print "Pixie 0.1 - Interactive REPL" print "(" + platform.name + ", " + platform.cc + ")" print "----------------------------" - #try: - # code = argv[1] - #except IndexError: - # print "must provide a program" - # return 1 - # rdr = StreamReader(sys.stdin) - # while True: - # #val = read(rdr, True) - # #if val is eof: - # # break - # val = read(StringReader(raw_input("user>")), True) - # print interpret(compile(val)) - # interpret(compile(read(StringReader(""" - # (do (def foo (fn [h v] (h 42))) - # ((create-stacklet foo) 0)) - # """), True))) - #rt.load_file(String(u"pixie/stdlib.lisp")) - - - - with_stacklets(repl) + + with_stacklets(ReplFn(args)) return 0 @@ -91,7 +84,6 @@ def entry_point(foo=None): from rpython.jit.metainterp import warmspot def run_child(glob, loc): - import sys, pdb interp = loc['interp'] graph = loc['graph'] interp.malloc_check = False @@ -153,6 +145,4 @@ def target(*args): print rpython.config.translationoption.get_combined_translation_config() if __name__ == "__main__": - #run_debug(sys.argv) - #with_stacklets(bootstrap) - entry_point() \ No newline at end of file + entry_point(sys.argv) \ No newline at end of file From 8f4d1084f16de591fcd5ebfbcf85fb99cca2d93f Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 30 Oct 2014 21:33:02 +0100 Subject: [PATCH 056/909] support string return values from ffi functions this is most likely not enough, because we must free the result? --- pixie/stdlib.lisp | 1 + pixie/vm/libs/ffi.py | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index ccfcbaec..3e2c76cf 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -411,6 +411,7 @@ (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) (def printf (ffi-fn libc "printf" [String] Integer)) + (def getenv (ffi-fn libc "getenv" [String] String)) (defn print [& args] (puts (apply str args))) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 9ea9cfd5..8b2c7f18 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -4,6 +4,7 @@ from pixie.vm.code import as_var, affirm import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype +from pixie.vm.primitives import nil from pixie.vm.numbers import Integer from pixie.vm.string import String from rpython.rlib.jit_libffi import CIF_DESCRIPTION @@ -76,6 +77,12 @@ def get_ret_val(ptr, tp): pnt = rffi.cast(rffi.LONGP, ptr) val = pnt[0] return Integer(val) + if tp == String._type: + pnt = rffi.cast(rffi.CCHARPP, ptr) + if pnt[0] == lltype.nullptr(rffi.CCHARP.TO): + return nil + else: + return String(rffi.charp2str(pnt[0])) assert False From 876efab3374448d3b47ee31e516aeca9f21ae51a Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 30 Oct 2014 00:11:48 +0100 Subject: [PATCH 057/909] implement '/' and '*' this also improves '+', it doesn't do an initial addition to 0 if it isn't necessary. --- pixie/stdlib.lisp | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index ccfcbaec..2b264295 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -203,15 +203,31 @@ (set-macro! defmacro) (defn + - [& args] - (reduce -add 0 args)) - + ([] 0) + ([x] x) + ([x y] (-add x y)) + ([x y & args] + (reduce -add (-add x y) args))) (defn - ([] 0) ([x] (-sub 0 x)) - ([x & args] - (reduce -sub x args))) + ([x y] (-sub x y)) + ([x y & args] + (reduce -sub (-sub x y) args))) + +(defn * + ([] 1) + ([x] x) + ([x y] (-mul x y)) + ([x y & args] + (reduce -mul (-mul x y) args))) + +(defn / + ([x] (-div 1 x)) + ([x y] (-div x y)) + ([x y & args] + (reduce -div (-div x y) args))) (defn = ([x] true) From 9b93ffa2d4090a7064439c8c09eebaedff3d1ca4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 01:48:47 +0100 Subject: [PATCH 058/909] fix octal literals, add some tests for numbers --- pixie/test-numbers.lisp | 38 ++++++++++++++++++++++++++++++++++++++ pixie/vm/reader.py | 2 +- 2 files changed, 39 insertions(+), 1 deletion(-) create mode 100644 pixie/test-numbers.lisp diff --git a/pixie/test-numbers.lisp b/pixie/test-numbers.lisp new file mode 100644 index 00000000..b634e8c6 --- /dev/null +++ b/pixie/test-numbers.lisp @@ -0,0 +1,38 @@ +(ns pixie.test-numbers + (require pixie.test :as t)) + +(t/deftest integer-literals + (t/assert= 0xa 10) + (t/assert= -0xa -10) + (t/assert= 012 10) + (t/assert= -012 -10) + (t/assert= 2r1010 10) + (t/assert= -2r1010 -10)) + +(t/deftest float-literals + (t/assert= 10. 10.0) + (t/assert= -10. -10.0) + (t/assert= 1e1 10.0) + (t/assert= -1e1 -10.0) + (t/assert= 1e-1 0.1) + (t/assert= -1e-1 -0.1)) + +(t/deftest mixed-float-ops + (t/assert= (+ 1/2 0.5) 1.0) + (t/assert= (+ 0 1.0) 1.0)) + +(t/deftest ratio-literals + (t/assert= 3/4 (/ 3 4)) + (t/assert= -3/4 (/ -3 4)) + (t/assert= 6/8 3/4) + (t/assert= 9/12 3/4) + (t/assert= 3/1 3)) + +(t/deftest ratio-from-divide + (t/assert= (/ 3 4) 3/4)) + +(t/deftest ratio-ops + (t/assert= (+ 1/2 1/2) 1) + (t/assert= (- 1/2 1/2) 0) + (t/assert= (* 1/2 1/2) 1/4) + (t/assert= (/ 1/2 1/2) 1)) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 944ffb1d..bb56ea16 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -415,7 +415,7 @@ def parse_int(m): radix = 16 num = m.group(3) elif m.group(4): - radix = 7 + radix = 8 num = m.group(4) elif m.group(5): radix = int(m.group(5)) From 1bddc1a7add5e050548a29d39a0ff20cafc2c64c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Oct 2014 21:14:09 -0600 Subject: [PATCH 059/909] a few tests and bug fixes --- pixie/test.lisp | 26 +++++++++++--------------- pixie/vm/libs/ffi.py | 2 +- pixie/vm/persistent_hash_map.py | 2 +- run-tests.lisp | 3 --- tests/collections/test-vectors.lisp | 2 ++ 5 files changed, 15 insertions(+), 20 deletions(-) diff --git a/pixie/test.lisp b/pixie/test.lisp index e283b301..f22f1fc3 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -10,13 +10,13 @@ (defmacro deftest [nm & body] `(do (defn ~nm [] - (print ".") - (try - ~@body - (swap! *stats* update-in [:pass] (fnil inc 0)) - (catch ex + (print "Running: " (str (namespace (var ~nm)) "/" (name (var ~nm)))) + (try + ~@body + (swap! *stats* update-in [:pass] (fnil inc 0)) + (catch ex (swap! *stats* update-in [:fail] (fnil inc 0)) - (swap! *stats* update-in [:errors] (fnil conj []) ex)))) + (swap! *stats* update-in [:errors] (fnil conj []) ex)))) (swap! tests assoc (symbol (str (namespace (var ~nm)) "/" (name (var ~nm)))) ~nm))) @@ -24,15 +24,11 @@ (defn run-tests [] (push-binding-frame!) (set! (var *stats*) (atom {:fail 0 :pass 0})) + (print @tests) + + (foreach [test @tests] + ((val test))) - (reduce - (fn - ([_ f] - ((val f)) - nil) - ([_] nil)) - nil - @tests) (let [stats @*stats*] (print stats) (pop-binding-frame!) @@ -42,7 +38,7 @@ (defn load-all-tests [] (print "Looking for tests...") (foreach [path @load-paths] - (print "Looking for tests in: " path) + (print "Looking for tests in: " path) (foreach [desc (pixie.path/file-list path)] (if (= (nth desc 1) :file) (let [filename (nth desc 2)] diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 9ea9cfd5..73b842f8 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -120,7 +120,7 @@ def thaw(self): ret_offset = transfer_size transfer_size += get_native_size(self._ret_type) - cd = lltype.malloc(CIF_DESCRIPTION, 1, flavor="raw") + cd = lltype.malloc(CIF_DESCRIPTION, len(self._arg_types), flavor="raw") cd.abi = clibffi.FFI_DEFAULT_ABI cd.nargs = len(self._arg_types) cd.rtype = get_clibffi_type(self._ret_type) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 06eae851..9719b24e 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -224,7 +224,7 @@ def reduce_inode(self, f, init): continue val = self._array[x + 1] - init = f.invoke([init, val]) + init = f.invoke([init, rt.map_entry(key_or_nil, val)]) if rt.reduced_QMARK_(init): return init return init diff --git a/run-tests.lisp b/run-tests.lisp index 05fed604..345b4a61 100644 --- a/run-tests.lisp +++ b/run-tests.lisp @@ -1,8 +1,5 @@ (require pixie.test :as t) - -(swap! load-paths conj "./tests/") - (print @load-paths) (t/load-all-tests) diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.lisp index 2775047e..a89fba1f 100644 --- a/tests/collections/test-vectors.lisp +++ b/tests/collections/test-vectors.lisp @@ -1,6 +1,8 @@ (ns collections.test-vectors (require pixie.test :as t)) + + (def MAX-SIZE 2000) (t/deftest vector-creation From 1056c556bb902960ea283d08c57b427b3514fa2c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 30 Oct 2014 22:00:10 -0600 Subject: [PATCH 060/909] added number comparison ops, added some tests, added range (exposed a compiler bug) --- pixie/stdlib.lisp | 53 ++++++++++++++++++++++++++++++++++++++++ pixie/vm/numbers.py | 11 +++++++-- tests/test-compiler.lisp | 18 ++++++++++++++ tests/test-numbers.lisp | 2 ++ 4 files changed, 82 insertions(+), 2 deletions(-) create mode 100644 tests/test-compiler.lisp create mode 100644 tests/test-numbers.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index ccfcbaec..fe1d1b3c 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -220,6 +220,35 @@ (apply = y rest) false))) +(defn < + ([x] true) + ([x y] (-lt x y)) + ([x y & rest] (if (-lt x y) + (apply < y rest) + false))) + +(defn > + ([x] true) + ([x y] (-gt x y)) + ([x y & rest] (if (-gt x y) + (apply > y rest) + false))) + +(defn <= + ([x] true) + ([x y] (-lte x y)) + ([x y & rest] (if (-lte x y) + (apply <= y rest) + false))) + +(defn >= + ([x] true) + ([x y] (-gte x y)) + ([x y & rest] (if (-gte x y) + (apply >= y rest) + false))) + + (def inc (fn [x] (+ x 1))) (def dec (fn [x] (- x 1))) @@ -471,3 +500,27 @@ (if r# r# ~y))) ([x y & more] `(let [r# ~x] (if r# r# (or ~y ~@more))))) + + +(deftype Range [:start :stop :step] + IReduce + (-reduce [self f init] + (let [start (. self :start) + stop (. self :stop) + step (. self :step)] + (loop [i start + acc init] + (if (or (and (> step 0) (< i stop)) + (and (< step 0) (> i stop)) + (and (= step 0))) + (let [acc (f acc i)] + (if (reduced? acc) + @acc + (recur (+ i step) acc))) + acc))))) + +(defn range + ([] (->Range 0 MAX-NUMBER 1)) + ([stop] (->Range 0 stop 1)) + ([start stop] (->Range start stop 1)) + ([start stop step] (->Range start stop step))) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 2b622f52..360db2bc 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -67,9 +67,15 @@ def type(self): _sub = as_var("-sub")(DoublePolymorphicFn(u"-sub", IMath)) _mul = as_var("-mul")(DoublePolymorphicFn(u"-mul", IMath)) _div = as_var("-div")(DoublePolymorphicFn(u"-div", IMath)) +_lt = as_var("-lt")(DoublePolymorphicFn(u"-lt", IMath)) +_gt = as_var("-gt")(DoublePolymorphicFn(u"-gt", IMath)) +_lte = as_var("-lte")(DoublePolymorphicFn(u"-lte", IMath)) +_gte = as_var("-gte")(DoublePolymorphicFn(u"-gte", IMath)) _num_eq = as_var("-num-eq")(DoublePolymorphicFn(u"-num-eq", IMath)) _num_eq.set_default_fn(wrap_fn(lambda a, b: false)) +as_var("MAX-NUMBER")(Integer(100000)) # TODO: set this to a real max number + num_op_template = """@extend({pfn}, {ty1}._type, {ty2}._type) def {pfn}_{ty1}_{ty2}(a, b): @@ -92,8 +98,9 @@ def define_num_ops(): if op == "_div" and c1 == Integer and c2 == Integer: continue extend_num_op(op, c1, c2, conv1, sym, conv2) - extend_num_op("_num_eq", c1, c2, conv1, "==", conv2, - wrap_start = "true if ", wrap_end = " else false") + for (op, sym) in [("_num_eq", "=="), ("_lt", "<"), ("_gt", ">"), ("_lte", "<="), ("_gte", ">=")]: + extend_num_op(op, c1, c2, conv1, sym, conv2, + wrap_start = "true if ", wrap_end = " else false") define_num_ops() diff --git a/tests/test-compiler.lisp b/tests/test-compiler.lisp new file mode 100644 index 00000000..8565defe --- /dev/null +++ b/tests/test-compiler.lisp @@ -0,0 +1,18 @@ +(ns pixie.test.test-compiler + (require pixie.test :as t)) + +(t/deftest test-do + (t/assert= (do 1) 1) + (t/assert= (do 1 2) 2) + (t/assert= (do) nil) + (t/assert= (do 1 2 3 4 5 6) 6)) + +(t/deftest test-if + (t/assert= (if true 42 nil) 42) + (t/assert= (if false 42 nil) nil) + (t/assert= (if false 42) nil)) + +(t/deftest test-let + (t/assert= (let [] 1) 1) + (t/assert= (let [x 1]) nil) + (t/assert= (let []) nil)) diff --git a/tests/test-numbers.lisp b/tests/test-numbers.lisp new file mode 100644 index 00000000..8755ab4b --- /dev/null +++ b/tests/test-numbers.lisp @@ -0,0 +1,2 @@ +(ns pixie.tests.test-numbers + (require pixie.test :as t)) From a88c97c4a07f8996c572b699f5f3aab380fb3ab9 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 09:26:19 +0100 Subject: [PATCH 061/909] move repl banner inside repl fn --- target.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/target.py b/target.py index 2b6fc9fc..b8a6e7c8 100644 --- a/target.py +++ b/target.py @@ -41,6 +41,10 @@ def inner_invoke(self, args): from pixie.vm.string import String import pixie.vm.persistent_vector as vector + print "Pixie 0.1 - Interactive REPL" + print "(" + platform.name + ", " + platform.cc + ")" + print "----------------------------" + with with_ns(u"user"): NS_VAR.deref().include_stdlib() @@ -70,10 +74,6 @@ def inner_invoke(self, args): print val._str def entry_point(args): - print "Pixie 0.1 - Interactive REPL" - print "(" + platform.name + ", " + platform.cc + ")" - print "----------------------------" - with_stacklets(ReplFn(args)) return 0 @@ -145,4 +145,4 @@ def target(*args): print rpython.config.translationoption.get_combined_translation_config() if __name__ == "__main__": - entry_point(sys.argv) \ No newline at end of file + entry_point(sys.argv) From 4e4a9ca754ffb370ad541493c58333037addb598 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 09:53:34 +0100 Subject: [PATCH 062/909] initial batch mode and cmdline args the argument parsing is not very sophisticated, but then why should it be? --- target.py | 49 ++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 48 insertions(+), 1 deletion(-) diff --git a/target.py b/target.py index b8a6e7c8..83cc883b 100644 --- a/target.py +++ b/target.py @@ -73,8 +73,55 @@ def inner_invoke(self, args): assert isinstance(val, String), "str should always return a string" print val._str +class BatchModeFn(NativeFn): + def __init__(self, args): + self._cmd = args[0] + self._argv = args[1:] + + def inner_invoke(self, args): + import pixie.vm.rt as rt + import pixie.vm.persistent_vector as vector + + with with_ns(u"user"): + NS_VAR.deref().include_stdlib() + + acc = vector.EMPTY + for x in self._argv: + acc = rt.conj(acc, rt.wrap(x)) + + PROGRAM_ARGUMENTS.set_root(acc) + + rt.load_file(rt.wrap(self._cmd)) + def entry_point(args): - with_stacklets(ReplFn(args)) + interactive = True + n = 0 + script_args = [] + + for arg in args[1:]: + if arg.startswith('-'): + if arg == '-v' or arg == '--version': + print "Pixie 0.1" + return 0 + elif arg == '-h' or arg == '--help': + print args[0] + " [] []" + print " -h|--help print this help" + print " -v|--version print the version number" + return 0 + else: + print "Unknown option " + arg + return 1 + else: + interactive = False + script_args = args[(n+1):] + break + + n += 1 + + if interactive: + with_stacklets(ReplFn(args)) + else: + with_stacklets(BatchModeFn(script_args)) return 0 From 68066e999b5db2d1e3194d6baf12b9c337f9fc91 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 11:23:46 +0100 Subject: [PATCH 063/909] add -e|--eval option --- target.py | 32 ++++++++++++++++++++++++++++---- 1 file changed, 28 insertions(+), 4 deletions(-) diff --git a/target.py b/target.py index 83cc883b..efbde812 100644 --- a/target.py +++ b/target.py @@ -93,12 +93,26 @@ def inner_invoke(self, args): rt.load_file(rt.wrap(self._cmd)) +class EvalFn(NativeFn): + def __init__(self, expr): + self._expr = expr + + def inner_invoke(self, args): + import pixie.vm.rt as rt + + with with_ns(u"user"): + NS_VAR.deref().include_stdlib() + + interpret(compile(read(StringReader(unicode(self._expr)), True))) + def entry_point(args): interactive = True - n = 0 script_args = [] - for arg in args[1:]: + i = 1 + while i < len(args): + arg = args[i] + if arg.startswith('-'): if arg == '-v' or arg == '--version': print "Pixie 0.1" @@ -107,16 +121,26 @@ def entry_point(args): print args[0] + " [] []" print " -h|--help print this help" print " -v|--version print the version number" + print " -e|--eval evaluate the given expression" return 0 + elif arg == '-e' or arg == '--eval': + i += 1 + if i < len(args): + expr = args[i] + with_stacklets(EvalFn(expr)) + return 0 + else: + print "Expected argument for " + arg + return 1 else: print "Unknown option " + arg return 1 else: interactive = False - script_args = args[(n+1):] + script_args = args[(i+1):] break - n += 1 + i += 1 if interactive: with_stacklets(ReplFn(args)) From cc44c80ac9fe64a37cc2f81e27a2e75fde2abc34 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 11:41:58 +0100 Subject: [PATCH 064/909] support reading code from stdin --- target.py | 15 +++++++++++---- 1 file changed, 11 insertions(+), 4 deletions(-) diff --git a/target.py b/target.py index efbde812..8912ef1f 100644 --- a/target.py +++ b/target.py @@ -3,6 +3,7 @@ from pixie.vm.interpreter import interpret from rpython.jit.codewriter.policy import JitPolicy from rpython.rlib.jit import JitHookInterface, Counters +from rpython.rlib.rfile import create_stdio from rpython.annotator.policy import AnnotatorPolicy from pixie.vm.code import wrap_fn, NativeFn, intern_var from pixie.vm.stacklet import with_stacklets @@ -75,7 +76,7 @@ def inner_invoke(self, args): class BatchModeFn(NativeFn): def __init__(self, args): - self._cmd = args[0] + self._file = args[0] self._argv = args[1:] def inner_invoke(self, args): @@ -91,7 +92,13 @@ def inner_invoke(self, args): PROGRAM_ARGUMENTS.set_root(acc) - rt.load_file(rt.wrap(self._cmd)) + with with_ns(u"user"): + if self._file == '-': + stdin, _, _ = create_stdio() + code = stdin.read() + interpret(compile(read(StringReader(unicode(code)), True))) + else: + rt.load_file(rt.wrap(self._file)) class EvalFn(NativeFn): def __init__(self, expr): @@ -113,7 +120,7 @@ def entry_point(args): while i < len(args): arg = args[i] - if arg.startswith('-'): + if arg.startswith('-') and arg != '-': if arg == '-v' or arg == '--version': print "Pixie 0.1" return 0 @@ -137,7 +144,7 @@ def entry_point(args): return 1 else: interactive = False - script_args = args[(i+1):] + script_args = args[i:] break i += 1 From 0bbcf1dfe3c0ba36f5ea2401885d5bcc86de7fe9 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 12:03:40 +0100 Subject: [PATCH 065/909] use batch mode in makefile --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 961ccd23..6c8c053f 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ run_interactive: PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run_built_tests: pixie-vm - cat run-tests.lisp | ./pixie-vm + ./pixie-vm run-tests.lisp run_interpreted_tests: pixie-vm - cat run-tests.lisp | PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py + PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.lisp From ad23256d268744b8b5f783844e8a132b702a4df3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 06:20:16 -0600 Subject: [PATCH 066/909] fixed translation via some odd compiler bugs --- pixie/vm/compiler.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7d89564b..a26daeab 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -578,10 +578,10 @@ def compile_recur(form, ctx): def compile_let(form, ctx): - form = next(form) + form = rt.next(form) bindings = rt.first(form) affirm(isinstance(bindings, PersistentVector), u"Bindings must be a vector") - body = next(form) + body = rt.next(form) ctc = ctx.can_tail_call ctx.disable_tail_call() @@ -614,10 +614,10 @@ def compile_let(form, ctx): ctx.bytecode.append(binding_count) def compile_loop(form, ctx): - form = next(form) + form = rt.next(form) bindings = rt.first(form) affirm(isinstance(bindings, PersistentVector), u"Loop bindings must be a vector") - body = next(form) + body = rt.next(form) ctc = ctx.can_tail_call ctx.disable_tail_call() @@ -639,7 +639,7 @@ def compile_loop(form, ctx): ctx.push_recur_point(LoopRecurPoint(binding_count, ctx)) while True: compile_form(rt.first(body), ctx) - body = next(body) + body = rt.next(body) if body is nil: break From 6372fc4d4f9c48b75dfd59af7e835e5a36a54754 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 08:00:21 -0600 Subject: [PATCH 067/909] fixed the odd compiler bug with range --- pixie/vm/compiler.py | 9 +++++++++ pixie/vm/interpreter.py | 6 ++---- 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index a26daeab..bc745642 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -85,6 +85,7 @@ def add_sp(self, v): self._max_sp = self._sp def sub_sp(self, v): + assert self._sp >= v if self._max_sp < self._sp: self._max_sp = self._sp self._sp -= v @@ -108,6 +109,10 @@ def push_arg(self, idx): self.add_sp(1) + def pop_locals(self, i=1): + for x in range(i): + self.locals.pop() + def add_local(self, name, arg): self.locals.append(self.locals[-1].copy()) self.locals[-1][name] = arg @@ -208,6 +213,7 @@ def __init__(self, sp): def emit(self, ctx): ctx.bytecode.append(code.DUP_NTH) + assert 0 <= ctx.sp() - self.sp < 100000 ctx.bytecode.append(r_uint(ctx.sp() - self.sp)) ctx.add_sp(1) @@ -437,6 +443,7 @@ def compile_fn(form, ctx): for x in arities: ctx.bytecode.append(r_uint(x)) + ctx.add_sp(1) # result ctx.sub_sp(len(arities)) else: @@ -612,6 +619,7 @@ def compile_let(form, ctx): ctx.bytecode.append(code.POP_UP_N) ctx.sub_sp(binding_count) ctx.bytecode.append(binding_count) + ctx.pop_locals(binding_count) def compile_loop(form, ctx): form = rt.next(form) @@ -650,6 +658,7 @@ def compile_loop(form, ctx): ctx.bytecode.append(code.POP_UP_N) ctx.sub_sp(binding_count) ctx.bytecode.append(binding_count) + ctx.pop_locals(binding_count) def compile_comment(form, ctx): ctx.push_const(nil) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index beb94d06..04a851a8 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -74,8 +74,8 @@ def pop(self): def nth(self, delta): - assert delta >= 0 - assert self.sp - 1 >= delta + affirm(delta >= 0, u"Invalid nth value, (compiler error)") + affirm(self.sp - 1 >= delta, u"interpreter nth out of range, (compiler error) ") return self.stack[self.sp - delta - 1] def push_nth(self, delta): @@ -142,8 +142,6 @@ def interpret(code_obj, args=[], self_obj = None): frame=frame) inst = frame.get_inst() - #print code.BYTECODES[inst] - if inst == code.LOAD_CONST: arg = frame.get_inst() frame.push_const(arg) From 267b8c59338a9def451dc07d005eeef85da13318 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 16:26:05 +0100 Subject: [PATCH 068/909] support ';' comments --- pixie/vm/reader.py | 12 ++++++++++++ pixie/vm/test/test_reader.py | 4 +++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 944ffb1d..a9c4a35a 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -510,6 +510,15 @@ def throw_syntax_error_with_data(rdr, txt): +def skip_line(rdr): + while True: + ch = rdr.read() + if ch == u"\n": + return + elif ch == u"\r": + ch2 = rdr.read() + if ch2 == u"\n": + return def read(rdr, error_on_eof): try: @@ -543,6 +552,9 @@ def read(rdr, error_on_eof): else: rdr.unread(ch2) itm = read_symbol(rdr, ch) + elif ch == u";": + skip_line(rdr) + return read(rdr, error_on_eof) else: itm = read_symbol(rdr, ch) diff --git a/pixie/vm/test/test_reader.py b/pixie/vm/test/test_reader.py index fa67007b..33a08152 100644 --- a/pixie/vm/test/test_reader.py +++ b/pixie/vm/test/test_reader.py @@ -15,7 +15,9 @@ u"2": 2, u"((42))": ((42,),), u"(platform+ 1 2)": (symbol(u"platform+"), 1, 2), - u"[42 43 44]": [42, 43, 44]} + u"[42 43 44]": [42, 43, 44], + u"(1 2 ; 7 8 9\n3)": (1, 2, 3,), + u"(1 2 ; 7 8 9\r\n3)": (1, 2, 3,)} class TestReader(unittest.TestCase): def _compare(self, frm, to): From 0b77e827ed90799173ff0b13ac62710e9bbed17c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 16:34:44 +0100 Subject: [PATCH 069/909] ignore compiled binary --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index dded1ddc..0e4825b4 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,4 @@ *.pyc externals +pixie-vm .idea From 074f6f61b446a246027ee8a61b22a0bcf136e7e9 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 16:38:20 +0100 Subject: [PATCH 070/909] handle 0 argument case in `and` and `or` as in clojure --- pixie/stdlib.lisp | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index fe1d1b3c..f8cb3593 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -490,11 +490,13 @@ (defmacro and + ([] true) ([x] x) ([x y] `(if ~x ~y nil)) ([x y & more] `(if ~x (and ~y ~@more)))) (defmacro or + ([] false) ([x] x) ([x y] `(let [r# ~x] (if r# r# ~y))) From 5e54929732c957923e25363c3fd3ca7570b1cc59 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 17:05:33 +0100 Subject: [PATCH 071/909] make gensym a multi arity function again, as in clojure. might be useful to make generated variable names in macros prettier. --- pixie/vm/compiler.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7d89564b..8df5e512 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -24,14 +24,17 @@ gensym_id = Atom(numbers.zero_int) +def gensym1(): + return gensym2(rt.wrap(u"gensym_")) -@code.as_var("gensym") -def gensym(): +def gensym2(prefix): rt.reset_BANG_(gensym_id, rt._add(rt.deref(gensym_id), rt.wrap(1))) i = rt.deref(gensym_id) - return rt.symbol(rt.str(rt.wrap(u"gensym_"), i)) + return rt.symbol(rt.str(prefix, i)) +gensym = code.intern_var(u"pixie.stdlib", u"gensym") +gensym.set_root(code.MultiArityFn({0: code.wrap_fn(gensym1), 1: code.wrap_fn(gensym2)})) class with_ns(object): def __init__(self, nm): From e80dc3f84ea60a2c959ac2e893deb055ac520f03 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 17:16:22 +0100 Subject: [PATCH 072/909] support -val-at on vectors --- pixie/vm/persistent_vector.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 870bc0fa..3ccae55f 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -372,6 +372,14 @@ def _nth(self, idx): assert isinstance(self, PersistentVector) return self.nth(idx.int_val()) +@extend(proto._val_at, PersistentVector) +def _val_at(self, key, not_found): + assert isinstance(self, PersistentVector) + if isinstance(key, Integer): + return self.nth(key.int_val()) + else: + return not_found + @extend(proto._conj, PersistentVector) def _conj(self, v): assert isinstance(self, PersistentVector) From 3b75b12bb4e7cbc97926d0ad42457dd9d476d74b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 17:50:28 +0100 Subject: [PATCH 073/909] implement get-in --- pixie/stdlib.lisp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index f8cb3593..3f0053cd 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -367,6 +367,20 @@ ([mp k not-found] (-val-at mp k not-found))) +(defn get-in + ([m ks] + (reduce get m ks)) + ([m ks not-found] + (loop [sentinel 'x + mi m + ks (seq ks)] + (if ks + (let [mi (get m (first ks) sentinel)] + (if (identical? sentinel mi) + not-found + (recur sentinel mi (next ks)))) + m)))) + (defmacro assert ([test] `(if ~test From 0a3002e0d593aab235363010f9be4405b110636f Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 31 Oct 2014 18:03:25 +0100 Subject: [PATCH 074/909] implement assoc-in --- pixie/stdlib.lisp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 3f0053cd..f37e1720 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -381,6 +381,15 @@ (recur sentinel mi (next ks)))) m)))) +(defn assoc-in + ([m ks v] + (let [ks (seq ks) + k (first ks) + ks (next ks)] + (if ks + (assoc m k (assoc-in (get m k) ks v)) + (assoc m k v))))) + (defmacro assert ([test] `(if ~test From 9cf798a6c8478dd960a12c3b8b9fe6facfb30c63 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 11:24:14 -0600 Subject: [PATCH 075/909] added shallow continuations --- pixie/stdlib.lisp | 30 ++++++++++++++++++++++++- pixie/vm/code.py | 3 ++- pixie/vm/compiler.py | 10 ++++++++- pixie/vm/interpreter.py | 49 ++++++++++++++++++++++++++++++++++++----- pixie/vm/stdlib.py | 22 +++++++++++++++++- 5 files changed, 105 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 5ea325f1..cbf4b9c0 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -517,6 +517,9 @@ ([x y & more] `(let [r# ~x] (if r# r# (or ~y ~@more))))) +(defmacro when [test & body] + `(if ~test (do ~@body))) + (deftype Range [:start :stop :step] IReduce @@ -533,10 +536,35 @@ (if (reduced? acc) @acc (recur (+ i step) acc))) - acc))))) + acc)))) + IIterable + (-iterator [self] + (let [start (. self :start) + stop (. self :stop) + step (. self :step)] + (loop [i start] + (when (or (and (> step 0) (< i stop)) + (and (< step 0) (> i stop)) + (and (= step 0))) + (yield i) + (recur (+ i step))))))) + + (defn range ([] (->Range 0 MAX-NUMBER 1)) ([stop] (->Range 0 stop 1)) ([start stop] (->Range start stop 1)) ([start stop step] (->Range start stop step))) + + +(extend -reduce ShallowContinuation + (fn [k f init] + (loop [acc init] + (if (reduced? init) + @init + (if (-at-end? k) + acc + (let [acc (f acc (-current k))] + (-move-next! k) + (recur acc))))))) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 01183c7e..388911fa 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -29,7 +29,8 @@ "PUSH_SELF", "POP_UP_N", "MAKE_MULTI_ARITY", - "MAKE_VARIADIC"] + "MAKE_VARIADIC", + "YIELD"] for x in range(len(BYTECODES)): globals()[BYTECODES[x]] = r_uint(x) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index bc745642..95ee23fd 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -695,6 +695,13 @@ def compile_var(form, ctx): def compile_catch(form, ctx): affirm(False, u"Catch used outside of try") +def compile_yield(form, ctx): + affirm(rt.count(form) == 2, u"yield takes a single argument") + arg = rt.first(rt.next(form)) + compile_form(arg, ctx) + ctx.bytecode.append(code.YIELD) + + builtins = {u"fn": compile_fn, u"if": compile_if, u"platform=": compile_platform_eq, @@ -708,7 +715,8 @@ def compile_catch(form, ctx): u"var": compile_var, u"__ns__": compile_ns, u"catch": compile_catch, - u"this-ns-name": compile_this_ns} + u"this-ns-name": compile_this_ns, + u"yield": compile_yield} def compiler_special(s): if isinstance(s, symbol.Symbol): diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 04a851a8..69da5de1 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -1,4 +1,4 @@ -from pixie.vm.object import Object, affirm, WrappedException +from pixie.vm.object import Object, affirm, WrappedException, Type import pixie.vm.code as code import pixie.vm.numbers as numbers from pixie.vm.primitives import nil, true, false @@ -29,7 +29,8 @@ class Frame(object): "debug_points", "args[*]", "base_code", - "closed_overs[*]" + "closed_overs[*]", + "finished" ] def __init__(self, code_obj, args): self = hint(self, access_directly=True, fresh_virtualizable=True) @@ -40,6 +41,7 @@ def __init__(self, code_obj, args): self.args = debug.make_sure_not_resized(args) self.base_code = code_obj.get_base_code() self.debug_points = code_obj.get_debug_points() + self.finished = False if code_obj is not None: self.unpack_code_obj() @@ -132,8 +134,37 @@ def make_multi_arity(frame, argc): return code.MultiArityFn(d, required_arity, rest_fn) -def interpret(code_obj, args=[], self_obj = None): - frame = Frame(code_obj, args) +class ShallowContinuation(Object): + _type = Type(u"pixie.stdlib.ShallowContinuation") + + def __init__(self, frame, val): + assert isinstance(frame, Frame) + self._frame = frame + self._val = val + + def type(self): + return ShallowContinuation._type + + + def is_finished(self): + return self._frame.finished + + def invoke(self, args): + affirm(len(args) == 1, u"Generators only take one argument") + arg = args[0] + self._frame.push(arg) + val = interpret(orig_frame=self._frame) + self._val = val + return self._val + +def interpret(code_obj=None, args=[], self_obj = None, orig_frame=None): + + if orig_frame is None: + assert code_obj is not None + frame = Frame(code_obj, args) + else: + frame = orig_frame + while True: jitdriver.jit_merge_point(bc=frame.bc, ip=frame.ip, @@ -188,7 +219,7 @@ def interpret(code_obj, args=[], self_obj = None): if inst == code.RETURN: val = frame.pop() - + frame.finished = True return val if inst == code.COND_BR: @@ -318,6 +349,14 @@ def interpret(code_obj, args=[], self_obj = None): continue + if inst == code.YIELD: + if orig_frame is None: + frame = jit.hint(frame, force_virtualizable=True) + k = ShallowContinuation(frame, frame.pop()) + return k + else: + return frame.pop() + affirm(False, u"NO DISPATCH FOR: " + unicode(code.BYTECODES[inst])) raise Exception() diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 6138af88..a58b8583 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -10,6 +10,7 @@ import os.path as path import rpython.rlib.rstacklet as rstacklet from rpython.rlib.rarithmetic import r_uint +from pixie.vm.interpreter import ShallowContinuation defprotocol("pixie.stdlib", "ISeq", ["-first", "-next"]) @@ -54,7 +55,7 @@ defprotocol("pixie.stdlib", "ITransientCollection", ["-conj!"]) defprotocol("pixie.stdlib", "IIterable", ["-iterator"]) -defprotocol("pixie.stdlib", "IIterator", ["-current", "-has-next?", "-move-next!"]) +defprotocol("pixie.stdlib", "IIterator", ["-current", "-at-end?", "-move-next!"]) def __make_code_overrides(x): @@ -165,12 +166,29 @@ def __name(self): def __name(_): return nil +@extend(_current, ShallowContinuation) +def _current(self): + assert isinstance(self, ShallowContinuation) + return self._val + +@extend(_at_end_QMARK_, ShallowContinuation) +def _(self): + assert isinstance(self, ShallowContinuation) + return true if self.is_finished() else false + +@extend(_move_next_BANG_, ShallowContinuation) +def _(self): + assert isinstance(self, ShallowContinuation) + self.invoke([nil]) + return self + @returns(r_uint) @as_var("hash") def __hash(x): return rt._hash(x) + _count_driver = jit.JitDriver(name="pixie.stdlib.count", greens=["tp"], reds="auto") @@ -472,3 +490,5 @@ def _merge__args(args): for x in range(1, len(args)): acc = rt._reduce(acc, merge_fn, args[x]) return acc + + From 3ba704a4e89b706b051a0d50504ca5f9d41fdd8c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 12:32:28 -0600 Subject: [PATCH 076/909] working on generators jit stuff --- pixie/vm/interpreter.py | 42 ++++++++++++++++++++++++++--------------- 1 file changed, 27 insertions(+), 15 deletions(-) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 69da5de1..5b862a44 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -8,10 +8,10 @@ import rpython.rlib.debug as debug import pixie.vm.rt as rt -def get_location(ip, sp, bc, base_code): +def get_location(ip, sp, is_continuation, bc, base_code): return code.BYTECODES[bc[ip]] + " in " + str(base_code._name) -jitdriver = JitDriver(greens=["ip", "sp", "bc", "base_code"], reds=["frame"], virtualizables=["frame"], +jitdriver = JitDriver(greens=["ip", "sp", "is_continuation", "bc", "base_code"], reds=["frame"], virtualizables=["frame"], get_printable_location=get_location) @@ -30,8 +30,9 @@ class Frame(object): "args[*]", "base_code", "closed_overs[*]", - "finished" -] + "finished", + "_is_continuation" + ] def __init__(self, code_obj, args): self = hint(self, access_directly=True, fresh_virtualizable=True) self.code_obj = code_obj @@ -42,9 +43,17 @@ def __init__(self, code_obj, args): self.base_code = code_obj.get_base_code() self.debug_points = code_obj.get_debug_points() self.finished = False + self._is_continuation = 0 if code_obj is not None: self.unpack_code_obj() + def set_continuation(self): + self._is_continuation = 1 + + def is_continuation(self): + return self._is_continuation == 1 + + def unpack_code_obj(self): self.bc = self.code_obj.get_bytecode() self.consts = self.code_obj.get_consts() @@ -151,26 +160,26 @@ def is_finished(self): def invoke(self, args): affirm(len(args) == 1, u"Generators only take one argument") - arg = args[0] - self._frame.push(arg) - val = interpret(orig_frame=self._frame) + self._frame.push(args[0]) + val = interpret(frame=self._frame) self._val = val return self._val -def interpret(code_obj=None, args=[], self_obj = None, orig_frame=None): +def interpret(code_obj=None, args=[], self_obj = None, frame=None): - if orig_frame is None: + if frame is None: assert code_obj is not None frame = Frame(code_obj, args) - else: - frame = orig_frame + + while True: jitdriver.jit_merge_point(bc=frame.bc, ip=frame.ip, sp=frame.sp, base_code=frame.base_code, - frame=frame) + frame=frame, + is_continuation=frame._is_continuation) inst = frame.get_inst() if inst == code.LOAD_CONST: @@ -299,7 +308,8 @@ def interpret(code_obj=None, args=[], self_obj = None, orig_frame=None): ip=frame.ip, sp=frame.sp, base_code=frame.base_code, - frame=frame) + frame=frame, + is_continuation=frame._is_continuation) continue if inst == code.PUSH_SELF: @@ -334,7 +344,8 @@ def interpret(code_obj=None, args=[], self_obj = None, orig_frame=None): ip=frame.ip, sp=frame.sp, base_code=frame.base_code, - frame=frame) + frame=frame, + is_continuation=frame._is_continuation) continue if inst == code.MAKE_MULTI_ARITY: @@ -350,7 +361,8 @@ def interpret(code_obj=None, args=[], self_obj = None, orig_frame=None): continue if inst == code.YIELD: - if orig_frame is None: + if not frame.is_continuation(): + frame.set_continuation() frame = jit.hint(frame, force_virtualizable=True) k = ShallowContinuation(frame, frame.pop()) return k From 1b9d84e57523bdc06db7e4faef17029c7a233452 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 13:29:28 -0600 Subject: [PATCH 077/909] remove the force_virtualizable hint, it now translates, but why? --- pixie/vm/interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 5b862a44..f9f39902 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -363,7 +363,7 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): if inst == code.YIELD: if not frame.is_continuation(): frame.set_continuation() - frame = jit.hint(frame, force_virtualizable=True) + #frame = jit.hint(frame, force_virtualizable=True) k = ShallowContinuation(frame, frame.pop()) return k else: From f9c65408cfb62c79b0da90a8e730144b1dd0e288 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 13:36:49 -0600 Subject: [PATCH 078/909] benchmark files, and fixes to the compiler for empty hashmaps --- benchmarks/ffi_test.lisp | 5 +++++ benchmarks/transduce_range_iterator.lisp | 1 + benchmarks/transduce_range_reduced.lisp | 1 + pixie/test.lisp | 2 +- pixie/vm/compiler.py | 5 +++-- pixie/vm/persistent_hash_map.py | 5 +++++ 6 files changed, 16 insertions(+), 3 deletions(-) create mode 100644 benchmarks/ffi_test.lisp create mode 100644 benchmarks/transduce_range_iterator.lisp create mode 100644 benchmarks/transduce_range_reduced.lisp diff --git a/benchmarks/ffi_test.lisp b/benchmarks/ffi_test.lisp new file mode 100644 index 00000000..a96d4d71 --- /dev/null +++ b/benchmarks/ffi_test.lisp @@ -0,0 +1,5 @@ +(loop [x 0] + (if (= x 10000) + x + (do (printf ".") + (recur (inc x))))) diff --git a/benchmarks/transduce_range_iterator.lisp b/benchmarks/transduce_range_iterator.lisp new file mode 100644 index 00000000..a30f9ab2 --- /dev/null +++ b/benchmarks/transduce_range_iterator.lisp @@ -0,0 +1 @@ +(reduce (fn [_ i] nil) nil (-iterator (range 10000000))) diff --git a/benchmarks/transduce_range_reduced.lisp b/benchmarks/transduce_range_reduced.lisp new file mode 100644 index 00000000..4939c02a --- /dev/null +++ b/benchmarks/transduce_range_reduced.lisp @@ -0,0 +1 @@ +(reduce (fn [_ i] nil) nil (range 10000000)) diff --git a/pixie/test.lisp b/pixie/test.lisp index f22f1fc3..f57cd45f 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -24,7 +24,7 @@ (defn run-tests [] (push-binding-frame!) (set! (var *stats*) (atom {:fail 0 :pass 0})) - (print @tests) + (print "Running: " (count@tests) " tests") (foreach [test @tests] ((val test))) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 95ee23fd..31999d28 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -85,7 +85,7 @@ def add_sp(self, v): self._max_sp = self._sp def sub_sp(self, v): - assert self._sp >= v + assert self._sp >= v, (v, self._sp) if self._max_sp < self._sp: self._max_sp = self._sp self._sp -= v @@ -289,7 +289,8 @@ def compile_map_literal(form, ctx): size = rt.count(form) * 2 ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(size) + 1) - ctx.sub_sp(size - 1) + if size > 0: + ctx.sub_sp(size - 1) compile_meta(rt.meta(form), ctx) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 9719b24e..6edddf7f 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -305,6 +305,11 @@ def hashmap__args(args): return acc +@extend(proto._count, PersistentHashMap) +def _count(self): + assert isinstance(self, PersistentHashMap) + return rt.wrap(self._cnt) + @extend(proto._val_at, PersistentHashMap) def _val_at(self, key, not_found): From ead754f0c20c54353b9b6afe864533f993ec9601 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 14:55:27 -0600 Subject: [PATCH 079/909] example implementation of mu-kanren, added a few functions to stdlib --- examples/mu-kanren.lisp | 63 +++++++++++++++++++++++++++++++++++++++++ pixie/stdlib.lisp | 51 ++++++++++++++++++++++++++++----- 2 files changed, 107 insertions(+), 7 deletions(-) create mode 100644 examples/mu-kanren.lisp diff --git a/examples/mu-kanren.lisp b/examples/mu-kanren.lisp new file mode 100644 index 00000000..3d48e9ab --- /dev/null +++ b/examples/mu-kanren.lisp @@ -0,0 +1,63 @@ +(ns examples.mu-kanren) + +(defn lvar [] + (gensym)) + +(defn lvar? [x] + (symbol? x)) + +(defn walk [s u] + (let [pr (get s u)] + (if (lvar? pr) + (recur s pr) + pr) + u)) + +(defn unify [s u v] + (let [u (walk s u) + v (walk s v)] + (cond + (and (lvar? u) + (lvar? v) + (= u v)) s + (lvar? u) (assoc s u v) + (lvar? v) (assoc s v u) + :else (and (= u v) s)))) + +(defn == [a b] + (keep (fn [s] (unify s a b)))) + + +(defn -disj [& xforms] + (fn [xf] + (let [xforms (transduce (map (fn [xform] + (xform xf))) + conj + xforms)] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (reduce + (fn [acc xform] + (xform acc i)) + acc + xforms)))))) + +(defn conde [& goals] + (apply -disj (map (fn [goals] + (apply comp goals)) + goals))) + + +"Use transduce to run eagerly" +(transduce (conde + [(== 'a 42)] + [ (== 'b 1)]) + conj + [{}]) + +"Use sequence to make it lazy (via stacklets)" +(sequence (conde + [(== 'a 42)] + [(== 'b 1)]) + [{}]) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index cbf4b9c0..9d21645a 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -6,13 +6,6 @@ (def program-arguments []) -(def map (fn ^{:doc "map - creates a transducer that applies f to every input element" :added "0.1"} - map [f] - (fn [xf] - (fn - ([] (xf)) - ([result] (xf result)) - ([result item] (xf result (f item))))))) (def conj (fn conj ([] []) @@ -39,6 +32,19 @@ result (-reduce coll f init)] (f result))))) +(def map (fn ^{:doc "map - creates a transducer that applies f to every input element" + :added "0.1"} + map + ([f] + (fn [xf] + (fn + ([] (xf)) + ([result] (xf result)) + ([result item] (xf result (f item)))))) + ([f coll] + (transduce (map f) conj coll)))) + + (def reduce (fn [rf init col] (-reduce col rf init))) @@ -185,6 +191,7 @@ data)))] (stacklet->lazy-seq f))))) + (extend -seq PersistentVector sequence) (extend -seq Array sequence) @@ -557,6 +564,17 @@ ([start stop] (->Range start stop 1)) ([start stop step] (->Range start stop step))) +(defn iterator [coll] + (-iterator coll)) + +(defn move-next! [i] + (-move-next! i) + i) + +(defn at-end? [i] + (-at-end? i) + i) + (extend -reduce ShallowContinuation (fn [k f init] @@ -568,3 +586,22 @@ (let [acc (f acc (-current k))] (-move-next! k) (recur acc))))))) + +(defn filter [f] + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (if (f i) + (xf acc i) + acc))))) + +(defn keep [f] + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (let [result (f i)] + (if result + (xf acc result) + acc)))))) From 23865d7c3d4a05cfed66fbd7ed60fce820c577a1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 31 Oct 2014 16:29:41 -0600 Subject: [PATCH 080/909] functions can now refer to themselves --- pixie/stdlib.lisp | 13 ++++---- pixie/vm/code.py | 56 ++++++++++++++++++++++------------ pixie/vm/compiler.py | 6 ++-- pixie/vm/interpreter.py | 12 +++++--- pixie/vm/stacklet.py | 2 +- pixie/vm/test/test_stacklet.py | 4 +-- 6 files changed, 56 insertions(+), 37 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 9d21645a..ceda4f44 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -474,15 +474,14 @@ (defn swap! [a f & args] (reset! a (apply f @a args))) -(def update-inner-f (fn inner-f - ([m f k] - (assoc m k (f (get m k)))) - ([m f k & ks] - (assoc m k (apply update-inner-f m f ks))))) - (defn update-in [m ks f & args] - (let [f (fn [m] (apply f m args))] + (let [f (fn [m] (apply f m args)) + update-inner-f (fn update-inner-f + ([m f k] + (assoc m k (f (get m k)))) + ([m f k & ks] + (assoc m k (apply update-inner-f m f ks))))] (apply update-inner-f m f ks))) (defn nil? [x] diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 388911fa..a1b818b1 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -102,9 +102,6 @@ def set_macro(self): def is_macro(self): return self._is_macro - def _invoke(self, args): - raise NotImplementedError() - def get_consts(self): raise NotImplementedError() @@ -116,9 +113,12 @@ def stack_size(self): return 0 def invoke(self, args): - result = self._invoke(args) + result = self.invoke(args) return result + def invoke_with(self, args, this_fn): + return self.invoke(args) + class MultiArityFn(BaseCode): _type = object.Type(u"pixie.stdlib.MultiArityFn") @@ -155,8 +155,11 @@ def get_fn(self, arity): affirm(False, u"Wrong number of args to fn: got " + unicode(str(arity)) + u" expected " + u",".join(acc)) - def _invoke(self, args): - return self.get_fn(len(args)).invoke(args) + def invoke(self, args): + return self.invoke_with(args, self) + + def invoke_with(self, args, self_fn): + return self.get_fn(len(args)).invoke_with(args, self_fn) @@ -171,12 +174,15 @@ def __init__(self): def type(self): return NativeFn._type - def _invoke(self, args): + def invoke(self, args): return self.inner_invoke(args) def inner_invoke(self, args): raise NotImplementedError() + def invoke_with(self, args, this_fn): + return self.invoke(args) + class Code(BaseCode): """Interpreted code block. Contains consts and """ @@ -201,9 +207,12 @@ def with_meta(self, meta): def get_debug_points(self): return self._debug_points - def _invoke(self, args): + def invoke(self, args): + return self.invoke_with(args, self) + + def invoke_with(self, args, this_fn): try: - return interpret(self, args) + return interpret(self, args, self_obj=this_fn) except object.WrappedException as ex: ex._ex._trace.append(object.PixieCodeInfo(self._name)) raise @@ -242,20 +251,23 @@ def __init__(self, code, required_arity, meta=nil): def with_meta(self, meta): return VariadicCode(self._code, self._required_arity, meta) - def _invoke(self, args): + def invoke(self, args): + return self.invoke_with(args, self) + + def invoke_with(self, args, self_fn): from pixie.vm.array import array argc = len(args) if self._required_arity == 0: - return self._code.invoke([array(args)]) + return self._code.invoke_with([array(args)], self_fn) if argc == self._required_arity: new_args = resize_list(args, len(args) + 1) new_args[len(args)] = array([]) - return self._code.invoke(new_args) + return self._code.invoke_with(new_args, self_fn) elif argc > self._required_arity: start = slice_from_start(args, self._required_arity, 1) rest = slice_to_end(args, self._required_arity) start[self._required_arity] = array(rest) - return self._code.invoke(start) + return self._code.invoke_with(start, self_fn) affirm(False, u"Got " + unicode(str(argc)) + u" arg(s) need at least " + unicode(str(self._required_arity))) class Closure(BaseCode): @@ -274,9 +286,12 @@ def __init__(self, code, closed_overs, meta=nil): def with_meta(self, meta): return Closure(self._code, self._closed_overs, meta) - def _invoke(self, args): + def invoke(self, args): + return self.invoke_with(args, self) + + def invoke_with(self, args, self_fn): try: - return interpret(self, args) + return interpret(self, args, self_obj=self_fn) except object.WrappedException as ex: ex._ex._trace.append(object.PixieCodeInfo(self._code._name)) raise @@ -381,7 +396,10 @@ def deref(self): def is_defined(self): return self._root is not undefined - def _invoke(self, args): + def invoke_with(self, args, this_fn): + return self.invoke(args) + + def invoke(self, args): return self.deref().invoke(args) class bindings(py_object): @@ -510,7 +528,7 @@ class DefaultProtocolFn(NativeFn): def __init__(self, pfn): self._pfn = pfn - def _invoke(self, args): + def invoke(self, args): from pixie.vm.string import String tp = args[0].type()._name affirm(False, u"No override for " + tp + u" on " + self._pfn._name + u" in protocol " + self._pfn._protocol._name) @@ -574,7 +592,7 @@ def get_protocol_fn(self, tp, rev): fn = self._dict.get(tp, self._default_fn) return promote(fn) - def _invoke(self, args): + def invoke(self, args): a = args[0].type() fn = self.get_protocol_fn(a, self._rev) try: @@ -621,7 +639,7 @@ def get_fn(self, tp1, tp2, _rev): fn = d1.get(tp2, self._default_fn) return promote(fn) - def _invoke(self, args): + def invoke(self, args): affirm(len(args) >= 2, u"DoublePolymorphicFunctions take at least two args") a = args[0].type() b = args[1].type() diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 31999d28..698932c3 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -275,7 +275,7 @@ def call_macro(var, form, ctx): class CompileMapRf(code.NativeFn): def __init__(self, ctx): self._ctx = ctx - def _invoke(self, args): + def invoke(self, args): map_entry = args[1] compile_form(rt.key(map_entry), self._ctx) compile_form(rt.val(map_entry), self._ctx) @@ -407,7 +407,7 @@ def compile_platform_eq(form, ctx): def add_args(name, args, ctx): required_args = -1 local_idx = 0 - #ctx.add_local(name, Self()) + ctx.add_local(name, Self()) for x in range(rt.count(args)): arg = rt.nth(args, rt.wrap(x)) affirm(isinstance(arg, symbol.Symbol), u"Argument names must be symbols") @@ -456,7 +456,7 @@ def compile_fn(form, ctx): def compile_fn_body(name, args, body, ctx): new_ctx = Context(name._str, rt.count(args), ctx) - required_args = add_args(name, args, new_ctx) + required_args = add_args(name._str, args, new_ctx) bc = 0 if name is not None: diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index f9f39902..1617d5a0 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -31,11 +31,13 @@ class Frame(object): "base_code", "closed_overs[*]", "finished", - "_is_continuation" + "_is_continuation", + "self_obj" ] - def __init__(self, code_obj, args): + def __init__(self, code_obj, args, self_obj): self = hint(self, access_directly=True, fresh_virtualizable=True) self.code_obj = code_obj + self.self_obj = self_obj self.sp = r_uint(0) self.ip = r_uint(0) self.stack = [None] * code_obj.stack_size() @@ -169,7 +171,7 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): if frame is None: assert code_obj is not None - frame = Frame(code_obj, args) + frame = Frame(code_obj, args, self_obj or code_obj) @@ -302,7 +304,7 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): argc = frame.get_inst() args = frame.pop_n(argc) - frame = Frame(frame.code_obj, args) + frame = Frame(frame.code_obj, args, frame.self_obj) jitdriver.can_enter_jit(bc=frame.bc, ip=frame.ip, @@ -313,7 +315,7 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): continue if inst == code.PUSH_SELF: - frame.push(frame.code_obj) + frame.push(frame.self_obj) continue if inst == code.DUP_NTH: diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index 199179d6..5cf85cdd 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -57,7 +57,7 @@ def __init__(self, h): def type(self): return WrappedHandler._type - def _invoke(self, args): + def invoke(self, args): affirm(len(args) == 1, u"Only one arg to continuation allowed") global_state._from = global_state._to global_state._to = self diff --git a/pixie/vm/test/test_stacklet.py b/pixie/vm/test/test_stacklet.py index 0ff66442..851efbf6 100644 --- a/pixie/vm/test/test_stacklet.py +++ b/pixie/vm/test/test_stacklet.py @@ -5,7 +5,7 @@ from pixie.vm.primitives import nil class YieldingFn(code.BaseCode): - def _invoke(self, args): + def invoke(self, args): assert len(args) == 2 hdler = args[0] arg = args[1] @@ -29,7 +29,7 @@ def yielding_fn(yld): class WrappingFn(code.NativeFn): def __init__(self, cont): self._cont = cont - def _invoke(self, args): + def invoke(self, args): ret = args[0] ret.invoke([self._cont.invoke([4])]) From 55d70dc0a952a5503ce737007affe150224993c3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 1 Nov 2014 00:09:19 +0100 Subject: [PATCH 081/909] implement `-seq`, `keys` and `vals` on PersistentHashMap --- pixie/stdlib.lisp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 9d21645a..bc1755e9 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -370,6 +370,16 @@ (fn [v] (transduce ordered-hash-reducing-fn v))) +(defn keys [m] + (reduce (fn [ks e] (conj ks (key e))) nil m)) + +(defn vals [m] + (reduce (fn [ks e] (conj ks (val e))) nil m)) + +(extend -seq PersistentHashMap + (fn [m] + (reduce conj nil m))) + (extend -str PersistentHashMap (fn [v] (apply str "{" (conj (transduce (comp cat (interpose " ")) conj v) "}")))) From 1c0a971c2e8a8b3823bd7b8802cf16d383414bc7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 1 Nov 2014 11:27:20 +0100 Subject: [PATCH 082/909] implement initial support for sets using maps internally, as clojure's sets do. it's rather simple, actually, but also quite surprising... --- pixie/stdlib.lisp | 6 ++++ pixie/vm/persistent_hash_set.py | 49 ++++++++++++++++++++++++++++++++ pixie/vm/rt.py | 1 + tests/collections/test-sets.lisp | 8 ++++++ 4 files changed, 64 insertions(+) create mode 100644 pixie/vm/persistent_hash_set.py create mode 100644 tests/collections/test-sets.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index bc1755e9..e03c0544 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -388,6 +388,12 @@ (fn [v] (transduce cat unordered-hash-reducing-fn v))) +(extend -seq PersistentHashSet sequence) + +(extend -str PersistentHashSet + (fn [s] + (apply str "#{" (conj (transduce (interpose " ") conj s) "}")))) + (extend -str Keyword (fn [k] (if (namespace k) diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py new file mode 100644 index 00000000..2756f189 --- /dev/null +++ b/pixie/vm/persistent_hash_set.py @@ -0,0 +1,49 @@ +py_object = object +import pixie.vm.object as object +from pixie.vm.object import affirm +from pixie.vm.primitives import nil, true, false +from pixie.vm.numbers import Integer +import pixie.vm.persistent_hash_map as persistent_hash_map +import pixie.vm.stdlib as proto +from pixie.vm.code import extend, as_var +from rpython.rlib.rarithmetic import r_uint, intmask +import rpython.rlib.jit as jit +import pixie.vm.rt as rt + +class PersistentHashSet(object.Object): + _type = object.Type(u"pixie.stdlib.PersistentHashSet") + + def type(self): + return PersistentHashSet._type + + def __init__(self, m): + self._map = m + + def conj(self, v): + return PersistentHashSet(self._map.assoc(v, v)) + +EMPTY = PersistentHashSet(persistent_hash_map.EMPTY) + +@as_var("set") +def _create(coll): + ret = EMPTY + coll = rt._seq(coll) + while coll is not nil: + ret = ret.conj(rt._first(coll)) + coll = rt._seq(rt._next(coll)) + return ret + +@extend(proto._count, PersistentHashSet) +def _count(self): + assert isinstance(self, PersistentHashSet) + return rt._count(self._map) + +@extend(proto._conj, PersistentHashSet) +def _conj(self, v): + assert isinstance(self, PersistentHashSet) + return self.conj(v) + +@extend(proto._reduce, PersistentHashSet) +def _reduce(self, f, init): + assert isinstance(self, PersistentHashSet) + return rt._reduce(rt.keys(self._map), f, init) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index e1e6b50c..6c7c2f9b 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -58,6 +58,7 @@ def wrapper(*args): import pixie.vm.lazy_seq import pixie.vm.persistent_list import pixie.vm.persistent_hash_map + import pixie.vm.persistent_hash_set import pixie.vm.custom_types import pixie.vm.compiler as compiler import pixie.vm.map_entry diff --git a/tests/collections/test-sets.lisp b/tests/collections/test-sets.lisp new file mode 100644 index 00000000..ff318c94 --- /dev/null +++ b/tests/collections/test-sets.lisp @@ -0,0 +1,8 @@ +(ns collections.test-sets + (require pixie.test :as t)) + +(t/deftest test-count + (t/assert= (count (set [])) 0) + (t/assert= (count (set [1 2 3])) 3) + (t/assert= (count (set [1 1 2 1])) 2)) + From 9800ec84f0392df8fe72617ce54182264f2a7dc2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 1 Nov 2014 11:53:27 +0100 Subject: [PATCH 083/909] make semicolon comments a reader macro --- pixie/vm/reader.py | 32 +++++++++++++++++--------------- 1 file changed, 17 insertions(+), 15 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index a9c4a35a..3fc4fe04 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -380,6 +380,21 @@ def invoke(self, rdr, ch): return obj +class LineCommentReader(ReaderHandler): + def invoke(self, rdr, ch): + self.skip_line(rdr) + return read(rdr, True) + + def skip_line(self, rdr): + while True: + ch = rdr.read() + if ch == u"\n": + return + elif ch == u"\r": + ch2 = rdr.read() + if ch2 == u"\n": + return + handlers = {u"(": ListReader(), u")": UnmachedListReader(), u"[": VectorReader(), @@ -392,7 +407,8 @@ def invoke(self, rdr, ch): u"@": DerefReader(), u"`": SyntaxQuoteReader(), u"~": UnquoteReader(), - u"^": MetaReader()} + u"^": MetaReader(), + u";": LineCommentReader()} # inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 # sign hex oct radix decimal @@ -509,17 +525,6 @@ def throw_syntax_error_with_data(rdr, txt): raise object.WrappedException(err) - -def skip_line(rdr): - while True: - ch = rdr.read() - if ch == u"\n": - return - elif ch == u"\r": - ch2 = rdr.read() - if ch2 == u"\n": - return - def read(rdr, error_on_eof): try: eat_whitespace(rdr) @@ -552,9 +557,6 @@ def read(rdr, error_on_eof): else: rdr.unread(ch2) itm = read_symbol(rdr, ch) - elif ch == u";": - skip_line(rdr) - return read(rdr, error_on_eof) else: itm = read_symbol(rdr, ch) From 442bfdb13fce1c79cc0b93ae7a8623ec66998878 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 1 Nov 2014 23:53:51 +0100 Subject: [PATCH 084/909] implement set literals this was surprisingly hard, especially the compiler bit. it turned out to be rather simple, but maybe there's a better way to do it. --- pixie/vm/compiler.py | 16 ++++++++++++++++ pixie/vm/reader.py | 34 ++++++++++++++++++++++++++++++++-- 2 files changed, 48 insertions(+), 2 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 31999d28..63207497 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -1,6 +1,7 @@ from pixie.vm.object import Object, _type_registry, affirm, InterpreterCodeInfo from pixie.vm.primitives import nil, true, false, Bool from pixie.vm.persistent_vector import EMPTY, PersistentVector +from pixie.vm.persistent_hash_set import PersistentHashSet import pixie.vm.numbers as numbers from pixie.vm.cons import cons, Cons import pixie.vm.symbol as symbol @@ -294,6 +295,17 @@ def compile_map_literal(form, ctx): compile_meta(rt.meta(form), ctx) +class ConsReduce(code.NativeFn): + def _invoke(self, args): + return rt.cons(args[1], args[0]) + +def compile_set_literal(form, ctx): + vals = rt.reduce(ConsReduce(), nil, form) + vector_call = rt.cons(rt.symbol(rt.wrap(u"vector")), vals) + set_call = rt.cons(rt.symbol(rt.wrap(u"set")), rt.cons(vector_call, nil)) + + compile_cons(set_call, ctx) + def compile_meta(meta, ctx): ctx.push_const(code.intern_var(u"pixie.stdlib", u'with-meta')) ctx.bytecode.append(code.DUP_NTH) @@ -368,6 +380,10 @@ def compile_form(form, ctx): return + if isinstance(form, PersistentHashSet): + compile_set_literal(form, ctx) + return + if rt.instance_QMARK_(rt.IMap.deref(), form): compile_map_literal(form, ctx) return diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index bb56ea16..034c49ad 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -13,6 +13,7 @@ from pixie.vm.string import String from pixie.vm.code import wrap_fn, extend from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP +from pixie.vm.persistent_hash_set import EMPTY as EMPTY_SET import pixie.vm.stdlib as proto import pixie.vm.compiler as compiler @@ -380,6 +381,30 @@ def invoke(self, rdr, ch): return obj +class SetReader(ReaderHandler): + def invoke(self, rdr, ch): + acc = EMPTY_SET + while True: + eat_whitespace(rdr) + ch = rdr.read() + if ch == u"}": + return acc + + rdr.unread(ch) + acc = acc.conj(read(rdr, True)) + +dispatch_handlers = { + u"{": SetReader() +} + +class DispatchReader(ReaderHandler): + def invoke(self, rdr, ch): + ch = rdr.read() + handler = dispatch_handlers[ch] + if handler is None: + raise Exception("unknown dispatch #" + ch) + return handler.invoke(rdr, ch) + handlers = {u"(": ListReader(), u")": UnmachedListReader(), u"[": VectorReader(), @@ -392,7 +417,9 @@ def invoke(self, rdr, ch): u"@": DerefReader(), u"`": SyntaxQuoteReader(), u"~": UnquoteReader(), - u"^": MetaReader()} + u"^": MetaReader(), + u"#": DispatchReader() +} # inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 # sign hex oct radix decimal @@ -466,12 +493,15 @@ def read_number(rdr, ch): return parsed return Symbol(joined) +def is_terminating_macro(ch): + return ch != u"#" and ch != u"'" and ch != u"%" and ch in handlers + def read_symbol(rdr, ch): acc = [ch] try: while True: ch = rdr.read() - if is_whitespace(ch) or ch in handlers: + if is_whitespace(ch) or is_terminating_macro(ch): rdr.unread(ch) break acc.append(ch) From 97729f263ecd36100fcc0dccfa472b094732fccf Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 01:05:59 +0100 Subject: [PATCH 085/909] implement abs, quot and rem these don't work for ratios yet and quot doesn't work properly with floats, but we they're all ok for integers. (and do what we need for now.) --- pixie/stdlib.lisp | 10 ++++++++++ pixie/vm/numbers.py | 4 +++- 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 2764c710..e70327ac 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -236,6 +236,12 @@ ([x y & args] (reduce -div (-div x y) args))) +(defn quot [num div] + (-quot num div)) + +(defn rem [num div] + (-rem num div)) + (defn = ([x] true) ([x y] (eq x y)) @@ -551,6 +557,10 @@ (defmacro when [test & body] `(if ~test (do ~@body))) +(defn abs [x] + (if (< x 0) + (* -1 x) + x)) (deftype Range [:start :stop :step] IReduce diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 360db2bc..c3d0b9d3 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -67,6 +67,8 @@ def type(self): _sub = as_var("-sub")(DoublePolymorphicFn(u"-sub", IMath)) _mul = as_var("-mul")(DoublePolymorphicFn(u"-mul", IMath)) _div = as_var("-div")(DoublePolymorphicFn(u"-div", IMath)) +_quot = as_var("-quot")(DoublePolymorphicFn(u"-quot", IMath)) +_rem = as_var("-rem")(DoublePolymorphicFn(u"-rem", IMath)) _lt = as_var("-lt")(DoublePolymorphicFn(u"-lt", IMath)) _gt = as_var("-gt")(DoublePolymorphicFn(u"-gt", IMath)) _lte = as_var("-lte")(DoublePolymorphicFn(u"-lte", IMath)) @@ -94,7 +96,7 @@ def define_num_ops(): num_classes = [(Integer, "int_val"), (Float, "float_val")] for (c1, conv1) in num_classes: for (c2, conv2) in num_classes: - for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/")]: + for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/"), ("_quot", "/"), ("_rem", "%")]: if op == "_div" and c1 == Integer and c2 == Integer: continue extend_num_op(op, c1, c2, conv1, sym, conv2) From 5f9320b75259bdbb80f37d21f8f40ec1081f4849 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 01:07:10 +0100 Subject: [PATCH 086/909] make ranges indexed and counted. --- pixie/stdlib.lisp | 22 +++++++++++++++++++++- 1 file changed, 21 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e70327ac..e0fd99ad 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -588,7 +588,27 @@ (and (< step 0) (> i stop)) (and (= step 0))) (yield i) - (recur (+ i step))))))) + (recur (+ i step)))))) + ICounted + (-count [self] + (let [start (. self :start) + stop (. self :stop) + step (. self :step)] + (if (or (and (< start stop) (< step 0)) + (and (> start stop) (> step 0)) + (= step 0)) + 0 + (abs (quot (- start stop) step))))) + IIndexed + (-nth [self idx] + (let [start (. self :start) + stop (. self :stop) + step (. self :step) + cmp (if (< start stop) < >) + val (+ start (* idx step))] + (if (cmp val stop) + val + nil)))) From 0a5f2618112cf78204a7414d8f455cefd639fde1 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 02:06:18 +0100 Subject: [PATCH 087/909] introduce satisfies?, change instance? to check class not sure about this, but this allows implementing type predicates for types that aren't based on protocols. --- pixie/vm/compiler.py | 6 +++--- pixie/vm/object.py | 18 +++++++++++++++--- pixie/vm/reader.py | 6 +++--- pixie/vm/stdlib.py | 21 ++++++++++++++------- 4 files changed, 35 insertions(+), 16 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index b0820b9c..d740833e 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -315,7 +315,7 @@ def compile_form(form, ctx): ctx.push_const(nil) return - if rt.instance_QMARK_(rt.ISeq.deref(), form) and form is not nil: + if rt.satisfies_QMARK_(rt.ISeq.deref(), form) and form is not nil: return compile_cons(form, ctx) if isinstance(form, numbers.Integer): ctx.push_const(form) @@ -371,7 +371,7 @@ def compile_form(form, ctx): return - if rt.instance_QMARK_(rt.IMap.deref(), form): + if rt.satisfies_QMARK_(rt.IMap.deref(), form): compile_map_literal(form, ctx) return @@ -434,7 +434,7 @@ def compile_fn(form, ctx): - if rt.instance_QMARK_(rt.ISeq.deref(), rt.first(form)): + if rt.satisfies_QMARK_(rt.ISeq.deref(), rt.first(form)): arities = [] while form is not nil: required_arity, argc = compile_fn_body(name, rt.first(rt.first(form)), rt.next(rt.first(form)), ctx) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 9a53f1c5..461edc6e 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -58,18 +58,30 @@ def get_by_name(self, nm, default=None): _type_registry = TypeRegistry() class Type(Object): - def __init__(self, name): + def __init__(self, name, parent = None): #assert isinstance(name, unicode), u"Type names must be unicode" _type_registry.register_type(name, self) self._name = name + self._parent = parent def type(self): return Type._type - - Type._type = Type(u"Type") +def istypeinstance(obj, t): + if obj._type == t: + return True + elif obj._type._parent != None: + obj_type = obj._type._parent + while obj_type != None: + if obj_type == t: + return True + obj_type = obj_type._parent + return False + else: + return False + class RuntimeException(Object): _type = Type(u"pixie.stdlib.RuntimeException") def __init__(self, data): diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index bd0a93e0..3f7386e9 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -290,12 +290,12 @@ def invoke(self, rdr, ch): LIST = symbol(u"list") def is_unquote(form): - return True if rt.instance_QMARK_(rt.ISeq.deref(), form) \ + return True if rt.satisfies_QMARK_(rt.ISeq.deref(), form) \ and rt.eq(rt.first(form), UNQUOTE) \ else False def is_unquote_splicing(form): - return True if rt.instance_QMARK_(rt.ISeq.deref(), form) \ + return True if rt.satisfies_QMARK_(rt.ISeq.deref(), form) \ and rt.eq(rt.first(form), UNQUOTE_SPLICING) \ else False @@ -375,7 +375,7 @@ def invoke(self, rdr, ch): if isinstance(meta, Keyword): meta = rt.hashmap(meta, true) - if rt.instance_QMARK_(rt.IMeta.deref(), obj): + if rt.satisfies_QMARK_(rt.IMeta.deref(), obj): return rt.with_meta(obj, rt.merge(meta, rt.meta(obj))) return obj diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index a58b8583..97ccaebf 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo +from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ resize_list, list_copy, returns, get_var_if_defined import pixie.vm.code as code @@ -101,7 +101,7 @@ def seq(x): @as_var("seq?") def seq_QMARK_(x): - return true if rt.instance_QMARK_(rt.ISeq.deref(), x) else false + return true if rt.satisfies_QMARK_(rt.ISeq.deref(), x) else false @as_var("type") @@ -221,7 +221,7 @@ def __with_meta(a, b): @returns(bool) @as_var("has-meta?") def __has_meta(a): - return true if rt.instance_QMARK_(rt.IMeta.deref(), a) else false + return true if rt.satisfies_QMARK_(rt.IMeta.deref(), a) else false @as_var("conj") def conj(a, b): @@ -244,8 +244,8 @@ def str__args(args): @jit.unroll_safe def apply__args(args): last_itm = args[len(args) - 1] - if not rt.instance_QMARK_(rt.IIndexed.deref(), last_itm) or \ - not rt.instance_QMARK_(rt.ICounted.deref(), last_itm): + if not rt.satisfies_QMARK_(rt.IIndexed.deref(), last_itm) or \ + not rt.satisfies_QMARK_(rt.ICounted.deref(), last_itm): raise ValueError("Last item to apply must be bost IIndexed and ICounted") fn = args[0] @@ -267,7 +267,14 @@ def apply__args(args): @returns(bool) @as_var("instance?") -def _instance(proto, o): +def _instance(c, o): + affirm(isinstance(c, Type), u"c must be a type") + + return true if istypeinstance(o, c) else false + +@returns(bool) +@as_var("satisfies?") +def _satisfies(proto, o): affirm(isinstance(proto, Protocol), u"proto must be a Protocol") return true if proto.satisfies(o.type()) else false @@ -373,7 +380,7 @@ def identical(a, b): @as_var("vector?") def vector_QMARK_(a): - return true if rt.instance_QMARK_(rt.IVector.deref(), a) else false + return true if rt.satisfies_QMARK_(rt.IVector.deref(), a) else false @returns(bool) @as_var("eq") From 6d35cbab58695bbb4f2fe3efef698f7d28fbf652 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 02:08:48 +0100 Subject: [PATCH 088/909] add common type predicates --- pixie/stdlib.lisp | 9 +++++++++ pixie/vm/numbers.py | 11 ++++++----- 2 files changed, 15 insertions(+), 5 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e0fd99ad..8374c6b6 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -353,6 +353,15 @@ ([obj sym & args] (apply (get-field obj sym) args))) +(defn number? [v] (instance? Number)) +(defn string? [v] (instance? String v)) +(defn keyword? [v] (instance? Keyword v)) + +(defn list? [v] (instance? PersistentList v)) +(defn map? [v] (satisfies? IMap v)) + +(defn indexed? [v] (satisfies? IIndexed v)) +(defn counted? [v] (satisfies? ICounted v)) (extend -count MapEntry (fn [self] 2)) (extend -nth MapEntry (fn [self idx not-found] diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index c3d0b9d3..073f5096 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -6,12 +6,13 @@ import pixie.vm.rt as rt class Number(object.Object): - pass - + _type = object.Type(u"pixie.stdlib.Number") + def type(self): + return Number._type class Integer(Number): - _type = object.Type(u"pixie.stdlib.Integer") + _type = object.Type(u"pixie.stdlib.Integer", Number._type) _immutable_fields_ = ["_int_val"] def __init__(self, i_val): @@ -33,7 +34,7 @@ def type(self): one_int = Integer(1) class Float(Number): - _type = object.Type(u"pixie.stdlib.Float") + _type = object.Type(u"pixie.stdlib.Float", Number._type) _immutable_fields_ = ["_float_val"] def __init__(self, f_val): @@ -46,7 +47,7 @@ def type(self): return Float._type class Ratio(Number): - _type = object.Type(u"pixie.stdlib.Ratio") + _type = object.Type(u"pixie.stdlib.Ratio", Number._type) _immutable_fields_ = ["_numerator", "_denominator"] def __init__(self, numerator, denominator): From 4fe5f231354aa54b65f14217a9efc250fd2d40cd Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 02:21:16 +0100 Subject: [PATCH 089/909] add missing argument --- pixie/stdlib.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 8374c6b6..72373033 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -353,7 +353,7 @@ ([obj sym & args] (apply (get-field obj sym) args))) -(defn number? [v] (instance? Number)) +(defn number? [v] (instance? Number v)) (defn string? [v] (instance? String v)) (defn keyword? [v] (instance? Keyword v)) From 77f4b795914e7aed48bfe8376ad2217a4b683a39 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 02:21:47 +0100 Subject: [PATCH 090/909] restrict quot and rem to integers for now rpython won't compile '%' on anything involving floats, so we'll have to divide and then floor. ratios are still remaining. --- pixie/vm/numbers.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 073f5096..5f8e1056 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -92,12 +92,15 @@ def extend_num_op(pfn, ty1, ty2, conv1, op, conv2, wrap_start = "rt.wrap(", wrap wrap_start=wrap_start, wrap_end=wrap_end) exec tp +extend_num_op("_quot", Integer, Integer, "int_val", "/", "int_val") +extend_num_op("_rem", Integer, Integer, "int_val", "%", "int_val") + def define_num_ops(): # maybe define get_val() instead of using tuples? num_classes = [(Integer, "int_val"), (Float, "float_val")] for (c1, conv1) in num_classes: for (c2, conv2) in num_classes: - for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/"), ("_quot", "/"), ("_rem", "%")]: + for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/")]: if op == "_div" and c1 == Integer and c2 == Integer: continue extend_num_op(op, c1, c2, conv1, sym, conv2) From 0089c1515d11caa2a5a54a36310ac16339f71080 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 02:22:57 +0100 Subject: [PATCH 091/909] fix instance check compilation we want the type objects to be equal, otherwise we'd have to implement eq/ne on types. --- pixie/vm/object.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 461edc6e..c7335400 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -70,12 +70,12 @@ def type(self): Type._type = Type(u"Type") def istypeinstance(obj, t): - if obj._type == t: + if obj._type is t: return True - elif obj._type._parent != None: + elif obj._type._parent is not None: obj_type = obj._type._parent - while obj_type != None: - if obj_type == t: + while obj_type is not None: + if obj_type is t: return True obj_type = obj_type._parent return False From ca6aa86296f872776c472b0504db0d1f787a84d7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 02:26:04 +0100 Subject: [PATCH 092/909] move number test to the new tests directory --- pixie/test-numbers.lisp | 38 -------------------------------------- tests/test-numbers.lisp | 38 +++++++++++++++++++++++++++++++++++++- 2 files changed, 37 insertions(+), 39 deletions(-) delete mode 100644 pixie/test-numbers.lisp diff --git a/pixie/test-numbers.lisp b/pixie/test-numbers.lisp deleted file mode 100644 index b634e8c6..00000000 --- a/pixie/test-numbers.lisp +++ /dev/null @@ -1,38 +0,0 @@ -(ns pixie.test-numbers - (require pixie.test :as t)) - -(t/deftest integer-literals - (t/assert= 0xa 10) - (t/assert= -0xa -10) - (t/assert= 012 10) - (t/assert= -012 -10) - (t/assert= 2r1010 10) - (t/assert= -2r1010 -10)) - -(t/deftest float-literals - (t/assert= 10. 10.0) - (t/assert= -10. -10.0) - (t/assert= 1e1 10.0) - (t/assert= -1e1 -10.0) - (t/assert= 1e-1 0.1) - (t/assert= -1e-1 -0.1)) - -(t/deftest mixed-float-ops - (t/assert= (+ 1/2 0.5) 1.0) - (t/assert= (+ 0 1.0) 1.0)) - -(t/deftest ratio-literals - (t/assert= 3/4 (/ 3 4)) - (t/assert= -3/4 (/ -3 4)) - (t/assert= 6/8 3/4) - (t/assert= 9/12 3/4) - (t/assert= 3/1 3)) - -(t/deftest ratio-from-divide - (t/assert= (/ 3 4) 3/4)) - -(t/deftest ratio-ops - (t/assert= (+ 1/2 1/2) 1) - (t/assert= (- 1/2 1/2) 0) - (t/assert= (* 1/2 1/2) 1/4) - (t/assert= (/ 1/2 1/2) 1)) diff --git a/tests/test-numbers.lisp b/tests/test-numbers.lisp index 8755ab4b..b634e8c6 100644 --- a/tests/test-numbers.lisp +++ b/tests/test-numbers.lisp @@ -1,2 +1,38 @@ -(ns pixie.tests.test-numbers +(ns pixie.test-numbers (require pixie.test :as t)) + +(t/deftest integer-literals + (t/assert= 0xa 10) + (t/assert= -0xa -10) + (t/assert= 012 10) + (t/assert= -012 -10) + (t/assert= 2r1010 10) + (t/assert= -2r1010 -10)) + +(t/deftest float-literals + (t/assert= 10. 10.0) + (t/assert= -10. -10.0) + (t/assert= 1e1 10.0) + (t/assert= -1e1 -10.0) + (t/assert= 1e-1 0.1) + (t/assert= -1e-1 -0.1)) + +(t/deftest mixed-float-ops + (t/assert= (+ 1/2 0.5) 1.0) + (t/assert= (+ 0 1.0) 1.0)) + +(t/deftest ratio-literals + (t/assert= 3/4 (/ 3 4)) + (t/assert= -3/4 (/ -3 4)) + (t/assert= 6/8 3/4) + (t/assert= 9/12 3/4) + (t/assert= 3/1 3)) + +(t/deftest ratio-from-divide + (t/assert= (/ 3 4) 3/4)) + +(t/deftest ratio-ops + (t/assert= (+ 1/2 1/2) 1) + (t/assert= (- 1/2 1/2) 0) + (t/assert= (* 1/2 1/2) 1/4) + (t/assert= (/ 1/2 1/2) 1)) From c67296c4763aece0f5a110f14cb39a952089c874 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 1 Nov 2014 11:31:52 +0100 Subject: [PATCH 093/909] allow keywords as functions on maps, retrieving the value for the keyword from it. --- pixie/stdlib.lisp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 72373033..f34316f7 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -399,6 +399,10 @@ (str ":" (namespace k) "/" (name k)) (str ":" (name k))))) +(extend -invoke Keyword + (fn [k m] + (-val-at m k nil))) + (defn get ([mp k] (get mp k nil)) From a65d58ac3dc9215f4884ccbc9900af43a6ab5c4e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 02:50:56 +0100 Subject: [PATCH 094/909] support quot and rem on floats --- pixie/vm/numbers.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 5f8e1056..1973b3ab 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -5,6 +5,8 @@ from pixie.vm.code import DoublePolymorphicFn, extend, Protocol, as_var, wrap_fn import pixie.vm.rt as rt +import math + class Number(object.Object): _type = object.Type(u"pixie.stdlib.Number") @@ -104,6 +106,8 @@ def define_num_ops(): if op == "_div" and c1 == Integer and c2 == Integer: continue extend_num_op(op, c1, c2, conv1, sym, conv2) + extend_num_op("_quot", c1, c2, conv1, "/", conv2, wrap_start = "rt.wrap(math.floor(", wrap_end = "))") + extend_num_op("_rem", c1, c2, conv1, ",", conv2, wrap_start = "rt.wrap(math.fmod(", wrap_end = "))") for (op, sym) in [("_num_eq", "=="), ("_lt", "<"), ("_gt", ">"), ("_lte", "<="), ("_gte", ">=")]: extend_num_op(op, c1, c2, conv1, sym, conv2, wrap_start = "true if ", wrap_end = " else false") From ec223fc7b66cfb2316e3dcd076c84ab1a7f0d823 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 12:04:46 +0100 Subject: [PATCH 095/909] extend quot and rem to all number types --- pixie/vm/numbers.py | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 1973b3ab..6404da0b 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -169,6 +169,17 @@ def _div(a, b): return rt._div(rt.wrap(b.denominator() * a.numerator()), rt.wrap(b.numerator() * a.denominator())) +@extend(_quot, Ratio._type, Ratio._type) +def _quot(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt.wrap((a.numerator() * b.denominator()) / (a.denominator() * b.numerator())) + +@extend(_rem, Ratio._type, Ratio._type) +def _rem(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + q = rt.wrap((a.numerator() * b.denominator()) / (a.denominator() * b.numerator())) + return rt._sub(a, rt._mul(q, b)) + @extend(_num_eq, Ratio._type, Ratio._type) def _num_eq(a, b): assert isinstance(a, Ratio) and isinstance(b, Ratio) @@ -206,12 +217,12 @@ def to_float_conv(c): def define_ratio_ops(): for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: - for op in ["_add", "_sub", "_mul", "_div", "_num_eq"]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_num_eq"]: code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) exec code for (c1, c2) in [(Float, Ratio), (Ratio, Float)]: - for op in ["_add", "_sub", "_mul", "_div", "_num_eq"]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_num_eq"]: code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) exec code From 3e990f1f4596bc5b3ae8cbcba2b5723f55f12abd Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 12:21:23 +0100 Subject: [PATCH 096/909] implement comparisons involving ratios --- pixie/vm/numbers.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 6404da0b..e0743eb2 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -180,6 +180,26 @@ def _rem(a, b): q = rt.wrap((a.numerator() * b.denominator()) / (a.denominator() * b.numerator())) return rt._sub(a, rt._mul(q, b)) +@extend(_lt, Ratio._type, Ratio._type) +def _lt(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if a.numerator() * b.denominator() < b.numerator() * a.denominator() else false + +@extend(_gt, Ratio._type, Ratio._type) +def _gt(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._lt(b, a) + +@extend(_lte, Ratio._type, Ratio._type) +def _lte(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if rt._lt(b, a) is false else false + +@extend(_gte, Ratio._type, Ratio._type) +def gte(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if rt._lt(a, b) is false else false + @extend(_num_eq, Ratio._type, Ratio._type) def _num_eq(a, b): assert isinstance(a, Ratio) and isinstance(b, Ratio) @@ -217,12 +237,12 @@ def to_float_conv(c): def define_ratio_ops(): for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: - for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_num_eq"]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) exec code for (c1, c2) in [(Float, Ratio), (Ratio, Float)]: - for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_num_eq"]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) exec code From de256c1695a0038a514869072ad506550a5a8e22 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 13:29:10 +0100 Subject: [PATCH 097/909] change -str implementations to be closer to clojure's that is, don't print commas except between map entries. --- pixie/stdlib.lisp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index f34316f7..7ecec378 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -140,14 +140,11 @@ (extend -str PersistentVector (fn [v] - (apply str "[" (conj (transduce (interpose ", ") conj v) "]")))) - - - + (apply str "[" (conj (transduce (interpose " ") conj v) "]")))) (extend -str Cons (fn [v] - (apply str "(" (conj (transduce (interpose ", ") conj v) ")")))) + (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) (extend -hash Cons (fn [v] @@ -155,11 +152,11 @@ (extend -str PersistentList (fn [v] - (apply str "(" (conj (transduce (interpose ", ") conj v) ")")))) + (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) (extend -str LazySeq (fn [v] - (apply str "(" (conj (transduce (interpose ", ") conj v) ")")))) + (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) (extend -hash PersistentVector (fn [v] @@ -379,7 +376,7 @@ (extend -str MapEntry (fn [v] - (apply str "[" (conj (transduce (interpose ", ") conj v) "]")))) + (apply str "[" (conj (transduce (interpose " ") conj v) "]")))) (extend -hash MapEntry (fn [v] @@ -387,7 +384,8 @@ (extend -str PersistentHashMap (fn [v] - (apply str "{" (conj (transduce (comp cat (interpose " ")) conj v) "}")))) + (let [entry->str (map (fn [e] (vector (key e) " " (val e))))] + (apply str "{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) (extend -hash PersistentHashMap (fn [v] From 62773438ace48eb0791f198d4631adb2cd358600 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 13:30:01 +0100 Subject: [PATCH 098/909] add true? and false? predicates --- pixie/stdlib.lisp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 7ecec378..eb3d962c 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -350,6 +350,9 @@ ([obj sym & args] (apply (get-field obj sym) args))) +(defn true? [v] (identical? v true)) +(defn false? [v] (identical? v false)) + (defn number? [v] (instance? Number v)) (defn string? [v] (instance? String v)) (defn keyword? [v] (instance? Keyword v)) From 20557723503646c582e55e739a0021f2e7b3957d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 14:35:52 +0100 Subject: [PATCH 099/909] fix the namespace of the numbers tests --- tests/test-numbers.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test-numbers.lisp b/tests/test-numbers.lisp index b634e8c6..ae93d7c1 100644 --- a/tests/test-numbers.lisp +++ b/tests/test-numbers.lisp @@ -1,4 +1,4 @@ -(ns pixie.test-numbers +(ns pixie.tests.test-numbers (require pixie.test :as t)) (t/deftest integer-literals From 351188319d0405896bb64a53b70cb5694afcbaa2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 16:20:46 +0100 Subject: [PATCH 100/909] make first and next work on ISeqable's --- pixie/vm/stdlib.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 97ccaebf..c9378898 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -89,11 +89,23 @@ def default_str(x): @as_var("first") def first(x): - return rt._first(x) + if rt.satisfies_QMARK_(ISeq, x): + return rt._first(x) + + seq = rt.seq(x) + if seq is nil: + return nil + return rt._first(seq) @as_var("next") def next(x): - return rt.seq(rt._next(x)) + if rt.satisfies_QMARK_(ISeq, x): + return rt._next(x) + seq = rt.seq(x) + if seq is nil: + return nil + else: + return rt.seq(rt._next(seq)) @as_var("seq") def seq(x): From 7d0e94a181ea0c490f2af0865e7c4a4730104743 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 2 Nov 2014 08:38:40 -0700 Subject: [PATCH 101/909] fix merge conflict --- pixie/vm/reader.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index a39701d3..b0a979b9 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -381,7 +381,6 @@ def invoke(self, rdr, ch): return obj -<<<<<<< HEAD class SetReader(ReaderHandler): def invoke(self, rdr, ch): acc = EMPTY_SET @@ -405,7 +404,7 @@ def invoke(self, rdr, ch): if handler is None: raise Exception("unknown dispatch #" + ch) return handler.invoke(rdr, ch) - + class LineCommentReader(ReaderHandler): def invoke(self, rdr, ch): self.skip_line(rdr) From ba3befbfca378adfcdf21114e809b0d7c817618d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 16:42:20 +0100 Subject: [PATCH 102/909] add second, third and fourth --- pixie/stdlib.lisp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index eb3d962c..597e69ae 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -279,6 +279,14 @@ (def dec (fn [x] (- x 1))) +(defn second [x] + (first (next x))) + +(defn third [x] + (first (next (next x)))) + +(defn fourth [x] + (first (next (next (next x))))) (defn assoc ([m] m) From 53c562da1945ca454fe25dbf830f971243f65e5f Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 09:37:55 +0100 Subject: [PATCH 103/909] allow running as a shebang executable we strip the shebang from *all* loaded files if it is present, but i think it should be ok. --- pixie/vm/stdlib.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 97ccaebf..24ed7a64 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -319,6 +319,10 @@ def load_file(filename): data = f.read() f.close() + if data.startswith("#!"): + newline_pos = data.find("\n") + if newline_pos > 0: + data = data[newline_pos:] rdr = reader.StringReader(unicode(data)) with compiler.with_ns(u"user"): From 037124aa35fc0229caa90321e57c485e37c6d494 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 3 Nov 2014 06:24:29 -0700 Subject: [PATCH 104/909] protocols can now be used in extend (instead of just types) --- pixie/stdlib.lisp | 13 +++++++++++-- pixie/vm/code.py | 35 ++++++++++++++++++++++++++++++++++- pixie/vm/object.py | 13 ++++++++++++- pixie/vm/stdlib.py | 2 +- 4 files changed, 58 insertions(+), 5 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 9784773d..13db5bd8 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -650,6 +650,7 @@ + (defn range ([] (->Range 0 MAX-NUMBER 1)) ([stop] (->Range 0 stop 1)) @@ -664,9 +665,17 @@ i) (defn at-end? [i] - (-at-end? i) - i) + (-at-end? i)) + +(defn current [i] + (-current i)) + +(defn iterator-seq [i] + (if (at-end? i) + nil + (cons (current i) (lazy-seq (iterator-seq (move-next! i)))))) +(extend -seq IIterator iterator-seq) (extend -reduce ShallowContinuation (fn [k f init] diff --git a/pixie/vm/code.py b/pixie/vm/code.py index a1b818b1..861f44cb 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -576,20 +576,53 @@ def __init__(self, name, protocol): self._rev = 0 self._protocol = protocol self._default_fn = DefaultProtocolFn(self) + self._fn_cache = {} protocol.add_method(self) def extend(self, tp, fn): self._dict[tp] = fn self._rev += 1 + self._fn_cache = {} self._protocol.add_satisfies(tp) + def _find_parent_fn(self, tp): + ## Search the entire object tree to find the function to execute + assert isinstance(tp, object.Type) + + protos = [] + for p, fn in self._dict.iteritems(): + if isinstance(p, Protocol): + protos.append(p) + + find_tp = tp + while True: + result = self._dict.get(find_tp, None) + if result is not None: + return result + + for proto in protos: + if proto.satisfies(find_tp): + return self._dict[proto] + + find_tp = find_tp._parent + if find_tp is None: + break + + return self._default_fn + + def set_default_fn(self, fn): self._default_fn = fn self._rev += 1 + self._fn_cache = {} @elidable_promote() def get_protocol_fn(self, tp, rev): - fn = self._dict.get(tp, self._default_fn) + fn = self._fn_cache.get(tp, None) + if fn is None: + fn = self._find_parent_fn(tp) + self._fn_cache[tp] = fn + return promote(fn) def invoke(self, args): diff --git a/pixie/vm/object.py b/pixie/vm/object.py index c7335400..80076bcc 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -59,16 +59,27 @@ def get_by_name(self, nm, default=None): class Type(Object): def __init__(self, name, parent = None): - #assert isinstance(name, unicode), u"Type names must be unicode" + assert isinstance(name, unicode), u"Type names must be unicode" _type_registry.register_type(name, self) self._name = name self._parent = parent + if parent is not None: + parent.add_subclass(self) + + self._subclasses = [] def type(self): return Type._type + def add_subclass(self, tp): + self._subclasses.append(tp) + + def subclasses(self): + return self._subclasses + Type._type = Type(u"Type") +@jit.elidable_promote() def istypeinstance(obj, t): if obj._type is t: return True diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index b93949d6..3fdefae6 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -366,7 +366,7 @@ def refer(ns, refer, alias): @as_var("extend") def extend(proto_fn, tp, fn): affirm(isinstance(proto_fn, PolymorphicFn), u"First argument to extend should be a PolymorphicFn") - affirm(isinstance(tp, Type), u"Second argument to extend must be a Type") + affirm(isinstance(tp, (Type, Protocol)), u"Second argument to extend must be a Type or Protocol") affirm(isinstance(fn, BaseCode), u"Last argument to extend must be a function") proto_fn.extend(tp, fn) return nil From bc5a6f055d59333cfce321f42496a6876a5817de Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 3 Nov 2014 06:40:45 -0700 Subject: [PATCH 105/909] extend seq to IIterator --- pixie/vm/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 3fdefae6..65295dd7 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -366,7 +366,7 @@ def refer(ns, refer, alias): @as_var("extend") def extend(proto_fn, tp, fn): affirm(isinstance(proto_fn, PolymorphicFn), u"First argument to extend should be a PolymorphicFn") - affirm(isinstance(tp, (Type, Protocol)), u"Second argument to extend must be a Type or Protocol") + affirm(isinstance(tp, Type) or isinstance(tp, Protocol), u"Second argument to extend must be a Type or Protocol") affirm(isinstance(fn, BaseCode), u"Last argument to extend must be a function") proto_fn.extend(tp, fn) return nil From 8e30a676173288db7d7ed2159729791d9999218a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 3 Nov 2014 21:28:53 -0700 Subject: [PATCH 106/909] reworked stacklets so that they satisfy IIterator --- pixie/stdlib.lisp | 15 ++++++------- pixie/vm/stacklet.py | 51 ++++++++++++++++++++++++++++++++++++++------ 2 files changed, 51 insertions(+), 15 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 13db5bd8..11286f81 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -164,19 +164,18 @@ (def stacklet->lazy-seq - (fn [f] - (let [val (f nil)] - (if (identical? val :end) - nil - (cons val (lazy-seq* (fn [] (stacklet->lazy-seq f)))))))) + (fn [k] + (if (-at-end? k) + nil + (cons (-current k) + (lazy-seq* (fn [] (stacklet->lazy-seq (-move-next! k)))))))) (def sequence (fn ([data] (let [f (create-stacklet (fn [h] - (reduce (fn ([h item] (h item) h)) h data) - (h :end)))] + (reduce (fn ([h item] (h item) h)) h data)))] (stacklet->lazy-seq f))) ([xform data] (let [f (create-stacklet @@ -184,7 +183,7 @@ (transduce xform (fn ([] h) ([h item] (h item) h) - ([h] (h :end))) + ([h] nil)) data)))] (stacklet->lazy-seq f))))) diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index 5cf85cdd..cea6db0d 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -49,22 +49,45 @@ def shutdown(): global_state.reset() +class FinishedToken(BaseCode): + _type = object.Type(u"pixie.stdlib.FinishedToken") + def __init__(self): + pass + + def type(self): + return FinishedToken._type + +finished_token = FinishedToken() + class WrappedHandler(BaseCode): _type = object.Type(u"Stacklet") def __init__(self, h): self._h = h + self._is_finished = False + self._val = nil + def type(self): return WrappedHandler._type + def is_finished(self): + return self._is_finished + def invoke(self, args): affirm(len(args) == 1, u"Only one arg to continuation allowed") + affirm(not self._is_finished, u"Execution of this stacklet has completed") + global_state._from = global_state._to global_state._to = self global_state._op = OP_SWITCH global_state._val = args[0] global_state._h = global_state._th.switch(global_state._h) + if global_state._val is finished_token: + global_state._from._is_finished = True + + global_state._from._val = global_state._val + return global_state._val @@ -85,17 +108,14 @@ def new_handler(h, o): global_state._val = None - global_state._op = OP_SWITCH global_state.switch_back() global_state._h = global_state._th.switch(h) + #try: f.invoke([global_state._from]) - #except Exception as ex: - # print "Uncaught Exception" + str(ex) - - + global_state._from.invoke([finished_token]) return global_state._h @@ -149,5 +169,22 @@ def with_stacklets(f): @as_var("create-stacklet") def _new_stacklet(f): - return new_stacklet(f) - + stacklet = new_stacklet(f) + stacklet.invoke([nil]) # prime it + return stacklet + +@extend(proto._at_end_QMARK_, WrappedHandler) +def _at_end(self): + assert isinstance(self, WrappedHandler) + return rt.wrap(self.is_finished()) + +@extend(proto._move_next_BANG_, WrappedHandler) +def _move_next(self): + assert isinstance(self, WrappedHandler) + self.invoke([nil]) + return self + +@extend(proto._current, WrappedHandler) +def _current(self): + assert isinstance(self, WrappedHandler) + return self._val From a237a0ad0bd14a447cf3e4d5f6516d0e0a5cc162 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 4 Nov 2014 05:55:14 -0700 Subject: [PATCH 107/909] fix for issue #36 --- pixie/vm/persistent_hash_map.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 6edddf7f..162e6b06 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -36,7 +36,7 @@ def assoc(self, key, val): added_leaf = Box() new_root = (BitmapIndexedNode_EMPTY if self._root is None else self._root) \ - .assoc_inode(r_uint(0), rt.hash(key), key, val, added_leaf) + .assoc_inode(r_uint(0), rt.hash(key) & MASK_32, key, val, added_leaf) if new_root is self._root: return self @@ -44,7 +44,7 @@ def assoc(self, key, val): return PersistentHashMap(self._cnt if added_leaf._val is None else self._cnt + 1, new_root, self._meta) def val_at(self, key, not_found): - return not_found if self._root is None else self._root.find(r_uint(0), rt.hash(key), key, not_found) + return not_found if self._root is None else self._root.find(r_uint(0), rt.hash(key) & MASK_32, key, not_found) @@ -124,7 +124,7 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): else: nodes[i] = BitmapIndexedNode_EMPTY.assoc_inode(shift + 5, rt.hash(self._array[j]), self._array[j], self._array[j + 1], added_leaf) - j += 1 + j += 2 return ArrayNode(None, n + 1, nodes) else: From 1a058b7d653817d292c92b2305f76ae6e56fc5bc Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 17:05:01 +0100 Subject: [PATCH 108/909] support = on strings --- pixie/vm/string.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index 13c050fc..ab4ba487 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -1,7 +1,7 @@ import pixie.vm.rt as rt from pixie.vm.object import Object, Type from pixie.vm.code import extend, as_var -from pixie.vm.primitives import nil +from pixie.vm.primitives import nil, true, false import pixie.vm.stdlib as proto import pixie.vm.numbers as numbers import pixie.vm.util as util @@ -38,6 +38,11 @@ def _nth(self, idx): return Character(ord(self._str[i])) raise IndexError() +@extend(proto._eq, String) +def _eq(self, v): + if not isinstance(v, String): + return false + return true if self._str == v._str else false class Character(Object): _type = Type(u"pixie.stdlib.Character") @@ -82,4 +87,4 @@ def _namespace(self): @extend(proto._hash, String) def _hash(self): - return rt.wrap(intmask(util.hash_unencoded_chars(self._str))) \ No newline at end of file + return rt.wrap(intmask(util.hash_unencoded_chars(self._str))) From ac11aa35460ee5e6ed2ae684a0157dd7e7c7595b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 18:04:12 +0100 Subject: [PATCH 109/909] add not= --- pixie/stdlib.lisp | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 11286f81..6af9f0f6 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -245,6 +245,11 @@ (apply = y rest) false))) +(defn not= + ([x] false) + ([x y] (not (eq x y))) + ([x y & rest] (not (apply = x y rest)))) + (defn < ([x] true) ([x y] (-lt x y)) From cbf1b880ffa50481982f58a52e6cb62dc54f16e7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 18:22:30 +0100 Subject: [PATCH 110/909] define not earlier to be able to use it in not=. the definition order is a problem in general, how does clojure handle that? (constraint solving? :) --- pixie/stdlib.lisp | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 6af9f0f6..2b438a41 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -205,6 +205,9 @@ (set-macro! defmacro) +(defn not [x] + (if x false true)) + (defn + ([] 0) ([x] x) @@ -317,10 +320,6 @@ (fn [& args] (f1 (f2 (apply f3 args)))))) - -(defn not [x] - (if x false true)) - (defmacro cond ([] nil) ([test then & clauses] From 4dfe7352be3889e7d094a346ababe3a20e7bbc64 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 18:51:17 +0100 Subject: [PATCH 111/909] add contains? for vectors and maps (and to sets once they're merged). this adds a new protocol method -contains-key to IAssociative. --- pixie/stdlib.lisp | 3 +++ pixie/vm/persistent_hash_map.py | 10 ++++++++++ pixie/vm/persistent_vector.py | 7 +++++++ pixie/vm/stdlib.py | 2 +- tests/collections/test-maps.lisp | 11 +++++++++++ tests/collections/test-vectors.lisp | 11 +++++++++-- 6 files changed, 41 insertions(+), 3 deletions(-) create mode 100644 tests/collections/test-maps.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 2b438a41..922784cf 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -302,6 +302,9 @@ ([m k v & rest] (apply assoc (-assoc m k v) rest))) +(defn contains? [coll key] + (-contains-key coll key)) + (def slot-tp (create-type :slot [:val])) (defn ->Slot [x] diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 162e6b06..e1a6aebc 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -11,6 +11,8 @@ MASK_32 = r_uint(0xFFFFFFFF) +NOT_FOUND = object.Object() + class Box(py_object): def __init__(self): self._val = None @@ -349,3 +351,11 @@ def _meta(self): def _with_meta(self, meta): assert isinstance(self, PersistentHashMap) return self.with_meta(meta) + +@extend(proto._contains_key, PersistentHashMap) +def _contains_key(self, key): + assert isinstance(self, PersistentHashMap) + if self._root is not None: + return true if self._root.find(r_uint(0), rt.hash(key), key, NOT_FOUND) is not NOT_FOUND else false + else: + return false diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 3ccae55f..388056b2 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -380,6 +380,13 @@ def _val_at(self, key, not_found): else: return not_found +@extend(proto._contains_key, PersistentVector) +def _contains_key(self, key): + if not isinstance(key, Integer): + return false + else: + return true if key.int_val() >= 0 and key.int_val() < intmask(self._cnt) else false + @extend(proto._conj, PersistentVector) def _conj(self, v): assert isinstance(self, PersistentVector) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 65295dd7..f9d708a3 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -33,7 +33,7 @@ defprotocol("pixie.stdlib", "INamed", ["-namespace", "-name"]) -defprotocol("pixie.stdlib", "IAssociative", ["-assoc"]) +defprotocol("pixie.stdlib", "IAssociative", ["-assoc", "-contains-key"]) defprotocol("pixie.stdlib", "ILookup", ["-val-at"]) diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp new file mode 100644 index 00000000..29d8ecd6 --- /dev/null +++ b/tests/collections/test-maps.lisp @@ -0,0 +1,11 @@ +(ns collections.test-maps + (require pixie.test :as t)) + +(t/deftest maps-contains + (let [m {:a 1, :b 2, :c 3} + c [:a :b :c] + n [:d 'a 1]] + (foreach [c c] + (t/assert= (contains? m c) true)) + (foreach [n n] + (t/assert= (contains? m c) false)))) diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.lisp index a89fba1f..69673284 100644 --- a/tests/collections/test-vectors.lisp +++ b/tests/collections/test-vectors.lisp @@ -1,8 +1,6 @@ (ns collections.test-vectors (require pixie.test :as t)) - - (def MAX-SIZE 2000) (t/deftest vector-creation @@ -12,3 +10,12 @@ (do (dotimes [j (count acc)] (t/assert= j (nth acc j))) (recur (conj acc (count acc))))))) + +(t/deftest vector-contains + (let [v [1 2 3] + c [0 1 2] + n [-1 3]] + (foreach [c c] + (t/assert= (contains? v c) true)) + (foreach [n n] + (t/assert= (contains? v n) false)))) From a83f446ef113148cf5de17324e88a1439d543b58 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 19:02:24 +0100 Subject: [PATCH 112/909] implement -eq for maps --- pixie/vm/persistent_hash_map.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index e1a6aebc..44742d2c 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -352,6 +352,25 @@ def _with_meta(self, meta): assert isinstance(self, PersistentHashMap) return self.with_meta(meta) +@extend(proto._eq, PersistentHashMap) +def _eq(self, obj): + if self is obj: + return true + elif not isinstance(obj, PersistentHashMap): + return false + elif self._cnt != obj._cnt: + return false + + seq = rt.seq(self) + while seq is not nil: + entry = rt.first(seq) + found = _contains_key(entry._key) + if not found or rt.eq(entry._value, rt.get(obj, entry._key)) is not true: + return false + seq = rt.next(seq) + + return True + @extend(proto._contains_key, PersistentHashMap) def _contains_key(self, key): assert isinstance(self, PersistentHashMap) From 4217c1a206d81fcbe5e1364f04448731ac485001 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 19:06:08 +0100 Subject: [PATCH 113/909] add some simple tests for map equality --- tests/collections/test-maps.lisp | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp index 29d8ecd6..3bd376b3 100644 --- a/tests/collections/test-maps.lisp +++ b/tests/collections/test-maps.lisp @@ -9,3 +9,13 @@ (t/assert= (contains? m c) true)) (foreach [n n] (t/assert= (contains? m c) false)))) + +(t/deftest map-equals + (let [m {:a 1, :b 2, :c 3}] + (t/assert= (-eq {} {}) true) + (t/assert= (-eq m m) true) + (t/assert= (-eq m {:a 1, :b 2, :c 3}) true) + + (t/assert= (-eq m {}) false) + (t/assert= (-eq m {:a 1, :b 2} false)) + (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false))) From 572353a870e4cfa989d35ebbdfd66bab5cae7232 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 19:45:50 +0100 Subject: [PATCH 114/909] try fixing -eq on maps --- pixie/vm/persistent_hash_map.py | 6 +++--- tests/collections/test-maps.lisp | 5 ++++- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 44742d2c..a5a2e59f 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -364,12 +364,12 @@ def _eq(self, obj): seq = rt.seq(self) while seq is not nil: entry = rt.first(seq) - found = _contains_key(entry._key) - if not found or rt.eq(entry._value, rt.get(obj, entry._key)) is not true: + found = _contains_key(obj, entry._key) + if not found or not rt.eq(entry._value, rt.get(obj, entry._key)): return false seq = rt.next(seq) - return True + return true @extend(proto._contains_key, PersistentHashMap) def _contains_key(self, key): diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp index 3bd376b3..e2caff8e 100644 --- a/tests/collections/test-maps.lisp +++ b/tests/collections/test-maps.lisp @@ -18,4 +18,7 @@ (t/assert= (-eq m {}) false) (t/assert= (-eq m {:a 1, :b 2} false)) - (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false))) + (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false) + + (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) + (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) From c1f4f4804dfc59b8ea0b25c550488393e7e6e087 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 19:58:04 +0100 Subject: [PATCH 115/909] fix compilation --- pixie/vm/persistent_hash_map.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index a5a2e59f..ff4a2d92 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -364,8 +364,9 @@ def _eq(self, obj): seq = rt.seq(self) while seq is not nil: entry = rt.first(seq) - found = _contains_key(obj, entry._key) - if not found or not rt.eq(entry._value, rt.get(obj, entry._key)): + key = rt._key(entry) + found = rt._contains_key(obj, key) + if not found or not rt.eq(rt._val(entry), rt.get(obj, key)): return false seq = rt.next(seq) From 8926eb08a5159ef32c1ee32bef32a57784d748df Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 2 Nov 2014 22:16:44 +0100 Subject: [PATCH 116/909] implement -eq for vectors --- pixie/vm/persistent_vector.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 388056b2..5747f223 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -380,6 +380,29 @@ def _val_at(self, key, not_found): else: return not_found +@extend(proto._eq, PersistentVector) +def _eq(self, obj): + if self is obj: + return true + elif isinstance(obj, PersistentVector): + if self._cnt != obj._cnt: + return false + for i in range(0, intmask(self._cnt)): + if not rt.eq(self.nth(i), obj.nth(i)): + return false + return true + else: + if not rt.instance_QMARK_(proto.ISeqable, obj): + return false + seq = rt.seq(obj) + for i in range(0, intmask(self._cnt)): + if seq is nil or not rt.eq(self.nth(i), rt.first(seq)): + return false + seq = rt.next(seq) + if seq is not nil: + return false + return true + @extend(proto._contains_key, PersistentVector) def _contains_key(self, key): if not isinstance(key, Integer): From 7fb6d810c626caf07aed14007af0c9d32ae65f4b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 17:41:00 +0100 Subject: [PATCH 117/909] use satisfies? instead of the old instance? --- pixie/vm/persistent_vector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 5747f223..8dbb3381 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -392,7 +392,7 @@ def _eq(self, obj): return false return true else: - if not rt.instance_QMARK_(proto.ISeqable, obj): + if not rt.satisfies_QMARK_(proto.ISeqable, obj): return false seq = rt.seq(obj) for i in range(0, intmask(self._cnt)): From 311d3f351826b0430cdb266908801e10bfe03ca4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 18:02:23 +0100 Subject: [PATCH 118/909] fix set literals the NativeFn interface changed from _invoke to invoke, i think. --- pixie/vm/compiler.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7b022f96..fb00ef1d 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -299,7 +299,7 @@ def compile_map_literal(form, ctx): compile_meta(rt.meta(form), ctx) class ConsReduce(code.NativeFn): - def _invoke(self, args): + def invoke(self, args): return rt.cons(args[1], args[0]) def compile_set_literal(form, ctx): From e12b87f9f1b742357ad9f7b39cc7fdfe310eff61 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 18:08:16 +0100 Subject: [PATCH 119/909] implement -contains-key on sets --- pixie/vm/persistent_hash_set.py | 5 +++++ tests/collections/test-sets.lisp | 8 ++++++++ 2 files changed, 13 insertions(+) diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 2756f189..5efdf89a 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -38,6 +38,11 @@ def _count(self): assert isinstance(self, PersistentHashSet) return rt._count(self._map) +@extend(proto._contains_key, PersistentHashSet) +def _contains_key(self, key): + assert isinstance(self, PersistentHashSet) + return rt._contains_key(self._map, key) + @extend(proto._conj, PersistentHashSet) def _conj(self, v): assert isinstance(self, PersistentHashSet) diff --git a/tests/collections/test-sets.lisp b/tests/collections/test-sets.lisp index ff318c94..f571cc04 100644 --- a/tests/collections/test-sets.lisp +++ b/tests/collections/test-sets.lisp @@ -6,3 +6,11 @@ (t/assert= (count (set [1 2 3])) 3) (t/assert= (count (set [1 1 2 1])) 2)) +(t/deftest test-contains + (let [s #{1 2 3} + c [1 2 3] + n [-1 0 4]] + (foreach [c c] + (t/assert= (contains? s c) true)) + (foreach [n n] + (t/assert= (contains? s n) false)))) From 62b70c86eaf96ec4f7f5fd07366e320c430c4ced Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 22:18:00 +0100 Subject: [PATCH 120/909] make next always return nil on empty collections this wasn't the case when we had a seqable that didn't return nil by itself. this is a bit different from clojure because our collections don't always return nil themselves. we need to take a closer look at this, though. --- pixie/vm/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index f9d708a3..62ded28d 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -100,7 +100,7 @@ def first(x): @as_var("next") def next(x): if rt.satisfies_QMARK_(ISeq, x): - return rt._next(x) + return rt.seq(rt._next(x)) seq = rt.seq(x) if seq is nil: return nil From f8d96c515206110b8e4050a66f37d078bebc779e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 22:22:15 +0100 Subject: [PATCH 121/909] implement -eq on sets --- pixie/vm/persistent_hash_set.py | 17 +++++++++++++++++ tests/collections/test-sets.lisp | 11 +++++++++++ 2 files changed, 28 insertions(+) diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 5efdf89a..2863712e 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -43,6 +43,23 @@ def _contains_key(self, key): assert isinstance(self, PersistentHashSet) return rt._contains_key(self._map, key) +@extend(proto._eq, PersistentHashSet) +def _eq(self, obj): + assert isinstance(self, PersistentHashSet) + if self is obj: + return true + if not isinstance(obj, PersistentHashSet): + return false + if self._map._cnt != obj._map._cnt: + return false + + seq = rt.seq(obj) + while seq is not nil: + if rt._contains_key(self, rt.first(seq)) is false: + return false + seq = rt.next(seq) + return true + @extend(proto._conj, PersistentHashSet) def _conj(self, v): assert isinstance(self, PersistentHashSet) diff --git a/tests/collections/test-sets.lisp b/tests/collections/test-sets.lisp index f571cc04..c1fdb0a2 100644 --- a/tests/collections/test-sets.lisp +++ b/tests/collections/test-sets.lisp @@ -14,3 +14,14 @@ (t/assert= (contains? s c) true)) (foreach [n n] (t/assert= (contains? s n) false)))) + +(t/deftest test-eq + (let [s #{1 2 3}] + (t/assert= s s) + (t/assert= s #{1 2 3}) + (t/assert= #{1 2 3} s) + + (t/assert= (= s [1 2 3]) false) + (t/assert= (= s '(1 2 3)) false) + (t/assert= (= s #{1 2}) false) + (t/assert= (= s #{1 2 3 4}) false))) From d1d8ec39d855e7d2eb66f332cd1c9847ee983c45 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 23:09:37 +0100 Subject: [PATCH 122/909] extend -eq to ISeqables i'd rather not expose -seq-eq, but it seems that we can't extend protocols from rpython yet. --- pixie/stdlib.lisp | 2 ++ pixie/vm/stdlib.py | 16 +++++++++++++++ tests/collections/test-seqables.lisp | 30 ++++++++++++++++++++++++++++ 3 files changed, 48 insertions(+) create mode 100644 tests/collections/test-seqables.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 922784cf..6a7088e6 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -683,6 +683,8 @@ (extend -seq IIterator iterator-seq) +(extend -eq ISeqable -seq-eq) + (extend -reduce ShallowContinuation (fn [k f init] (loop [acc init] diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 62ded28d..0ab51685 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -115,6 +115,22 @@ def seq(x): def seq_QMARK_(x): return true if rt.satisfies_QMARK_(rt.ISeq.deref(), x) else false +@as_var("-seq-eq") +def _seq_eq(a, b): + print "seq eq" + if a is b: + return true + if not (rt.satisfies_QMARK_(rt.ISeqable.deref(), b) or rt.satisfies_QMARK_(rt.ISeq.deref(), b)): + return false + + a = rt.seq(a) + b = rt.seq(b) + while a is not nil: + if b is nil or not rt.eq(rt.first(a), rt.first(b)): + return false + a = rt.next(a) + b = rt.next(b) + return true if b is nil else false @as_var("type") def type(x): diff --git a/tests/collections/test-seqables.lisp b/tests/collections/test-seqables.lisp new file mode 100644 index 00000000..e8a554fe --- /dev/null +++ b/tests/collections/test-seqables.lisp @@ -0,0 +1,30 @@ +(ns collections.test-seqables + (require pixie.test :as t)) + +(t/deftest test-seq + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (seq l) '(1 2 3)) + (t/assert= (seq v) [1 2 3]) + + (t/assert= (seq nil) nil) + (t/assert= (seq []) nil))) + +(t/deftest test-first + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (first l) 1) + (t/assert= (first v) 1) + (t/assert= (first (seq l)) 1) + (t/assert= (first (seq v)) 1))) + +(t/deftest test-next + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (next l) '(2 3)) + (t/assert= (next v) '(2 3)) + (t/assert= (next (seq l)) '(2 3)) + (t/assert= (next (seq v)) '(2 3)) + + (t/assert= (next (next (next l))) nil) + (t/assert= (next (next (next v))) nil))) From aaacfcf37f69215246e68b0195f45dd345c8a88d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 23:21:05 +0100 Subject: [PATCH 123/909] implement -eq on chars --- pixie/vm/string.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index ab4ba487..7c88277b 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -76,6 +76,14 @@ def _repr(self): return rt.wrap(u"\\"+unicode(chr(cv))) return rt.wrap(u"\\u"+unicode(str(cv))) +@extend(proto._eq, Character) +def _eq(self, obj): + assert isinstance(self, Character) + if self is obj: + return true + if not isinstance(obj, Character): + return false + return true if self.char_val() == obj.char_val() else false @extend(proto._name, String) def _name(self): From cafa5b1cb71841f4e4a6df9d67a23ebb353e83b4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 23:34:38 +0100 Subject: [PATCH 124/909] move superfluous print --- pixie/vm/stdlib.py | 1 - 1 file changed, 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 0ab51685..4780cdd7 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -117,7 +117,6 @@ def seq_QMARK_(x): @as_var("-seq-eq") def _seq_eq(a, b): - print "seq eq" if a is b: return true if not (rt.satisfies_QMARK_(rt.ISeqable.deref(), b) or rt.satisfies_QMARK_(rt.ISeq.deref(), b)): From ab98aa433cd7fe104ebac1bf224c78dadd53c423 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 23:46:06 +0100 Subject: [PATCH 125/909] allow using maps and sets as functions as well --- pixie/stdlib.lisp | 6 +++--- pixie/vm/persistent_hash_set.py | 5 +++++ 2 files changed, 8 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 6a7088e6..bded9e7f 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -430,9 +430,9 @@ (str ":" (namespace k) "/" (name k)) (str ":" (name k))))) -(extend -invoke Keyword - (fn [k m] - (-val-at m k nil))) +(extend -invoke Keyword (fn [k m] (-val-at m k nil))) +(extend -invoke PersistentHashMap (fn [m k] (-val-at m k nil))) +(extend -invoke PersistentHashSet (fn [m k] (-val-at m k nil))) (defn get ([mp k] diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 2863712e..0e7eea4a 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -38,6 +38,11 @@ def _count(self): assert isinstance(self, PersistentHashSet) return rt._count(self._map) +@extend(proto._val_at, PersistentHashSet) +def _val_at(self, key, not_found): + assert isinstance(self, PersistentHashSet) + return rt._val_at(self._map, key, not_found) + @extend(proto._contains_key, PersistentHashSet) def _contains_key(self, key): assert isinstance(self, PersistentHashSet) From 460c8b9ed857462c097a76056da0a06feab17b0b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 3 Nov 2014 23:46:50 +0100 Subject: [PATCH 126/909] add tests for keywords, maps and sets as functions --- tests/collections/test-maps.lisp | 8 ++++++++ tests/collections/test-sets.lisp | 9 +++++++++ tests/test-keywords.lisp | 10 ++++++++++ 3 files changed, 27 insertions(+) create mode 100644 tests/test-keywords.lisp diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp index e2caff8e..6b41cf79 100644 --- a/tests/collections/test-maps.lisp +++ b/tests/collections/test-maps.lisp @@ -22,3 +22,11 @@ (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) + +(t/deftest map-val-at-and-invoke + (let [m {:a 1, :b 2, :c 3}] + (foreach [e m] + (t/assert= (get m (key e)) (val e)) + (t/assert= (m (key e)) (val e))) + (t/assert= (get m :d) nil) + (t/assert= (m :d) nil))) diff --git a/tests/collections/test-sets.lisp b/tests/collections/test-sets.lisp index c1fdb0a2..27b02f3b 100644 --- a/tests/collections/test-sets.lisp +++ b/tests/collections/test-sets.lisp @@ -25,3 +25,12 @@ (t/assert= (= s '(1 2 3)) false) (t/assert= (= s #{1 2}) false) (t/assert= (= s #{1 2 3 4}) false))) + +(t/deftest test-invoke + (let [s #{1 2 3}] + (t/assert= (s 1) 1) + (t/assert= (s 2) 2) + (t/assert= (s 3) 3) + + (t/assert= (s -1) nil) + (t/assert= (s 4) nil))) diff --git a/tests/test-keywords.lisp b/tests/test-keywords.lisp new file mode 100644 index 00000000..5f336b73 --- /dev/null +++ b/tests/test-keywords.lisp @@ -0,0 +1,10 @@ +(ns pixie.tests.test-keywords + (require pixie.test :as t)) + +(t/deftest keyword-invoke + (let [m {:a 1, :b 2, :c 3}] + (t/assert= (:a m) 1) + (t/assert= (:b m) 2) + (t/assert= (:c m) 3) + + (t/assert= (:d m) nil))) From fdd1551c10c4eb3da083031cacc90fe31b709858 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 4 Nov 2014 19:06:43 +0100 Subject: [PATCH 127/909] make -eq on maps more general as suggested by @halgari. it now supports anything that implements IMap and is implemented in pixie itself, not in rpython. --- pixie/stdlib.lisp | 18 ++++++++++++++++++ pixie/vm/persistent_hash_map.py | 20 -------------------- 2 files changed, 18 insertions(+), 20 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index bded9e7f..abe90ca2 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -685,6 +685,24 @@ (extend -eq ISeqable -seq-eq) +(deftype Unknown []) +(def unknown (->Unknown)) + +(extend -eq PersistentHashMap + (fn [self other] + (cond + (not (map? other)) false + (not= (count self) (count other)) false + :else (reduce (fn + ([_] true) + ([_ entry] + (let [other-val (get other (key entry) unknown)] + (if (not= other-val (val entry)) + (reduced false) + true)))) + true + self)))) + (extend -reduce ShallowContinuation (fn [k f init] (loop [acc init] diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index ff4a2d92..e1a6aebc 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -352,26 +352,6 @@ def _with_meta(self, meta): assert isinstance(self, PersistentHashMap) return self.with_meta(meta) -@extend(proto._eq, PersistentHashMap) -def _eq(self, obj): - if self is obj: - return true - elif not isinstance(obj, PersistentHashMap): - return false - elif self._cnt != obj._cnt: - return false - - seq = rt.seq(self) - while seq is not nil: - entry = rt.first(seq) - key = rt._key(entry) - found = rt._contains_key(obj, key) - if not found or not rt.eq(rt._val(entry), rt.get(obj, key)): - return false - seq = rt.next(seq) - - return true - @extend(proto._contains_key, PersistentHashMap) def _contains_key(self, key): assert isinstance(self, PersistentHashMap) From 1860ebb589929c1bc6a160e81ce9b3a0f8a11f84 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 4 Nov 2014 19:07:53 +0100 Subject: [PATCH 128/909] add tests for -eq on iseqables --- tests/collections/test-seqables.lisp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/collections/test-seqables.lisp b/tests/collections/test-seqables.lisp index e8a554fe..ffb8b089 100644 --- a/tests/collections/test-seqables.lisp +++ b/tests/collections/test-seqables.lisp @@ -28,3 +28,12 @@ (t/assert= (next (next (next l))) nil) (t/assert= (next (next (next v))) nil))) + +(t/deftest test-equals + (let [l '(1 2 3)] + (t/assert= l l) + (t/assert= l '(1 2 3)) + (t/assert= l [1 2 3]) + + (t/assert= (= l '(1 2 3 4)) false) + (t/assert= (= l [1 2 3 4]) false))) From 23a723c81243d17c5513a22ea67444e63a7a2e48 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 4 Nov 2014 19:56:06 +0100 Subject: [PATCH 129/909] extend -seq to IIterables --- pixie/stdlib.lisp | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index abe90ca2..5e53fd7e 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -682,6 +682,7 @@ (cons (current i) (lazy-seq (iterator-seq (move-next! i)))))) (extend -seq IIterator iterator-seq) +(extend -seq IIterable (comp seq iterator)) (extend -eq ISeqable -seq-eq) From 11e782b8e05fa40589c1196df8d5316acd99b906 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 4 Nov 2014 19:57:00 +0100 Subject: [PATCH 130/909] add tests for -eq on vectors --- tests/collections/test-vectors.lisp | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.lisp index 69673284..0f43acca 100644 --- a/tests/collections/test-vectors.lisp +++ b/tests/collections/test-vectors.lisp @@ -19,3 +19,14 @@ (t/assert= (contains? v c) true)) (foreach [n n] (t/assert= (contains? v n) false)))) + +(t/deftest vector-equals + (let [v [1 2 3]] + (t/assert= v v) + (t/assert= v [1 2 3]) + (t/assert= v '(1 2 3)) + + (t/assert= (= v [1 2]) false) + (t/assert= (= v [1 2 3 4]) false) + (t/assert= (= v '(1 2)) false) + (t/assert= (= v '(1 2 3 4)) false))) From 99b711e91ef363427aec4f1ffc6fa39b53ede369 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 4 Nov 2014 15:30:17 -0700 Subject: [PATCH 131/909] fix translation --- pixie/vm/libs/ffi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index d0744efd..469d19b0 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -82,7 +82,7 @@ def get_ret_val(ptr, tp): if pnt[0] == lltype.nullptr(rffi.CCHARP.TO): return nil else: - return String(rffi.charp2str(pnt[0])) + return String(unicode(rffi.charp2str(pnt[0]))) assert False From 9383433fb13245001233f2e7f882b1d2623360bb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 4 Nov 2014 21:31:28 -0700 Subject: [PATCH 132/909] start of async io support. Started by creating a stacklet queue --- pixie/vm/libs/ring_buffer.py | 67 ++++++++++++++++++++++++++++++++++++ pixie/vm/libs/uv.py | 57 ++++++++++++++++++++++++++++++ pixie/vm/stacklet.py | 36 ++++++++++++++++++- 3 files changed, 159 insertions(+), 1 deletion(-) create mode 100644 pixie/vm/libs/ring_buffer.py create mode 100644 pixie/vm/libs/uv.py diff --git a/pixie/vm/libs/ring_buffer.py b/pixie/vm/libs/ring_buffer.py new file mode 100644 index 00000000..69cbd7e5 --- /dev/null +++ b/pixie/vm/libs/ring_buffer.py @@ -0,0 +1,67 @@ +from rpython.rlib.rarithmetic import r_uint + +class RingBuffer(object): + def __init__(self, size): + assert isinstance(size, r_uint) + self._array = [None] * size + self._array_len = size + self._length = size + self._head = 0 + self._tail = 0 + + def push(self, obj): + if self._count == self._size: + self.resize_larger(self._size * 2) + + self._array[self._idx] = obj + + def pop(self): + if not self._length == 0: + x = self._array[self._tail] + self._array[self._tail] = None + self._tail = (self._tail + 1) % self._array_len + self._length -= 1 + return x + return None + + def push(self, x): + self._array[self._head] = x + self._head = (self._head + 1) % self._array_len + self._length -= 1 + + def unbounded_push(self, x): + if self._length - 1 == 0: + self.resize() + self.push(x) + + def resize(self): + new_arr_size = self._array_len * 2 + new_arr = [None] * new_arr_size + + if self._tail < self._head: + array_copy(self._array, self._tail, new_arr, 0, self._length) + self._tail = 0 + self._head = self._length + self._array = new_arr + self._array_len = new_arr_size + + elif self._tail > self._head: + array_copy(self._array, self._tail, new_arr, 0, self._array_len - self._tail) + array_copy(self._array, 0, new_arr, self._array_len - self._tail, self._head) + self._tail = 0 + self._head = self._length + self._array = new_arr + self._array_len = new_arr_size + + else: + self._tail = 0 + self._head = 0 + self._array = new_arr + self._array_len = new_arr_size + + +def array_copy(src, src_pos, dest, dest_pos, count): + x = r_uint(0) + while x < count: + dest[dest_pos + x] = src[src_pos + x] + x += 1 diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py new file mode 100644 index 00000000..551a9d33 --- /dev/null +++ b/pixie/vm/libs/uv.py @@ -0,0 +1,57 @@ +import py + +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem.lloperation import llop +import rpython.rtyper.tool.rffi_platform as rffi_platform +from rpython.translator import cdir +from rpython.translator.tool.cbuild import ExternalCompilationInfo +from rpython.rtyper.annlowlevel import llhelper + +srcdir = py.path.local(cdir) / 'src' +compilation_info = ExternalCompilationInfo( + includes=['uv.h'], + libraries=["uv"]) + +def llexternal(*args, **kwargs): + return rffi.llexternal(*args, compilation_info=compilation_info, **kwargs) + + +uv_work = rffi_platform.Struct("uv_work_t", + [("data", rffi.VOIDP)]) + +uv_timer_t = rffi.COpaque("uv_timer_t", compilation_info=compilation_info) + +#print rffi.sizeof(uv_timer_t) + +uv_timer = lltype.Ptr(uv_timer_t) + +uv_timer_cb = lltype.Ptr(lltype.FuncType([uv_timer, rffi.INT], lltype.Void)) + +def as_cb(T, fn): + return llhelper(T, rffi._make_wrapper_for(T, fn)) + +def _timer_cb(timer_t, status): + pass + #lltype.free(timer_t) + print "foo!" + +def queue_something(loop): + print "starting" + timer = lltype.malloc(uv_timer_t, flavor="raw", track_allocation="false") + assert not timer_init(loop, timer) + assert not timer_start(timer, as_cb(uv_timer_cb, _timer_cb), 10000, 0) + print "Started" + + +loop_new = llexternal('uv_loop_new', [], rffi.VOIDP) +run = llexternal("uv_run", [rffi.VOIDP, rffi.INT], rffi.INT) + +timer_init = llexternal("uv_timer_init", [rffi.VOIDP, uv_timer], rffi.INT) + +timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.INT, rffi.INT], rffi.INT) + + + +RUN_DEFAULT = 0 +RUN_ONCE = 1 +RUN_NO_WAIT = 2 \ No newline at end of file diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index cea6db0d..0fd5fa44 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -6,9 +6,11 @@ from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint as r_uint32, intmask, widen +from rpython.rlib.rarithmetic import r_uint, intmask, widen import rpython.rlib.jit as jit import pixie.vm.rt as rt +import pixie.vm.libs.uv as uv +import pixie.vm.libs.ring_buffer as ring_buffer import rpython.rlib.rstacklet as rstacklet from rpython.rtyper.lltypesystem import lltype, rffi @@ -16,6 +18,8 @@ OP_SWITCH = 0x02 OP_CONTINUE = 0x03 OP_EXIT = 0x04 +OP_YIELD = 0x05 +OP_NEXT_PENDING = 0x06 class GlobalState(py_object): def __init__(self): @@ -99,6 +103,10 @@ def new_stacklet(f): val = global_state._val return val +def yield_stacklet(): + global_state._op = OP_YIELD + global_state._val = nil + global_state._h = global_state._th.switch(global_state._h) def new_handler(h, o): global_state._h = h @@ -137,6 +145,7 @@ def init_handler(h, o): def with_stacklets(f): + loop = uv.loop_new() init() global_state._init_fn = f @@ -144,7 +153,15 @@ def with_stacklets(f): main_h = global_state._th.new(init_handler) global_state._from = WrappedHandler(main_h) + pending_stacklets = ring_buffer.RingBuffer(r_uint(32)) + pending_stacklets.push(1) + assert pending_stacklets.pop() == 1 + pending_stacklets.push(2) + assert pending_stacklets.pop() == 2 + + print "Entrance" while True: + print global_state._op if global_state._op == OP_NEW: wh = WrappedHandler(global_state._th.get_null_handle()) global_state._to = wh @@ -161,6 +178,18 @@ def with_stacklets(f): shutdown() return global_state._val + elif global_state._op == OP_YIELD: + assert global_state._from + pending_stacklets.push(global_state._from) + global_state._op = OP_NEXT_PENDING + continue + + elif global_state._op == OP_NEXT_PENDING: + f = pending_stacklets.pop() + global_state._from = f + f._h = global_state._th.switch(f._h) + continue + else: break @@ -173,6 +202,11 @@ def _new_stacklet(f): stacklet.invoke([nil]) # prime it return stacklet +@as_var("yield-stacklet") +def _stacklet_yield(): + yield_stacklet() + return nil + @extend(proto._at_end_QMARK_, WrappedHandler) def _at_end(self): assert isinstance(self, WrappedHandler) From e784e993320f0e7685352fbfd67128af2534468d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Nov 2014 06:06:07 -0700 Subject: [PATCH 133/909] fixes for issue #38 --- pixie/vm/persistent_hash_map.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 162e6b06..c11c83d9 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -99,7 +99,7 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): return BitmapIndexedNode(None, self._bitmap, clone_and_set(self._array, 2 * idx + 1, n)) - if key_or_null is None or rt.eq(key, key_or_null): + if rt.eq(key, key_or_null): if val is val_or_node: return self return BitmapIndexedNode(None, self._bitmap, clone_and_set(self._array, 2 * idx + 1, val)) @@ -108,7 +108,7 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): return BitmapIndexedNode(None, self._bitmap, clone_and_set2(self._array, 2 * idx, None, - 2 * idx + 1, create_node(shift+ 5, key_or_null, val_or_node, hash_val, key, val))) + 2 * idx + 1, create_node(shift + 5, key_or_null, val_or_node, hash_val, key, val))) else: n = bit_count(self._bitmap) if n >= 16: @@ -237,7 +237,7 @@ def create_node(shift, key1, val1, key2hash, key2, val2): return HashCollisionNode(None, key1hash, [key1, val1, key2, val2]) added_leaf = Box() return BitmapIndexedNode_EMPTY.assoc_inode(shift, key1hash, key1, val1, added_leaf) \ - .assoc_inode(shift, key1hash, key1, val1, added_leaf) + .assoc_inode(shift, key2hash, key2, val2, added_leaf) def bit_count(i): assert isinstance(i, r_uint) From f6174b6bfe1b04924f5c88c25eaf4ea8968997d6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Nov 2014 06:31:56 -0700 Subject: [PATCH 134/909] implemented lightweight threads --- pixie/vm/stacklet.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index 0fd5fa44..074be283 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -20,6 +20,7 @@ OP_EXIT = 0x04 OP_YIELD = 0x05 OP_NEXT_PENDING = 0x06 +OP_NEW_THREAD = 0x07 class GlobalState(py_object): def __init__(self): @@ -103,6 +104,12 @@ def new_stacklet(f): val = global_state._val return val +def new_thread(f): + global_state._op = OP_NEW_THREAD + global_state._val = f + global_state._h = global_state._th.switch(global_state._h) + + def yield_stacklet(): global_state._op = OP_YIELD global_state._val = nil @@ -127,6 +134,25 @@ def new_handler(h, o): return global_state._h +def new_thread_handler(h, o): + global_state._h = h + + affirm(global_state._val is not None, u"Internal Stacklet Error") + f = global_state._val + global_state._val = None + + global_state._h = global_state._th.switch(global_state._h) + + + #try: + f.invoke([global_state._from]) + + global_state._op = OP_NEXT_PENDING + global_state._h = global_state._th.switch(global_state._h) + + return global_state._h + + def init_handler(h, o): global_state._h = h @@ -169,6 +195,15 @@ def with_stacklets(f): global_state._val = wh continue + elif global_state._op == OP_NEW_THREAD: + wh = WrappedHandler(global_state._th.get_null_handle()) + global_state._to = wh + wh._h = global_state._th.new(new_thread_handler) + pending_stacklets.push(wh) + pending_stacklets.push(global_state._from) + global_state._op = OP_NEXT_PENDING + continue + elif global_state._op == OP_SWITCH: to = global_state._to to._h = global_state._th.switch(global_state._to._h) @@ -202,6 +237,12 @@ def _new_stacklet(f): stacklet.invoke([nil]) # prime it return stacklet +@as_var("create-thread") +def _new_stacklet(f): + stacklet = new_thread(f) + + return nil + @as_var("yield-stacklet") def _stacklet_yield(): yield_stacklet() From 952cbc8df6c4e9a71860894ca36fa508152c4906 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Nov 2014 06:58:34 -0700 Subject: [PATCH 135/909] some improvements to help with finding the bug in the test suite --- pixie/test.lisp | 5 ++++- pixie/vm/stdlib.py | 7 ++++++- tests/collections/test-maps.lisp | 10 +++++++--- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/pixie/test.lisp b/pixie/test.lisp index f57cd45f..272d7e3e 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -15,8 +15,11 @@ ~@body (swap! *stats* update-in [:pass] (fnil inc 0)) (catch ex + (print "while running " ~(name nm) " " (quote (do ~@body))) + (swap! *stats* update-in [:fail] (fnil inc 0)) - (swap! *stats* update-in [:errors] (fnil conj []) ex)))) + (print (str ex)) + (swap! *stats* update-in [:errors] (fnil conj []) ex)))) (swap! tests assoc (symbol (str (namespace (var ~nm)) "/" (name (var ~nm)))) ~nm))) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 4780cdd7..7e71e5bc 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -379,7 +379,7 @@ def refer(ns, refer, alias): @as_var("extend") -def extend(proto_fn, tp, fn): +def _extend(proto_fn, tp, fn): affirm(isinstance(proto_fn, PolymorphicFn), u"First argument to extend should be a PolymorphicFn") affirm(isinstance(tp, Type) or isinstance(tp, Protocol), u"Second argument to extend must be a Type or Protocol") affirm(isinstance(fn, BaseCode), u"Last argument to extend must be a function") @@ -530,3 +530,8 @@ def _merge__args(args): return acc + +@extend(_str, RuntimeException) +def _str(self): + assert isinstance(self, RuntimeException) + return rt.wrap(self.__repr__()) \ No newline at end of file diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp index 6b41cf79..fc2928eb 100644 --- a/tests/collections/test-maps.lisp +++ b/tests/collections/test-maps.lisp @@ -16,12 +16,16 @@ (t/assert= (-eq m m) true) (t/assert= (-eq m {:a 1, :b 2, :c 3}) true) - (t/assert= (-eq m {}) false) - (t/assert= (-eq m {:a 1, :b 2} false)) + (comment (t/assert= (-eq m {}) false)) + + + (t/assert= (-eq m {:a 1, :b 2}) false) (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false) (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) - (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) + (t/assert= (-eq m {:a 3, :b 2, :c 1}) false) + )) + (t/deftest map-val-at-and-invoke (let [m {:a 1, :b 2, :c 3}] From 21640992a171b91eb72ef3b173fae783ec8441d9 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 5 Nov 2014 19:30:52 +0100 Subject: [PATCH 136/909] implement most bit-* on integers unsigned-bit-shift-right is missing, but the others are there. --- pixie/vm/bits.py | 77 ++++++++++++++++++++++++++++++++++++++++++++++++ pixie/vm/rt.py | 1 + 2 files changed, 78 insertions(+) create mode 100644 pixie/vm/bits.py diff --git a/pixie/vm/bits.py b/pixie/vm/bits.py new file mode 100644 index 00000000..9ab3b8e2 --- /dev/null +++ b/pixie/vm/bits.py @@ -0,0 +1,77 @@ +from pixie.vm.code import as_var +from pixie.vm.object import affirm + +from pixie.vm.numbers import Integer + +import pixie.vm.rt as rt + +@as_var("bit-clear") +def bit_clear(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() & ~(1 << n.int_val())) + +@as_var("bit-set") +def bit_set(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() | (1 << n.int_val())) + +@as_var("bit-flip") +def bit_flip(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() ^ (1 << n.int_val())) + +@as_var("bit-test") +def bit_test(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap((x.int_val() & (1 << n.int_val())) != 0) + +@as_var("bit-and") +def bit_and(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() & y.int_val()) + +@as_var("bit-or") +def bit_or(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() | y.int_val()) + +@as_var("bit-xor") +def bit_xor(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() ^ y.int_val()) + +@as_var("bit-shift-left") +def bit_shift_left(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() << n.int_val()) + +@as_var("bit-shift-right") +def bit_shift_right(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() >> n.int_val()) + +# unsigned-bit-shift-right (sets sign bit to zero) + +digits = "0123456789abcdefghijklmnopqrstuvwxyz" + +@as_var("bit-str") +def bit_str(x, shift): + affirm(isinstance(x, Integer) and isinstance(shift, Integer), u"x and shift must be Integers") + x = x.int_val() + shift = shift.int_val() + + buf = ['_'] * 32 + char_pos = 32 + radix = 1 << shift + mask = radix - 1 + while True: + char_pos -= 1 + buf[char_pos] = digits[x & mask] + x = x >> shift + if x == 0: + break + + res = "" + for i in range(char_pos, char_pos + 32 - char_pos): + res += buf[i] + return rt.wrap(res) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 6c7c2f9b..16ac308a 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -48,6 +48,7 @@ def wrapper(*args): sys.setrecursionlimit(10000) # Yeah we blow the stack sometimes, we promise it's not a bug import pixie.vm.numbers as numbers + import pixie.vm.bits as bits from pixie.vm.code import wrap_fn import pixie.vm.interpreter import pixie.vm.stacklet as stacklet From e024df2bf9609ed5a59ee18a3d8f7158e6076b54 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 5 Nov 2014 19:35:13 +0100 Subject: [PATCH 137/909] add bit-and-not for completeness --- pixie/vm/bits.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/vm/bits.py b/pixie/vm/bits.py index 9ab3b8e2..84500dd1 100644 --- a/pixie/vm/bits.py +++ b/pixie/vm/bits.py @@ -30,6 +30,11 @@ def bit_and(x, y): affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") return rt.wrap(x.int_val() & y.int_val()) +@as_var("bit-and-not") +def bit_and_not(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() & ~y.int_val()) + @as_var("bit-or") def bit_or(x, y): affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") From b0f0b1195bd0e81386f6d282986790b3362b8dbb Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 5 Nov 2014 19:50:33 +0100 Subject: [PATCH 138/909] add pos?, neg? and zero? --- pixie/stdlib.lisp | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e729d965..a72f545d 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -281,6 +281,14 @@ (apply >= y rest) false))) +(defn pos? [x] + (> x 0)) + +(defn neg? [x] + (< x 0)) + +(defn zero? [x] + (= x 0)) (def inc (fn [x] (+ x 1))) From 7f8185c77a360f52be5cf9e94c4049f091684ec2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 5 Nov 2014 19:50:54 +0100 Subject: [PATCH 139/909] add aget, aset and make-array --- pixie/vm/array.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 0a9f17e0..55e2f78d 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -1,5 +1,6 @@ import pixie.vm.rt as rt import pixie.vm.object as object +from pixie.vm.object import affirm from pixie.vm.code import extend, as_var from pixie.vm.numbers import Integer from pixie.vm.primitives import nil @@ -33,11 +34,6 @@ def reduce_large(self, f, init): init = f.invoke([init, self._list[x]]) return init - - - - - @extend(proto._count, Array) def _count(self): assert isinstance(self, Array) @@ -55,8 +51,22 @@ def reduce(self, f, init): return self.reduce_large(f, init) return self.reduce_small(f, init) - def array(lst): assert isinstance(lst, list) return Array(lst) +@as_var("aget") +def aget(self, idx): + assert isinstance(self, Array) + return self._list[idx.int_val()] + +@as_var("aset") +def aset(self, idx, val): + assert isinstance(self, Array) + self._list[idx.int_val()] = val + return val + +@as_var("make-array") +def make_array(l): + affirm(isinstance(l, Integer), u"l must be an Integer") + return Array([nil] * l.int_val()) From ea7c6da06b0fdac96d684a87f2657fe5ad0b1f64 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 5 Nov 2014 19:55:57 +0100 Subject: [PATCH 140/909] add alength for completeness --- pixie/vm/array.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 55e2f78d..204dff5b 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -66,6 +66,11 @@ def aset(self, idx, val): self._list[idx.int_val()] = val return val +@as_var("alength") +def alength(self): + assert isinstance(self, Array) + return rt.wrap(len(self._list)) + @as_var("make-array") def make_array(l): affirm(isinstance(l, Integer), u"l must be an Integer") From d19f7c4cff1409cc336ffb52215940de6a05c347 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 5 Nov 2014 20:09:12 +0100 Subject: [PATCH 141/909] add aslice and aconcat i'm not really happy with adding all these, but maybe they are useful to allow writing faster low-level code in pixie. --- pixie/vm/array.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 204dff5b..ff9d8e75 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -66,6 +66,17 @@ def aset(self, idx, val): self._list[idx.int_val()] = val return val +@as_var("aslice") +def aslice(self, offset): + assert isinstance(self, Array) + affirm(isinstance(offset, Integer), u"offset must be an integer") + return Array(self._list[offset.int_val():]) + +@as_var("aconcat") +def aconcat(self, other): + assert isinstance(self, Array) and isinstance(other, Array) + return Array(self._list + other._list) + @as_var("alength") def alength(self): assert isinstance(self, Array) From 24a702ae753000349ff17cd522720753ffdd1cf0 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 5 Nov 2014 20:22:57 +0100 Subject: [PATCH 142/909] fix aslice with negative integer arguments --- pixie/vm/array.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index ff9d8e75..8bd12932 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -68,9 +68,13 @@ def aset(self, idx, val): @as_var("aslice") def aslice(self, offset): - assert isinstance(self, Array) - affirm(isinstance(offset, Integer), u"offset must be an integer") - return Array(self._list[offset.int_val():]) + assert isinstance(self, Array) and isinstance(offset, Integer) + + offset = offset.int_val() + if offset >= 0: + return Array(self._list[offset:]) + else: + rt.throw(rt.wrap(u"offset must be an Integer >= 0")) @as_var("aconcat") def aconcat(self, other): From e0140e17e359cd748135d0938c28faf8b5d67b05 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Nov 2014 15:39:18 -0700 Subject: [PATCH 143/909] should fix issues with the tests --- pixie/test.lisp | 1 + pixie/vm/compiler.py | 7 +++---- pixie/vm/interpreter.py | 5 +++-- pixie/vm/object.py | 3 +++ tests/collections/test-maps.lisp | 6 ++---- tests/collections/test-vectors.lisp | 2 +- 6 files changed, 13 insertions(+), 11 deletions(-) diff --git a/pixie/test.lisp b/pixie/test.lisp index 272d7e3e..0452653e 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -54,3 +54,4 @@ (defn assert= [x y] (assert (= x y) (str x " != " y))) + diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index fb00ef1d..3623a16a 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -294,7 +294,7 @@ def compile_map_literal(form, ctx): ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(size) + 1) if size > 0: - ctx.sub_sp(size - 1) + ctx.sub_sp(size) compile_meta(rt.meta(form), ctx) @@ -492,10 +492,9 @@ def compile_fn_body(name, args, body, ctx): if rt.next(body) is nil: new_ctx.enable_tail_call() compile_form(rt.first(body), new_ctx) - if rt.next(body) is not nil: - new_ctx.pop() - bc += 1 body = rt.next(body) + if body is not nil: + new_ctx.pop() new_ctx.bytecode.append(code.RETURN) closed_overs = new_ctx.closed_overs diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 1617d5a0..bfee5760 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -1,4 +1,4 @@ -from pixie.vm.object import Object, affirm, WrappedException, Type +from pixie.vm.object import Object, affirm, WrappedException, Type, runtime_error import pixie.vm.code as code import pixie.vm.numbers as numbers from pixie.vm.primitives import nil, true, false @@ -88,7 +88,8 @@ def pop(self): def nth(self, delta): affirm(delta >= 0, u"Invalid nth value, (compiler error)") - affirm(self.sp - 1 >= delta, u"interpreter nth out of range, (compiler error) ") + if not self.sp - 1 >= delta: + runtime_error(u"Interpreter nth out of range: " + unicode(str(self.sp - 1)) + u", " + unicode(str(delta))) return self.stack[self.sp - delta - 1] def push_nth(self, delta): diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 80076bcc..07accbca 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -134,6 +134,9 @@ def affirm(val, msg): import pixie.vm.rt as rt raise WrappedException(RuntimeException(rt.wrap(msg))) +def runtime_error(msg): + import pixie.vm.rt as rt + raise WrappedException(RuntimeException(rt.wrap(msg))) class ErrorInfo(Object): _type = Type(u"pixie.stdlib.ErrorInfo") diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp index fc2928eb..c886146b 100644 --- a/tests/collections/test-maps.lisp +++ b/tests/collections/test-maps.lisp @@ -16,15 +16,13 @@ (t/assert= (-eq m m) true) (t/assert= (-eq m {:a 1, :b 2, :c 3}) true) - (comment (t/assert= (-eq m {}) false)) - + (t/assert= (-eq m {}) false) (t/assert= (-eq m {:a 1, :b 2}) false) (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false) (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) - (t/assert= (-eq m {:a 3, :b 2, :c 1}) false) - )) + (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) (t/deftest map-val-at-and-invoke diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.lisp index 0f43acca..6ef72691 100644 --- a/tests/collections/test-vectors.lisp +++ b/tests/collections/test-vectors.lisp @@ -29,4 +29,4 @@ (t/assert= (= v [1 2]) false) (t/assert= (= v [1 2 3 4]) false) (t/assert= (= v '(1 2)) false) - (t/assert= (= v '(1 2 3 4)) false))) + (t/assert= (= v '(1 2 3 4)) false))) \ No newline at end of file From 52f6d327007444a4287770a6f85ea265eb9d4a7e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 5 Nov 2014 17:15:15 -0700 Subject: [PATCH 144/909] we can dispatch to libuv, although the timeout values seem borked --- pixie/vm/libs/ring_buffer.py | 9 +++---- pixie/vm/libs/uv.py | 51 +++++++++++++++++++++++++++--------- pixie/vm/stacklet.py | 30 +++++++++++++++++++-- 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/pixie/vm/libs/ring_buffer.py b/pixie/vm/libs/ring_buffer.py index 69cbd7e5..70fd0e90 100644 --- a/pixie/vm/libs/ring_buffer.py +++ b/pixie/vm/libs/ring_buffer.py @@ -9,11 +9,8 @@ def __init__(self, size): self._head = 0 self._tail = 0 - def push(self, obj): - if self._count == self._size: - self.resize_larger(self._size * 2) - - self._array[self._idx] = obj + def pending(self): + return self._array_len - self._length def pop(self): if not self._length == 0: @@ -27,7 +24,7 @@ def pop(self): def push(self, x): self._array[self._head] = x self._head = (self._head + 1) % self._array_len - self._length -= 1 + self._length += 1 def unbounded_push(self, x): if self._length - 1 == 0: diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py index 551a9d33..d9e6e63c 100644 --- a/pixie/vm/libs/uv.py +++ b/pixie/vm/libs/uv.py @@ -6,6 +6,10 @@ from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.annlowlevel import llhelper +from rpython.rlib.rgc import pin, unpin +from pixie.vm.code import as_var, extend, NativeFn +from pixie.vm.primitives import nil + srcdir = py.path.local(cdir) / 'src' compilation_info = ExternalCompilationInfo( @@ -27,31 +31,54 @@ def llexternal(*args, **kwargs): uv_timer_cb = lltype.Ptr(lltype.FuncType([uv_timer, rffi.INT], lltype.Void)) +data_container = {} + def as_cb(T, fn): return llhelper(T, rffi._make_wrapper_for(T, fn)) def _timer_cb(timer_t, status): - pass - #lltype.free(timer_t) - print "foo!" - -def queue_something(loop): - print "starting" + print status, " status" + import pixie.vm.stacklet as stacklet + casted = rffi.cast(rffi.INT, timer_t) + data = data_container[casted] + del data_container[casted] + stacklet.pending_stacklets.push(data) + print "timeout completed" + +def set_timeout(loop, cont, timeout, repeat): timer = lltype.malloc(uv_timer_t, flavor="raw", track_allocation="false") + data_container[rffi.cast(rffi.INT, timer)] = cont assert not timer_init(loop, timer) - assert not timer_start(timer, as_cb(uv_timer_cb, _timer_cb), 10000, 0) - print "Started" + print "setting timeout", timeout, repeat + assert not timer_start(timer, as_cb(uv_timer_cb, _timer_cb), timeout, repeat) + loop_new = llexternal('uv_loop_new', [], rffi.VOIDP) -run = llexternal("uv_run", [rffi.VOIDP, rffi.INT], rffi.INT) +run = llexternal("uv_run", [rffi.VOIDP, rffi.INT], rffi.SIZE_T) timer_init = llexternal("uv_timer_init", [rffi.VOIDP, uv_timer], rffi.INT) - -timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.INT, rffi.INT], rffi.INT) +timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.UINT, rffi.UINT], rffi.INT) RUN_DEFAULT = 0 RUN_ONCE = 1 -RUN_NO_WAIT = 2 \ No newline at end of file +RUN_NO_WAIT = 2 + +class UVFunction(NativeFn): + pass + + +class SleepUVFunction(UVFunction): + def __init__(self, time): + self._time = time.int_val() + + def execute_uv(self, loop, k): + set_timeout(loop, k, self._time, 0) + +@as_var("sleep") +def _sleep(ms): + from pixie.vm.stacklet import execute_uv_func + execute_uv_func(SleepUVFunction(ms)) + return nil \ No newline at end of file diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index 074be283..caff6e1e 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -21,6 +21,7 @@ OP_YIELD = 0x05 OP_NEXT_PENDING = 0x06 OP_NEW_THREAD = 0x07 +OP_EXECUTE_UV = 0x08 class GlobalState(py_object): def __init__(self): @@ -109,12 +110,21 @@ def new_thread(f): global_state._val = f global_state._h = global_state._th.switch(global_state._h) +def enqueue_stacklet(k): + pending_stacklets.push(k) def yield_stacklet(): global_state._op = OP_YIELD global_state._val = nil global_state._h = global_state._th.switch(global_state._h) +def execute_uv_func(func): + assert isinstance(func, uv.UVFunction) + global_state._op = OP_EXECUTE_UV + global_state._val = func + global_state._h = global_state._th.switch(global_state._h) + + def new_handler(h, o): global_state._h = h @@ -169,6 +179,8 @@ def init_handler(h, o): global_state._op = OP_EXIT return global_state._h +pending_stacklets = ring_buffer.RingBuffer(r_uint(32)) + def with_stacklets(f): loop = uv.loop_new() @@ -179,13 +191,11 @@ def with_stacklets(f): main_h = global_state._th.new(init_handler) global_state._from = WrappedHandler(main_h) - pending_stacklets = ring_buffer.RingBuffer(r_uint(32)) pending_stacklets.push(1) assert pending_stacklets.pop() == 1 pending_stacklets.push(2) assert pending_stacklets.pop() == 2 - print "Entrance" while True: print global_state._op if global_state._op == OP_NEW: @@ -219,7 +229,23 @@ def with_stacklets(f): global_state._op = OP_NEXT_PENDING continue + elif global_state._op == OP_EXECUTE_UV: + assert global_state._from + k = global_state._from + f = global_state._val + f.execute_uv(loop, k) + global_state._op = OP_NEXT_PENDING + continue + elif global_state._op == OP_NEXT_PENDING: + print "running with", pending_stacklets.pending() + while True: + if pending_stacklets.pending() == 0: + uv.run(loop, uv.RUN_DEFAULT) + continue + else: + uv.run(loop, uv.RUN_DEFAULT | uv.RUN_NO_WAIT) + break f = pending_stacklets.pop() global_state._from = f f._h = global_state._th.switch(f._h) From f9bbc40fd60d731f77c0389012a27a4245d74839 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 6 Nov 2014 11:36:35 -0700 Subject: [PATCH 145/909] latest work --- pixie/vm/libs/c/uv_ffi.c | 56 ++++++++++++++++++++++++++++++++++++++++ pixie/vm/libs/ffi.py | 7 ++++- pixie/vm/libs/uv.py | 44 ++++++++++++++++++++++++++----- 3 files changed, 100 insertions(+), 7 deletions(-) create mode 100644 pixie/vm/libs/c/uv_ffi.c diff --git a/pixie/vm/libs/c/uv_ffi.c b/pixie/vm/libs/c/uv_ffi.c new file mode 100644 index 00000000..0e589547 --- /dev/null +++ b/pixie/vm/libs/c/uv_ffi.c @@ -0,0 +1,56 @@ +#include "uv.h" +#include "ffi.h" + +typedef struct cif_desc_t { + ffi_cif cif; + int abi; + int nargs; + ffi_type* rtype; + ffi_type** atypes; + int exchange_size; + int exchange_result; + int exchange_result_libffi; + int exchange_args[0]; +} cif_desc_t; + +void (*ffi_cb)(ffi_cif *cif); + +typedef struct work_baton_t +{ + uv_work_t work; + ffi_cif *cif; + void *fn_addr; + void *exb; +} work_baton_t; + +void ffi_call_impl_any_c(cif_desc_t *cif, void* func_addr, void* exchange_buffer) +{ + void **buffer_array = (void **)exchange_buffer; + for (int i = 0; i < cif->nargs; i += 1) + { + void **data = exchange_buffer + cif->exchange_args[i]; + buffer_array[i] = data; + } + void **result_data = exchange_buffer + cif->exchange_result_libffi; + ffi_call(&cif->cif, func_addr, (void*)result_data, buffer_array); +} + + + +void do_work(work_baton_t *req) +{ + ffi_call_impl_any_c(req->cif, req->fn_addr, req->exb); +} + +void* uv_ffi_make_baton() +{ + return malloc(sizeof(work_baton_t)); +} + +int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_addr, void *exb, uv_after_work_cb *after_work) +{ + w->cif = cif; + w->fn_addr = fn_addr; + w->exb = exb; + uv_queue_work(loop, w, do_work, after_work); +} \ No newline at end of file diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 469d19b0..4dd2619e 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -160,7 +160,7 @@ def _cleanup_(self): self._is_inited = False @jit.unroll_safe - def _invoke(self, args): + def prep_exb(self, args): if not self._is_inited: self.thaw() exb = lltype.malloc(rffi.CCHARP.TO, self._transfer_size, flavor="raw") @@ -168,7 +168,12 @@ def _invoke(self, args): for x in range(len(self._arg_types)): offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) + return exb + + @jit.unroll_safe + def _invoke(self, args): + exb = self.prep_exb(args) jit_ffi_call(self._cd, self._f_ptr, exb) offset_p = rffi.ptradd(exb, self._ret_offset) ret_val = get_ret_val(offset_p, self._ret_type) diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py index d9e6e63c..ba7d48fd 100644 --- a/pixie/vm/libs/uv.py +++ b/pixie/vm/libs/uv.py @@ -3,18 +3,27 @@ from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.lloperation import llop import rpython.rtyper.tool.rffi_platform as rffi_platform -from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo from rpython.rtyper.annlowlevel import llhelper from rpython.rlib.rgc import pin, unpin from pixie.vm.code import as_var, extend, NativeFn from pixie.vm.primitives import nil +import rpython.tool.udir as udir +import os +import shutil +import pixie.vm.libs.ffi as ffi + + +pkgpath = py.path.local(__file__).dirpath() +srcpath = pkgpath.join("c") + +shutil.copyfile(str(srcpath / "uv_ffi.c"), str(udir.udir / "uv_ffi.c")) -srcdir = py.path.local(cdir) / 'src' compilation_info = ExternalCompilationInfo( - includes=['uv.h'], - libraries=["uv"]) + includes=['uv.h', "ffi.h"], + libraries=["uv", "ffi"], + separate_module_files=[udir.udir / "uv_ffi.c"]).merge(ExternalCompilationInfo.from_pkg_config("libffi")) def llexternal(*args, **kwargs): return rffi.llexternal(*args, compilation_info=compilation_info, **kwargs) @@ -31,6 +40,8 @@ def llexternal(*args, **kwargs): uv_timer_cb = lltype.Ptr(lltype.FuncType([uv_timer, rffi.INT], lltype.Void)) +uv_callback = lltype.Ptr(lltype.FuncType([rffi.VOIDP, rffi.INT], lltype.Void)) + data_container = {} def as_cb(T, fn): @@ -58,9 +69,10 @@ def set_timeout(loop, cont, timeout, repeat): run = llexternal("uv_run", [rffi.VOIDP, rffi.INT], rffi.SIZE_T) timer_init = llexternal("uv_timer_init", [rffi.VOIDP, uv_timer], rffi.INT) -timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.UINT, rffi.UINT], rffi.INT) - +timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.INT, rffi.UINT], rffi.INT) +ffi_run = llexternal("uv_ffi_run", [rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, uv_callback], rffi.INT) +ffi_make_baton = llexternal("uv_ffi_make_baton", [], rffi.VOIDP) RUN_DEFAULT = 0 RUN_ONCE = 1 @@ -77,6 +89,26 @@ def __init__(self, time): def execute_uv(self, loop, k): set_timeout(loop, k, self._time, 0) +class RunFFIFunc(UVFunction): + def __init__(self, fn, args): + + assert isinstance(fn, ffi.FFIFn) + self._fn = fn + self._args = args + + def execute_uv(self, loop, k): + baton = ffi_make_baton() + data_container[rffi.cast(rffi.INT, baton)] = k + exb = self._fn.prep_exb(self._args) + + ffi_run(baton, loop, ffi._cd, ffi._f_ptr, exb, as_cb(uv_timer_cb, _timer_cb)) + +@as_var("run_blocking") +def _run_blocking(fn, arg): + from pixie.vm.stacklet import execute_uv_func + execute_uv_func(RunFFIFunc(fn, [arg])) + return nil + @as_var("sleep") def _sleep(ms): from pixie.vm.stacklet import execute_uv_func From 5f243e2eb445859536d0972f5d56135313ad1692 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 6 Nov 2014 21:36:54 +0100 Subject: [PATCH 146/909] add tests for the bit operations far too extensive, but a good demonstration of the base 2 literal syntax... --- tests/test-bits.lisp | 59 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 59 insertions(+) create mode 100644 tests/test-bits.lisp diff --git a/tests/test-bits.lisp b/tests/test-bits.lisp new file mode 100644 index 00000000..be044f9d --- /dev/null +++ b/tests/test-bits.lisp @@ -0,0 +1,59 @@ +(ns pixie.test.test-bits + (require pixie.test :as t)) + +(t/deftest test-bit-clear + (t/assert= (bit-clear 0 0) 0) + (t/assert= (bit-clear 2r11111 7) 2r11111) + + (t/assert= (bit-clear 2r111 0) 2r110) + (t/assert= (bit-clear 2r111 1) 2r101) + (t/assert= (bit-clear 2r111 2) 2r011)) + +(t/deftest test-bit-set + (t/assert= (bit-set 2r111 0) 2r111) + (t/assert= (bit-set 2r000 1) 2r010)) + +(t/deftest test-bit-flip + (t/assert= (bit-flip 2r101 0) 2r100) + (t/assert= (bit-flip 2r101 1) 2r111)) + +(t/deftest test-bit-test + (t/assert= (bit-test 2r101 0) true) + (t/assert= (bit-test 2r101 1) false)) + +(t/deftest test-bit-and + (t/assert= (bit-and 0 0) 0) + (t/assert= (bit-and 2r101 2r101) 2r101) + (t/assert= (bit-and 2r101 2r101) 2r101) + (t/assert= (bit-and 2r101 0) 0)) + +(t/deftest test-bit-or + (t/assert= (bit-or 0 0) 0) + (t/assert= (bit-or 2r101 2r010) 2r111) + (t/assert= (bit-or 2r111 2r010) 2r111) + (t/assert= (bit-or 2r111 2r111) 2r111) + (t/assert= (bit-or 2r101 0) 2r101)) + +(t/deftest test-bit-xor + (t/assert= (bit-xor 0 0) 0) + (t/assert= (bit-xor 2r101 2r010) 2r111) + (t/assert= (bit-xor 2r111 2r010) 2r101) + (t/assert= (bit-xor 2r111 2r111) 2r000) + (t/assert= (bit-xor 2r101 0) 2r101)) + +(t/deftest test-bit-shift-left + (t/assert= (bit-shift-left 0 0) 0) + (t/assert= (bit-shift-left 2r101 0) 2r101) + + (t/assert= (bit-shift-left 0 7) 0) + (t/assert= (bit-shift-left 2r001 2) 2r100) + (t/assert= (bit-shift-left 2r111 2) 2r11100)) + +(t/deftest test-bit-shift-right + (t/assert= (bit-shift-right 0 0) 0) + (t/assert= (bit-shift-right 2r101 0) 2r101) + + (t/assert= (bit-shift-right 0 7) 0) + (t/assert= (bit-shift-right 2r001 2) 0) + (t/assert= (bit-shift-right 2r111 2) 2r001) + (t/assert= (bit-shift-right 2r1011010 2) 2r10110)) From 6e3529f0708edb55da7489396c41be3e9d2c7b69 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 6 Nov 2014 21:38:27 +0100 Subject: [PATCH 147/909] add bit-not --- pixie/vm/bits.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/vm/bits.py b/pixie/vm/bits.py index 84500dd1..3154f859 100644 --- a/pixie/vm/bits.py +++ b/pixie/vm/bits.py @@ -20,6 +20,11 @@ def bit_flip(x, n): affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") return rt.wrap(x.int_val() ^ (1 << n.int_val())) +@as_var("bit-not") +def bit_not(x): + affirm(isinstance(x, Integer), u"x must be an Integer") + return rt.wrap(~x.int_val()) + @as_var("bit-test") def bit_test(x, n): affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") From 2a430d6eae423b25caabdb7c5d5670786e4f192e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 6 Nov 2014 21:55:12 +0100 Subject: [PATCH 148/909] add array tests --- tests/test-arrays.lisp | 45 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 tests/test-arrays.lisp diff --git a/tests/test-arrays.lisp b/tests/test-arrays.lisp new file mode 100644 index 00000000..5d7e2834 --- /dev/null +++ b/tests/test-arrays.lisp @@ -0,0 +1,45 @@ +(ns pixie.test.test-arrays + (require pixie.test :as t)) + +(t/deftest test-array-creation + (let [a (make-array 10)] + (t/assert= (count a) 10) + (t/assert= (alength a) 10) + (foreach [x a] + (t/assert= x nil)))) + +(t/deftest test-aget-and-aset + (let [a (make-array 10)] + (dotimes [i 10] + (t/assert= (aget a i) nil)) + + (dotimes [i 10] + (aset a i i)) + + (dotimes [i 10] + (t/assert= (aget a i) i)))) + +(t/deftest test-aconcat + (let [a1 (make-array 10) + a2 (make-array 10)] + (t/assert= (alength (aconcat a1 a2)) (+ (alength a1) (alength a2))) + + (dotimes [i 10] + (aset a1 i i) + (aset a2 i (+ 10 i))) + + (let [a3 (aconcat a1 a2)] + (dotimes [i 20] + (t/assert= (aget a3 i) i))))) + +(t/deftest test-aslice + (let [a (make-array 10)] + (dotimes [i 10] + (aset a i i)) + + (let [a1 (aslice a 3) + a2 (aslice a 7)] + (foreach [i (range 0 7)] + (t/assert= (aget a1 i) (+ i 3))) + (foreach [i (range 0 3)] + (t/assert= (aget a2 i) (+ i 7)))))) From a8f7447c9bed6320c80c806e2ec547ef1b613c7a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 6 Nov 2014 16:48:19 -0700 Subject: [PATCH 149/909] very rough wip, but run_blocking works --- pixie/stdlib.lisp | 6 +++ pixie/vm/code.py | 4 -- pixie/vm/libs/c/uv_ffi.c | 30 ++++++------- pixie/vm/libs/uv.py | 47 ++++++++++++++++---- pixie/vm/stacklet.py | 92 +++++++++++++++++++++------------------- 5 files changed, 105 insertions(+), 74 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e729d965..366b67ed 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -1,5 +1,11 @@ (__ns__ pixie.stdlib) + (def libc (ffi-library pixie.platform/lib-c-name)) + (def exit (ffi-fn libc "exit" [Integer] Integer)) + (def puts (ffi-fn libc "puts" [String] Integer)) + + (run_blocking puts "heyoooo-------------") + (def reset! -reset!) (def load-paths (atom ["./"])) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 861f44cb..e0f36ba7 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -112,10 +112,6 @@ def get_bytecode(self): def stack_size(self): return 0 - def invoke(self, args): - result = self.invoke(args) - return result - def invoke_with(self, args, this_fn): return self.invoke(args) diff --git a/pixie/vm/libs/c/uv_ffi.c b/pixie/vm/libs/c/uv_ffi.c index 0e589547..03ffaad6 100644 --- a/pixie/vm/libs/c/uv_ffi.c +++ b/pixie/vm/libs/c/uv_ffi.c @@ -1,8 +1,10 @@ #include "uv.h" #include "ffi.h" +#define EXPORT __attribute__((visibility("default"))) + typedef struct cif_desc_t { - ffi_cif cif; + ffi_cif *cif; int abi; int nargs; ffi_type* rtype; @@ -18,39 +20,31 @@ void (*ffi_cb)(ffi_cif *cif); typedef struct work_baton_t { uv_work_t work; - ffi_cif *cif; + cif_desc_t *cif; void *fn_addr; void *exb; + void *result; } work_baton_t; -void ffi_call_impl_any_c(cif_desc_t *cif, void* func_addr, void* exchange_buffer) -{ - void **buffer_array = (void **)exchange_buffer; - for (int i = 0; i < cif->nargs; i += 1) - { - void **data = exchange_buffer + cif->exchange_args[i]; - buffer_array[i] = data; - } - void **result_data = exchange_buffer + cif->exchange_result_libffi; - ffi_call(&cif->cif, func_addr, (void*)result_data, buffer_array); -} - - void do_work(work_baton_t *req) { - ffi_call_impl_any_c(req->cif, req->fn_addr, req->exb); + puts("doing work\n"); + ffi_call(req->cif, req->fn_addr, req->result, req->exb); + puts("work done\n"); } -void* uv_ffi_make_baton() +EXPORT void* uv_ffi_make_baton() { return malloc(sizeof(work_baton_t)); } -int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_addr, void *exb, uv_after_work_cb *after_work) +EXPORT int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_addr, void *exb, void *result, uv_after_work_cb *after_work) { w->cif = cif; w->fn_addr = fn_addr; w->exb = exb; + w->result = result; + puts("queuing work\n"); uv_queue_work(loop, w, do_work, after_work); } \ No newline at end of file diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py index ba7d48fd..21ca39aa 100644 --- a/pixie/vm/libs/uv.py +++ b/pixie/vm/libs/uv.py @@ -40,15 +40,18 @@ def llexternal(*args, **kwargs): uv_timer_cb = lltype.Ptr(lltype.FuncType([uv_timer, rffi.INT], lltype.Void)) -uv_callback = lltype.Ptr(lltype.FuncType([rffi.VOIDP, rffi.INT], lltype.Void)) +uv_callback_t = rffi.CCallback([rffi.VOIDP, rffi.INT], lltype.Void) data_container = {} -def as_cb(T, fn): - return llhelper(T, rffi._make_wrapper_for(T, fn)) +def as_cb(T): + def _inner(fn): + return llhelper(T, fn) + return _inner def _timer_cb(timer_t, status): - print status, " status" + print status, " status\n" + print "done" import pixie.vm.stacklet as stacklet casted = rffi.cast(rffi.INT, timer_t) data = data_container[casted] @@ -61,7 +64,7 @@ def set_timeout(loop, cont, timeout, repeat): data_container[rffi.cast(rffi.INT, timer)] = cont assert not timer_init(loop, timer) print "setting timeout", timeout, repeat - assert not timer_start(timer, as_cb(uv_timer_cb, _timer_cb), timeout, repeat) + assert not timer_start(timer, _timer_cb, timeout, repeat) @@ -69,9 +72,9 @@ def set_timeout(loop, cont, timeout, repeat): run = llexternal("uv_run", [rffi.VOIDP, rffi.INT], rffi.SIZE_T) timer_init = llexternal("uv_timer_init", [rffi.VOIDP, uv_timer], rffi.INT) -timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.INT, rffi.UINT], rffi.INT) +timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.ULONGLONG, rffi.ULONGLONG], rffi.INT) -ffi_run = llexternal("uv_ffi_run", [rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, uv_callback], rffi.INT) +ffi_run = llexternal("uv_ffi_run", [rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, uv_callback_t], rffi.INT) ffi_make_baton = llexternal("uv_ffi_make_baton", [], rffi.VOIDP) RUN_DEFAULT = 0 @@ -89,6 +92,18 @@ def __init__(self, time): def execute_uv(self, loop, k): set_timeout(loop, k, self._time, 0) + +def _work_cb(baton, status): + print "------------------done\n" + print status, " status\n" + print "done" + import pixie.vm.stacklet as stacklet + casted = rffi.cast(rffi.INT, baton) + data = data_container[casted] + del data_container[casted] + stacklet.pending_stacklets.push(data) + print "timeout completed" + class RunFFIFunc(UVFunction): def __init__(self, fn, args): @@ -100,8 +115,22 @@ def execute_uv(self, loop, k): baton = ffi_make_baton() data_container[rffi.cast(rffi.INT, baton)] = k exb = self._fn.prep_exb(self._args) - - ffi_run(baton, loop, ffi._cd, ffi._f_ptr, exb, as_cb(uv_timer_cb, _timer_cb)) + print "nargs inside", self._fn._cd.nargs + buffer_array = rffi.cast(rffi.VOIDPP, exb) + cif = self._fn._cd + for i in range(cif.nargs): + data = rffi.ptradd(exb, cif.exchange_args[i]) + buffer_array[i] = data + resultdata = rffi.ptradd(exb, + cif.exchange_result_libffi) + + ffi_run(baton, + loop, + rffi.cast(rffi.VOIDP, self._fn._cd), + rffi.cast(rffi.VOIDP, self._fn._f_ptr), + rffi.cast(rffi.VOIDP, exb), + rffi.cast(rffi.VOIDP, resultdata), + _work_cb) @as_var("run_blocking") def _run_blocking(fn, arg): diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index caff6e1e..2af4ed72 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -23,6 +23,9 @@ OP_NEW_THREAD = 0x07 OP_EXECUTE_UV = 0x08 +def to_main_loop(): + global_state._h = global_state._th.switch(global_state._h) + class GlobalState(py_object): def __init__(self): self.reset() @@ -35,11 +38,12 @@ def reset(self): self._op = 0x00 self._fn = None self._init_fn = None - self._from = None + self._current = None + self._parent = None def switch_back(self): - tmp = self._from - self._from = self._to + tmp = self._current + self._current = self._to self._to = tmp @@ -83,16 +87,21 @@ def invoke(self, args): affirm(len(args) == 1, u"Only one arg to continuation allowed") affirm(not self._is_finished, u"Execution of this stacklet has completed") - global_state._from = global_state._to - global_state._to = self + global_state._parent = global_state._current + global_state._current = self global_state._op = OP_SWITCH global_state._val = args[0] - global_state._h = global_state._th.switch(global_state._h) + print "from ", self + to_main_loop() + print "to ", self + + print global_state._val, self._val, self if global_state._val is finished_token: - global_state._from._is_finished = True + global_state._parent._is_finished = True + - global_state._from._val = global_state._val + global_state._parent._val = global_state._val return global_state._val @@ -100,15 +109,14 @@ def invoke(self, args): def new_stacklet(f): global_state._op = OP_NEW global_state._val = f - global_state.switch_back() - global_state._h = global_state._th.switch(global_state._h) + to_main_loop() val = global_state._val return val def new_thread(f): global_state._op = OP_NEW_THREAD global_state._val = f - global_state._h = global_state._th.switch(global_state._h) + to_main_loop() def enqueue_stacklet(k): pending_stacklets.push(k) @@ -116,32 +124,30 @@ def enqueue_stacklet(k): def yield_stacklet(): global_state._op = OP_YIELD global_state._val = nil - global_state._h = global_state._th.switch(global_state._h) + to_main_loop() def execute_uv_func(func): assert isinstance(func, uv.UVFunction) global_state._op = OP_EXECUTE_UV global_state._val = func - global_state._h = global_state._th.switch(global_state._h) + to_main_loop() def new_handler(h, o): global_state._h = h - + parent = global_state._parent affirm(global_state._val is not None, u"Internal Stacklet Error") f = global_state._val global_state._val = None - - global_state._op = OP_SWITCH - global_state.switch_back() - global_state._h = global_state._th.switch(h) + to_main_loop() #try: - f.invoke([global_state._from]) - global_state._from.invoke([finished_token]) - + f.invoke([global_state._parent]) + global_state._parent.invoke([finished_token]) + print "ENDED! Should never see this" + assert False return global_state._h def new_thread_handler(h, o): @@ -151,14 +157,14 @@ def new_thread_handler(h, o): f = global_state._val global_state._val = None - global_state._h = global_state._th.switch(global_state._h) + to_main_loop() #try: - f.invoke([global_state._from]) + f.invoke([global_state._current]) global_state._op = OP_NEXT_PENDING - global_state._h = global_state._th.switch(global_state._h) + to_main_loop() return global_state._h @@ -189,20 +195,19 @@ def with_stacklets(f): global_state._init_fn = f main_h = global_state._th.new(init_handler) - global_state._from = WrappedHandler(main_h) - - pending_stacklets.push(1) - assert pending_stacklets.pop() == 1 - pending_stacklets.push(2) - assert pending_stacklets.pop() == 2 + global_state._current = WrappedHandler(main_h) while True: - print global_state._op + print "OP - ", global_state._op if global_state._op == OP_NEW: - wh = WrappedHandler(global_state._th.get_null_handle()) - global_state._to = wh - wh._h = global_state._th.new(new_handler) + assert global_state._current + global_state._parent = global_state._current + new_h = global_state._th.new(new_handler) + wh = WrappedHandler(new_h) global_state._val = wh + assert global_state._parent + global_state._current = global_state._parent + global_state._op = OP_SWITCH continue elif global_state._op == OP_NEW_THREAD: @@ -210,13 +215,13 @@ def with_stacklets(f): global_state._to = wh wh._h = global_state._th.new(new_thread_handler) pending_stacklets.push(wh) - pending_stacklets.push(global_state._from) + pending_stacklets.push(global_state._current) global_state._op = OP_NEXT_PENDING continue elif global_state._op == OP_SWITCH: - to = global_state._to - to._h = global_state._th.switch(global_state._to._h) + cur = global_state._current + cur._h = global_state._th.switch(cur._h) continue elif global_state._op == OP_EXIT: @@ -224,14 +229,14 @@ def with_stacklets(f): return global_state._val elif global_state._op == OP_YIELD: - assert global_state._from - pending_stacklets.push(global_state._from) + assert global_state._current + pending_stacklets.push(global_state._current) global_state._op = OP_NEXT_PENDING continue elif global_state._op == OP_EXECUTE_UV: - assert global_state._from - k = global_state._from + assert global_state._current + k = global_state._current f = global_state._val f.execute_uv(loop, k) global_state._op = OP_NEXT_PENDING @@ -247,12 +252,13 @@ def with_stacklets(f): uv.run(loop, uv.RUN_DEFAULT | uv.RUN_NO_WAIT) break f = pending_stacklets.pop() - global_state._from = f + assert f + global_state._current = f f._h = global_state._th.switch(f._h) continue else: - break + assert False shutdown() return None From a35889b778e203e3614ea790c88ffb119eb741ba Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 6 Nov 2014 21:59:50 -0700 Subject: [PATCH 150/909] reading stdlib now happens at runtime --- Makefile | 2 +- pixie/vm/rt.py | 49 +++++++++++++++++++++++++++---------------------- target.py | 31 ++++++++++++++++++++++++++++++- 3 files changed, 58 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 6c8c053f..c1f44ef1 100644 --- a/Makefile +++ b/Makefile @@ -37,5 +37,5 @@ run_interactive: run_built_tests: pixie-vm ./pixie-vm run-tests.lisp -run_interpreted_tests: pixie-vm +run_interpreted_tests: target.py PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.lisp diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 16ac308a..ca753c62 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -17,7 +17,7 @@ def init(): _type_registry.set_registry(code._ns_registry) def unwrap(fn): - if isinstance(fn, code.Var) and hasattr(fn.deref(), "_returns"): + if isinstance(fn, code.Var) and fn.is_defined() and hasattr(fn.deref(), "_returns"): tp = fn.deref()._returns if tp is bool: def wrapper(*args): @@ -124,26 +124,32 @@ def reinit(): else: globals()[name] = var - f = open("pixie/stdlib.lisp") - data = f.read() - f.close() - rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") - result = nil - - @wrap_fn - def run_load_stdlib(): - with compiler.with_ns(u"pixie.stdlib"): - while True: - form = reader.read(rdr, False) - if form is reader.eof: - return result - result = compiler.compile(form).invoke([]) - reinit() - - stacklet.with_stacklets(run_load_stdlib) - - - + #f = open("pixie/stdlib.lisp") + #data = f.read() + #f.close() + #rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") + #result = nil + # + # @wrap_fn + # def run_load_stdlib(): + # with compiler.with_ns(u"pixie.stdlib"): + # while True: + # form = reader.read(rdr, False) + # if form is reader.eof: + # return result + # result = compiler.compile(form).invoke([]) + # reinit() + # + # stacklet.with_stacklets(run_load_stdlib) + + + init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals"] + for x in init_fns: + globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) + + init_vars = [u"load-paths"] + for x in init_vars: + globals()[py_str(code.munge(x))] = code.intern_var(u"pixie.stdlib", x) globals()["__inited__"] = True @@ -152,4 +158,3 @@ def run_load_stdlib(): - diff --git a/target.py b/target.py index 8912ef1f..24a0c9bf 100644 --- a/target.py +++ b/target.py @@ -112,10 +112,39 @@ def inner_invoke(self, args): interpret(compile(read(StringReader(unicode(self._expr)), True))) +@wrap_fn +def run_load_stdlib(): + import pixie.vm.compiler as compiler + import pixie.vm.reader as reader + f = open("pixie/stdlib.lisp") + data = f.read() + f.close() + rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") + result = nil + + with compiler.with_ns(u"pixie.stdlib"): + while True: + form = reader.read(rdr, False) + if form is reader.eof: + return result + result = compiler.compile(form).invoke([]) + +def load_stdlib(): + + + + + + + + stacklet.with_stacklets(run_load_stdlib) + def entry_point(args): interactive = True script_args = [] + load_stdlib() + i = 1 while i < len(args): arg = args[i] @@ -206,7 +235,7 @@ def run_debug(argv): import pixie.vm.rt as rt rt.init() -stacklet.global_state = stacklet.GlobalState() +#stacklet.global_state = stacklet.GlobalState() def target(*args): import pixie.vm.rt as rt From 944759cd3f39c3796d8f30acb24b6a72a0cf510b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 08:36:47 +0100 Subject: [PATCH 151/909] add ns-map to get the names & vars of a namespace it doesn't go through the refers yet, but i'm not sure if we want that. --- pixie/vm/stdlib.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 7e71e5bc..657162b1 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -366,6 +366,24 @@ def the_ns(ns_name): return code._ns_registry.get(rt.name(ns_name), nil) +@as_var("ns-map") +def ns_map(ns): + from pixie.vm.symbol import Symbol + + affirm(isinstance(ns, code.Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") + + if isinstance(ns, Symbol): + ns = rt.the_ns(ns) + if ns is nil: + return nil + + m = rt.hashmap() + for name in ns._registry: + var = ns._registry.get(name, nil) + m = rt.assoc(m, rt.symbol(rt.wrap(name)), var) + + return m + @as_var("refer") def refer(ns, refer, alias): from pixie.vm.symbol import Symbol From 2edf8e781d8ff1722633f4c11a938d8cb2b9c4a1 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 10:27:09 +0100 Subject: [PATCH 152/909] add when-not --- pixie/stdlib.lisp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e729d965..e7d9353e 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -602,6 +602,9 @@ (defmacro when [test & body] `(if ~test (do ~@body))) +(defmacro when-not [test & body] + `(if (not ~test) (do ~@body))) + (defn abs [x] (if (< x 0) (* -1 x) From 177ae8f1f2828c6b041b5f9dc5721034901711e7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 10:27:32 +0100 Subject: [PATCH 153/909] implement refer as in clojure the previous refer is now `refer-ns`. i'm not sure if i should have used the Refer class, but that would have required more code in rpython, i think. theoretically we could now use this to replace `include_stdlib` with a call to `(refer 'pixie.stdlib :refer :all)`. --- pixie/stdlib.lisp | 26 +++++++++++++++++++++++++- pixie/vm/code.py | 11 +++++++++++ pixie/vm/stdlib.py | 14 ++++++++++++-- 3 files changed, 48 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e7d9353e..83ccb668 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -492,7 +492,7 @@ (defmacro require [ns kw as-nm] (assert (= kw :as) "Require expects :as as the second argument") `(do (load-file (quote ~ns)) - (refer (this-ns-name) (the-ns (quote ~ns)) (quote ~as-nm)))) + (refer-ns (this-ns-name) (the-ns (quote ~ns)) (quote ~as-nm)))) (defmacro ns [nm & body] `(do (__ns__ ~nm) @@ -737,3 +737,27 @@ (if result (xf acc result) acc)))))) + +(defn refer [ns-sym & filters] + (let [ns (or (the-ns ns-sym) (throw (str "No such namespace: " ns-sym))) + filters (apply hashmap filters) + nsmap (ns-map ns) + rename (or (:rename filters) {}) + exclude (set (:exclude filters)) + refers (if (= :all (:refer filters)) + (keys nsmap) + (or (:refer filters) (:only filters) (keys nsmap)))] + (when (and refers (not (satisfies? ISeqable refers))) + (throw ":only/:refer must be a collection of symbols")) + (loop [syms (seq refers)] + (if (not syms) + nil + (do + (let [sym (first syms)] + (when-not (exclude sym) + (let [v (nsmap sym)] + (when-not v + (throw (str sym "does not exist"))) + (refer-symbol *ns* (or (rename sym) sym) v)))) + (recur (next syms))))) + nil)) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 861f44cb..e7315ecc 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -452,6 +452,17 @@ def add_refer(self, ns, as_nm=None, refer_all=False): self._refers[as_nm] = Refer(ns, refer_all=refer_all) + def add_refer_symbol(self, sym, var): + assert isinstance(self, Namespace) + + name = rt.name(sym) + prev_binding = self._registry.get(name, None) + if prev_binding is not None: + print rt.str(rt.wrap(u"Warning: "), sym, rt.wrap(u" already refers to "), prev_binding)._str + + self._registry[name] = var + return var + def include_stdlib(self): stdlib = _ns_registry.find_or_make(u"pixie.stdlib") self.add_refer(stdlib, refer_all=True) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 657162b1..00c00816 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -384,7 +384,7 @@ def ns_map(ns): return m -@as_var("refer") +@as_var("refer-ns") def refer(ns, refer, alias): from pixie.vm.symbol import Symbol @@ -395,6 +395,16 @@ def refer(ns, refer, alias): ns.add_refer(refer, rt.name(alias)) return nil +@as_var("refer-symbol") +def refer_symbol(ns, sym, var): + from pixie.vm.symbol import Symbol + + affirm(isinstance(ns, code.Namespace), u"First argument must be a namespace") + affirm(isinstance(sym, Symbol) and rt.namespace(sym) is None, u"Second argument must be a non-namespaced symbol") + affirm(isinstance(var, code.Var), u"Third argument must be a var") + + ns.add_refer_symbol(sym, var) + return nil @as_var("extend") def _extend(proto_fn, tp, fn): @@ -552,4 +562,4 @@ def _merge__args(args): @extend(_str, RuntimeException) def _str(self): assert isinstance(self, RuntimeException) - return rt.wrap(self.__repr__()) \ No newline at end of file + return rt.wrap(self.__repr__()) From eb8902b51cf48ab6e60d4bbc8636798cbfa587a4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 12:37:04 +0100 Subject: [PATCH 154/909] add more string functions index-of, upper-case, lower-case, capitalize, trim, triml, trimr --- pixie/vm/libs/string.py | 92 ++++++++++++++++++++++++++++++++++++++++- tests/test-strings.lisp | 73 ++++++++++++++++++++++++++++++++ 2 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 tests/test-strings.lisp diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index d2bf3231..69a3db3d 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -1,10 +1,12 @@ import pixie.vm.rt as rt from pixie.vm.string import String -from pixie.vm.code import as_var, extend -from pixie.vm.object import Object, Type +from pixie.vm.code import as_var, extend, intern_var, wrap_fn, MultiArityFn +from pixie.vm.object import affirm, runtime_error, Object, Type import pixie.vm.stdlib as proto from pixie.vm.keyword import keyword +from pixie.vm.numbers import Integer from rpython.rlib.clibffi import get_libc_name +from rpython.rlib.unicodedata import unicodedb_6_2_0 as unicodedb import os import pixie.vm.rt as rt @@ -18,3 +20,89 @@ def startswith(a, b): @as_var("pixie.string", "ends-with") def endswith(a, b): return rt.wrap(rt.name(a).endswith(rt.name(b))) + +# @as_var("pixie.string", "split") +# def split(a, b): +# v = rt.vector() +# for s in rt.name(a).split(rt.name(b)): +# v = rt.conj(v, rt.wrap(s)) +# return v + +def index_of2(a, sep): + return rt.wrap(rt.name(a).find(rt.name(sep))) + +def index_of3(a, sep, start): + affirm(isinstance(start, Integer), u"Third argument must be an integer") + start = start.int_val() + if start > 0: + return rt.wrap(rt.name(a).find(rt.name(sep), start)) + else: + runtime_error(u"Third argument must be a non-negative integer") + +def index_of4(a, sep, start, end): + affirm(isinstance(start, Integer) and isinstance(end, Integer), u"Third and fourth argument must be integers") + start = start.int_val() + end = end.int_val() + if start > 0 and end > 0: + return rt.wrap(rt.name(a).find(rt.name(sep), start, end)) + else: + runtime_error(u"Third and fourth argument must non-negative integers") + +index_of = intern_var(u"pixie.string", u"index-of") +index_of.set_root(MultiArityFn({2: wrap_fn(index_of2), 3: wrap_fn(index_of3), 4: wrap_fn(index_of4)}, + required_arity = 2)) + +@as_var("pixie.string", "upper-case") +def upper_case(a): + a = rt.name(a) + res = "" + for ch in a: + res += chr(unicodedb.toupper(ord(ch))) + return rt.wrap(res) + +@as_var("pixie.string", "lower-case") +def lower_case(a): + a = rt.name(a) + res = "" + for ch in a: + res += chr(unicodedb.tolower(ord(ch))) + return rt.wrap(res) + +@as_var("pixie.string", "capitalize") +def capitalize(a): + a = rt.name(a) + res = u"" + res += unichr(unicodedb.toupper(ord(a[0]))) + res += a[1:] + return rt.wrap(res) + +@as_var("pixie.string", "trim") +def trim(a): + a = rt.name(a) + i = 0 + while i < len(a) and unicodedb.isspace(ord(a[i])): + i += 1 + j = len(a) + while j > 0 and unicodedb.isspace(ord(a[j - 1])): + j -= 1 + if j <= i: + return rt.wrap(u"") + return rt.wrap(a[i:j]) + +@as_var("pixie.string", "triml") +def triml(a): + a = rt.name(a) + i = 0 + while i < len(a) and unicodedb.isspace(ord(a[i])): + i += 1 + return rt.wrap(a[i:len(a)]) + +@as_var("pixie.string", "trimr") +def trimr(a): + a = rt.name(a) + j = len(a) + while j > 0 and unicodedb.isspace(ord(a[j - 1])): + j -= 1 + if j <= 0: + return rt.wrap(u"") + return rt.wrap(a[0:j]) diff --git a/tests/test-strings.lisp b/tests/test-strings.lisp new file mode 100644 index 00000000..378de3d1 --- /dev/null +++ b/tests/test-strings.lisp @@ -0,0 +1,73 @@ +(ns pixie.test.test-strings + (require pixie.test :as t) + (require pixie.string :as s)) + +(t/deftest test-starts-with + (let [s "heyhohuh"] + (t/assert= (s/starts-with s "") true) + (t/assert= (s/starts-with s "hey") true) + (t/assert= (s/starts-with s "heyho") true) + (t/assert= (s/starts-with s s) true) + + (t/assert= (s/starts-with s "ho") false) + (t/assert= (s/starts-with s "foo") false))) + +(t/deftest test-ends-with + (let [s "heyhohuh"] + (t/assert= (s/ends-with s "") true) + (t/assert= (s/ends-with s "huh") true) + (t/assert= (s/ends-with s "hohuh") true) + (t/assert= (s/ends-with s s) true) + + (t/assert= (s/ends-with s "hey") false) + (t/assert= (s/ends-with s "foo") false))) + +(t/deftest test-index-of + (let [s "heyhohuh"] + (t/assert= (s/index-of s "hey") 0) + (t/assert= (s/index-of s "ho") 3) + (t/assert= (s/index-of s "foo") -1) + + (t/assert= (s/index-of s "h" 2) 3) + (t/assert= (s/index-of s "h" 4) 5) + (t/assert= (s/index-of s "hey" 1) -1) + + (t/assert= (s/index-of s "h" 1 2) -1))) + +(t/deftest test-upper-case + (t/assert= (s/lower-case "") "") + (t/assert= (s/upper-case "hey") "HEY") + (t/assert= (s/upper-case "hEy") "HEY") + (t/assert= (s/upper-case "HEY") "HEY") + (t/assert= (s/upper-case "hey?!") "HEY?!")) + +(t/deftest test-lower-case + (t/assert= (s/lower-case "") "") + (t/assert= (s/lower-case "hey") "hey") + (t/assert= (s/lower-case "hEy") "hey") + (t/assert= (s/lower-case "HEY") "hey") + (t/assert= (s/lower-case "HEY?!") "hey?!")) + +(t/deftest test-capitalize + (t/assert= (s/capitalize "timothy") "Timothy") + (t/assert= (s/capitalize "Timothy") "Timothy")) + +(t/deftest test-trim + (t/assert= (s/trim "") "") + (t/assert= (s/trim " ") "") + (t/assert= (s/trim " hey ") "hey") + (t/assert= (s/trim " h ey ") "h ey")) + +(t/deftest test-triml + (t/assert= (s/triml "") "") + (t/assert= (s/triml " ") "") + (t/assert= (s/triml " hey") "hey") + (t/assert= (s/triml " hey ") "hey ") + (t/assert= (s/triml " h ey ") "h ey ")) + +(t/deftest test-trimr + (t/assert= (s/trimr "") "") + (t/assert= (s/trimr " ") "") + (t/assert= (s/trimr "hey ") "hey") + (t/assert= (s/trimr " hey ") " hey") + (t/assert= (s/trimr " h ey ") " h ey")) From 10cf458ade748a8ef0b0ab9cc9781fce0e313a38 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 13:30:04 +0100 Subject: [PATCH 155/909] initial character literal support --- pixie/vm/compiler.py | 6 ++++- pixie/vm/reader.py | 61 ++++++++++++++++++++++++++++++++++++++++---- 2 files changed, 61 insertions(+), 6 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 3623a16a..6c213c16 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -7,7 +7,7 @@ import pixie.vm.symbol as symbol import pixie.vm.code as code from pixie.vm.keyword import Keyword, keyword -from pixie.vm.string import String +from pixie.vm.string import Character, String from pixie.vm.atom import Atom import pixie.vm.stdlib as proto from rpython.rlib.rarithmetic import r_uint @@ -395,6 +395,10 @@ def compile_form(form, ctx): ctx.push_const(form) return + if isinstance(form, Character): + ctx.push_const(form) + return + raise Exception("Can't compile ") def compile_platform_plus(form, ctx): diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index b0a979b9..23331d06 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -10,7 +10,7 @@ import pixie.vm.rt as rt from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR from pixie.vm.libs.readline import _readline -from pixie.vm.string import String +from pixie.vm.string import Character, String from pixie.vm.code import wrap_fn, extend from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP from pixie.vm.persistent_hash_set import EMPTY as EMPTY_SET @@ -184,6 +184,9 @@ def is_whitespace(ch): def is_digit(ch): return ch in u"0123456789" +def is_terminating_macro(ch): + return ch != u"#" and ch != u"'" and ch != u"%" and ch in handlers + def eat_whitespace(rdr): while True: ch = rdr.read() @@ -192,7 +195,6 @@ def eat_whitespace(rdr): rdr.unread(ch) return - class ReaderHandler(py_object): def invoke(self, rdr, ch): pass @@ -277,6 +279,57 @@ def invoke(self, rdr, ch): return rt.wrap(u"".join(acc)) acc.append(v) +def read_token(rdr): + acc = u"" + while True: + ch = rdr.read() + if is_whitespace(ch) or is_terminating_macro(ch): + rdr.unread(ch) + return acc + acc += ch + +def read_unicode_char(rdr, token, offset, length, base): + if len(token) != offset + length: + throw_syntax_error_with_data(rdr, u"Invalid unicode character: \\" + token) + c = 0 + for i in range(offset, offset + length): + d = int(token[i:i+1], base) + c = c * base + d + return c + +class LiteralCharacterReader(ReaderHandler): + def invoke(self, rdr, ch): + token = read_token(rdr) + if len(token) == 1: + return Character(ord(token[0])) + elif token == "newline": + return Character(ord(u"\n")) + elif token == "space": + return Character(ord(u" ")) + elif token == "tab": + return Character(ord(u"\t")) + elif token == "backspace": + return Character(ord(u"\b")) + elif token == "formfeed": + return Character(ord(u"\f")) + elif token == "return": + return Character(ord(u"\r")) + elif token.startswith(u"u"): + c = read_unicode_char(rdr, token, 1, 4, 16) + if c >= 0xd800 and c <= 0xdfff: + throw_syntax_error_with_data(rdr, u"Invalid character constant: \\" + token) + return Character(c) + elif token.startswith(u"o"): + l = len(token) - 1 + if l > 3: + throw_syntax_error_with_data(rdr, u"Invalid octal escape sequence: \\" + token) + c = read_unicode_char(rdr, token, 1, l, 8) + if c > 0377: + throw_syntax_error_with_data(rdr, u"Octal escape sequences must be in range [0, 377]") + return Character(c) + else: + throw_syntax_error_with_data(rdr, u"Unsupported character: \\" + unicode(token)) + class DerefReader(ReaderHandler): def invoke(self, rdr, ch): return rt.cons(symbol(u"-deref"), rt.cons(read(rdr, True), nil)) @@ -429,6 +482,7 @@ def skip_line(self, rdr): u"'": QuoteReader(), u":": KeywordReader(), u"\"": LiteralStringReader(), + u"\\": LiteralCharacterReader(), u"@": DerefReader(), u"`": SyntaxQuoteReader(), u"~": UnquoteReader(), @@ -509,9 +563,6 @@ def read_number(rdr, ch): return parsed return Symbol(joined) -def is_terminating_macro(ch): - return ch != u"#" and ch != u"'" and ch != u"%" and ch in handlers - def read_symbol(rdr, ch): acc = [ch] try: From d09611307dbee482ed05fd0877a4d2d2c61359a4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 06:40:27 -0700 Subject: [PATCH 156/909] more work on async-io --- pixie/stdlib.lisp | 9 ++++++++- pixie/vm/libs/ffi.py | 9 +++++++-- pixie/vm/libs/uv.py | 17 +++++++++++------ pixie/vm/stacklet.py | 12 +++++++----- 4 files changed, 33 insertions(+), 14 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 366b67ed..fec04c6a 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -4,7 +4,14 @@ (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) - (run_blocking puts "heyoooo-------------") + (def libreadline (ffi-library "libreadline.dylib")) + (def readline (ffi-fn libreadline "readline" [String] String)) + (def rand (ffi-fn libc "rand" [Integer] Integer)) + (def srand (ffi-fn libc "srand" [Integer] Integer)) + + (srand 44) + + (puts (str (run_blocking readline "heooo") "<-----")) (def reset! -reset!) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 4dd2619e..a7fa30a4 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -132,6 +132,7 @@ def thaw(self): cd.nargs = len(self._arg_types) cd.rtype = get_clibffi_type(self._ret_type) atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, len(self._arg_types), flavor="raw") + for x in range(len(self._arg_types)): atypes[x] = get_clibffi_type(self._arg_types[x]) @@ -170,13 +171,17 @@ def prep_exb(self, args): offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) return exb + def get_ret_val_from_buffer(self, exb): + offset_p = rffi.ptradd(exb, self._cd.exchange_result_libffi) + ret_val = get_ret_val(offset_p, self._ret_type) + return ret_val + @jit.unroll_safe def _invoke(self, args): exb = self.prep_exb(args) jit_ffi_call(self._cd, self._f_ptr, exb) - offset_p = rffi.ptradd(exb, self._ret_offset) - ret_val = get_ret_val(offset_p, self._ret_type) + ret_val = self.get_ret_val_from_buffer(exb) lltype.free(exb, flavor="raw") return ret_val diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py index 21ca39aa..24b05b66 100644 --- a/pixie/vm/libs/uv.py +++ b/pixie/vm/libs/uv.py @@ -93,15 +93,21 @@ def execute_uv(self, loop, k): set_timeout(loop, k, self._time, 0) +work_data_container = {} + def _work_cb(baton, status): print "------------------done\n" print status, " status\n" print "done" import pixie.vm.stacklet as stacklet casted = rffi.cast(rffi.INT, baton) - data = data_container[casted] - del data_container[casted] - stacklet.pending_stacklets.push(data) + (k, exb, fn) = work_data_container[casted] + del work_data_container[casted] + retval = fn.get_ret_val_from_buffer(exb) + print retval, "<----" + stacklet.pending_stacklets.push((k, retval)) + lltype.free(exb, flavor="raw") + #lltype.free(baton, flavor="raw") print "timeout completed" class RunFFIFunc(UVFunction): @@ -113,10 +119,10 @@ def __init__(self, fn, args): def execute_uv(self, loop, k): baton = ffi_make_baton() - data_container[rffi.cast(rffi.INT, baton)] = k exb = self._fn.prep_exb(self._args) print "nargs inside", self._fn._cd.nargs buffer_array = rffi.cast(rffi.VOIDPP, exb) + work_data_container[rffi.cast(rffi.INT, baton)] = (k, exb, self._fn) cif = self._fn._cd for i in range(cif.nargs): data = rffi.ptradd(exb, cif.exchange_args[i]) @@ -135,8 +141,7 @@ def execute_uv(self, loop, k): @as_var("run_blocking") def _run_blocking(fn, arg): from pixie.vm.stacklet import execute_uv_func - execute_uv_func(RunFFIFunc(fn, [arg])) - return nil + return execute_uv_func(RunFFIFunc(fn, [arg])) @as_var("sleep") def _sleep(ms): diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index 2af4ed72..03844cf0 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -34,7 +34,6 @@ def reset(self): self._th = None self._val = None self._ex = None - self._to = None self._op = 0x00 self._fn = None self._init_fn = None @@ -131,6 +130,7 @@ def execute_uv_func(func): global_state._op = OP_EXECUTE_UV global_state._val = func to_main_loop() + return global_state._val def new_handler(h, o): @@ -251,10 +251,12 @@ def with_stacklets(f): else: uv.run(loop, uv.RUN_DEFAULT | uv.RUN_NO_WAIT) break - f = pending_stacklets.pop() - assert f - global_state._current = f - f._h = global_state._th.switch(f._h) + (k, val) = pending_stacklets.pop() + assert k + print val, "---->" + global_state._val = val + global_state._current = k + k._h = global_state._th.switch(k._h) continue else: From 6f652543b9f70ff3af4a4e1ba573c65f4aa7ddff Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 08:11:30 -0700 Subject: [PATCH 157/909] some cleanup --- make-no-jit | 2 +- pixie/stdlib.lisp | 4 ---- pixie/vm/libs/ring_buffer.py | 15 +++++++++------ pixie/vm/libs/uv.py | 15 +++------------ pixie/vm/stacklet.py | 16 +++------------- 5 files changed, 16 insertions(+), 36 deletions(-) diff --git a/make-no-jit b/make-no-jit index 51decd0f..68874b6d 100755 --- a/make-no-jit +++ b/make-no-jit @@ -1 +1 @@ -python ../externals/pypy/rpython/bin/rpython --continuation --no-shared target.py +python ../externals/pypy/rpython/bin/rpython --continuation --no-shared --lldebug target.py diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index fec04c6a..d84dcd2c 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -9,10 +9,6 @@ (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) - (srand 44) - - (puts (str (run_blocking readline "heooo") "<-----")) - (def reset! -reset!) (def load-paths (atom ["./"])) diff --git a/pixie/vm/libs/ring_buffer.py b/pixie/vm/libs/ring_buffer.py index 70fd0e90..eab8ee8a 100644 --- a/pixie/vm/libs/ring_buffer.py +++ b/pixie/vm/libs/ring_buffer.py @@ -1,13 +1,16 @@ from rpython.rlib.rarithmetic import r_uint +from pixie.vm.primitives import nil + +empty_slot = (nil, nil) class RingBuffer(object): def __init__(self, size): assert isinstance(size, r_uint) - self._array = [None] * size + self._array = [empty_slot] * size self._array_len = size self._length = size - self._head = 0 - self._tail = 0 + self._head = r_uint(0) + self._tail = r_uint(0) def pending(self): return self._array_len - self._length @@ -15,11 +18,11 @@ def pending(self): def pop(self): if not self._length == 0: x = self._array[self._tail] - self._array[self._tail] = None + self._array[self._tail] = empty_slot self._tail = (self._tail + 1) % self._array_len self._length -= 1 return x - return None + return empty_slot def push(self, x): self._array[self._head] = x @@ -27,7 +30,7 @@ def push(self, x): self._length += 1 def unbounded_push(self, x): - if self._length - 1 == 0: + if self._length == 0: self.resize() self.push(x) diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py index 24b05b66..6fd2cc55 100644 --- a/pixie/vm/libs/uv.py +++ b/pixie/vm/libs/uv.py @@ -34,7 +34,6 @@ def llexternal(*args, **kwargs): uv_timer_t = rffi.COpaque("uv_timer_t", compilation_info=compilation_info) -#print rffi.sizeof(uv_timer_t) uv_timer = lltype.Ptr(uv_timer_t) @@ -50,20 +49,16 @@ def _inner(fn): return _inner def _timer_cb(timer_t, status): - print status, " status\n" - print "done" import pixie.vm.stacklet as stacklet casted = rffi.cast(rffi.INT, timer_t) data = data_container[casted] del data_container[casted] - stacklet.pending_stacklets.push(data) - print "timeout completed" + stacklet.pending_stacklets.push((data, nil)) def set_timeout(loop, cont, timeout, repeat): timer = lltype.malloc(uv_timer_t, flavor="raw", track_allocation="false") data_container[rffi.cast(rffi.INT, timer)] = cont assert not timer_init(loop, timer) - print "setting timeout", timeout, repeat assert not timer_start(timer, _timer_cb, timeout, repeat) @@ -96,19 +91,15 @@ def execute_uv(self, loop, k): work_data_container = {} def _work_cb(baton, status): - print "------------------done\n" - print status, " status\n" - print "done" import pixie.vm.stacklet as stacklet casted = rffi.cast(rffi.INT, baton) (k, exb, fn) = work_data_container[casted] del work_data_container[casted] retval = fn.get_ret_val_from_buffer(exb) - print retval, "<----" + stacklet.pending_stacklets.push((k, retval)) lltype.free(exb, flavor="raw") #lltype.free(baton, flavor="raw") - print "timeout completed" class RunFFIFunc(UVFunction): def __init__(self, fn, args): @@ -120,7 +111,7 @@ def __init__(self, fn, args): def execute_uv(self, loop, k): baton = ffi_make_baton() exb = self._fn.prep_exb(self._args) - print "nargs inside", self._fn._cd.nargs + buffer_array = rffi.cast(rffi.VOIDPP, exb) work_data_container[rffi.cast(rffi.INT, baton)] = (k, exb, self._fn) cif = self._fn._cd diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index 03844cf0..31fd759e 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -91,11 +91,8 @@ def invoke(self, args): global_state._op = OP_SWITCH global_state._val = args[0] - print "from ", self to_main_loop() - print "to ", self - print global_state._val, self._val, self if global_state._val is finished_token: global_state._parent._is_finished = True @@ -117,9 +114,6 @@ def new_thread(f): global_state._val = f to_main_loop() -def enqueue_stacklet(k): - pending_stacklets.push(k) - def yield_stacklet(): global_state._op = OP_YIELD global_state._val = nil @@ -198,7 +192,6 @@ def with_stacklets(f): global_state._current = WrappedHandler(main_h) while True: - print "OP - ", global_state._op if global_state._op == OP_NEW: assert global_state._current global_state._parent = global_state._current @@ -214,8 +207,8 @@ def with_stacklets(f): wh = WrappedHandler(global_state._th.get_null_handle()) global_state._to = wh wh._h = global_state._th.new(new_thread_handler) - pending_stacklets.push(wh) - pending_stacklets.push(global_state._current) + pending_stacklets.push((wh, nil)) + pending_stacklets.push((global_state._current, wh)) global_state._op = OP_NEXT_PENDING continue @@ -230,7 +223,7 @@ def with_stacklets(f): elif global_state._op == OP_YIELD: assert global_state._current - pending_stacklets.push(global_state._current) + pending_stacklets.push((global_state._current, nil)) global_state._op = OP_NEXT_PENDING continue @@ -243,7 +236,6 @@ def with_stacklets(f): continue elif global_state._op == OP_NEXT_PENDING: - print "running with", pending_stacklets.pending() while True: if pending_stacklets.pending() == 0: uv.run(loop, uv.RUN_DEFAULT) @@ -252,8 +244,6 @@ def with_stacklets(f): uv.run(loop, uv.RUN_DEFAULT | uv.RUN_NO_WAIT) break (k, val) = pending_stacklets.pop() - assert k - print val, "---->" global_state._val = val global_state._current = k k._h = global_state._th.switch(k._h) From b4a66cc6c390975403d0ae44802d5e04c6ae9ff3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 17:57:11 +0100 Subject: [PATCH 158/909] remove unnecessary unicode casts --- pixie/vm/reader.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 23331d06..0d5295d1 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -303,23 +303,23 @@ def invoke(self, rdr, ch): if len(token) == 1: return Character(ord(token[0])) elif token == "newline": - return Character(ord(u"\n")) + return Character(ord("\n")) elif token == "space": - return Character(ord(u" ")) + return Character(ord(" ")) elif token == "tab": - return Character(ord(u"\t")) + return Character(ord("\t")) elif token == "backspace": - return Character(ord(u"\b")) + return Character(ord("\b")) elif token == "formfeed": - return Character(ord(u"\f")) + return Character(ord("\f")) elif token == "return": - return Character(ord(u"\r")) - elif token.startswith(u"u"): + return Character(ord("\r")) + elif token.startswith("u"): c = read_unicode_char(rdr, token, 1, 4, 16) if c >= 0xd800 and c <= 0xdfff: throw_syntax_error_with_data(rdr, u"Invalid character constant: \\" + token) return Character(c) - elif token.startswith(u"o"): + elif token.startswith("o"): l = len(token) - 1 if l > 3: throw_syntax_error_with_data(rdr, u"Invalid octal escape sequence: \\" + token) @@ -328,7 +328,7 @@ def invoke(self, rdr, ch): throw_syntax_error_with_data(rdr, u"Octal escape sequences must be in range [0, 377]") return Character(c) else: - throw_syntax_error_with_data(rdr, u"Unsupported character: \\" + unicode(token)) + throw_syntax_error_with_data(rdr, u"Unsupported character: \\" + token) class DerefReader(ReaderHandler): def invoke(self, rdr, ch): From 591d8d00fc068f00397522e2a3c15aa45cc94e33 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 17:59:33 +0100 Subject: [PATCH 159/909] fix compilation that this was so difficult to track down is an issue with rpython. trying to fix that next. (thanks #pypy!) --- pixie/vm/reader.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 0d5295d1..c7caaf94 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -293,7 +293,7 @@ def read_unicode_char(rdr, token, offset, length, base): throw_syntax_error_with_data(rdr, u"Invalid unicode character: \\" + token) c = 0 for i in range(offset, offset + length): - d = int(token[i:i+1], base) + d = int(str(token[i:i+1]), base) c = c * base + d return c @@ -302,17 +302,17 @@ def invoke(self, rdr, ch): token = read_token(rdr) if len(token) == 1: return Character(ord(token[0])) - elif token == "newline": + elif token == u"newline": return Character(ord("\n")) - elif token == "space": + elif token == u"space": return Character(ord(" ")) - elif token == "tab": + elif token == u"tab": return Character(ord("\t")) - elif token == "backspace": + elif token == u"backspace": return Character(ord("\b")) - elif token == "formfeed": + elif token == u"formfeed": return Character(ord("\f")) - elif token == "return": + elif token == u"return": return Character(ord("\r")) elif token.startswith("u"): c = read_unicode_char(rdr, token, 1, 4, 16) From b3e17159e70acc3ea241038f20f209c707c3f6e5 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 18:00:39 +0100 Subject: [PATCH 160/909] add character literal tests --- tests/test-strings.lisp | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/tests/test-strings.lisp b/tests/test-strings.lisp index 378de3d1..ca1a5dfd 100644 --- a/tests/test-strings.lisp +++ b/tests/test-strings.lisp @@ -71,3 +71,17 @@ (t/assert= (s/trimr "hey ") "hey") (t/assert= (s/trimr " hey ") " hey") (t/assert= (s/trimr " h ey ") " h ey")) + +(t/deftest test-char-literals + (let [s "hey"] + (t/assert= (nth s 0) \h) + (t/assert= (nth s 0) \o150) + (t/assert= (nth s 0) \u0068) + + (t/assert= (nth s 1) \e) + (t/assert= (nth s 1) \o145) + (t/assert= (nth s 1) \u0065) + + (t/assert= (nth s 2) \y) + (t/assert= (nth s 2) \o171) + (t/assert= (nth s 2) \u0079))) From 0f86c8edca548e68f019d77f658e3a259df86fed Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 10:38:31 -0700 Subject: [PATCH 161/909] yah, run_blocking works when translated --- pixie/stdlib.lisp | 2 ++ pixie/vm/libs/c/uv_ffi.c | 32 ++++++-------------------------- pixie/vm/libs/c/uv_ffi.h | 25 +++++++++++++++++++++++++ pixie/vm/libs/uv.py | 13 +++++++++---- 4 files changed, 42 insertions(+), 30 deletions(-) create mode 100644 pixie/vm/libs/c/uv_ffi.h diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index d8f66348..5750a106 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -9,6 +9,8 @@ (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) + (run_blocking puts "async-io works") + (def reset! -reset!) (def load-paths (atom ["./"])) diff --git a/pixie/vm/libs/c/uv_ffi.c b/pixie/vm/libs/c/uv_ffi.c index 03ffaad6..7bc291e0 100644 --- a/pixie/vm/libs/c/uv_ffi.c +++ b/pixie/vm/libs/c/uv_ffi.c @@ -1,30 +1,10 @@ -#include "uv.h" -#include "ffi.h" +#include "uv_ffi.h" +#include "stdlib.h" +#include "stdio.h" #define EXPORT __attribute__((visibility("default"))) -typedef struct cif_desc_t { - ffi_cif *cif; - int abi; - int nargs; - ffi_type* rtype; - ffi_type** atypes; - int exchange_size; - int exchange_result; - int exchange_result_libffi; - int exchange_args[0]; -} cif_desc_t; -void (*ffi_cb)(ffi_cif *cif); - -typedef struct work_baton_t -{ - uv_work_t work; - cif_desc_t *cif; - void *fn_addr; - void *exb; - void *result; -} work_baton_t; void do_work(work_baton_t *req) @@ -39,12 +19,12 @@ EXPORT void* uv_ffi_make_baton() return malloc(sizeof(work_baton_t)); } -EXPORT int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_addr, void *exb, void *result, uv_after_work_cb *after_work) +EXPORT int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_addr, void *exb, void *result, uv_after_work_cb after_work) { - w->cif = cif; w->fn_addr = fn_addr; + w->cif = cif; w->exb = exb; w->result = result; puts("queuing work\n"); - uv_queue_work(loop, w, do_work, after_work); + return uv_queue_work(loop, (uv_work_t *)w, (uv_work_cb)do_work, after_work); } \ No newline at end of file diff --git a/pixie/vm/libs/c/uv_ffi.h b/pixie/vm/libs/c/uv_ffi.h new file mode 100644 index 00000000..ef0beac5 --- /dev/null +++ b/pixie/vm/libs/c/uv_ffi.h @@ -0,0 +1,25 @@ +#include "uv.h" +#include "ffi.h" + +typedef struct cif_desc_t { + ffi_cif *cif; + int abi; + int nargs; + ffi_type* rtype; + ffi_type** atypes; + int exchange_size; + int exchange_result; + int exchange_result_libffi; + int exchange_args[0]; +} cif_desc_t; + +void (*ffi_cb)(ffi_cif *cif); + +typedef struct work_baton_t +{ + uv_work_t work; + ffi_cif *cif; + void *fn_addr; + void *exb; + void *result; +} work_baton_t; \ No newline at end of file diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py index 6fd2cc55..860a33fe 100644 --- a/pixie/vm/libs/uv.py +++ b/pixie/vm/libs/uv.py @@ -12,16 +12,19 @@ import os import shutil import pixie.vm.libs.ffi as ffi +import rpython.rlib.rgc as rgc pkgpath = py.path.local(__file__).dirpath() srcpath = pkgpath.join("c") shutil.copyfile(str(srcpath / "uv_ffi.c"), str(udir.udir / "uv_ffi.c")) +shutil.copyfile(str(srcpath / "uv_ffi.h"), str(udir.udir / "uv_ffi.h")) compilation_info = ExternalCompilationInfo( - includes=['uv.h', "ffi.h"], + includes=['uv.h', "ffi.h", "uv_ffi.h"], + include_dirs=[srcpath], libraries=["uv", "ffi"], separate_module_files=[udir.udir / "uv_ffi.c"]).merge(ExternalCompilationInfo.from_pkg_config("libffi")) @@ -33,6 +36,7 @@ def llexternal(*args, **kwargs): [("data", rffi.VOIDP)]) uv_timer_t = rffi.COpaque("uv_timer_t", compilation_info=compilation_info) +uv_baton_t = rffi.COpaque("work_baton_t", compilation_info=compilation_info) uv_timer = lltype.Ptr(uv_timer_t) @@ -69,7 +73,7 @@ def set_timeout(loop, cont, timeout, repeat): timer_init = llexternal("uv_timer_init", [rffi.VOIDP, uv_timer], rffi.INT) timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.ULONGLONG, rffi.ULONGLONG], rffi.INT) -ffi_run = llexternal("uv_ffi_run", [rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, uv_callback_t], rffi.INT) +ffi_run = llexternal("uv_ffi_run", [rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, uv_callback_t], rffi.SIZE_T) ffi_make_baton = llexternal("uv_ffi_make_baton", [], rffi.VOIDP) RUN_DEFAULT = 0 @@ -109,7 +113,8 @@ def __init__(self, fn, args): self._args = args def execute_uv(self, loop, k): - baton = ffi_make_baton() + baton = lltype.malloc(uv_baton_t, flavor="raw") + rgc.pin(baton) exb = self._fn.prep_exb(self._args) buffer_array = rffi.cast(rffi.VOIDPP, exb) @@ -123,7 +128,7 @@ def execute_uv(self, loop, k): ffi_run(baton, loop, - rffi.cast(rffi.VOIDP, self._fn._cd), + rffi.cast(rffi.VOIDP, cif), rffi.cast(rffi.VOIDP, self._fn._f_ptr), rffi.cast(rffi.VOIDP, exb), rffi.cast(rffi.VOIDP, resultdata), From 2286656c56ffcd809f6aaa1ad91cfdad30432bb2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 7 Nov 2014 19:06:56 +0100 Subject: [PATCH 162/909] add pixie.string/split somehow the `.split` method on strings doesn't work here, so we have to use the rstring module directly. --- pixie/vm/libs/string.py | 14 ++++++++------ tests/test-strings.lisp | 5 +++++ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 69a3db3d..eb59133b 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -7,6 +7,7 @@ from pixie.vm.numbers import Integer from rpython.rlib.clibffi import get_libc_name from rpython.rlib.unicodedata import unicodedb_6_2_0 as unicodedb +import rpython.rlib.rstring as rstring import os import pixie.vm.rt as rt @@ -21,12 +22,13 @@ def startswith(a, b): def endswith(a, b): return rt.wrap(rt.name(a).endswith(rt.name(b))) -# @as_var("pixie.string", "split") -# def split(a, b): -# v = rt.vector() -# for s in rt.name(a).split(rt.name(b)): -# v = rt.conj(v, rt.wrap(s)) -# return v +@as_var("pixie.string", "split") +def split(a, b): + affirm(rt.count(b) > 0, u"seperator can't be empty") + v = rt.vector() + for s in rstring.split(rt.name(a), rt.name(b)): + v = rt.conj(v, rt.wrap(s)) + return v def index_of2(a, sep): return rt.wrap(rt.name(a).find(rt.name(sep))) diff --git a/tests/test-strings.lisp b/tests/test-strings.lisp index ca1a5dfd..1bdfed4c 100644 --- a/tests/test-strings.lisp +++ b/tests/test-strings.lisp @@ -22,6 +22,11 @@ (t/assert= (s/ends-with s "hey") false) (t/assert= (s/ends-with s "foo") false))) +(t/deftest test-split + (let [s "hey,ho,huh"] + (t/assert= (s/split s ",") ["hey" "ho" "huh"]) + (t/assert= (s/split s "h") ["" "ey," "o," "u" ""]))) + (t/deftest test-index-of (let [s "heyhohuh"] (t/assert= (s/index-of s "hey") 0) From 7e8f0b2f3d0b0e2c8339ffc23de60ccba21f8bfc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 11:16:01 -0700 Subject: [PATCH 163/909] some fixes and the like --- .travis.yml | 3 +++ pixie/stdlib.lisp | 2 -- pixie/vm/libs/c/uv_ffi.c | 11 ----------- pixie/vm/libs/uv.py | 19 +++++++++++++------ 4 files changed, 16 insertions(+), 19 deletions(-) diff --git a/.travis.yml b/.travis.yml index 78f27cda..a535dd4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -3,5 +3,8 @@ script: - ./make-with-jit - make run_built_tests +before_install: + - sudo apt-get install libffi-dev libuv-dev + os: - osx diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 3d3ce023..4e895dd4 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -9,8 +9,6 @@ (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) - (run_blocking puts "async-io works") - (def reset! -reset!) (def load-paths (atom ["./"])) diff --git a/pixie/vm/libs/c/uv_ffi.c b/pixie/vm/libs/c/uv_ffi.c index 7bc291e0..3ff67de4 100644 --- a/pixie/vm/libs/c/uv_ffi.c +++ b/pixie/vm/libs/c/uv_ffi.c @@ -4,19 +4,9 @@ #define EXPORT __attribute__((visibility("default"))) - - - void do_work(work_baton_t *req) { - puts("doing work\n"); ffi_call(req->cif, req->fn_addr, req->result, req->exb); - puts("work done\n"); -} - -EXPORT void* uv_ffi_make_baton() -{ - return malloc(sizeof(work_baton_t)); } EXPORT int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_addr, void *exb, void *result, uv_after_work_cb after_work) @@ -25,6 +15,5 @@ EXPORT int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_a w->cif = cif; w->exb = exb; w->result = result; - puts("queuing work\n"); return uv_queue_work(loop, (uv_work_t *)w, (uv_work_cb)do_work, after_work); } \ No newline at end of file diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py index 860a33fe..a981116e 100644 --- a/pixie/vm/libs/uv.py +++ b/pixie/vm/libs/uv.py @@ -9,6 +9,7 @@ from pixie.vm.code import as_var, extend, NativeFn from pixie.vm.primitives import nil import rpython.tool.udir as udir +from pixie.vm.object import affirm import os import shutil import pixie.vm.libs.ffi as ffi @@ -103,12 +104,11 @@ def _work_cb(baton, status): stacklet.pending_stacklets.push((k, retval)) lltype.free(exb, flavor="raw") - #lltype.free(baton, flavor="raw") + lltype.free(baton, flavor="raw") class RunFFIFunc(UVFunction): def __init__(self, fn, args): - - assert isinstance(fn, ffi.FFIFn) + affirm(isinstance(fn, ffi.FFIFn), u"Can only use blocking-call against a ffi function") self._fn = fn self._args = args @@ -134,10 +134,17 @@ def execute_uv(self, loop, k): rffi.cast(rffi.VOIDP, resultdata), _work_cb) -@as_var("run_blocking") -def _run_blocking(fn, arg): +@as_var("blocking-call") +def _run_blocking__args(args): + affirm(len(args) > 0, u"At least one arg must be supplied to blocking-call") + fn = args[0] + argc = len(args) - 1 + new_args = [None] * argc + for x in range(argc): + new_args[x] = args[x + 1] + from pixie.vm.stacklet import execute_uv_func - return execute_uv_func(RunFFIFunc(fn, [arg])) + return execute_uv_func(RunFFIFunc(fn, new_args)) @as_var("sleep") def _sleep(ms): From ae0c8630b3418022be2704e612d80f9aac0353e1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 11:17:44 -0700 Subject: [PATCH 164/909] fix travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a535dd4b..19409767 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,7 @@ script: - make run_built_tests before_install: - - sudo apt-get install libffi-dev libuv-dev + - sudo apt-get install libffi-dev libuv os: - osx From ed4daa7de9a0a2f4a3e2eb588b2f604e3caf7bd2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 11:28:11 -0700 Subject: [PATCH 165/909] more travis stuff --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 19409767..36d5e811 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,6 +4,8 @@ script: - make run_built_tests before_install: + - sudo apt-add-repository ppa:linuxjedi/ppa + - sudo apt-get update - sudo apt-get install libffi-dev libuv os: From ee151b3a32797bd6ac12f2ade9ff72d19628aa9d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 11:39:47 -0700 Subject: [PATCH 166/909] travis... --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 36d5e811..154bb0f4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -4,7 +4,8 @@ script: - make run_built_tests before_install: - - sudo apt-add-repository ppa:linuxjedi/ppa + - sudo apt-add-repository ppa:linuxjedi/ppa -y + - sudo apt-get update -qq - sudo apt-get update - sudo apt-get install libffi-dev libuv From 0e6615300c9fc51f2eb9e57eab1f37aedaf18c67 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 11:42:16 -0700 Subject: [PATCH 167/909] ...yeah.... --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 154bb0f4..cdd7d2cf 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - sudo apt-add-repository ppa:linuxjedi/ppa -y - sudo apt-get update -qq - sudo apt-get update - - sudo apt-get install libffi-dev libuv + - sudo apt-get install libffi-dev libuv-dev os: - osx From a1e74c0bb15b555a57ef986d632756bd79de3b76 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 12:34:07 -0700 Subject: [PATCH 168/909] fix how readline is loaded --- .travis.yml | 2 +- pixie/stdlib.lisp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index cdd7d2cf..e453c55c 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - sudo apt-add-repository ppa:linuxjedi/ppa -y - sudo apt-get update -qq - sudo apt-get update - - sudo apt-get install libffi-dev libuv-dev + - sudo apt-get install libffi-dev libuv-dev libreadline os: - osx diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 4e895dd4..72a46fba 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -4,7 +4,7 @@ (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) - (def libreadline (ffi-library "libreadline.dylib")) + (def libreadline (ffi-library (str "libreadline." pixie.platform/so-ext))) (def readline (ffi-fn libreadline "readline" [String] String)) (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) From 3b6002939e2c8a4e67d8a82c3500222643bfe36c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 12:40:37 -0700 Subject: [PATCH 169/909] . --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index e453c55c..465c7bb9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - sudo apt-add-repository ppa:linuxjedi/ppa -y - sudo apt-get update -qq - sudo apt-get update - - sudo apt-get install libffi-dev libuv-dev libreadline + - sudo apt-get install libffi-dev libuv-dev os: - osx From f3febcd0cb8adbfbb6a1f617ba27e77ce811e041 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 13:03:58 -0700 Subject: [PATCH 170/909] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d89c10b0..9b339c2f 100644 --- a/README.md +++ b/README.md @@ -104,7 +104,7 @@ However there are a few features of pixie that although may not be uncommon, are ## Contributing -Currently there isn't a whole lot that newcomers can help out with, since the entire codebase is in quite a bit of flux, and the main primitives are still under development. However, we do want to have a very open contribution process. If you have a feature you'd like to implement, submit a PR or file an issue and we'll see what we can do. +We have a very open contribution process. If you have a feature you'd like to implement, submit a PR or file an issue and we'll see what we can do. Most PRs are either rejected (if there is a techincal flaw) or accepted within a day, so send an improvement our way and see what happens. ## Implementation Notes From d7d49d5120139b33392bd64b5d122c95ba26cef5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 14:15:39 -0700 Subject: [PATCH 171/909] added dissoc for maps --- pixie/stdlib.lisp | 5 ++ pixie/vm/persistent_hash_map.py | 112 ++++++++++++++++++++++++++++ pixie/vm/stdlib.py | 2 +- tests/collections/test-maps.lisp | 6 ++ tests/collections/test-vectors.lisp | 4 +- 5 files changed, 126 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 72a46fba..b6c4c7c1 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -319,6 +319,11 @@ ([m k v & rest] (apply assoc (-assoc m k v) rest))) +(defn dissoc + ([m] m) + ([m & ks] + (reduce -dissoc m ks))) + (defn contains? [coll key] (-contains-key coll key)) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 029b1447..8d65bdc4 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -49,6 +49,16 @@ def val_at(self, key, not_found): return not_found if self._root is None else self._root.find(r_uint(0), rt.hash(key) & MASK_32, key, not_found) + def without(self, key): + if self._root is None: + return self + + new_root = self._root.without_inode(0, rt.hash(key) & MASK_32, key) + + if new_root is self._root: + return self + return PersistentHashMap(self._cnt - 1, new_root, self._meta) + @@ -67,6 +77,9 @@ def find(self, shift, hash_val, key, not_found): def reduce_inode(self, f, init): pass + def without(self, shift, hash, key): + pass + def mask(hash, shift): return (hash >> shift) & 0x01f @@ -151,6 +164,7 @@ def find(self, shift, hash_val, key, not_found): return val_or_node return not_found + def reduce_inode(self, f, init): for x in range(0, len(self._array), 2): key_or_none = self._array[x] @@ -163,6 +177,33 @@ def reduce_inode(self, f, init): return init return init + def without_inode(self, shift, hash, key): + bit = bitpos(hash, shift) + if self._bitmap & bit == 0: + return self + + idx = self.index(bit) + key_or_none = self._array[2 * idx] + val_or_node = self._array[2 * idx + 1] + + if key_or_none is None: + n = val_or_node.without_inode(shift + 5, hash, key) + if n is val_or_node: + return self + if n is not None: + return BitmapIndexedNode(None, self._bitmap, clone_and_set(self._array, 2 * idx + 1, n)) + + if self._bitmap == bit: + return None + + return BitmapIndexedNode(None, self._bitmap ^ bit, remove_pair(self._array, idx)) + + if rt.eq(key, key_or_none): + return BitmapIndexedNode(None, self._bitmap ^ bit, remove_pair(self._array, idx)) + + return self + + BitmapIndexedNode_EMPTY = BitmapIndexedNode(None, r_uint(0), []) @@ -185,6 +226,47 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): return self return ArrayNode(None, self._cnt, clone_and_set(self._array, idx, n)) + def without_inode(self, shift, hash, key): + idx = mask(hash, shift) + node = self._array[idx] + if node is None: + return self + n = node.without_inode(shift + 5, hash, key) + if n is node: + return self + if n is None: + if self._cnt <= 8: # shrink + return self.pack(None, idx) + return ArrayNode(None, self._cnt - 1, clone_and_set(self._array, idx, n)) + else: + return ArrayNode(None, self._cnt, clone_and_set(self._array, idx, n)) + + def pack(self, idx): + new_array = [None] * (2 * (self._cnt - 1)) + j = r_uint(1) + bitmap = r_uint(0) + + i = r_uint(0) + while i < idx: + if self._array[i] is not None: + new_array[j] = self._array[i] + bitmap |= 1 << i + j += 2 + + i += 1 + + i = r_uint(idx) + 1 + while i < len(self._array): + if self._array[i] is not None: + new_array[j] = self._array[i] + bitmap |= 1 << i + j += 2 + + i += 1 + + return BitmapIndexedNode(None, bitmap, new_array) + + def find(self, shift, hash_val, key, not_found): idx = mask(hash_val, shift) node = self._array[idx] @@ -231,6 +313,26 @@ def reduce_inode(self, f, init): return init return init + def find_index(self, key): + i = 0 + while i < len(self._array): + if rt.eq(key, self._array[i]): + return i + + i += 2 + + return -1 + + def without(self, shift, hash, key): + idx = self.find_index(key) + if idx == -1: + return self + + if len(self._array) == 1: + return None + + return HashCollisionNode(None, self._hash, remove_pair(self._array, idx / 2)) + def create_node(shift, key1, val1, key2hash, key2, val2): @@ -284,6 +386,11 @@ def clone_and_set2(array, i, a, j, b): clone[j] = b return clone +def remove_pair(array, i): + new_array = [None] * (len(array) - 2) + list_copy(array, 0, new_array, 0, 2 * i) + list_copy(array, 2 * (i + 1), new_array, 2 * i, len(new_array) - (2 * i)) + return new_array ### hook into RT @@ -334,6 +441,11 @@ def _assoc(self, key, val): assert isinstance(self, PersistentHashMap) return self.assoc(key, val) +@extend(proto._dissoc, PersistentHashMap) +def _dissoc(self, key): + assert isinstance(self, PersistentHashMap) + return self.without(key) + proto.IMap.add_satisfies(PersistentHashMap._type) @extend(proto._count, PersistentHashMap) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 00c00816..fa2c63ea 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -33,7 +33,7 @@ defprotocol("pixie.stdlib", "INamed", ["-namespace", "-name"]) -defprotocol("pixie.stdlib", "IAssociative", ["-assoc", "-contains-key"]) +defprotocol("pixie.stdlib", "IAssociative", ["-assoc", "-contains-key", "-dissoc"]) defprotocol("pixie.stdlib", "ILookup", ["-val-at"]) diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp index c886146b..2aa584e4 100644 --- a/tests/collections/test-maps.lisp +++ b/tests/collections/test-maps.lisp @@ -32,3 +32,9 @@ (t/assert= (m (key e)) (val e))) (t/assert= (get m :d) nil) (t/assert= (m :d) nil))) + +(t/deftest map-without + (let [m {:a 1 :b 2}] + (t/assert= m m) + (t/assert= (dissoc m :a) {:b 2}) + (t/assert= (dissoc m :a :b) {}))) diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.lisp index 6ef72691..2e060363 100644 --- a/tests/collections/test-vectors.lisp +++ b/tests/collections/test-vectors.lisp @@ -1,7 +1,7 @@ (ns collections.test-vectors (require pixie.test :as t)) -(def MAX-SIZE 2000) +(def MAX-SIZE 1064) (t/deftest vector-creation (loop [acc []] @@ -29,4 +29,4 @@ (t/assert= (= v [1 2]) false) (t/assert= (= v [1 2 3 4]) false) (t/assert= (= v '(1 2)) false) - (t/assert= (= v '(1 2 3 4)) false))) \ No newline at end of file + (t/assert= (= v '(1 2 3 4)) false))) From 4e5e7bce0c6e17f1e61a087432601bff02b7a76c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 7 Nov 2014 15:33:32 -0700 Subject: [PATCH 172/909] fixes to transient vectors, few other functions --- pixie/stdlib.lisp | 4 ++++ pixie/vm/code.py | 1 + pixie/vm/persistent_vector.py | 2 +- pixie/vm/stdlib.py | 7 +++++++ 4 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index b6c4c7c1..ef2d1d90 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -783,3 +783,7 @@ (refer-symbol *ns* (or (rename sym) sym) v)))) (recur (next syms))))) nil)) + + +(defn vec [coll] + (transduce conj! coll)) \ No newline at end of file diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 099484fc..bb615a3a 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -633,6 +633,7 @@ def get_protocol_fn(self, tp, rev): return promote(fn) def invoke(self, args): + affirm(len(args) >= 1, u"Wrong number of args") a = args[0].type() fn = self.get_protocol_fn(a, self._rev) try: diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 8dbb3381..822cc22b 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -226,7 +226,7 @@ def conj(self, val): new_root = Node(self._root._edit) new_root._array[0] = self._root new_root._array[1] = new_path(self._root._edit, self._shift, tail_node) - new_shift += 1 + new_shift += 5 else: new_root = self.push_tail(self._shift, self._root, tail_node) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index fa2c63ea..c2a0d40d 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -310,6 +310,13 @@ def _satisfies(proto, o): import pixie.vm.rt as rt +@as_var("read-string") +def _read_string(s): + import pixie.vm.reader as reader + + return reader.read(reader.StringReader(unicode(rt.name(s))), True) + + @as_var("load-file") def load_file(filename): import pixie.vm.reader as reader From 1a5cfad563cfdff874ddd88e91689a063880144d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 00:49:38 +0100 Subject: [PATCH 173/909] add function literals quite hairy, but also awesome to have. --- pixie/vm/reader.py | 74 ++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 72 insertions(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index b0a979b9..9913de41 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -28,6 +28,12 @@ GEN_SYM_ENV.set_dynamic() GEN_SYM_ENV.set_value(EMPTY_MAP) +ARG_AMP = symbol(u"&") +ARG_MAX = keyword(u"max-arg") +ARG_ENV = code.intern_var(u"pixie.stdlib.reader", u"*arg-env*") +ARG_ENV.set_dynamic() +ARG_ENV.set_value(nil) + class PlatformReader(object.Object): _type = object.Type(u"PlatformReader") @@ -381,6 +387,68 @@ def invoke(self, rdr, ch): return obj +class ArgReader(ReaderHandler): + def invoke(self, rdr, ch): + if ARG_ENV.deref() is nil: + return read_symbol(rdr, ch) + + ch = rdr.read() + rdr.unread(ch) + if is_whitespace(ch) or is_terminating_macro(ch): + return ArgReader.register_next_arg(1) + + n = read(rdr, True) + if rt.eq(n, ARG_AMP): + return ArgReader.register_next_arg(-1) + if not isinstance(n, numbers.Integer): + throw_syntax_error_with_data(rdr, u"%-arg must be %, % or %&") + return ArgReader.register_next_arg(n.int_val()) + + @staticmethod + def register_next_arg(n): + arg_env = ARG_ENV.deref() + max_arg = rt.get(arg_env, ARG_MAX) + if n > max_arg.int_val(): + arg_env = rt.assoc(arg_env, ARG_MAX, rt.wrap(n)) + arg = ArgReader.gen_arg(n) + arg_env = rt.assoc(arg_env, rt.wrap(n), arg) + ARG_ENV.set_value(arg_env) + return arg + + @staticmethod + def gen_arg(n): + s = unicode(str(n)) + if n == -1: + s = u"_rest" + return rt.gensym(rt.wrap(u"arg" + s + u"__")) + +class FnReader(ReaderHandler): + def invoke(self, rdr, ch): + if ARG_ENV.deref() is not nil: + throw_syntax_error_with_data(rdr, u"Nested #()s are not allowed") + + try: + ARG_ENV.set_value(rt.assoc(EMPTY_MAP, ARG_MAX, rt.wrap(-1))) + rdr.unread(ch) + form = read(rdr, True) + args = EMPTY_VECTOR + percent_args = ARG_ENV.deref() + max_arg = rt.get(percent_args, ARG_MAX) + for i in range(1, max_arg.int_val() + 1): + arg = rt.get(percent_args, rt.wrap(i)) + if arg is nil: + arg = ArgReader.gen_arg(i) + args = rt.conj(args, arg) + + rest_arg = rt.get(percent_args, rt.wrap(-1)) + if rest_arg is not nil: + args = rt.conj(args, ARG_AMP) + args = rt.conj(args, rest_arg) + print "fn_reader: ", rt.name(rt.str(rt.cons(symbol(u"fn"), rt.cons(args, rt.cons(form, nil))))) + return rt.cons(symbol(u"fn"), rt.cons(args, rt.cons(form, nil))) + finally: + ARG_ENV.set_value(nil) + class SetReader(ReaderHandler): def invoke(self, rdr, ch): acc = EMPTY_SET @@ -394,7 +462,8 @@ def invoke(self, rdr, ch): acc = acc.conj(read(rdr, True)) dispatch_handlers = { - u"{": SetReader() + u"{": SetReader(), + u"(": FnReader() } class DispatchReader(ReaderHandler): @@ -434,7 +503,8 @@ def skip_line(self, rdr): u"~": UnquoteReader(), u"^": MetaReader(), u"#": DispatchReader(), - u";": LineCommentReader() + u";": LineCommentReader(), + u"%": ArgReader() } # inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 From 18d214375e9f0c4849198f5c79f194995db659fd Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 00:52:39 +0100 Subject: [PATCH 174/909] cleanup a little --- pixie/vm/reader.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 9913de41..481fc750 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -429,11 +429,14 @@ def invoke(self, rdr, ch): try: ARG_ENV.set_value(rt.assoc(EMPTY_MAP, ARG_MAX, rt.wrap(-1))) + rdr.unread(ch) form = read(rdr, True) + args = EMPTY_VECTOR percent_args = ARG_ENV.deref() max_arg = rt.get(percent_args, ARG_MAX) + for i in range(1, max_arg.int_val() + 1): arg = rt.get(percent_args, rt.wrap(i)) if arg is nil: @@ -444,7 +447,7 @@ def invoke(self, rdr, ch): if rest_arg is not nil: args = rt.conj(args, ARG_AMP) args = rt.conj(args, rest_arg) - print "fn_reader: ", rt.name(rt.str(rt.cons(symbol(u"fn"), rt.cons(args, rt.cons(form, nil))))) + return rt.cons(symbol(u"fn"), rt.cons(args, rt.cons(form, nil))) finally: ARG_ENV.set_value(nil) From 30afa2829e0f3050ada4c26e426164aa9b19e089 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 00:54:50 +0100 Subject: [PATCH 175/909] display wrapped exception if we crash in batch mode i would like to display source information, though, as the repl does. --- target.py | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/target.py b/target.py index 24a0c9bf..27ed99f2 100644 --- a/target.py +++ b/target.py @@ -12,7 +12,7 @@ from rpython.translator.platform import platform from pixie.vm.primitives import nil import sys - +import os class DebugIFace(JitHookInterface): def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations): @@ -93,12 +93,16 @@ def inner_invoke(self, args): PROGRAM_ARGUMENTS.set_root(acc) with with_ns(u"user"): - if self._file == '-': - stdin, _, _ = create_stdio() - code = stdin.read() - interpret(compile(read(StringReader(unicode(code)), True))) - else: - rt.load_file(rt.wrap(self._file)) + try: + if self._file == '-': + stdin, _, _ = create_stdio() + code = stdin.read() + interpret(compile(read(StringReader(unicode(code)), True))) + else: + rt.load_file(rt.wrap(self._file)) + except WrappedException as ex: + print "Error: ", ex._ex.__repr__() + os._exit(1) class EvalFn(NativeFn): def __init__(self, expr): From a89839af0ccd559b7f50e0624e18cd96e0dd591c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 01:10:19 +0100 Subject: [PATCH 176/909] add shebang executable example --- examples/hello-world.lisp | 4 ++++ 1 file changed, 4 insertions(+) create mode 100755 examples/hello-world.lisp diff --git a/examples/hello-world.lisp b/examples/hello-world.lisp new file mode 100755 index 00000000..f4c17502 --- /dev/null +++ b/examples/hello-world.lisp @@ -0,0 +1,4 @@ +#!./pixie-vm + +(puts "hello, world!") +(print "arguments: " program-arguments) From ef4267908fc00d265a89fbff090e306909c84f11 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 01:20:20 +0100 Subject: [PATCH 177/909] provide *1, *2, *3 and *e in the repl --- target.py | 21 ++++++++++++++++++++- 1 file changed, 20 insertions(+), 1 deletion(-) diff --git a/target.py b/target.py index 27ed99f2..c58e7af5 100644 --- a/target.py +++ b/target.py @@ -31,6 +31,14 @@ def jitpolicy(driver): PROGRAM_ARGUMENTS = intern_var(u"pixie.stdlib", u"program-arguments") PROGRAM_ARGUMENTS.set_root(nil) +STAR_1 = intern_var(u"pixie.stdlib", u"*1") +STAR_1.set_root(nil) +STAR_2 = intern_var(u"pixie.stdlib", u"*2") +STAR_2.set_root(nil) +STAR_3 = intern_var(u"pixie.stdlib", u"*3") +STAR_3.set_root(nil) +STAR_E = intern_var(u"pixie.stdlib", u"*e") +STAR_E.set_root(nil) class ReplFn(NativeFn): def __init__(self, args): @@ -55,7 +63,6 @@ def inner_invoke(self, args): PROGRAM_ARGUMENTS.set_root(acc) - rdr = MetaDataReader(PromptReader()) with with_ns(u"user"): while True: @@ -64,9 +71,11 @@ def inner_invoke(self, args): if val is eof: break val = interpret(compile(val)) + self.set_recent_vars(val) except WrappedException as ex: print "Error: ", ex._ex.__repr__() rdr.reset_line() + self.set_error_var(ex._ex) continue if val is keyword(u"exit-repl"): break @@ -74,6 +83,16 @@ def inner_invoke(self, args): assert isinstance(val, String), "str should always return a string" print val._str + def set_recent_vars(self, val): + if rt.eq(val, STAR_1.deref()): + return + STAR_3.set_root(STAR_2.deref()) + STAR_2.set_root(STAR_1.deref()) + STAR_1.set_root(val) + + def set_error_var(self, ex): + STAR_E.set_root(ex) + class BatchModeFn(NativeFn): def __init__(self, args): self._file = args[0] From 0010e4c89bd8af04d72009ff5e88c8da8d90685a Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 01:23:51 +0100 Subject: [PATCH 178/909] make the help a tiny bit prettier --- target.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/target.py b/target.py index c58e7af5..15fa1c5c 100644 --- a/target.py +++ b/target.py @@ -178,9 +178,9 @@ def entry_point(args): return 0 elif arg == '-h' or arg == '--help': print args[0] + " [] []" - print " -h|--help print this help" - print " -v|--version print the version number" - print " -e|--eval evaluate the given expression" + print " -h, --help print this help" + print " -v, --version print the version number" + print " -e, --eval evaluate the given expression" return 0 elif arg == '-e' or arg == '--eval': i += 1 From 2817ae8c78215cf9e2a0c1d1acc2b1f83385cfea Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 01:34:17 +0100 Subject: [PATCH 179/909] add individual predicates for the num types --- pixie/stdlib.lisp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e9ad198b..fa7e6469 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -376,6 +376,10 @@ (defn false? [v] (identical? v false)) (defn number? [v] (instance? Number v)) +(defn integer? [v] (instance? Integer v)) +(defn float? [v] (instance? Float v)) +(defn ratio? [v] (instance? Ratio v)) + (defn string? [v] (instance? String v)) (defn keyword? [v] (instance? Keyword v)) From 1b29984194dce7b3042672903900d035024b9bfa Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 01:42:17 +0100 Subject: [PATCH 180/909] add convenience functions for transients --- pixie/stdlib.lisp | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index fa7e6469..bd288746 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -17,6 +17,9 @@ ([result] (-persistent! result)) ([result item] (-conj! result item)))) +(def transient (fn [coll] (-transient coll))) + +(def persistent! (fn [coll] (-persistent! coll))) (def transduce (fn transduce ([f coll] From 9fcbf16f28d90732abc04b267b2adb5a56052337 Mon Sep 17 00:00:00 2001 From: Michael Rubanov Date: Sat, 8 Nov 2014 05:10:13 +0200 Subject: [PATCH 181/909] Update README.md build dependencies --- README.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.md b/README.md index 9b339c2f..bd08d9c9 100644 --- a/README.md +++ b/README.md @@ -17,6 +17,11 @@ Some planned and implemented features: * Easy FFI (TODO) * Pattern matching (TODO) +## Dependencies + +* libuv +* libffi-dev + ## Building ./checkout-externals From aceb0528197f261b4fba513587253c34ab9dd927 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 10:05:27 +0100 Subject: [PATCH 182/909] implement into --- pixie/stdlib.lisp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index bd288746..ddffc9ab 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -51,6 +51,10 @@ (def reduce (fn [rf init col] (-reduce col rf init))) +(def into (fn [to from] + (if (satisfies? IToTransient to) + (persistent! (reduce conj! (transient to) from)) + (reduce conj to from)))) (def interpose (fn interpose [val] From a1b13c3839f1d53d7db7988aaa0d12fb9807864e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 10:16:54 +0100 Subject: [PATCH 183/909] add tests for fn literals --- tests/test-fns.lisp | 13 +++++++++++++ 1 file changed, 13 insertions(+) create mode 100644 tests/test-fns.lisp diff --git a/tests/test-fns.lisp b/tests/test-fns.lisp new file mode 100644 index 00000000..35d752c0 --- /dev/null +++ b/tests/test-fns.lisp @@ -0,0 +1,13 @@ +(ns pixie.test.test-fns + (require pixie.test :as t)) + +(t/deftest test-fn-literals + (t/assert= (#(+ 3 4)) 7) + (t/assert= (#(+ 3 %) 4) 7) + (t/assert= (#(+ 3 %1) 4) 7) + (t/assert= (#(+ %1 3) 4) 7) + (t/assert= (#(+ %1 %2) 3 4) 7) + (t/assert= (#(- %2 %1) 3 4) 1) + (t/assert= (#(+ %1 %3) 3 'ignored 4) 7) + (t/assert= (#(- %3 %1) 3 'ignored 4) 1) + (t/assert= (#(apply + %1 %2 %&) 1 2 3 4 5) (+ 1 2 3 4 5))) From 7f004af839db7d81e4fc614d8a25613bbb8ef799 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 12:51:11 +0100 Subject: [PATCH 184/909] introduce BigInteger (type, reader and -str/-repr) --- pixie/vm/compiler.py | 3 +++ pixie/vm/numbers.py | 21 +++++++++++++++++++++ pixie/vm/reader.py | 12 ++++++++---- pixie/vm/rt.py | 3 +++ 4 files changed, 35 insertions(+), 4 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 3623a16a..be3801d0 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -332,6 +332,9 @@ def compile_form(form, ctx): if isinstance(form, numbers.Integer): ctx.push_const(form) return + if isinstance(form, numbers.BigInteger): + ctx.push_const(form) + return if isinstance(form, numbers.Float): ctx.push_const(form) return diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index e0743eb2..0afe96f6 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -35,6 +35,19 @@ def type(self): zero_int = Integer(0) one_int = Integer(1) +class BigInteger(Number): + _type = object.Type(u"pixie.stdlib.BigInteger", Number._type) + _immutable_fields_ = ["_bigint_val"] + + def __init__(self, bi_val): + self._bigint_val = bi_val + + def bigint_val(self): + return self._bigint_val + + def type(self): + return self._type + class Float(Number): _type = object.Type(u"pixie.stdlib.Float", Number._type) _immutable_fields_ = ["_float_val"] @@ -275,6 +288,14 @@ def _str(i): def _repr(i): return rt.wrap(unicode(str(i.int_val()))) + @extend(proto._str, BigInteger._type) + def _str(b): + return rt.wrap(unicode(b.bigint_val().format('0123456789', suffix='N'))) + + @extend(proto._repr, BigInteger._type) + def _repr(b): + return rt.wrap(unicode(b.bigint_val().format('0123456789', suffix='N'))) + @extend(proto._str, Float._type) def _str(f): return rt.wrap(unicode(str(f.float_val()))) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index b0a979b9..697c82ac 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -17,6 +17,7 @@ import pixie.vm.stdlib as proto import pixie.vm.compiler as compiler +from rpython.rlib.rbigint import rbigint from rpython.rlib.rsre import rsre_re as re LINE_NUMBER_KW = keyword(u"line-number") @@ -438,9 +439,9 @@ def skip_line(self, rdr): } # inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 -# sign hex oct radix decimal -# 1 2 3 4 5 6 7 -int_matcher = re.compile(u'^([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))$') +# sign hex oct radix decimal biginteger +# 1 2 3 4 5 6 7 8 +int_matcher = re.compile(u'^([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))(N)?$') float_matcher = re.compile(u'^([-+]?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+)?)$') ratio_matcher = re.compile(u'^([-+]?[0-9]+)/([0-9]+)$') @@ -466,7 +467,10 @@ def parse_int(m): else: return None - return rt.wrap(sign * int(str(num), radix)) + if m.group(8): + return rt.wrap(rbigint.fromstr(str(m.group(1) + num), radix)) + else: + return rt.wrap(sign * int(str(num), radix)) def parse_float(m): return rt.wrap(float(str(m.group(0)))) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index ca753c62..405a5610 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -10,6 +10,7 @@ def init(): import pixie.vm.code as code from pixie.vm.object import affirm, _type_registry from rpython.rlib.rarithmetic import r_uint + from rpython.rlib.rbigint import rbigint from pixie.vm.primitives import nil, true, false from pixie.vm.string import String from pixie.vm.object import Object @@ -80,6 +81,8 @@ def wrap(x): return true if x else false if isinstance(x, int): return numbers.Integer(x) + if isinstance(x, rbigint): + return numbers.BigInteger(x) if isinstance(x, float): return numbers.Float(x) if isinstance(x, unicode): From 9eb89a0de10f201b08bb460b8bec457b7da37ca2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 13:40:03 +0100 Subject: [PATCH 185/909] support most operations on bigintegers missing: -quot, -rem and more importantly a proper divison. for that we'd need to extend Ratio, but i'm not sure how we should do that. i don't want to store pixie numbers in the ratio, so maybe it will be a BigRatio. we'll see. --- pixie/vm/numbers.py | 41 +++++++++++++++++++++++++++++++++++------ 1 file changed, 35 insertions(+), 6 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 0afe96f6..350d23bb 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -1,6 +1,7 @@ import pixie.vm.object as object from pixie.vm.primitives import nil, true, false from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rbigint import rbigint import rpython.rlib.jit as jit from pixie.vm.code import DoublePolymorphicFn, extend, Protocol, as_var, wrap_fn import pixie.vm.rt as rt @@ -127,6 +128,26 @@ def define_num_ops(): define_num_ops() +bigint_ops_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) +def _{pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + return rt.wrap({conv1}(a.{get1}()).{op}({conv2}(b.{get2}()))) +""" + +def define_bigint_ops(): + num_classes = [(Integer, "rbigint.fromint", "int_val"), (BigInteger, "", "bigint_val")] + for (c1, conv1, get1) in num_classes: + for (c2, conv2, get2) in num_classes: + if c1 == Integer and c2 == Integer: + continue + for (pfn, op) in [("_add", "add"), ("_sub", "sub"), ("_mul", "mul"), ("_div", "div"), + ("_num_eq", "eq"), ("_lt", "lt"), ("_gt", "gt"), ("_lte", "le"), ("_gte", "ge")]: + code = bigint_ops_tmpl.format(pfn=pfn, op=op, + ty1=c1.__name__, conv1=conv1, get1=get1, + ty2=c2.__name__, conv2=conv2, get2=get2) + exec code + +define_bigint_ops() def gcd(u, v): while v != 0: @@ -218,7 +239,7 @@ def _num_eq(a, b): assert isinstance(a, Ratio) and isinstance(b, Ratio) return true if a.numerator() == b.numerator() and a.denominator() == b.denominator() else false -ratio_op_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) +mixed_op_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) def {pfn}_{ty1}_{ty2}(a, b): assert isinstance(a, {ty1}) and isinstance(b, {ty2}) return rt.{pfn}({conv1}(a), {conv2}(b)) @@ -239,8 +260,11 @@ def to_ratio_conv(c): def to_float(x): if isinstance(x, Float): return x - else: + if isinstance(x, Ratio): return rt.wrap(x.numerator() / float(x.denominator())) + if isinstance(x, BigInteger): + return rt.wrap(x.bigint_val().tofloat()) + assert False def to_float_conv(c): if c == Float: @@ -248,18 +272,23 @@ def to_float_conv(c): else: return "to_float" -def define_ratio_ops(): +def define_mixed_ops(): for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: - code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) exec code for (c1, c2) in [(Float, Ratio), (Ratio, Float)]: for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: - code = ratio_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) + exec code + + for (c1, c2) in [(Float, BigInteger), (BigInteger, Float)]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) exec code -define_ratio_ops() +define_mixed_ops() # def add(a, b): # if isinstance(a, Integer): From b71f2ad2728ea4e4a3265dbe650473254b045cf0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 8 Nov 2014 07:17:47 -0700 Subject: [PATCH 186/909] basic support for byte arrays --- pixie/vm/array.py | 68 ++++++++++++++++++++++++++++++++++++++++++ pixie/vm/rt.py | 4 +++ tests/test-arrays.lisp | 6 ++++ 3 files changed, 78 insertions(+) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 8bd12932..062100cd 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -6,6 +6,8 @@ from pixie.vm.primitives import nil import pixie.vm.stdlib as proto import rpython.rlib.jit as jit +from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import build_int UNROLL_IF_SMALLER_THAN = 8 @@ -90,3 +92,69 @@ def alength(self): def make_array(l): affirm(isinstance(l, Integer), u"l must be an Integer") return Array([nil] * l.int_val()) + + +# ByteArray +ARRAY_OF_UCHAR = lltype.Array(lltype.Char) + +class ByteArray(object.Object): + _type = object.Type(u"pixie.stdlib.ByteArray") + + def __init__(self, size): + self._cnt = size + self._buffer = lltype.malloc(ARRAY_OF_UCHAR, size, flavor="raw") + for x in range(size): + self._buffer[x] = chr(0) + + def type(self): + return ByteArray._type + + + def __del__(self): + lltype.free(self._buffer, flavor="raw") + + + @jit.unroll_safe + def reduce_small(self, f, init): + for x in range(self._cnt): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, rt.wrap(ord(self._buffer[x]))]) + return init + + + def reduce_large(self, f, init): + for x in range(self._cnt): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, rt.wrap(ord(self._buffer[x]))]) + return init + + +@as_var("byte-array") +def _byte_array(size): + assert isinstance(size, Integer) + v = size.r_uint_val() + return ByteArray(v) + +@extend(proto._reduce, ByteArray) +def _reduce(self, f, init): + assert isinstance(self, ByteArray) + if self._cnt > UNROLL_IF_SMALLER_THAN: + return self.reduce_large(f, init) + return self.reduce_small(f, init) + +@extend(proto._nth, ByteArray) +def _nth(self, idx): + assert isinstance(self, ByteArray) + affirm(isinstance(idx, Integer), u"Index must be an integer") + ival = idx.r_uint_val() + if 0 <= ival < self._cnt: + return rt.wrap(ord(self._buffer[ival])) + + return nil + +@extend(proto._count, ByteArray) +def _nth(self): + assert isinstance(self, ByteArray) + return rt.wrap(self._cnt) \ No newline at end of file diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index ca753c62..f5c1c0f0 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -2,6 +2,7 @@ py_list = list py_str = str from rpython.rlib.objectmodel import specialize +from rpython.rtyper.lltypesystem import lltype, rffi @@ -88,6 +89,9 @@ def wrap(x): return String(unicode(x)) if isinstance(x, Object): return x + if x is None: + return nil + affirm(False, u"Bad wrap") globals()["wrap"] = wrap diff --git a/tests/test-arrays.lisp b/tests/test-arrays.lisp index 5d7e2834..fbd55937 100644 --- a/tests/test-arrays.lisp +++ b/tests/test-arrays.lisp @@ -43,3 +43,9 @@ (t/assert= (aget a1 i) (+ i 3))) (foreach [i (range 0 3)] (t/assert= (aget a2 i) (+ i 7)))))) + + +(t/deftest test-byte-array-creation + (let [ba (byte-array 10)] + (t/assert= (vec ba) [0 0 0 0 0 0 0 0 0 0]) + (t/assert= (count ba) 10))) \ No newline at end of file From c4aea28d376429f5d1ac765d9dc2e38a11b5886e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 15:35:29 +0100 Subject: [PATCH 187/909] implement unsigned-bit-shift-right --- pixie/vm/bits.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pixie/vm/bits.py b/pixie/vm/bits.py index 3154f859..d0688672 100644 --- a/pixie/vm/bits.py +++ b/pixie/vm/bits.py @@ -2,6 +2,7 @@ from pixie.vm.object import affirm from pixie.vm.numbers import Integer +from rpython.rlib.rarithmetic import intmask import pixie.vm.rt as rt @@ -60,7 +61,10 @@ def bit_shift_right(x, n): affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") return rt.wrap(x.int_val() >> n.int_val()) -# unsigned-bit-shift-right (sets sign bit to zero) +@as_var("unsigned-bit-shift-right") +def unsigned_bit_shift_right(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(intmask(x.r_uint_val() >> n.int_val())) digits = "0123456789abcdefghijklmnopqrstuvwxyz" From 0b00c6937f5769922cd59e8dc7e7e3a74222a5f4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 16:43:10 +0100 Subject: [PATCH 188/909] fix typos --- pixie/vm/libs/string.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index eb59133b..449abdb6 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -24,7 +24,7 @@ def endswith(a, b): @as_var("pixie.string", "split") def split(a, b): - affirm(rt.count(b) > 0, u"seperator can't be empty") + affirm(rt.count(b) > 0, u"separator can't be empty") v = rt.vector() for s in rstring.split(rt.name(a), rt.name(b)): v = rt.conj(v, rt.wrap(s)) @@ -48,7 +48,7 @@ def index_of4(a, sep, start, end): if start > 0 and end > 0: return rt.wrap(rt.name(a).find(rt.name(sep), start, end)) else: - runtime_error(u"Third and fourth argument must non-negative integers") + runtime_error(u"Third and fourth argument must be non-negative integers") index_of = intern_var(u"pixie.string", u"index-of") index_of.set_root(MultiArityFn({2: wrap_fn(index_of2), 3: wrap_fn(index_of3), 4: wrap_fn(index_of4)}, From 7e29c98192302040402c1aa8a44d77b0bb5b6c0e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 16:43:17 +0100 Subject: [PATCH 189/909] add substring (with an alias to subs) --- pixie/stdlib.lisp | 2 ++ pixie/vm/libs/string.py | 17 +++++++++++++++++ tests/test-strings.lisp | 10 ++++++++++ 3 files changed, 29 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index ef2d1d90..2d1fb5a4 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -485,6 +485,8 @@ (assoc m k (assoc-in (get m k) ks v)) (assoc m k v))))) +(def subs pixie.string/substring) + (defmacro assert ([test] `(if ~test diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 449abdb6..09eecf05 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -54,6 +54,23 @@ def index_of4(a, sep, start, end): index_of.set_root(MultiArityFn({2: wrap_fn(index_of2), 3: wrap_fn(index_of3), 4: wrap_fn(index_of4)}, required_arity = 2)) +def substring2(a, start): + return substring3(a, start, rt._count(a)) + +def substring3(a, start, end): + affirm(isinstance(a, String), u"First argument must be a string") + affirm(isinstance(start, Integer) and isinstance(end, Integer), u"Second and third argument must be integers") + start = start.int_val() + end = end.int_val() + if start >= 0 and end >= 0: + return rt.wrap(rt.name(a)[start:end]) + else: + runtime_error(u"Second and third argument must be non-negative integers") + +substring = intern_var(u"pixie.string", u"substring") +substring.set_root(MultiArityFn({2: wrap_fn(substring2), 3: wrap_fn(substring3)}, + required_arity = 2)) + @as_var("pixie.string", "upper-case") def upper_case(a): a = rt.name(a) diff --git a/tests/test-strings.lisp b/tests/test-strings.lisp index 1bdfed4c..6a622545 100644 --- a/tests/test-strings.lisp +++ b/tests/test-strings.lisp @@ -39,6 +39,16 @@ (t/assert= (s/index-of s "h" 1 2) -1))) +(t/deftest test-substring + (let [s "heyhohuh"] + (t/assert= (s/substring s 0) s) + (t/assert= (s/substring s 3) (s/substring s 3 (count s))) + (t/assert= (s/substring s 0 0) "") + (t/assert= (s/substring s 0 3) "hey") + (t/assert= (s/substring s 3 5) "ho") + (t/assert= (s/substring s 5 8) "huh") + (t/assert= (s/substring s 3 10000) "hohuh"))) + (t/deftest test-upper-case (t/assert= (s/lower-case "") "") (t/assert= (s/upper-case "hey") "HEY") From 019c7d0b6bbdf2d287b3571774fd667f23bac03e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 16:43:49 +0100 Subject: [PATCH 190/909] remove old test namespace --- pixie/test-tests.lisp | 3 --- 1 file changed, 3 deletions(-) delete mode 100644 pixie/test-tests.lisp diff --git a/pixie/test-tests.lisp b/pixie/test-tests.lisp deleted file mode 100644 index f3c4d784..00000000 --- a/pixie/test-tests.lisp +++ /dev/null @@ -1,3 +0,0 @@ -(ns pixie.test-tests - (require pixie.test :as t)) - From b2f72099a0a663f2a6420e73dc2414e5625358d9 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 17:17:36 +0100 Subject: [PATCH 191/909] show failing expression and result in assert= --- pixie/test.lisp | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pixie/test.lisp b/pixie/test.lisp index 0452653e..aaf26cc8 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -52,6 +52,14 @@ (load-file fullpath))))))))) -(defn assert= [x y] - (assert (= x y) (str x " != " y))) - +(defmacro assert= [x y] + `(let [xr# ~x + yr# ~y] + (assert (= xr# yr#) (str (show '~x xr#) " != " (show '~y yr#))))) + +(defn show + ([val] (if (instance? String val) (-repr val) val)) + ([orig res] + (if (= orig res) + (show orig) + (str (show orig) " = " (show res))))) From 5e96a2c224ffd8d5957d9105d1c20469d03129e3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 17:21:21 +0100 Subject: [PATCH 192/909] provide a friendlier variant of assert similar to assert= --- pixie/test.lisp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/test.lisp b/pixie/test.lisp index aaf26cc8..661e9331 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -57,6 +57,10 @@ yr# ~y] (assert (= xr# yr#) (str (show '~x xr#) " != " (show '~y yr#))))) +(defmacro assert [x] + `(let [x# ~x] + (assert x# (str '~x " is " x#)))) + (defn show ([val] (if (instance? String val) (-repr val) val)) ([orig res] From 23772251db4b471683d1506abd6d5933660f01a3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 17:29:21 +0100 Subject: [PATCH 193/909] add missing IMeta extend for PersistentList --- pixie/vm/persistent_list.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pixie/vm/persistent_list.py b/pixie/vm/persistent_list.py index 0b490256..a473b231 100644 --- a/pixie/vm/persistent_list.py +++ b/pixie/vm/persistent_list.py @@ -76,4 +76,12 @@ def list__args(args): i -= 1 return acc +@extend(proto._meta, PersistentList) +def _meta(self): + assert isinstance(self, PersistentList) + return self.meta() +@extend(proto._with_meta, PersistentList) +def _with_meta(self, meta): + assert isinstance(self, PersistentList) + return self.with_meta(meta) From 0c732bfaaaa68c5e6bbecbb4060b98f5256db219 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 17:29:49 +0100 Subject: [PATCH 194/909] add -> and ->> --- pixie/stdlib.lisp | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 2d1fb5a4..d033ac70 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -214,6 +214,28 @@ (set-macro! defmacro) +(defmacro -> + [x & forms] + (loop [x x, forms forms] + (if forms + (let [form (first forms) + threaded (if (seq? form) + (with-meta `(~(first form) ~x ~@(next form)) (meta form)) + (list form x))] + (recur threaded (next forms))) + x))) + +(defmacro ->> + [x & forms] + (loop [x x, forms forms] + (if forms + (let [form (first forms) + threaded (if (seq? form) + (with-meta `(~(first form) ~@(next form) ~x) (meta form)) + (list form x))] + (recur threaded (next forms))) + x))) + (defn not [x] (if x false true)) @@ -788,4 +810,4 @@ (defn vec [coll] - (transduce conj! coll)) \ No newline at end of file + (transduce conj! coll)) From c166a6d7d6dad9905ae1f37dcc9e861bade492b7 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 8 Nov 2014 10:35:09 -0700 Subject: [PATCH 195/909] bad Tim! run tests before pushing --- pixie/vm/rt.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index f5c1c0f0..124dac2e 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -6,11 +6,12 @@ + def init(): import pixie.vm.code as code from pixie.vm.object import affirm, _type_registry - from rpython.rlib.rarithmetic import r_uint + from rpython.rlib.rarithmetic import r_uint, intmask from pixie.vm.primitives import nil, true, false from pixie.vm.string import String from pixie.vm.object import Object @@ -77,6 +78,8 @@ def wrapper(*args): @specialize.argtype(0) def wrap(x): + if isinstance(x, r_uint): + return numbers.Integer(intmask(x)) if isinstance(x, bool): return true if x else false if isinstance(x, int): From 63a55b861396b2276b411bfe4489eecab164b8f9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 8 Nov 2014 10:43:24 -0700 Subject: [PATCH 196/909] fixed some more errors --- pixie/vm/rt.py | 2 +- tests/collections/test-vectors.lisp | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 2c028a40..e1aa9d3c 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -11,7 +11,7 @@ def init(): import pixie.vm.code as code from pixie.vm.object import affirm, _type_registry - from rpython.rlib.rarithmetic import r_uint + from rpython.rlib.rarithmetic import r_uint, intmask from rpython.rlib.rbigint import rbigint from pixie.vm.primitives import nil, true, false from pixie.vm.string import String diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.lisp index 2e060363..2299ee77 100644 --- a/tests/collections/test-vectors.lisp +++ b/tests/collections/test-vectors.lisp @@ -3,6 +3,8 @@ (def MAX-SIZE 1064) +(comment +;; Takes forever in interpreted mode but useful for debugging (t/deftest vector-creation (loop [acc []] (if (= (count acc) MAX-SIZE) @@ -10,7 +12,7 @@ (do (dotimes [j (count acc)] (t/assert= j (nth acc j))) (recur (conj acc (count acc))))))) - +) (t/deftest vector-contains (let [v [1 2 3] c [0 1 2] From ef127d573149a051602c1e30fc38ffcd39a5f882 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 8 Nov 2014 18:43:26 +0100 Subject: [PATCH 197/909] add last and butlast --- pixie/stdlib.lisp | 12 ++++++++++++ tests/test-stdlib.lisp | 25 +++++++++++++++++++++++++ 2 files changed, 37 insertions(+) create mode 100644 tests/test-stdlib.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index d033ac70..158edb48 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -421,6 +421,18 @@ (defn indexed? [v] (satisfies? IIndexed v)) (defn counted? [v] (satisfies? ICounted v)) +(defn last [coll] + (if (next coll) + (recur (next coll)) + (first coll))) + +(defn butlast [coll] + (loop [res [] + coll coll] + (if (next coll) + (recur (conj res (first coll)) (next coll)) + (seq res)))) + (extend -count MapEntry (fn [self] 2)) (extend -nth MapEntry (fn [self idx not-found] (cond (= idx 0) (-key self) diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.lisp new file mode 100644 index 00000000..c4f71a4f --- /dev/null +++ b/tests/test-stdlib.lisp @@ -0,0 +1,25 @@ +(ns pixie.tests.test-stdlib + (require pixie.test :as t)) + +(t/deftest test-last + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (t/assert= (last nil) nil) + (t/assert= (last []) nil) + (t/assert= (last (range 0 0)) nil) + (t/assert= (last v) 5) + (t/assert= (last l) 5) + (t/assert= (last r) 5))) + +(t/deftest test-butlast + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6) + res '(1 2 3 4)] + (t/assert= (butlast nil) nil) + (t/assert= (butlast []) nil) + (t/assert= (butlast (range 0 0)) nil) + (t/assert= (butlast v) res) + (t/assert= (butlast l) res) + (t/assert= (butlast r) res))) From 505c528c52f8e72c0f1b1eca988703a2c3a2590f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 8 Nov 2014 14:08:00 -0700 Subject: [PATCH 198/909] should fix issue #50 --- pixie/vm/array.py | 5 +++-- pixie/vm/rt.py | 2 -- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 062100cd..c3cadcbc 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -7,6 +7,7 @@ import pixie.vm.stdlib as proto import rpython.rlib.jit as jit from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rlib.rarithmetic import intmask from rpython.rlib.rarithmetic import build_int UNROLL_IF_SMALLER_THAN = 8 @@ -155,6 +156,6 @@ def _nth(self, idx): return nil @extend(proto._count, ByteArray) -def _nth(self): +def _count(self): assert isinstance(self, ByteArray) - return rt.wrap(self._cnt) \ No newline at end of file + return rt.wrap(intmask(self._cnt)) \ No newline at end of file diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index e1aa9d3c..8a5f958d 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -79,8 +79,6 @@ def wrapper(*args): @specialize.argtype(0) def wrap(x): - if isinstance(x, r_uint): - return numbers.Integer(intmask(x)) if isinstance(x, bool): return true if x else false if isinstance(x, int): From 6c60a3f4085584d68e43bdb34621dfe853ffbde9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 8 Nov 2014 15:47:45 -0700 Subject: [PATCH 199/909] Update .travis.yml --- .travis.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.travis.yml b/.travis.yml index 465c7bb9..a5788a52 100644 --- a/.travis.yml +++ b/.travis.yml @@ -11,3 +11,6 @@ before_install: os: - osx + +notifications: + irc: "chat.freenode.net#pixie-lang" From 2d2b7ed1b3bbb36bde9a061b63c12bd4f4d268d8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 01:28:09 +0100 Subject: [PATCH 200/909] implement meta/with-meta on sets --- pixie/vm/persistent_hash_set.py | 23 ++++++++++++++++++++--- tests/collections/test-sets.lisp | 6 ++++++ 2 files changed, 26 insertions(+), 3 deletions(-) diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 0e7eea4a..bc3ea9fa 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -16,13 +16,20 @@ class PersistentHashSet(object.Object): def type(self): return PersistentHashSet._type - def __init__(self, m): + def __init__(self, meta, m): + self._meta = meta self._map = m def conj(self, v): - return PersistentHashSet(self._map.assoc(v, v)) + return PersistentHashSet(self._meta, self._map.assoc(v, v)) -EMPTY = PersistentHashSet(persistent_hash_map.EMPTY) + def meta(self): + return self._meta + + def with_meta(self, meta): + return PersistentHashSet(meta, self._map) + +EMPTY = PersistentHashSet(nil, persistent_hash_map.EMPTY) @as_var("set") def _create(coll): @@ -74,3 +81,13 @@ def _conj(self, v): def _reduce(self, f, init): assert isinstance(self, PersistentHashSet) return rt._reduce(rt.keys(self._map), f, init) + +@extend(proto._meta, PersistentHashSet) +def _meta(self): + assert isinstance(self, PersistentHashSet) + return self.meta() + +@extend(proto._with_meta, PersistentHashSet) +def _with_meta(self, meta): + assert isinstance(self, PersistentHashSet) + return self.with_meta(meta) diff --git a/tests/collections/test-sets.lisp b/tests/collections/test-sets.lisp index 27b02f3b..27f055f0 100644 --- a/tests/collections/test-sets.lisp +++ b/tests/collections/test-sets.lisp @@ -34,3 +34,9 @@ (t/assert= (s -1) nil) (t/assert= (s 4) nil))) + +(t/deftest test-has-meta + (let [m {:has-meta true} + s (with-meta #{} m)] + (t/assert= (meta #{}) nil) + (t/assert= (meta s) m))) From 0967dfeba53aca6b3e9cb04b4352cb33d5abd8b8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 12:24:26 +0100 Subject: [PATCH 201/909] allow :tag metadata e.g. ^String -> {:tag String}. we don't use it (yet? :), but if we ever get "type hinting" to the jit compiler, it will be useful. or if we want to generate code... --- pixie/vm/reader.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 9f33c58a..8a9b6a43 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -436,6 +436,9 @@ def invoke(self, rdr, ch): if isinstance(meta, Keyword): meta = rt.hashmap(meta, true) + if isinstance(meta, Symbol): + meta = rt.hashmap(keyword(u"tag"), meta) + if rt.satisfies_QMARK_(rt.IMeta.deref(), obj): return rt.with_meta(obj, rt.merge(meta, rt.meta(obj))) From 619f492d7b2626f9532e617ab8b761479383c8b2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 13:09:04 +0100 Subject: [PATCH 202/909] support docstrings and meta in defn --- pixie/stdlib.lisp | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index ef2d1d90..db5b1b2b 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -204,7 +204,20 @@ (def concat (fn [& args] (transduce cat conj args))) -(def defn (fn [nm & rest] `(def ~nm (fn ~nm ~@rest)))) +(def key (fn [x] (-key x))) +(def val (fn [x] (-val x))) + +(def defn (fn [nm & rest] + (let [meta (if (instance? String (first rest)) + {:doc (first rest)} + {}) + rest (if (instance? String (first rest)) (next rest) rest) + meta (if (satisfies? IMap (first rest)) + (merge meta (first rest)) + meta) + rest (if (satisfies? IMap (first rest)) (next rest) rest) + nm (with-meta nm meta)] + `(def ~nm (fn ~nm ~@rest))))) (set-macro! defn) (defn defmacro [nm & rest] @@ -405,12 +418,6 @@ (= idx 1) (-val self) :else not-found))) -(defn key [x] - (-key x)) - -(defn val [x] - (-val x)) - (extend -reduce MapEntry indexed-reduce) (extend -str MapEntry From 8b8c6db0601edc0dc29897e9ee27923b97cb74c7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 14:11:14 +0100 Subject: [PATCH 203/909] allow specifying deftype fields as symbols --- pixie/stdlib.lisp | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index db5b1b2b..cab62578 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -537,6 +537,7 @@ (defmacro deftype [nm fields & body] (let [ctor-name (symbol (str "->" (name nm))) + fields (transduce (map (comp keyword name)) conj fields) type-decl `(def ~nm (create-type ~(keyword (name nm)) ~fields)) field-syms (transduce (map (comp symbol name)) conj fields) inst (gensym) From 09a9d7e44bc33d336bef995bdebfc9e944137bd6 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 14:11:37 +0100 Subject: [PATCH 204/909] provide deftype fields in fn bodies automatically --- pixie/stdlib.lisp | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index cab62578..f3237a33 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -549,11 +549,24 @@ conj fields) ~inst)) + mk-body (fn [body] + (let [fn-name (first body) + _ (assert (symbol? fn-name) "protocol override must have a name") + args (second body) + _ (assert (vector? args) "protocol override must have arguments") + self-arg (first args) + _ (assert (symbol? self-arg) "protocol override must have at least one `self' argument") + field-lets (transduce (comp (map (fn [f] + [(symbol (name f)) (list 'get-field self-arg f)])) + cat) + conj fields) + rest (next (next body))] + `(fn ~fn-name ~args (let ~field-lets ~@rest)))) proto-bodies (transduce (map (fn [body] (cond (symbol? body) `(satisfy ~body ~nm) - (seq? body) `(extend ~(first body) ~nm (fn ~@body)) + (seq? body) `(extend ~(first body) ~nm ~(mk-body body)) :else (assert false "Unknown body element in deftype, expected symbol or seq")))) conj body)] From 8c5a2a157b9d5940167bc769c35e50019d1a9115 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 14:12:16 +0100 Subject: [PATCH 205/909] use provided deftype fields --- pixie/stdlib.lisp | 52 ++++++++++++++++++----------------------------- 1 file changed, 20 insertions(+), 32 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index f3237a33..37b5b568 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -656,46 +656,34 @@ (deftype Range [:start :stop :step] IReduce (-reduce [self f init] - (let [start (. self :start) - stop (. self :stop) - step (. self :step)] - (loop [i start - acc init] - (if (or (and (> step 0) (< i stop)) - (and (< step 0) (> i stop)) - (and (= step 0))) - (let [acc (f acc i)] - (if (reduced? acc) - @acc - (recur (+ i step) acc))) - acc)))) + (loop [i start + acc init] + (if (or (and (> step 0) (< i stop)) + (and (< step 0) (> i stop)) + (and (= step 0))) + (let [acc (f acc i)] + (if (reduced? acc) + @acc + (recur (+ i step) acc))) + acc))) IIterable (-iterator [self] - (let [start (. self :start) - stop (. self :stop) - step (. self :step)] - (loop [i start] - (when (or (and (> step 0) (< i stop)) + (loop [i start] + (when (or (and (> step 0) (< i stop)) (and (< step 0) (> i stop)) (and (= step 0))) - (yield i) - (recur (+ i step)))))) + (yield i) + (recur (+ i step))))) ICounted (-count [self] - (let [start (. self :start) - stop (. self :stop) - step (. self :step)] - (if (or (and (< start stop) (< step 0)) - (and (> start stop) (> step 0)) - (= step 0)) - 0 - (abs (quot (- start stop) step))))) + (if (or (and (< start stop) (< step 0)) + (and (> start stop) (> step 0)) + (= step 0)) + 0 + (abs (quot (- start stop) step)))) IIndexed (-nth [self idx] - (let [start (. self :start) - stop (. self :stop) - step (. self :step) - cmp (if (< start stop) < >) + (let [cmp (if (< start stop) < >) val (+ start (* idx step))] (if (cmp val stop) val From c4f9bb402d6608a8de1b51e348ad36e5922c127c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 16:23:00 +0100 Subject: [PATCH 206/909] support calling "methods" of CustomTypes --- pixie/vm/compiler.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index bb56a14d..cf9e8c15 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -309,6 +309,22 @@ def compile_set_literal(form, ctx): compile_cons(set_call, ctx) +def macroexpand(form): + sym = rt.first(form) + if isinstance(sym, symbol.Symbol): + if rt.name(sym).startswith("."): + if rt.count(form) < 2: + raise Exception("malformed dot expression, expecting (.member obj ...)") + + method = rt.keyword(rt.wrap(rt.name(sym)[1:])) + obj = rt.first(rt.next(form)) + dot = rt.symbol(rt.wrap(u".")) + call = rt.cons(dot, rt.cons(obj, rt.cons(method, rt.next(rt.next(form))))) + + return call + + return form + def compile_meta(meta, ctx): ctx.push_const(code.intern_var(u"pixie.stdlib", u'with-meta')) ctx.bytecode.append(code.DUP_NTH) @@ -328,6 +344,7 @@ def compile_form(form, ctx): return if rt.satisfies_QMARK_(rt.ISeq.deref(), form) and form is not nil: + form = macroexpand(form) return compile_cons(form, ctx) if isinstance(form, numbers.Integer): ctx.push_const(form) From c2283f610b0f21499848973746a0c5043c9020a6 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 18:27:12 +0100 Subject: [PATCH 207/909] pass the custom type to "dot" method calls the underlying assumption is that you only want to deftype methods "on" the object itself (and possibly some arguments), not without it. not sure whether this is idiomatic in clojure, though. --- pixie/stdlib.lisp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 37b5b568..fc5a79fc 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -397,7 +397,7 @@ ([obj sym] (get-field obj sym)) ([obj sym & args] - (apply (get-field obj sym) args))) + (apply (get-field obj sym) obj args))) (defn true? [v] (identical? v true)) (defn false? [v] (identical? v false)) From e8f3f6efa8b6bc0f0c55705060ee69d9b34bd90d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 18:34:44 +0100 Subject: [PATCH 208/909] support non-protocol methods in deftype --- pixie/stdlib.lisp | 48 ++++++++++++++++++++------ tests/test-deftype.lisp | 76 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 11 deletions(-) create mode 100644 tests/test-deftype.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index fc5a79fc..a519d148 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -535,20 +535,15 @@ (defmacro lazy-seq [& body] `(lazy-seq* (fn [] ~@body))) +(def Protocol @(resolve (symbol "/Protocol"))) + +(defn protocol? [x] + (instance? Protocol x)) + (defmacro deftype [nm fields & body] (let [ctor-name (symbol (str "->" (name nm))) fields (transduce (map (comp keyword name)) conj fields) - type-decl `(def ~nm (create-type ~(keyword (name nm)) ~fields)) field-syms (transduce (map (comp symbol name)) conj fields) - inst (gensym) - ctor `(defn ~ctor-name ~field-syms - (let [~inst (new ~nm)] - ~@(transduce - (map (fn [field] - `(set-field! ~inst ~field ~(symbol (name field))))) - conj - fields) - ~inst)) mk-body (fn [body] (let [fn-name (first body) _ (assert (symbol? fn-name) "protocol override must have a name") @@ -562,6 +557,37 @@ conj fields) rest (next (next body))] `(fn ~fn-name ~args (let ~field-lets ~@rest)))) + bodies (reduce + (fn [res body] + (cond + (symbol? body) (cond + (= body 'Object) [body (second res) (third res)] + (protocol? @(resolve body)) [@(resolve body) (second res) (third res)] + :else (throw (str "can only extend protocols or Object, not " body))) + (seq? body) (let [proto (first res) tbs (second res) pbs (third res)] + (if (protocol? proto) + [proto tbs (conj pbs body)] + [proto (conj tbs body) pbs])))) + [nil [] []] + body) + type-bodies (second bodies) + proto-bodies (third bodies) + all-fields (reduce (fn [r tb] (conj r (keyword (name (first tb))))) fields type-bodies) + type-decl `(def ~nm (create-type ~(keyword (name nm)) ~all-fields)) + inst (gensym) + ctor `(defn ~ctor-name ~field-syms + (let [~inst (new ~nm)] + ~@(transduce + (map (fn [field] + `(set-field! ~inst ~field ~(symbol (name field))))) + conj + fields) + ~@(transduce + (map (fn [type-body] + `(set-field! ~inst ~(keyword (name (first type-body))) ~(mk-body type-body)))) + conj + type-bodies) + ~inst)) proto-bodies (transduce (map (fn [body] (cond @@ -569,7 +595,7 @@ (seq? body) `(extend ~(first body) ~nm ~(mk-body body)) :else (assert false "Unknown body element in deftype, expected symbol or seq")))) conj - body)] + proto-bodies)] `(do ~type-decl ~ctor ~@proto-bodies))) diff --git a/tests/test-deftype.lisp b/tests/test-deftype.lisp new file mode 100644 index 00000000..b6d92628 --- /dev/null +++ b/tests/test-deftype.lisp @@ -0,0 +1,76 @@ +(ns pixie.test.test-deftype + (require pixie.test :as t)) + +(deftype Simple [:val]) +(deftype Simple2 [val]) + +(t/deftest test-simple + (let [o1 (->Simple 1) + o2 (->Simple 2)] + (foreach [obj-and-val [[o1 1] [o2 2]]] + (let [o (first obj-and-val) + v (second obj-and-val)] + (t/assert= (. o :val) v) + (t/assert= (.val o) v))))) + +(deftype Count [:val] + ICounted + (-count [self] val)) + +(deftype Count2 [val] + ICounted + (-count [self] val)) + +(t/deftest test-extend + (let [o1 (->Count 1) + o2 (->Count 2)] + (foreach [obj-and-val [[o1 1] [o2 2]]] + (let [o (first obj-and-val) + v (second obj-and-val)] + (t/assert= (. o :val) v) + (t/assert= (.val o) v) + (t/assert= (-count o) v) + (t/assert= (count o) v))))) + +(deftype Three [:one :two :three] + Object + (add [self x & args] + (apply + x args)) + (one-plus [self x & xs] + (apply + one x xs)) + ICounted + (-count [self] (+ one two three))) + +(deftype Three2 [one two three] + Object + (add [self x & args] + (apply + x args)) + (one-plus [self x & xs] + (apply + one x xs)) + ICounted + (-count [self] (+ one two three))) + +(t/deftest test-complex + (let [o1 (->Three 1 2 3) + o2 (->Three2 3 4 5)] + (foreach [obj-and-vals [[o1 1 2 3] [o2 3 4 5]]] + (let [o (first obj-and-vals) + one (second obj-and-vals) + two (third obj-and-vals) + three (fourth obj-and-vals)] + (t/assert= (. o :one) one) + (t/assert= (.one o) one) + (t/assert= (. o :two) two) + (t/assert= (.two o) two) + (t/assert= (. o :three) three) + (t/assert= (.three o) three) + + (t/assert= (-count o) (+ one two three)) + (t/assert= (count o) (+ one two three)) + + (t/assert= (.add o 21 21) 42) + (t/assert= (.one-plus o 9) (+ one 9)) + + ; arity-1 (just the self arg) not supported for now + (t/assert= (.add o) (. o :add)) + (t/assert= (.one-plus o) (. o :one-plus)))))) From 98ec6846ea66f3193f58e517ebc98419e07dad54 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 18:36:12 +0100 Subject: [PATCH 209/909] fix possible endless loop when expanding dot expressions --- pixie/vm/compiler.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index cf9e8c15..6d1356d8 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -312,7 +312,8 @@ def compile_set_literal(form, ctx): def macroexpand(form): sym = rt.first(form) if isinstance(sym, symbol.Symbol): - if rt.name(sym).startswith("."): + s = rt.name(sym) + if s.startswith(".") and s != u".": if rt.count(form) < 2: raise Exception("malformed dot expression, expecting (.member obj ...)") From ac6eec34d7c79e74157d697255d51849e0b30519 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 9 Nov 2014 22:33:16 +0100 Subject: [PATCH 210/909] fix deftype satisfies --- pixie/stdlib.lisp | 2 +- tests/test-deftype.lisp | 9 +++++++++ 2 files changed, 10 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index a519d148..607c4ead 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -562,7 +562,7 @@ (cond (symbol? body) (cond (= body 'Object) [body (second res) (third res)] - (protocol? @(resolve body)) [@(resolve body) (second res) (third res)] + (protocol? @(resolve body)) [@(resolve body) (second res) (conj (third res) body)] :else (throw (str "can only extend protocols or Object, not " body))) (seq? body) (let [proto (first res) tbs (second res) pbs (third res)] (if (protocol? proto) diff --git a/tests/test-deftype.lisp b/tests/test-deftype.lisp index b6d92628..49f5e444 100644 --- a/tests/test-deftype.lisp +++ b/tests/test-deftype.lisp @@ -13,6 +13,13 @@ (t/assert= (. o :val) v) (t/assert= (.val o) v))))) +(deftype MagicalVectorMap [] IMap IVector) + +(t/deftest test-satisfies + (let [mvm (->MagicalVectorMap)] + (t/assert (satisfies? IVector mvm)) + (t/assert (satisfies? IMap mvm)))) + (deftype Count [:val] ICounted (-count [self] val)) @@ -29,6 +36,7 @@ v (second obj-and-val)] (t/assert= (. o :val) v) (t/assert= (.val o) v) + (t/assert (satisfies? ICounted o)) (t/assert= (-count o) v) (t/assert= (count o) v))))) @@ -65,6 +73,7 @@ (t/assert= (. o :three) three) (t/assert= (.three o) three) + (t/assert (satisfies? ICounted o)) (t/assert= (-count o) (+ one two three)) (t/assert= (count o) (+ one two three)) From cfc0e1e95862c545fb801c545b700816e06de21e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 10 Nov 2014 10:27:41 +0100 Subject: [PATCH 211/909] implement defrecord --- pixie/stdlib.lisp | 36 ++++++++++++++++++++++++++-- tests/test-defrecord.lisp | 50 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 84 insertions(+), 2 deletions(-) create mode 100644 tests/test-defrecord.lisp diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 607c4ead..7d2a5dac 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -600,6 +600,38 @@ ~ctor ~@proto-bodies))) +(defmacro defrecord [nm fields & body] + (let [ctor-name (symbol (str "->" (name nm))) + map-ctor-name (symbol (str "map" (name ctor-name))) + fields (transduce (map (comp keyword name)) conj fields) + type-from-map `(defn ~map-ctor-name [m] + (apply ~ctor-name (map #(get m %) ~fields))) + default-bodies ['IAssociative + `(-assoc [self k v] + (let [m (reduce (fn [m k] (assoc m k (. self k))) {} ~fields)] + (~map-ctor-name (assoc m k v)))) + `(-contains-key [self k] + (contains? ~(set fields) k)) + `(-dissoc [self k] + (throw "dissoc is not supported on defrecords")) + 'ILookup + `(-val-at [self k not-found] + (if (contains? ~(set fields) k) + (. self k) + not-found)) + 'IObject + `(-str [self] + (str "<" ~(name nm) " " (reduce (fn [m k] (assoc m k (. self k))) {} ~fields) ">")) + `(-eq [self other] + (and (instance? ~nm other) + ~@(map (fn [field] + `(= (. self ~field) (. other ~field))) + fields))) + `(-hash [self] + (throw "not implemented"))] + deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] + `(do ~type-from-map + ~deftype-decl))) (def libc (ffi-library pixie.platform/lib-c-name)) (def exit (ffi-fn libc "exit" [Integer] Integer)) @@ -657,7 +689,7 @@ (defmacro and ([] true) ([x] x) - ([x y] `(if ~x ~y nil)) + ([x y] `(if ~x ~y false)) ([x y & more] `(if ~x (and ~y ~@more)))) (defmacro or @@ -821,4 +853,4 @@ (defn vec [coll] - (transduce conj! coll)) \ No newline at end of file + (transduce conj! coll)) diff --git a/tests/test-defrecord.lisp b/tests/test-defrecord.lisp new file mode 100644 index 00000000..382247d1 --- /dev/null +++ b/tests/test-defrecord.lisp @@ -0,0 +1,50 @@ +(ns pixie.test.test-defrecord + (require pixie.test :as t)) + +(defrecord Three [one two three]) + +(def t1 (->Three 1 2 3)) +(def t2 (->Three 1 2 3)) +(def t3 (map->Three {:one 1, :two 2, :three 3})) +(def t4 (->Three 1 2 4)) +(def t5 (->Three 3 4 5)) + +(t/deftest test-satisfies + (foreach [t [t1 t2 t3 t4 t5]] + (t/assert= t t) + (t/assert (satisfies? IAssociative t)) + (t/assert (satisfies? ILookup t)))) + +(t/deftest test-eq + (t/assert= t1 t2) + (t/assert= t2 t3) + (t/assert= t3 t1) + (foreach [t [t1 t2 t3]] + (t/assert (not (= t (assoc t :one 42)))) + (t/assert (not (= t t4))) + (t/assert (not (= t t5))))) + +(t/deftest test-ilookup + (foreach [t [t1 t2 t3]] + (t/assert (satisfies? ILookup t)) + (t/assert= (get t :one) 1) + (t/assert= (get t :two) 2) + (t/assert= (get t :three) 3) + (t/assert= (get t :oops) nil) + (t/assert= (get t :oops 'not-found) 'not-found))) + +(t/deftest test-iassociative + (foreach [t [t1 t2 t3]] + (t/assert (satisfies? IAssociative t)) + (t/assert= t (assoc t4 :three 3)) + (let [t' (assoc t :one 42) + t-oops (assoc t :oops 'never-found)] + (t/assert (not (= t t'))) + (t/assert= (get t' :one) 42) + (t/assert= t t-oops) + (t/assert (not (contains? t-oops :oops))) + (t/assert= (get t-oops :oops) nil)) + + (t/assert (contains? t :one)) + (t/assert (contains? t :two)) + (t/assert (contains? t :three)))) From 820d573d35a7d83c0a3ce5ef86cfcd10ef6d76e8 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 10 Nov 2014 04:48:46 -0700 Subject: [PATCH 212/909] boost the stack size a bit --- pixie/vm/rt.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 8a5f958d..d81d8e3e 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -48,7 +48,7 @@ def wrapper(*args): return import sys - sys.setrecursionlimit(10000) # Yeah we blow the stack sometimes, we promise it's not a bug + sys.setrecursionlimit(100000) # Yeah we blow the stack sometimes, we promise it's not a bug import pixie.vm.numbers as numbers import pixie.vm.bits as bits From 5006274619a3d0a343885c9a2f475944972ebfc5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 10 Nov 2014 06:53:16 -0700 Subject: [PATCH 213/909] some compiler fixes for recur --- pixie/stdlib.lisp | 174 ++++++++++++++++++++++++++++++++--------- pixie/vm/compiler.py | 3 +- tests/test-stdlib.lisp | 27 +++++++ 3 files changed, 165 insertions(+), 39 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index cf885b56..53a6b928 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -256,10 +256,18 @@ (recur threaded (next forms))) x))) -(defn not [x] +(defn not + {:doc "Inverts the input, if a truthy value is supplied, returns false, otherwise + returns true" + :signatures [[arg]] + :added "0.1"} + [x] (if x false true)) (defn + + {:doc "Adds the arguments" + :signatures [[& args]] + :added "0.1"} ([] 0) ([x] x) ([x y] (-add x y)) @@ -293,6 +301,10 @@ (-rem num div)) (defn = + {:doc "Returns true if all the arguments are equivalent. Otherwise, returns false. Uses + -eq to perform equality checks." + :signatures [[& args]] + :added "0.1"} ([x] true) ([x y] (eq x y)) ([x y & rest] (if (eq x y) @@ -300,6 +312,10 @@ false))) (defn not= + {:doc "Returns true if one (or more) of the arguments are not equivalent to the others. Uses + -eq to perform equality checks." + :signatures [[& args]] + :added "0.1"} ([x] false) ([x y] (not (eq x y))) ([x y & rest] (not (apply = x y rest)))) @@ -332,27 +348,80 @@ (apply >= y rest) false))) -(defn pos? [x] +(defn pos? + {:doc "Returns true if x is greater than zero" + :signatures [[x]] + :added "0.1"} + [x] (> x 0)) -(defn neg? [x] +(defn neg? + {:doc "Returns true if x is less than zero" + :signatures [[x]] + :added "0.1"} + [x] (< x 0)) -(defn zero? [x] +(defn zero? + {:doc "Returns true if x is equal to zero" + :signatures [[x]] + :added "0.1"} + [x] (= x 0)) -(def inc (fn [x] (+ x 1))) - -(def dec (fn [x] (- x 1))) - -(defn second [x] - (first (next x))) - -(defn third [x] - (first (next (next x)))) - -(defn fourth [x] - (first (next (next (next x))))) +(defn inc + {:doc "Increments x by one" + :signatures [[x]] + :added "0.1"} + [x] + (+ x 1)) + +(defn dec + {:doc "Decrements x by one" + :signatures [[x]] + :added "0.1"} + [x] + (- x 1)) + +(defn first + {:doc "Returns the first item in coll, if coll implements IIndexed nth will be used to retreive + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 0) + (-first coll))) + +(defn second + {:doc "Returns the second item in coll, if coll implements IIndexed nth will be used to retreive + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 1) + (first (next coll)))) + +(defn third + {:doc "Returns the third item in coll, if coll implements IIndexed nth will be used to retreive + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 2) + (first (next (next coll))))) + +(defn fourth + {:doc "Returns the fourth item in coll, if coll implements IIndexed nth will be used to retreive + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 3) + (first (next (next (next coll)))))) (defn assoc ([m] m) @@ -369,12 +438,6 @@ (defn contains? [coll key] (-contains-key coll key)) -(def slot-tp (create-type :slot [:val])) - -(defn ->Slot [x] - (let [inst (new slot-tp)] - (set-field! inst :val x))) - (defn get-val [inst] (get-field inst :val)) @@ -473,11 +536,23 @@ (fn [v] (transduce ordered-hash-reducing-fn v))) -(defn keys [m] - (reduce (fn [ks e] (conj ks (key e))) nil m)) - -(defn vals [m] - (reduce (fn [ks e] (conj ks (val e))) nil m)) +(defn keys + {:doc "If called with no arguments returns a transducer that will extract the key from each map entry. If passed + a collection, will assume that it is a hashmap and return a vector of all keys from the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (map key)) + ([m] + (transduce (map key) conj! m))) + +(defn vals + {:doc "If called with no arguments returns a transducer that will extract the key from each map entry. If passed + a collection, will assume that it is a hashmap and return a vector of all keys from the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (map val)) + ([m] + (transduce (map val) conj! m))) (extend -seq PersistentHashMap (fn [m] @@ -722,6 +797,17 @@ nil ~(nth binding 1))) +(defmacro iterate [binding & body] + (assert (= 2 (count binding)) "binding and collection required") + `(let [i# (iterator ~(second binding))] + (loop [] + (if (at-end? i#) + nil + (let [~(first binding) (current i#)] + ~@body + (move-next! i#) + (recur)))))) + (defmacro dotimes [bind & body] (let [b (nth bind 0)] @@ -864,15 +950,21 @@ (xf acc i) acc))))) -(defn keep [f] - (fn [xf] - (fn - ([] (xf)) - ([acc] (xf acc)) - ([acc i] (let [result (f i)] - (if result - (xf acc result) - acc)))))) +(defn keep + ([f] + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (let [result (f i)] + (if result + (xf acc result) + acc)))))) + ([f coll] + (iterate [x coll] + (let [result (f x)] + (if result + (yield x)))))) (defn refer [ns-sym & filters] (let [ns (or (the-ns ns-sym) (throw (str "No such namespace: " ns-sym))) @@ -899,5 +991,11 @@ nil)) -(defn vec [coll] - (transduce conj! coll)) +(defn vec + {:doc "Converts a reducable collection into a vector using the (optional) transducer." + :signatures [[coll] [xform coll]] + :added "0.1"} + ([coll] + (transduce conj! coll)) + ([xform coll] + (transduce xform conj! coll))) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 6d1356d8..58917dee 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -625,7 +625,8 @@ def compile_recur(form, ctx): ctx.get_recur_point().emit(ctx, args) if ctc: ctx.enable_tail_call() - ctx.sub_sp(r_uint(args - 1)) + if args > 0: + ctx.sub_sp(r_uint(args - 1)) def compile_let(form, ctx): diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.lisp index c4f71a4f..7685a745 100644 --- a/tests/test-stdlib.lisp +++ b/tests/test-stdlib.lisp @@ -23,3 +23,30 @@ (t/assert= (butlast v) res) (t/assert= (butlast l) res) (t/assert= (butlast r) res))) + + +(t/deftest test-keys + (let [v {:a 1 :b 2 :c 3}] + (t/assert= (keys v) #{:a :b :c}) + (t/assert= (transduce (keys) conj! v) (keys v)))) + +(t/deftest test-vals + (let [v {:a 1 :b 2 :c 3}] + (t/assert= (vals v) #{1 2 3}) + (t/assert= (transduce (vals) conj! v) (vals v)))) + + +(t/deftest test-vec + (let [v '(1 2 3 4 5)] + (t/assert= (vec v) [1 2 3 4 5]) + (t/assert= (vec (map inc) v) [2 3 4 5 6]))) + + +(t/deftest test-keep + (let [v [-1 0 1 2 3 4 5]] + (t/assert= (vec (keep pos?) v) [true true true true true]) + (comment + + (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)) + )) +) From afeae43dd175ab42a4e54849e92ac65d637d2320 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 10 Nov 2014 17:59:25 +0100 Subject: [PATCH 214/909] fix multiple references to the same argument --- pixie/vm/reader.py | 7 +++++-- tests/test-fns.lisp | 2 ++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 8a9b6a43..82a3fdd7 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -467,8 +467,11 @@ def register_next_arg(n): max_arg = rt.get(arg_env, ARG_MAX) if n > max_arg.int_val(): arg_env = rt.assoc(arg_env, ARG_MAX, rt.wrap(n)) - arg = ArgReader.gen_arg(n) - arg_env = rt.assoc(arg_env, rt.wrap(n), arg) + + arg = rt.get(arg_env, rt.wrap(n), nil) + if arg is nil: + arg = ArgReader.gen_arg(n) + arg_env = rt.assoc(arg_env, rt.wrap(n), arg) ARG_ENV.set_value(arg_env) return arg diff --git a/tests/test-fns.lisp b/tests/test-fns.lisp index 35d752c0..e487ba89 100644 --- a/tests/test-fns.lisp +++ b/tests/test-fns.lisp @@ -6,8 +6,10 @@ (t/assert= (#(+ 3 %) 4) 7) (t/assert= (#(+ 3 %1) 4) 7) (t/assert= (#(+ %1 3) 4) 7) + (t/assert= (#(+ %1 %1) 3.5) 7.0) (t/assert= (#(+ %1 %2) 3 4) 7) (t/assert= (#(- %2 %1) 3 4) 1) + (t/assert= (#(+ %1 %1 %2 %2) 1.5 2) 7.0) (t/assert= (#(+ %1 %3) 3 'ignored 4) 7) (t/assert= (#(- %3 %1) 3 'ignored 4) 1) (t/assert= (#(apply + %1 %2 %&) 1 2 3 4 5) (+ 1 2 3 4 5))) From c46724824d2fc13a3da1d790033d430d5db9b44e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 10 Nov 2014 18:01:18 +0100 Subject: [PATCH 215/909] use fixed fn literals in defrecord as originally intended --- pixie/stdlib.lisp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index cf885b56..377038cc 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -655,7 +655,7 @@ (apply ~ctor-name (map #(get m %) ~fields))) default-bodies ['IAssociative `(-assoc [self k v] - (let [m (reduce (fn [m k] (assoc m k (. self k))) {} ~fields)] + (let [m (reduce #(assoc %1 %2 (. self %2)) {} ~fields)] (~map-ctor-name (assoc m k v)))) `(-contains-key [self k] (contains? ~(set fields) k)) @@ -668,7 +668,7 @@ not-found)) 'IObject `(-str [self] - (str "<" ~(name nm) " " (reduce (fn [m k] (assoc m k (. self k))) {} ~fields) ">")) + (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (. self %2)) {} ~fields) ">")) `(-eq [self other] (and (instance? ~nm other) ~@(map (fn [field] From 42e01e9ab4064106ef8d757f0d801c63a9ac553f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 10 Nov 2014 15:52:01 -0700 Subject: [PATCH 216/909] apply now works on any reducable, added a few more tests --- pixie/stdlib.lisp | 14 ++++++++++++-- pixie/vm/rt.py | 2 +- pixie/vm/stdlib.py | 6 +++--- target.py | 12 +++++++++++- tests/test-stdlib.lisp | 6 +----- 5 files changed, 28 insertions(+), 12 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index d1ed3b45..a2b77bbd 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -54,7 +54,13 @@ ([result] (xf result)) ([result item] (xf result (f item)))))) ([f coll] - (transduce (map f) conj coll)))) + (let [i (iterator coll)] + (loop [] + (if (at-end? i) + nil + (do (yield (f (current i))) + (move-next! i) + (recur)))))))) (def reduce (fn [rf init col] @@ -818,6 +824,10 @@ (do ~@body (recur (inc ~b)))))))) +(extend -iterator PersistentVector + (fn [v] + (dotimes [x (count v)] + (yield (nth v x))))) (defmacro and ([] true) @@ -964,7 +974,7 @@ (iterate [x coll] (let [result (f x)] (if result - (yield x)))))) + (yield result)))))) (defn refer [ns-sym & filters] (let [ns (or (the-ns ns-sym) (throw (str "No such namespace: " ns-sym))) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index d81d8e3e..5c0ddd3b 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -151,7 +151,7 @@ def reinit(): # stacklet.with_stacklets(run_load_stdlib) - init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals"] + init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec"] for x in init_fns: globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index c2a0d40d..320d859c 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -271,9 +271,9 @@ def str__args(args): @jit.unroll_safe def apply__args(args): last_itm = args[len(args) - 1] - if not rt.satisfies_QMARK_(rt.IIndexed.deref(), last_itm) or \ - not rt.satisfies_QMARK_(rt.ICounted.deref(), last_itm): - raise ValueError("Last item to apply must be bost IIndexed and ICounted") + if not (rt.satisfies_QMARK_(rt.IIndexed.deref(), last_itm) and + rt.satisfies_QMARK_(rt.ICounted.deref(), last_itm)): + last_itm = rt.vec(last_itm) fn = args[0] argc = r_uint(len(args) - 2) diff --git a/target.py b/target.py index 15fa1c5c..aff5ccb6 100644 --- a/target.py +++ b/target.py @@ -13,6 +13,7 @@ from pixie.vm.primitives import nil import sys import os +from rpython.rlib.objectmodel import we_are_translated class DebugIFace(JitHookInterface): def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations): @@ -145,13 +146,22 @@ def run_load_stdlib(): rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") result = nil + if not we_are_translated(): + print "Loading stdlib while interpreted, this will take some time..." + with compiler.with_ns(u"pixie.stdlib"): while True: + if not we_are_translated(): + sys.stdout.write(".") + sys.stdout.flush() form = reader.read(rdr, False) if form is reader.eof: - return result + break result = compiler.compile(form).invoke([]) + if not we_are_translated(): + print "done" + def load_stdlib(): diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.lisp index 7685a745..c5590b89 100644 --- a/tests/test-stdlib.lisp +++ b/tests/test-stdlib.lisp @@ -45,8 +45,4 @@ (t/deftest test-keep (let [v [-1 0 1 2 3 4 5]] (t/assert= (vec (keep pos?) v) [true true true true true]) - (comment - - (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)) - )) -) + (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)))) From 0338914f1bf6f6ed0ddb4d5701c904475b1b34bc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 10 Nov 2014 16:18:39 -0700 Subject: [PATCH 217/909] fixes issue #31 --- pixie/vm/compiler.py | 5 +++ pixie/vm/persistent_list.py | 74 +++++++++++++++++++++++++++++++++++++ pixie/vm/reader.py | 3 ++ tests/test-compiler.lisp | 4 ++ 4 files changed, 86 insertions(+) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 58917dee..335c3e9d 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -11,6 +11,7 @@ from pixie.vm.atom import Atom import pixie.vm.stdlib as proto from rpython.rlib.rarithmetic import r_uint +from pixie.vm.persistent_list import EmptyList import pixie.vm.rt as rt from pixie.vm.util import * @@ -775,6 +776,10 @@ def is_compiler_special(s): return True if compiler_special(s) is not None else False def compile_cons(form, ctx): + if isinstance(form, EmptyList): + ctx.push_const(form) + return + if isinstance(rt.first(form), symbol.Symbol): special = compiler_special(rt.first(form)) if special is not None: diff --git a/pixie/vm/persistent_list.py b/pixie/vm/persistent_list.py index a473b231..e15ca3a3 100644 --- a/pixie/vm/persistent_list.py +++ b/pixie/vm/persistent_list.py @@ -29,6 +29,7 @@ def meta(self): def with_meta(self, meta): return PersistentList(self._first, self._next, self._cnt, meta) + @extend(proto._first, PersistentList) @@ -69,6 +70,9 @@ def count(self): @as_var("list") def list__args(args): + if len(args) == 0: + return EmptyList() + i = r_uint(len(args)) acc = nil while i > 0: @@ -85,3 +89,73 @@ def _meta(self): def _with_meta(self, meta): assert isinstance(self, PersistentList) return self.with_meta(meta) + + + + +class EmptyList(object.Object): + _type = object.Type(u"pixie.stdlib.EmptyList") + def type(self): + return EmptyList._type + + def __init__(self, meta=nil): + self._meta = meta + + def meta(self): + return self._meta + + def with_meta(self, meta): + return EmptyList(meta) + + + + + +@extend(proto._first, EmptyList) +def _first(self): + assert isinstance(self, EmptyList) + return nil + +@extend(proto._next, EmptyList) +def _next(self): + assert isinstance(self, EmptyList) + return nil + +@extend(proto._seq, EmptyList) +def _seq(self): + assert isinstance(self, EmptyList) + return nil + +@extend(proto._count, EmptyList) +def _count(self): + assert isinstance(self, EmptyList) + return rt.wrap(0) + +@extend(proto._conj, EmptyList) +def _conj(self, itm): + assert isinstance(self, EmptyList) + return PersistentList(itm, nil, 1) + +@extend(proto._meta, EmptyList) +def _meta(self): + assert isinstance(self, EmptyList) + return self.meta() + +@extend(proto._with_meta, EmptyList) +def _with_meta(self, meta): + assert isinstance(self, EmptyList) + return self.with_meta(meta) + +@extend(proto._str, EmptyList) +def _str(self): + return rt.wrap(u"()") + +@extend(proto._repr, EmptyList) +def _str(self): + return rt.wrap(u"()") + +@extend(proto._reduce, EmptyList) +def _str(self, f, init): + return init + + diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 82a3fdd7..84113f8d 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -14,6 +14,7 @@ from pixie.vm.code import wrap_fn, extend from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP from pixie.vm.persistent_hash_set import EMPTY as EMPTY_SET +from pixie.vm.persistent_list import EmptyList import pixie.vm.stdlib as proto import pixie.vm.compiler as compiler @@ -213,6 +214,8 @@ def invoke(self, rdr, ch): eat_whitespace(rdr) ch = rdr.read() if ch == u")": + if len(lst) == 0: + return EmptyList() acc = nil for x in range(len(lst) - 1, -1, -1): acc = cons(lst[x], acc) diff --git a/tests/test-compiler.lisp b/tests/test-compiler.lisp index 8565defe..052bafe2 100644 --- a/tests/test-compiler.lisp +++ b/tests/test-compiler.lisp @@ -16,3 +16,7 @@ (t/assert= (let [] 1) 1) (t/assert= (let [x 1]) nil) (t/assert= (let []) nil)) + +(t/deftest test-lists + (t/assert= (vec '()) []) + (t/assert= (vec '()) ())) From 59fe7b61a92d07d88b9a63ed90a2637b2ff806ca Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 10 Nov 2014 16:26:40 -0700 Subject: [PATCH 218/909] Update README.md --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index bd08d9c9..9d9c89b3 100644 --- a/README.md +++ b/README.md @@ -106,6 +106,8 @@ However there are a few features of pixie that although may not be uncommon, are * CSP for concurrency. We already have stacklets, it's not that hard to use them for CSP style concurrency as well. +## Where do the devs hangout? +Mostly on FreeNode at `#pixie-lang` stop by and say "hello". ## Contributing From a50bd626701641a594a6753cedbe7a0bb5dc4400 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 10 Nov 2014 16:28:22 -0700 Subject: [PATCH 219/909] Update README.md --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 9d9c89b3..133f8445 100644 --- a/README.md +++ b/README.md @@ -10,6 +10,7 @@ The standard library is heavily inspired by Clojure as well as several other fun Some planned and implemented features: +* Immutable datastructures * Protocols first implementation * Transducers at-the-bottom (most primitves are based off of reduce) * Coroutines for transducer inversion of control (transducer to lazy-seq conversion) From 0e3316b70d045f2d0e21c1ac3161f60a96517377 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 11 Nov 2014 21:50:52 +0100 Subject: [PATCH 220/909] load stdlib relative to pixie-vm location --- pixie/stdlib.lisp | 1 - target.py | 21 +++++++++++++++++++-- 2 files changed, 19 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index a2b77bbd..8f0bc915 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -11,7 +11,6 @@ (def reset! -reset!) -(def load-paths (atom ["./"])) (def program-arguments []) diff --git a/target.py b/target.py index aff5ccb6..2699036c 100644 --- a/target.py +++ b/target.py @@ -5,14 +5,18 @@ from rpython.rlib.jit import JitHookInterface, Counters from rpython.rlib.rfile import create_stdio from rpython.annotator.policy import AnnotatorPolicy -from pixie.vm.code import wrap_fn, NativeFn, intern_var +from pixie.vm.code import wrap_fn, NativeFn, intern_var, Var from pixie.vm.stacklet import with_stacklets import pixie.vm.stacklet as stacklet from pixie.vm.object import RuntimeException, WrappedException from rpython.translator.platform import platform from pixie.vm.primitives import nil +from pixie.vm.atom import Atom +from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR import sys import os +import rpython.rlib.rpath as rpath +import rpython.rlib.rpath as rposix from rpython.rlib.objectmodel import we_are_translated class DebugIFace(JitHookInterface): @@ -32,6 +36,10 @@ def jitpolicy(driver): PROGRAM_ARGUMENTS = intern_var(u"pixie.stdlib", u"program-arguments") PROGRAM_ARGUMENTS.set_root(nil) +LOAD_PATHS = intern_var(u"pixie.stdlib", u"load-paths") +LOAD_PATHS.set_root(nil) +load_path = Var(u"", u"internal-load-path") + STAR_1 = intern_var(u"pixie.stdlib", u"*1") STAR_1.set_root(nil) STAR_2 = intern_var(u"pixie.stdlib", u"*2") @@ -140,7 +148,7 @@ def inner_invoke(self, args): def run_load_stdlib(): import pixie.vm.compiler as compiler import pixie.vm.reader as reader - f = open("pixie/stdlib.lisp") + f = open(rpath.rjoin(str(load_path.deref()._str), "pixie/stdlib.lisp")) data = f.read() f.close() rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") @@ -176,6 +184,7 @@ def entry_point(args): interactive = True script_args = [] + init_load_path(args[0]) load_stdlib() i = 1 @@ -218,7 +227,15 @@ def entry_point(args): return 0 +def init_load_path(self_path): + base_path = dirname(rpath.rabspath(self_path)) + # runtime is not loaded yet, so we have to do it manually + LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(base_path)))) + # just for run_load_stdlib (global variables can't be assigned to) + load_path.set_root(rt.wrap(base_path)) +def dirname(path): + return rpath.sep.join(path.split(rpath.sep)[0:-1]) from rpython.rtyper.lltypesystem import lltype from rpython.jit.metainterp import warmspot From 014705c6a2e0dedcb11cf074f7462f7baaad2f72 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 11 Nov 2014 21:51:21 +0100 Subject: [PATCH 221/909] add -l, --load-path option --- target.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/target.py b/target.py index 2699036c..2e9e0801 100644 --- a/target.py +++ b/target.py @@ -197,9 +197,10 @@ def entry_point(args): return 0 elif arg == '-h' or arg == '--help': print args[0] + " [] []" - print " -h, --help print this help" - print " -v, --version print the version number" - print " -e, --eval evaluate the given expression" + print " -h, --help print this help" + print " -v, --version print the version number" + print " -e, --eval= evaluate the given expression" + print " -l, --load-path= add to pixie.stdlib/load-paths" return 0 elif arg == '-e' or arg == '--eval': i += 1 @@ -210,6 +211,14 @@ def entry_point(args): else: print "Expected argument for " + arg return 1 + elif arg == '-l' or arg == '--load-path': + i += 1 + if i < len(args): + path = args[i] + add_to_load_paths(path) + else: + print "Expected argument for " + arg + return 1 else: print "Unknown option " + arg return 1 @@ -227,6 +236,9 @@ def entry_point(args): return 0 +def add_to_load_paths(path): + rt.reset_BANG_(LOAD_PATHS.deref(), rt.conj(rt.deref(LOAD_PATHS.deref()), rt.wrap(path))) + def init_load_path(self_path): base_path = dirname(rpath.rabspath(self_path)) # runtime is not loaded yet, so we have to do it manually From 70c0590d4ab79eaade850fab6a14935a866557bc Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 10:22:39 +0100 Subject: [PATCH 222/909] add IEmpty protocol for collections --- pixie/stdlib.lisp | 17 +++++++++++++++++ pixie/vm/stdlib.py | 2 ++ tests/test-stdlib.lisp | 11 +++++++++++ 3 files changed, 30 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index a2b77bbd..b00cb8f8 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -579,6 +579,23 @@ (fn [s] (apply str "#{" (conj (transduce (interpose " ") conj s) "}")))) +(extend -empty Cons (fn [_] '())) +(extend -empty LazySeq (fn [_] '())) +(extend -empty PersistentList (fn [_] '())) +(extend -empty EmptyList (fn [_] '())) +(extend -empty PersistentVector (fn [_] [])) +(extend -empty Array (fn [_] (make-array 0))) +(extend -empty PersistentHashMap (fn [_] {})) +(extend -empty PersistentHashSet (fn [_] #{})) + +(defn empty + {:doc "Returns an empty collection of the same type, or nil." + :added "0.1"} + [coll] + (if (satisfies? IEmpty coll) + (-empty coll) + nil)) + (extend -str Keyword (fn [k] (if (namespace k) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 320d859c..df42c6c4 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -22,6 +22,8 @@ defprotocol("pixie.stdlib", "IPersistentCollection", ["-conj"]) +defprotocol("pixie.stdlib", "IEmpty", ["-empty"]) + defprotocol("pixie.stdlib", "IObject", ["-hash", "-eq", "-str", "-repr"]) _eq.set_default_fn(wrap_fn(lambda a, b: false)) diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.lisp index c5590b89..138a31ea 100644 --- a/tests/test-stdlib.lisp +++ b/tests/test-stdlib.lisp @@ -36,6 +36,17 @@ (t/assert= (transduce (vals) conj! v) (vals v)))) +(t/deftest test-empty + (t/assert= (empty '(1 2 3)) '()) + (t/assert= (empty (list 1 2 3)) '()) + (t/assert= (empty (lazy-seq)) '()) + (t/assert= (empty '()) '()) + (t/assert= (empty [1 2 3]) []) + (t/assert= (empty (make-array 3)) (make-array 0)) + (t/assert= (empty {:a 1, :b 2, :c 3}) {}) + (t/assert= (empty #{1 2 3}) #{})) + + (t/deftest test-vec (let [v '(1 2 3 4 5)] (t/assert= (vec v) [1 2 3 4 5]) From bac40da584e9a5b990ccbf4c0f98c86214fe6707 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 11:03:15 +0100 Subject: [PATCH 223/909] add tests for first --- tests/test-stdlib.lisp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.lisp index 138a31ea..468be23c 100644 --- a/tests/test-stdlib.lisp +++ b/tests/test-stdlib.lisp @@ -1,6 +1,21 @@ (ns pixie.tests.test-stdlib (require pixie.test :as t)) +(t/deftest test-first + (t/assert= (first []) nil) + (t/assert= (first '()) nil) + (comment (t/assert= (first (make-array 0)) nil)) + (comment (t/assert= (first {}) nil)) + (comment (t/assert= (first #{}) nil)) + + (t/assert= (first [1 2 3]) 1) + (t/assert= (first '(1 2 3)) 1) + (let [a (make-array 3)] + (aset a 0 1) + (aset a 1 2) + (aset a 2 3) + (t/assert= (first a) 1))) + (t/deftest test-last (let [v [1 2 3 4 5] l '(1 2 3 4 5) From 7b4a37df6da68debbd3a9c1d256ca3c7ef07f5c5 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 11:03:50 +0100 Subject: [PATCH 224/909] return nil when nth is out of range on arrays this also fixes `first` on arrays, which is especially useful when destructuring arguments in a function. --- pixie/vm/array.py | 6 +++++- tests/test-stdlib.lisp | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index c3cadcbc..c7f813a5 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -45,7 +45,11 @@ def _count(self): @extend(proto._nth, Array) def _nth(self, idx): assert isinstance(self, Array) - return self._list[idx.int_val()] + ival = idx.int_val() + if ival < len(self._list): + return self._list[ival] + else: + return nil @extend(proto._reduce, Array) def reduce(self, f, init): diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.lisp index 468be23c..ad12772b 100644 --- a/tests/test-stdlib.lisp +++ b/tests/test-stdlib.lisp @@ -4,7 +4,7 @@ (t/deftest test-first (t/assert= (first []) nil) (t/assert= (first '()) nil) - (comment (t/assert= (first (make-array 0)) nil)) + (t/assert= (first (make-array 0)) nil) (comment (t/assert= (first {}) nil)) (comment (t/assert= (first #{}) nil)) From d7df7297c3643a769d83d1e798fd1ebef04e469c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 11:05:43 +0100 Subject: [PATCH 225/909] expose an internal version of -defprotocol it's internal because we probably want to enable multi-arity protocol methods and implement a `defprotocol` macro like clojure has. --- pixie/vm/stdlib.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index df42c6c4..00bed5ef 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ - resize_list, list_copy, returns, get_var_if_defined + resize_list, list_copy, returns, get_var_if_defined, intern_var import pixie.vm.code as code from types import MethodType from pixie.vm.primitives import true, false, nil @@ -59,6 +59,28 @@ defprotocol("pixie.stdlib", "IIterable", ["-iterator"]) defprotocol("pixie.stdlib", "IIterator", ["-current", "-at-end?", "-move-next!"]) +@as_var("pixie.stdlib.internal", "-defprotocol") +def _defprotocol(name, methods): + from pixie.vm.compiler import NS_VAR + from pixie.vm.persistent_vector import PersistentVector + from pixie.vm.symbol import Symbol + affirm(isinstance(name, Symbol), u"protocol name must be a symbol") + affirm(isinstance(methods, PersistentVector), u"protocol methods must be a vector of symbols") + method_list = [] + for i in range(0, rt.count(methods)): + method_sym = rt.nth(methods, rt.wrap(i)) + affirm(isinstance(method_sym, Symbol), u"protocol methods must be a vector of symbols") + method_list.append(rt.name(method_sym)) + + proto = Protocol(rt.name(name)) + ns = rt.name(NS_VAR.deref()) + intern_var(ns, rt.name(name)).set_root(proto) + for method in method_list: + method = unicode(method) + poly = PolymorphicFn(method, proto) + intern_var(ns, method).set_root(poly) + + return name def __make_code_overrides(x): @extend(_meta, x._type) From 4cf18776ad5700bb3be456b7d5098833b049d252 Mon Sep 17 00:00:00 2001 From: andrew chambers Date: Thu, 13 Nov 2014 00:21:22 +1300 Subject: [PATCH 226/909] working on read and eval --- pixie/vm/reader.py | 20 +++++++++++++++++++- pixie/vm/stdlib.py | 8 +++++++- tests/test-readeval.lisp | 16 ++++++++++++++++ 3 files changed, 42 insertions(+), 2 deletions(-) create mode 100644 tests/test-readeval.lisp diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 84113f8d..52cdd567 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -285,9 +285,27 @@ def invoke(self, rdr, ch): v = rdr.read() except EOFError: raise Exception("unmatched quote") + if v == "\"": return rt.wrap(u"".join(acc)) - acc.append(v) + elif v == "\\": + #inside escape... TODO handle everything. + try: + esc = rdr.read() + if esc == "\"": + acc.append("\"") + elif esc == "n": + acc.append("\n") + elif esc == "r": + acc.append("\r") + elif esc == "t": + acc.append("\t") + else: + acc.append(v) + except EOFError: + raise Exception("eof after escape character") + else: + acc.append(v) def read_token(rdr): acc = u"" diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 320d859c..d5649f39 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -313,9 +313,15 @@ def _satisfies(proto, o): @as_var("read-string") def _read_string(s): import pixie.vm.reader as reader - return reader.read(reader.StringReader(unicode(rt.name(s))), True) +# XXX seems broken under jit. +@as_var("eval") +def eval(form): + from pixie.vm.compiler import compile + from pixie.vm.interpreter import interpret + val = interpret(compile(form)) + return val @as_var("load-file") def load_file(filename): diff --git a/tests/test-readeval.lisp b/tests/test-readeval.lisp new file mode 100644 index 00000000..ece410f3 --- /dev/null +++ b/tests/test-readeval.lisp @@ -0,0 +1,16 @@ +(ns pixie.tests.test-readeval + (require pixie.test :as t)) + +(t/deftest test-read + (t/assert= (read-string "0xDEADBEEF") 3735928559) + (t/assert= (read-string "0xDeadBeef") 3735928559) + (t/assert= (read-string "0xdeadbeef") 3735928559) + (t/assert= (read-string "foo") 'foo) + (t/assert= (read-string "()") '()) + (t/assert= (read-string "(1 2 3)") '(1 2 3)) + (t/assert= (read-string "[1 2 3]") [1 2 3]) + (t/assert= (read-string "{:a 1 :b 2 :c 3}") {:a 1 :b 2 :c 3}) + (t/assert= (read-string "\"foo\"") "foo") + (t/assert= (read-string "false") false) + (t/assert= (read-string "true") true) + (t/assert= (read-string "(foo (bar (baz)))") '(foo (bar (baz))))) From a2e5b7215bdcb2352172cdcb1bfb1656f66f0f85 Mon Sep 17 00:00:00 2001 From: andrew chambers Date: Thu, 13 Nov 2014 00:28:56 +1300 Subject: [PATCH 227/909] another tiny test case --- tests/test-readeval.lisp | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/test-readeval.lisp b/tests/test-readeval.lisp index ece410f3..01be1a3d 100644 --- a/tests/test-readeval.lisp +++ b/tests/test-readeval.lisp @@ -11,6 +11,7 @@ (t/assert= (read-string "[1 2 3]") [1 2 3]) (t/assert= (read-string "{:a 1 :b 2 :c 3}") {:a 1 :b 2 :c 3}) (t/assert= (read-string "\"foo\"") "foo") + (t/assert= (read-string "\"fo\\\\o\"") "fo\\o") (t/assert= (read-string "false") false) (t/assert= (read-string "true") true) (t/assert= (read-string "(foo (bar (baz)))") '(foo (bar (baz))))) From 44a05ac46ffc88c01cacadbe512ea922f86f4ab1 Mon Sep 17 00:00:00 2001 From: andrew chambers Date: Thu, 13 Nov 2014 00:48:12 +1300 Subject: [PATCH 228/909] bug fix --- pixie/vm/reader.py | 14 ++++++++------ 1 file changed, 8 insertions(+), 6 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 52cdd567..d1936861 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -291,17 +291,19 @@ def invoke(self, rdr, ch): elif v == "\\": #inside escape... TODO handle everything. try: - esc = rdr.read() - if esc == "\"": + v = rdr.read() + if v == "\"": acc.append("\"") - elif esc == "n": + elif v == "n": acc.append("\n") - elif esc == "r": + elif v == "\\": + acc.append("\\") + elif v == "r": acc.append("\r") - elif esc == "t": + elif v == "t": acc.append("\t") else: - acc.append(v) + raise Exception("unhandled escape character") except EOFError: raise Exception("eof after escape character") else: From c957972a48128ed6b5e4bac62ed057bc36d9fb5f Mon Sep 17 00:00:00 2001 From: andrew chambers Date: Thu, 13 Nov 2014 00:51:44 +1300 Subject: [PATCH 229/909] reorder rules to look nicer --- pixie/vm/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index d1936861..d6118f29 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -294,10 +294,10 @@ def invoke(self, rdr, ch): v = rdr.read() if v == "\"": acc.append("\"") - elif v == "n": - acc.append("\n") elif v == "\\": acc.append("\\") + elif v == "n": + acc.append("\n") elif v == "r": acc.append("\r") elif v == "t": From 79befbb7451545572d7b20c6ebeae6c672e0d66d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 17:47:02 +0100 Subject: [PATCH 230/909] support running only a subset of the tests --- pixie/test.lisp | 17 ++++++++++++----- run-tests.lisp | 2 +- 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pixie/test.lisp b/pixie/test.lisp index 661e9331..ff5f5fdb 100644 --- a/pixie/test.lisp +++ b/pixie/test.lisp @@ -1,4 +1,5 @@ -(ns pixie.test) +(ns pixie.test + (require pixie.string :as s)) (def tests (atom {})) @@ -24,13 +25,19 @@ -(defn run-tests [] +(defn run-tests [& args] (push-binding-frame!) (set! (var *stats*) (atom {:fail 0 :pass 0})) - (print "Running: " (count@tests) " tests") - (foreach [test @tests] - ((val test))) + (let [match (or (first args) "") + tests (transduce (comp (filter #(>= (s/index-of (str (key %1)) match) 0)) + (map val)) + conj + @tests)] + (print "Running: " (count tests) " tests") + + (foreach [test tests] + (test))) (let [stats @*stats*] (print stats) diff --git a/run-tests.lisp b/run-tests.lisp index 345b4a61..157bc65b 100644 --- a/run-tests.lisp +++ b/run-tests.lisp @@ -4,5 +4,5 @@ (t/load-all-tests) -(let [result (t/run-tests)] +(let [result (apply t/run-tests program-arguments)] (exit (get result :fail))) From 89cb4368aacb38497525f7a9db424f8468b6cf71 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 19:25:00 +0100 Subject: [PATCH 231/909] improve load-file error message --- pixie/vm/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index d5649f39..271bdb52 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -354,7 +354,7 @@ def load_file(filename): break if f is None: - affirm(False, u"File does not exist in any directory found in load-paths") + affirm(False, u"File '" + rt.name(filename) + u"' does not exist in any directory found in load-paths") else: data = f.read() f.close() From c8a7bd874f7997bd7df421af741a0b73a4f40f12 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 19:32:13 +0100 Subject: [PATCH 232/909] add $PWD to the load-paths by default this allows loading "project-local" namespaces and files easily. --- target.py | 1 + 1 file changed, 1 insertion(+) diff --git a/target.py b/target.py index 2e9e0801..2d4977d1 100644 --- a/target.py +++ b/target.py @@ -186,6 +186,7 @@ def entry_point(args): init_load_path(args[0]) load_stdlib() + add_to_load_paths(".") i = 1 while i < len(args): From 3abf17d288b67a44ade4f84641bd243a9563d0c1 Mon Sep 17 00:00:00 2001 From: Joshua Davey Date: Wed, 12 Nov 2014 12:43:45 -0600 Subject: [PATCH 233/909] Update binary name --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 133f8445..3aa99f94 100644 --- a/README.md +++ b/README.md @@ -34,7 +34,7 @@ Some planned and implemented features: ### So this is written in Python? -It's actually written in the RPython, the same language PyPy is written in. The script `./make-with-jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `target-c` this executable is a flow blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... +It's actually written in the RPython, the same language PyPy is written in. The script `./make-with-jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm` this executable is a flow blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... ### What's this bit about "magical powers"? From ff50bf5b812dbfc8dd2c468768a7e8d24d91b17c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 12 Nov 2014 21:41:22 +0100 Subject: [PATCH 234/909] don't fail when being symlinked to --- target.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/target.py b/target.py index 2d4977d1..ca5f3478 100644 --- a/target.py +++ b/target.py @@ -15,6 +15,7 @@ from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR import sys import os +import os.path as path import rpython.rlib.rpath as rpath import rpython.rlib.rpath as rposix from rpython.rlib.objectmodel import we_are_translated @@ -241,7 +242,10 @@ def add_to_load_paths(path): rt.reset_BANG_(LOAD_PATHS.deref(), rt.conj(rt.deref(LOAD_PATHS.deref()), rt.wrap(path))) def init_load_path(self_path): - base_path = dirname(rpath.rabspath(self_path)) + base_path = rpath.rabspath(self_path) + if path.islink(base_path): + base_path = os.readlink(base_path) + base_path = dirname(base_path) # runtime is not loaded yet, so we have to do it manually LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(base_path)))) # just for run_load_stdlib (global variables can't be assigned to) From ee31a344a4504cc8df29f1d0703fd72f29b734bc Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 14 Nov 2014 01:35:07 +1100 Subject: [PATCH 235/909] Make PersistenHashMap conjable --- pixie/stdlib.lisp | 17 +++++++++++++++++ tests/collections/test-maps.lisp | 13 +++++++++++++ 2 files changed, 30 insertions(+) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index e3934cf8..97c18fa6 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -587,6 +587,23 @@ (extend -empty PersistentHashMap (fn [_] {})) (extend -empty PersistentHashSet (fn [_] #{})) +(extend -conj PersistentHashMap + (fn [coll x] + (cond + (instance? MapEntry x) + (assoc coll (key x) (val x)) + + (instance? PersistentVector x) + (if (= (count x) 2) + (assoc coll (nth x 0) (nth x 1)) + (throw "Vector arg to map conj must be a pair")) + + (satisfies? ISeqable x) + (reduce conj coll (-seq x)) + + :else + (throw (str (type x) " cannot be conjed to a map"))))) + (defn empty {:doc "Returns an empty collection of the same type, or nil." :added "0.1"} diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.lisp index 2aa584e4..8d449727 100644 --- a/tests/collections/test-maps.lisp +++ b/tests/collections/test-maps.lisp @@ -38,3 +38,16 @@ (t/assert= m m) (t/assert= (dissoc m :a) {:b 2}) (t/assert= (dissoc m :a :b) {}))) + +(t/deftest map-conj + (let [m {:a 1 :b 2}] + (t/assert= m m) + ;; Should conj vector of length 2 + (t/assert= (conj m [:c 3]) {:a 1 :b 2 :c 3}) + (t/assert= (conj m [:b 4]) {:a 1 :b 4}) + + ;; Should conj sequences of pairs + (t/assert= (conj {} '([:a 1] [:b 2] [:c 3])) {:a 1 :b 2 :c 3}) + + ;; Should conj sequences of MapEntries + (t/assert= (conj {} (seq {:a 1 :b 2 :c 3})) {:a 1 :b 2 :c 3}))) From 8a4689378140272d66e5bd0f7d1bdd975eec0921 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 13 Nov 2014 18:47:50 +0100 Subject: [PATCH 236/909] try finding the binary in $PATH if it isn't in the current directory --- target.py | 24 ++++++++++++++++++------ 1 file changed, 18 insertions(+), 6 deletions(-) diff --git a/target.py b/target.py index ca5f3478..73cbd5d9 100644 --- a/target.py +++ b/target.py @@ -242,18 +242,30 @@ def add_to_load_paths(path): rt.reset_BANG_(LOAD_PATHS.deref(), rt.conj(rt.deref(LOAD_PATHS.deref()), rt.wrap(path))) def init_load_path(self_path): - base_path = rpath.rabspath(self_path) - if path.islink(base_path): - base_path = os.readlink(base_path) - base_path = dirname(base_path) + if not path.isfile(self_path): + self_path = find_in_path(self_path) + assert self_path is not None + if path.islink(self_path): + self_path = os.readlink(self_path) + self_path = dirname(rpath.rabspath(self_path)) # runtime is not loaded yet, so we have to do it manually - LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(base_path)))) + LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(self_path)))) # just for run_load_stdlib (global variables can't be assigned to) - load_path.set_root(rt.wrap(base_path)) + load_path.set_root(rt.wrap(self_path)) def dirname(path): return rpath.sep.join(path.split(rpath.sep)[0:-1]) +def find_in_path(exe_name): + paths = os.environ.get('PATH') + print "PATH: " + paths + paths = paths.split(path.pathsep) + for p in paths: + exe_path = path.join(p, exe_name) + if path.isfile(exe_path): + return exe_path + return None + from rpython.rtyper.lltypesystem import lltype from rpython.jit.metainterp import warmspot From d7aafb3afc4f8d3a0cb305d714f450f2bdae71b3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 14 Nov 2014 12:12:20 +0100 Subject: [PATCH 237/909] don't use load-file for batch mode it's confusing because it interacts with load-paths in strange ways. (load-file doesn't simply load a file and evaluates it, but it looks through load-paths and tries to locate it in there.) --- target.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/target.py b/target.py index 73cbd5d9..b3664129 100644 --- a/target.py +++ b/target.py @@ -123,12 +123,20 @@ def inner_invoke(self, args): with with_ns(u"user"): try: + f = None if self._file == '-': - stdin, _, _ = create_stdio() - code = stdin.read() - interpret(compile(read(StringReader(unicode(code)), True))) + f, _, _ = create_stdio() else: - rt.load_file(rt.wrap(self._file)) + f = open(self._file) + data = f.read() + f.close() + + if data.startswith("#!"): + newline_pos = data.find("\n") + if newline_pos > 0: + data = data[newline_pos:] + + interpret(compile(read(StringReader(unicode(data)), True))) except WrappedException as ex: print "Error: ", ex._ex.__repr__() os._exit(1) @@ -248,6 +256,7 @@ def init_load_path(self_path): if path.islink(self_path): self_path = os.readlink(self_path) self_path = dirname(rpath.rabspath(self_path)) + # runtime is not loaded yet, so we have to do it manually LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(self_path)))) # just for run_load_stdlib (global variables can't be assigned to) @@ -258,7 +267,6 @@ def dirname(path): def find_in_path(exe_name): paths = os.environ.get('PATH') - print "PATH: " + paths paths = paths.split(path.pathsep) for p in paths: exe_path = path.join(p, exe_name) From dab1118d69fbd6026a64272bd39e105b40209810 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 14 Nov 2014 12:30:11 +0100 Subject: [PATCH 238/909] print an error if file isn't present doesn't work with inaccessible files, though. --- target.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/target.py b/target.py index b3664129..0f93cc89 100644 --- a/target.py +++ b/target.py @@ -127,6 +127,9 @@ def inner_invoke(self, args): if self._file == '-': f, _, _ = create_stdio() else: + if not path.isfile(self._file): + print "Error: Cannot open '" + self._file + "'" + os._exit(1) f = open(self._file) data = f.read() f.close() From eb1c077083c947df60a4fd40e96a8a776e1d9033 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 16 Nov 2014 20:09:36 +1100 Subject: [PATCH 239/909] All functions extend IFn --- pixie/stdlib.lisp | 9 +++++++++ pixie/vm/code.py | 16 ++++++++-------- tests/test-stdlib.lisp | 10 ++++++++++ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp index 97c18fa6..b8a2fe52 100644 --- a/pixie/stdlib.lisp +++ b/pixie/stdlib.lisp @@ -123,6 +123,14 @@ init (recur (f init (nth coll i)) (+ i 1)))))))) +;; Make all Function types extend IFn +(extend -invoke Code -invoke) +(extend -invoke NativeFn -invoke) +(extend -invoke VariadicCode -invoke) +(extend -invoke Closure -invoke) +(extend -invoke Var -invoke) +(extend -invoke PolymorphicFn -invoke) +(extend -invoke DoublePolymorphicFn -invoke) (extend -reduce Cons seq-reduce) (extend -reduce PersistentList seq-reduce) @@ -509,6 +517,7 @@ (defn list? [v] (instance? PersistentList v)) (defn map? [v] (satisfies? IMap v)) +(defn fn? [v] (satisfies? IFn v)) (defn indexed? [v] (satisfies? IIndexed v)) (defn counted? [v] (satisfies? ICounted v)) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index bb615a3a..b29f46bf 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -162,7 +162,7 @@ def invoke_with(self, args, self_fn): class NativeFn(BaseCode): """Wrapper for a native function""" - _type = object.Type(u"NativeFn") + _type = object.Type(u"pixie.stdlib.NativeFn") def __init__(self): BaseCode.__init__(self) @@ -182,7 +182,7 @@ def invoke_with(self, args, this_fn): class Code(BaseCode): """Interpreted code block. Contains consts and """ - _type = object.Type(u"Code") + _type = object.Type(u"pixie.stdlib.Code") __immutable_fields__ = ["_consts[*]", "_bytecode", "_stack_size", "_meta"] def type(self): @@ -267,7 +267,7 @@ def invoke_with(self, args, self_fn): affirm(False, u"Got " + unicode(str(argc)) + u" arg(s) need at least " + unicode(str(self._required_arity))) class Closure(BaseCode): - _type = object.Type(u"Closure") + _type = object.Type(u"pixie.stdlib.Closure") __immutable_fields__ = ["_closed_overs[*]", "_code", "_meta"] def type(self): return Closure._type @@ -314,7 +314,7 @@ def get_debug_points(self): return self._code.get_debug_points() class Undefined(object.Object): - _type = object.Type(u"Undefined") + _type = object.Type(u"pixie.stdlib.Undefined") def type(self): return Undefined._type @@ -340,7 +340,7 @@ def set_var_value(self, var, val): _dynamic_vars = DynamicVars() class Var(BaseCode): - _type = object.Type(u"Var") + _type = object.Type(u"pixie.stdlib.Var") _immutable_fields_ = ["_rev?"] def type(self): @@ -542,7 +542,7 @@ def invoke(self, args): class Protocol(object.Object): - _type = object.Type(u"Protocol") + _type = object.Type(u"pixie.stdlib.Protocol") __immutable_fields__ = ["_rev?"] def type(self): @@ -571,7 +571,7 @@ def satisfies(self, tp): class PolymorphicFn(BaseCode): - _type = object.Type(u"PolymorphicFn") + _type = object.Type(u"pixie.stdlib.PolymorphicFn") def type(self): return PolymorphicFn._type @@ -644,7 +644,7 @@ def invoke(self, args): class DoublePolymorphicFn(BaseCode): """A function that is polymorphic on the first two arguments""" - _type = object.Type(u"DoublePolymorphicFn") + _type = object.Type(u"pixie.stdlib.DoublePolymorphicFn") def type(self): return DefaultProtocolFn._type diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.lisp index ad12772b..9507cda6 100644 --- a/tests/test-stdlib.lisp +++ b/tests/test-stdlib.lisp @@ -72,3 +72,13 @@ (let [v [-1 0 1 2 3 4 5]] (t/assert= (vec (keep pos?) v) [true true true true true]) (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)))) + +(t/deftest test-fn? + (t/assert= (fn? inc) true) + (t/assert= (fn? {}) true) + (t/assert= (fn? #(%)) true) + (t/assert= (fn? :foo) true) + (t/assert= (fn? 1) false) + (t/assert= (fn? and) false) + (t/assert= (fn? "foo") false) + (t/assert (fn? (let [x 8] (fn [y] (+ x y)))) true)) From 03dd08f9e8323d53929a00c28808916ec96127a1 Mon Sep 17 00:00:00 2001 From: andrew chambers Date: Mon, 17 Nov 2014 00:43:20 +1300 Subject: [PATCH 240/909] renamed extensions to .pxi --- Makefile | 4 ++-- pixie/{stdlib.lisp => stdlib.pxi} | 0 pixie/{test.lisp => test.pxi} | 2 +- pixie/vm/bootstrap.py | 2 +- pixie/vm/rt.py | 2 +- pixie/vm/stdlib.py | 2 +- run-tests.lisp => run-tests.pxi | 0 target.py | 2 +- tests/collections/{test-maps.lisp => test-maps.pxi} | 0 tests/collections/{test-seqables.lisp => test-seqables.pxi} | 0 tests/collections/{test-sets.lisp => test-sets.pxi} | 0 tests/collections/{test-vectors.lisp => test-vectors.pxi} | 0 tests/{test-arrays.lisp => test-arrays.pxi} | 0 tests/{test-bits.lisp => test-bits.pxi} | 0 tests/{test-compiler.lisp => test-compiler.pxi} | 0 tests/{test-defrecord.lisp => test-defrecord.pxi} | 0 tests/{test-deftype.lisp => test-deftype.pxi} | 0 tests/{test-fns.lisp => test-fns.pxi} | 0 tests/{test-keywords.lisp => test-keywords.pxi} | 0 tests/{test-numbers.lisp => test-numbers.pxi} | 0 tests/{test-readeval.lisp => test-readeval.pxi} | 0 tests/{test-stdlib.lisp => test-stdlib.pxi} | 0 tests/{test-strings.lisp => test-strings.pxi} | 0 23 files changed, 7 insertions(+), 7 deletions(-) rename pixie/{stdlib.lisp => stdlib.pxi} (100%) rename pixie/{test.lisp => test.pxi} (99%) rename run-tests.lisp => run-tests.pxi (100%) rename tests/collections/{test-maps.lisp => test-maps.pxi} (100%) rename tests/collections/{test-seqables.lisp => test-seqables.pxi} (100%) rename tests/collections/{test-sets.lisp => test-sets.pxi} (100%) rename tests/collections/{test-vectors.lisp => test-vectors.pxi} (100%) rename tests/{test-arrays.lisp => test-arrays.pxi} (100%) rename tests/{test-bits.lisp => test-bits.pxi} (100%) rename tests/{test-compiler.lisp => test-compiler.pxi} (100%) rename tests/{test-defrecord.lisp => test-defrecord.pxi} (100%) rename tests/{test-deftype.lisp => test-deftype.pxi} (100%) rename tests/{test-fns.lisp => test-fns.pxi} (100%) rename tests/{test-keywords.lisp => test-keywords.pxi} (100%) rename tests/{test-numbers.lisp => test-numbers.pxi} (100%) rename tests/{test-readeval.lisp => test-readeval.pxi} (100%) rename tests/{test-stdlib.lisp => test-stdlib.pxi} (100%) rename tests/{test-strings.lisp => test-strings.pxi} (100%) diff --git a/Makefile b/Makefile index c1f44ef1..8de0b6e2 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ run_interactive: PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run_built_tests: pixie-vm - ./pixie-vm run-tests.lisp + ./pixie-vm run-tests.pxi run_interpreted_tests: target.py - PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.lisp + PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.pxi diff --git a/pixie/stdlib.lisp b/pixie/stdlib.pxi similarity index 100% rename from pixie/stdlib.lisp rename to pixie/stdlib.pxi diff --git a/pixie/test.lisp b/pixie/test.pxi similarity index 99% rename from pixie/test.lisp rename to pixie/test.pxi index ff5f5fdb..39db606a 100644 --- a/pixie/test.lisp +++ b/pixie/test.pxi @@ -53,7 +53,7 @@ (if (= (nth desc 1) :file) (let [filename (nth desc 2)] (if (pixie.string/starts-with filename "test-") - (if (pixie.string/ends-with filename ".lisp") + (if (pixie.string/ends-with filename ".pxi") (let [fullpath (str (nth desc 0) "/" filename)] (print "Loading " fullpath) (load-file fullpath))))))))) diff --git a/pixie/vm/bootstrap.py b/pixie/vm/bootstrap.py index 50831235..ca6ba124 100644 --- a/pixie/vm/bootstrap.py +++ b/pixie/vm/bootstrap.py @@ -7,7 +7,7 @@ def bootstrap(): import pixie.vm.rt as rt from pixie.vm.string import String assert False - rt.load_file(rt.wrap(u"pixie/stdlib.lisp")) + rt.load_file(rt.wrap(u"pixie/stdlib.pxi")) # run bootstrap diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 5c0ddd3b..835d5ec7 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -132,7 +132,7 @@ def reinit(): else: globals()[name] = var - #f = open("pixie/stdlib.lisp") + #f = open("pixie/stdlib.pxi") #data = f.read() #f.close() #rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 793df1c4..5b473db7 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -357,7 +357,7 @@ def load_file(filename): if isinstance(filename, symbol.Symbol): affirm(rt.namespace(filename) is None, u"load-file takes a un-namespaced symbol") - filename_str = rt.name(filename).replace(u".", u"/") + u".lisp" + filename_str = rt.name(filename).replace(u".", u"/") + u".pxi" loaded_ns = code._ns_registry.get(rt.name(filename), None) if loaded_ns is not None: diff --git a/run-tests.lisp b/run-tests.pxi similarity index 100% rename from run-tests.lisp rename to run-tests.pxi diff --git a/target.py b/target.py index 2e9e0801..917c827f 100644 --- a/target.py +++ b/target.py @@ -148,7 +148,7 @@ def inner_invoke(self, args): def run_load_stdlib(): import pixie.vm.compiler as compiler import pixie.vm.reader as reader - f = open(rpath.rjoin(str(load_path.deref()._str), "pixie/stdlib.lisp")) + f = open(rpath.rjoin(str(load_path.deref()._str), "pixie/stdlib.pxi")) data = f.read() f.close() rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") diff --git a/tests/collections/test-maps.lisp b/tests/collections/test-maps.pxi similarity index 100% rename from tests/collections/test-maps.lisp rename to tests/collections/test-maps.pxi diff --git a/tests/collections/test-seqables.lisp b/tests/collections/test-seqables.pxi similarity index 100% rename from tests/collections/test-seqables.lisp rename to tests/collections/test-seqables.pxi diff --git a/tests/collections/test-sets.lisp b/tests/collections/test-sets.pxi similarity index 100% rename from tests/collections/test-sets.lisp rename to tests/collections/test-sets.pxi diff --git a/tests/collections/test-vectors.lisp b/tests/collections/test-vectors.pxi similarity index 100% rename from tests/collections/test-vectors.lisp rename to tests/collections/test-vectors.pxi diff --git a/tests/test-arrays.lisp b/tests/test-arrays.pxi similarity index 100% rename from tests/test-arrays.lisp rename to tests/test-arrays.pxi diff --git a/tests/test-bits.lisp b/tests/test-bits.pxi similarity index 100% rename from tests/test-bits.lisp rename to tests/test-bits.pxi diff --git a/tests/test-compiler.lisp b/tests/test-compiler.pxi similarity index 100% rename from tests/test-compiler.lisp rename to tests/test-compiler.pxi diff --git a/tests/test-defrecord.lisp b/tests/test-defrecord.pxi similarity index 100% rename from tests/test-defrecord.lisp rename to tests/test-defrecord.pxi diff --git a/tests/test-deftype.lisp b/tests/test-deftype.pxi similarity index 100% rename from tests/test-deftype.lisp rename to tests/test-deftype.pxi diff --git a/tests/test-fns.lisp b/tests/test-fns.pxi similarity index 100% rename from tests/test-fns.lisp rename to tests/test-fns.pxi diff --git a/tests/test-keywords.lisp b/tests/test-keywords.pxi similarity index 100% rename from tests/test-keywords.lisp rename to tests/test-keywords.pxi diff --git a/tests/test-numbers.lisp b/tests/test-numbers.pxi similarity index 100% rename from tests/test-numbers.lisp rename to tests/test-numbers.pxi diff --git a/tests/test-readeval.lisp b/tests/test-readeval.pxi similarity index 100% rename from tests/test-readeval.lisp rename to tests/test-readeval.pxi diff --git a/tests/test-stdlib.lisp b/tests/test-stdlib.pxi similarity index 100% rename from tests/test-stdlib.lisp rename to tests/test-stdlib.pxi diff --git a/tests/test-strings.lisp b/tests/test-strings.pxi similarity index 100% rename from tests/test-strings.lisp rename to tests/test-strings.pxi From 258ee3eec36ee9e942cde100157fce949adbf009 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 16 Nov 2014 16:59:08 +0100 Subject: [PATCH 241/909] use new extension in examples and benchmarks as well --- benchmarks/{apply_varargs.lisp => apply_varargs.pxi} | 0 benchmarks/{dynamic_vars.lisp => dynamic_vars.pxi} | 0 benchmarks/{ffi_test.lisp => ffi_test.pxi} | 0 benchmarks/{get_from_hashmap.lisp => get_from_hashmap.pxi} | 0 benchmarks/{reduce_varargs.lisp => reduce_varargs.pxi} | 0 ...transduce_range_iterator.lisp => transduce_range_iterator.pxi} | 0 .../{transduce_range_reduced.lisp => transduce_range_reduced.pxi} | 0 benchmarks/{vector_assoc.lisp => vector_assoc.pxi} | 0 .../{vector_build_and_hash.lisp => vector_build_and_hash.pxi} | 0 examples/{hello-world.lisp => hello-world.pxi} | 0 examples/{mu-kanren.lisp => mu-kanren.pxi} | 0 11 files changed, 0 insertions(+), 0 deletions(-) rename benchmarks/{apply_varargs.lisp => apply_varargs.pxi} (100%) rename benchmarks/{dynamic_vars.lisp => dynamic_vars.pxi} (100%) rename benchmarks/{ffi_test.lisp => ffi_test.pxi} (100%) rename benchmarks/{get_from_hashmap.lisp => get_from_hashmap.pxi} (100%) rename benchmarks/{reduce_varargs.lisp => reduce_varargs.pxi} (100%) rename benchmarks/{transduce_range_iterator.lisp => transduce_range_iterator.pxi} (100%) rename benchmarks/{transduce_range_reduced.lisp => transduce_range_reduced.pxi} (100%) rename benchmarks/{vector_assoc.lisp => vector_assoc.pxi} (100%) rename benchmarks/{vector_build_and_hash.lisp => vector_build_and_hash.pxi} (100%) rename examples/{hello-world.lisp => hello-world.pxi} (100%) rename examples/{mu-kanren.lisp => mu-kanren.pxi} (100%) diff --git a/benchmarks/apply_varargs.lisp b/benchmarks/apply_varargs.pxi similarity index 100% rename from benchmarks/apply_varargs.lisp rename to benchmarks/apply_varargs.pxi diff --git a/benchmarks/dynamic_vars.lisp b/benchmarks/dynamic_vars.pxi similarity index 100% rename from benchmarks/dynamic_vars.lisp rename to benchmarks/dynamic_vars.pxi diff --git a/benchmarks/ffi_test.lisp b/benchmarks/ffi_test.pxi similarity index 100% rename from benchmarks/ffi_test.lisp rename to benchmarks/ffi_test.pxi diff --git a/benchmarks/get_from_hashmap.lisp b/benchmarks/get_from_hashmap.pxi similarity index 100% rename from benchmarks/get_from_hashmap.lisp rename to benchmarks/get_from_hashmap.pxi diff --git a/benchmarks/reduce_varargs.lisp b/benchmarks/reduce_varargs.pxi similarity index 100% rename from benchmarks/reduce_varargs.lisp rename to benchmarks/reduce_varargs.pxi diff --git a/benchmarks/transduce_range_iterator.lisp b/benchmarks/transduce_range_iterator.pxi similarity index 100% rename from benchmarks/transduce_range_iterator.lisp rename to benchmarks/transduce_range_iterator.pxi diff --git a/benchmarks/transduce_range_reduced.lisp b/benchmarks/transduce_range_reduced.pxi similarity index 100% rename from benchmarks/transduce_range_reduced.lisp rename to benchmarks/transduce_range_reduced.pxi diff --git a/benchmarks/vector_assoc.lisp b/benchmarks/vector_assoc.pxi similarity index 100% rename from benchmarks/vector_assoc.lisp rename to benchmarks/vector_assoc.pxi diff --git a/benchmarks/vector_build_and_hash.lisp b/benchmarks/vector_build_and_hash.pxi similarity index 100% rename from benchmarks/vector_build_and_hash.lisp rename to benchmarks/vector_build_and_hash.pxi diff --git a/examples/hello-world.lisp b/examples/hello-world.pxi similarity index 100% rename from examples/hello-world.lisp rename to examples/hello-world.pxi diff --git a/examples/mu-kanren.lisp b/examples/mu-kanren.pxi similarity index 100% rename from examples/mu-kanren.lisp rename to examples/mu-kanren.pxi From dffaaf1568adaff143b60abcf34b4a449b383d4e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 16 Nov 2014 17:10:25 +0100 Subject: [PATCH 242/909] fix batch mode in some (most?) cases it didn't read multiple expressions, although it did read and execute the two print expressions from the previous hello world example. doing it this way has the advantage of executing the statements as they are read. --- examples/hello-world.pxi | 6 ++++-- target.py | 7 ++++++- 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/examples/hello-world.pxi b/examples/hello-world.pxi index f4c17502..78f4c514 100755 --- a/examples/hello-world.pxi +++ b/examples/hello-world.pxi @@ -1,4 +1,6 @@ #!./pixie-vm -(puts "hello, world!") -(print "arguments: " program-arguments) +(defn greet [name] + (print (str "Hello, " (or name "World") "!"))) + +(greet (first program-arguments)) diff --git a/target.py b/target.py index 9724fd38..761e0afc 100644 --- a/target.py +++ b/target.py @@ -139,7 +139,12 @@ def inner_invoke(self, args): if newline_pos > 0: data = data[newline_pos:] - interpret(compile(read(StringReader(unicode(data)), True))) + rdr = StringReader(unicode(data)) + while True: + form = read(rdr, False) + if form is eof: + return + interpret(compile(form)) except WrappedException as ex: print "Error: ", ex._ex.__repr__() os._exit(1) From 0405fcef1cdec2d76b8ee6f8293196983272f2c0 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 17 Nov 2014 06:31:55 +1100 Subject: [PATCH 243/909] reudce -> reduce --- pixie/vm/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 01213293..d5a9e89c 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -186,7 +186,7 @@ def __assoc(_, k, v): return rt.hashmap(k, v) @extend(_reduce, nil._type) -def __reudce(self, f, init): +def __reduce(self, f, init): return init @extend(_val_at, nil._type) From cc3534f132289bc52904dd0963d66fd3947c9025 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 16 Nov 2014 23:55:28 -0500 Subject: [PATCH 244/909] variadic comp --- pixie/stdlib.pxi | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 97c18fa6..b01e4416 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -453,7 +453,10 @@ (f1 (apply f2 args)))) ([f1 f2 f3] (fn [& args] - (f1 (f2 (apply f3 args)))))) + (f1 (f2 (apply f3 args))))) + ([f1 f2 f3 & fs] + (fn [& args] + (apply (transduce comp (apply list f1 f2 f3 fs)) args)))) (defmacro cond ([] nil) From f835a889d6ab7e7980a73c4fe09f9f4a2c39b45f Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Mon, 17 Nov 2014 00:48:38 -0500 Subject: [PATCH 245/909] fix get-in behavior --- pixie/stdlib.pxi | 8 ++++---- tests/test-stdlib.pxi | 7 +++++++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 97c18fa6..055f3636 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -633,13 +633,13 @@ (reduce get m ks)) ([m ks not-found] (loop [sentinel 'x - mi m + m m ks (seq ks)] (if ks - (let [mi (get m (first ks) sentinel)] - (if (identical? sentinel mi) + (let [m (get m (first ks) sentinel)] + (if (identical? sentinel m) not-found - (recur sentinel mi (next ks)))) + (recur sentinel m (next ks)))) m)))) (defn assoc-in diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index ad12772b..394044ae 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -72,3 +72,10 @@ (let [v [-1 0 1 2 3 4 5]] (t/assert= (vec (keep pos?) v) [true true true true true]) (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)))) + +(t/deftest test-get-in + (let [m {:a 1 :b 2 :x {:a 2 :x [1 2 3]}}] + (t/assert= (get-in m [:a]) 1) + (t/assert= (get-in m [:missing]) nil) + (t/assert= (get-in m [:missing] :not-found) :not-found) + (t/assert= (get-in m [:x :x 0] :not-found) 1))) From 5438a4080815cfe6c841e1e00c14fe2d11aaaa74 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 17 Nov 2014 17:35:18 +0100 Subject: [PATCH 246/909] split load-file into load-ns and load-file load-file now just loads and runs a file at the given path, while load-ns searches for a file relative to load-paths. --- pixie/stdlib.pxi | 2 +- pixie/vm/bootstrap.py | 4 +-- pixie/vm/stdlib.py | 57 ++++++++++++++++++++++++++----------------- 3 files changed, 37 insertions(+), 26 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 97c18fa6..e6f99968 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -681,7 +681,7 @@ (defmacro require [ns kw as-nm] (assert (= kw :as) "Require expects :as as the second argument") - `(do (load-file (quote ~ns)) + `(do (load-ns (quote ~ns)) (refer-ns (this-ns-name) (the-ns (quote ~ns)) (quote ~as-nm)))) (defmacro ns [nm & body] diff --git a/pixie/vm/bootstrap.py b/pixie/vm/bootstrap.py index ca6ba124..f677409e 100644 --- a/pixie/vm/bootstrap.py +++ b/pixie/vm/bootstrap.py @@ -7,10 +7,10 @@ def bootstrap(): import pixie.vm.rt as rt from pixie.vm.string import String assert False - rt.load_file(rt.wrap(u"pixie/stdlib.pxi")) + rt.load_ns(rt.wrap(u"pixie/stdlib.pxi")) # run bootstrap #with_stacklets(bootstrap) # reset the stacklet state so we can translate with different settings -stacklet.global_state = stacklet.GlobalState() \ No newline at end of file +stacklet.global_state = stacklet.GlobalState() diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 01213293..da16b4c8 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -7,7 +7,6 @@ from pixie.vm.primitives import true, false, nil import pixie.vm.numbers as numbers import rpython.rlib.jit as jit -import os.path as path import rpython.rlib.rstacklet as rstacklet from rpython.rlib.rarithmetic import r_uint from pixie.vm.interpreter import ShallowContinuation @@ -347,22 +346,19 @@ def eval(form): val = interpret(compile(form)) return val -@as_var("load-file") -def load_file(filename): - import pixie.vm.reader as reader - import pixie.vm.compiler as compiler +@as_var("load-ns") +def load_ns(filename): import pixie.vm.string as string import pixie.vm.symbol as symbol import os.path as path - if isinstance(filename, symbol.Symbol): + if isinstance(filename, symbol.Symbol): affirm(rt.namespace(filename) is None, u"load-file takes a un-namespaced symbol") filename_str = rt.name(filename).replace(u".", u"/") + u".pxi" loaded_ns = code._ns_registry.get(rt.name(filename), None) if loaded_ns is not None: return loaded_ns - else: affirm(isinstance(filename, string.String), u"Filename must be string") filename_str = rt.name(filename) @@ -374,27 +370,42 @@ def load_file(filename): affirm(isinstance(path_x, string.String), u"Contents of load-paths must be strings") full_path = path.join(str(rt.name(path_x)), str(filename_str)) if path.isfile(full_path): - f = open(str(full_path)) + f = full_path break if f is None: affirm(False, u"File '" + rt.name(filename) + u"' does not exist in any directory found in load-paths") else: - data = f.read() - f.close() - - if data.startswith("#!"): - newline_pos = data.find("\n") - if newline_pos > 0: - data = data[newline_pos:] - rdr = reader.StringReader(unicode(data)) - - with compiler.with_ns(u"user"): - while True: - form = reader.read(rdr, False) - if form is reader.eof: - return nil - result = compiler.compile(form).invoke([]) + rt.load_file(rt.wrap(f)) + return nil + +@as_var("load-file") +def load_file(filename): + from pixie.vm.string import String + import pixie.vm.reader as reader + import pixie.vm.compiler as compiler + import os.path as path + + affirm(isinstance(filename, String), u"filename must be a string") + filename = str(rt.name(filename)) + affirm(path.isfile(filename), unicode(filename) + u" does not exist") + + f = open(filename) + data = f.read() + f.close() + + if data.startswith("#!"): + newline_pos = data.find("\n") + if newline_pos > 0: + data = data[newline_pos:] + rdr = reader.StringReader(unicode(data)) + + with compiler.with_ns(u"user"): + while True: + form = reader.read(rdr, False) + if form is reader.eof: + return nil + compiler.compile(form).invoke([]) return nil @as_var("the-ns") From 0af24e55f4c4731ec03f930cf340b5a2d9148347 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 17 Nov 2014 18:35:38 +0100 Subject: [PATCH 247/909] add load-reader --- pixie/vm/stdlib.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index da16b4c8..a0b47415 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -383,7 +383,6 @@ def load_ns(filename): def load_file(filename): from pixie.vm.string import String import pixie.vm.reader as reader - import pixie.vm.compiler as compiler import os.path as path affirm(isinstance(filename, String), u"filename must be a string") @@ -398,7 +397,14 @@ def load_file(filename): newline_pos = data.find("\n") if newline_pos > 0: data = data[newline_pos:] - rdr = reader.StringReader(unicode(data)) + + rt.load_reader(reader.StringReader(unicode(data))) + return nil + +@as_var("load-reader") +def load_reader(rdr): + import pixie.vm.reader as reader + import pixie.vm.compiler as compiler with compiler.with_ns(u"user"): while True: From 0969339ecb9286ed3de49a4a2b78cdbe8290d5f2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 17 Nov 2014 21:32:26 +0100 Subject: [PATCH 248/909] use load-reader in batch mode --- target.py | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/target.py b/target.py index 761e0afc..1705e536 100644 --- a/target.py +++ b/target.py @@ -139,12 +139,7 @@ def inner_invoke(self, args): if newline_pos > 0: data = data[newline_pos:] - rdr = StringReader(unicode(data)) - while True: - form = read(rdr, False) - if form is eof: - return - interpret(compile(form)) + rt.load_reader(StringReader(unicode(data))) except WrappedException as ex: print "Error: ", ex._ex.__repr__() os._exit(1) From 4c6946771bc5333147d15850a3e01af38ad5d484 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 18 Nov 2014 15:34:54 +0100 Subject: [PATCH 249/909] initial destructuring for vectors --- pixie/destructure.pxi | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 pixie/destructure.pxi diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi new file mode 100644 index 00000000..873e56f4 --- /dev/null +++ b/pixie/destructure.pxi @@ -0,0 +1,35 @@ +;(ns pixie.destructure) + +(defn nnext [coll] + (next (next coll))) + +(defn nthnext [coll n] + (loop [n n + xs (seq coll)] + (if (and xs (pos? n)) + (recur (dec n) (next xs)) + xs))) + +(defn destructure [binding expr] + (cond + (symbol? binding) [binding expr] + (vector? binding) (let [name (gensym "vec__")] + (reduce conj [name expr] + (destructure-vector binding name))) + :else (throw (str "unsupported binding form: " binding)))) + +(defn destructure-vector [binding-vector expr] + (loop [bindings (seq binding-vector) + i 0 + res []] + (if bindings + (let [binding (first bindings)] + (cond + (= binding '&) (recur (nnext bindings) + (inc (inc i)) + (reduce conj res (destructure (second bindings) `(nthnext ~expr ~i)))) + (= binding :as) (reduce conj res (destructure (second bindings) expr)) + :else (recur (next bindings) + (inc i) + (reduce conj res (destructure (first bindings) `(nth ~expr ~i)))))) + res))) From 4e6c00df63a450868fe258f065f8efbd470d6509 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 18 Nov 2014 16:59:11 +0100 Subject: [PATCH 250/909] initial destructuring for maps --- pixie/destructure.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi index 873e56f4..ebbee41c 100644 --- a/pixie/destructure.pxi +++ b/pixie/destructure.pxi @@ -16,6 +16,9 @@ (vector? binding) (let [name (gensym "vec__")] (reduce conj [name expr] (destructure-vector binding name))) + (map? binding) (let [name (gensym "map__")] + (reduce conj [name expr] + (destructure-map binding name))) :else (throw (str "unsupported binding form: " binding)))) (defn destructure-vector [binding-vector expr] @@ -33,3 +36,13 @@ (inc i) (reduce conj res (destructure (first bindings) `(nth ~expr ~i)))))) res))) + +(defn destructure-map [binding-map expr] + (loop [bindings (seq binding-map) + res []] + (if bindings + (let [binding (key (first bindings)) + binding-key (val (first bindings))] + (recur (next bindings) + (reduce conj res (destructure binding `(get ~expr ~binding-key))))) + res))) From 92c9e2f85304cb0a55814beb2d09d8a62b230d93 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 18 Nov 2014 17:41:25 +0100 Subject: [PATCH 251/909] add non-lazy implementations of take, drop and partition and when-let, because we use it for them. --- pixie/destructure.pxi | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi index ebbee41c..0c17f502 100644 --- a/pixie/destructure.pxi +++ b/pixie/destructure.pxi @@ -1,5 +1,10 @@ ;(ns pixie.destructure) +(defmacro when-let [binding & body] + `(let ~binding + (when ~(first binding) + ~@body))) + (defn nnext [coll] (next (next coll))) @@ -10,6 +15,23 @@ (recur (dec n) (next xs)) xs))) +(defn take [n coll] + (when (pos? n) + (when-let [s (seq coll)] + (cons (first s) (take (dec n) (next s)))))) + +(defn drop [n coll] + (let [s (seq coll)] + (if (and (pos? n) s) + (recur (dec n) (next s)) + s))) + +(defn partition + ([n coll] (partition n n coll)) + ([n step coll] + (when-let [s (seq coll)] + (cons (take n s) (partition n step (drop step s)))))) + (defn destructure [binding expr] (cond (symbol? binding) [binding expr] From ecbc5611118eddb1805956cbbb32de3fa6c544fb Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 18 Nov 2014 17:42:03 +0100 Subject: [PATCH 252/909] add let+ for trying out destructuring --- pixie/destructure.pxi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi index 0c17f502..4100dfad 100644 --- a/pixie/destructure.pxi +++ b/pixie/destructure.pxi @@ -68,3 +68,8 @@ (recur (next bindings) (reduce conj res (destructure binding `(get ~expr ~binding-key))))) res))) + +(defmacro let+ [bindings & body] + (let [destructured-bindings (transduce (map #(apply destructure %1)) concat [] (partition 2 bindings))] + `(let ~destructured-bindings + ~@body))) From ac9f28fd9447d9058cfb8e78eadabe003ada120c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 18 Nov 2014 17:51:00 +0100 Subject: [PATCH 253/909] support :as destructuring on maps --- pixie/destructure.pxi | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi index 4100dfad..c9a92e10 100644 --- a/pixie/destructure.pxi +++ b/pixie/destructure.pxi @@ -60,13 +60,19 @@ res))) (defn destructure-map [binding-map expr] - (loop [bindings (seq binding-map) - res []] - (if bindings - (let [binding (key (first bindings)) - binding-key (val (first bindings))] - (recur (next bindings) - (reduce conj res (destructure binding `(get ~expr ~binding-key))))) + (let [res + (loop [bindings (seq binding-map) + res []] + (if bindings + (let [binding (key (first bindings)) + binding-key (val (first bindings))] + (if (= binding :as) + (recur (next bindings) res) + (recur (next bindings) + (reduce conj res (destructure binding `(get ~expr ~binding-key)))))) + res))] + (if (contains? binding-map :as) + (reduce conj res [(get binding-map :as) expr]) res))) (defmacro let+ [bindings & body] From 3e0bc2019b2829cd9c4679f2223c014c08d084af Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 17 Nov 2014 22:42:35 +1100 Subject: [PATCH 254/909] Added macro? function --- pixie/vm/stdlib.py | 5 +++++ tests/test-stdlib.pxi | 12 ++++++++++-- 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 778fe0f5..93a1920b 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -509,6 +509,11 @@ def set_macro(f): f.set_macro() return f +@returns(bool) +@as_var("macro?") +def macro_QMARK_(f): + return true if isinstance(f, BaseCode) and f.is_macro() else false + @returns(unicode) @as_var("name") def name(s): diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 210104b1..6a688332 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -88,5 +88,13 @@ (t/assert= (fn? 1) false) (t/assert= (fn? and) false) (t/assert= (fn? "foo") false) - (t/assert (fn? (let [x 8] (fn [y] (+ x y)))) true)) - + (t/assert= (fn? (let [x 8] (fn [y] (+ x y)))) true)) + +(t/deftest test-macro? + (t/assert= (macro? and) true) + (t/assert= (macro? or) true) + (t/assert= (macro? defn) true) + (t/assert= (macro? inc) false) + (t/assert= (macro? 1) false) + (t/assert= (macro? :foo) false) + (t/assert= (macro? "foo") false)) From 50c997d7c83fdf1f3a3043fc87d25f7ac1fb94d8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 18 Nov 2014 18:11:51 +0100 Subject: [PATCH 255/909] support :or destructuring on maps --- pixie/destructure.pxi | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi index c9a92e10..4ca29b0d 100644 --- a/pixie/destructure.pxi +++ b/pixie/destructure.pxi @@ -66,14 +66,18 @@ (if bindings (let [binding (key (first bindings)) binding-key (val (first bindings))] - (if (= binding :as) + (if (or (= binding :as) (= binding :or)) (recur (next bindings) res) (recur (next bindings) (reduce conj res (destructure binding `(get ~expr ~binding-key)))))) - res))] - (if (contains? binding-map :as) - (reduce conj res [(get binding-map :as) expr]) - res))) + res)) + res (if (contains? binding-map :or) + (transduce (map #(vector (key %) `(or ~(key %) ~(val %)))) concat res (get binding-map :or)) + res) + res (if (contains? binding-map :as) + (reduce conj res [(get binding-map :as) expr]) + res)] + res)) (defmacro let+ [bindings & body] (let [destructured-bindings (transduce (map #(apply destructure %1)) concat [] (partition 2 bindings))] From 3e9403dab31dd837c044f8255d24063f1e2a682d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 18 Nov 2014 19:43:04 +0100 Subject: [PATCH 256/909] support for :keys destructuring we can't support :strs because strings aren't printed correctly in the expansion (without quotes, even in maps) and also not :syms because symbols can't be quoted in the expansion for now. --- pixie/destructure.pxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi index 4ca29b0d..0d1ea735 100644 --- a/pixie/destructure.pxi +++ b/pixie/destructure.pxi @@ -66,11 +66,13 @@ (if bindings (let [binding (key (first bindings)) binding-key (val (first bindings))] - (if (or (= binding :as) (= binding :or)) + (if (contains? #{:or :as :keys} binding) (recur (next bindings) res) (recur (next bindings) (reduce conj res (destructure binding `(get ~expr ~binding-key)))))) res)) + expand-with (fn [convert] #(vector % `(get ~expr ~(convert %)))) + res (if (contains? binding-map :keys) (transduce (map (expand-with (comp keyword name))) concat res (get binding-map :keys)) res) res (if (contains? binding-map :or) (transduce (map #(vector (key %) `(or ~(key %) ~(val %)))) concat res (get binding-map :or)) res) From 1334e368def489801489202e23ffd98c4288e2f2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 01:04:20 +0100 Subject: [PATCH 257/909] use fn* and let* as builtins we need this to be able to redefine fn and let later on. however, this does not work yet, because somehow the arity checking does not work. e.g. trying to use `defn` in the definition `defmacro` seems to broken. ("Wrong number of args to fn: got 3 expected") --- pixie/stdlib.pxi | 6 ++++++ pixie/vm/array.py | 43 +++++++++++++++++++++++++++++++++++++++++++ pixie/vm/compiler.py | 4 ++-- 3 files changed, 51 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d3f85c44..9f68921c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -13,7 +13,13 @@ (def program-arguments []) +(def fn (fn* [& args] + (cons 'fn* args))) +(set-macro! fn) +(def let (fn* [& args] + (cons 'let* args))) +(set-macro! let) (def conj (fn conj ([] []) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index c7f813a5..20b73035 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -58,6 +58,49 @@ def reduce(self, f, init): return self.reduce_large(f, init) return self.reduce_small(f, init) +@extend(proto._seq, Array) +def _seq(self): + assert isinstance(self, Array) + return ArraySeq(0, self._list) + +class ArraySeq(object.Object): + _type = object.Type(u"pixie.stdlib.ArraySeq") + __immutable_fields__ = ["_idx"] + + def __init__(self, idx, array): + self._idx = idx + self._array = array + + def first(self): + if self._idx < len(self._array): + return self._array[self._idx] + else: + return nil + + def next(self): + if self._idx < len(self._array): + return ArraySeq(self._idx + 1, self._array) + else: + return nil + + def type(self): + return self._type + +@extend(proto._first, ArraySeq) +def _first(self): + assert isinstance(self, ArraySeq) + return self.first() + +@extend(proto._next, ArraySeq) +def _next(self): + assert isinstance(self, ArraySeq) + return self.next() + +@extend(proto._seq, ArraySeq) +def _seq(self): + assert isinstance(self, ArraySeq) + return self + def array(lst): assert isinstance(lst, list) return Array(lst) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 335c3e9d..749513ff 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -748,14 +748,14 @@ def compile_yield(form, ctx): ctx.bytecode.append(code.YIELD) -builtins = {u"fn": compile_fn, +builtins = {u"fn*": compile_fn, u"if": compile_if, u"platform=": compile_platform_eq, u"def": compile_def, u"do": compile_do, u"quote": compile_quote, u"recur": compile_recur, - u"let": compile_let, + u"let*": compile_let, u"loop": compile_loop, u"comment": compile_comment, u"var": compile_var, From ad29e4a04a30cf6c229067e3d0831172be1b2cf5 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 12:45:17 +0100 Subject: [PATCH 258/909] don't extend -seq on Arrays twice now the expansion of ~@ in defn fails, because concat somehow doesn't deal well with nil arguments. --- pixie/stdlib.pxi | 1 - 1 file changed, 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 9f68921c..1dec69de 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -224,7 +224,6 @@ (extend -seq PersistentVector sequence) -(extend -seq Array sequence) From 96847571d92f8910f074fbec368f3aa7709beac4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 13:12:46 +0100 Subject: [PATCH 259/909] fix crashing while loading the stdlib --- pixie/vm/array.py | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 20b73035..cf85fb41 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -78,11 +78,18 @@ def first(self): return nil def next(self): - if self._idx < len(self._array): + if self._idx < len(self._array) - 1: return ArraySeq(self._idx + 1, self._array) else: return nil + def reduce(self, f, init): + for x in range(self._idx, len(self._array)): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, self._array[x]]) + return init + def type(self): return self._type @@ -101,6 +108,11 @@ def _seq(self): assert isinstance(self, ArraySeq) return self +@extend(proto._reduce, ArraySeq) +def _reduce(self, f, init): + assert isinstance(self, ArraySeq) + return self.reduce(f, init) + def array(lst): assert isinstance(lst, list) return Array(lst) From 8afd65728cbab410be28b20902a6f7e4fc9bac0c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 14:25:22 +0100 Subject: [PATCH 260/909] fix compilation i'm not sure why we have to copy the list, but using the original Array also does not work, so it's this for now. --- pixie/vm/array.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index cf85fb41..2505fc0f 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -61,11 +61,11 @@ def reduce(self, f, init): @extend(proto._seq, Array) def _seq(self): assert isinstance(self, Array) - return ArraySeq(0, self._list) + return ArraySeq(0, self._list[:]) class ArraySeq(object.Object): _type = object.Type(u"pixie.stdlib.ArraySeq") - __immutable_fields__ = ["_idx"] + __immutable_fields__ = ["_idx", "_array[*]"] def __init__(self, idx, array): self._idx = idx From caef475a6ec3f6f87f3ac10d6a5ab10be2f1f10b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 15:11:38 +0100 Subject: [PATCH 261/909] use a wrapped array, not a copy and fix -seq --- pixie/vm/array.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 2505fc0f..b8db8401 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -61,33 +61,33 @@ def reduce(self, f, init): @extend(proto._seq, Array) def _seq(self): assert isinstance(self, Array) - return ArraySeq(0, self._list[:]) + if rt.count(self) > 0: + return ArraySeq(0, self) + else: + return nil class ArraySeq(object.Object): _type = object.Type(u"pixie.stdlib.ArraySeq") - __immutable_fields__ = ["_idx", "_array[*]"] + __immutable_fields__ = ["_idx", "_w_array"] def __init__(self, idx, array): self._idx = idx - self._array = array + self._w_array = array def first(self): - if self._idx < len(self._array): - return self._array[self._idx] - else: - return nil + return rt.nth(self._w_array, rt.wrap(self._idx)) def next(self): - if self._idx < len(self._array) - 1: - return ArraySeq(self._idx + 1, self._array) + if self._idx < rt.count(self._w_array) - 1: + return ArraySeq(self._idx + 1, self._w_array) else: return nil def reduce(self, f, init): - for x in range(self._idx, len(self._array)): + for x in range(self._idx, rt.count(self._w_array)): if rt.reduced_QMARK_(init): return rt.deref(init) - init = f.invoke([init, self._array[x]]) + init = f.invoke([init, rt.nth(self._w_array, rt.wrap(x))]) return init def type(self): From 44f3dc77070416ec11f6983c0e071cb5aa07da47 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 15:34:17 +0100 Subject: [PATCH 262/909] move destructuring into stdlib i'd like to define it earlier, but it's difficult to figure out how to make everything work such that it can be defined as early as possible. there's likely an interesting dependency analysis algorithm in here. :) --- pixie/destructure.pxi | 87 ---------------------------------- pixie/stdlib.pxi | 108 ++++++++++++++++++++++++++++++++++++++---- 2 files changed, 98 insertions(+), 97 deletions(-) delete mode 100644 pixie/destructure.pxi diff --git a/pixie/destructure.pxi b/pixie/destructure.pxi deleted file mode 100644 index 0d1ea735..00000000 --- a/pixie/destructure.pxi +++ /dev/null @@ -1,87 +0,0 @@ -;(ns pixie.destructure) - -(defmacro when-let [binding & body] - `(let ~binding - (when ~(first binding) - ~@body))) - -(defn nnext [coll] - (next (next coll))) - -(defn nthnext [coll n] - (loop [n n - xs (seq coll)] - (if (and xs (pos? n)) - (recur (dec n) (next xs)) - xs))) - -(defn take [n coll] - (when (pos? n) - (when-let [s (seq coll)] - (cons (first s) (take (dec n) (next s)))))) - -(defn drop [n coll] - (let [s (seq coll)] - (if (and (pos? n) s) - (recur (dec n) (next s)) - s))) - -(defn partition - ([n coll] (partition n n coll)) - ([n step coll] - (when-let [s (seq coll)] - (cons (take n s) (partition n step (drop step s)))))) - -(defn destructure [binding expr] - (cond - (symbol? binding) [binding expr] - (vector? binding) (let [name (gensym "vec__")] - (reduce conj [name expr] - (destructure-vector binding name))) - (map? binding) (let [name (gensym "map__")] - (reduce conj [name expr] - (destructure-map binding name))) - :else (throw (str "unsupported binding form: " binding)))) - -(defn destructure-vector [binding-vector expr] - (loop [bindings (seq binding-vector) - i 0 - res []] - (if bindings - (let [binding (first bindings)] - (cond - (= binding '&) (recur (nnext bindings) - (inc (inc i)) - (reduce conj res (destructure (second bindings) `(nthnext ~expr ~i)))) - (= binding :as) (reduce conj res (destructure (second bindings) expr)) - :else (recur (next bindings) - (inc i) - (reduce conj res (destructure (first bindings) `(nth ~expr ~i)))))) - res))) - -(defn destructure-map [binding-map expr] - (let [res - (loop [bindings (seq binding-map) - res []] - (if bindings - (let [binding (key (first bindings)) - binding-key (val (first bindings))] - (if (contains? #{:or :as :keys} binding) - (recur (next bindings) res) - (recur (next bindings) - (reduce conj res (destructure binding `(get ~expr ~binding-key)))))) - res)) - expand-with (fn [convert] #(vector % `(get ~expr ~(convert %)))) - res (if (contains? binding-map :keys) (transduce (map (expand-with (comp keyword name))) concat res (get binding-map :keys)) res) - res (if (contains? binding-map :or) - (transduce (map #(vector (key %) `(or ~(key %) ~(val %)))) concat res (get binding-map :or)) - res) - res (if (contains? binding-map :as) - (reduce conj res [(get binding-map :as) expr]) - res)] - res)) - -(defmacro let+ [bindings & body] - (let [destructured-bindings (transduce (map #(apply destructure %1)) concat [] (partition 2 bindings))] - `(let ~destructured-bindings - ~@body))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1dec69de..6fc47ec2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -456,6 +456,15 @@ (defn contains? [coll key] (-contains-key coll key)) +(defn vec + {:doc "Converts a reducable collection into a vector using the (optional) transducer." + :signatures [[coll] [xform coll]] + :added "0.1"} + ([coll] + (transduce conj! coll)) + ([xform coll] + (transduce xform conj! coll))) + (defn get-val [inst] (get-field inst :val)) @@ -899,6 +908,95 @@ (defmacro when-not [test & body] `(if (not ~test) (do ~@body))) +(defmacro when-let [binding & body] + `(let ~binding + (when ~(first binding) + ~@body))) + +(defn nnext [coll] + (next (next coll))) + +(defn nthnext [coll n] + (loop [n n + xs (seq coll)] + (if (and xs (pos? n)) + (recur (dec n) (next xs)) + xs))) + +(defn take [n coll] + (when (pos? n) + (when-let [s (seq coll)] + (cons (first s) (take (dec n) (next s)))))) + +(defn drop [n coll] + (let [s (seq coll)] + (if (and (pos? n) s) + (recur (dec n) (next s)) + s))) + +(defn partition + ([n coll] (partition n n coll)) + ([n step coll] + (when-let [s (seq coll)] + (cons (take n s) (partition n step (drop step s)))))) + +(defn destructure [binding expr] + (cond + (symbol? binding) [binding expr] + (vector? binding) (let [name (gensym "vec__")] + (reduce conj [name expr] + (destructure-vector binding name))) + (map? binding) (let [name (gensym "map__")] + (reduce conj [name expr] + (destructure-map binding name))) + :else (throw (str "unsupported binding form: " binding)))) + +(defn destructure-vector [binding-vector expr] + (loop [bindings (seq binding-vector) + i 0 + res []] + (if bindings + (let [binding (first bindings)] + (cond + (= binding '&) (recur (nnext bindings) + (inc (inc i)) + (reduce conj res (destructure (second bindings) `(nthnext ~expr ~i)))) + (= binding :as) (reduce conj res (destructure (second bindings) expr)) + :else (recur (next bindings) + (inc i) + (reduce conj res (destructure (first bindings) `(nth ~expr ~i)))))) + res))) + +(defn destructure-map [binding-map expr] + (let [res + (loop [bindings (seq binding-map) + res []] + (if bindings + (let [binding (key (first bindings)) + binding-key (val (first bindings))] + (if (contains? #{:or :as :keys} binding) + (recur (next bindings) res) + (recur (next bindings) + (reduce conj res (destructure binding `(get ~expr ~binding-key)))))) + res)) + expand-with (fn [convert] #(vector % `(get ~expr ~(convert %)))) + res (if (contains? binding-map :keys) (transduce (map (expand-with (comp keyword name))) concat res (get binding-map :keys)) res) + res (if (contains? binding-map :or) + (transduce (map #(vector (key %) `(or ~(key %) ~(val %)))) concat res (get binding-map :or)) + res) + res (if (contains? binding-map :as) + (reduce conj res [(get binding-map :as) expr]) + res)] + res)) + +(defmacro let [bindings & body] + (let* [destructured-bindings (transduce (map #(apply destructure %1)) + concat + [] + (partition 2 bindings))] + `(let* ~destructured-bindings + ~@body))) + (defn abs [x] (if (< x 0) (* -1 x) @@ -1049,13 +1147,3 @@ (refer-symbol *ns* (or (rename sym) sym) v)))) (recur (next syms))))) nil)) - - -(defn vec - {:doc "Converts a reducable collection into a vector using the (optional) transducer." - :signatures [[coll] [xform coll]] - :added "0.1"} - ([coll] - (transduce conj! coll)) - ([xform coll] - (transduce xform conj! coll))) From 02cb61bf1ba9f74d37a1900cf50d14c3bf26daa8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 15:51:14 +0100 Subject: [PATCH 263/909] add tests for vector destructuring --- tests/test-destructuring.pxi | 31 +++++++++++++++++++++++++++++++ 1 file changed, 31 insertions(+) create mode 100644 tests/test-destructuring.pxi diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi new file mode 100644 index 00000000..005df4a2 --- /dev/null +++ b/tests/test-destructuring.pxi @@ -0,0 +1,31 @@ +(ns pixie.test.test-destructuring + (require pixie.test :as t)) + +(t/deftest test-let-simple + (t/assert= (let [x 1 y 2 z 3] [x y z]) [1 2 3]) + (t/assert= (let [x 1 y 2 z 3] [x y z]) (let* [x 1 y 2 z 3] [1 2 3])) + + (t/assert= (let [x 1 x 2] x) 2) + (t/assert= (let [x 1 y 2 x 3] x) 3) + + (t/assert= (let [x 1] (let [x 2] x)) 2)) + +(t/deftest test-let-vector-simple + (t/assert= (let [[x y z] [1 2 3]] [x y z]) [1 2 3]) + (t/assert= (let [[x y z] [1 2 3 4]] [x y z]) [1 2 3]) + + (t/assert= (let [[x y z & rest] [1 2 3 4]] + [x y z rest]) + [1 2 3 '(4)]) + (t/assert= (let [[x y z & rest] [1 2]] + [x y z rest]) + [1 2 nil nil])) + +(t/deftest test-let-vector-nested + (t/assert= (let [[[x y] z & rest] [[1 2] 3 4]] + [x y z rest]) + [1 2 3 '(4)]) + + (t/assert= (let [[[x [y]] z & rest] [[1 [2 3]] 4 5]] + [x y z rest]) + [1 2 4 '(5)])) From b9109e328094825388ade42d17af0078e1f75a82 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 16:09:20 +0100 Subject: [PATCH 264/909] support destructuring rest as a vector to support this nth is now supported on ISeqs. --- pixie/stdlib.pxi | 5 +++++ tests/test-destructuring.pxi | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6fc47ec2..aebabd5a 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -997,6 +997,11 @@ `(let* ~destructured-bindings ~@body))) +(extend -nth ISeq (fn [s n] + (if (and (pos? n) s) + (recur (next s) (dec n)) + (first s)))) + (defn abs [x] (if (< x 0) (* -1 x) diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi index 005df4a2..4621a828 100644 --- a/tests/test-destructuring.pxi +++ b/tests/test-destructuring.pxi @@ -29,3 +29,8 @@ (t/assert= (let [[[x [y]] z & rest] [[1 [2 3]] 4 5]] [x y z rest]) [1 2 4 '(5)])) + +(t/deftest test-let-vector-rest + (t/assert= (let [[x y & [z & rest]] [1 2 3 4 5]] + [x y z rest]) + [1 2 3 '(4 5)])) From dde6f354ea191c65e52e0588c77b8a88068c4d08 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 16:10:12 +0100 Subject: [PATCH 265/909] add simple tests for map destructuring --- tests/test-destructuring.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi index 4621a828..82ca17ff 100644 --- a/tests/test-destructuring.pxi +++ b/tests/test-destructuring.pxi @@ -34,3 +34,16 @@ (t/assert= (let [[x y & [z & rest]] [1 2 3 4 5]] [x y z rest]) [1 2 3 '(4 5)])) + +(t/deftest test-let-map + (t/assert= (let [{a :a, b :b, {c :c :as s} :d :as m} {:a 1, :b 2, :d {:c 3}}] + [a b c s m]) + [1 2 3 {:c 3} {:a 1, :b 2, :d {:c 3}}]) + + (t/assert= (let [{:keys [a b c] :as m} {:a 1, :b 2, :c 3, :d 4}] + [a b c (:d m)]) + [1 2 3 4])) + +(t/deftest test-let-map-defaults + (t/assert= (let [{a :a :or {a 42}} {:a 1}] a) 1) + (t/assert= (let [{a :a :or {a 42}} {}] a) 42)) From 1371cea1b176a50421eb4072bf8fac8f1ca63d85 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 19 Nov 2014 16:15:45 +0100 Subject: [PATCH 266/909] support non-truthy values in map destructuring previously, if defaults were passed, these would override the values from the map. --- pixie/stdlib.pxi | 8 +++----- tests/test-destructuring.pxi | 5 ++++- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index aebabd5a..148b2b55 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -968,7 +968,8 @@ res))) (defn destructure-map [binding-map expr] - (let [res + (let [defaults (or (:or binding-map) {}) + res (loop [bindings (seq binding-map) res []] (if bindings @@ -977,13 +978,10 @@ (if (contains? #{:or :as :keys} binding) (recur (next bindings) res) (recur (next bindings) - (reduce conj res (destructure binding `(get ~expr ~binding-key)))))) + (reduce conj res (destructure binding `(get ~expr ~binding-key ~(get defaults binding))))))) res)) expand-with (fn [convert] #(vector % `(get ~expr ~(convert %)))) res (if (contains? binding-map :keys) (transduce (map (expand-with (comp keyword name))) concat res (get binding-map :keys)) res) - res (if (contains? binding-map :or) - (transduce (map #(vector (key %) `(or ~(key %) ~(val %)))) concat res (get binding-map :or)) - res) res (if (contains? binding-map :as) (reduce conj res [(get binding-map :as) expr]) res)] diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi index 82ca17ff..8de451e1 100644 --- a/tests/test-destructuring.pxi +++ b/tests/test-destructuring.pxi @@ -46,4 +46,7 @@ (t/deftest test-let-map-defaults (t/assert= (let [{a :a :or {a 42}} {:a 1}] a) 1) - (t/assert= (let [{a :a :or {a 42}} {}] a) 42)) + (t/assert= (let [{a :a :or {a 42}} {}] a) 42) + + (t/assert= (let [{a :a :or {a 42}} {:a nil}] a) nil) + (t/assert= (let [{a :a :or {a 42}} {:a false}] a) false)) From 46196d03117f02209dfe5511157614ad491e0123 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 20 Nov 2014 23:37:15 +0100 Subject: [PATCH 267/909] generate fn literals using fn* they don't need destructuring. --- pixie/vm/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index d6118f29..b8d2eb80 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -531,7 +531,7 @@ def invoke(self, rdr, ch): args = rt.conj(args, ARG_AMP) args = rt.conj(args, rest_arg) - return rt.cons(symbol(u"fn"), rt.cons(args, rt.cons(form, nil))) + return rt.cons(symbol(u"fn*"), rt.cons(args, rt.cons(form, nil))) finally: ARG_ENV.set_value(nil) From f8d438fc4be36c028d5a9fc2ffe7272c5b862bd3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 20 Nov 2014 23:37:50 +0100 Subject: [PATCH 268/909] add destructuring for fn --- pixie/stdlib.pxi | 39 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 39 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 148b2b55..5bd03a97 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1150,3 +1150,42 @@ (refer-symbol *ns* (or (rename sym) sym) v)))) (recur (next syms))))) nil)) + +(extend -iterator ISeq (fn [s] + (loop [s s] + (when s + (yield (first s)) + (recur (next s)))))) + +(defn every? [pred coll] + (cond + (nil? (seq coll)) true + (pred (first coll)) (recur pred (next coll)) + :else false)) + +(defmacro fn [& decls] + (let [name (if (symbol? (first decls)) (first decls) nil) + decls (if name (next decls) decls) + name (or name '-fn) + decls (cond + (vector? (first decls)) (list decls) + ;(satisfies? ISeqable (first decls)) decls + ;:else (throw (str "expected a vector or a seq, got a " (type decls))) + :else decls) + decls (seq (map (fn* [decl] + (let [argv (first decl) + names (vec (map #(gensym "arg__") argv)) + bindings (loop [i 0 bindings []] + (if (< i (count argv)) + (recur (inc i) (reduce conj bindings [(nth argv i) (nth names i)])) + bindings)) + body (next decl)] + (if (every? symbol? argv) + `(~argv ~@body) + `(~names + (let ~bindings + ~@body))))) + decls))] + (if (= (count decls) 1) + `(fn* ~name ~(first (first decls)) ~@(next (first decls))) + `(fn* ~name ~@decls)))) From 57ab12e1e0923a76ac74a2a1ab99a70e11b4769d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 20 Nov 2014 23:53:12 +0100 Subject: [PATCH 269/909] fix the tests. --- pixie/stdlib.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 5bd03a97..bda6b00c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1156,6 +1156,7 @@ (when s (yield (first s)) (recur (next s)))))) +(extend -at-end? EmptyList (fn [_] true)) (defn every? [pred coll] (cond From 935a5f4d7161cc31508e6a7a9290b43c7f6212ec Mon Sep 17 00:00:00 2001 From: gigasquid Date: Thu, 20 Nov 2014 22:41:10 -0500 Subject: [PATCH 270/909] empty? and not-empty? tests for collections --- pixie/stdlib.pxi | 14 ++++++++++++++ tests/test-stdlib.pxi | 31 +++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8af5f13c..a4da5607 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -396,6 +396,20 @@ [x] (- x 1)) +(defn empty? + {:doc "returns true if the collection has no items, otherwise false" + :signatures [[coll]] + :added "0.1"} + [coll] + (not (seq coll))) + +(defn not-empty? + {:doc "returns true if the collection has items, otherwise false" + :signatures [[coll]] + :added "0.1"} + [coll] + (if (seq coll) true false)) + (defn first {:doc "Returns the first item in coll, if coll implements IIndexed nth will be used to retreive the item from the collection." diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 6a688332..2f7ac39a 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -39,6 +39,37 @@ (t/assert= (butlast l) res) (t/assert= (butlast r) res))) +(t/deftest test-empty? + (t/assert= (empty? []) true) + (t/assert= (empty? '()) true) + (t/assert= (empty? (make-array 0)) true) + (t/assert= (empty? {}) true) + (t/assert= (empty? #{}) true) + (t/assert= (empty? (range 1 5)) false) + + (t/assert= (empty? [1 2 3]) false) + (t/assert= (empty? '(1 2 3)) false) + (let [a (make-array 1)] + (aset a 0 1) + (t/assert= (empty? a) false)) + (t/assert= (empty? {:a 1}) false) + (t/assert= (empty? #{:a :b}) false)) + +(t/deftest test-not-empty? + (t/assert= (not-empty? []) false) + (t/assert= (not-empty? '()) false) + (t/assert= (not-empty? (make-array 0)) false) + (t/assert= (not-empty? {}) false) + (t/assert= (not-empty? #{}) false) + (t/assert= (not-empty? (range 1 5)) true) + + (t/assert= (not-empty? [1 2 3]) true) + (t/assert= (not-empty? '(1 2 3)) true) + (let [a (make-array 1)] + (aset a 0 1) + (t/assert= (not-empty? a) true)) + (t/assert= (not-empty? {:a 1}) true) + (t/assert= (not-empty? #{:a :b}) true)) (t/deftest test-keys (let [v {:a 1 :b 2 :c 3}] From a59ed51a3b49feb6959e66215880724c0e504153 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Thu, 20 Nov 2014 22:41:21 -0500 Subject: [PATCH 271/909] Update to run tests and examples --- README.md | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 3aa99f94..e9ea356b 100644 --- a/README.md +++ b/README.md @@ -28,7 +28,19 @@ Some planned and implemented features: ./checkout-externals ./make-with-jit ./pixie-vm - + + +## Running the tests + + ./pixie-vm run-tests.pxi + + +## Examples + +There are example in the /examples directory. +Try out "Hello World" with: + + ./examples/hello-world.pxi ## FAQ From 07d6938b358e1e45926348d32352ef6b559dd24c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 19 Nov 2014 22:01:40 +1100 Subject: [PATCH 272/909] Make Bool, PersistentHashSet and PersistentHashMap hashable. --- pixie/stdlib.pxi | 13 +++++++++++++ tests/collections/test-sets.pxi | 15 +++++++++++++++ 2 files changed, 28 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a4da5607..db1c9204 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -191,6 +191,19 @@ (fn [v] (transduce ordered-hash-reducing-fn v))) +(extend -hash PersistentHashSet + (fn [v] + (transduce ordered-hash-reducing-fn v))) + +(extend -hash PersistentHash + (fn [v] + (transduce ordered-hash-reducing-fn v))) + +(extend -hash Bool + (fn [v] + (if v + 1111111 + 3333333))) (def stacklet->lazy-seq (fn [k] diff --git a/tests/collections/test-sets.pxi b/tests/collections/test-sets.pxi index 27f055f0..440d2920 100644 --- a/tests/collections/test-sets.pxi +++ b/tests/collections/test-sets.pxi @@ -4,6 +4,21 @@ (t/deftest test-count (t/assert= (count (set [])) 0) (t/assert= (count (set [1 2 3])) 3) + (t/assert= (count (set [:foo :foo + :bar :bar + 'foo 'foo + 'bar 'bar + "foo" "foo" + "bar" "bar" + true true + false false + [:vec] [:vec] + ['another :vec] ['another :vec] + #{'set} #{'set} + #{:another 'set} #{:another 'set} + {:hash :map :number 1} {:hash :map :number 1} + {:hash :map :number 2} {:hash :map :number 2}])) + 14) (t/assert= (count (set [1 1 2 1])) 2)) (t/deftest test-contains From 30edcaeb9573f2715f063c5240a23238e40c2b0d Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 20 Nov 2014 00:11:19 +1100 Subject: [PATCH 273/909] Make characters hashable --- pixie/vm/string.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index 7c88277b..f7ea3f4b 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -5,7 +5,7 @@ import pixie.vm.stdlib as proto import pixie.vm.numbers as numbers import pixie.vm.util as util -from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rarithmetic import intmask, r_uint class String(Object): _type = Type(u"pixie.stdlib.String") @@ -85,6 +85,11 @@ def _eq(self, obj): return false return true if self.char_val() == obj.char_val() else false +@extend(proto._hash, Character) +def _hash(self): + return rt.wrap(intmask(util.hash_int(r_uint(self.char_val())))) + + @extend(proto._name, String) def _name(self): return self From 55fe82081ae0af23299007d8abc548161b463c2c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 20 Nov 2014 23:56:00 +1100 Subject: [PATCH 274/909] Make EmptyList and PersistentHashMap hashable --- pixie/stdlib.pxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index db1c9204..bdaad29e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -195,10 +195,12 @@ (fn [v] (transduce ordered-hash-reducing-fn v))) -(extend -hash PersistentHash +(extend -hash PersistentHashMap (fn [v] (transduce ordered-hash-reducing-fn v))) +(extend -hash EmptyList (fn [v] 5555555)) + (extend -hash Bool (fn [v] (if v From 32b1835ec6e262c484419dbbc42368e38052e14c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 20 Nov 2014 23:57:01 +1100 Subject: [PATCH 275/909] HashCollisionNodes now work --- pixie/vm/persistent_hash_map.py | 43 ++++++++++++++++++++++----------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 8d65bdc4..4a1a1052 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -5,7 +5,7 @@ from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rarithmetic import r_int, r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt @@ -108,7 +108,7 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): if key_or_null is None: assert isinstance(val_or_node, INode) - n = val_or_node.assoc_inode(shift + 5, hash_val, key, val, added_leaf) + n = val_or_node.assoc_inode(shift + 5, hash_val & MASK_32, key, val, added_leaf) if n is val_or_node: return self return BitmapIndexedNode(None, self._bitmap, clone_and_set(self._array, 2 * idx + 1, n)) @@ -226,17 +226,17 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): return self return ArrayNode(None, self._cnt, clone_and_set(self._array, idx, n)) - def without_inode(self, shift, hash, key): - idx = mask(hash, shift) + def without_inode(self, shift, hash_val, key): + idx = r_uint(mask(hash_val, shift)) node = self._array[idx] if node is None: return self - n = node.without_inode(shift + 5, hash, key) + n = node.without_inode(shift + 5, hash_val, key) if n is node: return self if n is None: if self._cnt <= 8: # shrink - return self.pack(None, idx) + return self.pack(idx) return ArrayNode(None, self._cnt - 1, clone_and_set(self._array, idx, n)) else: return ArrayNode(None, self._cnt, clone_and_set(self._array, idx, n)) @@ -250,7 +250,7 @@ def pack(self, idx): while i < idx: if self._array[i] is not None: new_array[j] = self._array[i] - bitmap |= 1 << i + bitmap |= r_uint(1) << i j += 2 i += 1 @@ -259,7 +259,7 @@ def pack(self, idx): while i < len(self._array): if self._array[i] is not None: new_array[j] = self._array[i] - bitmap |= 1 << i + bitmap |= r_uint(1) << i j += 2 i += 1 @@ -291,7 +291,22 @@ def __init__(self, edit, hash, array): self._array = array def assoc_inode(self, shift, hash_val, key, val, added_leaf): - assert False + if hash_val == self._hash: + count = len(self._array) + idx = self.find_index(key) + if idx != -1: + if self._array[idx + 1] == val: + return self; + return HashCollisionNode(None, hash_val, clone_and_set(self._array, r_uint(idx + 1), val)) + + new_array = [None] * (count + 2) + list_copy(self._array, 0, new_array, 0, count) + new_array[count] = key + added_leaf._val = added_leaf + new_array[count + 1] = val + return HashCollisionNode(self._edit, self._hash, new_array) + return BitmapIndexedNode(None, bitpos(self._hash, shift), [None, self]) \ + .assoc_inode(shift, hash_val, key, val, added_leaf) def find(self, shift, hash_val, key, not_found): for x in range(0, len(self._array), 2): @@ -314,16 +329,16 @@ def reduce_inode(self, f, init): return init def find_index(self, key): - i = 0 + i = r_int(0) while i < len(self._array): if rt.eq(key, self._array[i]): return i i += 2 - return -1 + return r_int(-1) - def without(self, shift, hash, key): + def without_inode(self, shift, hash, key): idx = self.find_index(key) if idx == -1: return self @@ -331,12 +346,12 @@ def without(self, shift, hash, key): if len(self._array) == 1: return None - return HashCollisionNode(None, self._hash, remove_pair(self._array, idx / 2)) + return HashCollisionNode(None, self._hash, remove_pair(self._array, r_uint(idx) / 2)) def create_node(shift, key1, val1, key2hash, key2, val2): - key1hash = rt.hash(key1) + key1hash = rt.hash(key1) & MASK_32 if key1hash == key2hash: return HashCollisionNode(None, key1hash, [key1, val1, key2, val2]) added_leaf = Box() From 123b28f29eb9e2a169c2521f481b8448cc4afaf0 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 21 Nov 2014 14:14:03 +0100 Subject: [PATCH 276/909] fix jit compilation this is quite weird, because it only fails after many tests, with a segfault. if i comment out tests, it fails somewhere else. --- pixie/test.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/test.pxi b/pixie/test.pxi index 39db606a..c109fe52 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -60,8 +60,8 @@ (defmacro assert= [x y] - `(let [xr# ~x - yr# ~y] + `(let* [xr# ~x + yr# ~y] (assert (= xr# yr#) (str (show '~x xr#) " != " (show '~y yr#))))) (defmacro assert [x] From b18e458d8b7178b58d5495c5da65721fe1baa446 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 22 Nov 2014 00:28:03 +1100 Subject: [PATCH 277/909] Added tests for collisions. --- tests/collections/test-sets.pxi | 38 ++++++++++++++++----------------- tests/utils.pxi | 13 +++++++++++ 2 files changed, 32 insertions(+), 19 deletions(-) create mode 100644 tests/utils.pxi diff --git a/tests/collections/test-sets.pxi b/tests/collections/test-sets.pxi index 440d2920..7815259f 100644 --- a/tests/collections/test-sets.pxi +++ b/tests/collections/test-sets.pxi @@ -1,34 +1,34 @@ (ns collections.test-sets - (require pixie.test :as t)) + (require pixie.test :as t) + (require tests.utils :as u)) + +(def worst-hashers (vec (map u/->WorstHasher) + (range 100))) (t/deftest test-count (t/assert= (count (set [])) 0) (t/assert= (count (set [1 2 3])) 3) - (t/assert= (count (set [:foo :foo - :bar :bar - 'foo 'foo - 'bar 'bar - "foo" "foo" - "bar" "bar" - true true - false false - [:vec] [:vec] - ['another :vec] ['another :vec] - #{'set} #{'set} - #{:another 'set} #{:another 'set} - {:hash :map :number 1} {:hash :map :number 1} - {:hash :map :number 2} {:hash :map :number 2}])) - 14) - (t/assert= (count (set [1 1 2 1])) 2)) + (t/assert= (count (set [1 1 2 1])) 2) + (t/assert= (count (set worst-hashers)) 100)) (t/deftest test-contains (let [s #{1 2 3} c [1 2 3] - n [-1 0 4]] + n [-1 0 4] + g (set worst-hashers)] (foreach [c c] (t/assert= (contains? s c) true)) (foreach [n n] - (t/assert= (contains? s n) false)))) + (t/assert= (contains? s n) false)) + (foreach [n worst-hashers] + (t/assert= (contains? g n) true)))) + +(t/deftest test-conj + (t/assert= (conj #{}) #{}) + (t/assert= (conj #{1 2} 3) #{1 2 3}) + (t/assert= (reduce conj #{} (range 10)) (set (vec (range 10)))) + (t/assert= (reduce conj #{} worst-hashers) (set worst-hashers))) + (t/deftest test-eq (let [s #{1 2 3}] diff --git a/tests/utils.pxi b/tests/utils.pxi new file mode 100644 index 00000000..4916736b --- /dev/null +++ b/tests/utils.pxi @@ -0,0 +1,13 @@ +(ns tests.utils) + +;; Here we create a new type which hashes poorly, in fact it's so bad we have a +;; hash space of only 1. +;; All members of WorstHasher return (hash "worst hasher") +;; when hash is called on them. +;; +;; This makes debugging, testing and benchmarking anything based off +;; PersistentHashMap trivial. + +;; X can be any thing you like. +(defrecord WorstHasher [x]) +(extend -hash WorstHasher (fn [self] (hash "worst hasher"))) From 358d391d2433159b04d06b5236531a03e11cd4fc Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 22 Nov 2014 02:15:00 +1100 Subject: [PATCH 278/909] Added disj to PersistentHashSet --- pixie/stdlib.pxi | 10 ++++++++++ pixie/vm/persistent_hash_set.py | 8 ++++++++ pixie/vm/stdlib.py | 2 +- tests/collections/test-sets.pxi | 6 ++++++ 4 files changed, 25 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index bdaad29e..d8ddfc2c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -25,6 +25,16 @@ ([result] (-persistent! result)) ([result item] (-conj! result item)))) +(def disj (fn disj + ([] []) + ([result] result) + ([result item] (-disj result item)))) + +(def disj! (fn conj! + ([] (-transient [])) + ([result] (-persistent! result)) + ([result item] (-disj! result item)))) + (def transient (fn [coll] (-transient coll))) (def persistent! (fn [coll] (-persistent! coll))) diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index bc3ea9fa..da311f2a 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -23,6 +23,9 @@ def __init__(self, meta, m): def conj(self, v): return PersistentHashSet(self._meta, self._map.assoc(v, v)) + def disj(self, k): + return PersistentHashSet(self._meta, self._map.without(k)) + def meta(self): return self._meta @@ -77,6 +80,11 @@ def _conj(self, v): assert isinstance(self, PersistentHashSet) return self.conj(v) +@extend(proto._disj, PersistentHashSet) +def _disj(self, v): + assert isinstance(self, PersistentHashSet) + return self.disj(v) + @extend(proto._reduce, PersistentHashSet) def _reduce(self, f, init): assert isinstance(self, PersistentHashSet) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 93a1920b..b3bd3c06 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -19,7 +19,7 @@ defprotocol("pixie.stdlib", "IIndexed", ["-nth"]) -defprotocol("pixie.stdlib", "IPersistentCollection", ["-conj"]) +defprotocol("pixie.stdlib", "IPersistentCollection", ["-conj", "-disj"]) defprotocol("pixie.stdlib", "IEmpty", ["-empty"]) diff --git a/tests/collections/test-sets.pxi b/tests/collections/test-sets.pxi index 7815259f..ad8f9315 100644 --- a/tests/collections/test-sets.pxi +++ b/tests/collections/test-sets.pxi @@ -29,6 +29,12 @@ (t/assert= (reduce conj #{} (range 10)) (set (vec (range 10)))) (t/assert= (reduce conj #{} worst-hashers) (set worst-hashers))) +(t/deftest test-disj + (t/assert= (disj #{}) #{}) + (t/assert= (disj #{1 2} 3) #{1 2}) + (t/assert= (disj #{1 2} 2) #{1}) + (t/assert= (reduce disj (set (vec (range 10))) (range 10)) #{}) + (t/assert= (reduce disj (set worst-hashers) worst-hashers) #{})) (t/deftest test-eq (let [s #{1 2 3}] From 845eb4dd0e9261348381a71913963320ba35d762 Mon Sep 17 00:00:00 2001 From: nisanharamati Date: Fri, 21 Nov 2014 08:50:09 -0800 Subject: [PATCH 279/909] Correct typo in FAQ s/flow blown/full blown/ --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index e9ea356b..56c7e9bb 100644 --- a/README.md +++ b/README.md @@ -46,7 +46,7 @@ Try out "Hello World" with: ### So this is written in Python? -It's actually written in the RPython, the same language PyPy is written in. The script `./make-with-jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm` this executable is a flow blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... +It's actually written in the RPython, the same language PyPy is written in. The script `./make-with-jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm` this executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... ### What's this bit about "magical powers"? From 2cb45180577cf6e6f158d7dac700a35bdd76eb42 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 21 Nov 2014 19:30:37 +0100 Subject: [PATCH 280/909] support :or together with :keys --- pixie/stdlib.pxi | 2 +- tests/test-destructuring.pxi | 5 ++++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d9c17fc3..4b1e1fd3 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1019,7 +1019,7 @@ (recur (next bindings) (reduce conj res (destructure binding `(get ~expr ~binding-key ~(get defaults binding))))))) res)) - expand-with (fn [convert] #(vector % `(get ~expr ~(convert %)))) + expand-with (fn [convert] #(vector % `(get ~expr ~(convert %) ~(get defaults %)))) res (if (contains? binding-map :keys) (transduce (map (expand-with (comp keyword name))) concat res (get binding-map :keys)) res) res (if (contains? binding-map :as) (reduce conj res [(get binding-map :as) expr]) diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi index 8de451e1..778707a9 100644 --- a/tests/test-destructuring.pxi +++ b/tests/test-destructuring.pxi @@ -49,4 +49,7 @@ (t/assert= (let [{a :a :or {a 42}} {}] a) 42) (t/assert= (let [{a :a :or {a 42}} {:a nil}] a) nil) - (t/assert= (let [{a :a :or {a 42}} {:a false}] a) false)) + (t/assert= (let [{a :a :or {a 42}} {:a false}] a) false) + + (t/assert= (let [{:keys [a], :or {a 42}} {:a 1}] a) 1) + (t/assert= (let [{:keys [a], :or {a 42}} {}] a) 42)) From 29131a51aab18e78661e7f20f495fcb205a43b34 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 22 Nov 2014 10:42:44 +0100 Subject: [PATCH 281/909] add a dockerfile and fix the fetch_externals command in the makefile. (at least on debian, i think it worked locally.) --- Dockerfile | 16 ++++++++++++++++ Makefile | 10 +++++----- 2 files changed, 21 insertions(+), 5 deletions(-) create mode 100644 Dockerfile diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..a01088b0 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,16 @@ +FROM debian:sid + +# install dependencies +RUN apt-get update \ + && apt-get install -y gcc pkg-config make curl bzip2 python2.7 \ + && apt-get install -y libffi-dev libuv-dev libreadline-dev \ + && ln -s /lib/x86_64-linux-gnu/libreadline.so.6 /lib/x86_64-linux-gnu/libreadline.so + +ADD . /usr/src/pixie + +# build the thing +RUN cd /usr/src/pixie \ + && make PYTHON=python2.7 build_with_jit + && ln -s $PWD/pixie-vm /usr/bin/pxi + +ENTRYPOINT ["/usr/bin/pxi"] \ No newline at end of file diff --git a/Makefile b/Makefile index 8de0b6e2..c0f41a37 100644 --- a/Makefile +++ b/Makefile @@ -21,11 +21,11 @@ build_no_jit: fetch_externals fetch_externals: $(EXTERNALS)/pypy $(EXTERNALS)/pypy: - mkdir $(EXTERNALS) - cd $(EXTERNALS) - curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2 - mkdir pypy - cd pypy + mkdir $(EXTERNALS); \ + cd $(EXTERNALS); \ + curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2; \ + mkdir pypy; \ + cd pypy; \ tar -jxf ../pypy.tar.bz2 --strip-components=1 run: From ff843593156d1a308ed9034d47e6bf9236c9e953 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 22 Nov 2014 10:51:41 +0100 Subject: [PATCH 282/909] fix the dockerfile $PWD doesn't seem to be set in RUN instructions in the Dockerfile. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a01088b0..1e3f1ce1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -11,6 +11,6 @@ ADD . /usr/src/pixie # build the thing RUN cd /usr/src/pixie \ && make PYTHON=python2.7 build_with_jit - && ln -s $PWD/pixie-vm /usr/bin/pxi + && ln -s /usr/src/pixie/pixie-vm /usr/bin/pxi ENTRYPOINT ["/usr/bin/pxi"] \ No newline at end of file From 002d25b2890fc614bb3f9ab8acc29a80c8030cf4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 22 Nov 2014 11:29:17 +0100 Subject: [PATCH 283/909] fix the dockerfile again (missing a backslash) --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 1e3f1ce1..f976ee9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -10,7 +10,7 @@ ADD . /usr/src/pixie # build the thing RUN cd /usr/src/pixie \ - && make PYTHON=python2.7 build_with_jit + && make PYTHON=python2.7 build_with_jit \ && ln -s /usr/src/pixie/pixie-vm /usr/bin/pxi ENTRYPOINT ["/usr/bin/pxi"] \ No newline at end of file From 12bef2fd49abe576c09a7954462404473d924642 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 17 Nov 2014 22:19:12 +1100 Subject: [PATCH 284/909] Added complement --- pixie/stdlib.pxi | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d9c17fc3..450f3da9 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -590,6 +590,18 @@ (recur (conj res (first coll)) (next coll)) (seq res)))) +(defn complement + {:doc "Given a function, return a new function which takes the same arguments + but returns the opposite truth value"} + [f] + (if (not (fn? f)) + (throw "Complement must be passed a function") + (fn + ([] (not (f))) + ([x] (not (f x))) + ([x y] (not (f x y))) + ([x y & more] (not (apply f x y more)))))) + (extend -count MapEntry (fn [self] 2)) (extend -nth MapEntry (fn [self idx not-found] (cond (= idx 0) (-key self) From 55876aac1e9d22cd775e2b3e08742e2a073ba90e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 23 Nov 2014 17:02:22 +0100 Subject: [PATCH 285/909] add a few tests for fn with destructuring. --- tests/test-destructuring.pxi | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi index 778707a9..8693edf9 100644 --- a/tests/test-destructuring.pxi +++ b/tests/test-destructuring.pxi @@ -53,3 +53,13 @@ (t/assert= (let [{:keys [a], :or {a 42}} {:a 1}] a) 1) (t/assert= (let [{:keys [a], :or {a 42}} {}] a) 42)) + +(t/deftest test-fn-simple + (t/assert= ((fn [[x y & rest]] [x y rest]) [1 2 3 4 5]) [1 2 '(3 4 5)]) + (t/assert= ((fn [{a :a, b :b :as m}] [a b m]) {:a 1, :b 2, :answer 42}) [1 2 {:a 1, :b 2, :answer 42}]) + + (t/assert= ((fn [[[x y] z & rest]] [x y z rest]) [[1 2] 3 4 5]) [1 2 3 '(4 5)])) + +(t/deftest test-fn-multiple-args + (t/assert= ((fn [[x y z] {:keys [a b c]}] [x y z a b c]) [1 2 3] {:a 4, :b 5, :c 6}) + [1 2 3 4 5 6])) From e5747dc3af3a017ef8bbe639cc0c4770d3a63bef Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 17 Nov 2014 22:55:27 +1100 Subject: [PATCH 286/909] Added some more usful functions * some * rest * distinct --- pixie/stdlib.pxi | 29 +++++++++++++++++++++++++++++ tests/test-stdlib.pxi | 20 ++++++++++++++++++++ 2 files changed, 49 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 450f3da9..1de6b43e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -139,6 +139,10 @@ init (recur (f init (nth coll i)) (+ i 1)))))))) +(def rest (fn [coll] + (let [next (next coll)] + (if next next '())))) + ;; Make all Function types extend IFn (extend -invoke Code -invoke) (extend -invoke NativeFn -invoke) @@ -440,6 +444,12 @@ [coll] (if (seq coll) true false)) +(defn even? [n] + (zero? (rem n 2))) + +(defn odd? [n] + (= (rem n 2) 1)) + (defn first {:doc "Returns the first item in coll, if coll implements IIndexed nth will be used to retreive the item from the collection." @@ -602,6 +612,12 @@ ([x y] (not (f x y))) ([x y & more] (not (apply f x y more)))))) +(defn some [pred coll] + (cond + (nil? (seq coll)) false + (pred (first coll)) true + :else (recur pred (next coll)))) + (extend -count MapEntry (fn [self] 2)) (extend -nth MapEntry (fn [self idx not-found] (cond (= idx 0) (-key self) @@ -1162,6 +1178,19 @@ (xf acc i) acc))))) +(defn distinct [] + (fn [xf] + (let [seen (atom #{})] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] + (if (contains? @seen i) + acc + (do + (swap! seen conj i) + (xf acc i)))))))) + (defn keep ([f] (fn [xf] diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 2f7ac39a..ed32354c 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -129,3 +129,23 @@ (t/assert= (macro? 1) false) (t/assert= (macro? :foo) false) (t/assert= (macro? "foo") false)) + +(t/deftest test-every? + (t/assert= (every? even? [2 4 6 8]) true) + (t/assert= (every? odd? [2 4 6 8]) false) + (t/assert= (every? even? [2 3 6 8]) false) + (t/assert= (every? even? []) true) + (t/assert= (every? odd? []) true)) + +(t/deftest test-some + (t/assert= (some even? [2 4 6 8]) true) + (t/assert= (some odd? [2 4 6 8]) false) + (t/assert= (some even? [2 3 6 8]) true) + (t/assert= (some even? [1 3 5 8]) true) + (t/assert= (some even? []) false) + (t/assert= (some odd? [2]) false)) + +(t/deftest test-distinct + (t/assert= (sequence (distinct) [1 2 3 2 1]) '(1 2 3)) + (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) + (t/assert= (vec (distinct) [nil nil nil]) [nil])) From 18ec766f5c1742db218760c00cedea59f60836c8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 23 Nov 2014 19:33:03 +0100 Subject: [PATCH 287/909] initial support for multimethods mhh... this might be enough, or not. it does some things already, though. --- pixie/stdlib.pxi | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d9c17fc3..47ff9a44 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1229,3 +1229,30 @@ (if (= (count decls) 1) `(fn* ~name ~(first (first decls)) ~@(next (first decls))) `(fn* ~name ~@decls)))) + +(deftype MultiMethod [dispatch-fn methods] + IFn + (-invoke [self dispatch-arg & args] + (let [dispatch-val (dispatch-fn dispatch-arg) + method (get @methods dispatch-val) + _ (assert method (str "no method defined for " dispatch-val))] + (apply method dispatch-arg args)))) + +(defmacro defmulti [name & args] + (let [[meta args] (if (string? (first args)) + [{:doc (first args)} (next args)] + [{} args]) + [meta args] (if (map? (first args)) + [(merge meta (first args)) (next args)] + [meta args]) + dispatch-fn (first args)] + `(def ~name (->MultiMethod ~dispatch-fn (atom {}))))) + +(defmacro defmethod [name dispatch-val params & body] + `(do + (let [methods (.methods ~name)] + (swap! methods + assoc + ~dispatch-val (fn ~params + ~@body)) + ~name))) From a34b24f9bade32df73b024cd240a2d548ae5923e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 23 Nov 2014 21:17:57 +0100 Subject: [PATCH 288/909] support default multimethods --- pixie/stdlib.pxi | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 47ff9a44..88c2ddde 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1230,11 +1230,13 @@ `(fn* ~name ~(first (first decls)) ~@(next (first decls))) `(fn* ~name ~@decls)))) -(deftype MultiMethod [dispatch-fn methods] +(deftype MultiMethod [dispatch-fn default-val methods] IFn (-invoke [self dispatch-arg & args] (let [dispatch-val (dispatch-fn dispatch-arg) - method (get @methods dispatch-val) + method (if (contains? @methods dispatch-val) + (get @methods dispatch-val) + (get @methods default-val)) _ (assert method (str "no method defined for " dispatch-val))] (apply method dispatch-arg args)))) @@ -1245,8 +1247,9 @@ [meta args] (if (map? (first args)) [(merge meta (first args)) (next args)] [meta args]) - dispatch-fn (first args)] - `(def ~name (->MultiMethod ~dispatch-fn (atom {}))))) + dispatch-fn (first args) + options (apply hashmap (next args))] + `(def ~name (->MultiMethod ~dispatch-fn ~(get options :default :default) (atom {}))))) (defmacro defmethod [name dispatch-val params & body] `(do From ec8617a996199b824e884a4a3110544db284867f Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 23 Nov 2014 20:47:49 -0500 Subject: [PATCH 289/909] Add multi-args for conj - also add conj for Cons type --- pixie/stdlib.pxi | 38 ++++++++++++++++++++--------- tests/collections/test-maps.pxi | 5 ++-- tests/collections/test-seqables.pxi | 4 +++ tests/collections/test-sets.pxi | 4 +++ tests/collections/test-vectors.pxi | 8 ++++++ 5 files changed, 45 insertions(+), 14 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a4da5607..dbbaaaa0 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -15,15 +15,25 @@ -(def conj (fn conj - ([] []) - ([result] result) - ([result item] (-conj result item)))) - -(def conj! (fn conj! - ([] (-transient [])) - ([result] (-persistent! result)) - ([result item] (-conj! result item)))) +(def conj + (fn conj ^{:doc "Adds elements to the collection. Elements are added to the end except in the case of Cons lists" + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([] []) + ([coll] coll) + ([coll item] (-conj coll item)) + ([coll item & args] + (reduce -conj (-conj coll item) args)))) + +(def conj! + (fn conj! ^{:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists" + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + ([] (-transient [])) + ([coll] (-persistent! coll)) + ([coll item] (-conj! coll item)) + ([coll item & args] + (reduce -conj! (-conj! coll item) args)))) (def transient (fn [coll] (-transient coll))) @@ -615,7 +625,7 @@ (extend -conj PersistentHashMap (fn [coll x] - (cond + (cond (instance? MapEntry x) (assoc coll (key x) (val x)) @@ -623,13 +633,17 @@ (if (= (count x) 2) (assoc coll (nth x 0) (nth x 1)) (throw "Vector arg to map conj must be a pair")) - + (satisfies? ISeqable x) (reduce conj coll (-seq x)) :else (throw (str (type x) " cannot be conjed to a map"))))) - + +(extend -conj Cons + (fn [coll x] + (cons x coll))) + (defn empty {:doc "Returns an empty collection of the same type, or nil." :added "0.1"} diff --git a/tests/collections/test-maps.pxi b/tests/collections/test-maps.pxi index 8d449727..c10bc44b 100644 --- a/tests/collections/test-maps.pxi +++ b/tests/collections/test-maps.pxi @@ -45,9 +45,10 @@ ;; Should conj vector of length 2 (t/assert= (conj m [:c 3]) {:a 1 :b 2 :c 3}) (t/assert= (conj m [:b 4]) {:a 1 :b 4}) - + (t/assert= (conj m [:b 4] [:c 5]) {:a 1 :b 4 :c 5}) + ;; Should conj sequences of pairs (t/assert= (conj {} '([:a 1] [:b 2] [:c 3])) {:a 1 :b 2 :c 3}) - + ;; Should conj sequences of MapEntries (t/assert= (conj {} (seq {:a 1 :b 2 :c 3})) {:a 1 :b 2 :c 3}))) diff --git a/tests/collections/test-seqables.pxi b/tests/collections/test-seqables.pxi index ffb8b089..35e47c1b 100644 --- a/tests/collections/test-seqables.pxi +++ b/tests/collections/test-seqables.pxi @@ -37,3 +37,7 @@ (t/assert= (= l '(1 2 3 4)) false) (t/assert= (= l [1 2 3 4]) false))) + +(t/deftest test-conj + (t/assert= '(3 1 2) (conj '(1 2) 3)) + (t/assert= '(5 4 3 1 2) (conj '(1 2) 3 4 5))) diff --git a/tests/collections/test-sets.pxi b/tests/collections/test-sets.pxi index 27f055f0..ad38c311 100644 --- a/tests/collections/test-sets.pxi +++ b/tests/collections/test-sets.pxi @@ -40,3 +40,7 @@ s (with-meta #{} m)] (t/assert= (meta #{}) nil) (t/assert= (meta s) m))) + +(t/deftest test-conj + (t/assert= #{1 2} (conj #{1} 2)) + (t/assert= #{1 2 3 4} (conj #{1} 2 3 4))) diff --git a/tests/collections/test-vectors.pxi b/tests/collections/test-vectors.pxi index 2299ee77..805155be 100644 --- a/tests/collections/test-vectors.pxi +++ b/tests/collections/test-vectors.pxi @@ -32,3 +32,11 @@ (t/assert= (= v [1 2 3 4]) false) (t/assert= (= v '(1 2)) false) (t/assert= (= v '(1 2 3 4)) false))) + +(t/deftest vector-conj + (t/assert= [1 2] (conj [1] 2)) + (t/assert= [1 2 3 4] (conj [1] 2 3 4))) + +(t/deftest vector-conj! + (t/assert= [1 2] (persistent! (conj! (transient [1]) 2))) + (t/assert= [1 2 3] (persistent! (conj! (transient [1]) 2 3)))) From 75cc8b9fda04fa63a8dd07b018a8dea47772146f Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 23 Nov 2014 21:35:55 -0500 Subject: [PATCH 290/909] add instructions on how to get out of the repl without having to ctl-c --- target.py | 1 + 1 file changed, 1 insertion(+) diff --git a/target.py b/target.py index 1705e536..bad53a17 100644 --- a/target.py +++ b/target.py @@ -61,6 +61,7 @@ def inner_invoke(self, args): import pixie.vm.persistent_vector as vector print "Pixie 0.1 - Interactive REPL" + print ":exit-repl to quit" print "(" + platform.name + ", " + platform.cc + ")" print "----------------------------" From 2a0eb1280a437d260dadc6d50fec34f76847bc60 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 12:27:18 +0100 Subject: [PATCH 291/909] implement -repr for most types --- pixie/stdlib.pxi | 29 +++++++++++++++++++++++++++-- 1 file changed, 27 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a985772c..ff97b189 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -173,8 +173,10 @@ (if (identical? x true) "true" "false"))) +(extend -repr Bool -str) (extend -str Nil (fn [x] "nil")) +(extend -repr Nil -str) (extend -reduce Nil (fn [self f init] init)) (extend -hash Nil (fn [self] 100000)) @@ -200,10 +202,16 @@ (extend -str PersistentVector (fn [v] (apply str "[" (conj (transduce (interpose " ") conj v) "]")))) +(extend -repr PersistentVector + (fn [v] + (apply str "[" (conj (transduce (comp (map -repr) (interpose " ")) conj v) "]")))) (extend -str Cons (fn [v] (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) +(extend -repr Cons + (fn [v] + (apply str "(" (conj (transduce (comp (map -repr) (interpose " ")) conj v) ")")))) (extend -hash Cons (fn [v] @@ -212,10 +220,16 @@ (extend -str PersistentList (fn [v] (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) +(extend -repr PersistentList + (fn [v] + (apply str "(" (conj (transduce (comp (map -repr) (interpose " ")) conj v) ")")))) (extend -str LazySeq (fn [v] (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) +(extend -repr LazySeq + (fn [v] + (apply str "(" (conj (transduce (comp (map -repr) (interpose " ")) conj v) ")")))) (extend -hash PersistentVector (fn [v] @@ -637,8 +651,11 @@ (extend -reduce MapEntry indexed-reduce) (extend -str MapEntry - (fn [v] - (apply str "[" (conj (transduce (interpose " ") conj v) "]")))) + (fn [v] + (apply str "[" (conj (transduce (interpose " ") conj v) "]")))) +(extend -repr MapEntry + (fn [v] + (apply str "[" (conj (transduce (comp (map -repr) (interpose " ")) conj v) "]")))) (extend -hash MapEntry (fn [v] @@ -670,6 +687,10 @@ (fn [v] (let [entry->str (map (fn [e] (vector (key e) " " (val e))))] (apply str "{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) +(extend -repr PersistentHashMap + (fn [v] + (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] + (apply str "{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) (extend -hash PersistentHashMap (fn [v] @@ -680,6 +701,9 @@ (extend -str PersistentHashSet (fn [s] (apply str "#{" (conj (transduce (interpose " ") conj s) "}")))) +(extend -repr PersistentHashSet + (fn [s] + (apply str "#{" (conj (transduce (comp (map -repr) (interpose " ")) conj s) "}")))) (extend -empty Cons (fn [_] '())) (extend -empty LazySeq (fn [_] '())) @@ -724,6 +748,7 @@ (if (namespace k) (str ":" (namespace k) "/" (name k)) (str ":" (name k))))) +(extend -repr Keyword -str) (extend -invoke Keyword (fn [k m] (-val-at m k nil))) (extend -invoke PersistentHashMap (fn [m k] (-val-at m k nil))) From 8216c28ee1a5eb347b7c681476a759fa97311c45 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 12:34:51 +0100 Subject: [PATCH 292/909] implement print, println, pr-str, pr and prn print does not add a newline anymore, but println does. print also inserts spaces between arguments now. --- pixie/stdlib.pxi | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ff97b189..bafa524a 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -933,8 +933,19 @@ (def getenv (ffi-fn libc "getenv" [String] String)) (defn print [& args] - (puts (apply str args))) + (printf (transduce (interpose " ") str args))) +(defn println [& args] + (puts (transduce (interpose " ") str args))) + +(defn pr-str [& args] + (transduce (comp (map -repr) (interpose " ")) str args)) + +(defn pr [& args] + (printf (apply pr-str args))) + +(defn prn [& args] + (puts (apply pr-str args))) (defn doc [x] (get (meta x) :doc)) From 0ba15461d274d1fd216981a235c257e0b66516e3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 12:44:56 +0100 Subject: [PATCH 293/909] fix usages of print that is, use println and don't add now extraneous spaces. --- examples/hello-world.pxi | 2 +- pixie/test.pxi | 19 +++++++++---------- run-tests.pxi | 2 +- 3 files changed, 11 insertions(+), 12 deletions(-) diff --git a/examples/hello-world.pxi b/examples/hello-world.pxi index 78f4c514..5c434843 100755 --- a/examples/hello-world.pxi +++ b/examples/hello-world.pxi @@ -1,6 +1,6 @@ #!./pixie-vm (defn greet [name] - (print (str "Hello, " (or name "World") "!"))) + (println (str "Hello, " (or name "World") "!"))) (greet (first program-arguments)) diff --git a/pixie/test.pxi b/pixie/test.pxi index c109fe52..f5378684 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -11,15 +11,15 @@ (defmacro deftest [nm & body] `(do (defn ~nm [] - (print "Running: " (str (namespace (var ~nm)) "/" (name (var ~nm)))) + (println "Running:" (str (namespace (var ~nm)) "/" (name (var ~nm)))) (try ~@body (swap! *stats* update-in [:pass] (fnil inc 0)) (catch ex - (print "while running " ~(name nm) " " (quote (do ~@body))) + (println "while running" ~(name nm) " " (quote (do ~@body))) (swap! *stats* update-in [:fail] (fnil inc 0)) - (print (str ex)) + (println (str ex)) (swap! *stats* update-in [:errors] (fnil conj []) ex)))) (swap! tests assoc (symbol (str (namespace (var ~nm)) "/" (name (var ~nm)))) ~nm))) @@ -34,28 +34,28 @@ (map val)) conj @tests)] - (print "Running: " (count tests) " tests") + (println "Running:" (count tests) "tests") (foreach [test tests] (test))) (let [stats @*stats*] - (print stats) + (println stats) (pop-binding-frame!) stats)) (defn load-all-tests [] - (print "Looking for tests...") + (println "Looking for tests...") (foreach [path @load-paths] - (print "Looking for tests in: " path) + (println "Looking for tests in:" path) (foreach [desc (pixie.path/file-list path)] (if (= (nth desc 1) :file) (let [filename (nth desc 2)] (if (pixie.string/starts-with filename "test-") (if (pixie.string/ends-with filename ".pxi") (let [fullpath (str (nth desc 0) "/" filename)] - (print "Loading " fullpath) + (println "Loading" fullpath) (load-file fullpath))))))))) @@ -69,8 +69,7 @@ (assert x# (str '~x " is " x#)))) (defn show - ([val] (if (instance? String val) (-repr val) val)) ([orig res] (if (= orig res) (show orig) - (str (show orig) " = " (show res))))) + (str (pr-str orig) " = " (pr-str res))))) diff --git a/run-tests.pxi b/run-tests.pxi index 157bc65b..a3b7a864 100644 --- a/run-tests.pxi +++ b/run-tests.pxi @@ -1,6 +1,6 @@ (require pixie.test :as t) -(print @load-paths) +(println @load-paths) (t/load-all-tests) From 45ca670a8c7d2ad08a5de667045a516dfa7cf652 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 12:45:43 +0100 Subject: [PATCH 294/909] print values using -repr on the repl --- target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target.py b/target.py index 1705e536..23f11f4e 100644 --- a/target.py +++ b/target.py @@ -89,7 +89,7 @@ def inner_invoke(self, args): continue if val is keyword(u"exit-repl"): break - val = rt.str(val) + val = rt._repr(val) assert isinstance(val, String), "str should always return a string" print val._str From 7db4d84b181a983cc4888b0b0793f590703d6e10 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 13:02:24 +0100 Subject: [PATCH 295/909] add tests for str and -repr and add -repr on symbols and fix pixie.test/show --- pixie/stdlib.pxi | 2 ++ pixie/test.pxi | 2 +- tests/test-stdlib.pxi | 34 ++++++++++++++++++++++++++++++++++ 3 files changed, 37 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index bafa524a..959c3a8b 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -750,6 +750,8 @@ (str ":" (name k))))) (extend -repr Keyword -str) +(extend -repr Symbol -str) + (extend -invoke Keyword (fn [k m] (-val-at m k nil))) (extend -invoke PersistentHashMap (fn [m k] (-val-at m k nil))) (extend -invoke PersistentHashSet (fn [m k] (-val-at m k nil))) diff --git a/pixie/test.pxi b/pixie/test.pxi index f5378684..e529f873 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -71,5 +71,5 @@ (defn show ([orig res] (if (= orig res) - (show orig) + (pr-str orig) (str (pr-str orig) " = " (pr-str res))))) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index ed32354c..ef3c18dd 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -1,6 +1,40 @@ (ns pixie.tests.test-stdlib (require pixie.test :as t)) +(t/deftest test-str + (t/assert= (str nil) "nil") + (t/assert= (str true) "true") + (t/assert= (str false) "false") + (t/assert= (str "hey") "hey") + (t/assert= (str :hey) ":hey") + (t/assert= (str 'hey) "hey") + + (t/assert= (str '()) "()") + (t/assert= (str '(1 2 3)) "(1 2 3)") + (t/assert= (str [1 2 3]) "[1 2 3]") + (t/assert= (str #{1}) "#{1}") + (t/assert= (str {}) "{}") + (t/assert= (str {:a 1}) "{:a 1}") + + (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]")) + +(t/deftest test-repr + (t/assert= (-repr nil) "nil") + (t/assert= (-repr true) "true") + (t/assert= (-repr false) "false") + (t/assert= (-repr "hey") "\"hey\"") + (t/assert= (-repr :hey) ":hey") + (t/assert= (-repr 'hey) "hey") + + (t/assert= (-repr '()) "()") + (t/assert= (-repr '(1 2 3)) "(1 2 3)") + (t/assert= (-repr [1 2 3]) "[1 2 3]") + (t/assert= (-repr #{1}) "#{1}") + (t/assert= (-repr {}) "{}") + (t/assert= (-repr {:a 1}) "{:a 1}") + + (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]")) + (t/deftest test-first (t/assert= (first []) nil) (t/assert= (first '()) nil) From 5b29784bc277a2ac9e75a953d0945ce3a59722a4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 14:06:02 +0100 Subject: [PATCH 296/909] implement -repr for Type --- pixie/vm/stdlib.py | 4 ++++ tests/test-stdlib.pxi | 2 ++ 2 files changed, 6 insertions(+) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index b3bd3c06..08d2bd7c 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -163,6 +163,10 @@ def _str(tp): import pixie.vm.string as string return string.rt.wrap(u"") +@extend(_repr, Type) +def _repr(tp): + import pixie.vm.string as string + return string.rt.wrap(tp._name) @extend(_first, nil._type) def _first(_): diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index ef3c18dd..2c5ea1a4 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -15,6 +15,7 @@ (t/assert= (str #{1}) "#{1}") (t/assert= (str {}) "{}") (t/assert= (str {:a 1}) "{:a 1}") + (t/assert= (str (type 3)) "") (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]")) @@ -32,6 +33,7 @@ (t/assert= (-repr #{1}) "#{1}") (t/assert= (-repr {}) "{}") (t/assert= (-repr {:a 1}) "{:a 1}") + (t/assert= (-repr (type 3)) "pixie.stdlib.Integer") (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]")) From 0015aadbecad40544d1ef163ce0cd1589cf29ff0 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Mon, 24 Nov 2014 08:30:05 -0500 Subject: [PATCH 297/909] add ctl-d --- target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target.py b/target.py index bad53a17..87d3586a 100644 --- a/target.py +++ b/target.py @@ -61,7 +61,7 @@ def inner_invoke(self, args): import pixie.vm.persistent_vector as vector print "Pixie 0.1 - Interactive REPL" - print ":exit-repl to quit" + print ":exit-repl or Ctl-D to quit" print "(" + platform.name + ", " + platform.cc + ")" print "----------------------------" From 8a2387ad1879eef2c4dd2bda0bf366204ceac3c3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 20:05:25 +0100 Subject: [PATCH 298/909] more docs lots more. it feels as if it makes loading slower, though. --- pixie/stdlib.pxi | 524 +++++++++++++++++++++++++++++++++++++++-------- 1 file changed, 433 insertions(+), 91 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index eca71f7a..397c8892 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -22,9 +22,10 @@ (set-macro! let) (def conj - (fn conj ^{:doc "Adds elements to the collection. Elements are added to the end except in the case of Cons lists" - :signatures [[] [coll] [coll item] [coll item & args]] - :added "0.1"} + (fn ^{:doc "Adds elements to the collection. Elements are added to the end except in the case of Cons lists" + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + conj ([] []) ([coll] coll) ([coll item] (-conj coll item)) @@ -32,41 +33,52 @@ (reduce -conj (-conj coll item) args)))) (def conj! - (fn conj! ^{:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists" - :signatures [[] [coll] [coll item] [coll item & args]] - :added "0.1"} + (fn ^{:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists" + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + conj! ([] (-transient [])) ([coll] (-persistent! coll)) ([coll item] (-conj! coll item)) ([coll item & args] (reduce -conj! (-conj! coll item) args)))) -(def disj (fn disj - ([] []) - ([result] result) - ([result item] (-disj result item)))) - -(def disj! (fn conj! +(def disj (fn ^{:doc "Removes elements from the collection." + :signatures [[] [coll] [coll item]] + :added "0.1"} + disj + ([] []) + ([coll] coll) + ([coll item] (-disj coll item)) + ([coll item & items] + (reduce -disj (-disj coll item) items)))) + +(def disj! (fn ^{:doc "Removes elements from the transient collection." + :signatures [[] [coll] [coll item] [coll item & items]] + :added "0.1"} + disj! ([] (-transient [])) ([result] (-persistent! result)) - ([result item] (-disj! result item)))) + ([result item] (-disj! result item)) + ([coll item & items] + (reduce -disj! (-disj! coll item) items)))) (def transient (fn [coll] (-transient coll))) (def persistent! (fn [coll] (-persistent! coll))) (def transduce (fn transduce - ([f coll] - (let [result (-reduce coll f (f))] + ([f coll] + (let [result (-reduce coll f (f))] (f result))) - ([xform rf coll] - (let [f (xform rf) - result (-reduce coll f (f))] + ([xform rf coll] + (let [f (xform rf) + result (-reduce coll f (f))] (f result))) - ([xform rf init coll] - (let [f (xform rf) - result (-reduce coll f init)] + ([xform rf init coll] + (let [f (xform rf) + result (-reduce coll f init)] (f result))))) (def map (fn ^{:doc "map - creates a transducer that applies f to every input element" @@ -91,22 +103,31 @@ (def reduce (fn [rf init col] (-reduce col rf init))) -(def into (fn [to from] +(def into (fn ^{:doc "Add the elements of `from` to the collection `to`." + :signatures [[to from]] + :added "0.1"} + into + [to from] (if (satisfies? IToTransient to) (persistent! (reduce conj! (transient to) from)) (reduce conj to from)))) (def interpose - (fn interpose [val] - (fn [xf] - (let [first? (atom true)] - (fn + (fn ^{:doc "Returns a transducer that inserts `val` in between elements of a collection." + :signatures [[val] [val coll]] + :added "0.1"} + interpose + ([val] (fn [xf] + (let [first? (atom true)] + (fn ([] (xf)) ([result] (xf result)) ([result item] (if @first? - (do (reset! first? false) - (xf result item)) - (xf (xf result val) item)))))))) + (do (reset! first? false) + (xf result item)) + (xf (xf result val) item))))))) + ([val coll] + (transduce (interpose val) conj coll)))) (def preserving-reduced @@ -118,7 +139,10 @@ ret))))) (def cat - (fn cat [rf] + (fn ^{:doc "A transducer that concatenates elements of a collection." + :added "0.1"} + cat + [rf] (let [rrf (preserving-reduced rf)] (fn cat-inner ([] (rf)) @@ -149,7 +173,10 @@ init (recur (f init (nth coll i)) (+ i 1)))))))) -(def rest (fn [coll] +(def rest (fn ^{:doc "Returns the elements after the first element, or () if there are no more elements." + :signatures [[coll]] + :added "0.1"} + [coll] (let [next (next coll)] (if next next '())))) @@ -259,7 +286,10 @@ (lazy-seq* (fn [] (stacklet->lazy-seq (-move-next! k)))))))) (def sequence - (fn + (fn ^{:doc "Returns a lazy sequence of the `data`, optionally transforming it using `xform`." + :signatures [[data] [xform data]] + :added "0.1"} + sequence ([data] (let [f (create-stacklet (fn [h] @@ -280,12 +310,20 @@ -(def concat (fn [& args] (transduce cat conj args))) +(def concat + (fn ^{:doc "Concatenates it's arguments." + :signatures [[& args]] + :added "0.1"} + concat + [& args] (transduce cat conj args))) (def key (fn [x] (-key x))) (def val (fn [x] (-val x))) -(def defn (fn [nm & rest] +(def defn (fn ^{:doc "Defines a new function." + :signatures [[nm doc? meta? & body]]} + defn + [nm & rest] (let [meta (if (instance? String (first rest)) {:doc (first rest)} {}) @@ -298,14 +336,24 @@ `(def ~nm (fn ~nm ~@rest))))) (set-macro! defn) -(defn defmacro [nm & rest] +(defn defmacro + {:doc "Defines a new macro." + :added "0.1"} + [nm & rest] `(do (defn ~nm ~@rest) (set-macro! ~nm) ~nm)) - (set-macro! defmacro) (defmacro -> + {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next. + +> (-> 3 inc inc) +; => 5 +> (-> \"James\" (str \"is \" \"awesome \") (str \"(and stuff)\" \"!\")) +; => \"James is awesome (and stuff)\"" + :signatures [[x & forms]] + :added "0.1"} [x & forms] (loop [x x, forms forms] (if forms @@ -317,6 +365,14 @@ x))) (defmacro ->> + {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next. + +> (->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \")) +; => \"you know what? we like James\" +> (->> 5 (range) (map inc) seq) +; => (1 2 3 4 5)" + :signatures [[x & forms]] + :added "0.1"} [x & forms] (loop [x x, forms forms] (if forms @@ -329,14 +385,14 @@ (defn not {:doc "Inverts the input, if a truthy value is supplied, returns false, otherwise - returns true" - :signatures [[arg]] +returns true" + :signatures [[x]] :added "0.1"} [x] (if x false true)) (defn + - {:doc "Adds the arguments" + {:doc "Adds the arguments, returning 0 if no arguments" :signatures [[& args]] :added "0.1"} ([] 0) @@ -373,7 +429,7 @@ (defn = {:doc "Returns true if all the arguments are equivalent. Otherwise, returns false. Uses - -eq to perform equality checks." +-eq to perform equality checks." :signatures [[& args]] :added "0.1"} ([x] true) @@ -384,7 +440,7 @@ (defn not= {:doc "Returns true if one (or more) of the arguments are not equivalent to the others. Uses - -eq to perform equality checks." +-eq to perform equality checks." :signatures [[& args]] :added "0.1"} ([x] false) @@ -468,10 +524,18 @@ [coll] (if (seq coll) true false)) -(defn even? [n] +(defn even? + {:doc "Returns true if n is even" + :signatures [[n]] + :added "0.1"} + [n] (zero? (rem n 2))) -(defn odd? [n] +(defn odd? + {:doc "Returns true of n is odd" + :signatures [[n]] + :added "0.1"} + [n] (= (rem n 2) 1)) (defn first @@ -515,6 +579,9 @@ (first (next (next (next coll)))))) (defn assoc + {:doc "Associates the key with the value in the collection" + :signatures [[m] [m k v] [m k v & kvs]] + :added "0.1"} ([m] m) ([m k v] (-assoc m k v)) @@ -522,11 +589,36 @@ (apply assoc (-assoc m k v) rest))) (defn dissoc + {:doc "Removes the value associated with the keys from the collection" + :signatures [[m] [m & ks]] + :addded "0.1"} ([m] m) ([m & ks] (reduce -dissoc m ks))) -(defn contains? [coll key] +(defn contains? + {:doc "Checks if there is a value associated with key in the collection. + +> (contains? {:a 1} :a) +; => true +> (contains? {:a 1} :b) +; => false +> (contains? #{:a :b :c} :a) +; => true + +Note the behaviour with indexed collections: + +> (contains? [:a :b :c] 0) +; => true +> (contains? [:a :b :c] 4) +; => false +> (contains? [:a :b :c] :a) +; => false + +Use some to check if a value is present in a collection." + :signatures [[coll key]] + :added "0.1"} + [coll key] (-contains-key coll key)) (defn vec @@ -542,6 +634,12 @@ (get-field inst :val)) (defn comp + {:doc "Composes the given functions, applying the last function first. + +> ((comp inc first) [41 2 3]) +; => 42" + :signatures [[f] [f & fs]] + :added "0.1"} ([f] f) ([f1 f2] (fn [& args] @@ -554,6 +652,9 @@ (apply (transduce comp (apply list f1 f2 f3 fs)) args)))) (defmacro cond + {:doc "Checks if any of the tests is truthy, if so, stops and returns the value of the corresponding body" + :signatures [[] [test then & clauses]] + :added "0.1"} ([] nil) ([test then & clauses] `(if ~test @@ -589,6 +690,11 @@ (fn [] ~@finally)))))) (defn . + {:doc "Access the field named by the symbol. + +If further arguments are passed, invokes the method named by symbol, passing the object and arguments." + :signatures [[obj sym] [obj sym & args]] + :added "0.1"} ([obj sym] (get-field obj sym)) ([obj sym & args] @@ -612,12 +718,20 @@ (defn indexed? [v] (satisfies? IIndexed v)) (defn counted? [v] (satisfies? ICounted v)) -(defn last [coll] +(defn last + {:doc "Returns the last element of the collection, or nil if none." + :signatures [[coll]] + :added "0.1"} + [coll] (if (next coll) (recur (next coll)) (first coll))) -(defn butlast [coll] +(defn butlast + {:doc "Returns all elements but the last from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] (loop [res [] coll coll] (if (next coll) @@ -636,7 +750,13 @@ ([x y] (not (f x y))) ([x y & more] (not (apply f x y more)))))) -(defn some [pred coll] +(defn some + {:doc "Checks if the predicate is true for any element of the collection. + +Stops if it finds such an element." + :signatures [[pred coll]] + :added "0.1"} + [pred coll] (cond (nil? (seq coll)) false (pred (first coll)) true @@ -757,12 +877,20 @@ (extend -invoke PersistentHashSet (fn [m k] (-val-at m k nil))) (defn get + {:doc "Get an element from a collection implementing ILookup, return nil or the default value if not found." + :added "0.1"} ([mp k] (get mp k nil)) ([mp k not-found] (-val-at mp k not-found))) (defn get-in + {:doc "Get a value from a nested collection at the \"path\" given by the keys. + +> (get-in {:a [{:b 42}]} [:a 0 :b]) +42" + :signatures [[m ks] [m ks not-found]] + :added "0.1"} ([m ks] (reduce get m ks)) ([m ks not-found] @@ -777,6 +905,12 @@ m)))) (defn assoc-in + {:doc "Associate a value in a nested collection given by the path. + +Creates new maps if the keys are not present. + +> (assoc-in {} [:a :b :c] 42) +{:a {:b {:c 42}}"} ([m ks v] (let [ks (seq ks) k (first ks) @@ -797,7 +931,10 @@ nil (throw (str "Assert failed " ~msg))))) -(defmacro resolve [sym] +(defmacro resolve + {:doc "Resolve the var associated with the symbol in the current namespace." + :added "0.1"} + [sym] `(resolve-in (this-ns-name) ~sym)) (defmacro with-bindings [binds & body] @@ -835,7 +972,24 @@ (defn protocol? [x] (instance? Protocol x)) -(defmacro deftype [nm fields & body] +(defmacro deftype + {:doc "Define a custom type. + +(deftype Person [name] + Object + (say-hi [self other-name] + (str \"Hi, I'm \" name \". You're \" other-name \", right?\")) + + IObject + (-str [self] + (str \"\")) + +> (.say-hi (->Person \"James\") \"Paul\") +; => \"Hi, I'm James. You're Paul, right?\" +> (str (->Person \"James\")) +; => \"" + :added "0.1"} + [nm fields & body] (let [ctor-name (symbol (str "->" (name nm))) fields (transduce (map (comp keyword name)) conj fields) field-syms (transduce (map (comp symbol name)) conj fields) @@ -895,7 +1049,13 @@ ~ctor ~@proto-bodies))) -(defmacro defrecord [nm fields & body] +(defmacro defrecord + {:doc "Define a record type. + +Similar to `deftype`, but supports construction from a map using `map->Type` +and implements IAssociative, ILookup and IObject." + :added "0.1"} + [nm fields & body] (let [ctor-name (symbol (str "->" (name nm))) map-ctor-name (symbol (str "map" (name ctor-name))) fields (transduce (map (comp keyword name)) conj fields) @@ -934,28 +1094,56 @@ (def printf (ffi-fn libc "printf" [String] Integer)) (def getenv (ffi-fn libc "getenv" [String] String)) -(defn print [& args] +(defn print + {:doc "Prints the arguments, seperated by spaces." + :added "0.1"} + [& args] (printf (transduce (interpose " ") str args))) -(defn println [& args] +(defn println + {:doc "Prints the arguments, separated by spaces, with a newline at the end." + :added "0.1"} + [& args] (puts (transduce (interpose " ") str args))) -(defn pr-str [& args] +(defn pr-str + {:doc "Formats the arguments using -repr, separated by spaces, returning a string." + :added "0.1"} + [& args] (transduce (comp (map -repr) (interpose " ")) str args)) -(defn pr [& args] +(defn pr + {:doc "Prints the arguments using -repr, separated by spaces." + :added "0.1"} + [& args] (printf (apply pr-str args))) -(defn prn [& args] +(defn prn + {:doc "Prints the arguments using -repr, separated by spaces, with a newline at the end." + :added "0.1"} + [& args] (puts (apply pr-str args))) -(defn doc [x] +(defn doc + {:doc "Returns the documentation of the given value." + :added "0.1"} + [x] (get (meta x) :doc)) -(defn swap! [a f & args] +(defn swap! + {:doc "Swaps the value in the atom, by applying f to the current value. + +The new value is thus `(apply f current-value-of-atom args)`." + :signatures [[atom f & args]] + :added "0.1"} + [a f & args] (reset! a (apply f @a args))) (defn update-in + {:doc "Update a value in a nested collection. + +> (update-in {:a [{:b 41}]} [:a 0 :b] inc) +{:a [{:b 42}]}"} [m ks f & args] (let [f (fn [m] (apply f m args)) update-inner-f (fn update-inner-f @@ -993,7 +1181,17 @@ (recur)))))) -(defmacro dotimes [bind & body] +(defmacro dotimes + {:doc "Execute the expressions in the body n times. + +> (dotimes [i 3] (println i)) +1 +2 +3 +; => nil" + :signatures [[[i n] & body]] + :added "0.1"} + [bind & body] (let [b (nth bind 0)] `(let [max# ~(nth bind 1)] (loop [~b 0] @@ -1008,12 +1206,30 @@ (yield (nth v x))))) (defmacro and + {:doc "Check if the given expressions return truthy values, returning the last, or false. + +> (and true false) +; => false +> (and 1 2 3) +; => 3 +> (and 1 false 3) +; => false" + :added "0.1"} ([] true) ([x] x) ([x y] `(if ~x ~y false)) ([x y & more] `(if ~x (and ~y ~@more)))) (defmacro or + {:doc "Returns the value of the first expression that returns a truthy value, or false. + +> (or 1 2 3) +; => 1 +> (or false 2) +; => 2 +> (or false nil) +; => nil" + :added "0.1"} ([] false) ([x] x) ([x y] `(let [r# ~x] @@ -1032,28 +1248,58 @@ (when ~(first binding) ~@body))) -(defn nnext [coll] +(defn nnext + {:doc "Equivalent to (next (next coll))" + :added "0.1"} + [coll] (next (next coll))) -(defn nthnext [coll n] +(defn nthnext + {:doc "Returns the result of calling next n times on the collection. + +> (nthnext [1 2 3 4 5] 2) +; => (3 4 5) +> (nthnext [1 2 3 4 5] 7) +; => nil" + :added "0.1"} + [coll n] (loop [n n xs (seq coll)] (if (and xs (pos? n)) (recur (dec n) (next xs)) xs))) -(defn take [n coll] +(defn take + {:doc "Takes n elements from the collection, or fewer, if not enough." + :added "0.1"} + [n coll] (when (pos? n) (when-let [s (seq coll)] (cons (first s) (take (dec n) (next s)))))) -(defn drop [n coll] +(defn drop + {:doc "Drops n elements from the start of the collection." + :added "0.1"} + [n coll] (let [s (seq coll)] (if (and (pos? n) s) (recur (dec n) (next s)) s))) (defn partition + {:doc "Separates the collection into collections of size n, starting at the beginning, with an optional step size. + +The last element of the result contains the remaining element, not necessarily of size n if +not enough elements were present. + +> (partition 2 [1 2 3 4 5 6]) +; => ((1 2) (3 4) (5 6)) +> (partition 2 [1 2 3 4 5]) +; => ((1 2) (3 4) (5)) +> (partition 2 1 [1 2 3 4 5]) +; => ((1 2) (2 3) (3 4) (4 5) (5))" + :signatures [[n coll] [n step coll]] + :added "0.1"} ([n coll] (partition n n coll)) ([n step coll] (when-let [s (seq coll)] @@ -1106,7 +1352,31 @@ res)] res)) -(defmacro let [bindings & body] +(defmacro let + {:doc "Makes the bindings availlable in the body. + +The bindings must be a vector of binding-expr pairs. The binding can be a destructuring +binding, as below. + +Vector destructuring: + [x y z] binds the first three elements of the collection to x, y and z + [x y & rest] binds rest to the elements after the first two elements of the collection + [x y :as v] binds the value of the complete collection to v + +Map destructuring: + {a :a, b :b} binds a and b to the values associated with :a and :b + {a :a :as m} binds the value of the complete collection to m + {a :a :or {a 42}} binds a to the value associated with :a, or 42, if not present + {:keys [a b c]} binds a, b and c to the values associated with :a, :b and :c + +All these forms can be combined and nested, in the example below: + +(let [[x y [z :as iv] :as v] [1 2 [3 4 5] 6 7] + {a :a [b c {:keys [d]}] :more :or {a 42}} {:a 1, :more [1 2 {:d 3, :e 4}]}] + ...) + +For more information, see http://clojure.org/special_forms#binding-forms"} + [bindings & body] (let* [destructured-bindings (transduce (map #(apply destructure %1)) concat [] @@ -1119,7 +1389,10 @@ (recur (next s) (dec n)) (first s)))) -(defn abs [x] +(defn abs + {:doc "Returns the absolute value of x." + :added "0.1"} + [x] (if (< x 0) (* -1 x) x)) @@ -1164,12 +1437,27 @@ (defn range + {:doc "Returns a range of numbers. + +> (seq (range 3)) +; => (0 1 2) +> (seq (range 3 5)) +; => (3 4) +> (seq (range 0 10 2)) +; => (0 2 4 6 8) +> (seq (range 5 -1 -1)) +; => (5 4 3 2 1 0)" + :signatures [[] [stop] [start stop] [start stop step]] + :added "0.1"} ([] (->Range 0 MAX-NUMBER 1)) ([stop] (->Range 0 stop 1)) ([start stop] (->Range start stop 1)) ([start stop step] (->Range start stop step))) -(defn iterator [coll] +(defn iterator + {:doc "Returns an iterator for the collection." + :added "0.1"} + [coll] (-iterator coll)) (defn move-next! [i] @@ -1221,27 +1509,37 @@ (-move-next! k) (recur acc))))))) -(defn filter [f] - (fn [xf] - (fn - ([] (xf)) - ([acc] (xf acc)) - ([acc i] (if (f i) - (xf acc i) - acc))))) - -(defn distinct [] - (fn [xf] - (let [seen (atom #{})] - (fn - ([] (xf)) - ([acc] (xf acc)) - ([acc i] - (if (contains? @seen i) - acc - (do - (swap! seen conj i) - (xf acc i)))))))) +(defn filter + {:doc "Filter the collection for elements matching the predicate." + :signatures [[pred] [pred coll]] + :added "0.1"} + ([f] (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (if (f i) + (xf acc i) + acc))))) + ([f coll] + (sequence (filter f) coll))) + +(defn distinct + {:doc "Returns the distinct elements in the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (fn [xf] + (let [seen (atom #{})] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] + (if (contains? @seen i) + acc + (do + (swap! seen conj i) + (xf acc i)))))))) + ([coll] + (sequence (distinct) coll))) (defn keep ([f] @@ -1259,7 +1557,24 @@ (if result (yield result)))))) -(defn refer [ns-sym & filters] +(defn refer + {:doc "Refer to the specified vars from a namespace directly. + +Supported filters: + :rename refer to the given vars under a different name + :exclude don't refer the given vars + :refer + :all refer all vars + :refer refer only the given vars + :only same as refer + +> (refer 'pixie.string :refer :all) +> (refer 'pixie.string :only '(index-of starts-with ends-with)) +> (refer 'pixie.string :rename '{index-of find}) +> (refer 'pixie.string :exclude '(substring)) +" + :added "0.1"} + [ns-sym & filters] (let [ns (or (the-ns ns-sym) (throw (str "No such namespace: " ns-sym))) filters (apply hashmap filters) nsmap (ns-map ns) @@ -1290,13 +1605,24 @@ (recur (next s)))))) (extend -at-end? EmptyList (fn [_] true)) -(defn every? [pred coll] +(defn every? + {:doc "Check if every element of the collection satisfies the predicate." + :added "0.1"} + [pred coll] (cond (nil? (seq coll)) true (pred (first coll)) (recur pred (next coll)) :else false)) -(defmacro fn [& decls] +(defmacro fn + {:doc "Creates a function. + +The following two forms are allowed: + (fn name? [param*] & body) + (fn name? ([param*] & body)+) + +The params can be destructuring bindings, see `(doc let)` for details."} + [& decls] (let [name (if (symbol? (first decls)) (first decls) nil) decls (if name (next decls) decls) name (or name '-fn) @@ -1333,7 +1659,19 @@ _ (assert method (str "no method defined for " dispatch-val))] (apply method dispatch-arg args)))) -(defmacro defmulti [name & args] +(defmacro defmulti + {:doc "Define a multimethod, which dispatches to it's methods based on dispatch-fn. + +(defmulti greet first) + +(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\")) +(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".)) + +> (greet [:hi \"Jane\"]) +; => \"Hi, Jane!\"" + :signatures [[name dispatch-fn & options]] + :added "0.1"} + [name & args] (let [[meta args] (if (string? (first args)) [{:doc (first args)} (next args)] [{} args]) @@ -1344,7 +1682,11 @@ options (apply hashmap (next args))] `(def ~name (->MultiMethod ~dispatch-fn ~(get options :default :default) (atom {}))))) -(defmacro defmethod [name dispatch-val params & body] +(defmacro defmethod + {:doc "Define a method of a multimethod. See `(doc defmulti)` for details." + :signatures [[name dispatch-val [param*] & body]] + :added "0.1"} + [name dispatch-val params & body] `(do (let [methods (.methods ~name)] (swap! methods From 017d59a7f2168f03bfb405bb8a737e79f2d32bac Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 20:07:26 +0100 Subject: [PATCH 299/909] fix typo --- target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target.py b/target.py index d611e74a..37c762cc 100644 --- a/target.py +++ b/target.py @@ -61,8 +61,8 @@ def inner_invoke(self, args): import pixie.vm.persistent_vector as vector print "Pixie 0.1 - Interactive REPL" - print ":exit-repl or Ctl-D to quit" print "(" + platform.name + ", " + platform.cc + ")" + print ":exit-repl or Ctrl-D to quit" print "----------------------------" with with_ns(u"user"): From 6c7ec82eee349b9ffd6d01c2ad070f71a7d1fd6d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 21:50:01 +0100 Subject: [PATCH 300/909] uncomment disj! for now, as -disj! doesn't exist yet --- pixie/stdlib.pxi | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 397c8892..f10cb95b 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -53,15 +53,15 @@ ([coll item & items] (reduce -disj (-disj coll item) items)))) -(def disj! (fn ^{:doc "Removes elements from the transient collection." - :signatures [[] [coll] [coll item] [coll item & items]] - :added "0.1"} - disj! - ([] (-transient [])) - ([result] (-persistent! result)) - ([result item] (-disj! result item)) - ([coll item & items] - (reduce -disj! (-disj! coll item) items)))) +(comment (def disj! (fn ^{:doc "Removes elements from the transient collection." + :signatures [[] [coll] [coll item] [coll item & items]] + :added "0.1"} + disj! + ([] (-transient [])) + ([result] (-persistent! result)) + ([result item] (-disj! result item)) + ([coll item & items] + (reduce -disj! (-disj! coll item) items))))) (def transient (fn [coll] (-transient coll))) From 30fe5d6e61b61825407fcff10b494c905ebb0316 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 21:53:47 +0100 Subject: [PATCH 301/909] add repeat --- pixie/stdlib.pxi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index f10cb95b..83cc88b9 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1124,6 +1124,12 @@ and implements IAssociative, ILookup and IObject." [& args] (puts (apply pr-str args))) +(defn repeat + ([x] + (cons x (lazy-seq* (fn [] (repeat x))))) + ([n x] + (take n (repeat x)))) + (defn doc {:doc "Returns the documentation of the given value." :added "0.1"} From a0024a150b0c6ac4480e24a87c7978ed22b09ab5 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 21:54:02 +0100 Subject: [PATCH 302/909] support printing the docs of all symbols in a namespace --- pixie/stdlib.pxi | 26 +++++++++++++++++++++++++- 1 file changed, 25 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 83cc88b9..8a3702da 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1134,7 +1134,31 @@ and implements IAssociative, ILookup and IObject." {:doc "Returns the documentation of the given value." :added "0.1"} [x] - (get (meta x) :doc)) + (let [doc (get (meta x) :doc)] + (cond + doc doc + (or (instance? Namespace x) (the-ns x)) (doc-ns x)))) + +(defn doc-ns + {:doc "Prints a summarizing documentation of the symbols in a namespace." + :added "0.1"} + [ns] + (let [ns (the-ns ns) + short-doc (fn [x] + (let [doc (get (meta x) :doc)] + (if doc + (let [newline (pixie.string/index-of doc "\n")] + (pixie.string/substring doc 0 (if (< newline 0) (count doc) newline))))))] + (println (str (name ns) ":")) + (vec (map (fn [sym] + (print (str " " (name sym))) + (let [doc (short-doc @(resolve-in ns sym))] + (if doc + (print (str (apply str (repeat (- 30 (count (name sym))) " ")) + doc)))) + (println)) + (keys (ns-map ns)))) + nil)) (defn swap! {:doc "Swaps the value in the atom, by applying f to the current value. From a591e2301d26f258b6e9c16adec93a156b57503c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 24 Nov 2014 22:21:18 +0100 Subject: [PATCH 303/909] print name and signatures of the value in question --- pixie/stdlib.pxi | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8a3702da..eff41d59 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1130,14 +1130,22 @@ and implements IAssociative, ILookup and IObject." ([n x] (take n (repeat x)))) -(defn doc +(defmacro doc {:doc "Returns the documentation of the given value." :added "0.1"} - [x] - (let [doc (get (meta x) :doc)] + [v] + (let [vr (resolve v) + x (if vr @vr) + doc (get (meta x) :doc)] (cond - doc doc - (or (instance? Namespace x) (the-ns x)) (doc-ns x)))) + doc (let [sigs (get (meta x) :signatures)] + (println (str (namespace vr) "/" (name vr))) + (if sigs + (prn (seq sigs))) + (println) + (println doc) + nil) + (the-ns v) (doc-ns v)))) (defn doc-ns {:doc "Prints a summarizing documentation of the symbols in a namespace." From 4e0f82af13985fa8fe8546e94577216ef359316b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 25 Nov 2014 17:08:50 +0100 Subject: [PATCH 304/909] fix typo --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index eff41d59..62901007 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -311,7 +311,7 @@ (def concat - (fn ^{:doc "Concatenates it's arguments." + (fn ^{:doc "Concatenates its arguments." :signatures [[& args]] :added "0.1"} concat From 0c3fe3969ca6119420283600c791daf50f69acd7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 25 Nov 2014 18:18:03 +0100 Subject: [PATCH 305/909] add doseq --- pixie/stdlib.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 62901007..4c7b8c7f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1130,6 +1130,19 @@ and implements IAssociative, ILookup and IObject." ([n x] (take n (repeat x)))) +(defmacro doseq + {:doc "Evaluates all elements of the seq, presumably for side effects. Returns nil." + :added "0.1"} + [binding & body] + (assert (= (count binding) 2) "expected a binding and a collection") + (let [b (first binding) + s (second binding)] + `(loop [s# (seq ~s)] + (if s# + (let [~b (first s#)] + ~@body + (recur (next s#))))))) + (defmacro doc {:doc "Returns the documentation of the given value." :added "0.1"} From 2af18284588216e769aca9e8083e425f6ad97f3d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 25 Nov 2014 18:18:11 +0100 Subject: [PATCH 306/909] move inline examples into a ^:examples metadata key they're vectors of strings (one per example) currently, and evaluated when being displayed. not sure about that, but at least it ensures they are correct. --- pixie/stdlib.pxi | 176 ++++++++++++++++------------------------------- 1 file changed, 59 insertions(+), 117 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4c7b8c7f..12f95d6a 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -346,12 +346,8 @@ (set-macro! defmacro) (defmacro -> - {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next. - -> (-> 3 inc inc) -; => 5 -> (-> \"James\" (str \"is \" \"awesome \") (str \"(and stuff)\" \"!\")) -; => \"James is awesome (and stuff)\"" + {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next." + :examples ["(-> 3 inc inc)" "(-> \"James\" (str \" is \" \"awesome \") (str \"(and stuff)\" \"!\"))"] :signatures [[x & forms]] :added "0.1"} [x & forms] @@ -365,12 +361,9 @@ x))) (defmacro ->> - {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next. - -> (->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \")) -; => \"you know what? we like James\" -> (->> 5 (range) (map inc) seq) -; => (1 2 3 4 5)" + {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next." + :examples ["(->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \"))" + "(->> 5 (range) (map inc) seq)"] :signatures [[x & forms]] :added "0.1"} [x & forms] @@ -599,23 +592,14 @@ returns true" (defn contains? {:doc "Checks if there is a value associated with key in the collection. -> (contains? {:a 1} :a) -; => true -> (contains? {:a 1} :b) -; => false -> (contains? #{:a :b :c} :a) -; => true - -Note the behaviour with indexed collections: - -> (contains? [:a :b :c] 0) -; => true -> (contains? [:a :b :c] 4) -; => false -> (contains? [:a :b :c] :a) -; => false - -Use some to check if a value is present in a collection." +Does *not* check for the presence of a value in the collection, only whether +there's a value associated with the key. Use `some` for checking for values." + :examples ["(contains? {:a 1} :a)" + "(contains? {:a 1} :b)" + "(contains? #{:a :b :c} :a)" + "(contains? [:a :b :c] 0)" + "(contains? [:a :b :c] 4)" + "(contains? [:a :b :c] :a)"] :signatures [[coll key]] :added "0.1"} [coll key] @@ -634,10 +618,8 @@ Use some to check if a value is present in a collection." (get-field inst :val)) (defn comp - {:doc "Composes the given functions, applying the last function first. - -> ((comp inc first) [41 2 3]) -; => 42" + {:doc "Composes the given functions, applying the last function first." + :examples ["((comp inc first) [41 2 3])"] :signatures [[f] [f & fs]] :added "0.1"} ([f] f) @@ -885,10 +867,8 @@ Stops if it finds such an element." (-val-at mp k not-found))) (defn get-in - {:doc "Get a value from a nested collection at the \"path\" given by the keys. - -> (get-in {:a [{:b 42}]} [:a 0 :b]) -42" + {:doc "Get a value from a nested collection at the \"path\" given by the keys." + :examples ["(get-in {:a [{:b 42}]} [:a 0 :b])"] :signatures [[m ks] [m ks not-found]] :added "0.1"} ([m ks] @@ -907,10 +887,9 @@ Stops if it finds such an element." (defn assoc-in {:doc "Associate a value in a nested collection given by the path. -Creates new maps if the keys are not present. - -> (assoc-in {} [:a :b :c] 42) -{:a {:b {:c 42}}"} +Creates new maps if the keys are not present." + :examples ["(assoc-in {} [:a :b :c] 42)"] + :added "0.1"} ([m ks v] (let [ks (seq ks) k (first ks) @@ -973,21 +952,17 @@ Creates new maps if the keys are not present. (instance? Protocol x)) (defmacro deftype - {:doc "Define a custom type. - -(deftype Person [name] + {:doc "Define a custom type." + :examples ["(deftype Person [name] Object (say-hi [self other-name] (str \"Hi, I'm \" name \". You're \" other-name \", right?\")) IObject (-str [self] - (str \"\")) - -> (.say-hi (->Person \"James\") \"Paul\") -; => \"Hi, I'm James. You're Paul, right?\" -> (str (->Person \"James\")) -; => \"" + (str \"\")))" + "(.say-hi (->Person \"James\") \"Paul\")" + "(str (->Person \"James\"))"] :added "0.1"} [nm fields & body] (let [ctor-name (symbol (str "->" (name nm))) @@ -1151,12 +1126,20 @@ and implements IAssociative, ILookup and IObject." x (if vr @vr) doc (get (meta x) :doc)] (cond - doc (let [sigs (get (meta x) :signatures)] + doc (let [sigs (get (meta x) :signatures) + examples (get (meta x) :examples)] (println (str (namespace vr) "/" (name vr))) (if sigs (prn (seq sigs))) (println) (println doc) + (if examples + (do + (println) + (doseq [example examples] + (println (str "user => " example)) + (println (eval (read-string example)))))) + (println) nil) (the-ns v) (doc-ns v)))) @@ -1191,10 +1174,9 @@ The new value is thus `(apply f current-value-of-atom args)`." (reset! a (apply f @a args))) (defn update-in - {:doc "Update a value in a nested collection. - -> (update-in {:a [{:b 41}]} [:a 0 :b] inc) -{:a [{:b 42}]}"} + {:doc "Update a value in a nested collection." + :examples ["(update-in {:a [{:b 41}]} [:a 0 :b] inc)"] + :added "0.1"} [m ks f & args] (let [f (fn [m] (apply f m args)) update-inner-f (fn update-inner-f @@ -1214,7 +1196,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro foreach [binding & body] (assert (= 2 (count binding)) "binding and collection required") `(reduce - (fn [_ ~ (nth binding 0)] + (fn [_ ~(nth binding 0)] ~@body nil) nil @@ -1233,13 +1215,8 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro dotimes - {:doc "Execute the expressions in the body n times. - -> (dotimes [i 3] (println i)) -1 -2 -3 -; => nil" + {:doc "Execute the expressions in the body n times." + :examples ["(dotimes [i 3] (println i))"] :signatures [[[i n] & body]] :added "0.1"} [bind & body] @@ -1257,14 +1234,8 @@ The new value is thus `(apply f current-value-of-atom args)`." (yield (nth v x))))) (defmacro and - {:doc "Check if the given expressions return truthy values, returning the last, or false. - -> (and true false) -; => false -> (and 1 2 3) -; => 3 -> (and 1 false 3) -; => false" + {:doc "Check if the given expressions return truthy values, returning the last, or false." + :examples ["(and true false)" "(and 1 2 3)" "(and 1 false 3)"] :added "0.1"} ([] true) ([x] x) @@ -1272,14 +1243,8 @@ The new value is thus `(apply f current-value-of-atom args)`." ([x y & more] `(if ~x (and ~y ~@more)))) (defmacro or - {:doc "Returns the value of the first expression that returns a truthy value, or false. - -> (or 1 2 3) -; => 1 -> (or false 2) -; => 2 -> (or false nil) -; => nil" + {:doc "Returns the value of the first expression that returns a truthy value, or false." + :examples ["(or 1 2 3)" "(or false 2)" "(or false nil)"] :added "0.1"} ([] false) ([x] x) @@ -1306,12 +1271,8 @@ The new value is thus `(apply f current-value-of-atom args)`." (next (next coll))) (defn nthnext - {:doc "Returns the result of calling next n times on the collection. - -> (nthnext [1 2 3 4 5] 2) -; => (3 4 5) -> (nthnext [1 2 3 4 5] 7) -; => nil" + {:doc "Returns the result of calling next n times on the collection." + :examples ["(nthnext [1 2 3 4 5] 2)" "(nthnext [1 2 3 4 5] 7)"] :added "0.1"} [coll n] (loop [n n @@ -1341,14 +1302,8 @@ The new value is thus `(apply f current-value-of-atom args)`." {:doc "Separates the collection into collections of size n, starting at the beginning, with an optional step size. The last element of the result contains the remaining element, not necessarily of size n if -not enough elements were present. - -> (partition 2 [1 2 3 4 5 6]) -; => ((1 2) (3 4) (5 6)) -> (partition 2 [1 2 3 4 5]) -; => ((1 2) (3 4) (5)) -> (partition 2 1 [1 2 3 4 5]) -; => ((1 2) (2 3) (3 4) (4 5) (5))" +not enough elements were present." + :examples ["(partition 2 [1 2 3 4 5 6])" "(partition 2 [1 2 3 4 5])" "(partition 2 1 [1 2 3 4 5])"] :signatures [[n coll] [n step coll]] :added "0.1"} ([n coll] (partition n n coll)) @@ -1488,16 +1443,8 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (defn range - {:doc "Returns a range of numbers. - -> (seq (range 3)) -; => (0 1 2) -> (seq (range 3 5)) -; => (3 4) -> (seq (range 0 10 2)) -; => (0 2 4 6 8) -> (seq (range 5 -1 -1)) -; => (5 4 3 2 1 0)" + {:doc "Returns a range of numbers." + :examples ["(seq (range 3))" "(seq (range 3 5))" "(seq (range 0 10 2))" "(seq (range 5 -1 -1))"] :signatures [[] [stop] [start stop] [start stop step]] :added "0.1"} ([] (->Range 0 MAX-NUMBER 1)) @@ -1619,11 +1566,10 @@ Supported filters: :refer refer only the given vars :only same as refer -> (refer 'pixie.string :refer :all) -> (refer 'pixie.string :only '(index-of starts-with ends-with)) -> (refer 'pixie.string :rename '{index-of find}) -> (refer 'pixie.string :exclude '(substring)) -" +user => (refer 'pixie.string :refer :all) +user => (refer 'pixie.string :only '(index-of starts-with ends-with)) +user => (refer 'pixie.string :rename '{index-of find}) +user => (refer 'pixie.string :exclude '(substring))" :added "0.1"} [ns-sym & filters] (let [ns (or (the-ns ns-sym) (throw (str "No such namespace: " ns-sym))) @@ -1711,15 +1657,11 @@ The params can be destructuring bindings, see `(doc let)` for details."} (apply method dispatch-arg args)))) (defmacro defmulti - {:doc "Define a multimethod, which dispatches to it's methods based on dispatch-fn. - -(defmulti greet first) - -(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\")) -(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".)) - -> (greet [:hi \"Jane\"]) -; => \"Hi, Jane!\"" + {:doc "Define a multimethod, which dispatches to it's methods based on dispatch-fn." + :examples ["(defmulti greet first)" + "(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\"))" + "(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".\"))" + "(greet [:hi \"Jane\"])"] :signatures [[name dispatch-fn & options]] :added "0.1"} [name & args] From ee5a4f0d4689d59c21c17a47f4cbf1610eed69f6 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 26 Nov 2014 09:48:16 +0100 Subject: [PATCH 307/909] represent ^:examples as a vector of vectors an example is a vector of [expr output val], where expr is a string containing a read-string'able expression, output is a string containing the output that is printed when evaluating the expr and val is the value returned. this allows checking the examples automatically, prettier doc output and potential use of this metadata. --- pixie/stdlib.pxi | 70 ++++++++++++++++++++++++++++-------------------- 1 file changed, 41 insertions(+), 29 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 12f95d6a..4bdb7b56 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -347,7 +347,8 @@ (defmacro -> {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next." - :examples ["(-> 3 inc inc)" "(-> \"James\" (str \" is \" \"awesome \") (str \"(and stuff)\" \"!\"))"] + :examples [["(-> 3 inc inc)" nil 5] + ["(-> \"James\" (str \" is \" \"awesome \") (str \"(and stuff)\" \"!\"))" nil "James is awesome (and stuff)!"]] :signatures [[x & forms]] :added "0.1"} [x & forms] @@ -362,8 +363,8 @@ (defmacro ->> {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next." - :examples ["(->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \"))" - "(->> 5 (range) (map inc) seq)"] + :examples [["(->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \"))" nil "you know what? we like James"] + ["(->> 5 (range) (map inc) seq)" nil '(1 2 3 4 5)]] :signatures [[x & forms]] :added "0.1"} [x & forms] @@ -594,12 +595,12 @@ returns true" Does *not* check for the presence of a value in the collection, only whether there's a value associated with the key. Use `some` for checking for values." - :examples ["(contains? {:a 1} :a)" - "(contains? {:a 1} :b)" - "(contains? #{:a :b :c} :a)" - "(contains? [:a :b :c] 0)" - "(contains? [:a :b :c] 4)" - "(contains? [:a :b :c] :a)"] + :examples [["(contains? {:a 1} :a)" nil true] + ["(contains? {:a 1} :b)" nil false] + ["(contains? #{:a :b :c} :a)" nil true] + ["(contains? [:a :b :c] 0)" nil true] + ["(contains? [:a :b :c] 4)" nil false] + ["(contains? [:a :b :c] :a)" nil false]] :signatures [[coll key]] :added "0.1"} [coll key] @@ -619,7 +620,7 @@ there's a value associated with the key. Use `some` for checking for values." (defn comp {:doc "Composes the given functions, applying the last function first." - :examples ["((comp inc first) [41 2 3])"] + :examples [["((comp inc first) [41 2 3])" nil 42]] :signatures [[f] [f & fs]] :added "0.1"} ([f] f) @@ -868,7 +869,7 @@ Stops if it finds such an element." (defn get-in {:doc "Get a value from a nested collection at the \"path\" given by the keys." - :examples ["(get-in {:a [{:b 42}]} [:a 0 :b])"] + :examples [["(get-in {:a [{:b 42}]} [:a 0 :b])" nil 42]] :signatures [[m ks] [m ks not-found]] :added "0.1"} ([m ks] @@ -888,7 +889,7 @@ Stops if it finds such an element." {:doc "Associate a value in a nested collection given by the path. Creates new maps if the keys are not present." - :examples ["(assoc-in {} [:a :b :c] 42)"] + :examples [["(assoc-in {} [:a :b :c] 42)" nil {:a {:b {:c 42}}}]] :added "0.1"} ([m ks v] (let [ks (seq ks) @@ -953,16 +954,16 @@ Creates new maps if the keys are not present." (defmacro deftype {:doc "Define a custom type." - :examples ["(deftype Person [name] + :examples [["(deftype Person [name] Object (say-hi [self other-name] (str \"Hi, I'm \" name \". You're \" other-name \", right?\")) IObject (-str [self] - (str \"\")))" - "(.say-hi (->Person \"James\") \"Paul\")" - "(str (->Person \"James\"))"] + (str \"\")))"] + ["(.say-hi (->Person \"James\") \"Paul\")" nil "Hi, I'm James. You're Paul, right?"] + ["(str (->Person \"James\"))" nil ""]] :added "0.1"} [nm fields & body] (let [ctor-name (symbol (str "->" (name nm))) @@ -1137,8 +1138,11 @@ and implements IAssociative, ILookup and IObject." (do (println) (doseq [example examples] - (println (str "user => " example)) - (println (eval (read-string example)))))) + (println (str " user => " (first example))) + (if (second example) + (print (apply str (map #(str " " % "\n") (pixie.string/split (second example) "\n"))))) + (if (contains? example 2) + (println (str " " (-repr (third example)))))))) (println) nil) (the-ns v) (doc-ns v)))) @@ -1175,7 +1179,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn update-in {:doc "Update a value in a nested collection." - :examples ["(update-in {:a [{:b 41}]} [:a 0 :b] inc)"] + :examples [["(update-in {:a [{:b 41}]} [:a 0 :b] inc)" nil {:a [{:b 42}]}]] :added "0.1"} [m ks f & args] (let [f (fn [m] (apply f m args)) @@ -1216,7 +1220,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro dotimes {:doc "Execute the expressions in the body n times." - :examples ["(dotimes [i 3] (println i))"] + :examples [["(dotimes [i 3] (println i))" "1\n2\n3\n"]] :signatures [[[i n] & body]] :added "0.1"} [bind & body] @@ -1235,7 +1239,9 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro and {:doc "Check if the given expressions return truthy values, returning the last, or false." - :examples ["(and true false)" "(and 1 2 3)" "(and 1 false 3)"] + :examples [["(and true false)" nil false] + ["(and 1 2 3)" nil 3] + ["(and 1 false 3)" nil false]] :added "0.1"} ([] true) ([x] x) @@ -1244,7 +1250,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro or {:doc "Returns the value of the first expression that returns a truthy value, or false." - :examples ["(or 1 2 3)" "(or false 2)" "(or false nil)"] + :examples [["(or 1 2 3)" nil 1] ["(or false 2)" nil 2] ["(or false nil)" nil nil]] :added "0.1"} ([] false) ([x] x) @@ -1272,7 +1278,8 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn nthnext {:doc "Returns the result of calling next n times on the collection." - :examples ["(nthnext [1 2 3 4 5] 2)" "(nthnext [1 2 3 4 5] 7)"] + :examples [["(nthnext [1 2 3 4 5] 2)" nil '(3 4 5)] + ["(nthnext [1 2 3 4 5] 7)" nil nil]] :added "0.1"} [coll n] (loop [n n @@ -1303,7 +1310,9 @@ The new value is thus `(apply f current-value-of-atom args)`." The last element of the result contains the remaining element, not necessarily of size n if not enough elements were present." - :examples ["(partition 2 [1 2 3 4 5 6])" "(partition 2 [1 2 3 4 5])" "(partition 2 1 [1 2 3 4 5])"] + :examples [["(partition 2 [1 2 3 4 5 6])" nil '((1 2) (3 4) (5 6))] + ["(partition 2 [1 2 3 4 5])" nil '((1 2) (3 4) (5))] + ["(partition 2 1 [1 2 3 4 5])" nil '((1 2) (2 3) (3 4) (4 5) (5))]] :signatures [[n coll] [n step coll]] :added "0.1"} ([n coll] (partition n n coll)) @@ -1444,7 +1453,10 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (defn range {:doc "Returns a range of numbers." - :examples ["(seq (range 3))" "(seq (range 3 5))" "(seq (range 0 10 2))" "(seq (range 5 -1 -1))"] + :examples [["(seq (range 3))" nil '(0 1 2)] + ["(seq (range 3 5))" nil '(3 4)] + ["(seq (range 0 10 2))" nil '(0 2 4 6 8)] + ["(seq (range 5 -1 -1))" nil '(5 4 3 2 1 0)]] :signatures [[] [stop] [start stop] [start stop step]] :added "0.1"} ([] (->Range 0 MAX-NUMBER 1)) @@ -1658,10 +1670,10 @@ The params can be destructuring bindings, see `(doc let)` for details."} (defmacro defmulti {:doc "Define a multimethod, which dispatches to it's methods based on dispatch-fn." - :examples ["(defmulti greet first)" - "(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\"))" - "(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".\"))" - "(greet [:hi \"Jane\"])"] + :examples [["(defmulti greet first)"] + ["(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\"))"] + ["(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".\"))"] + ["(greet [:hi \"Jane\"])" nil "Hi, Jane!"]] :signatures [[name dispatch-fn & options]] :added "0.1"} [name & args] From 8b5c7fc5abd9f04696a7c2dc033f8fe920b8a297 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 26 Nov 2014 11:21:16 +0100 Subject: [PATCH 308/909] test doc examples as part of the test suite --- pixie/stdlib.pxi | 20 ++++++++++---------- tests/test-docs.pxi | 21 +++++++++++++++++++++ 2 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 tests/test-docs.pxi diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4bdb7b56..4604ee9e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -364,7 +364,7 @@ (defmacro ->> {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next." :examples [["(->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \"))" nil "you know what? we like James"] - ["(->> 5 (range) (map inc) seq)" nil '(1 2 3 4 5)]] + ["(->> 5 (range) (map inc) seq)" nil (1 2 3 4 5)]] :signatures [[x & forms]] :added "0.1"} [x & forms] @@ -1179,7 +1179,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn update-in {:doc "Update a value in a nested collection." - :examples [["(update-in {:a [{:b 41}]} [:a 0 :b] inc)" nil {:a [{:b 42}]}]] +; :examples [["(update-in {:a [{:b 41}]} [:a 0 :b] inc)" nil {:a [{:b 42}]}]] :added "0.1"} [m ks f & args] (let [f (fn [m] (apply f m args)) @@ -1278,7 +1278,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn nthnext {:doc "Returns the result of calling next n times on the collection." - :examples [["(nthnext [1 2 3 4 5] 2)" nil '(3 4 5)] + :examples [["(nthnext [1 2 3 4 5] 2)" nil (3 4 5)] ["(nthnext [1 2 3 4 5] 7)" nil nil]] :added "0.1"} [coll n] @@ -1310,9 +1310,9 @@ The new value is thus `(apply f current-value-of-atom args)`." The last element of the result contains the remaining element, not necessarily of size n if not enough elements were present." - :examples [["(partition 2 [1 2 3 4 5 6])" nil '((1 2) (3 4) (5 6))] - ["(partition 2 [1 2 3 4 5])" nil '((1 2) (3 4) (5))] - ["(partition 2 1 [1 2 3 4 5])" nil '((1 2) (2 3) (3 4) (4 5) (5))]] + :examples [["(partition 2 [1 2 3 4 5 6])" nil ((1 2) (3 4) (5 6))] + ["(partition 2 [1 2 3 4 5])" nil ((1 2) (3 4) (5))] + ["(partition 2 1 [1 2 3 4 5])" nil ((1 2) (2 3) (3 4) (4 5) (5))]] :signatures [[n coll] [n step coll]] :added "0.1"} ([n coll] (partition n n coll)) @@ -1453,10 +1453,10 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (defn range {:doc "Returns a range of numbers." - :examples [["(seq (range 3))" nil '(0 1 2)] - ["(seq (range 3 5))" nil '(3 4)] - ["(seq (range 0 10 2))" nil '(0 2 4 6 8)] - ["(seq (range 5 -1 -1))" nil '(5 4 3 2 1 0)]] + :examples [["(seq (range 3))" nil (0 1 2)] + ["(seq (range 3 5))" nil (3 4)] + ["(seq (range 0 10 2))" nil (0 2 4 6 8)] + ["(seq (range 5 -1 -1))" nil (5 4 3 2 1 0)]] :signatures [[] [stop] [start stop] [start stop step]] :added "0.1"} ([] (->Range 0 MAX-NUMBER 1)) diff --git a/tests/test-docs.pxi b/tests/test-docs.pxi new file mode 100644 index 00000000..fe36c0e8 --- /dev/null +++ b/tests/test-docs.pxi @@ -0,0 +1,21 @@ +(ns pixie.tests.test-docs + (require pixie.test :as t)) + +; validate the examples in the docs by checking whether the included +; results match the actual results you get by evaluating the examples. + +(defn check-examples [ns] + (let [ns (the-ns ns) + syms (keys (ns-map ns))] + (doseq [sym syms] + (let [meta (meta @(resolve sym)) + examples (get meta :examples)] + (doseq [example examples] + (if (contains? example 2) + (do + (t/assert= (eval (read-string (first example))) + (third example))) + (eval (read-string (first example))))))))) + +(t/deftest test-stdlib-docs + (check-examples 'pixie.stdlib)) From 8fbabb3d892dc0f8c4137b67dfb346aa02097a69 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 26 Nov 2014 22:02:44 +0100 Subject: [PATCH 309/909] extract signatures from defn if not explicitely specified --- pixie/stdlib.pxi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4604ee9e..f0da865d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -332,6 +332,12 @@ (merge meta (first rest)) meta) rest (if (satisfies? IMap (first rest)) (next rest) rest) + meta (if (-contains-key meta :signatures) + meta + (merge meta {:signatures + (if (satisfies? IVector (first rest)) + [(first rest)] + (transduce (map first) conj rest))})) nm (with-meta nm meta)] `(def ~nm (fn ~nm ~@rest))))) (set-macro! defn) From 8137ca52c598f84db0acbd093f62f7ff8bfc03d1 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 27 Nov 2014 11:58:30 +0100 Subject: [PATCH 310/909] fix update-in --- pixie/stdlib.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index f0da865d..a98ae450 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1185,7 +1185,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn update-in {:doc "Update a value in a nested collection." -; :examples [["(update-in {:a [{:b 41}]} [:a 0 :b] inc)" nil {:a [{:b 42}]}]] + :examples [["(update-in {:a {:b {:c 41}}} [:a :b :c] inc)" nil {:a {:b {:c 42}}}]] :added "0.1"} [m ks f & args] (let [f (fn [m] (apply f m args)) @@ -1193,7 +1193,7 @@ The new value is thus `(apply f current-value-of-atom args)`." ([m f k] (assoc m k (f (get m k)))) ([m f k & ks] - (assoc m k (apply update-inner-f m f ks))))] + (assoc m k (apply update-inner-f (get m k) f ks))))] (apply update-inner-f m f ks))) (defn nil? [x] From b50a37bdefb9fd0be852f007252541fd0d57e2ce Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 27 Nov 2014 12:05:06 +0100 Subject: [PATCH 311/909] print signatures even if no docstring is present --- pixie/stdlib.pxi | 40 +++++++++++++++++++++------------------- 1 file changed, 21 insertions(+), 19 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a98ae450..cfce74e2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1131,26 +1131,28 @@ and implements IAssociative, ILookup and IObject." [v] (let [vr (resolve v) x (if vr @vr) - doc (get (meta x) :doc)] + doc (get (meta x) :doc) + has-doc? (if doc true (get (meta x) :signatures))] (cond - doc (let [sigs (get (meta x) :signatures) - examples (get (meta x) :examples)] - (println (str (namespace vr) "/" (name vr))) - (if sigs - (prn (seq sigs))) - (println) - (println doc) - (if examples - (do - (println) - (doseq [example examples] - (println (str " user => " (first example))) - (if (second example) - (print (apply str (map #(str " " % "\n") (pixie.string/split (second example) "\n"))))) - (if (contains? example 2) - (println (str " " (-repr (third example)))))))) - (println) - nil) + has-doc? (let [sigs (get (meta x) :signatures) + examples (get (meta x) :examples)] + (println (str (namespace vr) "/" (name vr))) + (if sigs + (prn (seq sigs))) + (if doc + (do (println) + (println doc))) + (if examples + (do + (println) + (doseq [example examples] + (println (str " user => " (first example))) + (if (second example) + (print (apply str (map #(str " " % "\n") (pixie.string/split (second example) "\n"))))) + (if (contains? example 2) + (println (str " " (-repr (third example)))))))) + (println) + nil) (the-ns v) (doc-ns v)))) (defn doc-ns From f43811f823918c60ea6c10ab06e19ea48d270b57 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 27 Nov 2014 12:06:57 +0100 Subject: [PATCH 312/909] add signatures for map --- pixie/stdlib.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index cfce74e2..13b6cfbd 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -82,6 +82,7 @@ (f result))))) (def map (fn ^{:doc "map - creates a transducer that applies f to every input element" + :signatures [[f] [f coll]] :added "0.1"} map ([f] From af18da2573ff3cd049692b266689b2a40b57b51a Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 27 Nov 2014 12:20:36 +0100 Subject: [PATCH 313/909] move update-in closer to *-in and add tests --- pixie/stdlib.pxi | 26 +++++++++++++------------- tests/test-stdlib.pxi | 8 ++++++++ 2 files changed, 21 insertions(+), 13 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 13b6cfbd..e7439988 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -906,6 +906,19 @@ Creates new maps if the keys are not present." (assoc m k (assoc-in (get m k) ks v)) (assoc m k v))))) +(defn update-in + {:doc "Update a value in a nested collection." + :examples [["(update-in {:a {:b {:c 41}}} [:a :b :c] inc)" nil {:a {:b {:c 42}}}]] + :added "0.1"} + [m ks f & args] + (let [f (fn [m] (apply f m args)) + update-inner-f (fn update-inner-f + ([m f k] + (assoc m k (f (get m k)))) + ([m f k & ks] + (assoc m k (apply update-inner-f (get m k) f ks))))] + (apply update-inner-f m f ks))) + (def subs pixie.string/substring) (defmacro assert @@ -1186,19 +1199,6 @@ The new value is thus `(apply f current-value-of-atom args)`." [a f & args] (reset! a (apply f @a args))) -(defn update-in - {:doc "Update a value in a nested collection." - :examples [["(update-in {:a {:b {:c 41}}} [:a :b :c] inc)" nil {:a {:b {:c 42}}}]] - :added "0.1"} - [m ks f & args] - (let [f (fn [m] (apply f m args)) - update-inner-f (fn update-inner-f - ([m f k] - (assoc m k (f (get m k)))) - ([m f k & ks] - (assoc m k (apply update-inner-f (get m k) f ks))))] - (apply update-inner-f m f ks))) - (defn nil? [x] (identical? x nil)) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 2c5ea1a4..2a14e03a 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -147,6 +147,14 @@ (t/assert= (get-in m [:missing] :not-found) :not-found) (t/assert= (get-in m [:x :x 0] :not-found) 1))) +(t/deftest test-assoc-in + (t/assert= (assoc-in {:a {:b 2}} [:a :b] 3) {:a {:b 3}}) + (t/assert= (assoc-in {} [:a :b] 3) {:a {:b 3}})) + +(t/deftest test-update-in + (t/assert= (update-in {} [:a :b] (fnil inc 0)) {:a {:b 1}}) + (t/assert= (update-in {:a {:b 2}} [:a :b] inc) {:a {:b 3}})) + (t/deftest test-fn? (t/assert= (fn? inc) true) (t/assert= (fn? {}) true) From bd51c59570c446dd8373afec7225b56316315084 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 27 Nov 2014 16:30:01 +0100 Subject: [PATCH 314/909] support assoc on vectors --- pixie/vm/persistent_vector.py | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 822cc22b..4315c668 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -154,6 +154,27 @@ def pop_tail(self, level, node): ret._array[sub_idx] = None return ret + def assoc_at(self, idx, val): + if idx >= 0 and idx < self._cnt: + if idx >= self.tailoff(): + new_tail = self._tail[:] + new_tail[idx & 0x01f] = val + return PersistentVector(self._meta, self._cnt, self._shift, self._root, new_tail) + return PersistentVector(self._meta, self._cnt, self._shift, do_assoc(self._shift, self._root, idx, val), self._tail) + if idx == self._cnt: + return self.conj(val) + else: + object.runtime_error(u"index out of range") + +def do_assoc(lvl, node, idx, val): + ret = Node(node._edit, node._array[:]) + if lvl == 0: + ret._array[idx & 0x01f] = val + else: + subidx = (idx >> lvl) & 0x01f + ret._array[subidx] = do_assoc(lvl - 5, node._array[subidx], idx, val) + return ret + def new_path(edit, level, node): if level == 0: return node @@ -425,6 +446,12 @@ def _push(self): assert isinstance(self, PersistentVector) return self.pop() +@extend(proto._assoc, PersistentVector) +def _assoc(self, idx, val): + assert isinstance(self, PersistentVector) + affirm(isinstance(idx, Integer), u"key must be an integer") + return self.assoc_at(r_uint(idx.int_val()), val) + @extend(proto._meta, PersistentVector) def _meta(self): assert isinstance(self, PersistentVector) From d344894dff345c813be224a34cf540d0f2159a7d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 13:26:31 +0100 Subject: [PATCH 315/909] test assoc, assoc-in and update-in on vectors --- tests/test-stdlib.pxi | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 2a14e03a..316dc4a6 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -140,6 +140,14 @@ (t/assert= (vec (keep pos?) v) [true true true true true]) (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)))) +(t/deftest test-assoc + (t/assert= (assoc {} :a 3) {:a 3}) + (t/assert= (assoc {:a 1} :a 3) {:a 3}) + + (t/assert= (assoc [] 0 :ok) [:ok]) + (t/assert= (assoc [1] 0 :ok) [:ok]) + (t/assert= (assoc [1 2 3] 1 :ok) [1 :ok 3])) + (t/deftest test-get-in (let [m {:a 1 :b 2 :x {:a 2 :x [1 2 3]}}] (t/assert= (get-in m [:a]) 1) @@ -149,11 +157,15 @@ (t/deftest test-assoc-in (t/assert= (assoc-in {:a {:b 2}} [:a :b] 3) {:a {:b 3}}) - (t/assert= (assoc-in {} [:a :b] 3) {:a {:b 3}})) + (t/assert= (assoc-in {:a [{:b 2}]} [:a 0 :b] 3) {:a [{:b 3}]}) + ; non existing keys create maps (not vectors, even if the keys are integers) + (t/assert= (assoc-in {} [:a :b] 3) {:a {:b 3}}) + (t/assert= (assoc-in {} [:a 0 :b] 3) {:a {0 {:b 3}}})) (t/deftest test-update-in (t/assert= (update-in {} [:a :b] (fnil inc 0)) {:a {:b 1}}) - (t/assert= (update-in {:a {:b 2}} [:a :b] inc) {:a {:b 3}})) + (t/assert= (update-in {:a {:b 2}} [:a :b] inc) {:a {:b 3}}) + (t/assert= (update-in {:a [{:b 2}]} [:a 0 :b] inc) {:a [{:b 3}]})) (t/deftest test-fn? (t/assert= (fn? inc) true) From c094bd3821afbb9aafdfa8f5e70818964deeeabd Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 15:04:01 +0100 Subject: [PATCH 316/909] export native string functions in an internal namespace to allow extending pixie.string using code written in pixie. (coming soon.) --- pixie/stdlib.pxi | 8 ++++---- pixie/vm/libs/string.py | 22 +++++++++++----------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e7439988..4212d381 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -919,7 +919,7 @@ Creates new maps if the keys are not present." (assoc m k (apply update-inner-f (get m k) f ks))))] (apply update-inner-f m f ks))) -(def subs pixie.string/substring) +(def subs pixie.string.internal/substring) (defmacro assert ([test] @@ -1162,7 +1162,7 @@ and implements IAssociative, ILookup and IObject." (doseq [example examples] (println (str " user => " (first example))) (if (second example) - (print (apply str (map #(str " " % "\n") (pixie.string/split (second example) "\n"))))) + (print (apply str (map #(str " " % "\n") (pixie.string.internal/split (second example) "\n"))))) (if (contains? example 2) (println (str " " (-repr (third example)))))))) (println) @@ -1177,8 +1177,8 @@ and implements IAssociative, ILookup and IObject." short-doc (fn [x] (let [doc (get (meta x) :doc)] (if doc - (let [newline (pixie.string/index-of doc "\n")] - (pixie.string/substring doc 0 (if (< newline 0) (count doc) newline))))))] + (let [newline (pixie.string.internal/index-of doc "\n")] + (pixie.string.internal/substring doc 0 (if (< newline 0) (count doc) newline))))))] (println (str (name ns) ":")) (vec (map (fn [sym] (print (str " " (name sym))) diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 09eecf05..28f2f281 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -13,16 +13,16 @@ -@as_var("pixie.string", "starts-with") +@as_var("pixie.string.internal", "starts-with") def startswith(a, b): return rt.wrap(rt.name(a).startswith(rt.name(b))) -@as_var("pixie.string", "ends-with") +@as_var("pixie.string.internal", "ends-with") def endswith(a, b): return rt.wrap(rt.name(a).endswith(rt.name(b))) -@as_var("pixie.string", "split") +@as_var("pixie.string.internal", "split") def split(a, b): affirm(rt.count(b) > 0, u"separator can't be empty") v = rt.vector() @@ -50,7 +50,7 @@ def index_of4(a, sep, start, end): else: runtime_error(u"Third and fourth argument must be non-negative integers") -index_of = intern_var(u"pixie.string", u"index-of") +index_of = intern_var(u"pixie.string.internal", u"index-of") index_of.set_root(MultiArityFn({2: wrap_fn(index_of2), 3: wrap_fn(index_of3), 4: wrap_fn(index_of4)}, required_arity = 2)) @@ -67,11 +67,11 @@ def substring3(a, start, end): else: runtime_error(u"Second and third argument must be non-negative integers") -substring = intern_var(u"pixie.string", u"substring") +substring = intern_var(u"pixie.string.internal", u"substring") substring.set_root(MultiArityFn({2: wrap_fn(substring2), 3: wrap_fn(substring3)}, required_arity = 2)) -@as_var("pixie.string", "upper-case") +@as_var("pixie.string.internal", "upper-case") def upper_case(a): a = rt.name(a) res = "" @@ -79,7 +79,7 @@ def upper_case(a): res += chr(unicodedb.toupper(ord(ch))) return rt.wrap(res) -@as_var("pixie.string", "lower-case") +@as_var("pixie.string.internal", "lower-case") def lower_case(a): a = rt.name(a) res = "" @@ -87,7 +87,7 @@ def lower_case(a): res += chr(unicodedb.tolower(ord(ch))) return rt.wrap(res) -@as_var("pixie.string", "capitalize") +@as_var("pixie.string.internal", "capitalize") def capitalize(a): a = rt.name(a) res = u"" @@ -95,7 +95,7 @@ def capitalize(a): res += a[1:] return rt.wrap(res) -@as_var("pixie.string", "trim") +@as_var("pixie.string.internal", "trim") def trim(a): a = rt.name(a) i = 0 @@ -108,7 +108,7 @@ def trim(a): return rt.wrap(u"") return rt.wrap(a[i:j]) -@as_var("pixie.string", "triml") +@as_var("pixie.string.internal", "triml") def triml(a): a = rt.name(a) i = 0 @@ -116,7 +116,7 @@ def triml(a): i += 1 return rt.wrap(a[i:len(a)]) -@as_var("pixie.string", "trimr") +@as_var("pixie.string.internal", "trimr") def trimr(a): a = rt.name(a) j = len(a) From 7a3ad9a47ddc908027e3329b0ef4c939e708a529 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 15:05:48 +0100 Subject: [PATCH 317/909] add pixie.string ns back again for now just reexporting the fns from pixie.string.internal, but now extendable from pixie. --- pixie/string.pxi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 pixie/string.pxi diff --git a/pixie/string.pxi b/pixie/string.pxi new file mode 100644 index 00000000..b0b2b069 --- /dev/null +++ b/pixie/string.pxi @@ -0,0 +1,18 @@ +(ns pixie.string + (require pixie.string.internal :as si)) + +; reexport native string functions +(def substring si/substring) +(def index-of si/index-of) +(def split si/split) + +(def ends-with si/ends-with) +(def starts-with si/starts-with) + +(def trim si/trim) +(def triml si/triml) +(def trimr si/trimr) + +(def capitalize si/capitalize) +(def lower-case si/lower-case) +(def upper-case si/upper-case) From a1115f966bd63650c387fcf4500b307e90e71b0d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 15:11:51 +0100 Subject: [PATCH 318/909] allow 0 as indices to index-of in all arities --- pixie/vm/libs/string.py | 4 ++-- tests/test-strings.pxi | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 28f2f281..4772118d 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -36,7 +36,7 @@ def index_of2(a, sep): def index_of3(a, sep, start): affirm(isinstance(start, Integer), u"Third argument must be an integer") start = start.int_val() - if start > 0: + if start >= 0: return rt.wrap(rt.name(a).find(rt.name(sep), start)) else: runtime_error(u"Third argument must be a non-negative integer") @@ -45,7 +45,7 @@ def index_of4(a, sep, start, end): affirm(isinstance(start, Integer) and isinstance(end, Integer), u"Third and fourth argument must be integers") start = start.int_val() end = end.int_val() - if start > 0 and end > 0: + if start >= 0 and end >= 0: return rt.wrap(rt.name(a).find(rt.name(sep), start, end)) else: runtime_error(u"Third and fourth argument must be non-negative integers") diff --git a/tests/test-strings.pxi b/tests/test-strings.pxi index 6a622545..b4daf9f7 100644 --- a/tests/test-strings.pxi +++ b/tests/test-strings.pxi @@ -35,8 +35,10 @@ (t/assert= (s/index-of s "h" 2) 3) (t/assert= (s/index-of s "h" 4) 5) + (t/assert= (s/index-of s "hey" 0) 0) (t/assert= (s/index-of s "hey" 1) -1) + (t/assert= (s/index-of s "h" 0 0) -1) (t/assert= (s/index-of s "h" 1 2) -1))) (t/deftest test-substring From a07efb02f76f309870808d6fcf7dddaa9df07e98 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 15:34:59 +0100 Subject: [PATCH 319/909] implement replace and replace-first in pixie.string --- pixie/string.pxi | 19 +++++++++++++++++++ tests/test-strings.pxi | 12 ++++++++++++ 2 files changed, 31 insertions(+) diff --git a/pixie/string.pxi b/pixie/string.pxi index b0b2b069..63242609 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -16,3 +16,22 @@ (def capitalize si/capitalize) (def lower-case si/lower-case) (def upper-case si/upper-case) + +(defn replace + "Replace all occurrences of x in s with r." + [s x r] + (let [offset (if (zero? (count x)) (+ 1 (count r)) (count r))] + (loop [start 0 + s s] + (let [i (index-of s x start)] + (if (neg? i) + s + (recur (+ i offset) (str (substring s 0 i) r (substring s (+ i (count x)))))))))) + +(defn replace-first + "Replace the first occurrence of x in s with r." + [s x r] + (let [i (index-of s x)] + (if (neg? i) + s + (str (substring s 0 i) r (substring s (+ i (count x))))))) diff --git a/tests/test-strings.pxi b/tests/test-strings.pxi index b4daf9f7..230f7547 100644 --- a/tests/test-strings.pxi +++ b/tests/test-strings.pxi @@ -89,6 +89,18 @@ (t/assert= (s/trimr " hey ") " hey") (t/assert= (s/trimr " h ey ") " h ey")) +(t/deftest test-replace + (t/assert= (s/replace "hey,you,there" "," ", ") "hey, you, there") + (t/assert= (s/replace "hey,you,there" "," "") "heyyouthere") + (t/assert= (s/replace "&&&" "&" "&&") "&&&&&&") + (t/assert= (s/replace "oops" "" "WAT") "WAToWAToWATpWATsWAT")) + +(t/deftest test-replace-first + (t/assert= (s/replace-first "hey,you,there" "," ", ") "hey, you,there") + (t/assert= (s/replace-first "hey,you,there" "," "") "heyyou,there") + (t/assert= (s/replace-first "&&&" "&" "&&") "&&&&") + (t/assert= (s/replace-first "oops" "" "WAT") "WAToops")) + (t/deftest test-char-literals (let [s "hey"] (t/assert= (nth s 0) \h) From 15eb83aca1b7005d1e1d5ecc7accaa3b1a692200 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 15:49:41 +0100 Subject: [PATCH 320/909] add pixie.string/join please welcome test #101, btw. :sparkles: --- pixie/string.pxi | 14 ++++++++++++++ tests/test-docs.pxi | 6 +++++- tests/test-strings.pxi | 10 ++++++++++ 3 files changed, 29 insertions(+), 1 deletion(-) diff --git a/pixie/string.pxi b/pixie/string.pxi index 63242609..574de008 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -35,3 +35,17 @@ (if (neg? i) s (str (substring s 0 i) r (substring s (+ i (count x))))))) + +(defn join + {:doc "Join the elements of the collection using an optional seperator." + :examples [["(require pixie.string :as s)"] + ["(s/join [1 2 3])" nil "123"] + ["(s/join \", \" [1 2 3])" nil "1, 2, 3"]]} + ([coll] (join "" coll)) + ([seperator coll] + (loop [s (seq coll) + res ""] + (cond + (nil? s) res + (nil? (next s)) (str res (first s)) + :else (recur (next s) (str res (first s) seperator)))))) diff --git a/tests/test-docs.pxi b/tests/test-docs.pxi index fe36c0e8..b8f66996 100644 --- a/tests/test-docs.pxi +++ b/tests/test-docs.pxi @@ -8,7 +8,7 @@ (let [ns (the-ns ns) syms (keys (ns-map ns))] (doseq [sym syms] - (let [meta (meta @(resolve sym)) + (let [meta (meta @(resolve-in ns sym)) examples (get meta :examples)] (doseq [example examples] (if (contains? example 2) @@ -19,3 +19,7 @@ (t/deftest test-stdlib-docs (check-examples 'pixie.stdlib)) + +(t/deftest test-string-docs + (load-ns 'pixie.string) + (check-examples 'pixie.string)) diff --git a/tests/test-strings.pxi b/tests/test-strings.pxi index 230f7547..5f8a5540 100644 --- a/tests/test-strings.pxi +++ b/tests/test-strings.pxi @@ -101,6 +101,16 @@ (t/assert= (s/replace-first "&&&" "&" "&&") "&&&&") (t/assert= (s/replace-first "oops" "" "WAT") "WAToops")) +(t/deftest test-join + (t/assert= (s/join []) "") + (t/assert= (s/join [1]) "1") + (t/assert= (s/join [1 2 3]) "123") + + (t/assert= (s/join ", " []) "") + (t/assert= (s/join ", " [1]) "1") + + (t/assert= (s/join ", " [1 2 3]) "1, 2, 3")) + (t/deftest test-char-literals (let [s "hey"] (t/assert= (nth s 0) \h) From 3ad9528eac9284a5ed0190aa9521b393d7f2d5ad Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 12:08:06 +0100 Subject: [PATCH 321/909] implement a simple version of `declare` it simply sets the vars to `nil`, maybe should throw an exception when trying to resolve them? --- pixie/stdlib.pxi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4212d381..eff1961b 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1708,3 +1708,7 @@ The params can be destructuring bindings, see `(doc let)` for details."} ~dispatch-val (fn ~params ~@body)) ~name))) + +(defmacro declare [& nms] + (let [defs (map (fn [nm] `(def ~nm)) (seq nms))] + `(do ~@defs))) From 637e1a969d5d329cb54ae46247e558c2d4f2f88c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 12:09:08 +0100 Subject: [PATCH 322/909] fix with-bindings, rename it to binding & test it we have to do the `set!` call generation outside of the syntax quote, otherwise the vars are already getting resolved. --- pixie/stdlib.pxi | 20 +++++++++++--------- tests/test-stdlib.pxi | 8 ++++++++ 2 files changed, 19 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index eff1961b..7e5085e8 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -937,15 +937,17 @@ Creates new maps if the keys are not present." [sym] `(resolve-in (this-ns-name) ~sym)) -(defmacro with-bindings [binds & body] - `(do (push-binding-frame!) - (reduce (fn [_ map-entry] - (set! (resolve (key map-entry)) (val map-entry))) - nil - (apply hashmap ~@binds)) - (let [ret (do ~@body)] - (pop-binding-frame!) - ret))) +(defmacro binding [bindings & body] + (let [bindings (apply hashmap bindings) + set-vars (reduce (fn [res binding] + (conj res `(set! (resolve (quote ~(key binding))) ~(val binding)))) + [] + bindings)] + `(do (push-binding-frame!) + ~@set-vars + (let [ret (do ~@body)] + (pop-binding-frame!) + ret)))) (def foo 42) (set-dynamic! (resolve 'pixie.stdlib/foo)) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 316dc4a6..22fb52bc 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -186,6 +186,14 @@ (t/assert= (macro? :foo) false) (t/assert= (macro? "foo") false)) +(def ^:dynamic *earmuffiness* :low) + +(t/deftest test-binding + (t/assert= *earmuffiness* :low) + (binding [*earmuffiness* :quite-high] + (t/assert= *earmuffiness* :quite-high)) + (t/assert= *earmuffiness* :low)) + (t/deftest test-every? (t/assert= (every? even? [2 4 6 8]) true) (t/assert= (every? odd? [2 4 6 8]) false) From 4cdd6f810a224b6dd0c3aa5d143912361489a183 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 13:03:21 +0100 Subject: [PATCH 323/909] add defprotocol, extend-type and extend-protocol defprotocol currently just ignores the bodies/argument counts, extend-type and extend-protocol expand to calls to extend. --- pixie/stdlib.pxi | 66 +++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 65 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7e5085e8..7d790409 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1711,6 +1711,70 @@ The params can be destructuring bindings, see `(doc let)` for details."} ~@body)) ~name))) -(defmacro declare [& nms] +(defmacro declare + {:doc "Forward declare the given variable names, setting them to nil." + :added "0.1"} + [& nms] (let [defs (map (fn [nm] `(def ~nm)) (seq nms))] `(do ~@defs))) + +(defmacro defprotocol + {:doc "Define a new protocol." + :examples [["(defprotocol SayHi (hi [x]))"] + ["(extend hi String (fn [name] (str \"Hi, \" name \"!\")))"] + ["(hi \"Jane\")" nil "Hi, Jane!"]] + :added "0.1"} + [nm & sigs] + `(pixie.stdlib.internal/-defprotocol (quote ~nm) + ~(reduce (fn [r sig] + (conj r `(quote ~(first sig)))) + [] + sigs))) + +(defmacro extend-type + {:doc "Extend the protocols to the given type. + +Expands to calls to `extend`." + :examples [["(defprotocol SayHi (hi [x]))"] + ["(extend-type String SayHi (hi [name] (str \"Hi, \" name \"!\")))"] + ["(hi \"Jane\")" nil "Hi, Jane!"]] + :added "0.1"} + [tp & extensions] + (let [[_ extends] (reduce (fn [[proto res] extend] + (cond + (symbol? extend) [extend res] + :else [proto (conj res `(extend ~(first extend) ~tp (fn ~@(next extend))))])) + [] + extensions)] + `(do + ~@extends))) + +(defmacro extend-protocol + {:doc "Extend the protocol to the given types. + +Expands to calls to `extend-type`." + :examples [["(defprotocol SayHi (hi [x]))"] + ["(extend-protocol SayHi + + String + (hi [name] + (str \"Hi, \" name \"!\")) + + Integer + (hi [n] + (str \"Hi, #\" n \"!\")))"] + ["(hi \"Jane\")" nil "Hi, Jane!"] + ["(hi 42)" nil "Hi, #42!"]] + :added "0.1"} + [protocol & extensions] + (let [[_ exts] (reduce (fn [[tp res] extend-body] + (cond + (symbol? extend-body) [extend-body (assoc res extend-body [])] + :else [tp (update-in res [tp] conj extend-body)])) + [nil {}] + extensions) + exts (reduce (fn [res [tp exts]] + (conj res `(extend-type ~tp ~protocol ~@exts))) + [] + exts)] + `(do ~@exts))) From eba022479a3a00b7f9704b0e6394b3bdc0bbd32e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 13:10:05 +0100 Subject: [PATCH 324/909] indent longer examples in the docs properly --- pixie/stdlib.pxi | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7d790409..4ef6f639 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1151,7 +1151,11 @@ and implements IAssociative, ILookup and IObject." has-doc? (if doc true (get (meta x) :signatures))] (cond has-doc? (let [sigs (get (meta x) :signatures) - examples (get (meta x) :examples)] + examples (get (meta x) :examples) + indent (fn [s] + (if (>= (pixie.string.internal/index-of s "\n") 0) + (apply str "\n" (map #(str " " % "\n") (pixie.string.internal/split s "\n"))) + s))] (println (str (namespace vr) "/" (name vr))) (if sigs (prn (seq sigs))) @@ -1162,9 +1166,9 @@ and implements IAssociative, ILookup and IObject." (do (println) (doseq [example examples] - (println (str " user => " (first example))) + (println (str " user => " (indent (first example)))) (if (second example) - (print (apply str (map #(str " " % "\n") (pixie.string.internal/split (second example) "\n"))))) + (print (indent (second example)))) (if (contains? example 2) (println (str " " (-repr (third example)))))))) (println) From e0308bcca3d770498c32e77bdb7959dc7f10cb6e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 16:34:10 +0100 Subject: [PATCH 325/909] use an ordering when resolving extended protocols previously, this depended on rpythons hashing, now it's the order of the calls to `extend`. this allows overriding more general protocols later. it could be difficult to coordinate ordering across namespaces, but that shouldn't be an issue because it was essentially random before. --- pixie/vm/code.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index b29f46bf..87906e91 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -580,6 +580,8 @@ def __init__(self, name, protocol): BaseCode.__init__(self) self._name = name self._dict = {} + # stored separately to allow ordered extending (e.g. more general protocols later) + self._protos = [] self._rev = 0 self._protocol = protocol self._default_fn = DefaultProtocolFn(self) @@ -588,6 +590,8 @@ def __init__(self, name, protocol): def extend(self, tp, fn): self._dict[tp] = fn + if isinstance(tp, Protocol): + self._protos.append(tp) self._rev += 1 self._fn_cache = {} self._protocol.add_satisfies(tp) @@ -596,18 +600,13 @@ def _find_parent_fn(self, tp): ## Search the entire object tree to find the function to execute assert isinstance(tp, object.Type) - protos = [] - for p, fn in self._dict.iteritems(): - if isinstance(p, Protocol): - protos.append(p) - find_tp = tp while True: result = self._dict.get(find_tp, None) if result is not None: return result - for proto in protos: + for proto in self._protos: if proto.satisfies(find_tp): return self._dict[proto] From 910fa227d7a19e8d982da9c8d6df60e8f7bcbfc7 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 16:37:08 +0100 Subject: [PATCH 326/909] ensure extend-protocol respects the order of the extensions --- pixie/stdlib.pxi | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4ef6f639..5703d76d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1771,14 +1771,15 @@ Expands to calls to `extend-type`." ["(hi 42)" nil "Hi, #42!"]] :added "0.1"} [protocol & extensions] - (let [[_ exts] (reduce (fn [[tp res] extend-body] - (cond - (symbol? extend-body) [extend-body (assoc res extend-body [])] - :else [tp (update-in res [tp] conj extend-body)])) - [nil {}] - extensions) - exts (reduce (fn [res [tp exts]] - (conj res `(extend-type ~tp ~protocol ~@exts))) + ; tps is used to ensure protocols are extended in order + (let [[_ tps exts] (reduce (fn [[tp tps res] extend-body] + (cond + (symbol? extend-body) [extend-body (conj tps extend-body) (assoc res extend-body [])] + :else [tp tps (update-in res [tp] conj extend-body)])) + [nil [] {}] + extensions) + exts (reduce (fn [res tp] + (conj res `(extend-type ~tp ~protocol ~@(get exts tp)))) [] - exts)] + tps)] `(do ~@exts))) From 94d16b51161c893997033924bf59126824a757dc Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 23:49:42 +0100 Subject: [PATCH 327/909] provide more precise error messages in load-*'d files we use the MetaDataReader for that, are there performance problems with that? --- pixie/vm/stdlib.py | 2 +- target.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 08d2bd7c..78971146 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -402,7 +402,7 @@ def load_file(filename): if newline_pos > 0: data = data[newline_pos:] - rt.load_reader(reader.StringReader(unicode(data))) + rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode(data)), unicode(filename))) return nil @as_var("load-reader") diff --git a/target.py b/target.py index 37c762cc..f533aaf0 100644 --- a/target.py +++ b/target.py @@ -164,7 +164,7 @@ def run_load_stdlib(): f = open(rpath.rjoin(str(load_path.deref()._str), "pixie/stdlib.pxi")) data = f.read() f.close() - rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") + rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pxi") result = nil if not we_are_translated(): From a5119ff4de9174fb2f952c29269a814f9012d938 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 17:49:02 +0100 Subject: [PATCH 328/909] add a simple version of for without `:when`, `:while` and `:let`. --- pixie/stdlib.pxi | 25 +++++++++++++++++++++++++ tests/test-stdlib.pxi | 7 +++++++ 2 files changed, 32 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 5703d76d..a02fbd8d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1783,3 +1783,28 @@ Expands to calls to `extend-type`." [] tps)] `(do ~@exts))) + +(defmacro for + {:doc "A list comprehension for the bindings." + :examples [["(for [x [1 2 3]] x)" nil [1 2 3]] + ["(for [x [1 2 3] y [:a :b :c]] [x y])" nil [[1 :a] [1 :b] [1 :c] [2 :a] [2 :b] [2 :c] [3 :a] [3 :b] [3 :c]]]] + :added "0.1"} + [bindings & body] + (assert (and (pos? (count bindings)) (even? (count bindings))) "for requires an even number of bindings") + (let [gen-loop (fn gen-loop [coll-bindings bindings] + (if (seq bindings) + (let [c (gensym "coll__") + binding (first bindings) + coll (second bindings)] + `(loop [res# [] + ~c (seq ~coll)] + (if ~c + (recur (into res# + ~(gen-loop (into coll-bindings + [binding `(first ~c)]) + (nnext bindings))) + (next ~c)) + res#))) + `(let ~coll-bindings + [~@body])))] + `(or (seq ~(gen-loop [] bindings)) '()))) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 22fb52bc..c1f6875b 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -213,3 +213,10 @@ (t/assert= (sequence (distinct) [1 2 3 2 1]) '(1 2 3)) (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) (t/assert= (vec (distinct) [nil nil nil]) [nil])) + +(t/deftest test-for + (t/assert= (for [x [1 2 3]] x) [1 2 3]) + (t/assert= (for [x [1 2 3] y [:a :b :c]] [x y]) + [[1 :a] [1 :b] [1 :c] + [2 :a] [2 :b] [2 :c] + [3 :a] [3 :b] [3 :c]])) From 5f535c50c073837f012e11d9254a7d0f0b91aa6c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 23:53:10 +0100 Subject: [PATCH 329/909] add identity --- pixie/stdlib.pxi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a02fbd8d..377d724c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -21,6 +21,13 @@ (cons 'let* args))) (set-macro! let) +(def identity + (fn ^{:doc "The identity function. Returns it's argument." + :added "0.1"} + identity + [x] + x)) + (def conj (fn ^{:doc "Adds elements to the collection. Elements are added to the end except in the case of Cons lists" :signatures [[] [coll] [coll item] [coll item & args]] From bc1166f9bfd08ab4bf761b50966aeb873d4cceaf Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 23:53:33 +0100 Subject: [PATCH 330/909] add mapcat and merge-with --- pixie/stdlib.pxi | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 377d724c..947aa725 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -158,6 +158,14 @@ ([result input] (reduce rrf result input)))))) +(def mapcat + (fn ^{:doc "Maps f over the elements of coll and concatenates the result" + :added "0.1"} + mapcat + ([f] + (comp (map f) cat)) + ([f coll] + (transduce (mapcat f) conj coll)))) (def seq-reduce (fn seq-reduce [coll f init] @@ -1636,6 +1644,21 @@ user => (refer 'pixie.string :exclude '(substring))" (recur (next s)))))) (extend -at-end? EmptyList (fn [_] true)) +(defn merge-with + [f & maps] + (cond + (empty? maps) nil + (= (count maps) 1) (first maps) + :else (let [merge2 (fn [m1 m2] + (reduce (fn [res e] + (let [k (key e) v (val e)] + (if (contains? m1 k) + (assoc res k (f (get m1 k) v)) + (assoc res k v)))) + (or m1 {}) + m2))] + (reduce merge2 (first maps) (next maps))))) + (defn every? {:doc "Check if every element of the collection satisfies the predicate." :added "0.1"} From 91e25eb160b2f0abe8fb2f4167fed7a0a7b55a5b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 23:53:47 +0100 Subject: [PATCH 331/909] add if-let --- pixie/stdlib.pxi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 947aa725..e121f2b6 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1300,6 +1300,11 @@ The new value is thus `(apply f current-value-of-atom args)`." (when ~(first binding) ~@body))) +(defmacro if-let [binding & body] + `(let ~binding + (if ~(first binding) + ~@body))) + (defn nnext {:doc "Equivalent to (next (next coll))" :added "0.1"} From 4e0ec8caeeb0db73938350d73ca0802e5e78749c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 23:53:53 +0100 Subject: [PATCH 332/909] add a very simple version of use --- pixie/stdlib.pxi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e121f2b6..96c16623 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1843,3 +1843,9 @@ Expands to calls to `extend-type`." `(let ~coll-bindings [~@body])))] `(or (seq ~(gen-loop [] bindings)) '()))) + +(defmacro use + [ns] + `(do + (load-ns ~ns) + (refer ~ns :refer :all))) From 1d47304d009f7bc70f3470a8182e5a1f150340b8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 29 Nov 2014 23:59:23 +0100 Subject: [PATCH 333/909] add html docs generation example --- examples/gen-docs.pxi | 58 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 58 insertions(+) create mode 100755 examples/gen-docs.pxi diff --git a/examples/gen-docs.pxi b/examples/gen-docs.pxi new file mode 100755 index 00000000..994a8239 --- /dev/null +++ b/examples/gen-docs.pxi @@ -0,0 +1,58 @@ +#!/usr/bin/env pixie-vm + +; generate html docs for a given namespace +(ns gen-ns-docs + (use 'hiccup.core)) + +(defn munge [nm] + (-> (str nm) + (replace "-" "_") + (replace "?" ""))) + +(defn generate-docs [ns] + (html [:html + [:head + [:title ns] + [:meta {:charset "utf-8"}] + [:style {:type "text/css"} +" +.version { + color: #aaa; +} +"]] + [:body + [:h1 ns] + (let [syms (ns-map ns) + infos (transduce (comp (map (fn [sym] + (when-let [info (meta @(resolve-in (the-ns ns) sym))] + (assoc info :name (hiccup.util/escape-html sym))))) + (filter (complement nil?))) + conj + (keys (ns-map ns)))] + (list [:section + [:h2 "Overview"] + [:ul + (for [{name :name} infos] + [:li [:a {:href (str "#" ns "/" name)} (str ns "/" name)]])]] + (seq (map (fn [{:keys [name doc signatures added examples]}] + [:article + [:h2 {:id (str ns "/" name)} name (when added [:span.version (str " (since " added ")")])] + (when signatures + [:pre (pr-str signatures)]) + (when doc + [:pre doc]) + (when examples + [:section.examples + [:ul + (for [[expr output result] examples] + [:pre (str "user => " expr (as-str output) "\n" (as-str result))])]])]) + infos))))]])) + +(defn main [file ns] + (load-file file) + (println (str "\n" (generate-docs (read-string ns))))) + +(if (< (count program-arguments) 2) + (println "Usage: gen-ns-docs ") + (let [[file ns] program-arguments] + (main file ns))) From 5ced6f6058be2f394b3ea9273b8930d92a973360 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 30 Nov 2014 00:07:40 +0100 Subject: [PATCH 334/909] fix tiny typos --- pixie/stdlib.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 96c16623..1e2d5665 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -22,7 +22,7 @@ (set-macro! let) (def identity - (fn ^{:doc "The identity function. Returns it's argument." + (fn ^{:doc "The identity function. Returns its argument." :added "0.1"} identity [x] @@ -1719,7 +1719,7 @@ The params can be destructuring bindings, see `(doc let)` for details."} (apply method dispatch-arg args)))) (defmacro defmulti - {:doc "Define a multimethod, which dispatches to it's methods based on dispatch-fn." + {:doc "Define a multimethod, which dispatches to its methods based on dispatch-fn." :examples [["(defmulti greet first)"] ["(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\"))"] ["(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".\"))"] From e2d6d97c8f50d9e8195923719b103736607da6d4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 30 Nov 2014 02:30:41 +0100 Subject: [PATCH 335/909] make the html doc signatures a bit prettier --- examples/gen-docs.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/examples/gen-docs.pxi b/examples/gen-docs.pxi index 994a8239..e98f3ec1 100755 --- a/examples/gen-docs.pxi +++ b/examples/gen-docs.pxi @@ -38,7 +38,7 @@ [:article [:h2 {:id (str ns "/" name)} name (when added [:span.version (str " (since " added ")")])] (when signatures - [:pre (pr-str signatures)]) + [:pre (pr-str (seq signatures))]) (when doc [:pre doc]) (when examples From ea9336e79abb6e9e05fa2bdba0fbc402aa94fce9 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 30 Nov 2014 13:20:50 +0100 Subject: [PATCH 336/909] add defn- equivalent to defn, but adds `^:private` to the metadata of the function being defined. tooling can the decide what to do with it. --- examples/gen-docs.pxi | 8 +++++++- pixie/stdlib.pxi | 15 ++++++++++++--- 2 files changed, 19 insertions(+), 4 deletions(-) diff --git a/examples/gen-docs.pxi b/examples/gen-docs.pxi index e98f3ec1..9cf965fa 100755 --- a/examples/gen-docs.pxi +++ b/examples/gen-docs.pxi @@ -19,6 +19,12 @@ .version { color: #aaa; } + +#overview ul { + -webkit-column-width: 15em; + -moz-column-width: 15em; + column-width: 15em; +} "]] [:body [:h1 ns] @@ -29,7 +35,7 @@ (filter (complement nil?))) conj (keys (ns-map ns)))] - (list [:section + (list [:section#overview [:h2 "Overview"] [:ul (for [{name :name} infos] diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1e2d5665..7005d58a 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -340,9 +340,10 @@ :signatures [[nm doc? meta? & body]]} defn [nm & rest] - (let [meta (if (instance? String (first rest)) - {:doc (first rest)} - {}) + (let [meta (if (meta nm) (meta nm) {}) + meta (if (instance? String (first rest)) + (assoc meta :doc (first rest)) + meta) rest (if (instance? String (first rest)) (next rest) rest) meta (if (satisfies? IMap (first rest)) (merge meta (first rest)) @@ -367,6 +368,14 @@ ~nm)) (set-macro! defmacro) +(defmacro defn- + {:doc "Define a new non-public function. Otherwise the same as defn" + :signatures [[nm doc? meta? & body]] + :added "0.1"} + [nm & rest] + (let [nm (with-meta nm (assoc (meta nm) :private true))] + (cons `defn (cons nm rest)))) + (defmacro -> {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next." :examples [["(-> 3 inc inc)" nil 5] From 91f6077f6b2166198913c9fe597c2719c527e74e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 2 Dec 2014 10:53:42 +0100 Subject: [PATCH 337/909] add test for identity --- tests/test-stdlib.pxi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index c1f6875b..884abd3d 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -1,6 +1,11 @@ (ns pixie.tests.test-stdlib (require pixie.test :as t)) +(t/deftest test-identity + (let [vs [nil true false [1 2 3] #{1 2 3} :oops]] + (doseq [v vs] + (t/assert= (identity v) v)))) + (t/deftest test-str (t/assert= (str nil) "nil") (t/assert= (str true) "true") From 2e8995a47a9ec66f23c42eaf8029a462158daad8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 2 Dec 2014 10:53:54 +0100 Subject: [PATCH 338/909] add tests for when, when-not, when-let and if-let --- tests/test-forms.pxi | 65 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) create mode 100644 tests/test-forms.pxi diff --git a/tests/test-forms.pxi b/tests/test-forms.pxi new file mode 100644 index 00000000..c2605788 --- /dev/null +++ b/tests/test-forms.pxi @@ -0,0 +1,65 @@ +(ns pixie.tests.test-forms + (require pixie.test :as t)) + +(t/deftest test-when + (t/assert= (when false :never) nil) + (t/assert= (when nil :never) nil) + (t/assert= (when (= 3 4) :never) nil) + + (t/assert= (when true :always) :always) + (t/assert= (when (+ 3 4) :always) :always) + (t/assert= (when {} :always) :always) + + (let [c (atom 0)] + (when (= 3 3) + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-not + (t/assert= (when-not false :always) :always) + (t/assert= (when-not nil :always) :always) + (t/assert= (when-not (= 3 4) :always) :always) + + (t/assert= (when-not true :never) nil) + (t/assert= (when-not (+ 3 4) :never) nil) + (t/assert= (when-not {} :never) nil) + + (let [c (atom 0)] + (when-not (= 3 4) + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-let + (t/assert= (when-let [v false] :never) nil) + (t/assert= (when-let [v nil] :never) nil) + (t/assert= (when-let [v (= 3 4)] :never) nil) + + (t/assert= (when-let [v true] :always) :always) + (t/assert= (when-let [v (+ 3 4)] :always) :always) + (t/assert= (when-let [v {}] :always) :always) + + (let [c (atom 0)] + (when-let [v @c] + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-if-let + (t/assert= (if-let [v false] :yay :nay) :nay) + (t/assert= (if-let [v false] :yay) nil) + (t/assert= (if-let [v nil] :yay :nay) :nay) + (t/assert= (if-let [v nil] :yay) nil) + (t/assert= (if-let [v (= 3 4)] :yay :nay) :nay) + (t/assert= (if-let [v (= 3 4)] :yay) nil) + + (t/assert= (if-let [v true] :yay :nay) :yay) + (t/assert= (if-let [v true] :yay) :yay) + (t/assset= (if-let [v (+ 3 4)] v :nay) 7) + (t/assset= (if-let [v (+ 3 4)] v) 7) + (t/assert= (if-let [v {}] v :nay) {}) + (t/assert= (if-let [v {}] v) {})) From b49e5a657df4428927d617659dcb53e2855fd514 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 2 Dec 2014 11:23:19 +0100 Subject: [PATCH 339/909] fix when-let and if-let with destructuring --- pixie/stdlib.pxi | 25 +++++++++++++++++-------- tests/test-forms.pxi | 29 +++++++++++++++++++++++++++-- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7005d58a..99a24c82 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1305,14 +1305,23 @@ The new value is thus `(apply f current-value-of-atom args)`." `(if (not ~test) (do ~@body))) (defmacro when-let [binding & body] - `(let ~binding - (when ~(first binding) - ~@body))) - -(defmacro if-let [binding & body] - `(let ~binding - (if ~(first binding) - ~@body))) + (let [bind (nth binding 0) + test (nth binding 1)] + `(let [tmp# ~test] + (when tmp# + (let [~bind tmp#] + ~@body))))) + +(defmacro if-let + ([binding then] `(if-let ~binding ~then nil)) + ([binding then else] + (let [bind (nth binding 0) + test (nth binding 1)] + `(let [tmp# ~test] + (if tmp# + (let [~bind tmp#] + ~then) + ~else))))) (defn nnext {:doc "Equivalent to (next (next coll))" diff --git a/tests/test-forms.pxi b/tests/test-forms.pxi index c2605788..b909869a 100644 --- a/tests/test-forms.pxi +++ b/tests/test-forms.pxi @@ -49,6 +49,17 @@ (swap! c inc)) (t/assert= @c 3))) +(t/deftest test-when-let-destructuring + (t/assert= (when-let [[x y & z] false] :yay) nil) + (t/assert= (when-let [[x y & z] nil] :yay) nil) + (t/assert= (when-let [{:keys [a b]} nil] :yay) nil) + + (t/assert= (when-let [[x y & z] [1 2 3]] :yay) :yay) + (t/assert= (when-let [[x y & z] [1 2 3]] [x y z]) [1 2 '(3)]) + (t/assert= (when-let [{:keys [a b]} {}] :yay) :yay) + (t/assert= (when-let [{:keys [a b]} {}] [a b]) [nil nil]) + (t/assert= (when-let [{:keys [a b]} {:a 1, :b 41}] [a b]) [1 41])) + (t/deftest test-if-let (t/assert= (if-let [v false] :yay :nay) :nay) (t/assert= (if-let [v false] :yay) nil) @@ -59,7 +70,21 @@ (t/assert= (if-let [v true] :yay :nay) :yay) (t/assert= (if-let [v true] :yay) :yay) - (t/assset= (if-let [v (+ 3 4)] v :nay) 7) - (t/assset= (if-let [v (+ 3 4)] v) 7) + (t/assert= (if-let [v (+ 3 4)] v :nay) 7) + (t/assert= (if-let [v (+ 3 4)] v) 7) (t/assert= (if-let [v {}] v :nay) {}) (t/assert= (if-let [v {}] v) {})) + +(t/deftest test-if-let-destructuring + (t/assert= (if-let [[x y & z] false] :yay :nay) :nay) + (t/assert= (if-let [[x y & z] false] :yay) nil) + (t/assert= (if-let [[x y & z] nil] :yay :nay) :nay) + (t/assert= (if-let [[x y & z] nil] :yay) nil) + (t/assert= (if-let [{:keys [a b]} nil] :yay :nay) :nay) + (t/assert= (if-let [{:keys [a b]} nil] :yay) nil) + + (t/assert= (if-let [[x y & z] [1 2 3]] :yay :nay) :yay) + (t/assert= (if-let [[x y & z] [1 2 3]] [x y z] :nay) [1 2 '(3)]) + (t/assert= (if-let [{:keys [a b]} {}] :yay :nay) :yay) + (t/assert= (if-let [{:keys [a b]} {}] [a b] :nay) [nil nil]) + (t/assert= (if-let [{:keys [a b]} {:a 1, :b 41}] [a b] :nay) [1 41])) From 516024f21b6f7bda0f1dba63387d31df04f8a56d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 3 Dec 2014 20:08:26 +0100 Subject: [PATCH 340/909] add a test for mapcat --- tests/test-stdlib.pxi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 884abd3d..69a02fc3 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -6,6 +6,10 @@ (doseq [v vs] (t/assert= (identity v) v)))) +(t/deftest test-mapcat + (t/assert= (mapcat identity []) []) + (t/assert= (mapcat first [[[1 2]] [[3] [:not :present]] [[4 5 6]]]) [1 2 3 4 5 6])) + (t/deftest test-str (t/assert= (str nil) "nil") (t/assert= (str true) "true") From 4577a5f31ac467b915674132fd9511b4abf16413 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 3 Dec 2014 20:41:37 +0100 Subject: [PATCH 341/909] change merge to merge it's arguments from left to right e.g. the values from later arguments take precendent over that of earlier ones. --- pixie/vm/stdlib.py | 3 +-- tests/test-stdlib.pxi | 11 +++++++++++ 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 78971146..78acb582 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -614,10 +614,9 @@ def merge_fn(acc, x): @jit.unroll_safe def _merge__args(args): affirm(len(args) > 0, u"Merge takes at least one arg") - x = 1 acc = args[0] for x in range(1, len(args)): - acc = rt._reduce(acc, merge_fn, args[x]) + acc = rt._reduce(args[x], merge_fn, acc) return acc diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 69a02fc3..ae4dd2ac 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -223,6 +223,17 @@ (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) (t/assert= (vec (distinct) [nil nil nil]) [nil])) +(t/deftest test-merge + (t/assert= (merge {}) {}) + (t/assert= (merge {:a 1} nil) {:a 1}) + + (t/assert= (merge {} {:a 1, :b 2}) {:a 1, :b 2}) + (t/assert= (merge {:a 1} {:b 2}) {:a 1, :b 2}) + (t/assert= (merge {} {:a 1} {:b 2}) {:a 1, :b 2}) + + (t/assert= (merge {:a 1} {:a 2, :b 3}) {:a 2, :b 3}) + (t/assert= (merge {:a 1, :b 4} {:a 2} {:a 3}) {:a 3, :b 4})) + (t/deftest test-for (t/assert= (for [x [1 2 3]] x) [1 2 3]) (t/assert= (for [x [1 2 3] y [:a :b :c]] [x y]) From 0c5c471580e0fb637dee87a2f4c99b9165f3cb3e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 3 Dec 2014 20:47:15 +0100 Subject: [PATCH 342/909] add a test for merge-with --- tests/test-stdlib.pxi | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index ae4dd2ac..23e1e869 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -224,6 +224,7 @@ (t/assert= (vec (distinct) [nil nil nil]) [nil])) (t/deftest test-merge + (t/assert= (merge nil) nil) (t/assert= (merge {}) {}) (t/assert= (merge {:a 1} nil) {:a 1}) @@ -234,6 +235,20 @@ (t/assert= (merge {:a 1} {:a 2, :b 3}) {:a 2, :b 3}) (t/assert= (merge {:a 1, :b 4} {:a 2} {:a 3}) {:a 3, :b 4})) +(t/deftest test-merge-with + (t/assert= (merge-with identity nil) nil) + (t/assert= (merge-with identity {}) {}) + + (t/assert= (merge-with identity {} {:a 1, :b 2}) {:a 1, :b 2}) + (t/assert= (merge-with identity {:a 1} {:b 2}) {:a 1, :b 2}) + + (t/assert= (merge-with #(identity %1) {:a 1} {:a 2}) {:a 1}) + (t/assert= (merge-with #(identity %1) {:a 1} {:a 2} {:a 3}) {:a 1}) + (t/assert= (merge-with #(identity %2) {:a 1} {:a 2}) {:a 2}) + + (t/assert= (merge-with + {:a 21} {:a 21}) {:a 42}) + (t/assert= (merge-with + {:a 21} {:a 21, :b 1}) {:a 42, :b 1})) + (t/deftest test-for (t/assert= (for [x [1 2 3]] x) [1 2 3]) (t/assert= (for [x [1 2 3] y [:a :b :c]] [x y]) From 3b1db1ed81a03f16c9621a34851ff6a0b80d7cf0 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 1 Dec 2014 16:46:05 +0100 Subject: [PATCH 343/909] fix destructuring with rest args --- pixie/stdlib.pxi | 6 ++++-- tests/test-destructuring.pxi | 11 +++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 99a24c82..fc85b44e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1710,10 +1710,12 @@ The params can be destructuring bindings, see `(doc let)` for details."} :else decls) decls (seq (map (fn* [decl] (let [argv (first decl) - names (vec (map #(gensym "arg__") argv)) + names (vec (map #(if (= % '&) '& (gensym "arg__")) argv)) bindings (loop [i 0 bindings []] (if (< i (count argv)) - (recur (inc i) (reduce conj bindings [(nth argv i) (nth names i)])) + (if (= (nth argv i) '&) + (recur (inc i) bindings) + (recur (inc i) (reduce conj bindings [(nth argv i) (nth names i)]))) bindings)) body (next decl)] (if (every? symbol? argv) diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi index 8693edf9..13eb93a0 100644 --- a/tests/test-destructuring.pxi +++ b/tests/test-destructuring.pxi @@ -63,3 +63,14 @@ (t/deftest test-fn-multiple-args (t/assert= ((fn [[x y z] {:keys [a b c]}] [x y z a b c]) [1 2 3] {:a 4, :b 5, :c 6}) [1 2 3 4 5 6])) + +(t/deftest test-fn-rest-args + (let [f1 (fn [& [status]] (or status :yay))] + (t/assert= (f1) :yay) + (t/assert= (f1 :nay) :nay) + (t/assert= (f1 :nay :something-else :whatever) :nay)) + (let [f2 (fn [x & [y]] + (+ x (or y 1)))] + (t/assert= (f2 41) 42) + (t/assert= (f2 21 21) 42) + (t/assert= (f2 21 21 :something-else :whatever) 42))) From 0de491f9970bc2113e55a758bedb2ed0cbde92ee Mon Sep 17 00:00:00 2001 From: gigasquid Date: Thu, 4 Dec 2014 22:50:10 -0500 Subject: [PATCH 344/909] add mac instructions and dust link --- README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/README.md b/README.md index 56c7e9bb..fba79dbe 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,15 @@ Some planned and implemented features: ./make-with-jit ./pixie-vm +## Special Note for Macs + +If you are having trouble building on Mac, checkout this issue +[https://github.com/pixie-lang/pixie/issues/49](https://github.com/pixie-lang/pixie/issues/49). +In particular, try this: + +``` +export PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig:/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig +``` ## Running the tests @@ -42,6 +51,10 @@ Try out "Hello World" with: ./examples/hello-world.pxi + +## Build Tool +Pixie now comes with a build tool called [dust](https://github.com/pixie-lang/dust). Try it and start making magic of your own. + ## FAQ ### So this is written in Python? From 71f693b3c1581350db1ed78bffe362a199066df8 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 5 Dec 2014 07:08:38 -0700 Subject: [PATCH 345/909] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 56c7e9bb..7ecb8d5f 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,4 @@ -[![Build Status](https://travis-ci.org/pixie-lang/pixie.svg?branch=master)](https://travis-ci.org/pixie-lang/pixie) +[![Build Status](https://travis-ci.org/pixie-lang/pixie.svg?branch=master)](https://travis-ci.org/pixie-lang/pixie)[![License: LGPL] (http://img.shields.io/badge/license-LGPL-green.svg)](http://img.shields.io/badge/license-LGPL-green.svg) # Pixie ## Intro From 193f90fc01ae4609cb6ecb80fb1742e2b7427800 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Fri, 5 Dec 2014 15:18:06 -0500 Subject: [PATCH 346/909] fix command remove dup --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index fba79dbe..8535731f 100644 --- a/README.md +++ b/README.md @@ -36,7 +36,7 @@ If you are having trouble building on Mac, checkout this issue In particular, try this: ``` -export PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig:/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig +PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' ./make-with-jit ``` ## Running the tests From 4891e0bd820b4fafb0f651a64058b698c7d81db5 Mon Sep 17 00:00:00 2001 From: Devin Walters Date: Fri, 5 Dec 2014 21:05:15 -0600 Subject: [PATCH 347/909] Add mac-specific run-interpreted info to README --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index c228fa3c..d2108e0f 100644 --- a/README.md +++ b/README.md @@ -31,12 +31,13 @@ Some planned and implemented features: ## Special Note for Macs -If you are having trouble building on Mac, checkout this issue +If you are having trouble building or running the interpreter on Mac, check out this issue [https://github.com/pixie-lang/pixie/issues/49](https://github.com/pixie-lang/pixie/issues/49). In particular, try this: ``` PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' ./make-with-jit +PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' ./run-interpreted ``` ## Running the tests From 09a4143eb6abc762655086d3f2a55dd86373806b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Dec 2014 00:12:01 -0700 Subject: [PATCH 348/909] fix the compiler tests --- pixie/vm/compiler.py | 22 +++++++++++-- pixie/vm/test/test_compile.py | 58 +++++++++++++++++++---------------- 2 files changed, 51 insertions(+), 29 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 749513ff..7d45b6c3 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -39,12 +39,16 @@ def gensym2(prefix): gensym.set_root(code.MultiArityFn({0: code.wrap_fn(gensym1), 1: code.wrap_fn(gensym2)})) class with_ns(object): - def __init__(self, nm): + def __init__(self, nm, include_stdlib=False): assert isinstance(nm, unicode) self._ns = nm + self._include_stdlib=include_stdlib + def __enter__(self): code._dynamic_vars.push_binding_frame() NS_VAR.set_value(code._ns_registry.find_or_make(self._ns)) + if self._include_stdlib: + NS_VAR.deref().include_stdlib() def __exit__(self, exc_type, exc_val, exc_tb): code._dynamic_vars.pop_binding_frame() @@ -747,6 +751,19 @@ def compile_yield(form, ctx): compile_form(arg, ctx) ctx.bytecode.append(code.YIELD) +def compile_effect(form, ctx): + form = rt.next(form) + affirm(isinstance(rt.first(form), Keyword), u"First argument to -effect must be a keyword") + affirm(rt.count(form) == 2, u"Must provide a value to -effect") + + eff_name = rt.first(form) + form = rt.next(form) + eff_val = rt.next(form) + compile_form(eff_val, ctx) + ctx.push_const(eff_name) + ctx.bytecode.append(code.EFFECT) + ctx.sub_sp(1) + builtins = {u"fn*": compile_fn, u"if": compile_if, @@ -762,7 +779,8 @@ def compile_yield(form, ctx): u"__ns__": compile_ns, u"catch": compile_catch, u"this-ns-name": compile_this_ns, - u"yield": compile_yield} + u"yield": compile_yield, + u"-effect": compile_effect} def compiler_special(s): if isinstance(s, symbol.Symbol): diff --git a/pixie/vm/test/test_compile.py b/pixie/vm/test/test_compile.py index 40073b9b..6f57f4e2 100644 --- a/pixie/vm/test/test_compile.py +++ b/pixie/vm/test/test_compile.py @@ -4,7 +4,7 @@ from pixie.vm.numbers import Integer from pixie.vm.symbol import symbol, Symbol from pixie.vm.keyword import Keyword -from pixie.vm.compiler import compile_form, compile +from pixie.vm.compiler import compile_form, compile, with_ns from pixie.vm.interpreter import interpret from pixie.vm.code import Code, Var from pixie.vm.primitives import nil, true, false @@ -14,41 +14,45 @@ import pixie.vm.libs.readline def read_code(s): - return read(StringReader(unicode(s)), False) + with with_ns(u"user"): + return read(StringReader(unicode(s)), False) def test_add_compilation(): - code = compile(read_code(u"(platform+ 1 2)")) - assert isinstance(code, Code) + with with_ns(u"user"): + code = compile(read_code(u"(platform+ 1 2)")) + assert isinstance(code, Code) #interpret(code) def eval_string(s): - rdr = StringReader(unicode(s)) - result = nil - while True: - form = read(rdr, False) - if form is eof: - return result + with with_ns(u"user", True): + rdr = StringReader(unicode(s)) + result = nil + while True: + form = read(rdr, False) + if form is eof: + return result - result = compile(form).invoke([]) + result = compile(form).invoke([]) def test_fn(): - code = compile(read_code("((fn [x y] (+ x y)) 1 2)")) - assert isinstance(code, Code) - retval = interpret(code) - assert isinstance(retval, Integer) and retval.int_val() == 3 + with with_ns(u"user", True): + code = compile(read_code("((fn* [x y] (-add x y)) 1 2)")) + assert isinstance(code, Code) + retval = interpret(code) + assert isinstance(retval, Integer) and retval.int_val() == 3 def test_multiarity_fn(): - retval = eval_string("""(let [v 1 - f (fn ([] v) + retval = eval_string("""(let* [v 1 + f (fn* ([] v) ([x] (+ v x)))] (f)) """) assert isinstance(retval, Integer) and retval.int_val() == 1 - retval = eval_string("""(let [v 1 - f (fn ([] v) + retval = eval_string("""(let* [v 1 + f (fn* ([] v) ([x] (+ v x)))] (f 2)) @@ -74,10 +78,10 @@ def test_if_eq(): assert eval_string("(if (platform= 1 2) true false)") is false def test_return_self(): - assert isinstance(eval_string("((fn r [] r))"), Code) + assert isinstance(eval_string("((fn* r [] r))"), Code) def test_recursive(): - retval = eval_string("""((fn rf [x] + retval = eval_string("""((fn* rf [x] (if (platform= x 10) x (recur (+ x 1)))) @@ -137,7 +141,7 @@ def test_loop(): assert retval.int_val() == 10 def test_closures(): - retval = eval_string("""((fn [x] ((fn [] x))) 42)""") + retval = eval_string("""((fn* [x] ((fn* [] x))) 42)""") assert isinstance(retval, Integer) assert retval.int_val() == 42 @@ -160,7 +164,7 @@ def test_native(): def test_build_list(): - retval = eval_string("""((fn [i lst] + retval = eval_string("""((fn* [i lst] (if (platform= i 10) (count lst) (recur (+ i 1) (cons i lst)))) 0 nil) @@ -169,7 +173,7 @@ def test_build_list(): assert isinstance(retval, Integer) and retval.int_val() == 10 def test_build_vector(): - retval = eval_string("""((fn [i lst] + retval = eval_string("""((fn* [i lst] (if (platform= i 10) (count lst) (recur (+ i 1) (conj lst i)))) 0 []) @@ -186,17 +190,17 @@ def test_build_vector(): # assert isinstance(retval, Integer) and retval.int_val() == 42 def test_let(): - retval = eval_string(""" (let [x 42] x) """) + retval = eval_string(""" (let* [x 42] x) """) assert isinstance(retval, Integer) and retval.int_val() == 42 - retval = eval_string(""" (let [x 42 y 1] (+ x y)) """) + retval = eval_string(""" (let* [x 42 y 1] (+ x y)) """) assert isinstance(retval, Integer) and retval.int_val() == 43 def test_variadic_fn(): from pixie.vm.array import Array - retval = eval_string(""" ((fn [& rest] rest) 1 2 3 4) """) + retval = eval_string(""" ((fn* [& rest] rest) 1 2 3 4) """) print retval assert isinstance(retval, Array) and len(retval._list) == 4 # From 6603e9b18195e6b94e017d96416abda9d63d1201 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Dec 2014 09:56:59 -0700 Subject: [PATCH 349/909] remove stacklets completely --- pixie/stdlib.pxi | 12 +- pixie/vm/bootstrap.py | 6 - pixie/vm/libs/uv.py | 153 ----------------- pixie/vm/rt.py | 1 - pixie/vm/stacklet.py | 289 --------------------------------- pixie/vm/test/test_stacklet.py | 73 --------- target.py | 15 +- 7 files changed, 13 insertions(+), 536 deletions(-) delete mode 100644 pixie/vm/libs/uv.py delete mode 100644 pixie/vm/stacklet.py delete mode 100644 pixie/vm/test/test_stacklet.py diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index fc85b44e..4bb25397 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -321,8 +321,16 @@ data)))] (stacklet->lazy-seq f))))) - -(extend -seq PersistentVector sequence) +(def = -eq) + +(extend -seq PersistentVector + (fn vector-seq + ([self] + (vector-seq self 0)) + ([self x] + (if (= x (count self)) + nil + (cons (nth self x) (lazy-seq* (fn [] (vector-seq self (+ x 1))))))))) diff --git a/pixie/vm/bootstrap.py b/pixie/vm/bootstrap.py index f677409e..623d9442 100644 --- a/pixie/vm/bootstrap.py +++ b/pixie/vm/bootstrap.py @@ -1,5 +1,3 @@ -from pixie.vm.stacklet import with_stacklets -import pixie.vm.stacklet as stacklet from pixie.vm.code import wrap_fn @wrap_fn @@ -10,7 +8,3 @@ def bootstrap(): rt.load_ns(rt.wrap(u"pixie/stdlib.pxi")) -# run bootstrap -#with_stacklets(bootstrap) -# reset the stacklet state so we can translate with different settings -stacklet.global_state = stacklet.GlobalState() diff --git a/pixie/vm/libs/uv.py b/pixie/vm/libs/uv.py deleted file mode 100644 index a981116e..00000000 --- a/pixie/vm/libs/uv.py +++ /dev/null @@ -1,153 +0,0 @@ -import py - -from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.lltypesystem.lloperation import llop -import rpython.rtyper.tool.rffi_platform as rffi_platform -from rpython.translator.tool.cbuild import ExternalCompilationInfo -from rpython.rtyper.annlowlevel import llhelper -from rpython.rlib.rgc import pin, unpin -from pixie.vm.code import as_var, extend, NativeFn -from pixie.vm.primitives import nil -import rpython.tool.udir as udir -from pixie.vm.object import affirm -import os -import shutil -import pixie.vm.libs.ffi as ffi -import rpython.rlib.rgc as rgc - - -pkgpath = py.path.local(__file__).dirpath() -srcpath = pkgpath.join("c") - -shutil.copyfile(str(srcpath / "uv_ffi.c"), str(udir.udir / "uv_ffi.c")) -shutil.copyfile(str(srcpath / "uv_ffi.h"), str(udir.udir / "uv_ffi.h")) - - -compilation_info = ExternalCompilationInfo( - includes=['uv.h', "ffi.h", "uv_ffi.h"], - include_dirs=[srcpath], - libraries=["uv", "ffi"], - separate_module_files=[udir.udir / "uv_ffi.c"]).merge(ExternalCompilationInfo.from_pkg_config("libffi")) - -def llexternal(*args, **kwargs): - return rffi.llexternal(*args, compilation_info=compilation_info, **kwargs) - - -uv_work = rffi_platform.Struct("uv_work_t", - [("data", rffi.VOIDP)]) - -uv_timer_t = rffi.COpaque("uv_timer_t", compilation_info=compilation_info) -uv_baton_t = rffi.COpaque("work_baton_t", compilation_info=compilation_info) - - -uv_timer = lltype.Ptr(uv_timer_t) - -uv_timer_cb = lltype.Ptr(lltype.FuncType([uv_timer, rffi.INT], lltype.Void)) - -uv_callback_t = rffi.CCallback([rffi.VOIDP, rffi.INT], lltype.Void) - -data_container = {} - -def as_cb(T): - def _inner(fn): - return llhelper(T, fn) - return _inner - -def _timer_cb(timer_t, status): - import pixie.vm.stacklet as stacklet - casted = rffi.cast(rffi.INT, timer_t) - data = data_container[casted] - del data_container[casted] - stacklet.pending_stacklets.push((data, nil)) - -def set_timeout(loop, cont, timeout, repeat): - timer = lltype.malloc(uv_timer_t, flavor="raw", track_allocation="false") - data_container[rffi.cast(rffi.INT, timer)] = cont - assert not timer_init(loop, timer) - assert not timer_start(timer, _timer_cb, timeout, repeat) - - - -loop_new = llexternal('uv_loop_new', [], rffi.VOIDP) -run = llexternal("uv_run", [rffi.VOIDP, rffi.INT], rffi.SIZE_T) - -timer_init = llexternal("uv_timer_init", [rffi.VOIDP, uv_timer], rffi.INT) -timer_start = llexternal("uv_timer_start", [uv_timer, uv_timer_cb, rffi.ULONGLONG, rffi.ULONGLONG], rffi.INT) - -ffi_run = llexternal("uv_ffi_run", [rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, rffi.VOIDP, uv_callback_t], rffi.SIZE_T) -ffi_make_baton = llexternal("uv_ffi_make_baton", [], rffi.VOIDP) - -RUN_DEFAULT = 0 -RUN_ONCE = 1 -RUN_NO_WAIT = 2 - -class UVFunction(NativeFn): - pass - - -class SleepUVFunction(UVFunction): - def __init__(self, time): - self._time = time.int_val() - - def execute_uv(self, loop, k): - set_timeout(loop, k, self._time, 0) - - -work_data_container = {} - -def _work_cb(baton, status): - import pixie.vm.stacklet as stacklet - casted = rffi.cast(rffi.INT, baton) - (k, exb, fn) = work_data_container[casted] - del work_data_container[casted] - retval = fn.get_ret_val_from_buffer(exb) - - stacklet.pending_stacklets.push((k, retval)) - lltype.free(exb, flavor="raw") - lltype.free(baton, flavor="raw") - -class RunFFIFunc(UVFunction): - def __init__(self, fn, args): - affirm(isinstance(fn, ffi.FFIFn), u"Can only use blocking-call against a ffi function") - self._fn = fn - self._args = args - - def execute_uv(self, loop, k): - baton = lltype.malloc(uv_baton_t, flavor="raw") - rgc.pin(baton) - exb = self._fn.prep_exb(self._args) - - buffer_array = rffi.cast(rffi.VOIDPP, exb) - work_data_container[rffi.cast(rffi.INT, baton)] = (k, exb, self._fn) - cif = self._fn._cd - for i in range(cif.nargs): - data = rffi.ptradd(exb, cif.exchange_args[i]) - buffer_array[i] = data - resultdata = rffi.ptradd(exb, - cif.exchange_result_libffi) - - ffi_run(baton, - loop, - rffi.cast(rffi.VOIDP, cif), - rffi.cast(rffi.VOIDP, self._fn._f_ptr), - rffi.cast(rffi.VOIDP, exb), - rffi.cast(rffi.VOIDP, resultdata), - _work_cb) - -@as_var("blocking-call") -def _run_blocking__args(args): - affirm(len(args) > 0, u"At least one arg must be supplied to blocking-call") - fn = args[0] - argc = len(args) - 1 - new_args = [None] * argc - for x in range(argc): - new_args[x] = args[x + 1] - - from pixie.vm.stacklet import execute_uv_func - return execute_uv_func(RunFFIFunc(fn, new_args)) - -@as_var("sleep") -def _sleep(ms): - from pixie.vm.stacklet import execute_uv_func - execute_uv_func(SleepUVFunction(ms)) - return nil \ No newline at end of file diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 835d5ec7..9b082f94 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -54,7 +54,6 @@ def wrapper(*args): import pixie.vm.bits as bits from pixie.vm.code import wrap_fn import pixie.vm.interpreter - import pixie.vm.stacklet as stacklet import pixie.vm.atom import pixie.vm.reduced import pixie.vm.util diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py deleted file mode 100644 index 31fd759e..00000000 --- a/pixie/vm/stacklet.py +++ /dev/null @@ -1,289 +0,0 @@ -py_object = object -import pixie.vm.object as object -from pixie.vm.object import affirm -from pixie.vm.primitives import nil, true, false -from pixie.vm.code import BaseCode -from pixie.vm.numbers import Integer -import pixie.vm.stdlib as proto -from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint, intmask, widen -import rpython.rlib.jit as jit -import pixie.vm.rt as rt -import pixie.vm.libs.uv as uv -import pixie.vm.libs.ring_buffer as ring_buffer -import rpython.rlib.rstacklet as rstacklet -from rpython.rtyper.lltypesystem import lltype, rffi - -OP_NEW = 0x01 -OP_SWITCH = 0x02 -OP_CONTINUE = 0x03 -OP_EXIT = 0x04 -OP_YIELD = 0x05 -OP_NEXT_PENDING = 0x06 -OP_NEW_THREAD = 0x07 -OP_EXECUTE_UV = 0x08 - -def to_main_loop(): - global_state._h = global_state._th.switch(global_state._h) - -class GlobalState(py_object): - def __init__(self): - self.reset() - - def reset(self): - self._th = None - self._val = None - self._ex = None - self._op = 0x00 - self._fn = None - self._init_fn = None - self._current = None - self._parent = None - - def switch_back(self): - tmp = self._current - self._current = self._to - self._to = tmp - - -global_state = GlobalState() - -def init(): - if global_state._th is None: - global_state._th = rstacklet.StackletThread(rt.__config__) - global_state._h = global_state._th.get_null_handle() - -def shutdown(): - global_state._h = global_state._th.get_null_handle() - - global_state.reset() - -class FinishedToken(BaseCode): - _type = object.Type(u"pixie.stdlib.FinishedToken") - def __init__(self): - pass - - def type(self): - return FinishedToken._type - -finished_token = FinishedToken() - -class WrappedHandler(BaseCode): - _type = object.Type(u"Stacklet") - def __init__(self, h): - self._h = h - self._is_finished = False - self._val = nil - - - def type(self): - return WrappedHandler._type - - def is_finished(self): - return self._is_finished - - def invoke(self, args): - affirm(len(args) == 1, u"Only one arg to continuation allowed") - affirm(not self._is_finished, u"Execution of this stacklet has completed") - - global_state._parent = global_state._current - global_state._current = self - global_state._op = OP_SWITCH - global_state._val = args[0] - - to_main_loop() - - if global_state._val is finished_token: - global_state._parent._is_finished = True - - - global_state._parent._val = global_state._val - - return global_state._val - - -def new_stacklet(f): - global_state._op = OP_NEW - global_state._val = f - to_main_loop() - val = global_state._val - return val - -def new_thread(f): - global_state._op = OP_NEW_THREAD - global_state._val = f - to_main_loop() - -def yield_stacklet(): - global_state._op = OP_YIELD - global_state._val = nil - to_main_loop() - -def execute_uv_func(func): - assert isinstance(func, uv.UVFunction) - global_state._op = OP_EXECUTE_UV - global_state._val = func - to_main_loop() - return global_state._val - - -def new_handler(h, o): - global_state._h = h - parent = global_state._parent - affirm(global_state._val is not None, u"Internal Stacklet Error") - f = global_state._val - global_state._val = None - - to_main_loop() - - - #try: - f.invoke([global_state._parent]) - global_state._parent.invoke([finished_token]) - print "ENDED! Should never see this" - assert False - return global_state._h - -def new_thread_handler(h, o): - global_state._h = h - - affirm(global_state._val is not None, u"Internal Stacklet Error") - f = global_state._val - global_state._val = None - - to_main_loop() - - - #try: - f.invoke([global_state._current]) - - global_state._op = OP_NEXT_PENDING - to_main_loop() - - return global_state._h - - - -def init_handler(h, o): - global_state._h = h - - affirm(global_state._init_fn is not None, u"Internal Stacklet error") - f = global_state._init_fn - global_state._init_fn = None - - #try: - f.invoke([]) - #except Exception as ex: - # print "Uncaught Exception" + str(ex) - - global_state._op = OP_EXIT - return global_state._h - -pending_stacklets = ring_buffer.RingBuffer(r_uint(32)) - - -def with_stacklets(f): - loop = uv.loop_new() - - init() - global_state._init_fn = f - - main_h = global_state._th.new(init_handler) - global_state._current = WrappedHandler(main_h) - - while True: - if global_state._op == OP_NEW: - assert global_state._current - global_state._parent = global_state._current - new_h = global_state._th.new(new_handler) - wh = WrappedHandler(new_h) - global_state._val = wh - assert global_state._parent - global_state._current = global_state._parent - global_state._op = OP_SWITCH - continue - - elif global_state._op == OP_NEW_THREAD: - wh = WrappedHandler(global_state._th.get_null_handle()) - global_state._to = wh - wh._h = global_state._th.new(new_thread_handler) - pending_stacklets.push((wh, nil)) - pending_stacklets.push((global_state._current, wh)) - global_state._op = OP_NEXT_PENDING - continue - - elif global_state._op == OP_SWITCH: - cur = global_state._current - cur._h = global_state._th.switch(cur._h) - continue - - elif global_state._op == OP_EXIT: - shutdown() - return global_state._val - - elif global_state._op == OP_YIELD: - assert global_state._current - pending_stacklets.push((global_state._current, nil)) - global_state._op = OP_NEXT_PENDING - continue - - elif global_state._op == OP_EXECUTE_UV: - assert global_state._current - k = global_state._current - f = global_state._val - f.execute_uv(loop, k) - global_state._op = OP_NEXT_PENDING - continue - - elif global_state._op == OP_NEXT_PENDING: - while True: - if pending_stacklets.pending() == 0: - uv.run(loop, uv.RUN_DEFAULT) - continue - else: - uv.run(loop, uv.RUN_DEFAULT | uv.RUN_NO_WAIT) - break - (k, val) = pending_stacklets.pop() - global_state._val = val - global_state._current = k - k._h = global_state._th.switch(k._h) - continue - - else: - assert False - - shutdown() - return None - -@as_var("create-stacklet") -def _new_stacklet(f): - stacklet = new_stacklet(f) - stacklet.invoke([nil]) # prime it - return stacklet - -@as_var("create-thread") -def _new_stacklet(f): - stacklet = new_thread(f) - - return nil - -@as_var("yield-stacklet") -def _stacklet_yield(): - yield_stacklet() - return nil - -@extend(proto._at_end_QMARK_, WrappedHandler) -def _at_end(self): - assert isinstance(self, WrappedHandler) - return rt.wrap(self.is_finished()) - -@extend(proto._move_next_BANG_, WrappedHandler) -def _move_next(self): - assert isinstance(self, WrappedHandler) - self.invoke([nil]) - return self - -@extend(proto._current, WrappedHandler) -def _current(self): - assert isinstance(self, WrappedHandler) - return self._val diff --git a/pixie/vm/test/test_stacklet.py b/pixie/vm/test/test_stacklet.py deleted file mode 100644 index 851efbf6..00000000 --- a/pixie/vm/test/test_stacklet.py +++ /dev/null @@ -1,73 +0,0 @@ -import pixie.vm.stacklet as stacklet -import pixie.vm.code as code -import pixie.vm.rt as rt -import pixie.vm.numbers as numbers -from pixie.vm.primitives import nil - -class YieldingFn(code.BaseCode): - def invoke(self, args): - assert len(args) == 2 - hdler = args[0] - arg = args[1] - - hdler.invoke([numbers.zero_int]) - hdler.invoke([numbers.one_int]) - hdler.invoke([rt.wrap(2)]) - - return - - -@code.wrap_fn -def yielding_fn(yld): - - yld.invoke([1]) - yld.invoke([2]) - yld.invoke([3]) - - return 42 - -class WrappingFn(code.NativeFn): - def __init__(self, cont): - self._cont = cont - def invoke(self, args): - ret = args[0] - ret.invoke([self._cont.invoke([4])]) - -def test_stacklets(): - @code.wrap_fn - def main(): - cont = stacklet.new_stacklet(yielding_fn) - assert cont.invoke([None]) == 1 - assert cont.invoke([None]) == 2 - assert cont.invoke([None]) == 3 - - stacklet.with_stacklets(main) - - pass - -def test_multi_stacklets(): - @code.wrap_fn - def main(): - cont1 = stacklet.new_stacklet(yielding_fn) - cont2 = stacklet.new_stacklet(yielding_fn) - assert cont1.invoke([None]) == 1 - assert cont2.invoke([None]) == 1 - assert cont1.invoke([None]) == 2 - assert cont2.invoke([None]) == 2 - assert cont1.invoke([None]) == 3 - assert cont2.invoke([None]) == 3 - - stacklet.with_stacklets(main) - - pass - -def test_stacklets2(): - @code.wrap_fn - def main(): - cont = stacklet.new_stacklet(yielding_fn) - cont2 = stacklet.new_stacklet(WrappingFn(cont)) - cont2.invoke([44]) - - stacklet.with_stacklets(main) - - pass \ No newline at end of file diff --git a/target.py b/target.py index f533aaf0..52690b31 100644 --- a/target.py +++ b/target.py @@ -6,8 +6,6 @@ from rpython.rlib.rfile import create_stdio from rpython.annotator.policy import AnnotatorPolicy from pixie.vm.code import wrap_fn, NativeFn, intern_var, Var -from pixie.vm.stacklet import with_stacklets -import pixie.vm.stacklet as stacklet from pixie.vm.object import RuntimeException, WrappedException from rpython.translator.platform import platform from pixie.vm.primitives import nil @@ -184,14 +182,7 @@ def run_load_stdlib(): print "done" def load_stdlib(): - - - - - - - - stacklet.with_stacklets(run_load_stdlib) + run_load_stdlib.invoke([]) def entry_point(args): interactive = True @@ -244,9 +235,9 @@ def entry_point(args): i += 1 if interactive: - with_stacklets(ReplFn(args)) + ReplFn(args).invoke([]) else: - with_stacklets(BatchModeFn(script_args)) + BatchModeFn(script_args).invoke([]) return 0 From 391e38d2bb887c2e1bfd0761adc4a69d2abf2a25 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Dec 2014 10:27:18 -0700 Subject: [PATCH 350/909] forgot two files in commit --- pixie/vm/compiler.py | 17 +---------------- target.py | 2 +- 2 files changed, 2 insertions(+), 17 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7d45b6c3..c5a126da 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -751,20 +751,6 @@ def compile_yield(form, ctx): compile_form(arg, ctx) ctx.bytecode.append(code.YIELD) -def compile_effect(form, ctx): - form = rt.next(form) - affirm(isinstance(rt.first(form), Keyword), u"First argument to -effect must be a keyword") - affirm(rt.count(form) == 2, u"Must provide a value to -effect") - - eff_name = rt.first(form) - form = rt.next(form) - eff_val = rt.next(form) - compile_form(eff_val, ctx) - ctx.push_const(eff_name) - ctx.bytecode.append(code.EFFECT) - ctx.sub_sp(1) - - builtins = {u"fn*": compile_fn, u"if": compile_if, u"platform=": compile_platform_eq, @@ -779,8 +765,7 @@ def compile_effect(form, ctx): u"__ns__": compile_ns, u"catch": compile_catch, u"this-ns-name": compile_this_ns, - u"yield": compile_yield, - u"-effect": compile_effect} + u"yield": compile_yield} def compiler_special(s): if isinstance(s, symbol.Symbol): diff --git a/target.py b/target.py index 52690b31..e0ea5f79 100644 --- a/target.py +++ b/target.py @@ -211,7 +211,7 @@ def entry_point(args): i += 1 if i < len(args): expr = args[i] - with_stacklets(EvalFn(expr)) + EvalFn(expr).invoke([]) return 0 else: print "Expected argument for " + arg From b40d5c5ec40d47e60ae0778a2fc8eeb4f934f709 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Dec 2014 10:28:29 -0700 Subject: [PATCH 351/909] remove some testing code --- pixie/vm/compiler.py | 17 +---------------- 1 file changed, 1 insertion(+), 16 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7d45b6c3..c5a126da 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -751,20 +751,6 @@ def compile_yield(form, ctx): compile_form(arg, ctx) ctx.bytecode.append(code.YIELD) -def compile_effect(form, ctx): - form = rt.next(form) - affirm(isinstance(rt.first(form), Keyword), u"First argument to -effect must be a keyword") - affirm(rt.count(form) == 2, u"Must provide a value to -effect") - - eff_name = rt.first(form) - form = rt.next(form) - eff_val = rt.next(form) - compile_form(eff_val, ctx) - ctx.push_const(eff_name) - ctx.bytecode.append(code.EFFECT) - ctx.sub_sp(1) - - builtins = {u"fn*": compile_fn, u"if": compile_if, u"platform=": compile_platform_eq, @@ -779,8 +765,7 @@ def compile_effect(form, ctx): u"__ns__": compile_ns, u"catch": compile_catch, u"this-ns-name": compile_this_ns, - u"yield": compile_yield, - u"-effect": compile_effect} + u"yield": compile_yield} def compiler_special(s): if isinstance(s, symbol.Symbol): From ac5743ec15047b0200ffe41d963c79bb9d10f1f8 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Dec 2014 11:30:54 -0700 Subject: [PATCH 352/909] Threads are here\! --- Makefile | 4 ++-- pixie/vm/rt.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c0f41a37..d3f08adf 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=../externals -PYTHON ?= python +PYTHON ?= pypy PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy help: @@ -16,7 +16,7 @@ build_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --continuation --no-shared target.py build_no_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --continuation --no-shared target.py + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target.py fetch_externals: $(EXTERNALS)/pypy diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 9b082f94..d0ac4be5 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -71,6 +71,7 @@ def wrapper(*args): import pixie.vm.symbol import pixie.vm.libs.path import pixie.vm.libs.string + import pixie.vm.threads From 8c484a360dedc5fbc5d252574f27ac9eea74c0fe Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Dec 2014 11:33:20 -0700 Subject: [PATCH 353/909] include --thread in Makefile --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index d3f08adf..6deac56f 100644 --- a/Makefile +++ b/Makefile @@ -13,7 +13,7 @@ help: @echo "make build_no_jit - build without jit" build_with_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --continuation --no-shared target.py + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target.py build_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target.py From 5aaf7386f37c7f395aedd0f81c430fe84baac656 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 6 Dec 2014 11:36:45 -0700 Subject: [PATCH 354/909] someday I'll figure out how git works --- pixie/vm/threads.py | 99 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 99 insertions(+) create mode 100644 pixie/vm/threads.py diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py new file mode 100644 index 00000000..c1589996 --- /dev/null +++ b/pixie/vm/threads.py @@ -0,0 +1,99 @@ +import rpython.rlib.rthread as rthread +from pixie.vm.primitives import nil +import rpython.rlib.rgil as rgil +from pixie.vm.code import wrap_fn, as_var + +from rpython.rlib.objectmodel import invoke_around_extcall +from rpython.rlib.rposix import get_errno, set_errno + + +class Bootstrapper(object): + def __init__(self): + self._is_inited = False + #_self.init() + + def init(self): + if not self._is_inited: + self._is_inited = True + self._lock = rthread.allocate_lock() + rgil.gil_allocate() + invoke_around_extcall(before_external_call, after_external_call) + + def aquire(self, fn): + self.init() + self._lock.acquire(True) + self._fn = fn + + def fn(self): + return self._fn + + def release(self): + self._fn = None + self._lock.release() + + + def __cleanup__(self): + self._lock = None + self._is_inited = False + + +def bootstrap(): + rthread.gc_thread_start() + fn = bootstrapper.fn() + bootstrapper.release() + fn.invoke([]) + rthread.gc_thread_die() + +bootstrapper = Bootstrapper() + +@as_var("-thread") +def new_thread(fn): + bootstrapper.aquire(fn) + ident = rthread.start_new_thread(bootstrap, ()) + return nil + + +## From PYPY + + +after_thread_switch = lambda: None # hook for signal.py + +# Fragile code below. We have to preserve the C-level errno manually... + +def before_external_call(): + # this function must not raise, in such a way that the exception + # transformer knows that it cannot raise! + rgil.gil_release() +before_external_call._gctransformer_hint_cannot_collect_ = True +before_external_call._dont_reach_me_in_del_ = True + +def after_external_call(): + e = get_errno() + rgil.gil_acquire() + rthread.gc_thread_run() + after_thread_switch() + set_errno(e) +after_external_call._gctransformer_hint_cannot_collect_ = True +after_external_call._dont_reach_me_in_del_ = True + +# The _gctransformer_hint_cannot_collect_ hack is needed for +# translations in which the *_external_call() functions are not inlined. +# They tell the gctransformer not to save and restore the local GC +# pointers in the shadow stack. This is necessary because the GIL is +# not held after the call to before_external_call() or before the call +# to after_external_call(). + +def do_yield_thread(): + # explicitly release the gil, in a way that tries to give more + # priority to other threads (as opposed to continuing to run in + # the same thread). + if rgil.gil_yield_thread(): + rthread.gc_thread_run() + after_thread_switch() +do_yield_thread._gctransformer_hint_close_stack_ = True +do_yield_thread._dont_reach_me_in_del_ = True +do_yield_thread._dont_inline_ = True + +# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_. +# The *_external_call() functions are themselves called only from the rffi +# module from a helper function that also has this hint. From d921c2886f007f5301cbd0ab7bcbb557c0b569bd Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sat, 6 Dec 2014 16:48:34 -0500 Subject: [PATCH 355/909] naive shell using ffi --- pixie/stdlib.pxi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index fc85b44e..cc981979 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -3,6 +3,7 @@ (def libc (ffi-library pixie.platform/lib-c-name)) (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) + (def shell (ffi-fn libc "system" [String] Integer)) (def libreadline (ffi-library (str "libreadline." pixie.platform/so-ext))) (def readline (ffi-fn libreadline "readline" [String] String)) @@ -1113,6 +1114,7 @@ and implements IAssociative, ILookup and IObject." (def libc (ffi-library pixie.platform/lib-c-name)) (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) + (def shell (ffi-fn libc "system" [String] Integer)) (def printf (ffi-fn libc "printf" [String] Integer)) (def getenv (ffi-fn libc "getenv" [String] String)) From 75554b8c6fad0c7b7ec982e79380655774b426c4 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 7 Dec 2014 16:25:09 -0500 Subject: [PATCH 356/909] remove duplication --- pixie/stdlib.pxi | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index cc981979..e4ed97fc 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -4,6 +4,8 @@ (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) (def shell (ffi-fn libc "system" [String] Integer)) + (def printf (ffi-fn libc "printf" [String] Integer)) + (def getenv (ffi-fn libc "getenv" [String] String)) (def libreadline (ffi-library (str "libreadline." pixie.platform/so-ext))) (def readline (ffi-fn libreadline "readline" [String] String)) @@ -1111,13 +1113,6 @@ and implements IAssociative, ILookup and IObject." `(do ~type-from-map ~deftype-decl))) - (def libc (ffi-library pixie.platform/lib-c-name)) - (def exit (ffi-fn libc "exit" [Integer] Integer)) - (def puts (ffi-fn libc "puts" [String] Integer)) - (def shell (ffi-fn libc "system" [String] Integer)) - (def printf (ffi-fn libc "printf" [String] Integer)) - (def getenv (ffi-fn libc "getenv" [String] String)) - (defn print {:doc "Prints the arguments, seperated by spaces." :added "0.1"} From df4960ab22946cac8a4783e400369182dea55f64 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 7 Dec 2014 16:45:50 -0500 Subject: [PATCH 357/909] =?UTF-8?q?shortened=20the=20name=20to=20=E2=80=9C?= =?UTF-8?q?sh=E2=80=9D?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e4ed97fc..a2e234e8 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -3,7 +3,7 @@ (def libc (ffi-library pixie.platform/lib-c-name)) (def exit (ffi-fn libc "exit" [Integer] Integer)) (def puts (ffi-fn libc "puts" [String] Integer)) - (def shell (ffi-fn libc "system" [String] Integer)) + (def sh (ffi-fn libc "system" [String] Integer)) (def printf (ffi-fn libc "printf" [String] Integer)) (def getenv (ffi-fn libc "getenv" [String] String)) From d22b9c3deb39d5f2dd11c20a93c21c07227d97b9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 8 Dec 2014 07:07:52 -0700 Subject: [PATCH 358/909] added hash set iterator --- pixie/stdlib.pxi | 27 +----- pixie/vm/persistent_hash_map.py | 145 +++++++++++++++++++++++++++++++- pixie/vm/persistent_hash_set.py | 16 +++- pixie/vm/rt.py | 4 +- pixie/vm/stdlib.py | 1 - pixie/vm/threads.py | 4 + tests/test-docs.pxi | 9 +- tests/test-stdlib.pxi | 2 +- 8 files changed, 176 insertions(+), 32 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4bb25397..45243cdc 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -301,26 +301,6 @@ (cons (-current k) (lazy-seq* (fn [] (stacklet->lazy-seq (-move-next! k)))))))) -(def sequence - (fn ^{:doc "Returns a lazy sequence of the `data`, optionally transforming it using `xform`." - :signatures [[data] [xform data]] - :added "0.1"} - sequence - ([data] - (let [f (create-stacklet - (fn [h] - (reduce (fn ([h item] (h item) h)) h data)))] - (stacklet->lazy-seq f))) - ([xform data] - (let [f (create-stacklet - (fn [h] - (transduce xform - (fn ([] h) - ([h item] (h item) h) - ([h] nil)) - data)))] - (stacklet->lazy-seq f))))) - (def = -eq) (extend -seq PersistentVector @@ -838,7 +818,7 @@ Stops if it finds such an element." (fn [v] (transduce cat unordered-hash-reducing-fn v))) -(extend -seq PersistentHashSet sequence) +(extend -seq PersistentHashSet (fn [self] (seq (iterator self)))) (extend -str PersistentHashSet (fn [s] @@ -1579,7 +1559,6 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (let [acc (f acc (-current k))] (-move-next! k) (recur acc))))))) - (defn filter {:doc "Filter the collection for elements matching the predicate." :signatures [[pred] [pred coll]] @@ -1592,7 +1571,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (xf acc i) acc))))) ([f coll] - (sequence (filter f) coll))) + nil)) (defn distinct {:doc "Returns the distinct elements in the collection." @@ -1610,7 +1589,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (swap! seen conj i) (xf acc i)))))))) ([coll] - (sequence (distinct) coll))) + nil)) (defn keep ([f] diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index 4a1a1052..adcccb0b 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -8,6 +8,7 @@ from rpython.rlib.rarithmetic import r_int, r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt +from pixie.vm.iterator import NativeIterator, empty_iterator MASK_32 = r_uint(0xFFFFFFFF) @@ -59,6 +60,14 @@ def without(self, key): return self return PersistentHashMap(self._cnt - 1, new_root, self._meta) + def iter(self): + if self._root is None: + return empty_iterator + else: + return self._root.iter() + + + @@ -177,6 +186,9 @@ def reduce_inode(self, f, init): return init return init + def iter(self): + return BitmapIndexedNodeIterator(self._array) + def without_inode(self, shift, hash, key): bit = bitpos(hash, shift) if self._bitmap & bit == 0: @@ -203,9 +215,59 @@ def without_inode(self, shift, hash, key): return self +BitmapIndexedNode_EMPTY = BitmapIndexedNode(None, r_uint(0), []) + + +class BitmapIndexedNodeIterator(NativeIterator): + def __init__(self, array): + self._array_w = array + self._idx = 0 + self._child_iterator_w = None + self._at_end = False + self._current = nil + self.move_next() + + def move_next(self): + while True: + if self._child_iterator_w is None: + while True: + if self._idx == len(self._array_w): + self._current = nil + self._at_end = True + self._array_w = None + return self + key_or_none = self._array_w[self._idx] + val_or_node = self._array_w[self._idx + 1] + + if key_or_none is not None: + self._idx += 2 + self._current = rt.map_entry(key_or_none, val_or_node) + return self + + elif val_or_node is not None: + iter = val_or_node.iter() + if iter.at_end(): + self._idx += 1 + continue + self._child_iterator_w = iter + self._current = self._child_iterator_w.current() + return self + + self._idx += 2 + else: + self._child_iterator_w.move_next() + if self._child_iterator_w.at_end(): + continue + self._current = self._child_iterator_w.current() + return self + + def at_end(self): + return self._at_end + + def current(self): + return self._current -BitmapIndexedNode_EMPTY = BitmapIndexedNode(None, r_uint(0), []) class ArrayNode(INode): @@ -284,6 +346,54 @@ def reduce_inode(self, f, init): return init + def iter(self): + return ArrayMapIterator(self._array) + +class ArrayMapIterator(NativeIterator): + def __init__(self, array): + self._array_w = array + self._idx = 0 + self._child_iterator_w = None + self._at_end = False + self._current = nil + self.move_next() + + def move_next(self): + while True: + if self._child_iterator_w is None: + while True: + if self._idx == len(self._array_w): + self._current = nil + self._at_end = True + self._array_w = None + return self + val = self._array_w[self._idx] + if val is not None: + iter = val.iter() + if iter.at_end(): + self._idx += 1 + continue + self._child_iterator_w = iter + self._current = self._child_iterator_w.current() + return self + + self._idx += 1 + else: + self._child_iterator_w.move_next() + if self._child_iterator_w.at_end(): + continue + self._current = self._child_iterator_w.current() + return self + + def at_end(self): + return self._at_end + + def current(self): + return self._current + + + + class HashCollisionNode(INode): def __init__(self, edit, hash, array): self._hash = hash @@ -328,6 +438,9 @@ def reduce_inode(self, f, init): return init return init + def iter(self): + return HashCollisionNodeIterator(self._array) + def find_index(self, key): i = r_int(0) while i < len(self._array): @@ -349,6 +462,36 @@ def without_inode(self, shift, hash, key): return HashCollisionNode(None, self._hash, remove_pair(self._array, r_uint(idx) / 2)) +class HashCollisionNodeIterator(NativeIterator): + def __init__(self, array): + self._w_array = array + self._w_idx = 0 + self._w_current = nil + self.move_next() + + def move_next(self): + while True: + if self._w_idx == len(self._w_array): + self._w_current = nil + self._w_array = None + return self + + key = self._w_array[self._w_idx] + val = self._w_array[self._w_idx + 1] + self._w_idx += 2 + if key is None: + continue + + self._w_current = rt.map_entry(key, val) + return + + def at_end(self): + return self._w_current is None + + def current(self): + return self._w_current + + def create_node(shift, key1, val1, key2hash, key2, val2): key1hash = rt.hash(key1) & MASK_32 diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index da311f2a..7ae23443 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -5,10 +5,14 @@ from pixie.vm.numbers import Integer import pixie.vm.persistent_hash_map as persistent_hash_map import pixie.vm.stdlib as proto -from pixie.vm.code import extend, as_var +from pixie.vm.code import extend, as_var, intern_var from rpython.rlib.rarithmetic import r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt +from pixie.vm.iterator import MapIterator + + +VAR_KEY = intern_var(u"pixie.stdlib", u"key") class PersistentHashSet(object.Object): _type = object.Type(u"pixie.stdlib.PersistentHashSet") @@ -32,6 +36,12 @@ def meta(self): def with_meta(self, meta): return PersistentHashSet(meta, self._map) + def iter(self): + return MapIterator(VAR_KEY.deref(), self._map.iter()) + + + + EMPTY = PersistentHashSet(nil, persistent_hash_map.EMPTY) @as_var("set") @@ -99,3 +109,7 @@ def _meta(self): def _with_meta(self, meta): assert isinstance(self, PersistentHashSet) return self.with_meta(meta) + +@extend(proto._iterator, PersistentHashSet) +def _iterator(self): + return self.iter() diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index d0ac4be5..c64e5f9a 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -112,7 +112,7 @@ def int_val(x): for name, var in _ns_registry._registry[u"pixie.stdlib"]._registry.iteritems(): name = munge(name) print name - if isinstance(var.deref(), BaseCode): + if var.is_defined() and isinstance(var.deref(), BaseCode): globals()[name] = unwrap(var) else: globals()[name] = var @@ -127,7 +127,7 @@ def reinit(): continue print "Found ->> ", name, var.deref() - if isinstance(var.deref(), BaseCode): + if var.is_defined() and isinstance(var.deref(), BaseCode): globals()[name] = unwrap(var) else: globals()[name] = var diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 78acb582..7bc81907 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -7,7 +7,6 @@ from pixie.vm.primitives import true, false, nil import pixie.vm.numbers as numbers import rpython.rlib.jit as jit -import rpython.rlib.rstacklet as rstacklet from rpython.rlib.rarithmetic import r_uint from pixie.vm.interpreter import ShallowContinuation diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index c1589996..3abab66f 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -52,6 +52,10 @@ def new_thread(fn): ident = rthread.start_new_thread(bootstrap, ()) return nil +@as_var("-yield-thread") +def yield_thread(): + do_yield_thread() + return nil ## From PYPY diff --git a/tests/test-docs.pxi b/tests/test-docs.pxi index b8f66996..2d4148d1 100644 --- a/tests/test-docs.pxi +++ b/tests/test-docs.pxi @@ -12,9 +12,14 @@ examples (get meta :examples)] (doseq [example examples] (if (contains? example 2) - (do + (try + (println (first example)) + (println ".") (t/assert= (eval (read-string (first example))) - (third example))) + (third example)) + (catch ex + (println "Example Failed") + (println example))) (eval (read-string (first example))))))))) (t/deftest test-stdlib-docs diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 23e1e869..509f98e6 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -118,7 +118,7 @@ (t/deftest test-keys (let [v {:a 1 :b 2 :c 3}] - (t/assert= (keys v) #{:a :b :c}) + (t/assert= (set (keys v)) #{:a :b :c}) (t/assert= (transduce (keys) conj! v) (keys v)))) (t/deftest test-vals From db7cf6aa7e25107078709f8cdf9f020a71078225 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 8 Dec 2014 16:09:34 +0100 Subject: [PATCH 359/909] fix nil in quotes and syntax quotes --- pixie/stdlib.pxi | 2 ++ pixie/vm/reader.py | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a2e234e8..d253c642 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -225,6 +225,8 @@ (extend -repr Nil -str) (extend -reduce Nil (fn [self f init] init)) (extend -hash Nil (fn [self] 100000)) +(extend -with-meta Nil (fn [self _] nil)) +(extend -at-end? Nil (fn [_] true)) (extend -hash Integer hash-int) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index b8d2eb80..d1a9b0ca 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -419,7 +419,7 @@ def syntax_quote(form): raise Exception("Unquote splicing not used inside list") elif rt.vector_QMARK_(form) is true: ret = rt.list(APPLY, CONCAT, SyntaxQuoteReader.expand_list(form)) - elif rt.seq_QMARK_(form) is true: + elif form is not nil and rt.seq_QMARK_(form) is true: ret = rt.list(APPLY, LIST, rt.cons(CONCAT, SyntaxQuoteReader.expand_list(rt.seq(form)))) else: ret = rt.list(QUOTE, form) From e9be20a31cbe5f1b4a951e5a019042d3723f9fc2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 8 Dec 2014 16:11:02 +0100 Subject: [PATCH 360/909] fix -eq involving nil and empty collections --- pixie/vm/persistent_vector.py | 2 +- pixie/vm/stdlib.py | 2 ++ tests/collections/test-maps.pxi | 1 + tests/collections/test-seqables.pxi | 2 ++ tests/collections/test-sets.pxi | 4 ++++ tests/collections/test-vectors.pxi | 3 +++ 6 files changed, 13 insertions(+), 1 deletion(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 4315c668..723366d9 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -413,7 +413,7 @@ def _eq(self, obj): return false return true else: - if not rt.satisfies_QMARK_(proto.ISeqable, obj): + if obj is nil or not rt.satisfies_QMARK_(proto.ISeqable, obj): return false seq = rt.seq(obj) for i in range(0, intmask(self._cnt)): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 78acb582..073fca00 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -142,6 +142,8 @@ def seq_QMARK_(x): def _seq_eq(a, b): if a is b: return true + if a is nil or b is nil: + return false if not (rt.satisfies_QMARK_(rt.ISeqable.deref(), b) or rt.satisfies_QMARK_(rt.ISeq.deref(), b)): return false diff --git a/tests/collections/test-maps.pxi b/tests/collections/test-maps.pxi index c10bc44b..a4072fe1 100644 --- a/tests/collections/test-maps.pxi +++ b/tests/collections/test-maps.pxi @@ -17,6 +17,7 @@ (t/assert= (-eq m {:a 1, :b 2, :c 3}) true) (t/assert= (-eq m {}) false) + (t/assert= (-eq m nil) false) (t/assert= (-eq m {:a 1, :b 2}) false) (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false) diff --git a/tests/collections/test-seqables.pxi b/tests/collections/test-seqables.pxi index 35e47c1b..dcf79354 100644 --- a/tests/collections/test-seqables.pxi +++ b/tests/collections/test-seqables.pxi @@ -35,6 +35,8 @@ (t/assert= l '(1 2 3)) (t/assert= l [1 2 3]) + (t/assert= (= nil '()) false) + (t/assert= (= '() nil) false) (t/assert= (= l '(1 2 3 4)) false) (t/assert= (= l [1 2 3 4]) false))) diff --git a/tests/collections/test-sets.pxi b/tests/collections/test-sets.pxi index cd80c906..62652da8 100644 --- a/tests/collections/test-sets.pxi +++ b/tests/collections/test-sets.pxi @@ -42,6 +42,10 @@ (t/assert= s #{1 2 3}) (t/assert= #{1 2 3} s) + (t/assert= (= #{} nil) false) + (t/assert= (= #{} []) false) + (t/assert= (= #{} '()) false) + (t/assert= (= s [1 2 3]) false) (t/assert= (= s '(1 2 3)) false) (t/assert= (= s #{1 2}) false) diff --git a/tests/collections/test-vectors.pxi b/tests/collections/test-vectors.pxi index 805155be..f69432fe 100644 --- a/tests/collections/test-vectors.pxi +++ b/tests/collections/test-vectors.pxi @@ -24,10 +24,13 @@ (t/deftest vector-equals (let [v [1 2 3]] + (t/assert= [] '()) (t/assert= v v) (t/assert= v [1 2 3]) (t/assert= v '(1 2 3)) + (t/assert= (= [] nil) false) + (t/assert= (= v []) false) (t/assert= (= v [1 2]) false) (t/assert= (= v [1 2 3 4]) false) (t/assert= (= v '(1 2)) false) From a44416f311e1858a8dfc2cc84b4e5a7493f51d0c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 8 Dec 2014 15:41:24 -0700 Subject: [PATCH 361/909] fixed filter and distinct, getting a few unknown runtime errors --- pixie/stdlib.pxi | 19 +++++++++++++++++-- pixie/vm/compiler.py | 2 ++ pixie/vm/stdlib.py | 4 +++- tests/test-docs.pxi | 8 +------- tests/test-stdlib.pxi | 4 ++-- 5 files changed, 25 insertions(+), 12 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 45243cdc..a209b313 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1571,7 +1571,12 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (xf acc i) acc))))) ([f coll] - nil)) + (let [iter (iterator coll)] + (loop [] + (when (not (at-end? iter)) + (yield (current iter)) + (move-next! iter) + (recur)))))) (defn distinct {:doc "Returns the distinct elements in the collection." @@ -1589,7 +1594,17 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (swap! seen conj i) (xf acc i)))))))) ([coll] - nil)) + (let [iter (iterator coll)] + (loop [acc #{}] + (when (not (at-end? iter)) + (if (contains? acc (current iter)) + (do (move-next! iter) + (recur acc)) + (let [val (current iter)] + (yield val) + (move-next! iter) + (recur (conj acc val))))))))) + (defn keep ([f] diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index c5a126da..dd5c40a8 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -632,6 +632,8 @@ def compile_recur(form, ctx): ctx.enable_tail_call() if args > 0: ctx.sub_sp(r_uint(args - 1)) + else: + ctx.add_sp(r_uint(1)) def compile_let(form, ctx): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 7bc81907..ec8afaaa 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -9,7 +9,7 @@ import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import r_uint from pixie.vm.interpreter import ShallowContinuation - +from rpython.rlib.objectmodel import we_are_translated defprotocol("pixie.stdlib", "ISeq", ["-first", "-next"]) defprotocol("pixie.stdlib", "ISeqable", ["-seq"]) @@ -536,6 +536,8 @@ def _try_catch(main_fn, catch_fn, final): from pixie.vm.string import String if isinstance(ex, Exception): ex = RuntimeException(rt.wrap(u"Some error")) + if not we_are_translated(): + print "Error", ex else: ex = RuntimeException(nil) return catch_fn.invoke([ex]) diff --git a/tests/test-docs.pxi b/tests/test-docs.pxi index 2d4148d1..dd6b96cf 100644 --- a/tests/test-docs.pxi +++ b/tests/test-docs.pxi @@ -12,14 +12,8 @@ examples (get meta :examples)] (doseq [example examples] (if (contains? example 2) - (try - (println (first example)) - (println ".") - (t/assert= (eval (read-string (first example))) + (t/assert= (eval (read-string (first example))) (third example)) - (catch ex - (println "Example Failed") - (println example))) (eval (read-string (first example))))))))) (t/deftest test-stdlib-docs diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi index 509f98e6..4f639268 100644 --- a/tests/test-stdlib.pxi +++ b/tests/test-stdlib.pxi @@ -123,7 +123,7 @@ (t/deftest test-vals (let [v {:a 1 :b 2 :c 3}] - (t/assert= (vals v) #{1 2 3}) + (t/assert= (set (vals v)) #{1 2 3}) (t/assert= (transduce (vals) conj! v) (vals v)))) @@ -219,7 +219,7 @@ (t/assert= (some odd? [2]) false)) (t/deftest test-distinct - (t/assert= (sequence (distinct) [1 2 3 2 1]) '(1 2 3)) + (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) (t/assert= (vec (distinct) [nil nil nil]) [nil])) From 21c090569b032be512435bd9a5ebb55c7f462dc5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 8 Dec 2014 15:53:11 -0700 Subject: [PATCH 362/909] some debugging help while interpreted --- pixie/vm/stdlib.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index ec8afaaa..c9e764f2 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -535,9 +535,9 @@ def _try_catch(main_fn, catch_fn, final): if not isinstance(ex, WrappedException): from pixie.vm.string import String if isinstance(ex, Exception): - ex = RuntimeException(rt.wrap(u"Some error")) if not we_are_translated(): - print "Error", ex + print "Python Error Info: ", ex.__dict__, type(ex) + ex = RuntimeException(rt.wrap(u"Some error")) else: ex = RuntimeException(nil) return catch_fn.invoke([ex]) From 4139be7427ec2f5ec592ee29e7aee9643c251051 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 8 Dec 2014 16:23:05 -0700 Subject: [PATCH 363/909] yay\! tests pass again --- pixie/vm/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 65ab618b..552e4278 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -538,7 +538,7 @@ def _try_catch(main_fn, catch_fn, final): from pixie.vm.string import String if isinstance(ex, Exception): if not we_are_translated(): - print "Python Error Info: ", ex.__dict__, type(ex) + print "Python Error Info: ", ex.__dict__, ex ex = RuntimeException(rt.wrap(u"Some error")) else: ex = RuntimeException(nil) From 182395db7c243784ad18cf267b78f5ebaad24dae Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 8 Dec 2014 16:36:22 -0700 Subject: [PATCH 364/909] really? I forgot a file again. --- pixie/vm/iterator.py | 76 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 76 insertions(+) create mode 100644 pixie/vm/iterator.py diff --git a/pixie/vm/iterator.py b/pixie/vm/iterator.py new file mode 100644 index 00000000..2d0b3e8b --- /dev/null +++ b/pixie/vm/iterator.py @@ -0,0 +1,76 @@ +from pixie.vm.object import Object, Type, runtime_error +from pixie.vm.code import extend, as_var +from pixie.vm.primitives import true, false, nil +import pixie.vm.stdlib as proto +import pixie.vm.rt as rt + + +class EmptyIterator(Object): + _type = Type(u"pixie.vm.Iterator") + + def type(self): + return EmptyIterator._type + + + def __init__(self): + pass + + def move_next(self): + runtime_error(u"Empty Iterator") + + def at_end(self): + return True + + +empty_iterator = EmptyIterator() + +@extend(proto._at_end_QMARK_, EmptyIterator) +def _at_end(_): + return true + + +class NativeIterator(Object): + _type = Type(u"pixie.vm.NativeIterator") + + def type(self): + return NativeIterator._type + + def __init__(self): + pass + +@extend(proto._at_end_QMARK_, NativeIterator) +def _at_end(self): + return rt.wrap(self.at_end()) + + +@extend(proto._move_next_BANG_, NativeIterator) +def _move_next(self): + return self.move_next() + + +@extend(proto._current, NativeIterator) +def _current(self): + return self.current() + + +class MapIterator(NativeIterator): + def __init__(self, fn, iter): + self._w_fn = fn + self._w_iter = iter + if not iter.at_end(): + self._w_current = self._w_fn.invoke([self._w_iter.current()]) + + def move_next(self): + self._w_iter.move_next() + + if not self._w_iter.at_end(): + self._w_current = self._w_fn.invoke([self._w_iter.current()]) + else: + self._w_current = nil + + def current(self): + return self._w_current + + def at_end(self): + return self._w_iter.at_end() + From 8ce7a4d730ad3e7f25652229ce7b4a53bdedc408 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 8 Dec 2014 17:04:10 -0700 Subject: [PATCH 365/909] update build script to include threads --- make-with-jit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make-with-jit b/make-with-jit index 8abafb1c..c5160670 100755 --- a/make-with-jit +++ b/make-with-jit @@ -1 +1 @@ -python ../externals/pypy/rpython/bin/rpython --opt=jit --continuation --no-shared target.py +python ../externals/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target.py From e6917b99014a2407d77dce945d145b763ab1a58d Mon Sep 17 00:00:00 2001 From: Scott Robinson Date: Tue, 9 Dec 2014 16:42:04 +1100 Subject: [PATCH 366/909] Fix typo in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d2108e0f..8913ca7a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ Some planned and implemented features: * Immutable datastructures * Protocols first implementation -* Transducers at-the-bottom (most primitves are based off of reduce) +* Transducers at-the-bottom (most primitives are based off of reduce) * Coroutines for transducer inversion of control (transducer to lazy-seq conversion) * A "good enough" JIT (implemented, tuning still a WIP, but not bad performance today) * Easy FFI (TODO) From af883309b3b3c2a62793d25b1954ca64f03447d1 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Tue, 9 Dec 2014 14:55:30 +0100 Subject: [PATCH 367/909] make iterators a bit friendlier to use `first` now works on any IIterator, returning `-current`, and `-iterator` is supported on IIterators as well to make the following possible: (map inc (distinct [1 2 3 1 -1 10])) --- pixie/stdlib.pxi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 0ec3ad94..5f398d92 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1525,6 +1525,9 @@ For more information, see http://clojure.org/special_forms#binding-forms"} nil (cons (current i) (lazy-seq (iterator-seq (move-next! i)))))) +(extend -first IIterator -current) +(extend -iterator IIterator identity) + (extend -seq IIterator iterator-seq) (extend -seq IIterable (comp seq iterator)) @@ -1558,6 +1561,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (let [acc (f acc (-current k))] (-move-next! k) (recur acc))))))) + (defn filter {:doc "Filter the collection for elements matching the predicate." :signatures [[pred] [pred coll]] From 41be76e15d14dd1c9fc312da1c2c2aa6ae5c9f3b Mon Sep 17 00:00:00 2001 From: Rui Carmo Date: Wed, 10 Dec 2014 12:34:58 +0000 Subject: [PATCH 368/909] Update README.md Used Ubuntu package naming for guidance. --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 8913ca7a..fbf625e8 100644 --- a/README.md +++ b/README.md @@ -20,8 +20,9 @@ Some planned and implemented features: ## Dependencies -* libuv +* libuv-dev * libffi-dev +* libreadline-dev ## Building From d6f92493c9d6a5296b39a5a38f092101a36cfd62 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 10 Dec 2014 07:22:14 -0700 Subject: [PATCH 369/909] added double support, confirming that strings are broken, WIP --- pixie/stdlib.pxi | 3 ++ pixie/vm/libs/ffi.py | 67 +++++++++++++++++++++++++++++++++++++------- 2 files changed, 60 insertions(+), 10 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 0ec3ad94..ee962d9f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -11,6 +11,9 @@ (def readline (ffi-fn libreadline "readline" [String] String)) (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) + (def fopen (ffi-fn libc "fopen" [String String] VoidP)) + (def atan2 (ffi-fn libc "atan2" [Float Float] Float)) + (def floor (ffi-fn libc "floor" [Float] Float)) (def reset! -reset!) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index a7fa30a4..14a21728 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -5,7 +5,7 @@ import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype from pixie.vm.primitives import nil -from pixie.vm.numbers import Integer +from pixie.vm.numbers import Integer, Float from pixie.vm.string import String from rpython.rlib.jit_libffi import CIF_DESCRIPTION from rpython.rlib import clibffi @@ -68,8 +68,12 @@ def _cleanup_(self): def get_native_size(tp): if tp == Integer._type: return rffi.sizeof(rffi.LONG) + if tp == Float._type: + return rffi.sizeof(rffi.DOUBLE) if tp == String._type: return rffi.sizeof(rffi.CCHARP) + if tp == FFIVoidP._type: + return rffi.sizeof(rffi.VOIDP) assert False def get_ret_val(ptr, tp): @@ -77,6 +81,10 @@ def get_ret_val(ptr, tp): pnt = rffi.cast(rffi.LONGP, ptr) val = pnt[0] return Integer(val) + if tp == Float._type: + pnt = rffi.cast(rffi.DOUBLEP, ptr) + val = pnt[0] + return Float(val) if tp == String._type: pnt = rffi.cast(rffi.CCHARPP, ptr) if pnt[0] == lltype.nullptr(rffi.CCHARP.TO): @@ -84,6 +92,11 @@ def get_ret_val(ptr, tp): else: return String(unicode(rffi.charp2str(pnt[0]))) + if tp == FFIVoidP._type: + pnt = rffi.cast(rffi.VOIDPP, ptr) + val = pnt[0] + return FFIVoidP(val) + assert False def set_native_value(ptr, val, tp): @@ -91,10 +104,18 @@ def set_native_value(ptr, val, tp): pnt = rffi.cast(rffi.LONGP, ptr) pnt[0] = rffi.cast(rffi.LONG, val.int_val()) return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.LONG))) + if tp is Float._type: + pnt = rffi.cast(rffi.DOUBLEP, ptr) + pnt[0] = rffi.cast(rffi.DOUBLE, val.float_val()) + return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.DOUBLE))) if tp is String._type: pnt = rffi.cast(rffi.CCHARPP, ptr) pnt[0] = rffi.str2charp(str(rt.name(val))) return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.CCHARP))) + if tp is FFIVoidP._type: + pnt = rffi.cast(rffi.VOIDPP, ptr) + pnt[0] = val.voidp_data() + return rffi.cast(rffi.CHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.VOIDP))) assert False class FFIFn(object.Object): @@ -117,15 +138,13 @@ def __init__(self, lib, name, arg_types, ret_type): def thaw(self): if not self._is_inited: self._f_ptr = self._lib.get_fn_ptr(self._name) - transfer_size = 0 arg0_offset = len(self._arg_types) * rffi.sizeof(rffi.CCHARP) - exchange_result = arg0_offset + exchange_buffer_size = arg0_offset for x in self._arg_types: - exchange_result = transfer_size - transfer_size += get_native_size(x) + exchange_buffer_size += get_native_size(x) - ret_offset = transfer_size - transfer_size += get_native_size(self._ret_type) + ret_offset = exchange_buffer_size + exchange_buffer_size += get_native_size(self._ret_type) cd = lltype.malloc(CIF_DESCRIPTION, len(self._arg_types), flavor="raw") cd.abi = clibffi.FFI_DEFAULT_ABI @@ -133,23 +152,33 @@ def thaw(self): cd.rtype = get_clibffi_type(self._ret_type) atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, len(self._arg_types), flavor="raw") + arg_x_offset = arg0_offset + idx = 0 + for x in self._arg_types: + cd.exchange_args[idx] = arg_x_offset + print arg_x_offset + arg_x_offset += get_native_size(x) + idx += 1 + + for x in range(len(self._arg_types)): atypes[x] = get_clibffi_type(self._arg_types[x]) cd.atypes = atypes - cd.exchange_size = transfer_size + cd.exchange_size = exchange_buffer_size cd.exchange_result = ret_offset cd.exchange_result_libffi = ret_offset - cd.exchange_args[0] = arg0_offset jit_ffi_prep_cif(cd) self._cd = cd - self._transfer_size = transfer_size + self._transfer_size = exchange_buffer_size self._arg0_offset = arg0_offset self._ret_offset = ret_offset + print exchange_buffer_size, arg0_offset, ret_offset + self._is_inited = True return self @@ -194,8 +223,12 @@ def invoke(self, args): def get_clibffi_type(arg): if arg == Integer._type: return clibffi.cast_type_to_ffitype(rffi.LONG) + if arg == Float._type: + return clibffi.cast_type_to_ffitype(rffi.DOUBLE) if arg == String._type: return clibffi.ffi_type_pointer + if arg == FFIVoidP._type: + return clibffi.ffi_type_pointer assert False @@ -220,4 +253,18 @@ def _ffi_fn(lib, nm, args, ret_type): f = FFIFn(lib, rt.name(nm), new_args, ret_type) return f +class FFIVoidP(object.Object): + _type = object.Type(u"pixie.stdlib.VoidP") + + def type(self): + return FFIVoidP._type + + def __init__(self, data): + self._voidp_data = data + + def voidp_data(self): + return self._voidp_data + + + From d640732abe5af27e6b9d9268c813626632740341 Mon Sep 17 00:00:00 2001 From: Ben Hughes Date: Thu, 11 Dec 2014 09:35:09 +1100 Subject: [PATCH 370/909] Add links for library dependencies to README --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index fbf625e8..6fb2857a 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ Some planned and implemented features: ## Dependencies -* libuv-dev -* libffi-dev -* libreadline-dev +* [libuv-dev](https://github.com/libuv/libuv) +* [libffi-dev](https://sourceware.org/libffi/) +* [libreadline-dev](http://cnswww.cns.cwru.edu/php/chet/readline/rltop.html) ## Building From 74f924843f0674c0d48372df4c4807c3e5c25134 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 10 Dec 2014 16:23:34 -0700 Subject: [PATCH 371/909] VERY WIP, but fixes a stupid issue with ffi --- pixie/stdlib.pxi | 3 +++ pixie/vm/libs/ffi.py | 44 +++++++++++++++++++++----------------------- 2 files changed, 24 insertions(+), 23 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ee962d9f..e8cec2d5 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -14,6 +14,9 @@ (def fopen (ffi-fn libc "fopen" [String String] VoidP)) (def atan2 (ffi-fn libc "atan2" [Float Float] Float)) (def floor (ffi-fn libc "floor" [Float] Float)) + (def strstr (ffi-fn libc "strstr" [String String] String)) + (def strlen (ffi-fn libc "strlen" [String] Integer)) + (def printf (ffi-fn libc "printf" [String String] Integer)) (def reset! -reset!) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 14a21728..702f712c 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -7,9 +7,8 @@ from pixie.vm.primitives import nil from pixie.vm.numbers import Integer, Float from pixie.vm.string import String -from rpython.rlib.jit_libffi import CIF_DESCRIPTION from rpython.rlib import clibffi -from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call +from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION import rpython.rlib.jit as jit @@ -111,11 +110,12 @@ def set_native_value(ptr, val, tp): if tp is String._type: pnt = rffi.cast(rffi.CCHARPP, ptr) pnt[0] = rffi.str2charp(str(rt.name(val))) - return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.CCHARP))) + print "sizeof", rffi.sizeof(rffi.CCHARP) + return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) if tp is FFIVoidP._type: pnt = rffi.cast(rffi.VOIDPP, ptr) pnt[0] = val.voidp_data() - return rffi.cast(rffi.CHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.VOIDP))) + return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.VOIDP))) assert False class FFIFn(object.Object): @@ -138,31 +138,26 @@ def __init__(self, lib, name, arg_types, ret_type): def thaw(self): if not self._is_inited: self._f_ptr = self._lib.get_fn_ptr(self._name) - arg0_offset = len(self._arg_types) * rffi.sizeof(rffi.CCHARP) - exchange_buffer_size = arg0_offset - for x in self._arg_types: - exchange_buffer_size += get_native_size(x) + nargs = len(self._arg_types) - ret_offset = exchange_buffer_size - exchange_buffer_size += get_native_size(self._ret_type) + exchange_buffer_size = nargs * rffi.sizeof(rffi.CCHARP) - cd = lltype.malloc(CIF_DESCRIPTION, len(self._arg_types), flavor="raw") + cd = lltype.malloc(CIF_DESCRIPTION, nargs, flavor="raw") cd.abi = clibffi.FFI_DEFAULT_ABI - cd.nargs = len(self._arg_types) + cd.nargs = nargs cd.rtype = get_clibffi_type(self._ret_type) - atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, len(self._arg_types), flavor="raw") - - arg_x_offset = arg0_offset - idx = 0 - for x in self._arg_types: - cd.exchange_args[idx] = arg_x_offset - print arg_x_offset - arg_x_offset += get_native_size(x) - idx += 1 + atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, nargs, flavor="raw") + arg0_offset = exchange_buffer_size + for idx in range(nargs): + cd.exchange_args[idx] = exchange_buffer_size + tp = self._arg_types[idx] + native_size = get_native_size(tp) + atypes[idx] = get_clibffi_type(tp) + exchange_buffer_size += native_size - for x in range(len(self._arg_types)): - atypes[x] = get_clibffi_type(self._arg_types[x]) + ret_offset = exchange_buffer_size + exchange_buffer_size += get_native_size(self._ret_type) @@ -197,6 +192,7 @@ def prep_exb(self, args): offset_p = rffi.ptradd(exb, self._arg0_offset) for x in range(len(self._arg_types)): + print offset_p offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) return exb @@ -209,7 +205,9 @@ def get_ret_val_from_buffer(self, exb): def _invoke(self, args): exb = self.prep_exb(args) + print "calling" jit_ffi_call(self._cd, self._f_ptr, exb) + print "done" ret_val = self.get_ret_val_from_buffer(exb) lltype.free(exb, flavor="raw") return ret_val From 53b97d03258ffb0237ef886af41c5eaa14735054 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 10 Dec 2014 16:27:18 -0700 Subject: [PATCH 372/909] removed debug stuff --- pixie/stdlib.pxi | 3 --- pixie/vm/libs/ffi.py | 12 +++--------- 2 files changed, 3 insertions(+), 12 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e8cec2d5..ee962d9f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -14,9 +14,6 @@ (def fopen (ffi-fn libc "fopen" [String String] VoidP)) (def atan2 (ffi-fn libc "atan2" [Float Float] Float)) (def floor (ffi-fn libc "floor" [Float] Float)) - (def strstr (ffi-fn libc "strstr" [String String] String)) - (def strlen (ffi-fn libc "strlen" [String] Integer)) - (def printf (ffi-fn libc "printf" [String String] Integer)) (def reset! -reset!) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 702f712c..3f3e9c7c 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -102,20 +102,19 @@ def set_native_value(ptr, val, tp): if tp is Integer._type: pnt = rffi.cast(rffi.LONGP, ptr) pnt[0] = rffi.cast(rffi.LONG, val.int_val()) - return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.LONG))) + return rffi.ptradd(rffi.cast(pnt, rffi.CCHARP), rffi.sizeof(rffi.LONG)) if tp is Float._type: pnt = rffi.cast(rffi.DOUBLEP, ptr) pnt[0] = rffi.cast(rffi.DOUBLE, val.float_val()) - return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.DOUBLE))) + return rffi.ptradd(rffi.cast(pnt, rffi.CCHARP), rffi.sizeof(rffi.DOUBLE)) if tp is String._type: pnt = rffi.cast(rffi.CCHARPP, ptr) pnt[0] = rffi.str2charp(str(rt.name(val))) - print "sizeof", rffi.sizeof(rffi.CCHARP) return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) if tp is FFIVoidP._type: pnt = rffi.cast(rffi.VOIDPP, ptr) pnt[0] = val.voidp_data() - return rffi.cast(rffi.CCHARP, rffi.ptradd(pnt, rffi.sizeof(rffi.VOIDP))) + return rffi.ptradd(rffi.cast(pnt, rffi.CCHARP), rffi.sizeof(rffi.VOIDP)) assert False class FFIFn(object.Object): @@ -172,8 +171,6 @@ def thaw(self): self._arg0_offset = arg0_offset self._ret_offset = ret_offset - print exchange_buffer_size, arg0_offset, ret_offset - self._is_inited = True return self @@ -192,7 +189,6 @@ def prep_exb(self, args): offset_p = rffi.ptradd(exb, self._arg0_offset) for x in range(len(self._arg_types)): - print offset_p offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) return exb @@ -205,9 +201,7 @@ def get_ret_val_from_buffer(self, exb): def _invoke(self, args): exb = self.prep_exb(args) - print "calling" jit_ffi_call(self._cd, self._f_ptr, exb) - print "done" ret_val = self.get_ret_val_from_buffer(exb) lltype.free(exb, flavor="raw") return ret_val From ddd79c8280956b82c7faf59286eef785ebbdbcd0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 10 Dec 2014 16:31:08 -0700 Subject: [PATCH 373/909] Get the arg order right --- pixie/vm/libs/ffi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 3f3e9c7c..dfd2fffe 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -102,11 +102,11 @@ def set_native_value(ptr, val, tp): if tp is Integer._type: pnt = rffi.cast(rffi.LONGP, ptr) pnt[0] = rffi.cast(rffi.LONG, val.int_val()) - return rffi.ptradd(rffi.cast(pnt, rffi.CCHARP), rffi.sizeof(rffi.LONG)) + return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.LONG)) if tp is Float._type: pnt = rffi.cast(rffi.DOUBLEP, ptr) pnt[0] = rffi.cast(rffi.DOUBLE, val.float_val()) - return rffi.ptradd(rffi.cast(pnt, rffi.CCHARP), rffi.sizeof(rffi.DOUBLE)) + return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.DOUBLE)) if tp is String._type: pnt = rffi.cast(rffi.CCHARPP, ptr) pnt[0] = rffi.str2charp(str(rt.name(val))) @@ -114,7 +114,7 @@ def set_native_value(ptr, val, tp): if tp is FFIVoidP._type: pnt = rffi.cast(rffi.VOIDPP, ptr) pnt[0] = val.voidp_data() - return rffi.ptradd(rffi.cast(pnt, rffi.CCHARP), rffi.sizeof(rffi.VOIDP)) + return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.VOIDP)) assert False class FFIFn(object.Object): From 05406e2af63a56e5facf43304ac1eb101fde1787 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 11 Dec 2014 06:39:38 -0700 Subject: [PATCH 374/909] added buffer, and tests for ffi --- pixie/stdlib.pxi | 2 ++ pixie/vm/libs/ffi.py | 58 +++++++++++++++++++++++++++++++++++++++++++- tests/test-ffi.pxi | 10 ++++++++ 3 files changed, 69 insertions(+), 1 deletion(-) create mode 100644 tests/test-ffi.pxi diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ee962d9f..3cadb2b8 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -12,9 +12,11 @@ (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) (def fopen (ffi-fn libc "fopen" [String String] VoidP)) + (def fread (ffi-fn libc "fread" [Buffer Integer Integer VoidP] Integer)) (def atan2 (ffi-fn libc "atan2" [Float Float] Float)) (def floor (ffi-fn libc "floor" [Float] Float)) + (def reset! -reset!) (def program-arguments []) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index dfd2fffe..a7e57929 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -1,7 +1,8 @@ import rpython.rlib.rdynload as dynload import pixie.vm.object as object import pixie.vm.code as code -from pixie.vm.code import as_var, affirm +import pixie.vm.stdlib as proto +from pixie.vm.code import as_var, affirm, extend import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype from pixie.vm.primitives import nil @@ -10,6 +11,7 @@ from rpython.rlib import clibffi from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION import rpython.rlib.jit as jit +from rpython.rlib.rarithmetic import intmask """ @@ -71,6 +73,8 @@ def get_native_size(tp): return rffi.sizeof(rffi.DOUBLE) if tp == String._type: return rffi.sizeof(rffi.CCHARP) + if tp == Buffer._type: + return rffi.sizeof(rffi.CCHARP) if tp == FFIVoidP._type: return rffi.sizeof(rffi.VOIDP) assert False @@ -111,6 +115,10 @@ def set_native_value(ptr, val, tp): pnt = rffi.cast(rffi.CCHARPP, ptr) pnt[0] = rffi.str2charp(str(rt.name(val))) return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) + if tp is Buffer._type: + pnt = rffi.cast(rffi.CCHARPP, ptr) + pnt[0] = val.buffer() + return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) if tp is FFIVoidP._type: pnt = rffi.cast(rffi.VOIDPP, ptr) pnt[0] = val.voidp_data() @@ -219,6 +227,8 @@ def get_clibffi_type(arg): return clibffi.cast_type_to_ffitype(rffi.DOUBLE) if arg == String._type: return clibffi.ffi_type_pointer + if arg == Buffer._type: + return clibffi.ffi_type_pointer if arg == FFIVoidP._type: return clibffi.ffi_type_pointer assert False @@ -260,3 +270,49 @@ def voidp_data(self): +class Buffer(object.Object): + """ Defines a byte buffer with non-gc'd (therefore non-movable) contents + """ + _type = object.Type(u"pixie.stdlib.Buffer") + + def type(self): + return Buffer._type + + def __init__(self, size): + self._size = size + self._used_size = 0 + self._buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor="raw") + + + def __del__(self): + lltype.free(self._buffer, flavor="raw") + + def set_used_size(self, size): + self._used_size = size + + def buffer(self): + return self._buffer + + def count(self): + return self._used_size + + def nth(self, idx): + return self._buffer[idx] + + +@extend(proto._nth, Buffer) +def _nth(self, idx): + return rt.wrap(ord(self.nth(idx.int_val()))) + +@extend(proto._count, Buffer) +def _count(self): + return self.count() + +@as_var("buffer") +def buffer(size): + return Buffer(size.int_val()) + +@as_var("set-buffer-count") +def set_buffer_size(self, size): + self.set_used_size(size.int_val()) + return self diff --git a/tests/test-ffi.pxi b/tests/test-ffi.pxi new file mode 100644 index 00000000..a99dd211 --- /dev/null +++ b/tests/test-ffi.pxi @@ -0,0 +1,10 @@ +(ns pixie.tests.test-ffi + (require pixie.test :as t)) + + + +(t/deftest test-buffer-ffi + (let [fp (fopen "README.md" "r") + b (buffer 1024)] + (t/assert= 10 (fread b 1 10 fp)) + (t/assert= 91 (nth b 0)))) From 2eee6054dde5e53ee2890ad59cf78342d35aa007 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 11 Dec 2014 06:46:43 -0700 Subject: [PATCH 375/909] fix a translation error --- pixie/vm/libs/ffi.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index a7e57929..f427d069 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -296,17 +296,17 @@ def buffer(self): def count(self): return self._used_size - def nth(self, idx): + def nth_char(self, idx): return self._buffer[idx] @extend(proto._nth, Buffer) def _nth(self, idx): - return rt.wrap(ord(self.nth(idx.int_val()))) + return rt.wrap(ord(self.nth_char(idx.int_val()))) @extend(proto._count, Buffer) def _count(self): - return self.count() + return rt.wrap(self.count()) @as_var("buffer") def buffer(size): From 2fc41c1cac4e5c0d496bdd4c828e72d7eae37652 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 11 Dec 2014 18:24:53 +0100 Subject: [PATCH 376/909] implement numerator and denominator --- pixie/vm/numbers.py | 11 +++++++++++ tests/test-numbers.pxi | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 350d23bb..3afe9505 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -1,4 +1,5 @@ import pixie.vm.object as object +from pixie.vm.object import affirm from pixie.vm.primitives import nil, true, false from rpython.rlib.rarithmetic import r_uint from rpython.rlib.rbigint import rbigint @@ -340,3 +341,13 @@ def _repr(r): @extend(proto._str, Ratio._type) def _str(r): return rt.wrap(unicode(str(r.numerator()) + "/" + str(r.denominator()))) + + @as_var("numerator") + def numerator(r): + affirm(isinstance(r, Ratio), u"First argument must be a Ratio") + return rt.wrap(r.numerator()) + + @as_var("denominator") + def denominator(r): + affirm(isinstance(r, Ratio), u"First argument must be a Ratio") + return rt.wrap(r.denominator()) diff --git a/tests/test-numbers.pxi b/tests/test-numbers.pxi index ae93d7c1..a318224b 100644 --- a/tests/test-numbers.pxi +++ b/tests/test-numbers.pxi @@ -36,3 +36,8 @@ (t/assert= (- 1/2 1/2) 0) (t/assert= (* 1/2 1/2) 1/4) (t/assert= (/ 1/2 1/2) 1)) + +(t/deftest ratio-accessors + (doseq [[r n d] [[3/2 3 2] [1/9 1 9] [-3/89 -3 89]]] + (t/assert= (numerator r) n) + (t/assert= (denominator r) d))) From ddbbe4d3bc19d599b1aefef25e3389e6e68ddb6d Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 11 Dec 2014 18:29:25 +0100 Subject: [PATCH 377/909] fix math ffi on linux On Linux, libm (and other libraries) are separate, so we have to use different libraries. One of these is libm, which contains things like `floor`, `atan2` and quite a few other things. There should be symlinks from `libm` et al to `libSystem` on Mac [1], so it should still work there. [1]: http://www.finkproject.org/doc/porting/porting.en.html#basics.libraries --- pixie/stdlib.pxi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ee962d9f..bdfa4d1b 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -12,8 +12,10 @@ (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) (def fopen (ffi-fn libc "fopen" [String String] VoidP)) - (def atan2 (ffi-fn libc "atan2" [Float Float] Float)) - (def floor (ffi-fn libc "floor" [Float] Float)) + + (def libm (ffi-library (str "libm." pixie.platform/so-ext))) + (def atan2 (ffi-fn libm "atan2" [Float Float] Float)) + (def floor (ffi-fn libm "floor" [Float] Float)) (def reset! -reset!) From b69f70b0a3e63a00dfecf4cc40512ab5d518579b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Thu, 11 Dec 2014 18:32:29 +0100 Subject: [PATCH 378/909] implement `int` and `float` --- pixie/stdlib.pxi | 19 +++++++++++++++++++ tests/test-numbers.pxi | 8 ++++++++ 2 files changed, 27 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index bdfa4d1b..1d5d21ad 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -16,6 +16,7 @@ (def libm (ffi-library (str "libm." pixie.platform/so-ext))) (def atan2 (ffi-fn libm "atan2" [Float Float] Float)) (def floor (ffi-fn libm "floor" [Float] Float)) + (def lround (ffi-fn libm "lround" [Float] Integer)) (def reset! -reset!) @@ -730,6 +731,24 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn indexed? [v] (satisfies? IIndexed v)) (defn counted? [v] (satisfies? ICounted v)) +(defn float + {:doc "Converts a number to a float." + :since "0.1"} + [x] + (cond + (number? x) (+ x 0.0) + :else (throw (str "Can't convert a value of type " (type x) " to a Float")))) + +(defn int + {:doc "Converts a number to an integer." + :since "0.1"} + [x] + (cond + (integer? x) x + (float? x) (lround (floor x)) + (ratio? x) (int (/ (float (numerator x)) (float (denominator x)))) + :else (throw (str "Can't convert a value of type " (type x) " to an Integer")))) + (defn last {:doc "Returns the last element of the collection, or nil if none." :signatures [[coll]] diff --git a/tests/test-numbers.pxi b/tests/test-numbers.pxi index a318224b..1d88d9d0 100644 --- a/tests/test-numbers.pxi +++ b/tests/test-numbers.pxi @@ -41,3 +41,11 @@ (doseq [[r n d] [[3/2 3 2] [1/9 1 9] [-3/89 -3 89]]] (t/assert= (numerator r) n) (t/assert= (denominator r) d))) + +(t/deftest test-int + (doseq [[x i] [[1 1] [3.0 3] [3.5 3] [3.999 3] [3/2 1]]] + (t/assert= (int x) i))) + +(t/deftest test-float + (doseq [[x f] [[1 1.0] [3 3.0] [3.333 3.333] [3/2 1.5] [1/7 (/ 1.0 7.0)]]] + (t/assert= (float x) f))) From bf923de96dfedd6b8c9ef541b79e051a3584f747 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 09:30:25 -0700 Subject: [PATCH 379/909] fixes getting io streams to work --- pixie/stdlib.pxi | 15 +++++++++++---- pixie/test.pxi | 1 + pixie/vm/code.py | 3 ++- pixie/vm/compiler.py | 3 ++- pixie/vm/interpreter.py | 5 +++++ pixie/vm/libs/ffi.py | 9 ++++++++- pixie/vm/object.py | 3 +++ pixie/vm/rt.py | 2 -- pixie/vm/stdlib.py | 7 +++++-- run-tests.pxi | 6 +++++- 10 files changed, 42 insertions(+), 12 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e7e8dd98..1eb36c7b 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -235,6 +235,7 @@ (extend -hash Nil (fn [self] 100000)) (extend -with-meta Nil (fn [self _] nil)) (extend -at-end? Nil (fn [_] true)) +(extend -deref Nil (fn [_] nil)) (extend -hash Integer hash-int) @@ -774,7 +775,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn complement {:doc "Given a function, return a new function which takes the same arguments but returns the opposite truth value"} - [f] + [f] (if (not (fn? f)) (throw "Complement must be passed a function") (fn @@ -974,11 +975,11 @@ Creates new maps if the keys are not present." nil (throw (str "Assert failed " ~msg))))) -(defmacro resolve +(defn resolve {:doc "Resolve the var associated with the symbol in the current namespace." :added "0.1"} [sym] - `(resolve-in (this-ns-name) ~sym)) + (resolve-in (this-ns-name) sym)) (defmacro binding [bindings & body] (let [bindings (apply hashmap bindings) @@ -1298,7 +1299,7 @@ The new value is thus `(apply f current-value-of-atom args)`." ([] true) ([x] x) ([x y] `(if ~x ~y false)) - ([x y & more] `(if ~x (and ~y ~@more)))) + ([x y & more] `(if ~x (and ~y ~@more) false))) (defmacro or {:doc "Returns the value of the first expression that returns a truthy value, or false." @@ -1900,3 +1901,9 @@ Expands to calls to `extend-type`." `(do (load-ns ~ns) (refer ~ns :refer :all))) + +(defn count-rf + "A Reducing function that counts the items reduced over" + ([] 0) + ([result] result) + ([result _] (inc result))) diff --git a/pixie/test.pxi b/pixie/test.pxi index e529f873..f482c64b 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -30,6 +30,7 @@ (set! (var *stats*) (atom {:fail 0 :pass 0})) (let [match (or (first args) "") + _ (println (keys @tests)) tests (transduce (comp (filter #(>= (s/index-of (str (key %1)) match) 0)) (map val)) conj diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 87906e91..235092ef 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -30,7 +30,8 @@ "POP_UP_N", "MAKE_MULTI_ARITY", "MAKE_VARIADIC", - "YIELD"] + "YIELD", + "PUSH_NS"] for x in range(len(BYTECODES)): globals()[BYTECODES[x]] = r_uint(x) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index dd5c40a8..4a03bcaf 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -729,7 +729,8 @@ def compile_ns(form, ctx): ctx.push_const(nil) def compile_this_ns(form, ctx): - ctx.push_const(NS_VAR.deref()) + ctx.bytecode.append(code.PUSH_NS) + ctx.add_sp(1) def compile_var(form, ctx): form = rt.next(form) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index bfee5760..137f6a85 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -372,6 +372,11 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): else: return frame.pop() + if inst == code.PUSH_NS: + from pixie.vm.compiler import NS_VAR + frame.push(NS_VAR.deref()) + continue + affirm(False, u"NO DISPATCH FOR: " + unicode(code.BYTECODES[inst])) raise Exception() diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index f427d069..def084bb 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -299,6 +299,9 @@ def count(self): def nth_char(self, idx): return self._buffer[idx] + def capacity(self): + return self._size + @extend(proto._nth, Buffer) def _nth(self, idx): @@ -312,7 +315,11 @@ def _count(self): def buffer(size): return Buffer(size.int_val()) -@as_var("set-buffer-count") +@as_var("buffer-capacity") +def buffer_capacity(buffer): + return rt.wrap(buffer.capacity()) + +@as_var("set-buffer-count!") def set_buffer_size(self, size): self.set_used_size(size.int_val()) return self diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 07accbca..14ffdfe0 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -68,6 +68,9 @@ def __init__(self, name, parent = None): self._subclasses = [] + def name(self): + return self._name + def type(self): return Type._type diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index c64e5f9a..af592704 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -111,7 +111,6 @@ def int_val(x): for name, var in _ns_registry._registry[u"pixie.stdlib"]._registry.iteritems(): name = munge(name) - print name if var.is_defined() and isinstance(var.deref(), BaseCode): globals()[name] = unwrap(var) else: @@ -126,7 +125,6 @@ def reinit(): if name in globals(): continue - print "Found ->> ", name, var.deref() if var.is_defined() and isinstance(var.deref(), BaseCode): globals()[name] = unwrap(var) else: diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 552e4278..90432fb6 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,5 +1,6 @@ # -*- coding: utf-8 -*- -from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance +from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ + runtime_error from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ resize_list, list_copy, returns, get_var_if_defined, intern_var import pixie.vm.code as code @@ -467,7 +468,9 @@ def refer_symbol(ns, sym, var): @as_var("extend") def _extend(proto_fn, tp, fn): - affirm(isinstance(proto_fn, PolymorphicFn), u"First argument to extend should be a PolymorphicFn") + if not isinstance(proto_fn, PolymorphicFn): + runtime_error(u"Fist argument to extend should be a PolymorphicFn not a " + proto_fn.type().name()) + affirm(isinstance(tp, Type) or isinstance(tp, Protocol), u"Second argument to extend must be a Type or Protocol") affirm(isinstance(fn, BaseCode), u"Last argument to extend must be a function") proto_fn.extend(tp, fn) diff --git a/run-tests.pxi b/run-tests.pxi index a3b7a864..f71acc1b 100644 --- a/run-tests.pxi +++ b/run-tests.pxi @@ -2,7 +2,11 @@ (println @load-paths) -(t/load-all-tests) +(if (= 0 (count program-arguments)) + (t/load-all-tests) + (doseq [nm program-arguments] + (println "Loading: " nm) + (load-ns (symbol nm)))) (let [result (apply t/run-tests program-arguments)] (exit (get result :fail))) From 6e210b88cd405ff480b6b4eec8e41a46a9612bf2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 09:34:06 -0700 Subject: [PATCH 380/909] move tests into a more sane directory structure. Now we don't have to load all tests just to run one ns --- run-tests.pxi | 3 + tests/collections/test-maps.pxi | 55 ------ tests/collections/test-seqables.pxi | 45 ----- tests/collections/test-sets.pxi | 71 -------- tests/collections/test-vectors.pxi | 45 ----- tests/test-arrays.pxi | 51 ------ tests/test-bits.pxi | 59 ------- tests/test-compiler.pxi | 22 --- tests/test-defrecord.pxi | 50 ------ tests/test-deftype.pxi | 85 --------- tests/test-destructuring.pxi | 76 -------- tests/test-docs.pxi | 24 --- tests/test-ffi.pxi | 10 -- tests/test-fns.pxi | 15 -- tests/test-forms.pxi | 90 ---------- tests/test-keywords.pxi | 10 -- tests/test-numbers.pxi | 51 ------ tests/test-readeval.pxi | 17 -- tests/test-stdlib.pxi | 257 ---------------------------- tests/test-strings.pxi | 126 -------------- tests/utils.pxi | 13 -- 21 files changed, 3 insertions(+), 1172 deletions(-) delete mode 100644 tests/collections/test-maps.pxi delete mode 100644 tests/collections/test-seqables.pxi delete mode 100644 tests/collections/test-sets.pxi delete mode 100644 tests/collections/test-vectors.pxi delete mode 100644 tests/test-arrays.pxi delete mode 100644 tests/test-bits.pxi delete mode 100644 tests/test-compiler.pxi delete mode 100644 tests/test-defrecord.pxi delete mode 100644 tests/test-deftype.pxi delete mode 100644 tests/test-destructuring.pxi delete mode 100644 tests/test-docs.pxi delete mode 100644 tests/test-ffi.pxi delete mode 100644 tests/test-fns.pxi delete mode 100644 tests/test-forms.pxi delete mode 100644 tests/test-keywords.pxi delete mode 100644 tests/test-numbers.pxi delete mode 100644 tests/test-readeval.pxi delete mode 100644 tests/test-stdlib.pxi delete mode 100644 tests/test-strings.pxi delete mode 100644 tests/utils.pxi diff --git a/run-tests.pxi b/run-tests.pxi index f71acc1b..d2c373aa 100644 --- a/run-tests.pxi +++ b/run-tests.pxi @@ -1,5 +1,8 @@ (require pixie.test :as t) + +(swap! load-paths conj "./tests/") + (println @load-paths) (if (= 0 (count program-arguments)) diff --git a/tests/collections/test-maps.pxi b/tests/collections/test-maps.pxi deleted file mode 100644 index a4072fe1..00000000 --- a/tests/collections/test-maps.pxi +++ /dev/null @@ -1,55 +0,0 @@ -(ns collections.test-maps - (require pixie.test :as t)) - -(t/deftest maps-contains - (let [m {:a 1, :b 2, :c 3} - c [:a :b :c] - n [:d 'a 1]] - (foreach [c c] - (t/assert= (contains? m c) true)) - (foreach [n n] - (t/assert= (contains? m c) false)))) - -(t/deftest map-equals - (let [m {:a 1, :b 2, :c 3}] - (t/assert= (-eq {} {}) true) - (t/assert= (-eq m m) true) - (t/assert= (-eq m {:a 1, :b 2, :c 3}) true) - - (t/assert= (-eq m {}) false) - (t/assert= (-eq m nil) false) - - (t/assert= (-eq m {:a 1, :b 2}) false) - (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false) - - (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) - (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) - - -(t/deftest map-val-at-and-invoke - (let [m {:a 1, :b 2, :c 3}] - (foreach [e m] - (t/assert= (get m (key e)) (val e)) - (t/assert= (m (key e)) (val e))) - (t/assert= (get m :d) nil) - (t/assert= (m :d) nil))) - -(t/deftest map-without - (let [m {:a 1 :b 2}] - (t/assert= m m) - (t/assert= (dissoc m :a) {:b 2}) - (t/assert= (dissoc m :a :b) {}))) - -(t/deftest map-conj - (let [m {:a 1 :b 2}] - (t/assert= m m) - ;; Should conj vector of length 2 - (t/assert= (conj m [:c 3]) {:a 1 :b 2 :c 3}) - (t/assert= (conj m [:b 4]) {:a 1 :b 4}) - (t/assert= (conj m [:b 4] [:c 5]) {:a 1 :b 4 :c 5}) - - ;; Should conj sequences of pairs - (t/assert= (conj {} '([:a 1] [:b 2] [:c 3])) {:a 1 :b 2 :c 3}) - - ;; Should conj sequences of MapEntries - (t/assert= (conj {} (seq {:a 1 :b 2 :c 3})) {:a 1 :b 2 :c 3}))) diff --git a/tests/collections/test-seqables.pxi b/tests/collections/test-seqables.pxi deleted file mode 100644 index dcf79354..00000000 --- a/tests/collections/test-seqables.pxi +++ /dev/null @@ -1,45 +0,0 @@ -(ns collections.test-seqables - (require pixie.test :as t)) - -(t/deftest test-seq - (let [l '(1 2 3) - v [1 2 3]] - (t/assert= (seq l) '(1 2 3)) - (t/assert= (seq v) [1 2 3]) - - (t/assert= (seq nil) nil) - (t/assert= (seq []) nil))) - -(t/deftest test-first - (let [l '(1 2 3) - v [1 2 3]] - (t/assert= (first l) 1) - (t/assert= (first v) 1) - (t/assert= (first (seq l)) 1) - (t/assert= (first (seq v)) 1))) - -(t/deftest test-next - (let [l '(1 2 3) - v [1 2 3]] - (t/assert= (next l) '(2 3)) - (t/assert= (next v) '(2 3)) - (t/assert= (next (seq l)) '(2 3)) - (t/assert= (next (seq v)) '(2 3)) - - (t/assert= (next (next (next l))) nil) - (t/assert= (next (next (next v))) nil))) - -(t/deftest test-equals - (let [l '(1 2 3)] - (t/assert= l l) - (t/assert= l '(1 2 3)) - (t/assert= l [1 2 3]) - - (t/assert= (= nil '()) false) - (t/assert= (= '() nil) false) - (t/assert= (= l '(1 2 3 4)) false) - (t/assert= (= l [1 2 3 4]) false))) - -(t/deftest test-conj - (t/assert= '(3 1 2) (conj '(1 2) 3)) - (t/assert= '(5 4 3 1 2) (conj '(1 2) 3 4 5))) diff --git a/tests/collections/test-sets.pxi b/tests/collections/test-sets.pxi deleted file mode 100644 index 62652da8..00000000 --- a/tests/collections/test-sets.pxi +++ /dev/null @@ -1,71 +0,0 @@ -(ns collections.test-sets - (require pixie.test :as t) - (require tests.utils :as u)) - -(def worst-hashers (vec (map u/->WorstHasher) - (range 100))) - -(t/deftest test-count - (t/assert= (count (set [])) 0) - (t/assert= (count (set [1 2 3])) 3) - (t/assert= (count (set [1 1 2 1])) 2) - (t/assert= (count (set worst-hashers)) 100)) - -(t/deftest test-contains - (let [s #{1 2 3} - c [1 2 3] - n [-1 0 4] - g (set worst-hashers)] - (foreach [c c] - (t/assert= (contains? s c) true)) - (foreach [n n] - (t/assert= (contains? s n) false)) - (foreach [n worst-hashers] - (t/assert= (contains? g n) true)))) - -(t/deftest test-conj - (t/assert= (conj #{}) #{}) - (t/assert= (conj #{1 2} 3) #{1 2 3}) - (t/assert= (reduce conj #{} (range 10)) (set (vec (range 10)))) - (t/assert= (reduce conj #{} worst-hashers) (set worst-hashers))) - -(t/deftest test-disj - (t/assert= (disj #{}) #{}) - (t/assert= (disj #{1 2} 3) #{1 2}) - (t/assert= (disj #{1 2} 2) #{1}) - (t/assert= (reduce disj (set (vec (range 10))) (range 10)) #{}) - (t/assert= (reduce disj (set worst-hashers) worst-hashers) #{})) - -(t/deftest test-eq - (let [s #{1 2 3}] - (t/assert= s s) - (t/assert= s #{1 2 3}) - (t/assert= #{1 2 3} s) - - (t/assert= (= #{} nil) false) - (t/assert= (= #{} []) false) - (t/assert= (= #{} '()) false) - - (t/assert= (= s [1 2 3]) false) - (t/assert= (= s '(1 2 3)) false) - (t/assert= (= s #{1 2}) false) - (t/assert= (= s #{1 2 3 4}) false))) - -(t/deftest test-invoke - (let [s #{1 2 3}] - (t/assert= (s 1) 1) - (t/assert= (s 2) 2) - (t/assert= (s 3) 3) - - (t/assert= (s -1) nil) - (t/assert= (s 4) nil))) - -(t/deftest test-has-meta - (let [m {:has-meta true} - s (with-meta #{} m)] - (t/assert= (meta #{}) nil) - (t/assert= (meta s) m))) - -(t/deftest test-conj - (t/assert= #{1 2} (conj #{1} 2)) - (t/assert= #{1 2 3 4} (conj #{1} 2 3 4))) diff --git a/tests/collections/test-vectors.pxi b/tests/collections/test-vectors.pxi deleted file mode 100644 index f69432fe..00000000 --- a/tests/collections/test-vectors.pxi +++ /dev/null @@ -1,45 +0,0 @@ -(ns collections.test-vectors - (require pixie.test :as t)) - -(def MAX-SIZE 1064) - -(comment -;; Takes forever in interpreted mode but useful for debugging -(t/deftest vector-creation - (loop [acc []] - (if (= (count acc) MAX-SIZE) - acc - (do (dotimes [j (count acc)] - (t/assert= j (nth acc j))) - (recur (conj acc (count acc))))))) -) -(t/deftest vector-contains - (let [v [1 2 3] - c [0 1 2] - n [-1 3]] - (foreach [c c] - (t/assert= (contains? v c) true)) - (foreach [n n] - (t/assert= (contains? v n) false)))) - -(t/deftest vector-equals - (let [v [1 2 3]] - (t/assert= [] '()) - (t/assert= v v) - (t/assert= v [1 2 3]) - (t/assert= v '(1 2 3)) - - (t/assert= (= [] nil) false) - (t/assert= (= v []) false) - (t/assert= (= v [1 2]) false) - (t/assert= (= v [1 2 3 4]) false) - (t/assert= (= v '(1 2)) false) - (t/assert= (= v '(1 2 3 4)) false))) - -(t/deftest vector-conj - (t/assert= [1 2] (conj [1] 2)) - (t/assert= [1 2 3 4] (conj [1] 2 3 4))) - -(t/deftest vector-conj! - (t/assert= [1 2] (persistent! (conj! (transient [1]) 2))) - (t/assert= [1 2 3] (persistent! (conj! (transient [1]) 2 3)))) diff --git a/tests/test-arrays.pxi b/tests/test-arrays.pxi deleted file mode 100644 index fbd55937..00000000 --- a/tests/test-arrays.pxi +++ /dev/null @@ -1,51 +0,0 @@ -(ns pixie.test.test-arrays - (require pixie.test :as t)) - -(t/deftest test-array-creation - (let [a (make-array 10)] - (t/assert= (count a) 10) - (t/assert= (alength a) 10) - (foreach [x a] - (t/assert= x nil)))) - -(t/deftest test-aget-and-aset - (let [a (make-array 10)] - (dotimes [i 10] - (t/assert= (aget a i) nil)) - - (dotimes [i 10] - (aset a i i)) - - (dotimes [i 10] - (t/assert= (aget a i) i)))) - -(t/deftest test-aconcat - (let [a1 (make-array 10) - a2 (make-array 10)] - (t/assert= (alength (aconcat a1 a2)) (+ (alength a1) (alength a2))) - - (dotimes [i 10] - (aset a1 i i) - (aset a2 i (+ 10 i))) - - (let [a3 (aconcat a1 a2)] - (dotimes [i 20] - (t/assert= (aget a3 i) i))))) - -(t/deftest test-aslice - (let [a (make-array 10)] - (dotimes [i 10] - (aset a i i)) - - (let [a1 (aslice a 3) - a2 (aslice a 7)] - (foreach [i (range 0 7)] - (t/assert= (aget a1 i) (+ i 3))) - (foreach [i (range 0 3)] - (t/assert= (aget a2 i) (+ i 7)))))) - - -(t/deftest test-byte-array-creation - (let [ba (byte-array 10)] - (t/assert= (vec ba) [0 0 0 0 0 0 0 0 0 0]) - (t/assert= (count ba) 10))) \ No newline at end of file diff --git a/tests/test-bits.pxi b/tests/test-bits.pxi deleted file mode 100644 index be044f9d..00000000 --- a/tests/test-bits.pxi +++ /dev/null @@ -1,59 +0,0 @@ -(ns pixie.test.test-bits - (require pixie.test :as t)) - -(t/deftest test-bit-clear - (t/assert= (bit-clear 0 0) 0) - (t/assert= (bit-clear 2r11111 7) 2r11111) - - (t/assert= (bit-clear 2r111 0) 2r110) - (t/assert= (bit-clear 2r111 1) 2r101) - (t/assert= (bit-clear 2r111 2) 2r011)) - -(t/deftest test-bit-set - (t/assert= (bit-set 2r111 0) 2r111) - (t/assert= (bit-set 2r000 1) 2r010)) - -(t/deftest test-bit-flip - (t/assert= (bit-flip 2r101 0) 2r100) - (t/assert= (bit-flip 2r101 1) 2r111)) - -(t/deftest test-bit-test - (t/assert= (bit-test 2r101 0) true) - (t/assert= (bit-test 2r101 1) false)) - -(t/deftest test-bit-and - (t/assert= (bit-and 0 0) 0) - (t/assert= (bit-and 2r101 2r101) 2r101) - (t/assert= (bit-and 2r101 2r101) 2r101) - (t/assert= (bit-and 2r101 0) 0)) - -(t/deftest test-bit-or - (t/assert= (bit-or 0 0) 0) - (t/assert= (bit-or 2r101 2r010) 2r111) - (t/assert= (bit-or 2r111 2r010) 2r111) - (t/assert= (bit-or 2r111 2r111) 2r111) - (t/assert= (bit-or 2r101 0) 2r101)) - -(t/deftest test-bit-xor - (t/assert= (bit-xor 0 0) 0) - (t/assert= (bit-xor 2r101 2r010) 2r111) - (t/assert= (bit-xor 2r111 2r010) 2r101) - (t/assert= (bit-xor 2r111 2r111) 2r000) - (t/assert= (bit-xor 2r101 0) 2r101)) - -(t/deftest test-bit-shift-left - (t/assert= (bit-shift-left 0 0) 0) - (t/assert= (bit-shift-left 2r101 0) 2r101) - - (t/assert= (bit-shift-left 0 7) 0) - (t/assert= (bit-shift-left 2r001 2) 2r100) - (t/assert= (bit-shift-left 2r111 2) 2r11100)) - -(t/deftest test-bit-shift-right - (t/assert= (bit-shift-right 0 0) 0) - (t/assert= (bit-shift-right 2r101 0) 2r101) - - (t/assert= (bit-shift-right 0 7) 0) - (t/assert= (bit-shift-right 2r001 2) 0) - (t/assert= (bit-shift-right 2r111 2) 2r001) - (t/assert= (bit-shift-right 2r1011010 2) 2r10110)) diff --git a/tests/test-compiler.pxi b/tests/test-compiler.pxi deleted file mode 100644 index 052bafe2..00000000 --- a/tests/test-compiler.pxi +++ /dev/null @@ -1,22 +0,0 @@ -(ns pixie.test.test-compiler - (require pixie.test :as t)) - -(t/deftest test-do - (t/assert= (do 1) 1) - (t/assert= (do 1 2) 2) - (t/assert= (do) nil) - (t/assert= (do 1 2 3 4 5 6) 6)) - -(t/deftest test-if - (t/assert= (if true 42 nil) 42) - (t/assert= (if false 42 nil) nil) - (t/assert= (if false 42) nil)) - -(t/deftest test-let - (t/assert= (let [] 1) 1) - (t/assert= (let [x 1]) nil) - (t/assert= (let []) nil)) - -(t/deftest test-lists - (t/assert= (vec '()) []) - (t/assert= (vec '()) ())) diff --git a/tests/test-defrecord.pxi b/tests/test-defrecord.pxi deleted file mode 100644 index 382247d1..00000000 --- a/tests/test-defrecord.pxi +++ /dev/null @@ -1,50 +0,0 @@ -(ns pixie.test.test-defrecord - (require pixie.test :as t)) - -(defrecord Three [one two three]) - -(def t1 (->Three 1 2 3)) -(def t2 (->Three 1 2 3)) -(def t3 (map->Three {:one 1, :two 2, :three 3})) -(def t4 (->Three 1 2 4)) -(def t5 (->Three 3 4 5)) - -(t/deftest test-satisfies - (foreach [t [t1 t2 t3 t4 t5]] - (t/assert= t t) - (t/assert (satisfies? IAssociative t)) - (t/assert (satisfies? ILookup t)))) - -(t/deftest test-eq - (t/assert= t1 t2) - (t/assert= t2 t3) - (t/assert= t3 t1) - (foreach [t [t1 t2 t3]] - (t/assert (not (= t (assoc t :one 42)))) - (t/assert (not (= t t4))) - (t/assert (not (= t t5))))) - -(t/deftest test-ilookup - (foreach [t [t1 t2 t3]] - (t/assert (satisfies? ILookup t)) - (t/assert= (get t :one) 1) - (t/assert= (get t :two) 2) - (t/assert= (get t :three) 3) - (t/assert= (get t :oops) nil) - (t/assert= (get t :oops 'not-found) 'not-found))) - -(t/deftest test-iassociative - (foreach [t [t1 t2 t3]] - (t/assert (satisfies? IAssociative t)) - (t/assert= t (assoc t4 :three 3)) - (let [t' (assoc t :one 42) - t-oops (assoc t :oops 'never-found)] - (t/assert (not (= t t'))) - (t/assert= (get t' :one) 42) - (t/assert= t t-oops) - (t/assert (not (contains? t-oops :oops))) - (t/assert= (get t-oops :oops) nil)) - - (t/assert (contains? t :one)) - (t/assert (contains? t :two)) - (t/assert (contains? t :three)))) diff --git a/tests/test-deftype.pxi b/tests/test-deftype.pxi deleted file mode 100644 index 49f5e444..00000000 --- a/tests/test-deftype.pxi +++ /dev/null @@ -1,85 +0,0 @@ -(ns pixie.test.test-deftype - (require pixie.test :as t)) - -(deftype Simple [:val]) -(deftype Simple2 [val]) - -(t/deftest test-simple - (let [o1 (->Simple 1) - o2 (->Simple 2)] - (foreach [obj-and-val [[o1 1] [o2 2]]] - (let [o (first obj-and-val) - v (second obj-and-val)] - (t/assert= (. o :val) v) - (t/assert= (.val o) v))))) - -(deftype MagicalVectorMap [] IMap IVector) - -(t/deftest test-satisfies - (let [mvm (->MagicalVectorMap)] - (t/assert (satisfies? IVector mvm)) - (t/assert (satisfies? IMap mvm)))) - -(deftype Count [:val] - ICounted - (-count [self] val)) - -(deftype Count2 [val] - ICounted - (-count [self] val)) - -(t/deftest test-extend - (let [o1 (->Count 1) - o2 (->Count 2)] - (foreach [obj-and-val [[o1 1] [o2 2]]] - (let [o (first obj-and-val) - v (second obj-and-val)] - (t/assert= (. o :val) v) - (t/assert= (.val o) v) - (t/assert (satisfies? ICounted o)) - (t/assert= (-count o) v) - (t/assert= (count o) v))))) - -(deftype Three [:one :two :three] - Object - (add [self x & args] - (apply + x args)) - (one-plus [self x & xs] - (apply + one x xs)) - ICounted - (-count [self] (+ one two three))) - -(deftype Three2 [one two three] - Object - (add [self x & args] - (apply + x args)) - (one-plus [self x & xs] - (apply + one x xs)) - ICounted - (-count [self] (+ one two three))) - -(t/deftest test-complex - (let [o1 (->Three 1 2 3) - o2 (->Three2 3 4 5)] - (foreach [obj-and-vals [[o1 1 2 3] [o2 3 4 5]]] - (let [o (first obj-and-vals) - one (second obj-and-vals) - two (third obj-and-vals) - three (fourth obj-and-vals)] - (t/assert= (. o :one) one) - (t/assert= (.one o) one) - (t/assert= (. o :two) two) - (t/assert= (.two o) two) - (t/assert= (. o :three) three) - (t/assert= (.three o) three) - - (t/assert (satisfies? ICounted o)) - (t/assert= (-count o) (+ one two three)) - (t/assert= (count o) (+ one two three)) - - (t/assert= (.add o 21 21) 42) - (t/assert= (.one-plus o 9) (+ one 9)) - - ; arity-1 (just the self arg) not supported for now - (t/assert= (.add o) (. o :add)) - (t/assert= (.one-plus o) (. o :one-plus)))))) diff --git a/tests/test-destructuring.pxi b/tests/test-destructuring.pxi deleted file mode 100644 index 13eb93a0..00000000 --- a/tests/test-destructuring.pxi +++ /dev/null @@ -1,76 +0,0 @@ -(ns pixie.test.test-destructuring - (require pixie.test :as t)) - -(t/deftest test-let-simple - (t/assert= (let [x 1 y 2 z 3] [x y z]) [1 2 3]) - (t/assert= (let [x 1 y 2 z 3] [x y z]) (let* [x 1 y 2 z 3] [1 2 3])) - - (t/assert= (let [x 1 x 2] x) 2) - (t/assert= (let [x 1 y 2 x 3] x) 3) - - (t/assert= (let [x 1] (let [x 2] x)) 2)) - -(t/deftest test-let-vector-simple - (t/assert= (let [[x y z] [1 2 3]] [x y z]) [1 2 3]) - (t/assert= (let [[x y z] [1 2 3 4]] [x y z]) [1 2 3]) - - (t/assert= (let [[x y z & rest] [1 2 3 4]] - [x y z rest]) - [1 2 3 '(4)]) - (t/assert= (let [[x y z & rest] [1 2]] - [x y z rest]) - [1 2 nil nil])) - -(t/deftest test-let-vector-nested - (t/assert= (let [[[x y] z & rest] [[1 2] 3 4]] - [x y z rest]) - [1 2 3 '(4)]) - - (t/assert= (let [[[x [y]] z & rest] [[1 [2 3]] 4 5]] - [x y z rest]) - [1 2 4 '(5)])) - -(t/deftest test-let-vector-rest - (t/assert= (let [[x y & [z & rest]] [1 2 3 4 5]] - [x y z rest]) - [1 2 3 '(4 5)])) - -(t/deftest test-let-map - (t/assert= (let [{a :a, b :b, {c :c :as s} :d :as m} {:a 1, :b 2, :d {:c 3}}] - [a b c s m]) - [1 2 3 {:c 3} {:a 1, :b 2, :d {:c 3}}]) - - (t/assert= (let [{:keys [a b c] :as m} {:a 1, :b 2, :c 3, :d 4}] - [a b c (:d m)]) - [1 2 3 4])) - -(t/deftest test-let-map-defaults - (t/assert= (let [{a :a :or {a 42}} {:a 1}] a) 1) - (t/assert= (let [{a :a :or {a 42}} {}] a) 42) - - (t/assert= (let [{a :a :or {a 42}} {:a nil}] a) nil) - (t/assert= (let [{a :a :or {a 42}} {:a false}] a) false) - - (t/assert= (let [{:keys [a], :or {a 42}} {:a 1}] a) 1) - (t/assert= (let [{:keys [a], :or {a 42}} {}] a) 42)) - -(t/deftest test-fn-simple - (t/assert= ((fn [[x y & rest]] [x y rest]) [1 2 3 4 5]) [1 2 '(3 4 5)]) - (t/assert= ((fn [{a :a, b :b :as m}] [a b m]) {:a 1, :b 2, :answer 42}) [1 2 {:a 1, :b 2, :answer 42}]) - - (t/assert= ((fn [[[x y] z & rest]] [x y z rest]) [[1 2] 3 4 5]) [1 2 3 '(4 5)])) - -(t/deftest test-fn-multiple-args - (t/assert= ((fn [[x y z] {:keys [a b c]}] [x y z a b c]) [1 2 3] {:a 4, :b 5, :c 6}) - [1 2 3 4 5 6])) - -(t/deftest test-fn-rest-args - (let [f1 (fn [& [status]] (or status :yay))] - (t/assert= (f1) :yay) - (t/assert= (f1 :nay) :nay) - (t/assert= (f1 :nay :something-else :whatever) :nay)) - (let [f2 (fn [x & [y]] - (+ x (or y 1)))] - (t/assert= (f2 41) 42) - (t/assert= (f2 21 21) 42) - (t/assert= (f2 21 21 :something-else :whatever) 42))) diff --git a/tests/test-docs.pxi b/tests/test-docs.pxi deleted file mode 100644 index dd6b96cf..00000000 --- a/tests/test-docs.pxi +++ /dev/null @@ -1,24 +0,0 @@ -(ns pixie.tests.test-docs - (require pixie.test :as t)) - -; validate the examples in the docs by checking whether the included -; results match the actual results you get by evaluating the examples. - -(defn check-examples [ns] - (let [ns (the-ns ns) - syms (keys (ns-map ns))] - (doseq [sym syms] - (let [meta (meta @(resolve-in ns sym)) - examples (get meta :examples)] - (doseq [example examples] - (if (contains? example 2) - (t/assert= (eval (read-string (first example))) - (third example)) - (eval (read-string (first example))))))))) - -(t/deftest test-stdlib-docs - (check-examples 'pixie.stdlib)) - -(t/deftest test-string-docs - (load-ns 'pixie.string) - (check-examples 'pixie.string)) diff --git a/tests/test-ffi.pxi b/tests/test-ffi.pxi deleted file mode 100644 index a99dd211..00000000 --- a/tests/test-ffi.pxi +++ /dev/null @@ -1,10 +0,0 @@ -(ns pixie.tests.test-ffi - (require pixie.test :as t)) - - - -(t/deftest test-buffer-ffi - (let [fp (fopen "README.md" "r") - b (buffer 1024)] - (t/assert= 10 (fread b 1 10 fp)) - (t/assert= 91 (nth b 0)))) diff --git a/tests/test-fns.pxi b/tests/test-fns.pxi deleted file mode 100644 index e487ba89..00000000 --- a/tests/test-fns.pxi +++ /dev/null @@ -1,15 +0,0 @@ -(ns pixie.test.test-fns - (require pixie.test :as t)) - -(t/deftest test-fn-literals - (t/assert= (#(+ 3 4)) 7) - (t/assert= (#(+ 3 %) 4) 7) - (t/assert= (#(+ 3 %1) 4) 7) - (t/assert= (#(+ %1 3) 4) 7) - (t/assert= (#(+ %1 %1) 3.5) 7.0) - (t/assert= (#(+ %1 %2) 3 4) 7) - (t/assert= (#(- %2 %1) 3 4) 1) - (t/assert= (#(+ %1 %1 %2 %2) 1.5 2) 7.0) - (t/assert= (#(+ %1 %3) 3 'ignored 4) 7) - (t/assert= (#(- %3 %1) 3 'ignored 4) 1) - (t/assert= (#(apply + %1 %2 %&) 1 2 3 4 5) (+ 1 2 3 4 5))) diff --git a/tests/test-forms.pxi b/tests/test-forms.pxi deleted file mode 100644 index b909869a..00000000 --- a/tests/test-forms.pxi +++ /dev/null @@ -1,90 +0,0 @@ -(ns pixie.tests.test-forms - (require pixie.test :as t)) - -(t/deftest test-when - (t/assert= (when false :never) nil) - (t/assert= (when nil :never) nil) - (t/assert= (when (= 3 4) :never) nil) - - (t/assert= (when true :always) :always) - (t/assert= (when (+ 3 4) :always) :always) - (t/assert= (when {} :always) :always) - - (let [c (atom 0)] - (when (= 3 3) - (swap! c inc) - (swap! c inc) - (swap! c inc)) - (t/assert= @c 3))) - -(t/deftest test-when-not - (t/assert= (when-not false :always) :always) - (t/assert= (when-not nil :always) :always) - (t/assert= (when-not (= 3 4) :always) :always) - - (t/assert= (when-not true :never) nil) - (t/assert= (when-not (+ 3 4) :never) nil) - (t/assert= (when-not {} :never) nil) - - (let [c (atom 0)] - (when-not (= 3 4) - (swap! c inc) - (swap! c inc) - (swap! c inc)) - (t/assert= @c 3))) - -(t/deftest test-when-let - (t/assert= (when-let [v false] :never) nil) - (t/assert= (when-let [v nil] :never) nil) - (t/assert= (when-let [v (= 3 4)] :never) nil) - - (t/assert= (when-let [v true] :always) :always) - (t/assert= (when-let [v (+ 3 4)] :always) :always) - (t/assert= (when-let [v {}] :always) :always) - - (let [c (atom 0)] - (when-let [v @c] - (swap! c inc) - (swap! c inc) - (swap! c inc)) - (t/assert= @c 3))) - -(t/deftest test-when-let-destructuring - (t/assert= (when-let [[x y & z] false] :yay) nil) - (t/assert= (when-let [[x y & z] nil] :yay) nil) - (t/assert= (when-let [{:keys [a b]} nil] :yay) nil) - - (t/assert= (when-let [[x y & z] [1 2 3]] :yay) :yay) - (t/assert= (when-let [[x y & z] [1 2 3]] [x y z]) [1 2 '(3)]) - (t/assert= (when-let [{:keys [a b]} {}] :yay) :yay) - (t/assert= (when-let [{:keys [a b]} {}] [a b]) [nil nil]) - (t/assert= (when-let [{:keys [a b]} {:a 1, :b 41}] [a b]) [1 41])) - -(t/deftest test-if-let - (t/assert= (if-let [v false] :yay :nay) :nay) - (t/assert= (if-let [v false] :yay) nil) - (t/assert= (if-let [v nil] :yay :nay) :nay) - (t/assert= (if-let [v nil] :yay) nil) - (t/assert= (if-let [v (= 3 4)] :yay :nay) :nay) - (t/assert= (if-let [v (= 3 4)] :yay) nil) - - (t/assert= (if-let [v true] :yay :nay) :yay) - (t/assert= (if-let [v true] :yay) :yay) - (t/assert= (if-let [v (+ 3 4)] v :nay) 7) - (t/assert= (if-let [v (+ 3 4)] v) 7) - (t/assert= (if-let [v {}] v :nay) {}) - (t/assert= (if-let [v {}] v) {})) - -(t/deftest test-if-let-destructuring - (t/assert= (if-let [[x y & z] false] :yay :nay) :nay) - (t/assert= (if-let [[x y & z] false] :yay) nil) - (t/assert= (if-let [[x y & z] nil] :yay :nay) :nay) - (t/assert= (if-let [[x y & z] nil] :yay) nil) - (t/assert= (if-let [{:keys [a b]} nil] :yay :nay) :nay) - (t/assert= (if-let [{:keys [a b]} nil] :yay) nil) - - (t/assert= (if-let [[x y & z] [1 2 3]] :yay :nay) :yay) - (t/assert= (if-let [[x y & z] [1 2 3]] [x y z] :nay) [1 2 '(3)]) - (t/assert= (if-let [{:keys [a b]} {}] :yay :nay) :yay) - (t/assert= (if-let [{:keys [a b]} {}] [a b] :nay) [nil nil]) - (t/assert= (if-let [{:keys [a b]} {:a 1, :b 41}] [a b] :nay) [1 41])) diff --git a/tests/test-keywords.pxi b/tests/test-keywords.pxi deleted file mode 100644 index 5f336b73..00000000 --- a/tests/test-keywords.pxi +++ /dev/null @@ -1,10 +0,0 @@ -(ns pixie.tests.test-keywords - (require pixie.test :as t)) - -(t/deftest keyword-invoke - (let [m {:a 1, :b 2, :c 3}] - (t/assert= (:a m) 1) - (t/assert= (:b m) 2) - (t/assert= (:c m) 3) - - (t/assert= (:d m) nil))) diff --git a/tests/test-numbers.pxi b/tests/test-numbers.pxi deleted file mode 100644 index 1d88d9d0..00000000 --- a/tests/test-numbers.pxi +++ /dev/null @@ -1,51 +0,0 @@ -(ns pixie.tests.test-numbers - (require pixie.test :as t)) - -(t/deftest integer-literals - (t/assert= 0xa 10) - (t/assert= -0xa -10) - (t/assert= 012 10) - (t/assert= -012 -10) - (t/assert= 2r1010 10) - (t/assert= -2r1010 -10)) - -(t/deftest float-literals - (t/assert= 10. 10.0) - (t/assert= -10. -10.0) - (t/assert= 1e1 10.0) - (t/assert= -1e1 -10.0) - (t/assert= 1e-1 0.1) - (t/assert= -1e-1 -0.1)) - -(t/deftest mixed-float-ops - (t/assert= (+ 1/2 0.5) 1.0) - (t/assert= (+ 0 1.0) 1.0)) - -(t/deftest ratio-literals - (t/assert= 3/4 (/ 3 4)) - (t/assert= -3/4 (/ -3 4)) - (t/assert= 6/8 3/4) - (t/assert= 9/12 3/4) - (t/assert= 3/1 3)) - -(t/deftest ratio-from-divide - (t/assert= (/ 3 4) 3/4)) - -(t/deftest ratio-ops - (t/assert= (+ 1/2 1/2) 1) - (t/assert= (- 1/2 1/2) 0) - (t/assert= (* 1/2 1/2) 1/4) - (t/assert= (/ 1/2 1/2) 1)) - -(t/deftest ratio-accessors - (doseq [[r n d] [[3/2 3 2] [1/9 1 9] [-3/89 -3 89]]] - (t/assert= (numerator r) n) - (t/assert= (denominator r) d))) - -(t/deftest test-int - (doseq [[x i] [[1 1] [3.0 3] [3.5 3] [3.999 3] [3/2 1]]] - (t/assert= (int x) i))) - -(t/deftest test-float - (doseq [[x f] [[1 1.0] [3 3.0] [3.333 3.333] [3/2 1.5] [1/7 (/ 1.0 7.0)]]] - (t/assert= (float x) f))) diff --git a/tests/test-readeval.pxi b/tests/test-readeval.pxi deleted file mode 100644 index 01be1a3d..00000000 --- a/tests/test-readeval.pxi +++ /dev/null @@ -1,17 +0,0 @@ -(ns pixie.tests.test-readeval - (require pixie.test :as t)) - -(t/deftest test-read - (t/assert= (read-string "0xDEADBEEF") 3735928559) - (t/assert= (read-string "0xDeadBeef") 3735928559) - (t/assert= (read-string "0xdeadbeef") 3735928559) - (t/assert= (read-string "foo") 'foo) - (t/assert= (read-string "()") '()) - (t/assert= (read-string "(1 2 3)") '(1 2 3)) - (t/assert= (read-string "[1 2 3]") [1 2 3]) - (t/assert= (read-string "{:a 1 :b 2 :c 3}") {:a 1 :b 2 :c 3}) - (t/assert= (read-string "\"foo\"") "foo") - (t/assert= (read-string "\"fo\\\\o\"") "fo\\o") - (t/assert= (read-string "false") false) - (t/assert= (read-string "true") true) - (t/assert= (read-string "(foo (bar (baz)))") '(foo (bar (baz))))) diff --git a/tests/test-stdlib.pxi b/tests/test-stdlib.pxi deleted file mode 100644 index 4f639268..00000000 --- a/tests/test-stdlib.pxi +++ /dev/null @@ -1,257 +0,0 @@ -(ns pixie.tests.test-stdlib - (require pixie.test :as t)) - -(t/deftest test-identity - (let [vs [nil true false [1 2 3] #{1 2 3} :oops]] - (doseq [v vs] - (t/assert= (identity v) v)))) - -(t/deftest test-mapcat - (t/assert= (mapcat identity []) []) - (t/assert= (mapcat first [[[1 2]] [[3] [:not :present]] [[4 5 6]]]) [1 2 3 4 5 6])) - -(t/deftest test-str - (t/assert= (str nil) "nil") - (t/assert= (str true) "true") - (t/assert= (str false) "false") - (t/assert= (str "hey") "hey") - (t/assert= (str :hey) ":hey") - (t/assert= (str 'hey) "hey") - - (t/assert= (str '()) "()") - (t/assert= (str '(1 2 3)) "(1 2 3)") - (t/assert= (str [1 2 3]) "[1 2 3]") - (t/assert= (str #{1}) "#{1}") - (t/assert= (str {}) "{}") - (t/assert= (str {:a 1}) "{:a 1}") - (t/assert= (str (type 3)) "") - - (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]")) - -(t/deftest test-repr - (t/assert= (-repr nil) "nil") - (t/assert= (-repr true) "true") - (t/assert= (-repr false) "false") - (t/assert= (-repr "hey") "\"hey\"") - (t/assert= (-repr :hey) ":hey") - (t/assert= (-repr 'hey) "hey") - - (t/assert= (-repr '()) "()") - (t/assert= (-repr '(1 2 3)) "(1 2 3)") - (t/assert= (-repr [1 2 3]) "[1 2 3]") - (t/assert= (-repr #{1}) "#{1}") - (t/assert= (-repr {}) "{}") - (t/assert= (-repr {:a 1}) "{:a 1}") - (t/assert= (-repr (type 3)) "pixie.stdlib.Integer") - - (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]")) - -(t/deftest test-first - (t/assert= (first []) nil) - (t/assert= (first '()) nil) - (t/assert= (first (make-array 0)) nil) - (comment (t/assert= (first {}) nil)) - (comment (t/assert= (first #{}) nil)) - - (t/assert= (first [1 2 3]) 1) - (t/assert= (first '(1 2 3)) 1) - (let [a (make-array 3)] - (aset a 0 1) - (aset a 1 2) - (aset a 2 3) - (t/assert= (first a) 1))) - -(t/deftest test-last - (let [v [1 2 3 4 5] - l '(1 2 3 4 5) - r (range 1 6)] - (t/assert= (last nil) nil) - (t/assert= (last []) nil) - (t/assert= (last (range 0 0)) nil) - (t/assert= (last v) 5) - (t/assert= (last l) 5) - (t/assert= (last r) 5))) - -(t/deftest test-butlast - (let [v [1 2 3 4 5] - l '(1 2 3 4 5) - r (range 1 6) - res '(1 2 3 4)] - (t/assert= (butlast nil) nil) - (t/assert= (butlast []) nil) - (t/assert= (butlast (range 0 0)) nil) - (t/assert= (butlast v) res) - (t/assert= (butlast l) res) - (t/assert= (butlast r) res))) - -(t/deftest test-empty? - (t/assert= (empty? []) true) - (t/assert= (empty? '()) true) - (t/assert= (empty? (make-array 0)) true) - (t/assert= (empty? {}) true) - (t/assert= (empty? #{}) true) - (t/assert= (empty? (range 1 5)) false) - - (t/assert= (empty? [1 2 3]) false) - (t/assert= (empty? '(1 2 3)) false) - (let [a (make-array 1)] - (aset a 0 1) - (t/assert= (empty? a) false)) - (t/assert= (empty? {:a 1}) false) - (t/assert= (empty? #{:a :b}) false)) - -(t/deftest test-not-empty? - (t/assert= (not-empty? []) false) - (t/assert= (not-empty? '()) false) - (t/assert= (not-empty? (make-array 0)) false) - (t/assert= (not-empty? {}) false) - (t/assert= (not-empty? #{}) false) - (t/assert= (not-empty? (range 1 5)) true) - - (t/assert= (not-empty? [1 2 3]) true) - (t/assert= (not-empty? '(1 2 3)) true) - (let [a (make-array 1)] - (aset a 0 1) - (t/assert= (not-empty? a) true)) - (t/assert= (not-empty? {:a 1}) true) - (t/assert= (not-empty? #{:a :b}) true)) - -(t/deftest test-keys - (let [v {:a 1 :b 2 :c 3}] - (t/assert= (set (keys v)) #{:a :b :c}) - (t/assert= (transduce (keys) conj! v) (keys v)))) - -(t/deftest test-vals - (let [v {:a 1 :b 2 :c 3}] - (t/assert= (set (vals v)) #{1 2 3}) - (t/assert= (transduce (vals) conj! v) (vals v)))) - - -(t/deftest test-empty - (t/assert= (empty '(1 2 3)) '()) - (t/assert= (empty (list 1 2 3)) '()) - (t/assert= (empty (lazy-seq)) '()) - (t/assert= (empty '()) '()) - (t/assert= (empty [1 2 3]) []) - (t/assert= (empty (make-array 3)) (make-array 0)) - (t/assert= (empty {:a 1, :b 2, :c 3}) {}) - (t/assert= (empty #{1 2 3}) #{})) - - -(t/deftest test-vec - (let [v '(1 2 3 4 5)] - (t/assert= (vec v) [1 2 3 4 5]) - (t/assert= (vec (map inc) v) [2 3 4 5 6]))) - - -(t/deftest test-keep - (let [v [-1 0 1 2 3 4 5]] - (t/assert= (vec (keep pos?) v) [true true true true true]) - (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)))) - -(t/deftest test-assoc - (t/assert= (assoc {} :a 3) {:a 3}) - (t/assert= (assoc {:a 1} :a 3) {:a 3}) - - (t/assert= (assoc [] 0 :ok) [:ok]) - (t/assert= (assoc [1] 0 :ok) [:ok]) - (t/assert= (assoc [1 2 3] 1 :ok) [1 :ok 3])) - -(t/deftest test-get-in - (let [m {:a 1 :b 2 :x {:a 2 :x [1 2 3]}}] - (t/assert= (get-in m [:a]) 1) - (t/assert= (get-in m [:missing]) nil) - (t/assert= (get-in m [:missing] :not-found) :not-found) - (t/assert= (get-in m [:x :x 0] :not-found) 1))) - -(t/deftest test-assoc-in - (t/assert= (assoc-in {:a {:b 2}} [:a :b] 3) {:a {:b 3}}) - (t/assert= (assoc-in {:a [{:b 2}]} [:a 0 :b] 3) {:a [{:b 3}]}) - ; non existing keys create maps (not vectors, even if the keys are integers) - (t/assert= (assoc-in {} [:a :b] 3) {:a {:b 3}}) - (t/assert= (assoc-in {} [:a 0 :b] 3) {:a {0 {:b 3}}})) - -(t/deftest test-update-in - (t/assert= (update-in {} [:a :b] (fnil inc 0)) {:a {:b 1}}) - (t/assert= (update-in {:a {:b 2}} [:a :b] inc) {:a {:b 3}}) - (t/assert= (update-in {:a [{:b 2}]} [:a 0 :b] inc) {:a [{:b 3}]})) - -(t/deftest test-fn? - (t/assert= (fn? inc) true) - (t/assert= (fn? {}) true) - (t/assert= (fn? #(%)) true) - (t/assert= (fn? :foo) true) - (t/assert= (fn? 1) false) - (t/assert= (fn? and) false) - (t/assert= (fn? "foo") false) - (t/assert= (fn? (let [x 8] (fn [y] (+ x y)))) true)) - -(t/deftest test-macro? - (t/assert= (macro? and) true) - (t/assert= (macro? or) true) - (t/assert= (macro? defn) true) - (t/assert= (macro? inc) false) - (t/assert= (macro? 1) false) - (t/assert= (macro? :foo) false) - (t/assert= (macro? "foo") false)) - -(def ^:dynamic *earmuffiness* :low) - -(t/deftest test-binding - (t/assert= *earmuffiness* :low) - (binding [*earmuffiness* :quite-high] - (t/assert= *earmuffiness* :quite-high)) - (t/assert= *earmuffiness* :low)) - -(t/deftest test-every? - (t/assert= (every? even? [2 4 6 8]) true) - (t/assert= (every? odd? [2 4 6 8]) false) - (t/assert= (every? even? [2 3 6 8]) false) - (t/assert= (every? even? []) true) - (t/assert= (every? odd? []) true)) - -(t/deftest test-some - (t/assert= (some even? [2 4 6 8]) true) - (t/assert= (some odd? [2 4 6 8]) false) - (t/assert= (some even? [2 3 6 8]) true) - (t/assert= (some even? [1 3 5 8]) true) - (t/assert= (some even? []) false) - (t/assert= (some odd? [2]) false)) - -(t/deftest test-distinct - (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) - (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) - (t/assert= (vec (distinct) [nil nil nil]) [nil])) - -(t/deftest test-merge - (t/assert= (merge nil) nil) - (t/assert= (merge {}) {}) - (t/assert= (merge {:a 1} nil) {:a 1}) - - (t/assert= (merge {} {:a 1, :b 2}) {:a 1, :b 2}) - (t/assert= (merge {:a 1} {:b 2}) {:a 1, :b 2}) - (t/assert= (merge {} {:a 1} {:b 2}) {:a 1, :b 2}) - - (t/assert= (merge {:a 1} {:a 2, :b 3}) {:a 2, :b 3}) - (t/assert= (merge {:a 1, :b 4} {:a 2} {:a 3}) {:a 3, :b 4})) - -(t/deftest test-merge-with - (t/assert= (merge-with identity nil) nil) - (t/assert= (merge-with identity {}) {}) - - (t/assert= (merge-with identity {} {:a 1, :b 2}) {:a 1, :b 2}) - (t/assert= (merge-with identity {:a 1} {:b 2}) {:a 1, :b 2}) - - (t/assert= (merge-with #(identity %1) {:a 1} {:a 2}) {:a 1}) - (t/assert= (merge-with #(identity %1) {:a 1} {:a 2} {:a 3}) {:a 1}) - (t/assert= (merge-with #(identity %2) {:a 1} {:a 2}) {:a 2}) - - (t/assert= (merge-with + {:a 21} {:a 21}) {:a 42}) - (t/assert= (merge-with + {:a 21} {:a 21, :b 1}) {:a 42, :b 1})) - -(t/deftest test-for - (t/assert= (for [x [1 2 3]] x) [1 2 3]) - (t/assert= (for [x [1 2 3] y [:a :b :c]] [x y]) - [[1 :a] [1 :b] [1 :c] - [2 :a] [2 :b] [2 :c] - [3 :a] [3 :b] [3 :c]])) diff --git a/tests/test-strings.pxi b/tests/test-strings.pxi deleted file mode 100644 index 5f8a5540..00000000 --- a/tests/test-strings.pxi +++ /dev/null @@ -1,126 +0,0 @@ -(ns pixie.test.test-strings - (require pixie.test :as t) - (require pixie.string :as s)) - -(t/deftest test-starts-with - (let [s "heyhohuh"] - (t/assert= (s/starts-with s "") true) - (t/assert= (s/starts-with s "hey") true) - (t/assert= (s/starts-with s "heyho") true) - (t/assert= (s/starts-with s s) true) - - (t/assert= (s/starts-with s "ho") false) - (t/assert= (s/starts-with s "foo") false))) - -(t/deftest test-ends-with - (let [s "heyhohuh"] - (t/assert= (s/ends-with s "") true) - (t/assert= (s/ends-with s "huh") true) - (t/assert= (s/ends-with s "hohuh") true) - (t/assert= (s/ends-with s s) true) - - (t/assert= (s/ends-with s "hey") false) - (t/assert= (s/ends-with s "foo") false))) - -(t/deftest test-split - (let [s "hey,ho,huh"] - (t/assert= (s/split s ",") ["hey" "ho" "huh"]) - (t/assert= (s/split s "h") ["" "ey," "o," "u" ""]))) - -(t/deftest test-index-of - (let [s "heyhohuh"] - (t/assert= (s/index-of s "hey") 0) - (t/assert= (s/index-of s "ho") 3) - (t/assert= (s/index-of s "foo") -1) - - (t/assert= (s/index-of s "h" 2) 3) - (t/assert= (s/index-of s "h" 4) 5) - (t/assert= (s/index-of s "hey" 0) 0) - (t/assert= (s/index-of s "hey" 1) -1) - - (t/assert= (s/index-of s "h" 0 0) -1) - (t/assert= (s/index-of s "h" 1 2) -1))) - -(t/deftest test-substring - (let [s "heyhohuh"] - (t/assert= (s/substring s 0) s) - (t/assert= (s/substring s 3) (s/substring s 3 (count s))) - (t/assert= (s/substring s 0 0) "") - (t/assert= (s/substring s 0 3) "hey") - (t/assert= (s/substring s 3 5) "ho") - (t/assert= (s/substring s 5 8) "huh") - (t/assert= (s/substring s 3 10000) "hohuh"))) - -(t/deftest test-upper-case - (t/assert= (s/lower-case "") "") - (t/assert= (s/upper-case "hey") "HEY") - (t/assert= (s/upper-case "hEy") "HEY") - (t/assert= (s/upper-case "HEY") "HEY") - (t/assert= (s/upper-case "hey?!") "HEY?!")) - -(t/deftest test-lower-case - (t/assert= (s/lower-case "") "") - (t/assert= (s/lower-case "hey") "hey") - (t/assert= (s/lower-case "hEy") "hey") - (t/assert= (s/lower-case "HEY") "hey") - (t/assert= (s/lower-case "HEY?!") "hey?!")) - -(t/deftest test-capitalize - (t/assert= (s/capitalize "timothy") "Timothy") - (t/assert= (s/capitalize "Timothy") "Timothy")) - -(t/deftest test-trim - (t/assert= (s/trim "") "") - (t/assert= (s/trim " ") "") - (t/assert= (s/trim " hey ") "hey") - (t/assert= (s/trim " h ey ") "h ey")) - -(t/deftest test-triml - (t/assert= (s/triml "") "") - (t/assert= (s/triml " ") "") - (t/assert= (s/triml " hey") "hey") - (t/assert= (s/triml " hey ") "hey ") - (t/assert= (s/triml " h ey ") "h ey ")) - -(t/deftest test-trimr - (t/assert= (s/trimr "") "") - (t/assert= (s/trimr " ") "") - (t/assert= (s/trimr "hey ") "hey") - (t/assert= (s/trimr " hey ") " hey") - (t/assert= (s/trimr " h ey ") " h ey")) - -(t/deftest test-replace - (t/assert= (s/replace "hey,you,there" "," ", ") "hey, you, there") - (t/assert= (s/replace "hey,you,there" "," "") "heyyouthere") - (t/assert= (s/replace "&&&" "&" "&&") "&&&&&&") - (t/assert= (s/replace "oops" "" "WAT") "WAToWAToWATpWATsWAT")) - -(t/deftest test-replace-first - (t/assert= (s/replace-first "hey,you,there" "," ", ") "hey, you,there") - (t/assert= (s/replace-first "hey,you,there" "," "") "heyyou,there") - (t/assert= (s/replace-first "&&&" "&" "&&") "&&&&") - (t/assert= (s/replace-first "oops" "" "WAT") "WAToops")) - -(t/deftest test-join - (t/assert= (s/join []) "") - (t/assert= (s/join [1]) "1") - (t/assert= (s/join [1 2 3]) "123") - - (t/assert= (s/join ", " []) "") - (t/assert= (s/join ", " [1]) "1") - - (t/assert= (s/join ", " [1 2 3]) "1, 2, 3")) - -(t/deftest test-char-literals - (let [s "hey"] - (t/assert= (nth s 0) \h) - (t/assert= (nth s 0) \o150) - (t/assert= (nth s 0) \u0068) - - (t/assert= (nth s 1) \e) - (t/assert= (nth s 1) \o145) - (t/assert= (nth s 1) \u0065) - - (t/assert= (nth s 2) \y) - (t/assert= (nth s 2) \o171) - (t/assert= (nth s 2) \u0079))) diff --git a/tests/utils.pxi b/tests/utils.pxi deleted file mode 100644 index 4916736b..00000000 --- a/tests/utils.pxi +++ /dev/null @@ -1,13 +0,0 @@ -(ns tests.utils) - -;; Here we create a new type which hashes poorly, in fact it's so bad we have a -;; hash space of only 1. -;; All members of WorstHasher return (hash "worst hasher") -;; when hash is called on them. -;; -;; This makes debugging, testing and benchmarking anything based off -;; PersistentHashMap trivial. - -;; X can be any thing you like. -(defrecord WorstHasher [x]) -(extend -hash WorstHasher (fn [self] (hash "worst hasher"))) From 94e7a52b675299d5d38ad60a1ad116d9267004cb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 09:35:11 -0700 Subject: [PATCH 381/909] move...don't delete --- tests/pixie/test/collections/test-maps.pxi | 55 ++++ .../pixie/test/collections/test-seqables.pxi | 45 +++ tests/pixie/test/collections/test-sets.pxi | 71 +++++ tests/pixie/test/collections/test-vectors.pxi | 45 +++ tests/pixie/test/test-arrays.pxi | 51 ++++ tests/pixie/test/test-bits.pxi | 59 ++++ tests/pixie/test/test-compiler.pxi | 22 ++ tests/pixie/test/test-defrecord.pxi | 50 ++++ tests/pixie/test/test-deftype.pxi | 85 ++++++ tests/pixie/test/test-destructuring.pxi | 76 ++++++ tests/pixie/test/test-docs.pxi | 24 ++ tests/pixie/test/test-ffi.pxi | 10 + tests/pixie/test/test-fns.pxi | 15 + tests/pixie/test/test-forms.pxi | 90 ++++++ tests/pixie/test/test-io.pxi | 10 + tests/pixie/test/test-io.txt | 1 + tests/pixie/test/test-keywords.pxi | 10 + tests/pixie/test/test-numbers.pxi | 51 ++++ tests/pixie/test/test-readeval.pxi | 17 ++ tests/pixie/test/test-stdlib.pxi | 257 ++++++++++++++++++ tests/pixie/test/test-strings.pxi | 126 +++++++++ tests/pixie/test/utils.pxi | 13 + 22 files changed, 1183 insertions(+) create mode 100644 tests/pixie/test/collections/test-maps.pxi create mode 100644 tests/pixie/test/collections/test-seqables.pxi create mode 100644 tests/pixie/test/collections/test-sets.pxi create mode 100644 tests/pixie/test/collections/test-vectors.pxi create mode 100644 tests/pixie/test/test-arrays.pxi create mode 100644 tests/pixie/test/test-bits.pxi create mode 100644 tests/pixie/test/test-compiler.pxi create mode 100644 tests/pixie/test/test-defrecord.pxi create mode 100644 tests/pixie/test/test-deftype.pxi create mode 100644 tests/pixie/test/test-destructuring.pxi create mode 100644 tests/pixie/test/test-docs.pxi create mode 100644 tests/pixie/test/test-ffi.pxi create mode 100644 tests/pixie/test/test-fns.pxi create mode 100644 tests/pixie/test/test-forms.pxi create mode 100644 tests/pixie/test/test-io.pxi create mode 100644 tests/pixie/test/test-io.txt create mode 100644 tests/pixie/test/test-keywords.pxi create mode 100644 tests/pixie/test/test-numbers.pxi create mode 100644 tests/pixie/test/test-readeval.pxi create mode 100644 tests/pixie/test/test-stdlib.pxi create mode 100644 tests/pixie/test/test-strings.pxi create mode 100644 tests/pixie/test/utils.pxi diff --git a/tests/pixie/test/collections/test-maps.pxi b/tests/pixie/test/collections/test-maps.pxi new file mode 100644 index 00000000..a4072fe1 --- /dev/null +++ b/tests/pixie/test/collections/test-maps.pxi @@ -0,0 +1,55 @@ +(ns collections.test-maps + (require pixie.test :as t)) + +(t/deftest maps-contains + (let [m {:a 1, :b 2, :c 3} + c [:a :b :c] + n [:d 'a 1]] + (foreach [c c] + (t/assert= (contains? m c) true)) + (foreach [n n] + (t/assert= (contains? m c) false)))) + +(t/deftest map-equals + (let [m {:a 1, :b 2, :c 3}] + (t/assert= (-eq {} {}) true) + (t/assert= (-eq m m) true) + (t/assert= (-eq m {:a 1, :b 2, :c 3}) true) + + (t/assert= (-eq m {}) false) + (t/assert= (-eq m nil) false) + + (t/assert= (-eq m {:a 1, :b 2}) false) + (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false) + + (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) + (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) + + +(t/deftest map-val-at-and-invoke + (let [m {:a 1, :b 2, :c 3}] + (foreach [e m] + (t/assert= (get m (key e)) (val e)) + (t/assert= (m (key e)) (val e))) + (t/assert= (get m :d) nil) + (t/assert= (m :d) nil))) + +(t/deftest map-without + (let [m {:a 1 :b 2}] + (t/assert= m m) + (t/assert= (dissoc m :a) {:b 2}) + (t/assert= (dissoc m :a :b) {}))) + +(t/deftest map-conj + (let [m {:a 1 :b 2}] + (t/assert= m m) + ;; Should conj vector of length 2 + (t/assert= (conj m [:c 3]) {:a 1 :b 2 :c 3}) + (t/assert= (conj m [:b 4]) {:a 1 :b 4}) + (t/assert= (conj m [:b 4] [:c 5]) {:a 1 :b 4 :c 5}) + + ;; Should conj sequences of pairs + (t/assert= (conj {} '([:a 1] [:b 2] [:c 3])) {:a 1 :b 2 :c 3}) + + ;; Should conj sequences of MapEntries + (t/assert= (conj {} (seq {:a 1 :b 2 :c 3})) {:a 1 :b 2 :c 3}))) diff --git a/tests/pixie/test/collections/test-seqables.pxi b/tests/pixie/test/collections/test-seqables.pxi new file mode 100644 index 00000000..dcf79354 --- /dev/null +++ b/tests/pixie/test/collections/test-seqables.pxi @@ -0,0 +1,45 @@ +(ns collections.test-seqables + (require pixie.test :as t)) + +(t/deftest test-seq + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (seq l) '(1 2 3)) + (t/assert= (seq v) [1 2 3]) + + (t/assert= (seq nil) nil) + (t/assert= (seq []) nil))) + +(t/deftest test-first + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (first l) 1) + (t/assert= (first v) 1) + (t/assert= (first (seq l)) 1) + (t/assert= (first (seq v)) 1))) + +(t/deftest test-next + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (next l) '(2 3)) + (t/assert= (next v) '(2 3)) + (t/assert= (next (seq l)) '(2 3)) + (t/assert= (next (seq v)) '(2 3)) + + (t/assert= (next (next (next l))) nil) + (t/assert= (next (next (next v))) nil))) + +(t/deftest test-equals + (let [l '(1 2 3)] + (t/assert= l l) + (t/assert= l '(1 2 3)) + (t/assert= l [1 2 3]) + + (t/assert= (= nil '()) false) + (t/assert= (= '() nil) false) + (t/assert= (= l '(1 2 3 4)) false) + (t/assert= (= l [1 2 3 4]) false))) + +(t/deftest test-conj + (t/assert= '(3 1 2) (conj '(1 2) 3)) + (t/assert= '(5 4 3 1 2) (conj '(1 2) 3 4 5))) diff --git a/tests/pixie/test/collections/test-sets.pxi b/tests/pixie/test/collections/test-sets.pxi new file mode 100644 index 00000000..62652da8 --- /dev/null +++ b/tests/pixie/test/collections/test-sets.pxi @@ -0,0 +1,71 @@ +(ns collections.test-sets + (require pixie.test :as t) + (require tests.utils :as u)) + +(def worst-hashers (vec (map u/->WorstHasher) + (range 100))) + +(t/deftest test-count + (t/assert= (count (set [])) 0) + (t/assert= (count (set [1 2 3])) 3) + (t/assert= (count (set [1 1 2 1])) 2) + (t/assert= (count (set worst-hashers)) 100)) + +(t/deftest test-contains + (let [s #{1 2 3} + c [1 2 3] + n [-1 0 4] + g (set worst-hashers)] + (foreach [c c] + (t/assert= (contains? s c) true)) + (foreach [n n] + (t/assert= (contains? s n) false)) + (foreach [n worst-hashers] + (t/assert= (contains? g n) true)))) + +(t/deftest test-conj + (t/assert= (conj #{}) #{}) + (t/assert= (conj #{1 2} 3) #{1 2 3}) + (t/assert= (reduce conj #{} (range 10)) (set (vec (range 10)))) + (t/assert= (reduce conj #{} worst-hashers) (set worst-hashers))) + +(t/deftest test-disj + (t/assert= (disj #{}) #{}) + (t/assert= (disj #{1 2} 3) #{1 2}) + (t/assert= (disj #{1 2} 2) #{1}) + (t/assert= (reduce disj (set (vec (range 10))) (range 10)) #{}) + (t/assert= (reduce disj (set worst-hashers) worst-hashers) #{})) + +(t/deftest test-eq + (let [s #{1 2 3}] + (t/assert= s s) + (t/assert= s #{1 2 3}) + (t/assert= #{1 2 3} s) + + (t/assert= (= #{} nil) false) + (t/assert= (= #{} []) false) + (t/assert= (= #{} '()) false) + + (t/assert= (= s [1 2 3]) false) + (t/assert= (= s '(1 2 3)) false) + (t/assert= (= s #{1 2}) false) + (t/assert= (= s #{1 2 3 4}) false))) + +(t/deftest test-invoke + (let [s #{1 2 3}] + (t/assert= (s 1) 1) + (t/assert= (s 2) 2) + (t/assert= (s 3) 3) + + (t/assert= (s -1) nil) + (t/assert= (s 4) nil))) + +(t/deftest test-has-meta + (let [m {:has-meta true} + s (with-meta #{} m)] + (t/assert= (meta #{}) nil) + (t/assert= (meta s) m))) + +(t/deftest test-conj + (t/assert= #{1 2} (conj #{1} 2)) + (t/assert= #{1 2 3 4} (conj #{1} 2 3 4))) diff --git a/tests/pixie/test/collections/test-vectors.pxi b/tests/pixie/test/collections/test-vectors.pxi new file mode 100644 index 00000000..f69432fe --- /dev/null +++ b/tests/pixie/test/collections/test-vectors.pxi @@ -0,0 +1,45 @@ +(ns collections.test-vectors + (require pixie.test :as t)) + +(def MAX-SIZE 1064) + +(comment +;; Takes forever in interpreted mode but useful for debugging +(t/deftest vector-creation + (loop [acc []] + (if (= (count acc) MAX-SIZE) + acc + (do (dotimes [j (count acc)] + (t/assert= j (nth acc j))) + (recur (conj acc (count acc))))))) +) +(t/deftest vector-contains + (let [v [1 2 3] + c [0 1 2] + n [-1 3]] + (foreach [c c] + (t/assert= (contains? v c) true)) + (foreach [n n] + (t/assert= (contains? v n) false)))) + +(t/deftest vector-equals + (let [v [1 2 3]] + (t/assert= [] '()) + (t/assert= v v) + (t/assert= v [1 2 3]) + (t/assert= v '(1 2 3)) + + (t/assert= (= [] nil) false) + (t/assert= (= v []) false) + (t/assert= (= v [1 2]) false) + (t/assert= (= v [1 2 3 4]) false) + (t/assert= (= v '(1 2)) false) + (t/assert= (= v '(1 2 3 4)) false))) + +(t/deftest vector-conj + (t/assert= [1 2] (conj [1] 2)) + (t/assert= [1 2 3 4] (conj [1] 2 3 4))) + +(t/deftest vector-conj! + (t/assert= [1 2] (persistent! (conj! (transient [1]) 2))) + (t/assert= [1 2 3] (persistent! (conj! (transient [1]) 2 3)))) diff --git a/tests/pixie/test/test-arrays.pxi b/tests/pixie/test/test-arrays.pxi new file mode 100644 index 00000000..fbd55937 --- /dev/null +++ b/tests/pixie/test/test-arrays.pxi @@ -0,0 +1,51 @@ +(ns pixie.test.test-arrays + (require pixie.test :as t)) + +(t/deftest test-array-creation + (let [a (make-array 10)] + (t/assert= (count a) 10) + (t/assert= (alength a) 10) + (foreach [x a] + (t/assert= x nil)))) + +(t/deftest test-aget-and-aset + (let [a (make-array 10)] + (dotimes [i 10] + (t/assert= (aget a i) nil)) + + (dotimes [i 10] + (aset a i i)) + + (dotimes [i 10] + (t/assert= (aget a i) i)))) + +(t/deftest test-aconcat + (let [a1 (make-array 10) + a2 (make-array 10)] + (t/assert= (alength (aconcat a1 a2)) (+ (alength a1) (alength a2))) + + (dotimes [i 10] + (aset a1 i i) + (aset a2 i (+ 10 i))) + + (let [a3 (aconcat a1 a2)] + (dotimes [i 20] + (t/assert= (aget a3 i) i))))) + +(t/deftest test-aslice + (let [a (make-array 10)] + (dotimes [i 10] + (aset a i i)) + + (let [a1 (aslice a 3) + a2 (aslice a 7)] + (foreach [i (range 0 7)] + (t/assert= (aget a1 i) (+ i 3))) + (foreach [i (range 0 3)] + (t/assert= (aget a2 i) (+ i 7)))))) + + +(t/deftest test-byte-array-creation + (let [ba (byte-array 10)] + (t/assert= (vec ba) [0 0 0 0 0 0 0 0 0 0]) + (t/assert= (count ba) 10))) \ No newline at end of file diff --git a/tests/pixie/test/test-bits.pxi b/tests/pixie/test/test-bits.pxi new file mode 100644 index 00000000..be044f9d --- /dev/null +++ b/tests/pixie/test/test-bits.pxi @@ -0,0 +1,59 @@ +(ns pixie.test.test-bits + (require pixie.test :as t)) + +(t/deftest test-bit-clear + (t/assert= (bit-clear 0 0) 0) + (t/assert= (bit-clear 2r11111 7) 2r11111) + + (t/assert= (bit-clear 2r111 0) 2r110) + (t/assert= (bit-clear 2r111 1) 2r101) + (t/assert= (bit-clear 2r111 2) 2r011)) + +(t/deftest test-bit-set + (t/assert= (bit-set 2r111 0) 2r111) + (t/assert= (bit-set 2r000 1) 2r010)) + +(t/deftest test-bit-flip + (t/assert= (bit-flip 2r101 0) 2r100) + (t/assert= (bit-flip 2r101 1) 2r111)) + +(t/deftest test-bit-test + (t/assert= (bit-test 2r101 0) true) + (t/assert= (bit-test 2r101 1) false)) + +(t/deftest test-bit-and + (t/assert= (bit-and 0 0) 0) + (t/assert= (bit-and 2r101 2r101) 2r101) + (t/assert= (bit-and 2r101 2r101) 2r101) + (t/assert= (bit-and 2r101 0) 0)) + +(t/deftest test-bit-or + (t/assert= (bit-or 0 0) 0) + (t/assert= (bit-or 2r101 2r010) 2r111) + (t/assert= (bit-or 2r111 2r010) 2r111) + (t/assert= (bit-or 2r111 2r111) 2r111) + (t/assert= (bit-or 2r101 0) 2r101)) + +(t/deftest test-bit-xor + (t/assert= (bit-xor 0 0) 0) + (t/assert= (bit-xor 2r101 2r010) 2r111) + (t/assert= (bit-xor 2r111 2r010) 2r101) + (t/assert= (bit-xor 2r111 2r111) 2r000) + (t/assert= (bit-xor 2r101 0) 2r101)) + +(t/deftest test-bit-shift-left + (t/assert= (bit-shift-left 0 0) 0) + (t/assert= (bit-shift-left 2r101 0) 2r101) + + (t/assert= (bit-shift-left 0 7) 0) + (t/assert= (bit-shift-left 2r001 2) 2r100) + (t/assert= (bit-shift-left 2r111 2) 2r11100)) + +(t/deftest test-bit-shift-right + (t/assert= (bit-shift-right 0 0) 0) + (t/assert= (bit-shift-right 2r101 0) 2r101) + + (t/assert= (bit-shift-right 0 7) 0) + (t/assert= (bit-shift-right 2r001 2) 0) + (t/assert= (bit-shift-right 2r111 2) 2r001) + (t/assert= (bit-shift-right 2r1011010 2) 2r10110)) diff --git a/tests/pixie/test/test-compiler.pxi b/tests/pixie/test/test-compiler.pxi new file mode 100644 index 00000000..052bafe2 --- /dev/null +++ b/tests/pixie/test/test-compiler.pxi @@ -0,0 +1,22 @@ +(ns pixie.test.test-compiler + (require pixie.test :as t)) + +(t/deftest test-do + (t/assert= (do 1) 1) + (t/assert= (do 1 2) 2) + (t/assert= (do) nil) + (t/assert= (do 1 2 3 4 5 6) 6)) + +(t/deftest test-if + (t/assert= (if true 42 nil) 42) + (t/assert= (if false 42 nil) nil) + (t/assert= (if false 42) nil)) + +(t/deftest test-let + (t/assert= (let [] 1) 1) + (t/assert= (let [x 1]) nil) + (t/assert= (let []) nil)) + +(t/deftest test-lists + (t/assert= (vec '()) []) + (t/assert= (vec '()) ())) diff --git a/tests/pixie/test/test-defrecord.pxi b/tests/pixie/test/test-defrecord.pxi new file mode 100644 index 00000000..382247d1 --- /dev/null +++ b/tests/pixie/test/test-defrecord.pxi @@ -0,0 +1,50 @@ +(ns pixie.test.test-defrecord + (require pixie.test :as t)) + +(defrecord Three [one two three]) + +(def t1 (->Three 1 2 3)) +(def t2 (->Three 1 2 3)) +(def t3 (map->Three {:one 1, :two 2, :three 3})) +(def t4 (->Three 1 2 4)) +(def t5 (->Three 3 4 5)) + +(t/deftest test-satisfies + (foreach [t [t1 t2 t3 t4 t5]] + (t/assert= t t) + (t/assert (satisfies? IAssociative t)) + (t/assert (satisfies? ILookup t)))) + +(t/deftest test-eq + (t/assert= t1 t2) + (t/assert= t2 t3) + (t/assert= t3 t1) + (foreach [t [t1 t2 t3]] + (t/assert (not (= t (assoc t :one 42)))) + (t/assert (not (= t t4))) + (t/assert (not (= t t5))))) + +(t/deftest test-ilookup + (foreach [t [t1 t2 t3]] + (t/assert (satisfies? ILookup t)) + (t/assert= (get t :one) 1) + (t/assert= (get t :two) 2) + (t/assert= (get t :three) 3) + (t/assert= (get t :oops) nil) + (t/assert= (get t :oops 'not-found) 'not-found))) + +(t/deftest test-iassociative + (foreach [t [t1 t2 t3]] + (t/assert (satisfies? IAssociative t)) + (t/assert= t (assoc t4 :three 3)) + (let [t' (assoc t :one 42) + t-oops (assoc t :oops 'never-found)] + (t/assert (not (= t t'))) + (t/assert= (get t' :one) 42) + (t/assert= t t-oops) + (t/assert (not (contains? t-oops :oops))) + (t/assert= (get t-oops :oops) nil)) + + (t/assert (contains? t :one)) + (t/assert (contains? t :two)) + (t/assert (contains? t :three)))) diff --git a/tests/pixie/test/test-deftype.pxi b/tests/pixie/test/test-deftype.pxi new file mode 100644 index 00000000..49f5e444 --- /dev/null +++ b/tests/pixie/test/test-deftype.pxi @@ -0,0 +1,85 @@ +(ns pixie.test.test-deftype + (require pixie.test :as t)) + +(deftype Simple [:val]) +(deftype Simple2 [val]) + +(t/deftest test-simple + (let [o1 (->Simple 1) + o2 (->Simple 2)] + (foreach [obj-and-val [[o1 1] [o2 2]]] + (let [o (first obj-and-val) + v (second obj-and-val)] + (t/assert= (. o :val) v) + (t/assert= (.val o) v))))) + +(deftype MagicalVectorMap [] IMap IVector) + +(t/deftest test-satisfies + (let [mvm (->MagicalVectorMap)] + (t/assert (satisfies? IVector mvm)) + (t/assert (satisfies? IMap mvm)))) + +(deftype Count [:val] + ICounted + (-count [self] val)) + +(deftype Count2 [val] + ICounted + (-count [self] val)) + +(t/deftest test-extend + (let [o1 (->Count 1) + o2 (->Count 2)] + (foreach [obj-and-val [[o1 1] [o2 2]]] + (let [o (first obj-and-val) + v (second obj-and-val)] + (t/assert= (. o :val) v) + (t/assert= (.val o) v) + (t/assert (satisfies? ICounted o)) + (t/assert= (-count o) v) + (t/assert= (count o) v))))) + +(deftype Three [:one :two :three] + Object + (add [self x & args] + (apply + x args)) + (one-plus [self x & xs] + (apply + one x xs)) + ICounted + (-count [self] (+ one two three))) + +(deftype Three2 [one two three] + Object + (add [self x & args] + (apply + x args)) + (one-plus [self x & xs] + (apply + one x xs)) + ICounted + (-count [self] (+ one two three))) + +(t/deftest test-complex + (let [o1 (->Three 1 2 3) + o2 (->Three2 3 4 5)] + (foreach [obj-and-vals [[o1 1 2 3] [o2 3 4 5]]] + (let [o (first obj-and-vals) + one (second obj-and-vals) + two (third obj-and-vals) + three (fourth obj-and-vals)] + (t/assert= (. o :one) one) + (t/assert= (.one o) one) + (t/assert= (. o :two) two) + (t/assert= (.two o) two) + (t/assert= (. o :three) three) + (t/assert= (.three o) three) + + (t/assert (satisfies? ICounted o)) + (t/assert= (-count o) (+ one two three)) + (t/assert= (count o) (+ one two three)) + + (t/assert= (.add o 21 21) 42) + (t/assert= (.one-plus o 9) (+ one 9)) + + ; arity-1 (just the self arg) not supported for now + (t/assert= (.add o) (. o :add)) + (t/assert= (.one-plus o) (. o :one-plus)))))) diff --git a/tests/pixie/test/test-destructuring.pxi b/tests/pixie/test/test-destructuring.pxi new file mode 100644 index 00000000..13eb93a0 --- /dev/null +++ b/tests/pixie/test/test-destructuring.pxi @@ -0,0 +1,76 @@ +(ns pixie.test.test-destructuring + (require pixie.test :as t)) + +(t/deftest test-let-simple + (t/assert= (let [x 1 y 2 z 3] [x y z]) [1 2 3]) + (t/assert= (let [x 1 y 2 z 3] [x y z]) (let* [x 1 y 2 z 3] [1 2 3])) + + (t/assert= (let [x 1 x 2] x) 2) + (t/assert= (let [x 1 y 2 x 3] x) 3) + + (t/assert= (let [x 1] (let [x 2] x)) 2)) + +(t/deftest test-let-vector-simple + (t/assert= (let [[x y z] [1 2 3]] [x y z]) [1 2 3]) + (t/assert= (let [[x y z] [1 2 3 4]] [x y z]) [1 2 3]) + + (t/assert= (let [[x y z & rest] [1 2 3 4]] + [x y z rest]) + [1 2 3 '(4)]) + (t/assert= (let [[x y z & rest] [1 2]] + [x y z rest]) + [1 2 nil nil])) + +(t/deftest test-let-vector-nested + (t/assert= (let [[[x y] z & rest] [[1 2] 3 4]] + [x y z rest]) + [1 2 3 '(4)]) + + (t/assert= (let [[[x [y]] z & rest] [[1 [2 3]] 4 5]] + [x y z rest]) + [1 2 4 '(5)])) + +(t/deftest test-let-vector-rest + (t/assert= (let [[x y & [z & rest]] [1 2 3 4 5]] + [x y z rest]) + [1 2 3 '(4 5)])) + +(t/deftest test-let-map + (t/assert= (let [{a :a, b :b, {c :c :as s} :d :as m} {:a 1, :b 2, :d {:c 3}}] + [a b c s m]) + [1 2 3 {:c 3} {:a 1, :b 2, :d {:c 3}}]) + + (t/assert= (let [{:keys [a b c] :as m} {:a 1, :b 2, :c 3, :d 4}] + [a b c (:d m)]) + [1 2 3 4])) + +(t/deftest test-let-map-defaults + (t/assert= (let [{a :a :or {a 42}} {:a 1}] a) 1) + (t/assert= (let [{a :a :or {a 42}} {}] a) 42) + + (t/assert= (let [{a :a :or {a 42}} {:a nil}] a) nil) + (t/assert= (let [{a :a :or {a 42}} {:a false}] a) false) + + (t/assert= (let [{:keys [a], :or {a 42}} {:a 1}] a) 1) + (t/assert= (let [{:keys [a], :or {a 42}} {}] a) 42)) + +(t/deftest test-fn-simple + (t/assert= ((fn [[x y & rest]] [x y rest]) [1 2 3 4 5]) [1 2 '(3 4 5)]) + (t/assert= ((fn [{a :a, b :b :as m}] [a b m]) {:a 1, :b 2, :answer 42}) [1 2 {:a 1, :b 2, :answer 42}]) + + (t/assert= ((fn [[[x y] z & rest]] [x y z rest]) [[1 2] 3 4 5]) [1 2 3 '(4 5)])) + +(t/deftest test-fn-multiple-args + (t/assert= ((fn [[x y z] {:keys [a b c]}] [x y z a b c]) [1 2 3] {:a 4, :b 5, :c 6}) + [1 2 3 4 5 6])) + +(t/deftest test-fn-rest-args + (let [f1 (fn [& [status]] (or status :yay))] + (t/assert= (f1) :yay) + (t/assert= (f1 :nay) :nay) + (t/assert= (f1 :nay :something-else :whatever) :nay)) + (let [f2 (fn [x & [y]] + (+ x (or y 1)))] + (t/assert= (f2 41) 42) + (t/assert= (f2 21 21) 42) + (t/assert= (f2 21 21 :something-else :whatever) 42))) diff --git a/tests/pixie/test/test-docs.pxi b/tests/pixie/test/test-docs.pxi new file mode 100644 index 00000000..dd6b96cf --- /dev/null +++ b/tests/pixie/test/test-docs.pxi @@ -0,0 +1,24 @@ +(ns pixie.tests.test-docs + (require pixie.test :as t)) + +; validate the examples in the docs by checking whether the included +; results match the actual results you get by evaluating the examples. + +(defn check-examples [ns] + (let [ns (the-ns ns) + syms (keys (ns-map ns))] + (doseq [sym syms] + (let [meta (meta @(resolve-in ns sym)) + examples (get meta :examples)] + (doseq [example examples] + (if (contains? example 2) + (t/assert= (eval (read-string (first example))) + (third example)) + (eval (read-string (first example))))))))) + +(t/deftest test-stdlib-docs + (check-examples 'pixie.stdlib)) + +(t/deftest test-string-docs + (load-ns 'pixie.string) + (check-examples 'pixie.string)) diff --git a/tests/pixie/test/test-ffi.pxi b/tests/pixie/test/test-ffi.pxi new file mode 100644 index 00000000..a99dd211 --- /dev/null +++ b/tests/pixie/test/test-ffi.pxi @@ -0,0 +1,10 @@ +(ns pixie.tests.test-ffi + (require pixie.test :as t)) + + + +(t/deftest test-buffer-ffi + (let [fp (fopen "README.md" "r") + b (buffer 1024)] + (t/assert= 10 (fread b 1 10 fp)) + (t/assert= 91 (nth b 0)))) diff --git a/tests/pixie/test/test-fns.pxi b/tests/pixie/test/test-fns.pxi new file mode 100644 index 00000000..e487ba89 --- /dev/null +++ b/tests/pixie/test/test-fns.pxi @@ -0,0 +1,15 @@ +(ns pixie.test.test-fns + (require pixie.test :as t)) + +(t/deftest test-fn-literals + (t/assert= (#(+ 3 4)) 7) + (t/assert= (#(+ 3 %) 4) 7) + (t/assert= (#(+ 3 %1) 4) 7) + (t/assert= (#(+ %1 3) 4) 7) + (t/assert= (#(+ %1 %1) 3.5) 7.0) + (t/assert= (#(+ %1 %2) 3 4) 7) + (t/assert= (#(- %2 %1) 3 4) 1) + (t/assert= (#(+ %1 %1 %2 %2) 1.5 2) 7.0) + (t/assert= (#(+ %1 %3) 3 'ignored 4) 7) + (t/assert= (#(- %3 %1) 3 'ignored 4) 1) + (t/assert= (#(apply + %1 %2 %&) 1 2 3 4 5) (+ 1 2 3 4 5))) diff --git a/tests/pixie/test/test-forms.pxi b/tests/pixie/test/test-forms.pxi new file mode 100644 index 00000000..b909869a --- /dev/null +++ b/tests/pixie/test/test-forms.pxi @@ -0,0 +1,90 @@ +(ns pixie.tests.test-forms + (require pixie.test :as t)) + +(t/deftest test-when + (t/assert= (when false :never) nil) + (t/assert= (when nil :never) nil) + (t/assert= (when (= 3 4) :never) nil) + + (t/assert= (when true :always) :always) + (t/assert= (when (+ 3 4) :always) :always) + (t/assert= (when {} :always) :always) + + (let [c (atom 0)] + (when (= 3 3) + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-not + (t/assert= (when-not false :always) :always) + (t/assert= (when-not nil :always) :always) + (t/assert= (when-not (= 3 4) :always) :always) + + (t/assert= (when-not true :never) nil) + (t/assert= (when-not (+ 3 4) :never) nil) + (t/assert= (when-not {} :never) nil) + + (let [c (atom 0)] + (when-not (= 3 4) + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-let + (t/assert= (when-let [v false] :never) nil) + (t/assert= (when-let [v nil] :never) nil) + (t/assert= (when-let [v (= 3 4)] :never) nil) + + (t/assert= (when-let [v true] :always) :always) + (t/assert= (when-let [v (+ 3 4)] :always) :always) + (t/assert= (when-let [v {}] :always) :always) + + (let [c (atom 0)] + (when-let [v @c] + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-let-destructuring + (t/assert= (when-let [[x y & z] false] :yay) nil) + (t/assert= (when-let [[x y & z] nil] :yay) nil) + (t/assert= (when-let [{:keys [a b]} nil] :yay) nil) + + (t/assert= (when-let [[x y & z] [1 2 3]] :yay) :yay) + (t/assert= (when-let [[x y & z] [1 2 3]] [x y z]) [1 2 '(3)]) + (t/assert= (when-let [{:keys [a b]} {}] :yay) :yay) + (t/assert= (when-let [{:keys [a b]} {}] [a b]) [nil nil]) + (t/assert= (when-let [{:keys [a b]} {:a 1, :b 41}] [a b]) [1 41])) + +(t/deftest test-if-let + (t/assert= (if-let [v false] :yay :nay) :nay) + (t/assert= (if-let [v false] :yay) nil) + (t/assert= (if-let [v nil] :yay :nay) :nay) + (t/assert= (if-let [v nil] :yay) nil) + (t/assert= (if-let [v (= 3 4)] :yay :nay) :nay) + (t/assert= (if-let [v (= 3 4)] :yay) nil) + + (t/assert= (if-let [v true] :yay :nay) :yay) + (t/assert= (if-let [v true] :yay) :yay) + (t/assert= (if-let [v (+ 3 4)] v :nay) 7) + (t/assert= (if-let [v (+ 3 4)] v) 7) + (t/assert= (if-let [v {}] v :nay) {}) + (t/assert= (if-let [v {}] v) {})) + +(t/deftest test-if-let-destructuring + (t/assert= (if-let [[x y & z] false] :yay :nay) :nay) + (t/assert= (if-let [[x y & z] false] :yay) nil) + (t/assert= (if-let [[x y & z] nil] :yay :nay) :nay) + (t/assert= (if-let [[x y & z] nil] :yay) nil) + (t/assert= (if-let [{:keys [a b]} nil] :yay :nay) :nay) + (t/assert= (if-let [{:keys [a b]} nil] :yay) nil) + + (t/assert= (if-let [[x y & z] [1 2 3]] :yay :nay) :yay) + (t/assert= (if-let [[x y & z] [1 2 3]] [x y z] :nay) [1 2 '(3)]) + (t/assert= (if-let [{:keys [a b]} {}] :yay :nay) :yay) + (t/assert= (if-let [{:keys [a b]} {}] [a b] :nay) [nil nil]) + (t/assert= (if-let [{:keys [a b]} {:a 1, :b 41}] [a b] :nay) [1 41])) diff --git a/tests/pixie/test/test-io.pxi b/tests/pixie/test/test-io.pxi new file mode 100644 index 00000000..f8766a18 --- /dev/null +++ b/tests/pixie/test/test-io.pxi @@ -0,0 +1,10 @@ +(ns pixie.test.test-io + (require pixie.test :as t) + (require pixie.io :as io)) + +(t/deftest test-file-reduction + (let [f (io/open-read "test/test-io.txt")] + (t/assert= (transduce (map identity) + count-rf + f) + 10))) diff --git a/tests/pixie/test/test-io.txt b/tests/pixie/test/test-io.txt new file mode 100644 index 00000000..c5187b04 --- /dev/null +++ b/tests/pixie/test/test-io.txt @@ -0,0 +1 @@ +This is a test file used in testing the io routines. Please do not remove it. diff --git a/tests/pixie/test/test-keywords.pxi b/tests/pixie/test/test-keywords.pxi new file mode 100644 index 00000000..5f336b73 --- /dev/null +++ b/tests/pixie/test/test-keywords.pxi @@ -0,0 +1,10 @@ +(ns pixie.tests.test-keywords + (require pixie.test :as t)) + +(t/deftest keyword-invoke + (let [m {:a 1, :b 2, :c 3}] + (t/assert= (:a m) 1) + (t/assert= (:b m) 2) + (t/assert= (:c m) 3) + + (t/assert= (:d m) nil))) diff --git a/tests/pixie/test/test-numbers.pxi b/tests/pixie/test/test-numbers.pxi new file mode 100644 index 00000000..1d88d9d0 --- /dev/null +++ b/tests/pixie/test/test-numbers.pxi @@ -0,0 +1,51 @@ +(ns pixie.tests.test-numbers + (require pixie.test :as t)) + +(t/deftest integer-literals + (t/assert= 0xa 10) + (t/assert= -0xa -10) + (t/assert= 012 10) + (t/assert= -012 -10) + (t/assert= 2r1010 10) + (t/assert= -2r1010 -10)) + +(t/deftest float-literals + (t/assert= 10. 10.0) + (t/assert= -10. -10.0) + (t/assert= 1e1 10.0) + (t/assert= -1e1 -10.0) + (t/assert= 1e-1 0.1) + (t/assert= -1e-1 -0.1)) + +(t/deftest mixed-float-ops + (t/assert= (+ 1/2 0.5) 1.0) + (t/assert= (+ 0 1.0) 1.0)) + +(t/deftest ratio-literals + (t/assert= 3/4 (/ 3 4)) + (t/assert= -3/4 (/ -3 4)) + (t/assert= 6/8 3/4) + (t/assert= 9/12 3/4) + (t/assert= 3/1 3)) + +(t/deftest ratio-from-divide + (t/assert= (/ 3 4) 3/4)) + +(t/deftest ratio-ops + (t/assert= (+ 1/2 1/2) 1) + (t/assert= (- 1/2 1/2) 0) + (t/assert= (* 1/2 1/2) 1/4) + (t/assert= (/ 1/2 1/2) 1)) + +(t/deftest ratio-accessors + (doseq [[r n d] [[3/2 3 2] [1/9 1 9] [-3/89 -3 89]]] + (t/assert= (numerator r) n) + (t/assert= (denominator r) d))) + +(t/deftest test-int + (doseq [[x i] [[1 1] [3.0 3] [3.5 3] [3.999 3] [3/2 1]]] + (t/assert= (int x) i))) + +(t/deftest test-float + (doseq [[x f] [[1 1.0] [3 3.0] [3.333 3.333] [3/2 1.5] [1/7 (/ 1.0 7.0)]]] + (t/assert= (float x) f))) diff --git a/tests/pixie/test/test-readeval.pxi b/tests/pixie/test/test-readeval.pxi new file mode 100644 index 00000000..01be1a3d --- /dev/null +++ b/tests/pixie/test/test-readeval.pxi @@ -0,0 +1,17 @@ +(ns pixie.tests.test-readeval + (require pixie.test :as t)) + +(t/deftest test-read + (t/assert= (read-string "0xDEADBEEF") 3735928559) + (t/assert= (read-string "0xDeadBeef") 3735928559) + (t/assert= (read-string "0xdeadbeef") 3735928559) + (t/assert= (read-string "foo") 'foo) + (t/assert= (read-string "()") '()) + (t/assert= (read-string "(1 2 3)") '(1 2 3)) + (t/assert= (read-string "[1 2 3]") [1 2 3]) + (t/assert= (read-string "{:a 1 :b 2 :c 3}") {:a 1 :b 2 :c 3}) + (t/assert= (read-string "\"foo\"") "foo") + (t/assert= (read-string "\"fo\\\\o\"") "fo\\o") + (t/assert= (read-string "false") false) + (t/assert= (read-string "true") true) + (t/assert= (read-string "(foo (bar (baz)))") '(foo (bar (baz))))) diff --git a/tests/pixie/test/test-stdlib.pxi b/tests/pixie/test/test-stdlib.pxi new file mode 100644 index 00000000..4f639268 --- /dev/null +++ b/tests/pixie/test/test-stdlib.pxi @@ -0,0 +1,257 @@ +(ns pixie.tests.test-stdlib + (require pixie.test :as t)) + +(t/deftest test-identity + (let [vs [nil true false [1 2 3] #{1 2 3} :oops]] + (doseq [v vs] + (t/assert= (identity v) v)))) + +(t/deftest test-mapcat + (t/assert= (mapcat identity []) []) + (t/assert= (mapcat first [[[1 2]] [[3] [:not :present]] [[4 5 6]]]) [1 2 3 4 5 6])) + +(t/deftest test-str + (t/assert= (str nil) "nil") + (t/assert= (str true) "true") + (t/assert= (str false) "false") + (t/assert= (str "hey") "hey") + (t/assert= (str :hey) ":hey") + (t/assert= (str 'hey) "hey") + + (t/assert= (str '()) "()") + (t/assert= (str '(1 2 3)) "(1 2 3)") + (t/assert= (str [1 2 3]) "[1 2 3]") + (t/assert= (str #{1}) "#{1}") + (t/assert= (str {}) "{}") + (t/assert= (str {:a 1}) "{:a 1}") + (t/assert= (str (type 3)) "") + + (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]")) + +(t/deftest test-repr + (t/assert= (-repr nil) "nil") + (t/assert= (-repr true) "true") + (t/assert= (-repr false) "false") + (t/assert= (-repr "hey") "\"hey\"") + (t/assert= (-repr :hey) ":hey") + (t/assert= (-repr 'hey) "hey") + + (t/assert= (-repr '()) "()") + (t/assert= (-repr '(1 2 3)) "(1 2 3)") + (t/assert= (-repr [1 2 3]) "[1 2 3]") + (t/assert= (-repr #{1}) "#{1}") + (t/assert= (-repr {}) "{}") + (t/assert= (-repr {:a 1}) "{:a 1}") + (t/assert= (-repr (type 3)) "pixie.stdlib.Integer") + + (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]")) + +(t/deftest test-first + (t/assert= (first []) nil) + (t/assert= (first '()) nil) + (t/assert= (first (make-array 0)) nil) + (comment (t/assert= (first {}) nil)) + (comment (t/assert= (first #{}) nil)) + + (t/assert= (first [1 2 3]) 1) + (t/assert= (first '(1 2 3)) 1) + (let [a (make-array 3)] + (aset a 0 1) + (aset a 1 2) + (aset a 2 3) + (t/assert= (first a) 1))) + +(t/deftest test-last + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (t/assert= (last nil) nil) + (t/assert= (last []) nil) + (t/assert= (last (range 0 0)) nil) + (t/assert= (last v) 5) + (t/assert= (last l) 5) + (t/assert= (last r) 5))) + +(t/deftest test-butlast + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6) + res '(1 2 3 4)] + (t/assert= (butlast nil) nil) + (t/assert= (butlast []) nil) + (t/assert= (butlast (range 0 0)) nil) + (t/assert= (butlast v) res) + (t/assert= (butlast l) res) + (t/assert= (butlast r) res))) + +(t/deftest test-empty? + (t/assert= (empty? []) true) + (t/assert= (empty? '()) true) + (t/assert= (empty? (make-array 0)) true) + (t/assert= (empty? {}) true) + (t/assert= (empty? #{}) true) + (t/assert= (empty? (range 1 5)) false) + + (t/assert= (empty? [1 2 3]) false) + (t/assert= (empty? '(1 2 3)) false) + (let [a (make-array 1)] + (aset a 0 1) + (t/assert= (empty? a) false)) + (t/assert= (empty? {:a 1}) false) + (t/assert= (empty? #{:a :b}) false)) + +(t/deftest test-not-empty? + (t/assert= (not-empty? []) false) + (t/assert= (not-empty? '()) false) + (t/assert= (not-empty? (make-array 0)) false) + (t/assert= (not-empty? {}) false) + (t/assert= (not-empty? #{}) false) + (t/assert= (not-empty? (range 1 5)) true) + + (t/assert= (not-empty? [1 2 3]) true) + (t/assert= (not-empty? '(1 2 3)) true) + (let [a (make-array 1)] + (aset a 0 1) + (t/assert= (not-empty? a) true)) + (t/assert= (not-empty? {:a 1}) true) + (t/assert= (not-empty? #{:a :b}) true)) + +(t/deftest test-keys + (let [v {:a 1 :b 2 :c 3}] + (t/assert= (set (keys v)) #{:a :b :c}) + (t/assert= (transduce (keys) conj! v) (keys v)))) + +(t/deftest test-vals + (let [v {:a 1 :b 2 :c 3}] + (t/assert= (set (vals v)) #{1 2 3}) + (t/assert= (transduce (vals) conj! v) (vals v)))) + + +(t/deftest test-empty + (t/assert= (empty '(1 2 3)) '()) + (t/assert= (empty (list 1 2 3)) '()) + (t/assert= (empty (lazy-seq)) '()) + (t/assert= (empty '()) '()) + (t/assert= (empty [1 2 3]) []) + (t/assert= (empty (make-array 3)) (make-array 0)) + (t/assert= (empty {:a 1, :b 2, :c 3}) {}) + (t/assert= (empty #{1 2 3}) #{})) + + +(t/deftest test-vec + (let [v '(1 2 3 4 5)] + (t/assert= (vec v) [1 2 3 4 5]) + (t/assert= (vec (map inc) v) [2 3 4 5 6]))) + + +(t/deftest test-keep + (let [v [-1 0 1 2 3 4 5]] + (t/assert= (vec (keep pos?) v) [true true true true true]) + (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)))) + +(t/deftest test-assoc + (t/assert= (assoc {} :a 3) {:a 3}) + (t/assert= (assoc {:a 1} :a 3) {:a 3}) + + (t/assert= (assoc [] 0 :ok) [:ok]) + (t/assert= (assoc [1] 0 :ok) [:ok]) + (t/assert= (assoc [1 2 3] 1 :ok) [1 :ok 3])) + +(t/deftest test-get-in + (let [m {:a 1 :b 2 :x {:a 2 :x [1 2 3]}}] + (t/assert= (get-in m [:a]) 1) + (t/assert= (get-in m [:missing]) nil) + (t/assert= (get-in m [:missing] :not-found) :not-found) + (t/assert= (get-in m [:x :x 0] :not-found) 1))) + +(t/deftest test-assoc-in + (t/assert= (assoc-in {:a {:b 2}} [:a :b] 3) {:a {:b 3}}) + (t/assert= (assoc-in {:a [{:b 2}]} [:a 0 :b] 3) {:a [{:b 3}]}) + ; non existing keys create maps (not vectors, even if the keys are integers) + (t/assert= (assoc-in {} [:a :b] 3) {:a {:b 3}}) + (t/assert= (assoc-in {} [:a 0 :b] 3) {:a {0 {:b 3}}})) + +(t/deftest test-update-in + (t/assert= (update-in {} [:a :b] (fnil inc 0)) {:a {:b 1}}) + (t/assert= (update-in {:a {:b 2}} [:a :b] inc) {:a {:b 3}}) + (t/assert= (update-in {:a [{:b 2}]} [:a 0 :b] inc) {:a [{:b 3}]})) + +(t/deftest test-fn? + (t/assert= (fn? inc) true) + (t/assert= (fn? {}) true) + (t/assert= (fn? #(%)) true) + (t/assert= (fn? :foo) true) + (t/assert= (fn? 1) false) + (t/assert= (fn? and) false) + (t/assert= (fn? "foo") false) + (t/assert= (fn? (let [x 8] (fn [y] (+ x y)))) true)) + +(t/deftest test-macro? + (t/assert= (macro? and) true) + (t/assert= (macro? or) true) + (t/assert= (macro? defn) true) + (t/assert= (macro? inc) false) + (t/assert= (macro? 1) false) + (t/assert= (macro? :foo) false) + (t/assert= (macro? "foo") false)) + +(def ^:dynamic *earmuffiness* :low) + +(t/deftest test-binding + (t/assert= *earmuffiness* :low) + (binding [*earmuffiness* :quite-high] + (t/assert= *earmuffiness* :quite-high)) + (t/assert= *earmuffiness* :low)) + +(t/deftest test-every? + (t/assert= (every? even? [2 4 6 8]) true) + (t/assert= (every? odd? [2 4 6 8]) false) + (t/assert= (every? even? [2 3 6 8]) false) + (t/assert= (every? even? []) true) + (t/assert= (every? odd? []) true)) + +(t/deftest test-some + (t/assert= (some even? [2 4 6 8]) true) + (t/assert= (some odd? [2 4 6 8]) false) + (t/assert= (some even? [2 3 6 8]) true) + (t/assert= (some even? [1 3 5 8]) true) + (t/assert= (some even? []) false) + (t/assert= (some odd? [2]) false)) + +(t/deftest test-distinct + (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) + (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) + (t/assert= (vec (distinct) [nil nil nil]) [nil])) + +(t/deftest test-merge + (t/assert= (merge nil) nil) + (t/assert= (merge {}) {}) + (t/assert= (merge {:a 1} nil) {:a 1}) + + (t/assert= (merge {} {:a 1, :b 2}) {:a 1, :b 2}) + (t/assert= (merge {:a 1} {:b 2}) {:a 1, :b 2}) + (t/assert= (merge {} {:a 1} {:b 2}) {:a 1, :b 2}) + + (t/assert= (merge {:a 1} {:a 2, :b 3}) {:a 2, :b 3}) + (t/assert= (merge {:a 1, :b 4} {:a 2} {:a 3}) {:a 3, :b 4})) + +(t/deftest test-merge-with + (t/assert= (merge-with identity nil) nil) + (t/assert= (merge-with identity {}) {}) + + (t/assert= (merge-with identity {} {:a 1, :b 2}) {:a 1, :b 2}) + (t/assert= (merge-with identity {:a 1} {:b 2}) {:a 1, :b 2}) + + (t/assert= (merge-with #(identity %1) {:a 1} {:a 2}) {:a 1}) + (t/assert= (merge-with #(identity %1) {:a 1} {:a 2} {:a 3}) {:a 1}) + (t/assert= (merge-with #(identity %2) {:a 1} {:a 2}) {:a 2}) + + (t/assert= (merge-with + {:a 21} {:a 21}) {:a 42}) + (t/assert= (merge-with + {:a 21} {:a 21, :b 1}) {:a 42, :b 1})) + +(t/deftest test-for + (t/assert= (for [x [1 2 3]] x) [1 2 3]) + (t/assert= (for [x [1 2 3] y [:a :b :c]] [x y]) + [[1 :a] [1 :b] [1 :c] + [2 :a] [2 :b] [2 :c] + [3 :a] [3 :b] [3 :c]])) diff --git a/tests/pixie/test/test-strings.pxi b/tests/pixie/test/test-strings.pxi new file mode 100644 index 00000000..5f8a5540 --- /dev/null +++ b/tests/pixie/test/test-strings.pxi @@ -0,0 +1,126 @@ +(ns pixie.test.test-strings + (require pixie.test :as t) + (require pixie.string :as s)) + +(t/deftest test-starts-with + (let [s "heyhohuh"] + (t/assert= (s/starts-with s "") true) + (t/assert= (s/starts-with s "hey") true) + (t/assert= (s/starts-with s "heyho") true) + (t/assert= (s/starts-with s s) true) + + (t/assert= (s/starts-with s "ho") false) + (t/assert= (s/starts-with s "foo") false))) + +(t/deftest test-ends-with + (let [s "heyhohuh"] + (t/assert= (s/ends-with s "") true) + (t/assert= (s/ends-with s "huh") true) + (t/assert= (s/ends-with s "hohuh") true) + (t/assert= (s/ends-with s s) true) + + (t/assert= (s/ends-with s "hey") false) + (t/assert= (s/ends-with s "foo") false))) + +(t/deftest test-split + (let [s "hey,ho,huh"] + (t/assert= (s/split s ",") ["hey" "ho" "huh"]) + (t/assert= (s/split s "h") ["" "ey," "o," "u" ""]))) + +(t/deftest test-index-of + (let [s "heyhohuh"] + (t/assert= (s/index-of s "hey") 0) + (t/assert= (s/index-of s "ho") 3) + (t/assert= (s/index-of s "foo") -1) + + (t/assert= (s/index-of s "h" 2) 3) + (t/assert= (s/index-of s "h" 4) 5) + (t/assert= (s/index-of s "hey" 0) 0) + (t/assert= (s/index-of s "hey" 1) -1) + + (t/assert= (s/index-of s "h" 0 0) -1) + (t/assert= (s/index-of s "h" 1 2) -1))) + +(t/deftest test-substring + (let [s "heyhohuh"] + (t/assert= (s/substring s 0) s) + (t/assert= (s/substring s 3) (s/substring s 3 (count s))) + (t/assert= (s/substring s 0 0) "") + (t/assert= (s/substring s 0 3) "hey") + (t/assert= (s/substring s 3 5) "ho") + (t/assert= (s/substring s 5 8) "huh") + (t/assert= (s/substring s 3 10000) "hohuh"))) + +(t/deftest test-upper-case + (t/assert= (s/lower-case "") "") + (t/assert= (s/upper-case "hey") "HEY") + (t/assert= (s/upper-case "hEy") "HEY") + (t/assert= (s/upper-case "HEY") "HEY") + (t/assert= (s/upper-case "hey?!") "HEY?!")) + +(t/deftest test-lower-case + (t/assert= (s/lower-case "") "") + (t/assert= (s/lower-case "hey") "hey") + (t/assert= (s/lower-case "hEy") "hey") + (t/assert= (s/lower-case "HEY") "hey") + (t/assert= (s/lower-case "HEY?!") "hey?!")) + +(t/deftest test-capitalize + (t/assert= (s/capitalize "timothy") "Timothy") + (t/assert= (s/capitalize "Timothy") "Timothy")) + +(t/deftest test-trim + (t/assert= (s/trim "") "") + (t/assert= (s/trim " ") "") + (t/assert= (s/trim " hey ") "hey") + (t/assert= (s/trim " h ey ") "h ey")) + +(t/deftest test-triml + (t/assert= (s/triml "") "") + (t/assert= (s/triml " ") "") + (t/assert= (s/triml " hey") "hey") + (t/assert= (s/triml " hey ") "hey ") + (t/assert= (s/triml " h ey ") "h ey ")) + +(t/deftest test-trimr + (t/assert= (s/trimr "") "") + (t/assert= (s/trimr " ") "") + (t/assert= (s/trimr "hey ") "hey") + (t/assert= (s/trimr " hey ") " hey") + (t/assert= (s/trimr " h ey ") " h ey")) + +(t/deftest test-replace + (t/assert= (s/replace "hey,you,there" "," ", ") "hey, you, there") + (t/assert= (s/replace "hey,you,there" "," "") "heyyouthere") + (t/assert= (s/replace "&&&" "&" "&&") "&&&&&&") + (t/assert= (s/replace "oops" "" "WAT") "WAToWAToWATpWATsWAT")) + +(t/deftest test-replace-first + (t/assert= (s/replace-first "hey,you,there" "," ", ") "hey, you,there") + (t/assert= (s/replace-first "hey,you,there" "," "") "heyyou,there") + (t/assert= (s/replace-first "&&&" "&" "&&") "&&&&") + (t/assert= (s/replace-first "oops" "" "WAT") "WAToops")) + +(t/deftest test-join + (t/assert= (s/join []) "") + (t/assert= (s/join [1]) "1") + (t/assert= (s/join [1 2 3]) "123") + + (t/assert= (s/join ", " []) "") + (t/assert= (s/join ", " [1]) "1") + + (t/assert= (s/join ", " [1 2 3]) "1, 2, 3")) + +(t/deftest test-char-literals + (let [s "hey"] + (t/assert= (nth s 0) \h) + (t/assert= (nth s 0) \o150) + (t/assert= (nth s 0) \u0068) + + (t/assert= (nth s 1) \e) + (t/assert= (nth s 1) \o145) + (t/assert= (nth s 1) \u0065) + + (t/assert= (nth s 2) \y) + (t/assert= (nth s 2) \o171) + (t/assert= (nth s 2) \u0079))) diff --git a/tests/pixie/test/utils.pxi b/tests/pixie/test/utils.pxi new file mode 100644 index 00000000..4916736b --- /dev/null +++ b/tests/pixie/test/utils.pxi @@ -0,0 +1,13 @@ +(ns tests.utils) + +;; Here we create a new type which hashes poorly, in fact it's so bad we have a +;; hash space of only 1. +;; All members of WorstHasher return (hash "worst hasher") +;; when hash is called on them. +;; +;; This makes debugging, testing and benchmarking anything based off +;; PersistentHashMap trivial. + +;; X can be any thing you like. +(defrecord WorstHasher [x]) +(extend -hash WorstHasher (fn [self] (hash "worst hasher"))) From 4170a463f65dee577a93ab4eddb4eba24740c0ed Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 09:39:52 -0700 Subject: [PATCH 382/909] io test passes now --- pixie/stdlib.pxi | 5 ++++- tests/pixie/test/test-io.pxi | 4 ++-- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1eb36c7b..24b92405 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -222,6 +222,8 @@ (comment (extend -reduce Array indexed-reduce)) +(extend -reduce Buffer indexed-reduce) + (extend -str Bool (fn [x] (if (identical? x true) @@ -1039,7 +1041,8 @@ Creates new maps if the keys are not present." (let [fn-name (first body) _ (assert (symbol? fn-name) "protocol override must have a name") args (second body) - _ (assert (vector? args) "protocol override must have arguments") + _ (assert (or (vector? args) + (seq? args)) "protocol override must have arguments") self-arg (first args) _ (assert (symbol? self-arg) "protocol override must have at least one `self' argument") field-lets (transduce (comp (map (fn [f] diff --git a/tests/pixie/test/test-io.pxi b/tests/pixie/test/test-io.pxi index f8766a18..5b9d15f3 100644 --- a/tests/pixie/test/test-io.pxi +++ b/tests/pixie/test/test-io.pxi @@ -3,8 +3,8 @@ (require pixie.io :as io)) (t/deftest test-file-reduction - (let [f (io/open-read "test/test-io.txt")] + (let [f (io/open-read "tests/pixie/test/test-io.txt")] (t/assert= (transduce (map identity) count-rf f) - 10))) + 78))) From d60411f3ded5dcb5263b738bfb8e131b9b12e606 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 12 Dec 2014 18:24:52 +0100 Subject: [PATCH 383/909] support converting characters to integers should be useful for reading from files/streams. incidentally, you can now add characters together. --- pixie/stdlib.pxi | 2 ++ pixie/vm/string.py | 14 ++++++++++++-- 2 files changed, 14 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e32e109f..8bfbc0d6 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -726,6 +726,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn float? [v] (instance? Float v)) (defn ratio? [v] (instance? Ratio v)) +(defn char? [v] (instance? Character v)) (defn string? [v] (instance? String v)) (defn keyword? [v] (instance? Keyword v)) @@ -752,6 +753,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (integer? x) x (float? x) (lround (floor x)) (ratio? x) (int (/ (float (numerator x)) (float (denominator x)))) + (char? x) (+ x 0) :else (throw (str "Can't convert a value of type " (type x) " to an Integer")))) (defn last diff --git a/pixie/vm/string.py b/pixie/vm/string.py index f7ea3f4b..ca10194c 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -1,9 +1,9 @@ import pixie.vm.rt as rt -from pixie.vm.object import Object, Type +from pixie.vm.object import Object, Type, affirm from pixie.vm.code import extend, as_var from pixie.vm.primitives import nil, true, false +from pixie.vm.numbers import Integer, _add import pixie.vm.stdlib as proto -import pixie.vm.numbers as numbers import pixie.vm.util as util from rpython.rlib.rarithmetic import intmask, r_uint @@ -89,6 +89,16 @@ def _eq(self, obj): def _hash(self): return rt.wrap(intmask(util.hash_int(r_uint(self.char_val())))) +@extend(_add, Character._type, Integer._type) +def _add(a, b): + assert isinstance(a, Character) and isinstance(b, Integer) + return rt._add(rt.wrap(a.char_val()), b) + +@extend(_add, Character._type, Character._type) +def _add(a, b): + assert isinstance(a, Character) and isinstance(b, Character) + return Character(a.char_val() + b.char_val()) + @extend(proto._name, String) def _name(self): From ec2a65e75a07e55564d1aa65758943e815efd54e Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 12 Dec 2014 18:30:29 +0100 Subject: [PATCH 384/909] support converting integers to characters --- pixie/vm/string.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index ca10194c..81ca34ac 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -89,6 +89,11 @@ def _eq(self, obj): def _hash(self): return rt.wrap(intmask(util.hash_int(r_uint(self.char_val())))) +@as_var("char") +def char(val): + affirm(isinstance(val, Integer), u"First argument must be an Integer") + return Character(val.int_val()) + @extend(_add, Character._type, Integer._type) def _add(a, b): assert isinstance(a, Character) and isinstance(b, Integer) From f952ced08497daf4ea2e2d7f3600d8121ce45ff0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 11:06:03 -0700 Subject: [PATCH 385/909] moved some files again, all tests should pass now --- pixie/io.pxi | 50 +++++++++++++++++++ pixie/stdlib.pxi | 12 ++--- pixie/test.pxi | 1 - pixie/vm/compiler.py | 3 +- pixie/vm/stdlib.py | 15 +++++- .../{test => tests}/collections/test-maps.pxi | 0 .../collections/test-seqables.pxi | 0 .../{test => tests}/collections/test-sets.pxi | 6 +-- .../collections/test-vectors.pxi | 2 +- tests/pixie/{test => tests}/test-arrays.pxi | 0 tests/pixie/{test => tests}/test-bits.pxi | 0 tests/pixie/{test => tests}/test-compiler.pxi | 0 .../pixie/{test => tests}/test-defrecord.pxi | 0 tests/pixie/{test => tests}/test-deftype.pxi | 0 .../{test => tests}/test-destructuring.pxi | 0 tests/pixie/{test => tests}/test-docs.pxi | 0 tests/pixie/{test => tests}/test-ffi.pxi | 0 tests/pixie/{test => tests}/test-fns.pxi | 0 tests/pixie/{test => tests}/test-forms.pxi | 0 tests/pixie/{test => tests}/test-io.pxi | 4 +- tests/pixie/{test => tests}/test-io.txt | 0 tests/pixie/{test => tests}/test-keywords.pxi | 0 tests/pixie/{test => tests}/test-numbers.pxi | 0 tests/pixie/{test => tests}/test-readeval.pxi | 0 tests/pixie/{test => tests}/test-stdlib.pxi | 0 tests/pixie/{test => tests}/test-strings.pxi | 0 tests/pixie/{test => tests}/utils.pxi | 8 +-- 27 files changed, 80 insertions(+), 21 deletions(-) create mode 100644 pixie/io.pxi rename tests/pixie/{test => tests}/collections/test-maps.pxi (100%) rename tests/pixie/{test => tests}/collections/test-seqables.pxi (100%) rename tests/pixie/{test => tests}/collections/test-sets.pxi (95%) rename tests/pixie/{test => tests}/collections/test-vectors.pxi (96%) rename tests/pixie/{test => tests}/test-arrays.pxi (100%) rename tests/pixie/{test => tests}/test-bits.pxi (100%) rename tests/pixie/{test => tests}/test-compiler.pxi (100%) rename tests/pixie/{test => tests}/test-defrecord.pxi (100%) rename tests/pixie/{test => tests}/test-deftype.pxi (100%) rename tests/pixie/{test => tests}/test-destructuring.pxi (100%) rename tests/pixie/{test => tests}/test-docs.pxi (100%) rename tests/pixie/{test => tests}/test-ffi.pxi (100%) rename tests/pixie/{test => tests}/test-fns.pxi (100%) rename tests/pixie/{test => tests}/test-forms.pxi (100%) rename tests/pixie/{test => tests}/test-io.pxi (72%) rename tests/pixie/{test => tests}/test-io.txt (100%) rename tests/pixie/{test => tests}/test-keywords.pxi (100%) rename tests/pixie/{test => tests}/test-numbers.pxi (100%) rename tests/pixie/{test => tests}/test-readeval.pxi (100%) rename tests/pixie/{test => tests}/test-stdlib.pxi (100%) rename tests/pixie/{test => tests}/test-strings.pxi (100%) rename tests/pixie/{test => tests}/utils.pxi (74%) diff --git a/pixie/io.pxi b/pixie/io.pxi new file mode 100644 index 00000000..1f3ef65c --- /dev/null +++ b/pixie/io.pxi @@ -0,0 +1,50 @@ +(ns pixie.io) + +(def fopen (ffi-fn libc "fopen" [String String] VoidP)) +(def fread (ffi-fn libc "fread" [Buffer Integer Integer VoidP] Integer)) +(def fgetc (ffi-fn libc "fgetc" [VoidP] Integer)) +(def fclose (ffi-fn libc "fclose" [VoidP] Integer)) + + +(defprotocol IInputStream + (read [this] "Read a single character") + (read-byte [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) + +(defprotocol IClosable + (close [this] "Closes the stream")) + +(def DEFAULT-BUFFER-SIZE 1024) + +(deftype FileStream [fp] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [read-count (fread buffer 1 len fp)] + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (fgetc buffer)) + IClosable + (close [this] + (fclose buffer)) + IReduce + (-reduce [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc)))))) + + +(defn open-read + {:doc "Open a file for reading, returning a IInputStream" + :added "0.1"} + [filename] + (assert (string? filename) "Filename must be a string") + (->FileStream (fopen filename "r"))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 24b92405..44696299 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -977,11 +977,11 @@ Creates new maps if the keys are not present." nil (throw (str "Assert failed " ~msg))))) -(defn resolve +(defmacro resolve {:doc "Resolve the var associated with the symbol in the current namespace." :added "0.1"} [sym] - (resolve-in (this-ns-name) sym)) + `(resolve-in (this-ns-name) ~sym)) (defmacro binding [bindings & body] (let [bindings (apply hashmap bindings) @@ -995,12 +995,10 @@ Creates new maps if the keys are not present." (pop-binding-frame!) ret)))) -(def foo 42) -(set-dynamic! (resolve 'pixie.stdlib/foo)) - (defmacro require [ns kw as-nm] (assert (= kw :as) "Require expects :as as the second argument") `(do (load-ns (quote ~ns)) + (assert (the-ns (quote ~ns)) (str "Couldn't find the namespace " (quote ~ns) " after loading the file")) (refer-ns (this-ns-name) (the-ns (quote ~ns)) (quote ~as-nm)))) (defmacro ns [nm & body] @@ -1056,7 +1054,9 @@ Creates new maps if the keys are not present." (cond (symbol? body) (cond (= body 'Object) [body (second res) (third res)] - (protocol? @(resolve body)) [@(resolve body) (second res) (conj (third res) body)] + (protocol? @(resolve-in *ns* body)) [@(resolve-in *ns* body) + (second res) + (conj (third res) body)] :else (throw (str "can only extend protocols or Object, not " body))) (seq? body) (let [proto (first res) tbs (second res) pbs (third res)] (if (protocol? proto) diff --git a/pixie/test.pxi b/pixie/test.pxi index f482c64b..e529f873 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -30,7 +30,6 @@ (set! (var *stats*) (atom {:fail 0 :pass 0})) (let [match (or (first args) "") - _ (println (keys @tests)) tests (transduce (comp (filter #(>= (s/index-of (str (key %1)) match) 0)) (map val)) conj diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 4a03bcaf..dd5c40a8 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -729,8 +729,7 @@ def compile_ns(form, ctx): ctx.push_const(nil) def compile_this_ns(form, ctx): - ctx.bytecode.append(code.PUSH_NS) - ctx.add_sp(1) + ctx.push_const(NS_VAR.deref()) def compile_var(form, ctx): form = rt.next(form) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 90432fb6..9af9c49d 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -447,9 +447,19 @@ def ns_map(ns): @as_var("refer-ns") def refer(ns, refer, alias): from pixie.vm.symbol import Symbol + from pixie.vm.string import String + from pixie.vm.code import _ns_registry + + if isinstance(ns, Symbol) or isinstance(ns, String): + ns = _ns_registry.find_or_make(rt.name(ns)) + + if isinstance(refer, Symbol) or isinstance(refer, String): + refer = _ns_registry.find_or_make(rt.name(refer)) affirm(isinstance(ns, code.Namespace), u"First argument must be a namespace") - affirm(isinstance(refer, code.Namespace), u"Second argument must be a namespace") + if not isinstance(refer, code.Namespace): + runtime_error(u"Second argument must be a namespace not a " + refer.type().name()) + affirm(isinstance(alias, Symbol), u"Third argument must be a symbol") ns.add_refer(refer, rt.name(alias)) @@ -572,7 +582,8 @@ def set_dynamic(var): @as_var("set!") def set(var, val): - affirm(isinstance(var, Var), u"Can only set the dynamic value of a var") + if not isinstance(var, Var): + runtime_error(u"Can only set the dynamic value of a var. Not a: " + var.type().name()) var.set_value(val) return var diff --git a/tests/pixie/test/collections/test-maps.pxi b/tests/pixie/tests/collections/test-maps.pxi similarity index 100% rename from tests/pixie/test/collections/test-maps.pxi rename to tests/pixie/tests/collections/test-maps.pxi diff --git a/tests/pixie/test/collections/test-seqables.pxi b/tests/pixie/tests/collections/test-seqables.pxi similarity index 100% rename from tests/pixie/test/collections/test-seqables.pxi rename to tests/pixie/tests/collections/test-seqables.pxi diff --git a/tests/pixie/test/collections/test-sets.pxi b/tests/pixie/tests/collections/test-sets.pxi similarity index 95% rename from tests/pixie/test/collections/test-sets.pxi rename to tests/pixie/tests/collections/test-sets.pxi index 62652da8..376f95bf 100644 --- a/tests/pixie/test/collections/test-sets.pxi +++ b/tests/pixie/tests/collections/test-sets.pxi @@ -1,8 +1,8 @@ (ns collections.test-sets (require pixie.test :as t) - (require tests.utils :as u)) + (require pixie.tests.utils :as u)) -(def worst-hashers (vec (map u/->WorstHasher) +(def worst-hashers (vec (map u/->WorstHasher) (range 100))) (t/deftest test-count @@ -10,7 +10,7 @@ (t/assert= (count (set [1 2 3])) 3) (t/assert= (count (set [1 1 2 1])) 2) (t/assert= (count (set worst-hashers)) 100)) - + (t/deftest test-contains (let [s #{1 2 3} c [1 2 3] diff --git a/tests/pixie/test/collections/test-vectors.pxi b/tests/pixie/tests/collections/test-vectors.pxi similarity index 96% rename from tests/pixie/test/collections/test-vectors.pxi rename to tests/pixie/tests/collections/test-vectors.pxi index f69432fe..84927140 100644 --- a/tests/pixie/test/collections/test-vectors.pxi +++ b/tests/pixie/tests/collections/test-vectors.pxi @@ -1,4 +1,4 @@ -(ns collections.test-vectors +(ns pixie.tests.collections.test-vectors (require pixie.test :as t)) (def MAX-SIZE 1064) diff --git a/tests/pixie/test/test-arrays.pxi b/tests/pixie/tests/test-arrays.pxi similarity index 100% rename from tests/pixie/test/test-arrays.pxi rename to tests/pixie/tests/test-arrays.pxi diff --git a/tests/pixie/test/test-bits.pxi b/tests/pixie/tests/test-bits.pxi similarity index 100% rename from tests/pixie/test/test-bits.pxi rename to tests/pixie/tests/test-bits.pxi diff --git a/tests/pixie/test/test-compiler.pxi b/tests/pixie/tests/test-compiler.pxi similarity index 100% rename from tests/pixie/test/test-compiler.pxi rename to tests/pixie/tests/test-compiler.pxi diff --git a/tests/pixie/test/test-defrecord.pxi b/tests/pixie/tests/test-defrecord.pxi similarity index 100% rename from tests/pixie/test/test-defrecord.pxi rename to tests/pixie/tests/test-defrecord.pxi diff --git a/tests/pixie/test/test-deftype.pxi b/tests/pixie/tests/test-deftype.pxi similarity index 100% rename from tests/pixie/test/test-deftype.pxi rename to tests/pixie/tests/test-deftype.pxi diff --git a/tests/pixie/test/test-destructuring.pxi b/tests/pixie/tests/test-destructuring.pxi similarity index 100% rename from tests/pixie/test/test-destructuring.pxi rename to tests/pixie/tests/test-destructuring.pxi diff --git a/tests/pixie/test/test-docs.pxi b/tests/pixie/tests/test-docs.pxi similarity index 100% rename from tests/pixie/test/test-docs.pxi rename to tests/pixie/tests/test-docs.pxi diff --git a/tests/pixie/test/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi similarity index 100% rename from tests/pixie/test/test-ffi.pxi rename to tests/pixie/tests/test-ffi.pxi diff --git a/tests/pixie/test/test-fns.pxi b/tests/pixie/tests/test-fns.pxi similarity index 100% rename from tests/pixie/test/test-fns.pxi rename to tests/pixie/tests/test-fns.pxi diff --git a/tests/pixie/test/test-forms.pxi b/tests/pixie/tests/test-forms.pxi similarity index 100% rename from tests/pixie/test/test-forms.pxi rename to tests/pixie/tests/test-forms.pxi diff --git a/tests/pixie/test/test-io.pxi b/tests/pixie/tests/test-io.pxi similarity index 72% rename from tests/pixie/test/test-io.pxi rename to tests/pixie/tests/test-io.pxi index 5b9d15f3..adebd4f8 100644 --- a/tests/pixie/test/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -1,9 +1,9 @@ -(ns pixie.test.test-io +(ns pixie.tests.test-io (require pixie.test :as t) (require pixie.io :as io)) (t/deftest test-file-reduction - (let [f (io/open-read "tests/pixie/test/test-io.txt")] + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] (t/assert= (transduce (map identity) count-rf f) diff --git a/tests/pixie/test/test-io.txt b/tests/pixie/tests/test-io.txt similarity index 100% rename from tests/pixie/test/test-io.txt rename to tests/pixie/tests/test-io.txt diff --git a/tests/pixie/test/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi similarity index 100% rename from tests/pixie/test/test-keywords.pxi rename to tests/pixie/tests/test-keywords.pxi diff --git a/tests/pixie/test/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi similarity index 100% rename from tests/pixie/test/test-numbers.pxi rename to tests/pixie/tests/test-numbers.pxi diff --git a/tests/pixie/test/test-readeval.pxi b/tests/pixie/tests/test-readeval.pxi similarity index 100% rename from tests/pixie/test/test-readeval.pxi rename to tests/pixie/tests/test-readeval.pxi diff --git a/tests/pixie/test/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi similarity index 100% rename from tests/pixie/test/test-stdlib.pxi rename to tests/pixie/tests/test-stdlib.pxi diff --git a/tests/pixie/test/test-strings.pxi b/tests/pixie/tests/test-strings.pxi similarity index 100% rename from tests/pixie/test/test-strings.pxi rename to tests/pixie/tests/test-strings.pxi diff --git a/tests/pixie/test/utils.pxi b/tests/pixie/tests/utils.pxi similarity index 74% rename from tests/pixie/test/utils.pxi rename to tests/pixie/tests/utils.pxi index 4916736b..b7d6fce6 100644 --- a/tests/pixie/test/utils.pxi +++ b/tests/pixie/tests/utils.pxi @@ -1,11 +1,11 @@ -(ns tests.utils) +(ns pixie.tests.utils) ;; Here we create a new type which hashes poorly, in fact it's so bad we have a -;; hash space of only 1. -;; All members of WorstHasher return (hash "worst hasher") +;; hash space of only 1. +;; All members of WorstHasher return (hash "worst hasher") ;; when hash is called on them. ;; -;; This makes debugging, testing and benchmarking anything based off +;; This makes debugging, testing and benchmarking anything based off ;; PersistentHashMap trivial. ;; X can be any thing you like. From 2b0df581f3d98b5de2d0fd1c92913470030bc53f Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 12 Dec 2014 19:13:09 +0100 Subject: [PATCH 386/909] print char escapes in -repr on strings --- pixie/vm/string.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index 81ca34ac..faa2c442 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -24,8 +24,23 @@ def _str(x): @extend(proto._repr, String) def _repr(self): - return rt.wrap(u"\"" + self._str + u"\"") - + res = u"" + for c in self._str: + if c == "\"": + res += u"\\\"" + elif c == "\n": + res += u"\\n" + elif c == "\t": + res += u"\\t" + elif c == "\b": + res += u"\\b" + elif c == "\f": + res += u"\\f" + elif c == "\r": + res += u"\\r" + else: + res += c + return rt.wrap(u"\"" + res + u"\"") @extend(proto._count, String) def _count(self): From 1ef8ab6d77120ecca5b5b835147628ab52111991 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Fri, 12 Dec 2014 19:55:56 +0100 Subject: [PATCH 387/909] initial support for printing and reading utf8 --- pixie/vm/libs/ffi.py | 5 ++++- pixie/vm/libs/readline.py | 8 +++++++- target.py | 3 ++- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index def084bb..9962fe47 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -9,6 +9,7 @@ from pixie.vm.numbers import Integer, Float from pixie.vm.string import String from rpython.rlib import clibffi +from rpython.rlib.runicode import unicode_encode_utf_8 from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import intmask @@ -113,7 +114,9 @@ def set_native_value(ptr, val, tp): return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.DOUBLE)) if tp is String._type: pnt = rffi.cast(rffi.CCHARPP, ptr) - pnt[0] = rffi.str2charp(str(rt.name(val))) + s = rt.name(val) + s = unicode_encode_utf_8(s, len(s), 'strict') + pnt[0] = rffi.str2charp(s) return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) if tp is Buffer._type: pnt = rffi.cast(rffi.CCHARPP, ptr) diff --git a/pixie/vm/libs/readline.py b/pixie/vm/libs/readline.py index 5299f47f..10684fe1 100644 --- a/pixie/vm/libs/readline.py +++ b/pixie/vm/libs/readline.py @@ -1,5 +1,6 @@ import py +from rpython.rlib.runicode import str_decode_utf_8 from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.translator import cdir @@ -22,4 +23,9 @@ def _readline(prompt): if result == lltype.nullptr(rffi.CCHARP.TO): return u"" else: - return unicode(rffi.charp2str(result)) + u"\n" + s = rffi.charp2str(result) + try: + return unicode(s) + u"\n" + except UnicodeDecodeError: + res, _ = str_decode_utf_8(s, len(s), 'strict') + return res + u"\n" diff --git a/target.py b/target.py index 9a9b92f1..a6f05391 100644 --- a/target.py +++ b/target.py @@ -16,6 +16,7 @@ import os.path as path import rpython.rlib.rpath as rpath import rpython.rlib.rpath as rposix +import rpython.rlib.runicode as runicode from rpython.rlib.objectmodel import we_are_translated class DebugIFace(JitHookInterface): @@ -90,7 +91,7 @@ def inner_invoke(self, args): break val = rt._repr(val) assert isinstance(val, String), "str should always return a string" - print val._str + print runicode.unicode_encode_utf_8(val._str, len(val._str), 'strict') def set_recent_vars(self, val): if rt.eq(val, STAR_1.deref()): From 53131ed1a347f62072f110a2776cc4a02666471c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 13:26:11 -0700 Subject: [PATCH 388/909] implemented preloading --- Makefile | 6 ++++++ target.py | 8 ++++++++ target_preload.py | 27 +++++++++++++++++++++++++++ 3 files changed, 41 insertions(+) create mode 100644 target_preload.py diff --git a/Makefile b/Makefile index 6deac56f..d011ab32 100644 --- a/Makefile +++ b/Makefile @@ -18,6 +18,12 @@ build_with_jit: fetch_externals build_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target.py +build_preload_with_jit: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py + +build_preload_no_jit: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target_preload.py + fetch_externals: $(EXTERNALS)/pypy $(EXTERNALS)/pypy: diff --git a/target.py b/target.py index e0ea5f79..438a5a22 100644 --- a/target.py +++ b/target.py @@ -155,8 +155,13 @@ def inner_invoke(self, args): interpret(compile(read(StringReader(unicode(self._expr)), True))) +stdlib_loaded = False + @wrap_fn def run_load_stdlib(): + global stdlib_loaded + if stdlib_loaded: + return import pixie.vm.compiler as compiler import pixie.vm.reader as reader f = open(rpath.rjoin(str(load_path.deref()._str), "pixie/stdlib.pxi")) @@ -181,6 +186,8 @@ def run_load_stdlib(): if not we_are_translated(): print "done" + stdlib_loaded = True + def load_stdlib(): run_load_stdlib.invoke([]) @@ -326,6 +333,7 @@ def target(*args): rt.__config__ = args[0].config + print "ARG INFO: ", args return entry_point, None diff --git a/target_preload.py b/target_preload.py new file mode 100644 index 00000000..343c9b69 --- /dev/null +++ b/target_preload.py @@ -0,0 +1,27 @@ +from target import entry_point, load_stdlib, init_load_path, LOAD_PATHS, load_path +from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR + +import pixie.vm.rt as rt +rt.init() + +load_path.set_root(rt.wrap(u"./")) +LOAD_PATHS.set_root(EMPTY_VECTOR.conj(rt.wrap(u"./"))) +load_stdlib() + +def target(*args): + import pixie.vm.rt as rt + driver = args[0] + driver.exe_name = "pixie-vm" + rt.__config__ = args[0].config + + + + + + return entry_point, None + +import rpython.config.translationoption +print rpython.config.translationoption.get_combined_translation_config() + +if __name__ == "__main__": + entry_point(sys.argv) \ No newline at end of file From 91f437737920a3af884fb324d149b2ad1503008d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 13:26:42 -0700 Subject: [PATCH 389/909] updated travis build script --- make-with-jit | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/make-with-jit b/make-with-jit index c5160670..26da38e4 100755 --- a/make-with-jit +++ b/make-with-jit @@ -1 +1 @@ -python ../externals/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target.py +python ../externals/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py From 72954a5872ea229d862f8195ce71a407ccc8e23c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 13:48:17 -0700 Subject: [PATCH 390/909] fix the large log issue --- make-with-jit | 2 +- target.py | 18 +++++++++++++++--- 2 files changed, 16 insertions(+), 4 deletions(-) diff --git a/make-with-jit b/make-with-jit index 26da38e4..59c9057e 100755 --- a/make-with-jit +++ b/make-with-jit @@ -1 +1 @@ -python ../externals/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py +python ../externals/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' diff --git a/target.py b/target.py index 438a5a22..9a9b92f1 100644 --- a/target.py +++ b/target.py @@ -155,12 +155,24 @@ def inner_invoke(self, args): interpret(compile(read(StringReader(unicode(self._expr)), True))) -stdlib_loaded = False + +class IsPreloadFlag(object): + def __init__(self): + self._is_true = False + + def is_true(self): + return self._is_true + + def set_true(self): + self._is_true = True + + +stdlib_loaded = IsPreloadFlag() @wrap_fn def run_load_stdlib(): global stdlib_loaded - if stdlib_loaded: + if stdlib_loaded.is_true(): return import pixie.vm.compiler as compiler import pixie.vm.reader as reader @@ -186,7 +198,7 @@ def run_load_stdlib(): if not we_are_translated(): print "done" - stdlib_loaded = True + stdlib_loaded.set_true() def load_stdlib(): run_load_stdlib.invoke([]) From 4a208fd0976dd4250929b00e4d3e1970cc8fd1dd Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 12 Dec 2014 14:38:33 -0700 Subject: [PATCH 391/909] add preloading of all .pxi files --- pixie/preload.pxi | 21 +++++++++++++++++++++ pixie/stdlib.pxi | 2 ++ target_preload.py | 9 +++++++-- 3 files changed, 30 insertions(+), 2 deletions(-) create mode 100644 pixie/preload.pxi diff --git a/pixie/preload.pxi b/pixie/preload.pxi new file mode 100644 index 00000000..55abe180 --- /dev/null +++ b/pixie/preload.pxi @@ -0,0 +1,21 @@ +(ns pixie.preload + (require pixie.string :as s)) + +(defn load-all-pxi-files + "Only used by preloading, loads all found .pxi files" + [] + (println "Looking for pxi files...") + (foreach [path @load-paths] + (println "Looking for files in:" path) + (foreach [desc (pixie.path/file-list path)] + (if (= (nth desc 1) :file) + (let [filename (nth desc 2)] + (if (and (pixie.string/ends-with filename ".pxi") + (not (= filename "preload.pxi")) + (not (= filename "stdlib.pxi"))) + (if (pixie.string/starts-with (nth desc 0) "./pixie") + (let [fullpath (str (nth desc 0) "/" filename)] + (println "Loading" fullpath) + (load-file fullpath))))))))) + +(load-all-pxi-files) \ No newline at end of file diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 44696299..e32e109f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1910,3 +1910,5 @@ Expands to calls to `extend-type`." ([] 0) ([result] result) ([result _] (inc result))) + + diff --git a/target_preload.py b/target_preload.py index 343c9b69..3bb0fbe8 100644 --- a/target_preload.py +++ b/target_preload.py @@ -1,13 +1,18 @@ -from target import entry_point, load_stdlib, init_load_path, LOAD_PATHS, load_path +from target import entry_point, load_stdlib, init_load_path, LOAD_PATHS, load_path, BatchModeFn +from pixie.vm.atom import Atom from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR +from pixie.vm.symbol import symbol +from pixie.vm.code import intern_var import pixie.vm.rt as rt rt.init() load_path.set_root(rt.wrap(u"./")) -LOAD_PATHS.set_root(EMPTY_VECTOR.conj(rt.wrap(u"./"))) +LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(u"./")))) load_stdlib() +BatchModeFn(["pixie/preload.pxi"]).invoke([]) + def target(*args): import pixie.vm.rt as rt driver = args[0] From c583ba63aaac6bebc80a7fef5a45d40a0efa7931 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 13 Dec 2014 13:19:47 +0100 Subject: [PATCH 392/909] fix str on characters MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit e.g. `(= (str \a \b \c \u269b) "abc⚛")`. --- pixie/vm/string.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index faa2c442..017f0d88 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -78,10 +78,7 @@ def char_val(self): @extend(proto._str, Character) def _str(self): assert isinstance(self, Character) - cv = self.char_val() - if cv < 128: - return rt.wrap(u"\\"+unicode(chr(cv))) - return rt.wrap(u"\\u"+unicode(str(cv))) + return rt.wrap(u"" + unichr(self.char_val())) @extend(proto._repr, Character) def _repr(self): From 77ab01fdad77163a8dcb46db6ed862c6303a8abf Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 13 Dec 2014 13:48:07 +0100 Subject: [PATCH 393/909] introduce helpers for encoding from and to utf8 --- pixie/vm/libs/ffi.py | 6 ++---- pixie/vm/libs/readline.py | 10 +++------- pixie/vm/util.py | 12 ++++++++++++ target.py | 4 ++-- 4 files changed, 19 insertions(+), 13 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 9962fe47..f2adac5b 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -8,8 +8,8 @@ from pixie.vm.primitives import nil from pixie.vm.numbers import Integer, Float from pixie.vm.string import String +from pixie.vm.util import unicode_to_utf8 from rpython.rlib import clibffi -from rpython.rlib.runicode import unicode_encode_utf_8 from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import intmask @@ -114,9 +114,7 @@ def set_native_value(ptr, val, tp): return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.DOUBLE)) if tp is String._type: pnt = rffi.cast(rffi.CCHARPP, ptr) - s = rt.name(val) - s = unicode_encode_utf_8(s, len(s), 'strict') - pnt[0] = rffi.str2charp(s) + pnt[0] = rffi.str2charp(unicode_to_utf8(rt.name(val))) return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) if tp is Buffer._type: pnt = rffi.cast(rffi.CCHARPP, ptr) diff --git a/pixie/vm/libs/readline.py b/pixie/vm/libs/readline.py index 10684fe1..a8420098 100644 --- a/pixie/vm/libs/readline.py +++ b/pixie/vm/libs/readline.py @@ -1,6 +1,7 @@ import py -from rpython.rlib.runicode import str_decode_utf_8 +from pixie.vm.util import unicode_from_utf8 + from rpython.rtyper.lltypesystem import lltype, rffi from rpython.rtyper.lltypesystem.lloperation import llop from rpython.translator import cdir @@ -23,9 +24,4 @@ def _readline(prompt): if result == lltype.nullptr(rffi.CCHARP.TO): return u"" else: - s = rffi.charp2str(result) - try: - return unicode(s) + u"\n" - except UnicodeDecodeError: - res, _ = str_decode_utf_8(s, len(s), 'strict') - return res + u"\n" + return unicode_from_utf8(rffi.charp2str(result)) + u"\n" diff --git a/pixie/vm/util.py b/pixie/vm/util.py index 4d52a09b..da4314a2 100644 --- a/pixie/vm/util.py +++ b/pixie/vm/util.py @@ -1,4 +1,5 @@ from rpython.rlib.rarithmetic import r_uint, LONG_BIT, intmask, LONG_MASK +from rpython.rlib.runicode import str_decode_utf_8, unicode_encode_utf_8 from pixie.vm.object import affirm seed = 0 @@ -123,3 +124,14 @@ def finish_hash_state(acc): def _hash_int(acc): return rt.wrap(intmask(hash_int(acc.r_uint_val()))) + +# unicode utils + +def unicode_from_utf8(s): + """Converts a `str` value to a `unicode` value assuming it's encoded in UTF8.""" + res, _ = str_decode_utf_8(s, len(s), 'strict') + return res + +def unicode_to_utf8(s): + """Converts a `unicode` value to a UTF8 encoded `str` value.""" + return unicode_encode_utf_8(s, len(s), 'strict') diff --git a/target.py b/target.py index a6f05391..b91674b7 100644 --- a/target.py +++ b/target.py @@ -11,12 +11,12 @@ from pixie.vm.primitives import nil from pixie.vm.atom import Atom from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR +from pixie.vm.util import unicode_to_utf8 import sys import os import os.path as path import rpython.rlib.rpath as rpath import rpython.rlib.rpath as rposix -import rpython.rlib.runicode as runicode from rpython.rlib.objectmodel import we_are_translated class DebugIFace(JitHookInterface): @@ -91,7 +91,7 @@ def inner_invoke(self, args): break val = rt._repr(val) assert isinstance(val, String), "str should always return a string" - print runicode.unicode_encode_utf_8(val._str, len(val._str), 'strict') + print unicode_to_utf8(val._str) def set_recent_vars(self, val): if rt.eq(val, STAR_1.deref()): From c98d5cd47b6cb17e039ec3331e6ea9fb1e47eb94 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 13 Dec 2014 13:57:00 +0100 Subject: [PATCH 394/909] support unicode in source files --- pixie/vm/stdlib.py | 3 ++- target.py | 6 +++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 9af9c49d..4f5b3c60 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -388,6 +388,7 @@ def load_ns(filename): @as_var("load-file") def load_file(filename): from pixie.vm.string import String + from pixie.vm.util import unicode_from_utf8 import pixie.vm.reader as reader import os.path as path @@ -404,7 +405,7 @@ def load_file(filename): if newline_pos > 0: data = data[newline_pos:] - rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode(data)), unicode(filename))) + rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) return nil @as_var("load-reader") diff --git a/target.py b/target.py index b91674b7..afc52f21 100644 --- a/target.py +++ b/target.py @@ -11,7 +11,7 @@ from pixie.vm.primitives import nil from pixie.vm.atom import Atom from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR -from pixie.vm.util import unicode_to_utf8 +from pixie.vm.util import unicode_from_utf8, unicode_to_utf8 import sys import os import os.path as path @@ -139,7 +139,7 @@ def inner_invoke(self, args): if newline_pos > 0: data = data[newline_pos:] - rt.load_reader(StringReader(unicode(data))) + rt.load_reader(StringReader(unicode_from_utf8(data))) except WrappedException as ex: print "Error: ", ex._ex.__repr__() os._exit(1) @@ -154,7 +154,7 @@ def inner_invoke(self, args): with with_ns(u"user"): NS_VAR.deref().include_stdlib() - interpret(compile(read(StringReader(unicode(self._expr)), True))) + interpret(compile(read(StringReader(unicode_from_utf8(self._expr)), True))) class IsPreloadFlag(object): From 0de392c27dcd4b618d06a8535b80311d8ee502a8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 13 Dec 2014 14:57:21 +0100 Subject: [PATCH 395/909] fix -repr on characters outside the ascii range --- pixie/vm/string.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index 017f0d88..8aa04a17 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -86,7 +86,8 @@ def _repr(self): cv = self.char_val() if cv < 128: return rt.wrap(u"\\"+unicode(chr(cv))) - return rt.wrap(u"\\u"+unicode(str(cv))) + hexv = rt.name(rt.bit_str(rt.wrap(self.char_val()), rt.wrap(4))) + return rt.wrap(u"\\u" + u"0" * (4 - len(hexv)) + hexv) @extend(proto._eq, Character) def _eq(self, obj): From 01b734c12a9220ec518e4cc6d3d0c4011c084f56 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 14 Dec 2014 14:16:28 +0100 Subject: [PATCH 396/909] add missing tests --- tests/pixie/tests/test-stdlib.pxi | 6 ++++++ tests/pixie/tests/test-strings.pxi | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 4f639268..3d72f43d 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -14,6 +14,8 @@ (t/assert= (str nil) "nil") (t/assert= (str true) "true") (t/assert= (str false) "false") + (t/assert= (str \a) "a") + (t/assert= (str \u269b) "⚛") (t/assert= (str "hey") "hey") (t/assert= (str :hey) ":hey") (t/assert= (str 'hey) "hey") @@ -32,6 +34,10 @@ (t/assert= (-repr nil) "nil") (t/assert= (-repr true) "true") (t/assert= (-repr false) "false") + (t/assert= (-repr \a) "\\a") + (t/assert= (-repr \u00fa) "\\u00fa") + (t/assert= (-repr \u0fab) "\\u0fab") + (t/assert= (-repr \u269b) "\\u269b") (t/assert= (-repr "hey") "\"hey\"") (t/assert= (-repr :hey) ":hey") (t/assert= (-repr 'hey) "hey") diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 5f8a5540..2f88965b 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -124,3 +124,13 @@ (t/assert= (nth s 2) \y) (t/assert= (nth s 2) \o171) (t/assert= (nth s 2) \u0079))) + +(t/deftest test-char-conversions + (t/assert= (int \a) 97) + (t/assert= (char 97) \a) + + (t/assert= (int \u269b) 0x269b) + (t/assert= (char 0x269b) \u269b)) + +(t/deftest test-unicode + (t/assert= "hâllo" "hâllo")) From 85afd59a70c23d9124cdd9f692b69888d3df5bfd Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Sun, 14 Dec 2014 10:37:46 -0600 Subject: [PATCH 397/909] PEP8 whitespace changes, remove unused imports --- pixie/vm/code.py | 55 +++++++++++++++++++++++++++++------------------- 1 file changed, 33 insertions(+), 22 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 235092ef..62ca0cbe 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -1,9 +1,9 @@ py_object = object import pixie.vm.object as object from pixie.vm.object import affirm -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import nil, false from rpython.rlib.rarithmetic import r_uint -from rpython.rlib.jit import elidable, elidable_promote, promote +from rpython.rlib.jit import elidable_promote, promote import rpython.rlib.jit as jit import pixie.vm.rt as rt @@ -48,6 +48,7 @@ def resize_list(lst, new_size): i += 1 return new_list + @jit.unroll_safe def list_copy(from_lst, from_loc, to_list, to_loc, count): from_loc = r_uint(from_loc) @@ -56,10 +57,11 @@ def list_copy(from_lst, from_loc, to_list, to_loc, count): i = r_uint(0) while i < count: - to_list[to_loc + i] = from_lst[from_loc+i] + to_list[to_loc + i] = from_lst[from_loc + i] i += 1 return to_list + @jit.unroll_safe def slice_to_end(from_list, start_pos): start_pos = r_uint(start_pos) @@ -68,12 +70,14 @@ def slice_to_end(from_list, start_pos): list_copy(from_list, start_pos, new_lst, 0, items_to_copy) return new_lst + @jit.unroll_safe def slice_from_start(from_list, count, extra=r_uint(0)): new_lst = [None] * (count + extra) list_copy(from_list, 0, new_lst, 0, count) return new_lst + # class TailCall(object.Object): # _type = object.Type("TailCall") # __immutable_fields_ = ["_f", "_args"] @@ -159,8 +163,6 @@ def invoke_with(self, args, self_fn): return self.get_fn(len(args)).invoke_with(args, self_fn) - - class NativeFn(BaseCode): """Wrapper for a native function""" _type = object.Type(u"pixie.stdlib.NativeFn") @@ -231,7 +233,6 @@ def get_base_code(self): return self - class VariadicCode(BaseCode): __immutable_fields__ = ["_required_arity", "_code", "_meta"] _type = object.Type(u"pixie.stdlib.VariadicCode") @@ -267,9 +268,11 @@ def invoke_with(self, args, self_fn): return self._code.invoke_with(start, self_fn) affirm(False, u"Got " + unicode(str(argc)) + u" arg(s) need at least " + unicode(str(self._required_arity))) + class Closure(BaseCode): _type = object.Type(u"pixie.stdlib.Closure") __immutable_fields__ = ["_closed_overs[*]", "_code", "_meta"] + def type(self): return Closure._type @@ -314,6 +317,7 @@ def get_base_code(self): def get_debug_points(self): return self._code.get_debug_points() + class Undefined(object.Object): _type = object.Type(u"pixie.stdlib.Undefined") @@ -322,6 +326,7 @@ def type(self): undefined = Undefined() + class DynamicVars(py_object): def __init__(self): self._vars = [{}] @@ -340,6 +345,7 @@ def set_var_value(self, var, val): _dynamic_vars = DynamicVars() + class Var(BaseCode): _type = object.Type(u"pixie.stdlib.Var") _immutable_fields_ = ["_rev?"] @@ -365,7 +371,6 @@ def set_value(self, val): _dynamic_vars.set_var_value(self, val) return self - def set_dynamic(self): self._dynamic = True self._rev += 1 @@ -381,7 +386,6 @@ def is_dynamic(self, rev): def get_root(self, rev): return self._root - def deref(self): if self.is_dynamic(self._rev): return self.get_dynamic_value() @@ -399,9 +403,10 @@ def invoke_with(self, args, this_fn): def invoke(self, args): return self.deref().invoke(args) + class bindings(py_object): def __init__(self, *args): - self._args = list(args) + self._args = list(args) def __enter__(self): _dynamic_vars.push_binding_frame() @@ -493,11 +498,10 @@ def resolve(self, s, use_refers=True): return None return var - - def get(self, name, default): return self._registry.get(name, default) + class NamespaceRegistry(py_object): def __init__(self): self._registry = {} @@ -510,7 +514,6 @@ def find_or_make(self, name): self._registry[name] = v return v - def get(self, name, default): return self._registry.get(name, default) @@ -524,6 +527,7 @@ def intern_var(ns, name=None): return _ns_registry.find_or_make(ns).intern_or_make(name) + def get_var_if_defined(ns, name, els=None): w_ns = _ns_registry.get(ns, None) if w_ns is None: @@ -531,13 +535,11 @@ def get_var_if_defined(ns, name, els=None): return w_ns.get(name, els) - class DefaultProtocolFn(NativeFn): def __init__(self, pfn): self._pfn = pfn def invoke(self, args): - from pixie.vm.string import String tp = args[0].type()._name affirm(False, u"No override for " + tp + u" on " + self._pfn._name + u" in protocol " + self._pfn._protocol._name) @@ -546,6 +548,7 @@ class Protocol(object.Object): _type = object.Type(u"pixie.stdlib.Protocol") __immutable_fields__ = ["_rev?"] + def type(self): return Protocol._type @@ -555,7 +558,6 @@ def __init__(self, name): self._satisfies = {} self._rev = 0 - def add_method(self, pfn): self._polyfns[pfn] = pfn @@ -573,10 +575,12 @@ def satisfies(self, tp): class PolymorphicFn(BaseCode): _type = object.Type(u"pixie.stdlib.PolymorphicFn") + def type(self): return PolymorphicFn._type __immutable_fields__ = ["_rev?"] + def __init__(self, name, protocol): BaseCode.__init__(self) self._name = name @@ -617,7 +621,6 @@ def _find_parent_fn(self, tp): return self._default_fn - def set_default_fn(self, fn): self._default_fn = fn self._rev += 1 @@ -642,6 +645,7 @@ def invoke(self, args): ex._ex._trace.append(object.PolymorphicCodeInfo(self._name, args[0].type())) raise + class DoublePolymorphicFn(BaseCode): """A function that is polymorphic on the first two arguments""" _type = object.Type(u"pixie.stdlib.DoublePolymorphicFn") @@ -650,6 +654,7 @@ def type(self): return DefaultProtocolFn._type __immutable_fields__ = ["_rev?"] + def __init__(self, name, protocol): BaseCode.__init__(self) self._name = name @@ -688,7 +693,6 @@ def invoke(self, args): return fn.invoke(args) - # class ElidableFn(object.Object): # _type = object.Type(u"pixie.stdlib.ElidableFn") # __immutable_fields__ = ["_boxed_fn"] @@ -722,10 +726,14 @@ def invoke(self, args): # return self._elidable_invoke_2(fn, args[0].promote(), args[1].promote()).promote() # affirm(False, u"Too many args to Elidable Fn") + def munge(s): return s.replace("-", "_").replace("?", "_QMARK_").replace("!", "_BANG_") + import inspect + + def defprotocol(ns, name, methods): """Define a protocol in the given namespace with the given name and methods, vars will be created in the namespace for the protocol and methods. This function will dump @@ -734,14 +742,15 @@ def defprotocol(ns, name, methods): name = unicode(name) methods = map(unicode, methods) gbls = inspect.currentframe().f_back.f_globals - proto = Protocol(name) + proto = Protocol(name) intern_var(ns, name).set_root(proto) gbls[munge(name)] = proto for method in methods: - poly = PolymorphicFn(method, proto) + poly = PolymorphicFn(method, proto) intern_var(ns, method).set_root(poly) gbls[munge(method)] = poly + def assert_type(x, tp): affirm(isinstance(x, tp), u"Fatal Error, this should never happen") return x @@ -749,13 +758,15 @@ def assert_type(x, tp): ## PYTHON FLAGS CO_VARARGS = 0x4 + + def wrap_fn(fn, tp=object.Object): """Converts a native Python function into a pixie function.""" def as_native_fn(f): - return type("W"+fn.__name__, (NativeFn,), {"inner_invoke": f})() + return type("W" + fn.__name__, (NativeFn,), {"inner_invoke": f})() def as_variadic_fn(f): - return type("W"+fn.__name__[:len("__args")], (NativeFn,), {"inner_invoke": f})() + return type("W" + fn.__name__[:len("__args")], (NativeFn,), {"inner_invoke": f})() code = fn.func_code if fn.__name__.endswith("__args"): @@ -841,7 +852,6 @@ def extend_inner(fn): return extend_inner - def as_var(ns, name=None): """Locates a var with the given name (defaulting to the namespace pixie.stdlib), sets the root to the decorated function. If the function is not an instance of BaseCode it will @@ -854,6 +864,7 @@ def as_var(ns, name=None): ns = ns if isinstance(ns, unicode) else unicode(ns) var = intern_var(ns, name) + def with_fn(fn): fn.__real_name__ = name if not isinstance(fn, object.Object): From 14d9e2921af591f02644a2fb521656bcbfabdafc Mon Sep 17 00:00:00 2001 From: John Walker Date: Sun, 14 Dec 2014 10:30:05 -0800 Subject: [PATCH 398/909] readline -> libedit --- pixie/stdlib.pxi | 4 ++-- pixie/vm/libs/{readline.py => libedit.py} | 4 ++-- pixie/vm/reader.py | 2 +- pixie/vm/test/test_compile.py | 2 +- 4 files changed, 6 insertions(+), 6 deletions(-) rename pixie/vm/libs/{readline.py => libedit.py} (91%) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8bfbc0d6..c7063bde 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -7,8 +7,8 @@ (def printf (ffi-fn libc "printf" [String] Integer)) (def getenv (ffi-fn libc "getenv" [String] String)) - (def libreadline (ffi-library (str "libreadline." pixie.platform/so-ext))) - (def readline (ffi-fn libreadline "readline" [String] String)) + (def libedit (ffi-library (str "libedit." pixie.platform/so-ext))) + (def readline (ffi-fn libedit "readline" [String] String)) (def rand (ffi-fn libc "rand" [Integer] Integer)) (def srand (ffi-fn libc "srand" [Integer] Integer)) (def fopen (ffi-fn libc "fopen" [String String] VoidP)) diff --git a/pixie/vm/libs/readline.py b/pixie/vm/libs/libedit.py similarity index 91% rename from pixie/vm/libs/readline.py rename to pixie/vm/libs/libedit.py index a8420098..c32580f2 100644 --- a/pixie/vm/libs/readline.py +++ b/pixie/vm/libs/libedit.py @@ -11,8 +11,8 @@ srcdir = py.path.local(cdir) / 'src' compilation_info = ExternalCompilationInfo( - includes=['readline/readline.h'], - libraries=["readline"]) + includes=['editline/readline.h'], + libraries=["edit"]) def llexternal(*args, **kwargs): return rffi.llexternal(*args, compilation_info=compilation_info, **kwargs) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index d1a9b0ca..198e450d 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -9,7 +9,7 @@ from pixie.vm.keyword import keyword, Keyword import pixie.vm.rt as rt from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR -from pixie.vm.libs.readline import _readline +from pixie.vm.libs.libedit import _readline from pixie.vm.string import Character, String from pixie.vm.code import wrap_fn, extend from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP diff --git a/pixie/vm/test/test_compile.py b/pixie/vm/test/test_compile.py index 6f57f4e2..410f846a 100644 --- a/pixie/vm/test/test_compile.py +++ b/pixie/vm/test/test_compile.py @@ -11,7 +11,7 @@ from pixie.vm.custom_types import CustomTypeInstance import unittest -import pixie.vm.libs.readline +import pixie.vm.libs.libedit def read_code(s): with with_ns(u"user"): From b1839c5942c3b1b94c164ce30810da862e8f28a1 Mon Sep 17 00:00:00 2001 From: John Walker Date: Sun, 14 Dec 2014 10:30:37 -0800 Subject: [PATCH 399/909] Update travis for libedit --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index a5788a52..3483f31d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ before_install: - sudo apt-add-repository ppa:linuxjedi/ppa -y - sudo apt-get update -qq - sudo apt-get update - - sudo apt-get install libffi-dev libuv-dev + - sudo apt-get install libffi-dev libuv-dev libedit-dev os: - osx From 9dce2da05255895440d953c6ce3286d38134ffff Mon Sep 17 00:00:00 2001 From: John Walker Date: Sun, 14 Dec 2014 10:32:19 -0800 Subject: [PATCH 400/909] Update README for libedit --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 6fb2857a..f2a96703 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Some planned and implemented features: * [libuv-dev](https://github.com/libuv/libuv) * [libffi-dev](https://sourceware.org/libffi/) -* [libreadline-dev](http://cnswww.cns.cwru.edu/php/chet/readline/rltop.html) +* [libedit-dev](http://thrysoee.dk/editline/) ## Building From adf6b1ede5f53def070fe70f0dc3036adcab01d4 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 15 Dec 2014 11:40:42 +0100 Subject: [PATCH 401/909] deprecate the shell build scripts they print out a message with the appropriate `make ...` command to use instead. --- .travis.yml | 3 +-- Makefile | 2 +- checkout-externals | 8 +------- make-no-jit | 2 +- make-with-jit | 2 +- run-interpreted | 2 +- 6 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.travis.yml b/.travis.yml index 3483f31d..ebd74511 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,5 @@ script: - - ./checkout-externals - - ./make-with-jit + - make build_preload_with_jit - make run_built_tests before_install: diff --git a/Makefile b/Makefile index d011ab32..12b22e7e 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ build_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target.py build_preload_with_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' build_preload_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target_preload.py diff --git a/checkout-externals b/checkout-externals index 719a808c..db5f24d8 100755 --- a/checkout-externals +++ b/checkout-externals @@ -1,7 +1 @@ -mkdir ../externals -cd ../externals -curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2 -mkdir pypy -cd pypy -tar -jxf ../pypy.tar.bz2 --strip-components=1 -cd - +echo "This script will be removed soon, please use 'make build_with_jit' or one of it's variants instead." \ No newline at end of file diff --git a/make-no-jit b/make-no-jit index 68874b6d..c10d5f4b 100755 --- a/make-no-jit +++ b/make-no-jit @@ -1 +1 @@ -python ../externals/pypy/rpython/bin/rpython --continuation --no-shared --lldebug target.py +echo "This script will be removed soon, please use 'make build_no_jit' instead." \ No newline at end of file diff --git a/make-with-jit b/make-with-jit index 59c9057e..2d955b51 100755 --- a/make-with-jit +++ b/make-with-jit @@ -1 +1 @@ -python ../externals/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' +echo "This script will be removed soon, please use 'make build_with_jit' instead." diff --git a/run-interpreted b/run-interpreted index 1f62325c..cef13f95 100755 --- a/run-interpreted +++ b/run-interpreted @@ -1 +1 @@ -PYTHONPATH=$PYTHONPATH:../externals/pypy python target.py +echo "This script will be removed soon, please use 'make run_interactive' instead." From 0891f45c202a06223f925c08a176d1adb5700138 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 15 Dec 2014 11:48:33 +0100 Subject: [PATCH 402/909] update 'make help' to mention new targets --- Makefile | 14 +++++++++----- 1 file changed, 9 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 12b22e7e..7bbe67ca 100644 --- a/Makefile +++ b/Makefile @@ -6,11 +6,15 @@ PYTHON ?= pypy PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy help: - @echo "make help - display this message" - @echo "make run - run the compiled interpreter" - @echo "make run_interactive - run without compiling (slow)" - @echo "make build_with_jit - build with jit enabled" - @echo "make build_no_jit - build without jit" + @echo "make help - display this message" + @echo "make run - run the compiled interpreter" + @echo "make run_interactive - run without compiling (slow)" + @echo "make build_with_jit - build with jit enabled" + @echo "make build_no_jit - build without jit" + @echo "make build_preload_with_jit - build with jit enabled and preload the stdlib" + @echo " (this means that pixie-vm will run as a standalone binary," + @echo " without having to load 'stdlib.pxi' and friends.)" + @echo "make build_preload_no_jit - build without jit and preload the stdlib" build_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target.py From 9b82068b190fd1c2268e14fc408401dbff565a07 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 15 Dec 2014 12:48:40 +0100 Subject: [PATCH 403/909] remove references to build scripts from the README --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index f2a96703..066962b8 100644 --- a/README.md +++ b/README.md @@ -26,8 +26,7 @@ Some planned and implemented features: ## Building - ./checkout-externals - ./make-with-jit + make build_with_jit ./pixie-vm ## Special Note for Macs @@ -37,8 +36,8 @@ If you are having trouble building or running the interpreter on Mac, check out In particular, try this: ``` -PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' ./make-with-jit -PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' ./run-interpreted +PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' make build_with_jit +PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' make run_interactive ``` ## Running the tests @@ -61,7 +60,8 @@ Pixie now comes with a build tool called [dust](https://github.com/pixie-lang/du ### So this is written in Python? -It's actually written in the RPython, the same language PyPy is written in. The script `./make-with-jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm` this executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... +It's actually written in the RPython, the same language PyPy is written in. `make build_with_jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm` this executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... + ### What's this bit about "magical powers"? From 9a220c5ef46557d10200cc87dc31ab45f915daea Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 15 Dec 2014 12:54:26 +0100 Subject: [PATCH 404/909] build with python on travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ebd74511..85642c80 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,5 +1,5 @@ script: - - make build_preload_with_jit + - make PYTHON=python build_preload_with_jit - make run_built_tests before_install: From bd5fbf99c71998bc8e9d48d7fb9455c720f2b96e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 15 Dec 2014 09:11:05 -0700 Subject: [PATCH 405/909] start of cstruct functions, also a refactor of ffi --- pixie/vm/libs/ffi.py | 57 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 57 insertions(+) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index f2adac5b..170f6931 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -324,3 +324,60 @@ def buffer_capacity(buffer): def set_buffer_size(self, size): self.set_used_size(size.int_val()) return self + + + +class CStructType(object.Type): + def __init__(self, name, size, desc): + object.Type.__init__(self, name) + self._desc = desc + self._size = size + #offsets is a dict of {nm, (type, offset)} + + def get_offset(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return offset + + def get_type(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return tp + + def get_desc(self, nm): + return self._desc[nm] + +class CType(object.Type): + def __init__(self, name): + object.Type.__init__(self, name) + +class CInt(CType): + def __init__(self): + CType.__init__(self, "pixie.stdlib.CInt") + + def ffi_load_from(self, ptr): + casted = rffi.cast(rffi.INTP, ptr) + return rt.wrap(casted[0]) + + def ffi_save_to(self, ptr, val): + casted = rffi.cast(rffi.INTP, ptr) + casted[0] = val.int_val() + return ptr + + +class CStruct(object.Object): + def __init__(self, tp, buffer): + self._type = tp + self._buffer = buffer + + def type(self): + return self._type + + def get_item(self, nm): + (tp, offset) = self._type.get_desc(nm) + ptr = rffi.ptradd(self._buffer, offset) + return tp.ffi_load_from(ptr) From 1ded06d8c3e22d35ef5d456d792444b65bf9b9b8 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 15 Dec 2014 17:46:33 +0100 Subject: [PATCH 406/909] fix the docker build e.g. use libedit there as well --- Dockerfile | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index f976ee9a..ea969e9a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,8 +3,7 @@ FROM debian:sid # install dependencies RUN apt-get update \ && apt-get install -y gcc pkg-config make curl bzip2 python2.7 \ - && apt-get install -y libffi-dev libuv-dev libreadline-dev \ - && ln -s /lib/x86_64-linux-gnu/libreadline.so.6 /lib/x86_64-linux-gnu/libreadline.so + && apt-get install -y libffi-dev libuv-dev libedit-dev ADD . /usr/src/pixie From 4e91e4a29442697e4a1eb870f8b39b9bf7a71296 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 15 Dec 2014 21:09:30 -0700 Subject: [PATCH 407/909] switch to polymorphic c types in ffi --- pixie/stdlib.pxi | 27 ++--- pixie/vm/libs/ffi.py | 250 +++++++++++++++++++++++++++---------------- 2 files changed, 173 insertions(+), 104 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8bfbc0d6..b3b88d4d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1,23 +1,24 @@ (__ns__ pixie.stdlib) (def libc (ffi-library pixie.platform/lib-c-name)) - (def exit (ffi-fn libc "exit" [Integer] Integer)) - (def puts (ffi-fn libc "puts" [String] Integer)) - (def sh (ffi-fn libc "system" [String] Integer)) - (def printf (ffi-fn libc "printf" [String] Integer)) - (def getenv (ffi-fn libc "getenv" [String] String)) + (def exit (ffi-fn libc "exit" [CInt] CInt)) + (def puts (ffi-fn libc "puts" [CCharP] CInt)) + + (def sh (ffi-fn libc "system" [CCharP] CInt)) + (def printf (ffi-fn libc "printf" [CCharP] CInt)) + (def getenv (ffi-fn libc "getenv" [CCharP] CCharP)) (def libreadline (ffi-library (str "libreadline." pixie.platform/so-ext))) - (def readline (ffi-fn libreadline "readline" [String] String)) - (def rand (ffi-fn libc "rand" [Integer] Integer)) - (def srand (ffi-fn libc "srand" [Integer] Integer)) - (def fopen (ffi-fn libc "fopen" [String String] VoidP)) - (def fread (ffi-fn libc "fread" [Buffer Integer Integer VoidP] Integer)) + (def readline (ffi-fn libreadline "readline" [CCharP] CCharP)) + (def rand (ffi-fn libc "rand" [CInt] CInt)) + (def srand (ffi-fn libc "srand" [CInt] CInt)) + (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) + (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) (def libm (ffi-library (str "libm." pixie.platform/so-ext))) - (def atan2 (ffi-fn libm "atan2" [Float Float] Float)) - (def floor (ffi-fn libm "floor" [Float] Float)) - (def lround (ffi-fn libm "lround" [Float] Integer)) + (def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) + (def floor (ffi-fn libm "floor" [CDouble] CDouble)) + (def lround (ffi-fn libm "lround" [CDouble] CInt)) (def reset! -reset!) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 170f6931..d8a55d48 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -67,64 +67,64 @@ def _cleanup_(self): self._is_inited = False -def get_native_size(tp): - if tp == Integer._type: - return rffi.sizeof(rffi.LONG) - if tp == Float._type: - return rffi.sizeof(rffi.DOUBLE) - if tp == String._type: - return rffi.sizeof(rffi.CCHARP) - if tp == Buffer._type: - return rffi.sizeof(rffi.CCHARP) - if tp == FFIVoidP._type: - return rffi.sizeof(rffi.VOIDP) - assert False - -def get_ret_val(ptr, tp): - if tp == Integer._type: - pnt = rffi.cast(rffi.LONGP, ptr) - val = pnt[0] - return Integer(val) - if tp == Float._type: - pnt = rffi.cast(rffi.DOUBLEP, ptr) - val = pnt[0] - return Float(val) - if tp == String._type: - pnt = rffi.cast(rffi.CCHARPP, ptr) - if pnt[0] == lltype.nullptr(rffi.CCHARP.TO): - return nil - else: - return String(unicode(rffi.charp2str(pnt[0]))) - - if tp == FFIVoidP._type: - pnt = rffi.cast(rffi.VOIDPP, ptr) - val = pnt[0] - return FFIVoidP(val) - - assert False - -def set_native_value(ptr, val, tp): - if tp is Integer._type: - pnt = rffi.cast(rffi.LONGP, ptr) - pnt[0] = rffi.cast(rffi.LONG, val.int_val()) - return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.LONG)) - if tp is Float._type: - pnt = rffi.cast(rffi.DOUBLEP, ptr) - pnt[0] = rffi.cast(rffi.DOUBLE, val.float_val()) - return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.DOUBLE)) - if tp is String._type: - pnt = rffi.cast(rffi.CCHARPP, ptr) - pnt[0] = rffi.str2charp(unicode_to_utf8(rt.name(val))) - return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) - if tp is Buffer._type: - pnt = rffi.cast(rffi.CCHARPP, ptr) - pnt[0] = val.buffer() - return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) - if tp is FFIVoidP._type: - pnt = rffi.cast(rffi.VOIDPP, ptr) - pnt[0] = val.voidp_data() - return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.VOIDP)) - assert False +# def get_native_size(tp): +# if tp == Integer._type: +# return rffi.sizeof(rffi.LONG) +# if tp == Float._type: +# return rffi.sizeof(rffi.DOUBLE) +# if tp == String._type: +# return rffi.sizeof(rffi.CCHARP) +# if tp == Buffer._type: +# return rffi.sizeof(rffi.CCHARP) +# if tp == FFIVoidP._type: +# return rffi.sizeof(rffi.VOIDP) +# assert False +# +# def get_ret_val(ptr, tp): +# if tp == Integer._type: +# pnt = rffi.cast(rffi.LONGP, ptr) +# val = pnt[0] +# return Integer(val) +# if tp == Float._type: +# pnt = rffi.cast(rffi.DOUBLEP, ptr) +# val = pnt[0] +# return Float(val) +# if tp == String._type: +# pnt = rffi.cast(rffi.CCHARPP, ptr) +# if pnt[0] == lltype.nullptr(rffi.CCHARP.TO): +# return nil +# else: +# return String(unicode(rffi.charp2str(pnt[0]))) +# +# if tp == FFIVoidP._type: +# pnt = rffi.cast(rffi.VOIDPP, ptr) +# val = pnt[0] +# return FFIVoidP(val) +# +# assert False + +# def set_native_value(ptr, val, tp): +# if tp is Integer._type: +# pnt = rffi.cast(rffi.LONGP, ptr) +# pnt[0] = rffi.cast(rffi.LONG, val.int_val()) +# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.LONG)) +# if tp is Float._type: +# pnt = rffi.cast(rffi.DOUBLEP, ptr) +# pnt[0] = rffi.cast(rffi.DOUBLE, val.float_val()) +# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.DOUBLE)) +# if tp is String._type: +# pnt = rffi.cast(rffi.CCHARPP, ptr) +# pnt[0] = rffi.str2charp(unicode_to_utf8(rt.name(val))) +# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) +# if tp is Buffer._type: +# pnt = rffi.cast(rffi.CCHARPP, ptr) +# pnt[0] = val.buffer() +# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) +# if tp is FFIVoidP._type: +# pnt = rffi.cast(rffi.VOIDPP, ptr) +# pnt[0] = val.voidp_data() +# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.VOIDP)) +# assert False class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") @@ -153,19 +153,19 @@ def thaw(self): cd = lltype.malloc(CIF_DESCRIPTION, nargs, flavor="raw") cd.abi = clibffi.FFI_DEFAULT_ABI cd.nargs = nargs - cd.rtype = get_clibffi_type(self._ret_type) + cd.rtype = self._ret_type.ffi_type() atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, nargs, flavor="raw") arg0_offset = exchange_buffer_size for idx in range(nargs): cd.exchange_args[idx] = exchange_buffer_size tp = self._arg_types[idx] - native_size = get_native_size(tp) - atypes[idx] = get_clibffi_type(tp) + native_size = tp.ffi_size() + atypes[idx] = tp.ffi_type() exchange_buffer_size += native_size ret_offset = exchange_buffer_size - exchange_buffer_size += get_native_size(self._ret_type) + exchange_buffer_size += self._ret_type.ffi_size() @@ -198,12 +198,13 @@ def prep_exb(self, args): offset_p = rffi.ptradd(exb, self._arg0_offset) for x in range(len(self._arg_types)): - offset_p = set_native_value(offset_p, args[x], self._arg_types[x]) + self._arg_types[x].ffi_set_value(offset_p, args[x]) + offset_p = rffi.ptradd(offset_p, self._arg_types[x].ffi_size()) return exb def get_ret_val_from_buffer(self, exb): offset_p = rffi.ptradd(exb, self._cd.exchange_result_libffi) - ret_val = get_ret_val(offset_p, self._ret_type) + ret_val = self._ret_type.ffi_get_value(offset_p) return ret_val @jit.unroll_safe @@ -221,18 +222,18 @@ def invoke(self, args): -def get_clibffi_type(arg): - if arg == Integer._type: - return clibffi.cast_type_to_ffitype(rffi.LONG) - if arg == Float._type: - return clibffi.cast_type_to_ffitype(rffi.DOUBLE) - if arg == String._type: - return clibffi.ffi_type_pointer - if arg == Buffer._type: - return clibffi.ffi_type_pointer - if arg == FFIVoidP._type: - return clibffi.ffi_type_pointer - assert False +# def get_clibffi_type(arg): +# if arg == Integer._type: +# return clibffi.cast_type_to_ffitype(rffi.LONG) +# if arg == Float._type: +# return clibffi.cast_type_to_ffitype(rffi.DOUBLE) +# if arg == String._type: +# return clibffi.ffi_type_pointer +# if arg == Buffer._type: +# return clibffi.ffi_type_pointer +# if arg == FFIVoidP._type: +# return clibffi.ffi_type_pointer +# assert False @as_var("ffi-library") @@ -256,17 +257,7 @@ def _ffi_fn(lib, nm, args, ret_type): f = FFIFn(lib, rt.name(nm), new_args, ret_type) return f -class FFIVoidP(object.Object): - _type = object.Type(u"pixie.stdlib.VoidP") - - def type(self): - return FFIVoidP._type - def __init__(self, data): - self._voidp_data = data - - def voidp_data(self): - return self._voidp_data @@ -357,17 +348,94 @@ def __init__(self, name): class CInt(CType): def __init__(self): - CType.__init__(self, "pixie.stdlib.CInt") + CType.__init__(self, u"pixie.stdlib.CInt") - def ffi_load_from(self, ptr): + def ffi_get_value(self, ptr): casted = rffi.cast(rffi.INTP, ptr) - return rt.wrap(casted[0]) + return Integer(casted[0]) - def ffi_save_to(self, ptr, val): + def ffi_set_value(self, ptr, val): casted = rffi.cast(rffi.INTP, ptr) - casted[0] = val.int_val() + casted[0] = rffi.cast(rffi.INT, val.int_val()) + return ptr + + def ffi_size(self): + return rffi.sizeof(rffi.INT) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.INT) +CInt() + +class CDouble(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CDouble") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.DOUBLEP, ptr) + return Integer(casted[0]) + + def ffi_set_value(self, ptr, val): + casted = rffi.cast(rffi.DOUBLEP, ptr) + casted[0] = rffi.cast(rffi.DOUBLE, val.float_val()) + return ptr + + def ffi_size(self): + return rffi.sizeof(rffi.DOUBLE) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.DOUBLE) +CDouble() + +class CCharP(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CCharP") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.CCHARPP, ptr) + if casted[0] == lltype.nullptr(rffi.CCHARP.TO): + return nil + else: + return String(unicode(rffi.charp2str(casted[0]))) + + def ffi_set_value(self, ptr, val): + pnt = rffi.cast(rffi.CCHARPP, ptr) + pnt[0] = rffi.str2charp(unicode_to_utf8(rt.name(val))) return ptr + def ffi_size(self): + return rffi.sizeof(rffi.CCHARP) + + def ffi_type(self): + return clibffi.ffi_type_pointer +CCharP() + + +class CVoidP(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CVoidP") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.VOIDPP, ptr) + if casted[0] == lltype.nullptr(rffi.VOIDP.TO): + return nil + else: + return Buffer(casted[0]) + + def ffi_set_value(self, ptr, val): + pnt = rffi.cast(rffi.VOIDPP, ptr) + if isinstance(val, Buffer): + pnt[0] = val.buffer() + else: + affirm(False, u"Cannot encode this type") + return ptr + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer + +CCharP() class CStruct(object.Object): def __init__(self, tp, buffer): From c4559e4cb2066f54832e1a56db87455b8792dbf0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 16 Dec 2014 09:51:37 -0700 Subject: [PATCH 408/909] update travis..some odd build error here --- .travis.yml | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 85642c80..2607ad32 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,15 +1,10 @@ + script: - make PYTHON=python build_preload_with_jit - make run_built_tests before_install: - - sudo apt-add-repository ppa:linuxjedi/ppa -y - - sudo apt-get update -qq - - sudo apt-get update - - sudo apt-get install libffi-dev libuv-dev libedit-dev - -os: - - osx + - sudo apt-get install libffi-dev libedit-dev notifications: irc: "chat.freenode.net#pixie-lang" From 15a98742c8cdf22be47d879af18d7afbad1e38a0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 16 Dec 2014 15:15:08 -0700 Subject: [PATCH 409/909] added string builder and popen --- pixie/io.pxi | 48 +++++++++++++++++++++++++++++++++++ pixie/stdlib.pxi | 5 ++++ pixie/vm/rt.py | 1 + pixie/vm/string_builder.py | 37 +++++++++++++++++++++++++++ tests/pixie/tests/test-io.pxi | 4 +++ 5 files changed, 95 insertions(+) create mode 100644 pixie/vm/string_builder.py diff --git a/pixie/io.pxi b/pixie/io.pxi index df8fa046..4574ffa7 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -4,6 +4,8 @@ (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) (def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) (def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) +(def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) +(def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) (defprotocol IInputStream @@ -48,3 +50,49 @@ [filename] (assert (string? filename) "Filename must be a string") (->FileStream (fopen filename "r"))) + + +(deftype ProcessInputStream [fp] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [read-count (fread buffer 1 len fp)] + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (fgetc buffer)) + IClosable + (close [this] + (pclose fp)) + IReduce + (-reduce [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc)))))) + + +(defn popen-read + {:doc "Open a file for reading, returning a IInputStream" + :added "0.1"} + [command] + (assert (string? command) "Command must be a string") + (->ProcessInputStream (popen command "r"))) + + +(defn run-command [command] + (let [c (->ProcessInputStream (popen command "r")) + result (transduce + (map char) + string-builder + c)] + (close c) + result)) + diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 937b110c..2517b6fb 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1915,4 +1915,9 @@ Expands to calls to `extend-type`." ([result] result) ([result _] (inc result))) +(defn string-builder + "Creates a reducing function that builds a string based on calling str on the transduced collection" + ([] (-string-builder)) + ([sb] (str sb)) + ([sb item] (conj! sb item))) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index a61fc793..72d1bef7 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -71,6 +71,7 @@ def wrapper(*args): import pixie.vm.libs.path import pixie.vm.libs.string import pixie.vm.threads + import pixie.vm.string_builder diff --git a/pixie/vm/string_builder.py b/pixie/vm/string_builder.py new file mode 100644 index 00000000..d2a94296 --- /dev/null +++ b/pixie/vm/string_builder.py @@ -0,0 +1,37 @@ +import pixie.vm.rt as rt +from pixie.vm.object import Object, Type +from pixie.vm.code import as_var, extend +import pixie.vm.stdlib as proto + +class StringBuilder(Object): + _type = Type(u"pixie.stdlib.StringBuilder") + + def type(self): + return StringBuilder._type + + def __init__(self): + self._strs = [] + + def add_str(self, s): + self._strs.append(s) + return self + + def to_string(self): + return u"".join(self._strs) + + +@extend(proto._conj_BANG_, StringBuilder) +def _conj(self, val): + return self.add_str(rt.name(rt._str(val))) + +@extend(proto._persistent_BANG_, StringBuilder) +def _persistent(self): + return rt._str(self) + +@extend(proto._str, StringBuilder) +def _str(self): + return rt.wrap(self.to_string()) + +@as_var("-string-builder") +def _string_builder(): + return StringBuilder() \ No newline at end of file diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index adebd4f8..c0a0f720 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -8,3 +8,7 @@ count-rf f) 78))) + +(t/deftest test-process-reduction + (let [f (io/run-command "ls tests/pixie/tests/test-io.txt")] + (t/assert= f "tests/pixie/tests/test-io.txt\n"))) From 9106710450fe614371da7fe21c3fe433121ac35c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 16 Dec 2014 16:41:48 -0700 Subject: [PATCH 410/909] preload causes build failures? --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2607ad32..18558fba 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,6 @@ script: - - make PYTHON=python build_preload_with_jit + - make PYTHON=python build_with_jit - make run_built_tests before_install: From c8543c029174aad0038960946869fe2150dd744f Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Tue, 16 Dec 2014 21:07:21 -0600 Subject: [PATCH 411/909] Refactor build script to include travis build matrix --- .travis.yml | 10 +++++++++- Makefile | 15 +++++++++++---- 2 files changed, 20 insertions(+), 5 deletions(-) diff --git a/.travis.yml b/.travis.yml index 18558fba..181c17c3 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,6 +1,14 @@ +env: + - JIT_OPTS='--opt=jit' TARGET_OPTS='target.py' + - JIT_OPTS='' TARGET_OPTS='target.py' + - JIT_OPTS='--opt=jit' TARGET_OPTS='target_preload.py' + - JIT_OPTS='' TARGET_OPTS='target_preload.py' + +matrix: + fast_finish: true script: - - make PYTHON=python build_with_jit + - make PYTHON=python build - make run_built_tests before_install: diff --git a/Makefile b/Makefile index 7bbe67ca..fd995992 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,10 @@ EXTERNALS=../externals PYTHON ?= pypy PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy +COMMON_BUILD_OPTS?=--thread --no-shared +JIT_OPTS?=--opt=jit +TARGET_OPTS?=target.py + help: @echo "make help - display this message" @echo "make run - run the compiled interpreter" @@ -17,16 +21,19 @@ help: @echo "make build_preload_no_jit - build without jit and preload the stdlib" build_with_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target.py + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py build_no_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target.py + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target.py build_preload_with_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --opt=jit --thread --no-shared target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' build_preload_no_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython --thread --no-shared target_preload.py + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target_preload.py + +build: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) 2>&1 >/dev/null | grep -v 'WARNING' fetch_externals: $(EXTERNALS)/pypy From 12bdd515e2f9ad34149ba91c4ab5ca9ea6333324 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 17 Dec 2014 14:53:23 -0700 Subject: [PATCH 412/909] added spit and slurp --- pixie/io.pxi | 50 ++++++++++++++++++++++++++++++++--- pixie/stdlib.pxi | 8 +++++- tests/pixie/tests/test-io.pxi | 4 +++ 3 files changed, 57 insertions(+), 5 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 4574ffa7..98eb242f 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -3,14 +3,20 @@ (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) (def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) +(def fputc (ffi-fn libc "fputc" [CInt CVoidP] CInt)) +(def fwrite (ffi-fn libc "fwrite" [CVoidP CInt CInt CVoidP] CInt)) (def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) (def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) (def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) (defprotocol IInputStream - (read [this] "Read a single character") - (read-byte [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) + (read-byte [this] "Read a single character") + (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) + +(defprotocol IOutputStream + (write-byte [this byte]) + (write [this buffer])) (defprotocol IClosable (close [this] "Closes the stream")) @@ -29,7 +35,7 @@ (fgetc buffer)) IClosable (close [this] - (fclose buffer)) + (fclose fp)) IReduce (-reduce [this f init] (let [buf (buffer DEFAULT-BUFFER-SIZE) @@ -44,6 +50,7 @@ acc)))))) + (defn open-read {:doc "Open a file for reading, returning a IInputStream" :added "0.1"} @@ -51,6 +58,42 @@ (assert (string? filename) "Filename must be a string") (->FileStream (fopen filename "r"))) +(deftype FileOutputStream [fp] + IOutputStream + (write-byte [this val] + (assert (integer? val) "Value must be a int") + (fputc val fp)) + (write [this buffer] + (fwrite buffer 1 (count buffer) fp)) + IClosable + (close [this] + (fclose fp))) + +(defn file-output-rf [filename] + (let [fp (->FileOutputStream (fopen filename "w"))] + (fn ([] 0) + ([cnt] (close fp) cnt) + ([cnt chr] + (assert (integer? chr)) + (let [written (write-byte fp chr)] + (if (= written 0) + (reduced cnt) + (+ cnt written))))))) + + +(defn spit [filename val] + (transduce (map int) + (file-output-rf filename) + (str val))) + +(defn slurp [filename] + (let [c (->FileStream (fopen filename "r")) + result (transduce + (map char) + string-builder + c)] + (close c) + result)) (deftype ProcessInputStream [fp] IInputStream @@ -95,4 +138,3 @@ c)] (close c) result)) - diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 2517b6fb..3f6e7d2d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -222,9 +222,11 @@ (extend -reduce PersistentList seq-reduce) (extend -reduce LazySeq seq-reduce) + (comment (extend -reduce Array indexed-reduce)) (extend -reduce Buffer indexed-reduce) +(extend -reduce String indexed-reduce) (extend -str Bool (fn [x] @@ -1297,6 +1299,11 @@ The new value is thus `(apply f current-value-of-atom args)`." (dotimes [x (count v)] (yield (nth v x))))) +(extend -iterator Array + (fn [v] + (dotimes [x (count v)] + (yield (nth v x))))) + (defmacro and {:doc "Check if the given expressions return truthy values, returning the last, or false." :examples [["(and true false)" nil false] @@ -1920,4 +1927,3 @@ Expands to calls to `extend-type`." ([] (-string-builder)) ([sb] (str sb)) ([sb item] (conj! sb item))) - diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index c0a0f720..1cee7b01 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -12,3 +12,7 @@ (t/deftest test-process-reduction (let [f (io/run-command "ls tests/pixie/tests/test-io.txt")] (t/assert= f "tests/pixie/tests/test-io.txt\n"))) + +(t/deftest test-slurp-spit + (let [val (vec (range 1024))] + (t/assert= val (read-string (io/slurp "test.tmp" (io/spit "test.tmp" val)))))) From 0308b0b5b8ed2651bbccba92a76497fc67fff343 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 17 Dec 2014 14:55:07 -0700 Subject: [PATCH 413/909] started working on rffi-esque C introspection --- pixie/ffi-infer.pxi | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 pixie/ffi-infer.pxi diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi new file mode 100644 index 00000000..be1460a7 --- /dev/null +++ b/pixie/ffi-infer.pxi @@ -0,0 +1,26 @@ +(ns pixie.ffi-infer + (require pixie.io :as io)) + +(defmulti emit-infer-code :op) + +(defmethod emit-infer-code :constant + [{:keys [name]}] + (str "cout << \"{:name " (keyword name) " :value \" << " name " << \"}\" << std::endl; \n")) + +(defn start-string [] + " #include + #include + int main() { +") + +(defn end-string [] + " return 0; + }") + + +(println (str (start-string) + (emit-infer-code {:op :constant :name "foo"}) + (end-string))) + +() +(io/run-command "ls") From 3a9f61d83b6e29eb8f15523c3dccf8a64b0181f5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 17 Dec 2014 15:16:55 -0700 Subject: [PATCH 414/909] add allowed failure to preload --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 181c17c3..275e9fbc 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,6 +6,8 @@ env: matrix: fast_finish: true + allow_failures: + - TARGET_OPTS='target_preload.py' script: - make PYTHON=python build From 1b2149fae128e24404005cee31518e089be3628d Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Wed, 17 Dec 2014 19:03:51 -0600 Subject: [PATCH 415/909] Clean up whitespace and remove unused imports --- pixie/vm/rt.py | 22 +++++----------------- 1 file changed, 5 insertions(+), 17 deletions(-) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 72d1bef7..8d18e12e 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -2,15 +2,12 @@ py_list = list py_str = str from rpython.rlib.objectmodel import specialize, we_are_translated -from rpython.rtyper.lltypesystem import lltype, rffi - def init(): - import pixie.vm.code as code from pixie.vm.object import affirm, _type_registry - from rpython.rlib.rarithmetic import r_uint, intmask + from rpython.rlib.rarithmetic import r_uint from rpython.rlib.rbigint import rbigint from pixie.vm.primitives import nil, true, false from pixie.vm.string import String @@ -42,16 +39,14 @@ def wrapper(*args): assert False, "Don't know how to convert" + str(tp) return lambda *args: fn.invoke(py_list(args)) - - if globals().has_key("__inited__"): + if "__inited__" in globals(): return import sys - sys.setrecursionlimit(100000) # Yeah we blow the stack sometimes, we promise it's not a bug + sys.setrecursionlimit(100000) # Yeah we blow the stack sometimes, we promise it's not a bug import pixie.vm.numbers as numbers - import pixie.vm.bits as bits - from pixie.vm.code import wrap_fn + import pixie.vm.bits import pixie.vm.interpreter import pixie.vm.atom import pixie.vm.reduced @@ -62,9 +57,7 @@ def wrapper(*args): import pixie.vm.persistent_hash_map import pixie.vm.persistent_hash_set import pixie.vm.custom_types - import pixie.vm.compiler as compiler import pixie.vm.map_entry - import pixie.vm.reader as reader import pixie.vm.libs.platform import pixie.vm.libs.ffi import pixie.vm.symbol @@ -73,8 +66,6 @@ def wrapper(*args): import pixie.vm.threads import pixie.vm.string_builder - - numbers.init() @specialize.argtype(0) @@ -102,7 +93,6 @@ def wrap(x): globals()["wrap"] = wrap - def int_val(x): affirm(isinstance(x, numbers.Number), u"Expected number") return x.int_val() @@ -118,7 +108,6 @@ def int_val(x): else: globals()[name] = var - import pixie.vm.bootstrap def reinit(): @@ -127,7 +116,7 @@ def reinit(): if name in globals(): continue - if var.is_defined() and isinstance(var.deref(), BaseCode): + if var.is_defined() and isinstance(var.deref(), BaseCode): globals()[name] = unwrap(var) else: globals()[name] = var @@ -150,7 +139,6 @@ def reinit(): # # stacklet.with_stacklets(run_load_stdlib) - init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec"] for x in init_fns: globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) From f0748e8571e6f6446f0d1bb1917b33b42ee021c1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 17 Dec 2014 21:22:38 -0700 Subject: [PATCH 416/909] added tokens to allow strings to be gc'd after a call to ffi --- pixie/vm/libs/ffi.py | 118 +++++++++++++------------------------------ 1 file changed, 35 insertions(+), 83 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 36e44e0e..3839cbf2 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -1,3 +1,4 @@ +py_object = object import rpython.rlib.rdynload as dynload import pixie.vm.object as object import pixie.vm.code as code @@ -67,65 +68,6 @@ def _cleanup_(self): self._is_inited = False -# def get_native_size(tp): -# if tp == Integer._type: -# return rffi.sizeof(rffi.LONG) -# if tp == Float._type: -# return rffi.sizeof(rffi.DOUBLE) -# if tp == String._type: -# return rffi.sizeof(rffi.CCHARP) -# if tp == Buffer._type: -# return rffi.sizeof(rffi.CCHARP) -# if tp == FFIVoidP._type: -# return rffi.sizeof(rffi.VOIDP) -# assert False -# -# def get_ret_val(ptr, tp): -# if tp == Integer._type: -# pnt = rffi.cast(rffi.LONGP, ptr) -# val = pnt[0] -# return Integer(val) -# if tp == Float._type: -# pnt = rffi.cast(rffi.DOUBLEP, ptr) -# val = pnt[0] -# return Float(val) -# if tp == String._type: -# pnt = rffi.cast(rffi.CCHARPP, ptr) -# if pnt[0] == lltype.nullptr(rffi.CCHARP.TO): -# return nil -# else: -# return String(unicode(rffi.charp2str(pnt[0]))) -# -# if tp == FFIVoidP._type: -# pnt = rffi.cast(rffi.VOIDPP, ptr) -# val = pnt[0] -# return FFIVoidP(val) -# -# assert False - -# def set_native_value(ptr, val, tp): -# if tp is Integer._type: -# pnt = rffi.cast(rffi.LONGP, ptr) -# pnt[0] = rffi.cast(rffi.LONG, val.int_val()) -# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.LONG)) -# if tp is Float._type: -# pnt = rffi.cast(rffi.DOUBLEP, ptr) -# pnt[0] = rffi.cast(rffi.DOUBLE, val.float_val()) -# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.DOUBLE)) -# if tp is String._type: -# pnt = rffi.cast(rffi.CCHARPP, ptr) -# pnt[0] = rffi.str2charp(unicode_to_utf8(rt.name(val))) -# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) -# if tp is Buffer._type: -# pnt = rffi.cast(rffi.CCHARPP, ptr) -# pnt[0] = val.buffer() -# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.CCHARP)) -# if tp is FFIVoidP._type: -# pnt = rffi.cast(rffi.VOIDPP, ptr) -# pnt[0] = val.voidp_data() -# return rffi.ptradd(rffi.cast(rffi.CCHARP, pnt), rffi.sizeof(rffi.VOIDP)) -# assert False - class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") __immutable_fields__ = ["_is_inited?", "_lib", "_name", "_arg_types[*]", "_ret_type", \ @@ -196,11 +138,13 @@ def prep_exb(self, args): self.thaw() exb = lltype.malloc(rffi.CCHARP.TO, self._transfer_size, flavor="raw") offset_p = rffi.ptradd(exb, self._arg0_offset) + tokens = [None] * len(args) for x in range(len(self._arg_types)): - self._arg_types[x].ffi_set_value(offset_p, args[x]) + tokens[x] = self._arg_types[x].ffi_set_value(offset_p, args[x]) offset_p = rffi.ptradd(offset_p, self._arg_types[x].ffi_size()) - return exb + + return exb, tokens def get_ret_val_from_buffer(self, exb): offset_p = rffi.ptradd(exb, self._cd.exchange_result_libffi) @@ -210,9 +154,15 @@ def get_ret_val_from_buffer(self, exb): @jit.unroll_safe def _invoke(self, args): - exb = self.prep_exb(args) + exb, tokens = self.prep_exb(args) jit_ffi_call(self._cd, self._f_ptr, exb) ret_val = self.get_ret_val_from_buffer(exb) + + for x in range(len(args)): + t = tokens[x] + if t is not None: + t.finalize_token() + lltype.free(exb, flavor="raw") return ret_val @@ -220,22 +170,6 @@ def invoke(self, args): self = jit.promote(self) return self._invoke(args) - - -# def get_clibffi_type(arg): -# if arg == Integer._type: -# return clibffi.cast_type_to_ffitype(rffi.LONG) -# if arg == Float._type: -# return clibffi.cast_type_to_ffitype(rffi.DOUBLE) -# if arg == String._type: -# return clibffi.ffi_type_pointer -# if arg == Buffer._type: -# return clibffi.ffi_type_pointer -# if arg == FFIVoidP._type: -# return clibffi.ffi_type_pointer -# assert False - - @as_var("ffi-library") def _ffi_library(ns): nm = rt.name(ns) @@ -343,10 +277,18 @@ def get_type(self, nm): def get_desc(self, nm): return self._desc[nm] +class Token(py_object): + """ Tokens are returned by ffi_set_value and are called when ffi is ready to clean up resources + """ + def finalize_token(self): + pass + + class CType(object.Type): def __init__(self, name): object.Type.__init__(self, name) + class CInt(CType): def __init__(self): CType.__init__(self, u"pixie.stdlib.CInt") @@ -358,7 +300,6 @@ def ffi_get_value(self, ptr): def ffi_set_value(self, ptr, val): casted = rffi.cast(rffi.INTP, ptr) casted[0] = rffi.cast(rffi.INT, val.int_val()) - return ptr def ffi_size(self): return rffi.sizeof(rffi.INT) @@ -378,7 +319,6 @@ def ffi_get_value(self, ptr): def ffi_set_value(self, ptr, val): casted = rffi.cast(rffi.DOUBLEP, ptr) casted[0] = rffi.cast(rffi.DOUBLE, val.float_val()) - return ptr def ffi_size(self): return rffi.sizeof(rffi.DOUBLE) @@ -400,8 +340,10 @@ def ffi_get_value(self, ptr): def ffi_set_value(self, ptr, val): pnt = rffi.cast(rffi.CCHARPP, ptr) - pnt[0] = rffi.str2charp(unicode_to_utf8(rt.name(val))) - return ptr + utf8 = unicode_to_utf8(rt.name(val)) + data, pinned, raw = rffi.get_nonmovingbuffer(utf8) + pnt[0] = data + return CCharPToken(utf8, data, pinned, raw) def ffi_size(self): return rffi.sizeof(rffi.CCHARP) @@ -410,6 +352,17 @@ def ffi_type(self): return clibffi.ffi_type_pointer CCharP() +class CCharPToken(Token): + def __init__(self, s, data, pinned, raw): + self._s = s + self._data = data + self._pinned = pinned + self._raw = raw + + def finalize_token(self): + rffi.free_nonmovingbuffer(self._s, self._data, self._pinned, self._raw) + + class CVoidP(CType): def __init__(self): @@ -430,7 +383,6 @@ def ffi_set_value(self, ptr, val): pnt[0] = val.raw_data() else: affirm(False, u"Cannot encode this type") - return ptr def ffi_size(self): return rffi.sizeof(rffi.VOIDP) From 401a8e134de5dc6e02f7a064ded70f92c7cca789 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 17 Dec 2014 21:34:51 -0700 Subject: [PATCH 417/909] debugging a bit of ffi --- pixie/vm/libs/ffi.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 3839cbf2..97e28ad2 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -382,6 +382,7 @@ def ffi_set_value(self, ptr, val): elif isinstance(val, VoidP): pnt[0] = val.raw_data() else: + print val affirm(False, u"Cannot encode this type") def ffi_size(self): From 684a85703944a02d518789062056f1b6ce19013d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 17 Dec 2014 21:45:19 -0700 Subject: [PATCH 418/909] use str2charp --- pixie/vm/libs/ffi.py | 13 +++++-------- tests/pixie/tests/test-io.pxi | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 97e28ad2..8c35c46e 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -341,9 +341,9 @@ def ffi_get_value(self, ptr): def ffi_set_value(self, ptr, val): pnt = rffi.cast(rffi.CCHARPP, ptr) utf8 = unicode_to_utf8(rt.name(val)) - data, pinned, raw = rffi.get_nonmovingbuffer(utf8) - pnt[0] = data - return CCharPToken(utf8, data, pinned, raw) + raw = rffi.str2charp(utf8) + pnt[0] = raw + return CCharPToken(raw) def ffi_size(self): return rffi.sizeof(rffi.CCHARP) @@ -353,14 +353,11 @@ def ffi_type(self): CCharP() class CCharPToken(Token): - def __init__(self, s, data, pinned, raw): - self._s = s - self._data = data - self._pinned = pinned + def __init__(self, raw): self._raw = raw def finalize_token(self): - rffi.free_nonmovingbuffer(self._s, self._data, self._pinned, self._raw) + rffi.free_charp(self._raw) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 1cee7b01..930acbac 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -14,5 +14,5 @@ (t/assert= f "tests/pixie/tests/test-io.txt\n"))) (t/deftest test-slurp-spit - (let [val (vec (range 1024))] + (let [val (vec (range 128))] (t/assert= val (read-string (io/slurp "test.tmp" (io/spit "test.tmp" val)))))) From 2a001ea57a0bf6f10527d4c4fdab1bbdf404e937 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 18 Dec 2014 16:04:44 -0700 Subject: [PATCH 419/909] latest work on hte ffi infer code --- pixie/ffi-infer.pxi | 56 ++++++++++++++++++++++++++++++++++++++++----- 1 file changed, 50 insertions(+), 6 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index be1460a7..e128b23c 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -5,22 +5,66 @@ (defmethod emit-infer-code :constant [{:keys [name]}] - (str "cout << \"{:name " (keyword name) " :value \" << " name " << \"}\" << std::endl; \n")) + (str "std::cout << \"{:name " (keyword name)" :value \" << " name + "<< \" :type \" << TypeOf( " name ") " + "<< \"}\" << std::endl; \n")) + +(defmethod emit-infer-code :sizeof-struct + [{:keys [name]}] + (str "std::cout << \"{:name " (keyword name)" :sizeof \" << sizeof(struct " name + ") << \"}\" << std::endl; \n")) + +(defmethod emit-infer-code :sizeof + [{:keys [name]}] + (str "std::cout << \"{:name " (keyword name)" :sizeof \" << sizeof( " name + ") << \"}\" << std::endl; \n")) (defn start-string [] " #include + #include + #include #include + + template + std::string TypeOf(T v) + { + return \":unknown\"; + } + + template<> + std::string TypeOf(size_t v) + { + return \":CSizeT\"; + } + + template<> + std::string TypeOf(int v) + { + return \":CIntT\"; + } + int main() { + std::cout << \"[\"; ") (defn end-string [] - " return 0; + " std::cout << \"]\" << std::endl; + return 0; }") -(println (str (start-string) - (emit-infer-code {:op :constant :name "foo"}) - (end-string))) +(io/spit "/tmp/tmp.cpp" (str (start-string) + (apply str (map emit-infer-code + [ {:op :constant :name "RAND_MAX"} + {:op :sizeof-struct :name "stat"} + {:op :sizeof :name "int"} + {:op :offsetof-struct :struct "stat" :member ""}])) + (end-string))) + +(do (io/spit "/tmp/tmp.cpp" (str (start-string) + (emit-infer-code {:op :constant :name "RAND_MAX"}) + (end-string))) + (read-string (io/run-command "c++ /tmp/tmp.cpp && ./a.out"))) -() +(spit ) (io/run-command "ls") From f92e7d48e3a15cb600f24fec5f912eee078cdb8d Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 20 Dec 2014 00:08:19 +1100 Subject: [PATCH 420/909] Make ITransientStack interface TransientVector now supports -pop! and -push! --- pixie/stdlib.pxi | 7 +++++++ pixie/vm/persistent_vector.py | 11 +++++++++++ pixie/vm/stdlib.py | 1 + tests/pixie/tests/collections/test-vectors.pxi | 4 ++++ 4 files changed, 23 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 3f6e7d2d..f12f55c5 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -83,6 +83,13 @@ ([coll item & items] (reduce -disj! (-disj! coll item) items))))) +(def pop! + (fn ^{:doc "Pops elements off a transient stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + pop! + ([coll] (-pop! coll)))) + (def transient (fn [coll] (-transient coll))) (def persistent! (fn [coll] (-persistent! coll))) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 723366d9..a90b25ea 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -508,6 +508,17 @@ def _conj(self, val): assert isinstance(self, TransientVector) return self.conj(val) +@extend(proto._pop_BANG_, TransientVector) +def _pop(self): + assert isinstance(self, TransientVector) + return self.pop() + +@extend(proto._push_BANG_, TransientVector) +def _push(self, val): + assert isinstance(self, TransientVector) + return self.conj(val) + + proto.IVector.add_satisfies(PersistentVector._type) EMPTY = PersistentVector(nil, r_uint(0), r_uint(5), EMPTY_NODE, []) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 4f5b3c60..77a16605 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -54,6 +54,7 @@ defprotocol("pixie.stdlib", "IToTransient", ["-transient"]) defprotocol("pixie.stdlib", "ITransientCollection", ["-conj!"]) +defprotocol("pixie.stdlib", "ITransientStack", ["-push!", "-pop!"]) defprotocol("pixie.stdlib", "IIterable", ["-iterator"]) defprotocol("pixie.stdlib", "IIterator", ["-current", "-at-end?", "-move-next!"]) diff --git a/tests/pixie/tests/collections/test-vectors.pxi b/tests/pixie/tests/collections/test-vectors.pxi index 84927140..2d760984 100644 --- a/tests/pixie/tests/collections/test-vectors.pxi +++ b/tests/pixie/tests/collections/test-vectors.pxi @@ -43,3 +43,7 @@ (t/deftest vector-conj! (t/assert= [1 2] (persistent! (conj! (transient [1]) 2))) (t/assert= [1 2 3] (persistent! (conj! (transient [1]) 2 3)))) + +(t/deftest vector-pop! + (t/assert= [1 2] (persistent! (pop! (transient [1 2 3])))) + (t/assert= [1 2 3] (persistent! (pop! (transient [1 2 3 4]))))) From 183e656a90a6a21f753df8f12f8aac8358745a51 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 20 Dec 2014 00:54:14 +1100 Subject: [PATCH 421/909] Fixed bug in editable_array_for --- pixie/vm/persistent_vector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index a90b25ea..84877edc 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -297,7 +297,7 @@ def editable_array_for(self, i): node = self._root level = self._shift while level > 0: - node = self.ensure_node_editable(node._array[(i >> self._level) & 0x1f]) + node = self.ensure_node_editable(node._array[(i >> level) & 0x1f]) level -= 5 return node._array From 64d9f83a27c697131065a42688ee237d6c3826a9 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 20 Dec 2014 00:54:58 +1100 Subject: [PATCH 422/909] Fixed miss named function --- pixie/vm/persistent_vector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 84877edc..4105b713 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -442,7 +442,7 @@ def _push(self, v): return self.conj(v) @extend(proto._pop, PersistentVector) -def _push(self): +def _pop(self): assert isinstance(self, PersistentVector) return self.pop() From 1a2d421c630bf067e72339803f1bb1ff6cb5a237 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 20 Dec 2014 00:56:12 +1100 Subject: [PATCH 423/909] Added push! to stdlib.pxi --- pixie/stdlib.pxi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index f12f55c5..36b14e49 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -90,6 +90,13 @@ pop! ([coll] (-pop! coll)))) +(def push! + (fn ^{:doc "Push an element on to a transient stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + push! + ([coll x] (-push! coll x)))) + (def transient (fn [coll] (-transient coll))) (def persistent! (fn [coll] (-persistent! coll))) From dabe776441ff58db86f70aabc5d1347cfa354f62 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 20 Dec 2014 01:12:05 +1100 Subject: [PATCH 424/909] Added tests for ITransientStack operations on vectors --- tests/pixie/tests/collections/test-vectors.pxi | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/tests/pixie/tests/collections/test-vectors.pxi b/tests/pixie/tests/collections/test-vectors.pxi index 2d760984..b08d585d 100644 --- a/tests/pixie/tests/collections/test-vectors.pxi +++ b/tests/pixie/tests/collections/test-vectors.pxi @@ -44,6 +44,11 @@ (t/assert= [1 2] (persistent! (conj! (transient [1]) 2))) (t/assert= [1 2 3] (persistent! (conj! (transient [1]) 2 3)))) +(t/deftest vector-push! + (t/assert= [1] (persistent! (push! (transient []) 1))) + (t/assert= [1 2] (persistent! (push! (transient [1]) 2)))) + (t/deftest vector-pop! - (t/assert= [1 2] (persistent! (pop! (transient [1 2 3])))) - (t/assert= [1 2 3] (persistent! (pop! (transient [1 2 3 4]))))) + (t/assert= [] (persistent! (pop! (transient [1])))) + (t/assert= [1] (persistent! (pop! (transient [1 2])))) + (t/assert= [1 2] (persistent! (pop! (transient [1 2 3]))))) From 85cd41e1eb6d7a4b254d0bf0fa4084e988f09c15 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 20 Dec 2014 01:13:14 +1100 Subject: [PATCH 425/909] Subtract 1 not 2 from _cnt when creating a new_tail for TransientVectors --- pixie/vm/persistent_vector.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 4105b713..ead07db2 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -326,7 +326,7 @@ def pop(self): self._cnt -= 1 return self - new_tail = self.editable_array_for(self._cnt - 2) + new_tail = self.editable_array_for(self._cnt - 1) new_root = self.pop_tail(self._shift, self._root) new_shift = self._shift From 85dc8ec5f3999b8051426182073558b29e846c04 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 20 Dec 2014 03:43:54 +1100 Subject: [PATCH 426/909] Transient vectors implement ICounted --- pixie/vm/persistent_vector.py | 4 ++++ tests/pixie/tests/collections/test-vectors.pxi | 7 +++++++ 2 files changed, 11 insertions(+) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index ead07db2..25967142 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -518,6 +518,10 @@ def _push(self, val): assert isinstance(self, TransientVector) return self.conj(val) +@extend(proto._count, TransientVector) +def _count(self): + assert isinstance(self, TransientVector) + return rt.wrap(intmask(self._cnt)) proto.IVector.add_satisfies(PersistentVector._type) diff --git a/tests/pixie/tests/collections/test-vectors.pxi b/tests/pixie/tests/collections/test-vectors.pxi index b08d585d..9ef2914d 100644 --- a/tests/pixie/tests/collections/test-vectors.pxi +++ b/tests/pixie/tests/collections/test-vectors.pxi @@ -52,3 +52,10 @@ (t/assert= [] (persistent! (pop! (transient [1])))) (t/assert= [1] (persistent! (pop! (transient [1 2])))) (t/assert= [1 2] (persistent! (pop! (transient [1 2 3]))))) + +(t/deftest transient-vector-count + (t/assert= 0 (count (transient []))) + (t/assert= 1 (count (transient [1]))) + (t/assert= 2 (count (transient [1 2]))) + (t/assert= 1 (count (pop! (transient [1 2])))) + (t/assert= 100 (count (reduce conj! (transient []) (range 0 100))))) \ No newline at end of file From 6348ede963e59627d3e4ea24d8cd5d39c28508c9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 19 Dec 2014 16:23:28 -0700 Subject: [PATCH 427/909] fixed quite a few of the warnings, a few left however --- pixie/vm/code.py | 13 ++++-- pixie/vm/compiler.py | 16 ++++---- pixie/vm/custom_types.py | 10 ++--- pixie/vm/interpreter.py | 4 +- pixie/vm/keyword.py | 4 +- pixie/vm/object.py | 7 +++- pixie/vm/persistent_vector.py | 74 ++++++++++++++++++++++++++++------- pixie/vm/reader.py | 2 +- pixie/vm/rt.py | 5 ++- pixie/vm/stdlib.py | 14 +++++-- pixie/vm/string.py | 5 +++ pixie/vm/symbol.py | 4 +- target.py | 2 +- 13 files changed, 116 insertions(+), 44 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 62ca0cbe..d1c0d296 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -293,7 +293,9 @@ def invoke_with(self, args, self_fn): try: return interpret(self, args, self_obj=self_fn) except object.WrappedException as ex: - ex._ex._trace.append(object.PixieCodeInfo(self._code._name)) + code = self._code + assert isinstance(code, Code) + ex._ex._trace.append(object.PixieCodeInfo(code._name)) raise def get_closed_over(self, idx): @@ -460,7 +462,7 @@ def add_refer_symbol(self, sym, var): name = rt.name(sym) prev_binding = self._registry.get(name, None) if prev_binding is not None: - print rt.str(rt.wrap(u"Warning: "), sym, rt.wrap(u" already refers to "), prev_binding)._str + print rt.name(rt.str(rt.wrap(u"Warning: "), sym, rt.wrap(u" already refers to "), prev_binding)) self._registry[name] = var return var @@ -487,6 +489,8 @@ def resolve(self, s, use_refers=True): else: resolved_ns = self + assert isinstance(resolved_ns, Namespace) + var = resolved_ns._registry.get(name, None) if var is None and use_refers: for refer_nm in self._refers: @@ -540,8 +544,9 @@ def __init__(self, pfn): self._pfn = pfn def invoke(self, args): - tp = args[0].type()._name - affirm(False, u"No override for " + tp + u" on " + self._pfn._name + u" in protocol " + self._pfn._protocol._name) + tp = args[0].type() + assert isinstance(tp, object.Type) + affirm(False, u"No override for " + tp._name + u" on " + self._pfn._name + u" in protocol " + self._pfn._protocol._name) class Protocol(object.Object): diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index dd5c40a8..7268d46e 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -260,7 +260,7 @@ def resolve_local(ctx, name): def is_macro_call(form, ctx): if rt.seq_QMARK_(form) is true and isinstance(rt.first(form), symbol.Symbol): - name = rt.first(form)._str + name = rt.name(rt.first(form)) if resolve_local(ctx, name): return None var = resolve_var(ctx, rt.first(form)) @@ -366,7 +366,7 @@ def compile_form(form, ctx): return if isinstance(form, symbol.Symbol): - name = form._str + name = rt.name(form) loc = resolve_local(ctx, name) if loc is None: var = resolve_var(ctx, form) @@ -460,11 +460,11 @@ def add_args(name, args, ctx): for x in range(rt.count(args)): arg = rt.nth(args, rt.wrap(x)) affirm(isinstance(arg, symbol.Symbol), u"Argument names must be symbols") - if arg._str == u"&": + if rt.name(arg) == u"&": required_args = intmask(x) continue - ctx.add_local(arg._str, Arg(local_idx)) + ctx.add_local(rt.name(arg), Arg(local_idx)) local_idx += 1 return required_args @@ -504,8 +504,8 @@ def compile_fn(form, ctx): def compile_fn_body(name, args, body, ctx): - new_ctx = Context(name._str, rt.count(args), ctx) - required_args = add_args(name._str, args, new_ctx) + new_ctx = Context(rt.name(name), rt.count(args), ctx) + required_args = add_args(rt.name(name), args, new_ctx) bc = 0 if name is not None: @@ -654,7 +654,7 @@ def compile_let(form, ctx): compile_form(bind, ctx) - ctx.add_local(name._str, LetBinding(ctx.sp())) + ctx.add_local(rt.name(name), LetBinding(ctx.sp())) if ctc: ctx.enable_tail_call() @@ -691,7 +691,7 @@ def compile_loop(form, ctx): compile_form(bind, ctx) - ctx.add_local(name._str, LetBinding(ctx.sp())) + ctx.add_local(rt.name(name), LetBinding(ctx.sp())) if ctc: ctx.enable_tail_call() diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index debffa3f..cf8f985d 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -28,19 +28,19 @@ class CustomTypeInstance(Object): __immutable_fields__ = ["_type"] def __init__(self, type): affirm(isinstance(type, CustomType), u"Can't create a instance of a non custom type") - self._type = type - self._fields = [None] * self._type.get_num_slots() + self._custom_type = type + self._fields = [None] * self._custom_type.get_num_slots() def type(self): - return self._type + return self._custom_type def set_field(self, name, val): - idx = self._type.get_slot_idx(name) + idx = self._custom_type.get_slot_idx(name) self._fields[idx] = val return self def get_field(self, name): - idx = self._type.get_slot_idx(name) + idx = self._custom_type.get_slot_idx(name) return self._fields[idx] def set_field_by_idx(self, idx, val): diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 137f6a85..f4f5e5ad 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -290,7 +290,9 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): debug_ip = frame.ip var = frame.pop() if not isinstance(var, code.Var): - affirm(False, u"Can't deref " + var.type()._name) + tp = var.type() + assert isinstance(tp, Type) + affirm(False, u"Can't deref " + tp._name) try: frame.push(var.deref()) continue diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index 59df31e6..866757ba 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -74,5 +74,7 @@ def _hash(self): @as_var("keyword") def _keyword(s): - affirm(isinstance(s, String), u"Symbol name must be a string") + if not isinstance(s, String): + from pixie.vm.object import runtime_error + runtime_error(u"Symbol name must be a string") return keyword(s._str) \ No newline at end of file diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 14ffdfe0..7c40eee6 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -3,6 +3,7 @@ class Object(object): """ Base Object for all VM objects """ + _attrs_ = () def type(self): affirm(False, u".type isn't overloaded") @@ -114,7 +115,7 @@ def __repr__(self): s.append(x.__repr__()) s.append(u"\n") - s.extend([u"RuntimeException: " + rt.str(self._data)._str + u"\n"]) + s.extend([u"RuntimeException: " + rt.name(rt.str(self._data)) + u"\n"]) return u"".join(s) @@ -180,7 +181,9 @@ def __init__(self, name, tp): self._tp = tp def __repr__(self): - return u"in polymorphic function " + self._name + u" dispatching on " + self._tp._name + u"\n" + tp = self._tp + assert isinstance(tp, Type) + return u"in polymorphic function " + self._name + u" dispatching on " + tp._name + u"\n" diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index ead07db2..438fbdbc 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -59,6 +59,7 @@ def array_for(self, i): assert isinstance(node, Node) node = node._array[(i >> level) & 0x01f] level -= 5 + assert isinstance(node, Node) return node._array affirm(False, u"Index out of Range") @@ -79,13 +80,19 @@ def conj(self, val): new_tail.append(val) return PersistentVector(self._meta, self._cnt + 1, self._shift, self._root, new_tail) - tail_node = Node(self._root._edit, self._tail) + root = self._root + assert isinstance(root, Node) + tail_node = Node(root._edit, self._tail) new_shift = self._shift if (self._cnt >> 5) > (r_uint(1) << self._shift): - new_root = Node(self._root._edit) + root = self._root + assert isinstance(root, Node) + new_root = Node(root._edit) new_root._array[0] = self._root - new_root._array[1] = new_path(self._root._edit, self._shift, tail_node) + root = self._root + assert isinstance(root, Node) + new_root._array[1] = new_path(root._edit, self._shift, tail_node) new_shift += 5 else: @@ -95,7 +102,13 @@ def conj(self, val): def push_tail(self, level, parent, tail_node): subidx = ((self._cnt - 1) >> level) & 0x01f + assert isinstance(parent, Node) ret = Node(parent._edit, parent._array[:]) + + root = self._root + assert isinstance(root, Node) + + if (level == 5): node_to_insert = tail_node else: @@ -103,7 +116,7 @@ def push_tail(self, level, parent, tail_node): if child is not None: node_to_insert = self.push_tail(level - 5, child, tail_node) else: - node_to_insert = new_path(self._root._edit, level - 5, tail_node) + node_to_insert = new_path(root._edit, level - 5, tail_node) ret._array[subidx] = node_to_insert return ret @@ -138,11 +151,14 @@ def pop(self): def pop_tail(self, level, node): sub_idx = ((self._cnt - 1) >> level) & 0x01f if level > 5: + assert isinstance(node, Node) new_child = self.pop_tail(level - 5, node._array[sub_idx]) if new_child is None or sub_idx == 0: return None else: - ret = Node(self._root._edit, node._array[:]) + root = self._root + assert isinstance(root, Node) + ret = Node(root._edit, node._array[:]) ret._array[sub_idx] = new_child return ret @@ -150,7 +166,10 @@ def pop_tail(self, level, node): return None else: - ret = Node(self._root._edit, node._array[:]) + root = self._root + assert isinstance(root, Node) + assert isinstance(node, Node) + ret = Node(root._edit, node._array[:]) ret._array[sub_idx] = None return ret @@ -167,6 +186,7 @@ def assoc_at(self, idx, val): object.runtime_error(u"index out of range") def do_assoc(lvl, node, idx, val): + assert isinstance(node, Node) ret = Node(node._edit, node._array[:]) if lvl == 0: ret._array[idx & 0x01f] = val @@ -198,16 +218,24 @@ def __init__(self, cnt, shift, root, tail): @staticmethod def editable_root(node): + assert isinstance(node, Node) return Node(edited, node._array[:]) def ensure_editable(self): - affirm(self._root._edit is not None, u"Transient used after call to persist!") + root = self._root + assert isinstance(root, Node) + affirm(root._edit is not None, u"Transient used after call to persist!") def ensure_node_editable(self, node): - if node._edit is self._root._edit: + assert isinstance(node, Node) + root = self._root + assert isinstance(root, Node) + if node._edit is root._edit: return node - return Node(self._root._edit, node._array[:]) + root = self._root + assert isinstance(root, Node) + return Node(root._edit, node._array[:]) def tailoff(self): @@ -218,7 +246,10 @@ def tailoff(self): def persistent(self): self.ensure_editable() - self._root._edit = None + root = self._root + assert isinstance(root, Node) + + root._edit = None trimmed = [None] * (self._cnt - self.tailoff()) list_copy(self._tail, 0, trimmed, 0, len(trimmed)) return PersistentVector(nil, self._cnt, self._shift, self._root, trimmed) @@ -238,15 +269,19 @@ def conj(self, val): self._cnt += 1 return self - tail_node = Node(self._root._edit, self._tail) + root = self._root + assert isinstance(root, Node) + + + tail_node = Node(root._edit, self._tail) self._tail = [None] * 32 self._tail[0] = val new_shift = self._shift if (self._cnt >> 5) > (r_uint(1) << self._shift): - new_root = Node(self._root._edit) + new_root = Node(root._edit) new_root._array[0] = self._root - new_root._array[1] = new_path(self._root._edit, self._shift, tail_node) + new_root._array[1] = new_path(root._edit, self._shift, tail_node) new_shift += 5 else: @@ -260,6 +295,9 @@ def conj(self, val): def push_tail(self, level, parent, tail_node): parent = self.ensure_node_editable(parent) + root = self._root + assert isinstance(root, Node) + sub_idx = ((self._cnt - 1) >> level) & 0x01f ret = parent @@ -270,7 +308,7 @@ def push_tail(self, level, parent, tail_node): if child is not None: node_to_insert = self.push_tail(level - 5, child, tail_node) else: - node_to_insert = new_path(self._root._edit, level-5, tail_node) + node_to_insert = new_path(root._edit, level-5, tail_node) ret._array[sub_idx] = node_to_insert return ret @@ -331,8 +369,12 @@ def pop(self): new_root = self.pop_tail(self._shift, self._root) new_shift = self._shift + root = self._root + assert isinstance(root, Node) + + if new_root is None: - new_root = Node(self._root._edit) + new_root = Node(root._edit) if self._shift > 5 and new_root._array[1] is None: new_root = self.ensure_node_editable(new_root._array[0]) @@ -403,6 +445,7 @@ def _val_at(self, key, not_found): @extend(proto._eq, PersistentVector) def _eq(self, obj): + assert isinstance(self, PersistentVector) if self is obj: return true elif isinstance(obj, PersistentVector): @@ -426,6 +469,7 @@ def _eq(self, obj): @extend(proto._contains_key, PersistentVector) def _contains_key(self, key): + assert isinstance(self, PersistentVector) if not isinstance(key, Integer): return false else: diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 198e450d..5a7515f3 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -273,7 +273,7 @@ def invoke(self, rdr, ch): itm = read(rdr, True) affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") - return keyword(itm._str) + return keyword(rt.name(itm)) class LiteralStringReader(ReaderHandler): def invoke(self, rdr, ch): diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 8d18e12e..13839e88 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -32,7 +32,10 @@ def wrapper(*args): ret = fn.invoke(py_list(args)) if ret is nil: return None - affirm(isinstance(ret, String), u"Invalid return value, expected String") + + if not isinstance(ret, String): + from pixie.vm.object import runtime_error + runtime_error(u"Invalid return value, expected String") return ret._str return wrapper else: diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 77a16605..646c9ebf 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -100,8 +100,9 @@ def __meta(self, meta): def default_str(x): from pixie.vm.string import String - - return rt.wrap(u"") + tp = x.type() + assert isinstance(tp, Type) + return rt.wrap(u"") _str.set_default_fn(wrap_fn(default_str)) _repr.set_default_fn(wrap_fn(default_str)) @@ -164,11 +165,13 @@ def type(x): @extend(_str, Type) def _str(tp): import pixie.vm.string as string + assert isinstance(tp, Type) return string.rt.wrap(u"") @extend(_repr, Type) def _repr(tp): import pixie.vm.string as string + assert isinstance(tp, Type) return string.rt.wrap(tp._name) @extend(_first, nil._type) @@ -294,7 +297,7 @@ def str__args(args): from pixie.vm.string import String acc = [] for x in args: - acc.append(rt._str(x)._str) + acc.append(rt.name(rt._str(x))) return rt.wrap(u"".join(acc)) @as_var("apply") @@ -430,6 +433,8 @@ def the_ns(ns_name): @as_var("ns-map") def ns_map(ns): + from pixie.vm.code import Namespace + assert isinstance(ns, Namespace) from pixie.vm.symbol import Symbol affirm(isinstance(ns, code.Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") @@ -498,7 +503,8 @@ def satisfy(protocol, tp): @as_var("type-by-name") def type_by_name(nm): import pixie.vm.string as string - affirm(isinstance(nm, string.String), u"type name must be string") + if not isinstance(nm, string.String): + runtime_error(u"type name must be string") return _type_registry.get_by_name(nm._str, nil) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index 8aa04a17..88d756b8 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -25,6 +25,7 @@ def _str(x): @extend(proto._repr, String) def _repr(self): res = u"" + assert isinstance(self, String) for c in self._str: if c == "\"": res += u"\\\"" @@ -44,10 +45,12 @@ def _repr(self): @extend(proto._count, String) def _count(self): + assert isinstance(self, String) return rt.wrap(len(self._str)) @extend(proto._nth, String) def _nth(self, idx): + assert isinstance(self, String) i = idx.int_val() if 0 <= i < len(self._str): return Character(ord(self._str[i])) @@ -55,6 +58,7 @@ def _nth(self, idx): @extend(proto._eq, String) def _eq(self, v): + assert isinstance(self, String) if not isinstance(v, String): return false return true if self._str == v._str else false @@ -128,4 +132,5 @@ def _namespace(self): @extend(proto._hash, String) def _hash(self): + assert isinstance(self, String) return rt.wrap(intmask(util.hash_unencoded_chars(self._str))) diff --git a/pixie/vm/symbol.py b/pixie/vm/symbol.py index c052cf80..59d31ec4 100644 --- a/pixie/vm/symbol.py +++ b/pixie/vm/symbol.py @@ -81,7 +81,9 @@ def _hash(self): @as_var("symbol") def _symbol(s): - affirm(isinstance(s, String), u"Symbol name must be a string") + if not isinstance(s, String): + from pixie.vm.object import runtime_error + runtime_error(u"Symbol name must be a string") return symbol(s._str) diff --git a/target.py b/target.py index afc52f21..6cea8b38 100644 --- a/target.py +++ b/target.py @@ -177,7 +177,7 @@ def run_load_stdlib(): return import pixie.vm.compiler as compiler import pixie.vm.reader as reader - f = open(rpath.rjoin(str(load_path.deref()._str), "pixie/stdlib.pxi")) + f = open(rpath.rjoin(str(rt.name(load_path.deref())), "pixie/stdlib.pxi")) data = f.read() f.close() rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pxi") From 2b965e51b24a6540560ee37d5694b7bdd93710d5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 19 Dec 2014 16:29:49 -0700 Subject: [PATCH 428/909] removed _is_macro warnings --- pixie/vm/code.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index d1c0d296..35031d01 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -356,6 +356,7 @@ def type(self): return Var._type def __init__(self, ns, name): + BaseCode.__init__(self) self._ns = ns self._name = name self._rev = 0 From 3e23e6b26966e0bc8bca294f87ffab6dcadc7c77 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 19 Dec 2014 16:54:34 -0700 Subject: [PATCH 429/909] last warnings removed --- pixie/vm/code.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 35031d01..b7c36770 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -92,6 +92,7 @@ def slice_from_start(from_list, count, extra=r_uint(0)): class BaseCode(object.Object): def __init__(self): assert isinstance(self, BaseCode) + self._name = u"unknown" self._is_macro = False self._meta = nil @@ -105,6 +106,7 @@ def set_macro(self): self._is_macro = True def is_macro(self): + assert isinstance(self, BaseCode) return self._is_macro def get_consts(self): @@ -542,12 +544,21 @@ def get_var_if_defined(ns, name, els=None): class DefaultProtocolFn(NativeFn): def __init__(self, pfn): + BaseCode.__init__(self) self._pfn = pfn def invoke(self, args): tp = args[0].type() assert isinstance(tp, object.Type) - affirm(False, u"No override for " + tp._name + u" on " + self._pfn._name + u" in protocol " + self._pfn._protocol._name) + pfn = self._pfn + if isinstance(pfn, PolymorphicFn): + protocol = pfn._protocol + elif isinstance(pfn, DoublePolymorphicFn): + protocol = pfn._protocol + else: + assert False + assert isinstance(protocol, Protocol) + affirm(False, u"No override for " + tp._name + u" on " + self._pfn._name + u" in protocol " + protocol._name) class Protocol(object.Object): From a7047932ee6eb4128bdd94c2fa65eda83bba3233 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 19 Dec 2014 17:47:25 -0700 Subject: [PATCH 430/909] a few bug fixes for translation --- pixie/vm/numbers.py | 2 +- pixie/vm/object.py | 6 +++--- pixie/vm/stdlib.py | 1 + 3 files changed, 5 insertions(+), 4 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 3afe9505..624c4dc3 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -48,7 +48,7 @@ def bigint_val(self): return self._bigint_val def type(self): - return self._type + return BigInteger._type class Float(Number): _type = object.Type(u"pixie.stdlib.Float", Number._type) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 7c40eee6..006f578f 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -85,10 +85,10 @@ def subclasses(self): @jit.elidable_promote() def istypeinstance(obj, t): - if obj._type is t: + if obj.type() is t: return True - elif obj._type._parent is not None: - obj_type = obj._type._parent + elif obj.type()._parent is not None: + obj_type = obj.type()._parent while obj_type is not None: if obj_type is t: return True diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 646c9ebf..768229c8 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -560,6 +560,7 @@ def _try_catch(main_fn, catch_fn, final): if isinstance(ex, Exception): if not we_are_translated(): print "Python Error Info: ", ex.__dict__, ex + raise ex = RuntimeException(rt.wrap(u"Some error")) else: ex = RuntimeException(nil) From fe43dadb0ee79a3340424215ccff98882436b397 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 19 Dec 2014 17:48:11 -0700 Subject: [PATCH 431/909] disable a flakey test --- tests/pixie/tests/test-io.pxi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 930acbac..4c81ab0a 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -13,6 +13,8 @@ (let [f (io/run-command "ls tests/pixie/tests/test-io.txt")] (t/assert= f "tests/pixie/tests/test-io.txt\n"))) +(comment (t/deftest test-slurp-spit (let [val (vec (range 128))] (t/assert= val (read-string (io/slurp "test.tmp" (io/spit "test.tmp" val)))))) +) \ No newline at end of file From 107863bc8269d07c9ed7f75696dbe81e00bc27f5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 19 Dec 2014 18:01:21 -0700 Subject: [PATCH 432/909] translation fix --- pixie/vm/object.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 006f578f..4a78cbec 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -85,10 +85,12 @@ def subclasses(self): @jit.elidable_promote() def istypeinstance(obj, t): - if obj.type() is t: + obj_type = obj.type() + assert isinstance(obj_type, Type) + if obj_type is t: return True - elif obj.type()._parent is not None: - obj_type = obj.type()._parent + elif obj_type._parent is not None: + obj_type = obj_type._parent while obj_type is not None: if obj_type is t: return True From cdc19cccae0d4e3321558b424703afaf32f154a3 Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Fri, 19 Dec 2014 20:25:33 -0600 Subject: [PATCH 433/909] Remove PEP8 warnings in persistent_vector.py --- pixie/vm/persistent_vector.py | 49 +++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 20 deletions(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index d4dd6348..0f070df9 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -4,15 +4,15 @@ from pixie.vm.primitives import nil, true, false from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto -from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint, intmask, widen +from pixie.vm.code import extend, as_var +from rpython.rlib.rarithmetic import r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt - class Node(object.Object): _type = object.Type(u"pixie.stdlib.PersistentVectorNode") + def type(self): return Node._type @@ -73,7 +73,6 @@ def nth(self, i, not_found=nil): def conj(self, val): assert self._cnt < r_uint(0xFFFFFFFF) - i = self._cnt if self._cnt - self.tailoff() < 32: new_tail = self._tail[:] @@ -108,7 +107,6 @@ def push_tail(self, level, parent, tail_node): root = self._root assert isinstance(root, Node) - if (level == 5): node_to_insert = tail_node else: @@ -121,8 +119,6 @@ def push_tail(self, level, parent, tail_node): ret._array[subidx] = node_to_insert return ret - - def pop(self): affirm(self._cnt != 0, u"Can't pop an empty vector") @@ -131,7 +127,7 @@ def pop(self): if self._cnt - self.tailoff() > 1: size = len(self._tail) - 1 - assert size >= 0 # for translation + assert size >= 0 # for translation new_tail = self._tail[:size] return PersistentVector(self._meta, self._cnt - 1, self._shift, self._root, new_tail) @@ -185,6 +181,7 @@ def assoc_at(self, idx, val): else: object.runtime_error(u"index out of range") + def do_assoc(lvl, node, idx, val): assert isinstance(node, Node) ret = Node(node._edit, node._array[:]) @@ -195,6 +192,7 @@ def do_assoc(lvl, node, idx, val): ret._array[subidx] = do_assoc(lvl - 5, node._array[subidx], idx, val) return ret + def new_path(edit, level, node): if level == 0: return node @@ -202,8 +200,10 @@ def new_path(edit, level, node): ret._array[0] = new_path(edit, level - 5, node) return ret + edited = u"edited" + class TransientVector(object.Object): _type = object.Type(u"pixie.stdlib.TransientVector") @@ -237,7 +237,6 @@ def ensure_node_editable(self, node): assert isinstance(root, Node) return Node(root._edit, node._array[:]) - def tailoff(self): if self._cnt < 32: return 0 @@ -272,7 +271,6 @@ def conj(self, val): root = self._root assert isinstance(root, Node) - tail_node = Node(root._edit, self._tail) self._tail = [None] * 32 self._tail[0] = val @@ -308,7 +306,7 @@ def push_tail(self, level, parent, tail_node): if child is not None: node_to_insert = self.push_tail(level - 5, child, tail_node) else: - node_to_insert = new_path(root._edit, level-5, tail_node) + node_to_insert = new_path(root._edit, level - 5, tail_node) ret._array[sub_idx] = node_to_insert return ret @@ -372,7 +370,6 @@ def pop(self): root = self._root assert isinstance(root, Node) - if new_root is None: new_root = Node(root._edit) @@ -408,9 +405,6 @@ def pop_tail(self, level, node): return ret - - - @jit.unroll_safe def list_copy(from_lst, from_loc, to_list, to_loc, count): from_loc = r_uint(from_loc) @@ -419,22 +413,23 @@ def list_copy(from_lst, from_loc, to_list, to_loc, count): i = r_uint(0) while i < count: - to_list[to_loc + i] = from_lst[from_loc+i] + to_list[to_loc + i] = from_lst[from_loc + i] i += 1 return to_list - @extend(proto._count, PersistentVector) def _count(self): assert isinstance(self, PersistentVector) return rt.wrap(intmask(self._cnt)) + @extend(proto._nth, PersistentVector) def _nth(self, idx): assert isinstance(self, PersistentVector) return self.nth(idx.int_val()) + @extend(proto._val_at, PersistentVector) def _val_at(self, key, not_found): assert isinstance(self, PersistentVector) @@ -443,6 +438,7 @@ def _val_at(self, key, not_found): else: return not_found + @extend(proto._eq, PersistentVector) def _eq(self, obj): assert isinstance(self, PersistentVector) @@ -467,6 +463,7 @@ def _eq(self, obj): return false return true + @extend(proto._contains_key, PersistentVector) def _contains_key(self, key): assert isinstance(self, PersistentVector) @@ -475,32 +472,38 @@ def _contains_key(self, key): else: return true if key.int_val() >= 0 and key.int_val() < intmask(self._cnt) else false + @extend(proto._conj, PersistentVector) def _conj(self, v): assert isinstance(self, PersistentVector) return self.conj(v) + @extend(proto._push, PersistentVector) def _push(self, v): assert isinstance(self, PersistentVector) return self.conj(v) + @extend(proto._pop, PersistentVector) def _pop(self): assert isinstance(self, PersistentVector) return self.pop() + @extend(proto._assoc, PersistentVector) def _assoc(self, idx, val): assert isinstance(self, PersistentVector) affirm(isinstance(idx, Integer), u"key must be an integer") return self.assoc_at(r_uint(idx.int_val()), val) + @extend(proto._meta, PersistentVector) def _meta(self): assert isinstance(self, PersistentVector) return self.meta() + @extend(proto._with_meta, PersistentVector) def _with_meta(self, meta): assert isinstance(self, PersistentVector) @@ -508,8 +511,9 @@ def _with_meta(self, meta): _reduce_driver = jit.JitDriver(name="pixie.stdlib.PersistentVector_reduce", - greens=["f"], - reds="auto") + greens=["f"], + reds="auto") + @extend(proto._reduce, PersistentVector) def _reduce(self, f, init): @@ -518,7 +522,6 @@ def _reduce(self, f, init): while i < self._cnt: array = self.array_for(i) for j in range(len(array)): - item = array[j] _reduce_driver.jit_merge_point(f=f) init = f.invoke([init, array[j]]) @@ -537,31 +540,37 @@ def vector__args(args): acc = rt._conj_BANG_(acc, args[x]) return rt._persistent_BANG_(acc) + @extend(proto._transient, PersistentVector) def _transient(self): assert isinstance(self, PersistentVector) return TransientVector(self._cnt, self._shift, TransientVector.editable_root(self._root), TransientVector.editable_tail(self._tail)) + @extend(proto._persistent_BANG_, TransientVector) def _persistent(self): assert isinstance(self, TransientVector) return self.persistent() + @extend(proto._conj_BANG_, TransientVector) def _conj(self, val): assert isinstance(self, TransientVector) return self.conj(val) + @extend(proto._pop_BANG_, TransientVector) def _pop(self): assert isinstance(self, TransientVector) return self.pop() + @extend(proto._push_BANG_, TransientVector) def _push(self, val): assert isinstance(self, TransientVector) return self.conj(val) + @extend(proto._count, TransientVector) def _count(self): assert isinstance(self, TransientVector) From c4ec867ae058968dab055b815b786632f3cc1b8f Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Fri, 19 Dec 2014 19:57:59 -0600 Subject: [PATCH 434/909] Remove pep8 warnings in atom.py --- pixie/vm/atom.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pixie/vm/atom.py b/pixie/vm/atom.py index 46223da3..d372e920 100644 --- a/pixie/vm/atom.py +++ b/pixie/vm/atom.py @@ -1,14 +1,12 @@ -import pixie.vm.rt as rt import pixie.vm.object as object from pixie.vm.code import extend, as_var from pixie.vm.primitives import nil import pixie.vm.stdlib as proto -import rpython.rlib.jit as jit -import pixie.vm.stdlib as proto class Atom(object.Object): _type = object.Type(u"pixie.stdlib.Atom") + def type(self): return Atom._type @@ -16,18 +14,19 @@ def __init__(self, boxed_value): self._boxed_value = boxed_value - @extend(proto._reset_BANG_, Atom) def _reset(self, v): assert isinstance(self, Atom) self._boxed_value = v return self + @extend(proto._deref, Atom) def _deref(self): assert isinstance(self, Atom) return self._boxed_value + @as_var("atom") def atom(val=nil): return Atom(val) From 7c0890d3cb3f8ac740d31f8723691e8eb5f92230 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 20 Dec 2014 13:38:19 -0700 Subject: [PATCH 435/909] fixed the bug with recur by removing fn-recur, it didn't JIT well anywas --- pixie/stdlib.pxi | 9 ++++----- pixie/vm/code.py | 1 - pixie/vm/compiler.py | 11 ++++++++++- pixie/vm/interpreter.py | 25 ++++++++----------------- pixie/vm/rt.py | 2 +- 5 files changed, 23 insertions(+), 25 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 36b14e49..994231be 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -166,8 +166,8 @@ (def preserving-reduced - (fn [rf] - (fn [a b] + (fn preserving-reduced [rf] + (fn pr-inner [a b] (let [ret (rf a b)] (if (reduced? ret) (reduced ret) @@ -819,10 +819,9 @@ Stops if it finds such an element." :else (recur pred (next coll)))) (extend -count MapEntry (fn [self] 2)) -(extend -nth MapEntry (fn [self idx not-found] +(extend -nth MapEntry (fn map-entry-nth [self idx] (cond (= idx 0) (-key self) - (= idx 1) (-val self) - :else not-found))) + (= idx 1) (-val self)))) (extend -reduce MapEntry indexed-reduce) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index b7c36770..32679227 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -23,7 +23,6 @@ "POP", "DEREF_VAR", "INSTALL", - "RECUR", "LOOP_RECUR", "ARG", "PUSH_SELF", diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7268d46e..7a984d11 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -502,6 +502,7 @@ def compile_fn(form, ctx): if rt.meta(name) is not nil: compile_meta(rt.meta(name), ctx) +LOOP = symbol.symbol(u"loop") def compile_fn_body(name, args, body, ctx): new_ctx = Context(rt.name(name), rt.count(args), ctx) @@ -512,7 +513,15 @@ def compile_fn_body(name, args, body, ctx): affirm(isinstance(name, symbol.Symbol), u"Function names must be symbols") #new_ctx.add_local(name._str, Self()) - new_ctx.push_recur_point(FunctionRecurPoint()) + arg_syms = EMPTY + for x in range(rt.count(args)): + sym = rt.nth(args, rt.wrap(x)) + if not rt.name(sym) == u"&": + arg_syms = rt.conj(rt.conj(arg_syms, sym), sym) + + body = rt.list(rt.cons(LOOP, rt.cons(arg_syms, body))) + + #new_ctx.push_recur_point(FunctionRecurPoint()) new_ctx.disable_tail_call() if body is nil: diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index f4f5e5ad..6397fbe1 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -80,7 +80,10 @@ def push(self, val): def pop(self): #print type(self.sp), self.sp self.sp -= 1 - assert 0 <= self.sp < len(self.stack), u"Stack out of range: " + unicode(str(self.sp)) + + if not 0 <= self.sp < len(self.stack): + runtime_error(u"Stack out of range: " + unicode(str(self.sp))) + v = self.stack[self.sp] self.stack[self.sp] = None return v @@ -96,7 +99,10 @@ def push_nth(self, delta): self.push(self.nth(delta)) def push_arg(self, idx): - affirm(0 <= idx < len(self.args), u"Argument doesn't exist") + if not 0 <= idx < len(self.args): + runtime_error(u"Invalid argument " + unicode(str(idx)) + u" function takes " + + unicode(str(len(self.args))) + u" args") + self.push(self.args[r_uint(idx)]) @unroll_safe @@ -302,21 +308,6 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): ex._ex._trace.append(dp) raise - - if inst == code.RECUR: - argc = frame.get_inst() - args = frame.pop_n(argc) - - frame = Frame(frame.code_obj, args, frame.self_obj) - - jitdriver.can_enter_jit(bc=frame.bc, - ip=frame.ip, - sp=frame.sp, - base_code=frame.base_code, - frame=frame, - is_continuation=frame._is_continuation) - continue - if inst == code.PUSH_SELF: frame.push(frame.self_obj) continue diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 13839e88..6eb52544 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -46,7 +46,7 @@ def wrapper(*args): return import sys - sys.setrecursionlimit(100000) # Yeah we blow the stack sometimes, we promise it's not a bug + #sys.setrecursionlimit(10000) # Yeah we blow the stack sometimes, we promise it's not a bug import pixie.vm.numbers as numbers import pixie.vm.bits From bc70a33ca76f8389d14963183dcfd76f972dcec2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 20 Dec 2014 15:07:26 -0700 Subject: [PATCH 436/909] added groupings --- pixie/ffi-infer.pxi | 24 +++++++++++++++++++++--- 1 file changed, 21 insertions(+), 3 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index e128b23c..d89550b2 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -19,6 +19,18 @@ (str "std::cout << \"{:name " (keyword name)" :sizeof \" << sizeof( " name ") << \"}\" << std::endl; \n")) +(defmethod emit-infer-code :offsetof-struct + [{:keys [struct-name member-name]}] + (str "std::cout << \"{:name " (keyword member-name)" :offsetof \" << offsetof(struct " struct-name ", " + member-name + ") << \"}\" << std::endl; \n")) + +(defmethod emit-infer-code :group + [{:keys [ops]}] + (str "std::cout << \"[\" << std::endl; " + (apply str (map emit-infer-code ops)) + "std::cout << \"]\" << std::endl; ")) + (defn start-string [] " #include #include @@ -56,13 +68,19 @@ (io/spit "/tmp/tmp.cpp" (str (start-string) (apply str (map emit-infer-code [ {:op :constant :name "RAND_MAX"} - {:op :sizeof-struct :name "stat"} {:op :sizeof :name "int"} - {:op :offsetof-struct :struct "stat" :member ""}])) + {:op :group + :ops [ {:op :sizeof-struct :name "stat"} + {:op :offsetof-struct :struct-name "stat" :member-name "st_size"}]}])) (end-string))) (do (io/spit "/tmp/tmp.cpp" (str (start-string) - (emit-infer-code {:op :constant :name "RAND_MAX"}) + (apply str (map emit-infer-code + [ {:op :constant :name "RAND_MAX"} + {:op :sizeof :name "int"} + {:op :group + :ops [ {:op :sizeof-struct :name "stat"} + {:op :offsetof-struct :struct-name "stat" :member-name "st_size"}]}])) (end-string))) (read-string (io/run-command "c++ /tmp/tmp.cpp && ./a.out"))) From f8d86b11cbdbac9c2c36058e2e892f2b3e2d377b Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Sat, 20 Dec 2014 14:01:00 -0600 Subject: [PATCH 437/909] Use destructure form in test.py --- pixie/test.pxi | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/pixie/test.pxi b/pixie/test.pxi index e529f873..560631c1 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -49,14 +49,13 @@ (println "Looking for tests...") (foreach [path @load-paths] (println "Looking for tests in:" path) - (foreach [desc (pixie.path/file-list path)] - (if (= (nth desc 1) :file) - (let [filename (nth desc 2)] - (if (pixie.string/starts-with filename "test-") - (if (pixie.string/ends-with filename ".pxi") - (let [fullpath (str (nth desc 0) "/" filename)] - (println "Loading" fullpath) - (load-file fullpath))))))))) + (foreach [[dir desc filename] (pixie.path/file-list path)] + (if (= desc :file) + (if (pixie.string/starts-with filename "test-") + (if (pixie.string/ends-with filename ".pxi") + (let [fullpath (str dir "/" filename)] + (println "Loading" fullpath) + (load-file fullpath)))))))) (defmacro assert= [x y] From fbe59cd88e5fa120c752218d595289fca5c2bf1f Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Sun, 21 Dec 2014 01:09:05 -0600 Subject: [PATCH 438/909] Fix bug in filter, add tests --- pixie/stdlib.pxi | 3 ++- tests/pixie/tests/test-stdlib.pxi | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 994231be..955f7906 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1631,7 +1631,8 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (let [iter (iterator coll)] (loop [] (when (not (at-end? iter)) - (yield (current iter)) + (if (f (current iter)) + (yield (current iter))) (move-next! iter) (recur)))))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 3d72f43d..dc958839 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -224,6 +224,22 @@ (t/assert= (some even? []) false) (t/assert= (some odd? [2]) false)) +(t/deftest test-filter + (t/assert= (vec (filter (fn [x] true) [])) []) + (t/assert= (vec (filter (fn [x] false) [])) []) + (t/assert= (vec (filter (fn [x] true) [1 2 3 4])) [1 2 3 4]) + (t/assert= (vec (filter (fn [x] false) [1 2 3 4])) []) + (t/assert= (vec (filter (fn [x] true)) + [1 2 3 4]) + [1 2 3 4]) + (t/assert= (vec (filter (fn [x] false)) + [1 2 3 4]) + []) + (t/assert= (seq (filter (fn [x] true) [])) nil) + (t/assert= (seq (filter (fn [x] false) [])) nil) + (t/assert= (seq (filter (fn [x] true) [1 2 3 4])) '(1 2 3 4)) + (t/assert= (seq (filter (fn [x] false) [1 2 3 4])) nil)) + (t/deftest test-distinct (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) From 755a5722c25c70835a57f1817a93ca9c98e6a498 Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Mon, 22 Dec 2014 16:49:38 -0600 Subject: [PATCH 439/909] Fix travis.yml allow build failure syntax --- .travis.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 275e9fbc..cd706325 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,8 @@ env: matrix: fast_finish: true allow_failures: - - TARGET_OPTS='target_preload.py' + - env: JIT_OPTS='--opt=jit' TARGET_OPTS='target_preload.py' + - env: JIT_OPTS='' TARGET_OPTS='target_preload.py' script: - make PYTHON=python build From b969ca8dbf90e04b1ff451d9588da814096018b8 Mon Sep 17 00:00:00 2001 From: Frank Licea Date: Mon, 22 Dec 2014 16:49:56 -0600 Subject: [PATCH 440/909] Don't spam the IRC channel from travis --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index cd706325..1fe62851 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,4 @@ before_install: - sudo apt-get install libffi-dev libedit-dev notifications: - irc: "chat.freenode.net#pixie-lang" + # irc: "chat.freenode.net#pixie-lang" From 7c48019576ca24faa569b602cebd1cdb52cd2f8b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 23 Dec 2014 16:08:11 -0700 Subject: [PATCH 441/909] fixing some ffi stuff via copying the pypy ffi code --- pixie/io.pxi | 2 +- pixie/stdlib.pxi | 2 + pixie/vm/libs/ffi.py | 202 +++++++++++++++++++++++++++++++++---------- 3 files changed, 161 insertions(+), 45 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 98eb242f..51e6d3e7 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -104,7 +104,7 @@ (set-buffer-count! buffer read-count) read-count)) (read-byte [this] - (fgetc buffer)) + (fgetc fp)) IClosable (close [this] (pclose fp)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 955f7906..4673fbc0 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -21,6 +21,8 @@ (def floor (ffi-fn libm "floor" [CDouble] CDouble)) (def lround (ffi-fn libm "lround" [CDouble] CInt)) + (puts "testing ffi....") + (def reset! -reset!) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 8c35c46e..632399bf 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -5,13 +5,16 @@ import pixie.vm.stdlib as proto from pixie.vm.code import as_var, affirm, extend import pixie.vm.rt as rt -from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.lltypesystem import rffi, lltype, llmemory from pixie.vm.primitives import nil from pixie.vm.numbers import Integer, Float from pixie.vm.string import String from pixie.vm.util import unicode_to_utf8 from rpython.rlib import clibffi -from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION +from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION, CIF_DESCRIPTION_P, \ + FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG +import rpython.rlib.jit_libffi as jit_libffi +from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import intmask, r_uint @@ -88,44 +91,9 @@ def __init__(self, lib, name, arg_types, ret_type): def thaw(self): if not self._is_inited: self._f_ptr = self._lib.get_fn_ptr(self._name) - nargs = len(self._arg_types) - - exchange_buffer_size = nargs * rffi.sizeof(rffi.CCHARP) - - cd = lltype.malloc(CIF_DESCRIPTION, nargs, flavor="raw") - cd.abi = clibffi.FFI_DEFAULT_ABI - cd.nargs = nargs - cd.rtype = self._ret_type.ffi_type() - - atypes = lltype.malloc(clibffi.FFI_TYPE_PP.TO, nargs, flavor="raw") - arg0_offset = exchange_buffer_size - for idx in range(nargs): - cd.exchange_args[idx] = exchange_buffer_size - tp = self._arg_types[idx] - native_size = tp.ffi_size() - atypes[idx] = tp.ffi_type() - exchange_buffer_size += native_size - - ret_offset = exchange_buffer_size - exchange_buffer_size += self._ret_type.ffi_size() - - - - cd.atypes = atypes - cd.exchange_size = exchange_buffer_size - cd.exchange_result = ret_offset - cd.exchange_result_libffi = ret_offset - - jit_ffi_prep_cif(cd) - self._cd = cd - self._transfer_size = exchange_buffer_size - self._arg0_offset = arg0_offset - self._ret_offset = ret_offset - + CifDescrBuilder(self._arg_types, self._ret_type).rawallocate(self) self._is_inited = True - return self - def _cleanup_(self): self._rev += 1 self._f_ptr = lltype.nullptr(rffi.VOIDP.TO) @@ -136,13 +104,13 @@ def _cleanup_(self): def prep_exb(self, args): if not self._is_inited: self.thaw() - exb = lltype.malloc(rffi.CCHARP.TO, self._transfer_size, flavor="raw") - offset_p = rffi.ptradd(exb, self._arg0_offset) + size = self._cd.exchange_size + exb = lltype.malloc(rffi.CCHARP.TO, size, flavor="raw") tokens = [None] * len(args) - for x in range(len(self._arg_types)): - tokens[x] = self._arg_types[x].ffi_set_value(offset_p, args[x]) - offset_p = rffi.ptradd(offset_p, self._arg_types[x].ffi_size()) + for i, tp in enumerate(self._arg_types): + offset_p = rffi.ptradd(exb, self._cd.exchange_args[i]) + tokens[i] = tp.ffi_set_value(offset_p, args[i]) return exb, tokens @@ -164,6 +132,7 @@ def _invoke(self, args): t.finalize_token() lltype.free(exb, flavor="raw") + keepalive_until_here(args) return ret_val def invoke(self, args): @@ -211,7 +180,8 @@ def __init__(self, size): def __del__(self): - lltype.free(self._buffer, flavor="raw") + #lltype.free(self._buffer, flavor="raw") + pass def set_used_size(self, size): self._used_size = size @@ -378,6 +348,8 @@ def ffi_set_value(self, ptr, val): pnt[0] = val.buffer() elif isinstance(val, VoidP): pnt[0] = val.raw_data() + elif val is nil: + pnt[0] = rffi.cast(rffi.VOIDP, 0) else: print val affirm(False, u"Cannot encode this type") @@ -409,7 +381,149 @@ def __init__(self, tp, buffer): def type(self): return self._type +import sys + +BIG_ENDIAN = sys.byteorder == 'big' +USE_C_LIBFFI_MSVC = getattr(clibffi, 'USE_C_LIBFFI_MSVC', False) + + + +class CifDescrBuilder(py_object): + rawmem = lltype.nullptr(rffi.CCHARP.TO) + + def __init__(self, fargs, fresult): + self.fargs = fargs + self.fresult = fresult + + def fb_alloc(self, size): + size = llmemory.raw_malloc_usage(size) + if not self.bufferp: + self.nb_bytes += size + return lltype.nullptr(rffi.CCHARP.TO) + else: + result = self.bufferp + self.bufferp = rffi.ptradd(result, size) + return result + + def fb_fill_type(self, ctype, is_result_type): + return ctype.ffi_type() + + + def fb_build(self): + # Build a CIF_DESCRIPTION. Actually this computes the size and + # allocates a larger amount of data. It starts with a + # CIF_DESCRIPTION and continues with data needed for the CIF: + # + # - the argument types, as an array of 'ffi_type *'. + # + # - optionally, the result's and the arguments' ffi type data + # (this is used only for 'struct' ffi types; in other cases the + # 'ffi_type *' just points to static data like 'ffi_type_sint32'). + # + nargs = len(self.fargs) + + # start with a cif_description (cif and exchange_* fields) + self.fb_alloc(llmemory.sizeof(CIF_DESCRIPTION, nargs)) + + # next comes an array of 'ffi_type*', one per argument + atypes = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * nargs) + self.atypes = rffi.cast(FFI_TYPE_PP, atypes) + + # next comes the result type data + self.rtype = self.fb_fill_type(self.fresult, True) + + # next comes each argument's type data + for i, farg in enumerate(self.fargs): + atype = self.fb_fill_type(farg, False) + if self.atypes: + self.atypes[i] = atype + + def align_arg(self, n): + return (n + 7) & ~7 + + def fb_build_exchange(self, cif_descr): + nargs = len(self.fargs) + + # first, enough room for an array of 'nargs' pointers + exchange_offset = rffi.sizeof(rffi.CCHARP) * nargs + exchange_offset = self.align_arg(exchange_offset) + cif_descr.exchange_result = exchange_offset + cif_descr.exchange_result_libffi = exchange_offset + + if BIG_ENDIAN and self.fresult.is_primitive_integer: + # For results of precisely these types, libffi has a + # strange rule that they will be returned as a whole + # 'ffi_arg' if they are smaller. The difference + # only matters on big-endian. + if self.fresult.size < SIZE_OF_FFI_ARG: + diff = SIZE_OF_FFI_ARG - self.fresult.size + cif_descr.exchange_result += diff + + # then enough room for the result, rounded up to sizeof(ffi_arg) + exchange_offset += max(rffi.getintfield(self.rtype, 'c_size'), + SIZE_OF_FFI_ARG) + + # loop over args + for i, farg in enumerate(self.fargs): + #if isinstance(farg, W_CTypePointer): + # exchange_offset += 1 # for the "must free" flag + exchange_offset = self.align_arg(exchange_offset) + cif_descr.exchange_args[i] = exchange_offset + exchange_offset += rffi.getintfield(self.atypes[i], 'c_size') + + # store the exchange data size + cif_descr.exchange_size = exchange_offset + + def fb_extra_fields(self, cif_descr): + cif_descr.abi = clibffi.FFI_DEFAULT_ABI # XXX + cif_descr.nargs = len(self.fargs) + cif_descr.rtype = self.rtype + cif_descr.atypes = self.atypes + + @jit.dont_look_inside + def rawallocate(self, ctypefunc): + + # compute the total size needed in the CIF_DESCRIPTION buffer + self.nb_bytes = 0 + self.bufferp = lltype.nullptr(rffi.CCHARP.TO) + self.fb_build() + + # allocate the buffer + if we_are_translated(): + rawmem = lltype.malloc(rffi.CCHARP.TO, self.nb_bytes, + flavor='raw') + rawmem = rffi.cast(CIF_DESCRIPTION_P, rawmem) + else: + # gross overestimation of the length below, but too bad + rawmem = lltype.malloc(CIF_DESCRIPTION_P.TO, self.nb_bytes, + flavor='raw') + + # the buffer is automatically managed from the W_CTypeFunc instance + ctypefunc._cd = rawmem + + # call again fb_build() to really build the libffi data structures + self.bufferp = rffi.cast(rffi.CCHARP, rawmem) + self.fb_build() + assert self.bufferp == rffi.ptradd(rffi.cast(rffi.CCHARP, rawmem), + self.nb_bytes) + + # fill in the 'exchange_*' fields + self.fb_build_exchange(rawmem) + + # fill in the extra fields + self.fb_extra_fields(rawmem) + + # call libffi's ffi_prep_cif() function + res = jit_libffi.jit_ffi_prep_cif(rawmem) + if res != clibffi.FFI_OK: + raise OperationError(space.w_SystemError, + space.wrap("libffi failed to build this function type")) + def get_item(self, nm): (tp, offset) = self._type.get_desc(nm) ptr = rffi.ptradd(self._buffer, offset) return tp.ffi_load_from(ptr) + + +# Taken from PyPy + From 939c71f9fca8287bbf19f0d538b549b2530d723c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 27 Dec 2014 16:11:44 -0700 Subject: [PATCH 442/909] implemented a c++ based c type inferencer. This should really help with ffi --- pixie/PixieChecker.hpp | 275 +++++++++++++++++++++++++++++++++++++++++ pixie/ffi-infer.pxi | 50 ++++---- pixie/vm/libs/ffi.py | 4 +- target.py | 2 +- 4 files changed, 302 insertions(+), 29 deletions(-) create mode 100644 pixie/PixieChecker.hpp diff --git a/pixie/PixieChecker.hpp b/pixie/PixieChecker.hpp new file mode 100644 index 00000000..27e2a48d --- /dev/null +++ b/pixie/PixieChecker.hpp @@ -0,0 +1,275 @@ +// +// Pixie C++ typer +// +// This code (when compiled) will emit EDN data for a given C++ type. This includes C functions +// So calling this: +// PixieChecker::DumpType() +// will emit this in the compiled program: +// {:type :function :arity 1 :returns {:type :float :size 8} :arguments [{:type :pointer :of-type {:type :int :size 1 :signed? true}} ]} + +#include +#include "boost/type_traits.hpp" + +namespace PixieChecker { + +template +struct GenericChecker; + + + +// Function Checker +template +struct FunctionTyper +{ + static std::string getType() + { + return "{:type :function :arity :unknown}"; + } +}; + +template +struct FunctionTyper<0, T> +{ + static std::string getType() + { + return "{:type :function :arity 0 :returns " + + GenericChecker::result_type>::getType() + + " :arguments []}"; + + } +}; + + + template + struct FunctionTyper<1, T> + { + static std::string getType() + { + return "{:type :function :arity 1 :returns " + + GenericChecker::result_type>::getType() + + " :arguments [" + + GenericChecker::arg1_type>::getType() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<2, T> + { + static std::string getType() + { + return "{:type :function :arity 2 :returns " + + GenericChecker::result_type>::getType() + + " :arguments [" + + GenericChecker::arg1_type>::getType() + " " + + GenericChecker::arg2_type>::getType() + " " + + + " ]}"; + } + }; + + + template + struct FunctionTyper<3, T> + { + static std::string getType() + { + return "{:type :function :arity 3 :returns " + + GenericChecker::result_type>::getType() + + " :arguments [" + + GenericChecker::arg1_type>::getType() + " " + + GenericChecker::arg2_type>::getType() + " " + + GenericChecker::arg3_type>::getType() + " " + + " ]}"; + } + }; + +// End Function Typer + +// Is Void + template + struct GenericCheckerIsVoid + { + static std::string getType() + { + return "Shouldn't happen in void checker"; + } + }; + + template + struct GenericCheckerIsVoid + { + static std::string getType() + { + return "{:type :void}"; + } + }; + + template + struct GenericCheckerIsVoid + { + static std::string getType() + { + return "{:type :unknown}"; + } + }; +// End Is Void + +// Is Pointer + +template +struct GenericCheckerIsPointer +{ + static std::string getType() + { + return "Shouldn't happen in pointer checker"; + } +}; + +template +struct GenericCheckerIsPointer +{ + static std::string getType() + { + return "{:type :pointer :of-type " + GenericChecker::type>::getType() + "}"; + } +}; + +template +struct GenericCheckerIsPointer +{ + static std::string getType() + { + return GenericCheckerIsVoid::type, T>::getType(); + } +}; + + +// End Is Pointer + + + +// Is Function + +template +struct GenericCheckerIsFunction +{ + static std::string getType() + { + return "Shouldn't happen in function checker"; + } +}; + +template +struct GenericCheckerIsFunction +{ + static std::string getType() + { + return FunctionTyper::arity, T>::getType(); + } +}; + +template +struct GenericCheckerIsFunction +{ + static std::string getType() + { + return GenericCheckerIsPointer::type, T>::getType(); + } +}; + +// End Is Function + + +// Is Float + +template +struct GenericCheckerIsFloat +{ + static std::string getType() + { + throw 42; + } +}; + +template +struct GenericCheckerIsFloat +{ + static std::string getType() + { + return "{:type :float :size " + std::to_string(sizeof(T)) + "}"; + } +}; + +template +struct GenericCheckerIsFloat +{ + static std::string getType() + { + return GenericCheckerIsFunction::type, T>::getType(); + } +}; + +// End is Float + +// IsInt + +template +struct GenericCheckerIsInt +{ + static std::string getType() + { + return "Shouldn't happen"; + } +}; + +template +struct GenericCheckerIsInt +{ + static std::string getType() + { + return "{:type :int :size " + std::to_string(sizeof(T)) + + " :signed? " + (boost::is_signed::value ? "true" : "false") + + "}"; + } +}; + +template +struct GenericCheckerIsInt +{ + static std::string getType() + { + return GenericCheckerIsFloat::type, T>::getType(); + } +}; + +// End is Int + + +// Generic Typer +template +struct GenericChecker +{ + static std::string getType() + { + return GenericCheckerIsInt::type, T>::getType(); + } +}; + +// End Generic Typer + +template +void DumpValue(T t) +{ + std::cout << GenericChecker::getType() << std::endl; +} + +template +void DumpType() +{ + std::cout << GenericChecker::getType() << std::endl; +} + +} + + diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index d89550b2..cbad90c7 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -25,45 +25,43 @@ member-name ") << \"}\" << std::endl; \n")) +(defmethod emit-infer-code :struct-member-type + [{:keys [struct-name member-name]}] + (str "std::cout << \"{:name " (keyword member-name)" :type \" << WeakTypeOf((new struct " struct-name ")->" + member-name + ") << \"}\" << std::endl; \n")) + (defmethod emit-infer-code :group [{:keys [ops]}] (str "std::cout << \"[\" << std::endl; " (apply str (map emit-infer-code ops)) "std::cout << \"]\" << std::endl; ")) +(defmethod emit-infer-code :function + [{:keys [name]}] + (str "PixieChecker::DumpType);")) + (defn start-string [] - " #include - #include - #include - #include - - template - std::string TypeOf(T v) - { - return \":unknown\"; - } - - template<> - std::string TypeOf(size_t v) - { - return \":CSizeT\"; - } - - template<> - std::string TypeOf(int v) - { - return \":CIntT\"; - } + " #include \"pixie/PixieChecker.hpp\" + + int main() { - std::cout << \"[\"; + std::cout << \"[\"; ") (defn end-string [] " std::cout << \"]\" << std::endl; - return 0; +return 0; }") +(do (require pixie.io :as io ) + (io/run-command "cat /tmp/tmp.cpp")) + +{:op :struct + :name "stat" + :interesting-fields #{:st_size}} + (io/spit "/tmp/tmp.cpp" (str (start-string) (apply str (map emit-infer-code @@ -80,9 +78,9 @@ {:op :sizeof :name "int"} {:op :group :ops [ {:op :sizeof-struct :name "stat"} - {:op :offsetof-struct :struct-name "stat" :member-name "st_size"}]}])) + {:op :struct-member-type :struct-name "stat" :member-name "st_size"}]}])) (end-string))) - (read-string (io/run-command "c++ /tmp/tmp.cpp && ./a.out"))) + (io/run-command "c++ /tmp/tmp.cpp -o /tmp/a.out && /tmp/a.out")) (spit ) (io/run-command "ls") diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 632399bf..f29e02e5 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -1,6 +1,7 @@ py_object = object import rpython.rlib.rdynload as dynload import pixie.vm.object as object +from pixie.vm.object import runtime_error import pixie.vm.code as code import pixie.vm.stdlib as proto from pixie.vm.code import as_var, affirm, extend @@ -516,8 +517,7 @@ def rawallocate(self, ctypefunc): # call libffi's ffi_prep_cif() function res = jit_libffi.jit_ffi_prep_cif(rawmem) if res != clibffi.FFI_OK: - raise OperationError(space.w_SystemError, - space.wrap("libffi failed to build this function type")) + runtime_error(u"libffi failed to build function type") def get_item(self, nm): (tp, offset) = self._type.get_desc(nm) diff --git a/target.py b/target.py index 6cea8b38..fb041241 100644 --- a/target.py +++ b/target.py @@ -38,7 +38,7 @@ def jitpolicy(driver): LOAD_PATHS = intern_var(u"pixie.stdlib", u"load-paths") LOAD_PATHS.set_root(nil) -load_path = Var(u"", u"internal-load-path") +load_path = Var(u"pixie.stdlib", u"internal-load-path") STAR_1 = intern_var(u"pixie.stdlib", u"*1") STAR_1.set_root(nil) From 2748f26bfca4078f59e1f7b0e252f1212ecea126 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 27 Dec 2014 16:36:53 -0700 Subject: [PATCH 443/909] added some more examples and fixed a multimethod bug --- pixie/stdlib.pxi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4673fbc0..c334d5c2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1792,13 +1792,13 @@ The params can be destructuring bindings, see `(doc let)` for details."} (deftype MultiMethod [dispatch-fn default-val methods] IFn - (-invoke [self dispatch-arg & args] - (let [dispatch-val (dispatch-fn dispatch-arg) + (-invoke [self & args] + (let [dispatch-val (apply dispatch-fn args) method (if (contains? @methods dispatch-val) (get @methods dispatch-val) (get @methods default-val)) _ (assert method (str "no method defined for " dispatch-val))] - (apply method dispatch-arg args)))) + (apply method args)))) (defmacro defmulti {:doc "Define a multimethod, which dispatches to its methods based on dispatch-fn." From 62031a71526db3c36e68adfaf699252c3a2ee94b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 27 Dec 2014 17:03:42 -0700 Subject: [PATCH 444/909] added basic support for functions --- pixie/ffi-infer.pxi | 96 +++++++++++++++++++++++++++++++++------------ 1 file changed, 72 insertions(+), 24 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index cbad90c7..3780bf15 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -3,11 +3,10 @@ (defmulti emit-infer-code :op) -(defmethod emit-infer-code :constant +(defmethod emit-infer-code :const [{:keys [name]}] - (str "std::cout << \"{:name " (keyword name)" :value \" << " name - "<< \" :type \" << TypeOf( " name ") " - "<< \"}\" << std::endl; \n")) + (str "std::cout << \"{:value \" << " name " << \" :type \" " + ";\n PixieChecker::DumpValue(" name "); \n std::cout << \"}\" << std::endl; ")) (defmethod emit-infer-code :sizeof-struct [{:keys [name]}] @@ -39,11 +38,13 @@ (defmethod emit-infer-code :function [{:keys [name]}] - (str "PixieChecker::DumpType);")) + (str "PixieChecker::DumpType(); \n")) + + (defn start-string [] " #include \"pixie/PixieChecker.hpp\" - +#include \"stdlib.h\" int main() { @@ -55,32 +56,79 @@ return 0; }") -(do (require pixie.io :as io ) - (io/run-command "cat /tmp/tmp.cpp")) +;; To Ctype conversion + +(defmulti edn-to-ctype :type) + +(defmethod edn-to-ctype :pointer + [{:keys [of-type] :as ptr}] + (cond + (= of-type {:signed? true :size 1 :type :int}) 'pixie.stdlib/CCharP)) + +(defmethod edn-to-ctype :float + [{:keys [size]}] + (cond + (= size 8) 'pixie.stdlib/CDouble + :else (assert False "unknown type"))) + +;; Code Generation +(defmulti generate-code (fn [input output] + (:op input))) + +(defmethod generate-code :const + [{:keys [name]} {:keys [value type]}] + `(def ~(symbol name) ~value)) + +(defmethod generate-code :function + [{:keys [name]} {:keys [type arguments returns]}] + (assert (= type :function) (str name " is not infered to be a function")) + `(def ~(symbol name) (ffi-fn libc ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) -{:op :struct - :name "stat" - :interesting-fields #{:st_size}} (io/spit "/tmp/tmp.cpp" (str (start-string) (apply str (map emit-infer-code - [ {:op :constant :name "RAND_MAX"} - {:op :sizeof :name "int"} - {:op :group - :ops [ {:op :sizeof-struct :name "stat"} - {:op :offsetof-struct :struct-name "stat" :member-name "st_size"}]}])) + [{:op :const :name "RAND_MAX"}])) + (end-string))) + +(do (io/spit "/tmp/tmp.cpp" (str (start-string) + (apply str (map emit-infer-code + [ {:op :const :name "RAND_MAX"}])) (end-string))) + (io/run-command (str "c++ /tmp/tmp.cpp -I" (first @load-paths) " -o /tmp/a.out && /tmp/a.out")) + + ) (do (io/spit "/tmp/tmp.cpp" (str (start-string) (apply str (map emit-infer-code - [ {:op :constant :name "RAND_MAX"} - {:op :sizeof :name "int"} - {:op :group - :ops [ {:op :sizeof-struct :name "stat"} - {:op :struct-member-type :struct-name "stat" :member-name "st_size"}]}])) + [ {:op :const :name "RAND_MAX"}])) (end-string))) - (io/run-command "c++ /tmp/tmp.cpp -o /tmp/a.out && /tmp/a.out")) + (read-string (io/run-command (str "c++ /tmp/tmp.cpp -I" (first @load-paths) " -o /tmp/a.out && /tmp/a.out"))) + + ) + +(defn run-infer [cmd] + (io/spit "/tmp/tmp.cpp" (str (start-string) + (apply str (map emit-infer-code + [ cmd])) + (end-string))) + (let [result (first (read-string (io/run-command (str "c++ /tmp/tmp.cpp -I" + (first @load-paths) + " -o /tmp/a.out && /tmp/a.out"))))] + (println result) + (generate-code cmd result))) + +(run-infer {:op :function :name "atof"}) + + +(defmacro defcglobal [nm] + (run-infer {:op :const :name (name nm)})) + +(defmacro deflibcfn [nm] + (run-infer {:op :function :name (name nm)})) + +(defcglobal RAND_MAX) + +(deflibcfn "atof") -(spit ) -(io/run-command "ls") +RAND_MAX From 085b75474cb86cf3bc1a6a57404839eedb1cfa5a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 27 Dec 2014 21:59:59 -0700 Subject: [PATCH 445/909] added multi arity map --- pixie/stdlib.pxi | 20 +++++++++++++++++++- pixie/vm/compiler.py | 2 +- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index c334d5c2..e43fb554 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -134,7 +134,25 @@ nil (do (yield (f (current i))) (move-next! i) - (recur)))))))) + (recur)))))) + ([f & colls] + (let [its (vec (map iterator colls))] + (loop [] + (let [args (if (at-end? (first its)) + nil + (reduce + (fn [acc i] + (if (at-end? i) + (reduced nil) + (let [new-acc (conj acc (current i))] + (move-next! i) + new-acc))) + [] + its))] + + (if args + (do (yield (apply f args)) + (recur))))))))) (def reduce (fn [rf init col] diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7a984d11..195d0e78 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -625,7 +625,7 @@ def compile_quote(form, ctx): def compile_recur(form, ctx): form = form.next() - affirm(ctx.can_tail_call, u"Can't recur in non-tail position") + #affirm(ctx.can_tail_call, u"Can't recur in non-tail position") ctc = ctx.can_tail_call ctx.disable_tail_call() args = 0 From c20851fde5dc37db0f6a816ed9237012269ae25a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 29 Dec 2014 14:41:21 -0700 Subject: [PATCH 446/909] a bit of cleanup and helper macros --- pixie/ffi-infer.pxi | 87 +++++++++++++++------------------------------ 1 file changed, 29 insertions(+), 58 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 3780bf15..e6961515 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -8,28 +8,6 @@ (str "std::cout << \"{:value \" << " name " << \" :type \" " ";\n PixieChecker::DumpValue(" name "); \n std::cout << \"}\" << std::endl; ")) -(defmethod emit-infer-code :sizeof-struct - [{:keys [name]}] - (str "std::cout << \"{:name " (keyword name)" :sizeof \" << sizeof(struct " name - ") << \"}\" << std::endl; \n")) - -(defmethod emit-infer-code :sizeof - [{:keys [name]}] - (str "std::cout << \"{:name " (keyword name)" :sizeof \" << sizeof( " name - ") << \"}\" << std::endl; \n")) - -(defmethod emit-infer-code :offsetof-struct - [{:keys [struct-name member-name]}] - (str "std::cout << \"{:name " (keyword member-name)" :offsetof \" << offsetof(struct " struct-name ", " - member-name - ") << \"}\" << std::endl; \n")) - -(defmethod emit-infer-code :struct-member-type - [{:keys [struct-name member-name]}] - (str "std::cout << \"{:name " (keyword member-name)" :type \" << WeakTypeOf((new struct " struct-name ")->" - member-name - ") << \"}\" << std::endl; \n")) - (defmethod emit-infer-code :group [{:keys [ops]}] (str "std::cout << \"[\" << std::endl; " @@ -85,50 +63,43 @@ return 0; `(def ~(symbol name) (ffi-fn libc ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) +(defn run-infer [config cmds] + (io/spit "/tmp/tmp.cpp" (str (start-string) + (apply str (map emit-infer-code + cmds)) + (end-string))) + (let [result (read-string (io/run-command (str "c++ /tmp/tmp.cpp -I" + (first @load-paths) + " -o /tmp/a.out && /tmp/a.out")))] + `(do ~@(map generate-code cmds result)))) -(io/spit "/tmp/tmp.cpp" (str (start-string) - (apply str (map emit-infer-code - [{:op :const :name "RAND_MAX"}])) - (end-string))) - -(do (io/spit "/tmp/tmp.cpp" (str (start-string) - (apply str (map emit-infer-code - [ {:op :const :name "RAND_MAX"}])) - (end-string))) - (io/run-command (str "c++ /tmp/tmp.cpp -I" (first @load-paths) " -o /tmp/a.out && /tmp/a.out")) - - ) -(do (io/spit "/tmp/tmp.cpp" (str (start-string) - (apply str (map emit-infer-code - [ {:op :const :name "RAND_MAX"}])) - (end-string))) - (read-string (io/run-command (str "c++ /tmp/tmp.cpp -I" (first @load-paths) " -o /tmp/a.out && /tmp/a.out"))) +(def *config* nil) +(set-dynamic! (var *config*)) +(def *bodies* nil) +(set-dynamic! (var *bodies*)) - ) -(defn run-infer [cmd] - (io/spit "/tmp/tmp.cpp" (str (start-string) - (apply str (map emit-infer-code - [ cmd])) - (end-string))) - (let [result (first (read-string (io/run-command (str "c++ /tmp/tmp.cpp -I" - (first @load-paths) - " -o /tmp/a.out && /tmp/a.out"))))] - (println result) - (generate-code cmd result))) +(defmacro with-config [config & body] + `(binding [*config* ~config + *bodies* (atom [])] + ~@body + (eval (run-infer *config* @*bodies*)))) -(run-infer {:op :function :name "atof"}) +(defmacro defcfn [nm] + (let [name-str (name nm)] + `(swap! *bodies* conj (assoc {:op :function} :name ~name-str)))) +(defmacro defconst [nm] + (let [name-str (name nm)] + `(swap! *bodies* conj (assoc {:op :const} :name ~name-str))) ) -(defmacro defcglobal [nm] - (run-infer {:op :const :name (name nm)})) -(defmacro deflibcfn [nm] - (run-infer {:op :function :name (name nm)})) +(comment -(defcglobal RAND_MAX) + (with-config {} + (defconst RAND_MAX) + (defcfn atof)) -(deflibcfn "atof") -RAND_MAX + ) From 2aee2f369b5a3bf8567f42f688d30fcde9b88bd4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 29 Dec 2014 16:25:16 -0700 Subject: [PATCH 447/909] added more config options --- pixie/ffi-infer.pxi | 54 ++++++++++++++++++++++++++++++++------------- 1 file changed, 39 insertions(+), 15 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index e6961515..297585a8 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -1,6 +1,14 @@ (ns pixie.ffi-infer (require pixie.io :as io)) +(def *config* nil) +(set-dynamic! (var *config*)) +(def *bodies* nil) +(set-dynamic! (var *bodies*)) +(def *library* nil) +(set-dynamic! (var *library*)) + + (defmulti emit-infer-code :op) (defmethod emit-infer-code :const @@ -21,13 +29,16 @@ (defn start-string [] - " #include \"pixie/PixieChecker.hpp\" -#include \"stdlib.h\" + (str (apply str (map (fn [i] + (str "#include \"" i "\"\n")) + (:includes *config*))) + "#include \"pixie/PixieChecker.hpp\" + #include \"stdlib.h\" - int main() { + int main(int argc, char* argv[]) { std::cout << \"[\"; -") +")) (defn end-string [] " std::cout << \"]\" << std::endl; @@ -49,6 +60,12 @@ return 0; (= size 8) 'pixie.stdlib/CDouble :else (assert False "unknown type"))) +(defmethod edn-to-ctype :int + [{:keys [size]}] + (cond + (= size 4) 'pixie.stdlib/CInt + :else (assert False "unknown type"))) + ;; Code Generation (defmulti generate-code (fn [input output] (:op input))) @@ -60,7 +77,7 @@ return 0; (defmethod generate-code :function [{:keys [name]} {:keys [type arguments returns]}] (assert (= type :function) (str name " is not infered to be a function")) - `(def ~(symbol name) (ffi-fn libc ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) + `(def ~(symbol name) (ffi-fn *library* ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) (defn run-infer [config cmds] @@ -68,21 +85,21 @@ return 0; (apply str (map emit-infer-code cmds)) (end-string))) - (let [result (read-string (io/run-command (str "c++ /tmp/tmp.cpp -I" + (let [cmd-str (str "c++ /tmp/tmp.cpp -I" (first @load-paths) - " -o /tmp/a.out && /tmp/a.out")))] + (apply str " " (interpose " " (:cxx-flags *config*))) + " -o /tmp/a.out && /tmp/a.out") + _ (print cmd-str) + result (read-string (io/run-command cmd-str))] `(do ~@(map generate-code cmds result)))) -(def *config* nil) -(set-dynamic! (var *config*)) -(def *bodies* nil) -(set-dynamic! (var *bodies*)) (defmacro with-config [config & body] `(binding [*config* ~config - *bodies* (atom [])] + *bodies* (atom []) + *library* (ffi-library ~(str "lib" (:library config) "." pixie.platform/so-ext))] ~@body (eval (run-infer *config* @*bodies*)))) @@ -95,11 +112,18 @@ return 0; `(swap! *bodies* conj (assoc {:op :const} :name ~name-str))) ) + (comment - (with-config {} - (defconst RAND_MAX) - (defcfn atof)) +(with-config {:library "SDL" + :cxx-flags ["`sdl2-config --cflags --libs`"] + :includes ["SDL.h"] + } + (defconst SDL_INIT_EVERYTHING) + (defcfn SDL_Init) + (defcfn SDL_CreateWindow) + (defconst SDL_WINDOW_SHOWN)) + ) From 1ee2aed2ae7efe3b8012dace7e967aa1b88712d2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 29 Dec 2014 18:08:14 -0700 Subject: [PATCH 448/909] remove testing code --- pixie/stdlib.pxi | 2 -- 1 file changed, 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e43fb554..ad480691 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -21,8 +21,6 @@ (def floor (ffi-fn libm "floor" [CDouble] CDouble)) (def lround (ffi-fn libm "lround" [CDouble] CInt)) - (puts "testing ffi....") - (def reset! -reset!) From f0da86182929636fe51b27ad87ed86bfaa0fbc92 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Dec 2014 10:28:11 -0700 Subject: [PATCH 449/909] fix crash caused by reader failure --- pixie/vm/reader.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 5a7515f3..3267b5e3 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -1,6 +1,6 @@ py_object = object import pixie.vm.object as object -from pixie.vm.object import affirm +from pixie.vm.object import affirm, runtime_error import pixie.vm.code as code from pixie.vm.primitives import nil, true, false import pixie.vm.numbers as numbers @@ -717,7 +717,7 @@ def read(rdr, error_on_eof): eat_whitespace(rdr) except EOFError as ex: if error_on_eof: - raise ex + runtime_error("Unexpected EOF while reading") return eof From 7f5d591b4d4c24d36b82d0edd0a9634dacb9d90b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Dec 2014 10:41:58 -0700 Subject: [PATCH 450/909] unicode strings --- pixie/vm/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 3267b5e3..dc6533e9 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -717,7 +717,7 @@ def read(rdr, error_on_eof): eat_whitespace(rdr) except EOFError as ex: if error_on_eof: - runtime_error("Unexpected EOF while reading") + runtime_error(u"Unexpected EOF while reading") return eof From 2ae2a96feb2046e321f9b8f321d1e4454288fa71 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Dec 2014 13:51:14 -0700 Subject: [PATCH 451/909] improve performance of ffi, this will require us to disable preload --- .travis.yml | 6 +++--- pixie/vm/libs/ffi.py | 28 +++++++++++++++------------- 2 files changed, 18 insertions(+), 16 deletions(-) diff --git a/.travis.yml b/.travis.yml index 1fe62851..8bc329d2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,8 +1,8 @@ env: - JIT_OPTS='--opt=jit' TARGET_OPTS='target.py' - JIT_OPTS='' TARGET_OPTS='target.py' - - JIT_OPTS='--opt=jit' TARGET_OPTS='target_preload.py' - - JIT_OPTS='' TARGET_OPTS='target_preload.py' + #- JIT_OPTS='--opt=jit' TARGET_OPTS='target_preload.py' + #- JIT_OPTS='' TARGET_OPTS='target_preload.py' matrix: fast_finish: true @@ -18,4 +18,4 @@ before_install: - sudo apt-get install libffi-dev libedit-dev notifications: - # irc: "chat.freenode.net#pixie-lang" + irc: "chat.freenode.net#pixie-lang" diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index f29e02e5..ca33941e 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -74,8 +74,8 @@ def _cleanup_(self): class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") - __immutable_fields__ = ["_is_inited?", "_lib", "_name", "_arg_types[*]", "_ret_type", \ - "_transfer_size?", "_arg0_offset?", "_ret_offset?", "_cd?"] + _immutable_fields_ = ["_is_inited", "_lib", "_name", "_arg_types[*]", "_ret_type", \ + "_transfer_size", "_arg0_offset", "_ret_offset", "_cd"] def type(self): return FFIFn._type @@ -86,14 +86,14 @@ def __init__(self, lib, name, arg_types, ret_type): self._lib = lib self._arg_types = arg_types self._ret_type = ret_type - self._is_inited = False + #self._is_inited = False + self.thaw() def thaw(self): - if not self._is_inited: - self._f_ptr = self._lib.get_fn_ptr(self._name) - CifDescrBuilder(self._arg_types, self._ret_type).rawallocate(self) - self._is_inited = True + #if not self._is_inited: + self._f_ptr = self._lib.get_fn_ptr(self._name) + CifDescrBuilder(self._arg_types, self._ret_type).rawallocate(self) def _cleanup_(self): self._rev += 1 @@ -103,20 +103,18 @@ def _cleanup_(self): @jit.unroll_safe def prep_exb(self, args): - if not self._is_inited: - self.thaw() - size = self._cd.exchange_size + size = jit.promote(self._cd.exchange_size) exb = lltype.malloc(rffi.CCHARP.TO, size, flavor="raw") tokens = [None] * len(args) for i, tp in enumerate(self._arg_types): - offset_p = rffi.ptradd(exb, self._cd.exchange_args[i]) + offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_args[i])) tokens[i] = tp.ffi_set_value(offset_p, args[i]) return exb, tokens def get_ret_val_from_buffer(self, exb): - offset_p = rffi.ptradd(exb, self._cd.exchange_result_libffi) + offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_result_libffi)) ret_val = self._ret_type.ffi_get_value(offset_p) return ret_val @@ -124,7 +122,11 @@ def get_ret_val_from_buffer(self, exb): def _invoke(self, args): exb, tokens = self.prep_exb(args) - jit_ffi_call(self._cd, self._f_ptr, exb) + cd = jit.promote(self._cd) + #fp = jit.promote(self._f_ptr) + jit_ffi_call(cd, + self._f_ptr, + exb) ret_val = self.get_ret_val_from_buffer(exb) for x in range(len(args)): From 748fe00fbd90eb93577a4b99c425d1784d9b1bb3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Dec 2014 13:57:54 -0700 Subject: [PATCH 452/909] don't use a tab in the .travis.yml file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 8bc329d2..ffd0496f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -18,4 +18,4 @@ before_install: - sudo apt-get install libffi-dev libedit-dev notifications: - irc: "chat.freenode.net#pixie-lang" + irc: "chat.freenode.net#pixie-lang" From 3ad536876278591d0c0eaf7fb2ac6ffe6cba4aeb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Dec 2014 17:11:22 -0700 Subject: [PATCH 453/909] Update README.md --- README.md | 37 +------------------------------------ 1 file changed, 1 insertion(+), 36 deletions(-) diff --git a/README.md b/README.md index 066962b8..0a56143b 100644 --- a/README.md +++ b/README.md @@ -13,14 +13,12 @@ Some planned and implemented features: * Immutable datastructures * Protocols first implementation * Transducers at-the-bottom (most primitives are based off of reduce) -* Coroutines for transducer inversion of control (transducer to lazy-seq conversion) * A "good enough" JIT (implemented, tuning still a WIP, but not bad performance today) -* Easy FFI (TODO) +* Easy FFI * Pattern matching (TODO) ## Dependencies -* [libuv-dev](https://github.com/libuv/libuv) * [libffi-dev](https://sourceware.org/libffi/) * [libedit-dev](http://thrysoee.dk/editline/) @@ -90,39 +88,6 @@ However there are a few features of pixie that although may not be uncommon, are ``` -* Inversion of control via stacklets. Most of Pixie makes heavy use of transducers. However there are times when transducers need to be converted into a cons style lazy sequence. For this we make use of stacklets to invert control of the loops. Because of this, any data collection need only define a method for the -reduce protocol and most collections functions will "just work" - -```clojure - -(def stacklet->lazy-seq - (fn [f] - (let [val (f nil)] - (if (identical? val :end) - nil - (cons val (lazy-seq* (fn [] (stacklet->lazy-seq f)))))))) - -(def sequence - (fn - ([data] - (let [f (create-stacklet - (fn [h] - (reduce (fn ([h item] (h item) h)) h data) - (h :end)))] - (stacklet->lazy-seq f))) - ([xform data] - (let [f (create-stacklet - (fn [h] - (transduce xform - (fn ([] h) - ([h item] (h item) h) - ([h] (h :end))) - data)))] - (stacklet->lazy-seq f))))) - -(comment - (sequence [1 2 3 4]) now produces '(1 2 3 4)) - -``` * Math system is fully polymorphic. Math primitives (+,-, etc.) are built off of polymorphic functions that dispatch on the types of the first two arguments. This allows the math system to be extended to complex numbers, matricies, etc. The performance penalty of such a polymorphic call is completely removed by the RPython generated JIT. From 2329b72df6dab6dc2d84ac03d27cf9f6285206f3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Dec 2014 17:14:51 -0700 Subject: [PATCH 454/909] added ffi to math.h --- pixie/math.pxi | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 pixie/math.pxi diff --git a/pixie/math.pxi b/pixie/math.pxi new file mode 100644 index 00000000..7c16889d --- /dev/null +++ b/pixie/math.pxi @@ -0,0 +1,26 @@ +(ns pixie.math + (require pixie.ffi-infer :as i)) + +(i/with-config {:library "m" + :cxx-flags ["-lm"] + :includes ["math.h"]} + (i/defcfn acos) + (i/defcfn asin) + (i/defcfn atan) + (i/defcfn atan2) + (i/defcfn cos) + (i/defcfn cosh) + (i/defcfn sin) + (i/defcfn sinh) + (i/defcfn tanh) + (i/defcfn exp) + (i/defcfn ldexp) + (i/defcfn log) + (i/defcfn log10) + ;(i/defcfn modf) ;; Needs ffi support + (i/defcfn pow) + (i/defcfn sqrt) + (i/defcfn ceil) + (i/defcfn fabs) + (i/defcfn floor) + (i/defcfn fmod)) From ad1b88ab29ba4486c10dff9b47a786074fbc4d97 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 30 Dec 2014 21:54:11 -0700 Subject: [PATCH 455/909] add start of stuct support --- pixie/math.pxi | 26 +++++++++++++++++ pixie/vm/libs/ffi.py | 66 +++++++++++++++++++++++++++++--------------- 2 files changed, 70 insertions(+), 22 deletions(-) create mode 100644 pixie/math.pxi diff --git a/pixie/math.pxi b/pixie/math.pxi new file mode 100644 index 00000000..7c16889d --- /dev/null +++ b/pixie/math.pxi @@ -0,0 +1,26 @@ +(ns pixie.math + (require pixie.ffi-infer :as i)) + +(i/with-config {:library "m" + :cxx-flags ["-lm"] + :includes ["math.h"]} + (i/defcfn acos) + (i/defcfn asin) + (i/defcfn atan) + (i/defcfn atan2) + (i/defcfn cos) + (i/defcfn cosh) + (i/defcfn sin) + (i/defcfn sinh) + (i/defcfn tanh) + (i/defcfn exp) + (i/defcfn ldexp) + (i/defcfn log) + (i/defcfn log10) + ;(i/defcfn modf) ;; Needs ffi support + (i/defcfn pow) + (i/defcfn sqrt) + (i/defcfn ceil) + (i/defcfn fabs) + (i/defcfn floor) + (i/defcfn fmod)) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index ca33941e..e1053ee8 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -3,6 +3,7 @@ import pixie.vm.object as object from pixie.vm.object import runtime_error import pixie.vm.code as code +from pixie.vm.keyword import Keyword import pixie.vm.stdlib as proto from pixie.vm.code import as_var, affirm, extend import pixie.vm.rt as rt @@ -226,29 +227,7 @@ def set_buffer_size(self, size): -class CStructType(object.Type): - def __init__(self, name, size, desc): - object.Type.__init__(self, name) - self._desc = desc - self._size = size - #offsets is a dict of {nm, (type, offset)} - - def get_offset(self, nm): - (tp, offset) = self._desc.get(nm, (None, 0)) - - assert tp is not None - - return offset - def get_type(self, nm): - (tp, offset) = self._desc.get(nm, (None, 0)) - - assert tp is not None - - return tp - - def get_desc(self, nm): - return self._desc[nm] class Token(py_object): """ Tokens are returned by ffi_set_value and are called when ffi is ready to clean up resources @@ -375,6 +354,33 @@ def raw_data(self): return self._raw_data +class CStructType(object.Type): + def __init__(self, name, size, desc): + object.Type.__init__(self, name) + self._desc = desc + self._size = size + #offsets is a dict of {nm, (type, offset)} + + def get_offset(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return offset + + def get_type(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return tp + + def get_desc(self, nm): + return self._desc[nm] + + def invoke(self, args): + return CStruct(self, lltype.malloc(rffi.CCHARP.TO, self._size, flavor="raw")) + class CStruct(object.Object): def __init__(self, tp, buffer): @@ -384,6 +390,22 @@ def __init__(self, tp, buffer): def type(self): return self._type +@as_var("pixie.ffi", "c-struct") +def c_struct(name, size, spec): + d = {} + for x in range(rt.count(spec)): + row = rt.nth(spec, rt.wrap(x)) + nm = rt.nth(row, rt.wrap(0)) + tp = rt.nth(row, rt.wrap(1)) + offset = rt.nth(row, rt.wrap(2)) + + affirm(isinstance(nm, Keyword), u"c-struct field names must be keywords") + affirm(isinstance(tp, CType), u"c-struct field types must be c types") + + d[nm] = (tp, offset.int_val()) + + return CStructType(rt.name(name), size.int_val(), d) + import sys BIG_ENDIAN = sys.byteorder == 'big' From a19728736e14af9043a190e2afb10613ccbe1f68 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 1 Jan 2015 14:51:52 -0700 Subject: [PATCH 456/909] more work on ffi structs --- pixie/ffi-infer.pxi | 44 +++++++++++++++++++++++++++++++++++++++++--- pixie/vm/libs/ffi.py | 43 +++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 297585a8..0342371e 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -26,6 +26,16 @@ [{:keys [name]}] (str "PixieChecker::DumpType(); \n")) +(defmethod emit-infer-code :struct + [{:keys [name members]}] + (str "std::cout << \"{:size \" << sizeof(struct " name ")" + " << \" :infered-members [\" << " + (apply str + (map (fn [member] + (str "\"{:type \"; PixieChecker::DumpValue((new (struct " name "))->" member "); " + " std::cout << \":offset \" << offsetof(struct " name ", " member ") << \" }\" << \n ")) + members)) + "\"]}\" << std::endl;")) (defn start-string [] @@ -52,7 +62,8 @@ return 0; (defmethod edn-to-ctype :pointer [{:keys [of-type] :as ptr}] (cond - (= of-type {:signed? true :size 1 :type :int}) 'pixie.stdlib/CCharP)) + (= of-type {:signed? true :size 1 :type :int}) 'pixie.stdlib/CCharP + :else 'pixie.stdlib/CVoidP)) (defmethod edn-to-ctype :float [{:keys [size]}] @@ -64,6 +75,7 @@ return 0; [{:keys [size]}] (cond (= size 4) 'pixie.stdlib/CInt + (= size 8) 'pixie.stdlib/CInt :else (assert False "unknown type"))) ;; Code Generation @@ -79,6 +91,14 @@ return 0; (assert (= type :function) (str name " is not infered to be a function")) `(def ~(symbol name) (ffi-fn *library* ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) +(defmethod generate-code :struct + [{:keys [name members]} {:keys [size infered-members]}] + `(def ~(symbol name) + (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] + `[~(keyword name) ~(edn-to-ctype type) ~offset]) + members infered-members)]))) + + (defn run-infer [config cmds] (io/spit "/tmp/tmp.cpp" (str (start-string) @@ -94,6 +114,10 @@ return 0; `(do ~@(map generate-code cmds result)))) +(binding [*config* {:includes ["sys/stat.h"]}] + (run-infer nil + [{:op :struct :name "stat" + :members ["st_size"]}])) (defmacro with-config [config & body] @@ -109,12 +133,25 @@ return 0; (defmacro defconst [nm] (let [name-str (name nm)] - `(swap! *bodies* conj (assoc {:op :const} :name ~name-str))) ) + `(swap! *bodies* conj (assoc {:op :const} :name ~name-str)))) + +(defmacro defcstruct [nm members] + `(swap! *bodies* conj (assoc {:op :struct} + :name ~(name nm) + :members ~(vec (map name members))) )) +(with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["sys/stat.h"] + } + (defcstruct stat [:st_size]) + (defcfn lstat)) +(let [s (stat)] + (lstat "/tmp/a.out" s) + (println "filesize " (:st_size s))) (comment - (with-config {:library "SDL" :cxx-flags ["`sdl2-config --cflags --libs`"] :includes ["SDL.h"] @@ -126,4 +163,5 @@ return 0; + ) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index e1053ee8..02e9db90 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -332,6 +332,8 @@ def ffi_set_value(self, ptr, val): pnt[0] = val.raw_data() elif val is nil: pnt[0] = rffi.cast(rffi.VOIDP, 0) + elif isinstance(val, CStruct): + pnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) else: print val affirm(False, u"Cannot encode this type") @@ -355,8 +357,11 @@ def raw_data(self): class CStructType(object.Type): + base_type = object.Type(u"pixie.ffi.CStruct") + _immutable_fields_ = ["_desc", "_size"] + def __init__(self, name, size, desc): - object.Type.__init__(self, name) + object.Type.__init__(self, name, CStructType.base_type) self._desc = desc self._size = size #offsets is a dict of {nm, (type, offset)} @@ -375,13 +380,15 @@ def get_type(self, nm): return tp + @jit.elidable_promote() def get_desc(self, nm): - return self._desc[nm] + return self._desc.get(nm, (None, 0)) def invoke(self, args): return CStruct(self, lltype.malloc(rffi.CCHARP.TO, self._size, flavor="raw")) + class CStruct(object.Object): def __init__(self, tp, buffer): self._type = tp @@ -390,6 +397,30 @@ def __init__(self, tp, buffer): def type(self): return self._type + def raw_data(self): + return self._buffer + + def val_at(self, k, not_found): + (tp, offset) = self._type.get_desc(k) + + if tp is None: + return not_found + + offset = rffi.ptradd(self._buffer, offset) + return tp.ffi_get_value(offset) + + def set_val(self, k, v): + (tp, offset) = self._type.get_desc(k) + + if tp is None: + runtime_error(u"Invalid field name: " + rt.name(rt.str(tp))) + + offset = rffi.ptradd(self._buffer, offset) + tp.ffi_set_value(offset, v) + + return nil + + @as_var("pixie.ffi", "c-struct") def c_struct(name, size, spec): d = {} @@ -406,6 +437,14 @@ def c_struct(name, size, spec): return CStructType(rt.name(name), size.int_val(), d) +@extend(proto._val_at, CStructType.base_type) +def val_at(self, k, not_found): + return self.val_at(k, not_found) + +@as_var("pixie.ffi", "set!") +def set_(self, k, val): + return self.set_val(k, val) + import sys BIG_ENDIAN = sys.byteorder == 'big' From 045cfc288b3f84722ecaac9fec1148459cc30815 Mon Sep 17 00:00:00 2001 From: Akos Gyimesi Date: Thu, 1 Jan 2015 22:39:31 +0100 Subject: [PATCH 457/909] check ffi function arity --- pixie/vm/libs/ffi.py | 7 ++++++- tests/pixie/tests/test-ffi.pxi | 6 ++++++ 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index ca33941e..da8af2e1 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -74,7 +74,7 @@ def _cleanup_(self): class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") - _immutable_fields_ = ["_is_inited", "_lib", "_name", "_arg_types[*]", "_ret_type", \ + _immutable_fields_ = ["_is_inited", "_lib", "_name", "_arg_types[*]", "_arity", "_ret_type", \ "_transfer_size", "_arg0_offset", "_ret_offset", "_cd"] def type(self): @@ -85,6 +85,7 @@ def __init__(self, lib, name, arg_types, ret_type): self._name = name self._lib = lib self._arg_types = arg_types + self._arity = len(arg_types) self._ret_type = ret_type #self._is_inited = False self.thaw() @@ -120,6 +121,10 @@ def get_ret_val_from_buffer(self, exb): @jit.unroll_safe def _invoke(self, args): + arity = len(args) + if arity < self._arity: + runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + + u", expected at least " + unicode(str(self._arity))) exb, tokens = self.prep_exb(args) cd = jit.promote(self._cd) diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index a99dd211..629fd68d 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -8,3 +8,9 @@ b (buffer 1024)] (t/assert= 10 (fread b 1 10 fp)) (t/assert= 91 (nth b 0)))) + +(t/deftest test-arity-check + (try + (puts) + (t/assert false) + (catch ex (t/assert= (type ex) RuntimeException)))) From 5ea27fd5fcb87502a4217d182aea0ac24ce8a977 Mon Sep 17 00:00:00 2001 From: Akos Gyimesi Date: Thu, 1 Jan 2015 22:47:44 +0100 Subject: [PATCH 458/909] fix arity of rand function --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ad480691..4a2836ba 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -10,7 +10,7 @@ (def libedit (ffi-library (str "libedit." pixie.platform/so-ext))) (def readline (ffi-fn libedit "readline" [CCharP] CCharP)) - (def rand (ffi-fn libc "rand" [CInt] CInt)) + (def rand (ffi-fn libc "rand" [] CInt)) (def srand (ffi-fn libc "srand" [CInt] CInt)) (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) From 9792aeb88436057ed0e5552fbafe86975e059266 Mon Sep 17 00:00:00 2001 From: Akos Gyimesi Date: Fri, 2 Jan 2015 01:43:42 +0100 Subject: [PATCH 459/909] added :variadic? flag to ffi-fn, check arity based on this flag --- pixie/stdlib.pxi | 2 +- pixie/vm/libs/ffi.py | 47 ++++++++++++++++++++++++++-------- tests/pixie/tests/test-ffi.pxi | 25 +++++++++++++++--- 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4a2836ba..4768171c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -5,7 +5,7 @@ (def puts (ffi-fn libc "puts" [CCharP] CInt)) (def sh (ffi-fn libc "system" [CCharP] CInt)) - (def printf (ffi-fn libc "printf" [CCharP] CInt)) + (def printf (ffi-fn libc "printf" [CCharP] CInt :variadic? true)) (def getenv (ffi-fn libc "getenv" [CCharP] CCharP)) (def libedit (ffi-library (str "libedit." pixie.platform/so-ext))) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index da8af2e1..c75585a3 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -7,9 +7,10 @@ from pixie.vm.code import as_var, affirm, extend import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype, llmemory -from pixie.vm.primitives import nil +from pixie.vm.primitives import nil, true, false from pixie.vm.numbers import Integer, Float from pixie.vm.string import String +from pixie.vm.keyword import Keyword, keyword from pixie.vm.util import unicode_to_utf8 from rpython.rlib import clibffi from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION, CIF_DESCRIPTION_P, \ @@ -74,19 +75,20 @@ def _cleanup_(self): class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") - _immutable_fields_ = ["_is_inited", "_lib", "_name", "_arg_types[*]", "_arity", "_ret_type", \ + _immutable_fields_ = ["_is_inited", "_lib", "_name", "_arg_types[*]", "_arity", "_ret_type", "_is_variadic", \ "_transfer_size", "_arg0_offset", "_ret_offset", "_cd"] def type(self): return FFIFn._type - def __init__(self, lib, name, arg_types, ret_type): + def __init__(self, lib, name, arg_types, ret_type, is_variadic): self._rev = 0 self._name = name self._lib = lib self._arg_types = arg_types self._arity = len(arg_types) self._ret_type = ret_type + self._is_variadic = is_variadic #self._is_inited = False self.thaw() @@ -122,9 +124,14 @@ def get_ret_val_from_buffer(self, exb): @jit.unroll_safe def _invoke(self, args): arity = len(args) - if arity < self._arity: - runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + - u", expected at least " + unicode(str(self._arity))) + if self._is_variadic: + if arity < self._arity: + runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + + u", expected at least " + unicode(str(self._arity))) + else: + if arity != self._arity: + runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + + u", expected " + unicode(str(self._arity))) exb, tokens = self.prep_exb(args) cd = jit.promote(self._cd) @@ -153,19 +160,39 @@ def _ffi_library(ns): return ExternalLib(nm) @as_var("ffi-fn") -def _ffi_fn(lib, nm, args, ret_type): +def _ffi_fn__args(args): + affirm(len(args) >= 4, u"ffi-fn requires at least 4 arguments") + lib, nm, arg_types, ret_type = args[:4] + affirm(isinstance(lib, ExternalLib), u"First argument must be an ExternalLib") affirm(isinstance(ret_type, object.Type), u"Ret type must be a type") affirm(rt.namespace(nm) is None, u"Name must not be namespaced") - cnt = rt.count(args) + cnt = rt.count(arg_types) new_args = [None] * cnt for x in range(cnt): - t = rt.nth(args, rt.wrap(x)) + t = rt.nth(arg_types, rt.wrap(x)) affirm(isinstance(t, object.Type), u"Arg defs must be types") new_args[x] = t - f = FFIFn(lib, rt.name(nm), new_args, ret_type) + kwargs = args[4:] + affirm(len(kwargs) & 0x1 == 0, u"ffi-fn requires even number of options") + + is_variadic = False + for i in range(0, len(kwargs)/2, 2): + key = kwargs[i] + val = kwargs[i+1] + + affirm(isinstance(key, Keyword), u"ffi-fn options should be keyword/bool pairs") + affirm(val is true or val is false, u"ffi-fn options should be keyword/bool pairs") + + k = rt.name(key) + if k == u"variadic?": + is_variadic = True if val is true else False + else: + affirm(False, u"unknown ffi-fn option: :" + k) + + f = FFIFn(lib, rt.name(nm), new_args, ret_type, is_variadic) return f diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index 629fd68d..fb9cd872 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -10,7 +10,24 @@ (t/assert= 91 (nth b 0)))) (t/deftest test-arity-check - (try - (puts) - (t/assert false) - (catch ex (t/assert= (type ex) RuntimeException)))) + (let [sscanf-2 (ffi-fn libc "sscanf" [CCharP CCharP] CInt) + sscanf-* (ffi-fn libc "sscanf" [CCharP CCharP] CInt :variadic? true)] + (try + (sscanf-2 "too few arguments") + (t/assert false) + (catch ex (t/assert= (type ex) RuntimeException))) + + (try + (sscanf-2 "too" "many" "arguments") + (t/assert false) + (catch ex (t/assert= (type ex) RuntimeException))) + + (try + (sscanf-* "too few arguments") + (t/assert false) + (catch ex (t/assert= (type ex) RuntimeException))) + + (sscanf-2 "string" "fmt") + (sscanf-* "string" "fmt") + (sscanf-* "string" "fmt" "optional arg1" "optional arg2") + (t/assert true))) From ce64b79ac0b04dde2f09fa26756b28583a912fc5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 1 Jan 2015 21:56:13 -0700 Subject: [PATCH 460/909] more work on ffi structs --- pixie/ffi-infer.pxi | 33 +++++++++++++++++++++++--------- pixie/vm/libs/ffi.py | 45 +++++++++++++++++++++++++++++++++++++++----- pixie/vm/reader.py | 1 + 3 files changed, 65 insertions(+), 14 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 0342371e..2d4af4b3 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -71,12 +71,21 @@ return 0; (= size 8) 'pixie.stdlib/CDouble :else (assert False "unknown type"))) + +(def int-types {[8 true] 'pixie.stdlib/CInt8 + [8 false] 'pixie.stdlib/CUInt8 + [16 true] 'pixie.stdlib/CInt16 + [16 false] 'pixie.stdlib/CUInt16 + [32 true] 'pixie.stdlib/CInt32 + [32 false] 'pixie.stdlib/CUInt32 + [64 true] 'pixie.stdlib/CInt64 + [64 false] 'pixie.stdlib/CUInt64}) + (defmethod edn-to-ctype :int - [{:keys [size]}] - (cond - (= size 4) 'pixie.stdlib/CInt - (= size 8) 'pixie.stdlib/CInt - :else (assert False "unknown type"))) + [{:keys [size signed?] :as tp}] + (let [tp-found (get int-types [(* 8 size) signed?])] + (assert tp-found (str "No type found for " tp)) + tp-found)) ;; Code Generation (defmulti generate-code (fn [input output] @@ -105,7 +114,7 @@ return 0; (apply str (map emit-infer-code cmds)) (end-string))) - (let [cmd-str (str "c++ /tmp/tmp.cpp -I" + (let [cmd-str (str "c++ -arch x86_64 /tmp/tmp.cpp -I" (first @load-paths) (apply str " " (interpose " " (:cxx-flags *config*))) " -o /tmp/a.out && /tmp/a.out") @@ -144,12 +153,18 @@ return 0; :cxx-flags ["-lc"] :includes ["sys/stat.h"] } - (defcstruct stat [:st_size]) + (defcstruct stat [:st_dev + :st_ino + :st_mode + :st_nlink + :st_uid + :st_gid + :st_size]) (defcfn lstat)) (let [s (stat)] - (lstat "/tmp/a.out" s) - (println "filesize " (:st_size s))) + (println (str "\n" (lstat "/tmp/a.out" s))) + (println "filesize " (:st_size s) " " (:st_uid s))) (comment (with-config {:library "SDL" diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 02e9db90..5f5fdd88 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -34,6 +34,12 @@ """ + +class CType(object.Type): + def __init__(self, name): + object.Type.__init__(self, name) + + class ExternalLib(object.Object): _type = object.Type(u"pixie.stdlib.ExternalLib") @@ -225,6 +231,38 @@ def set_buffer_size(self, size): self.set_used_size(size.int_val()) return self +def make_itype(name, ctype, llt): + lltp = lltype.Ptr(lltype.Array(llt, hints={'nolength': True})) + class GenericCInt(CType): + def __init__(self): + CType.__init__(self, name) + + def ffi_get_value(self, ptr): + casted = rffi.cast(lltp, ptr) + return Integer(rffi.cast(rffi.LONG, casted[0])) + + def ffi_set_value(self, ptr, val): + casted = rffi.cast(lltp, ptr) + casted[0] = rffi.cast(llt, val.int_val()) + + def ffi_size(self): + return rffi.sizeof(llt) + + def ffi_type(self): + return ctype + + return GenericCInt() + +from rpython.rlib.rarithmetic import build_int +for x in [8, 16, 32, 64]: + for s in [True, False]: + nm = "C" + ("" if s else "U") + "Int" + str(x) + int_tp = lltype.build_number(None, build_int(nm, s, x)) + ctype = clibffi.cast_type_to_ffitype(int_tp) + make_itype(unicode("pixie.stdlib." + nm), ctype, int_tp) + + + @@ -236,10 +274,6 @@ def finalize_token(self): pass -class CType(object.Type): - def __init__(self, name): - object.Type.__init__(self, name) - class CInt(CType): def __init__(self): @@ -431,7 +465,8 @@ def c_struct(name, size, spec): offset = rt.nth(row, rt.wrap(2)) affirm(isinstance(nm, Keyword), u"c-struct field names must be keywords") - affirm(isinstance(tp, CType), u"c-struct field types must be c types") + if not isinstance(tp, CType): + runtime_error(u"c-struct field types must be c types, got: " + rt.name(rt.str(tp))) d[nm] = (tp, offset.int_val()) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index dc6533e9..8816d66e 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -712,6 +712,7 @@ def throw_syntax_error_with_data(rdr, txt): raise object.WrappedException(err) + def read(rdr, error_on_eof): try: eat_whitespace(rdr) From 8e7d5fe08daf7966db2b0fcdddd992863cf31cd9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 2 Jan 2015 20:57:13 -0700 Subject: [PATCH 461/909] more work on generic ints and ffi stuff --- pixie/ffi-infer.pxi | 68 +++++++++++++++++++++++++++++++++----------- pixie/vm/libs/ffi.py | 29 +++++++++++++++++-- 2 files changed, 78 insertions(+), 19 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 2d4af4b3..17ebe4f2 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -149,22 +149,6 @@ return 0; :name ~(name nm) :members ~(vec (map name members))) )) -(with-config {:library "c" - :cxx-flags ["-lc"] - :includes ["sys/stat.h"] - } - (defcstruct stat [:st_dev - :st_ino - :st_mode - :st_nlink - :st_uid - :st_gid - :st_size]) - (defcfn lstat)) - -(let [s (stat)] - (println (str "\n" (lstat "/tmp/a.out" s))) - (println "filesize " (:st_size s) " " (:st_uid s))) (comment (with-config {:library "SDL" @@ -177,6 +161,58 @@ return 0; (defconst SDL_WINDOW_SHOWN)) +(f/with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["sys/stat.h"] + } + (f/defcstruct stat [:st_dev + :st_ino + :st_mode + :st_nlink + :st_uid + :st_gid + :st_size]) + (f/defcfn lstat64)) + + + + + +(let [s (stat)] + (pixie.ffi/set! s :st_size 42) + (println (str "\n" (:st_size s))) + (println (str "\n" (lstat64 "/tmp/tmp.cpp" s))) + (println "filesize " (:st_size s) " " (:st_uid s) " " (:st_gid s))) + +(with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["ctime.h"]}) + + +(f/with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["time.h"] + } + (def time_t (pixie.ffi/c-struct :time_t 8 [[:val CInt 0 ]])) + (f/defcfn time) + (f/defcstruct tm [:tm_sec + :tm_min + :tm_hour + :tm_mday + :tm_mon + :tm_year + :tm_wday + :tm_yday + :tm_isdst]) + (f/defcfn localtime)) + +(let [t (time_t) + tmi (pixie.ffi/cast (localtime t) tm)] + + (println (:tm_sec tmi))) + +(type (pixie.ffi/cast (localtime (time_t)) tm)) + ) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 5f5fdd88..ddcd0e87 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -414,16 +414,20 @@ def get_type(self, nm): return tp + def cast_to(self, frm): + return CStruct(self, frm.raw_data()) + @jit.elidable_promote() def get_desc(self, nm): return self._desc.get(nm, (None, 0)) def invoke(self, args): - return CStruct(self, lltype.malloc(rffi.CCHARP.TO, self._size, flavor="raw")) + return CStruct(self, rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, self._size, flavor="raw"))) class CStruct(object.Object): + _immutable_fields_ = ["_type", "_buffer"] def __init__(self, tp, buffer): self._type = tp self._buffer = buffer @@ -441,7 +445,7 @@ def val_at(self, k, not_found): return not_found offset = rffi.ptradd(self._buffer, offset) - return tp.ffi_get_value(offset) + return tp.ffi_get_value(rffi.cast(rffi.CCHARP, offset)) def set_val(self, k, v): (tp, offset) = self._type.get_desc(k) @@ -450,7 +454,7 @@ def set_val(self, k, v): runtime_error(u"Invalid field name: " + rt.name(rt.str(tp))) offset = rffi.ptradd(self._buffer, offset) - tp.ffi_set_value(offset, v) + tp.ffi_set_value(rffi.cast(rffi.CCHARP, offset), v) return nil @@ -472,6 +476,25 @@ def c_struct(name, size, spec): return CStructType(rt.name(name), size.int_val(), d) +@as_var("pixie.ffi", "cast") +def c_cast(frm, to): + if not isinstance(to, CStructType): + runtime_error(u"Expected a CStruct type to cast to, got " + rt.name(rt.str(to))) + + if not isinstance(frm, CStruct) and not isinstance(frm, VoidP): + runtime_error(u"From must be a CVoidP or a CStruct, got " + rt.name(rt.str(frm))) + + return to.cast_to(frm) + +@as_var("pixie.ffi", "free") +def c_free(frm): + if not isinstance(frm, CStruct) and not isinstance(frm, VoidP): + runtime_error(u"Can only free CStructs or CVoidP") + + lltype.free(frm.raw_data(), flavor="raw") + + return nil + @extend(proto._val_at, CStructType.base_type) def val_at(self, k, not_found): return self.val_at(k, not_found) From 8c5101089027abfd60318ffa218515db9c1eee0e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 3 Jan 2015 10:58:23 -0700 Subject: [PATCH 462/909] latest changes, structs seem to be working correctly --- pixie/ffi-infer.pxi | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 17ebe4f2..21390f6a 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -206,13 +206,15 @@ return 0; :tm_isdst]) (f/defcfn localtime)) -(let [t (time_t) - tmi (pixie.ffi/cast (localtime t) tm)] - (println (:tm_sec tmi))) (type (pixie.ffi/cast (localtime (time_t)) tm)) +(let [t (time_t) + _ (time t) + tmi (pixie.ffi/cast (localtime t) tm)] + + (println (- (:tm_hour tmi) 12) " " (:tm_min tmi))) ) From d7f2232e9a70448fdb1aca73a723f461381121b7 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 19 Nov 2014 03:56:44 +1100 Subject: [PATCH 463/909] Added filesystem functions to handle FS stuff better --- pixie/fs.pxi | 207 ++++++++++++++++++++++ pixie/test.pxi | 24 ++- pixie/vm/libs/path.py | 82 +++++++-- tests/pixie/tests/fs/parent/bar.txt | 0 tests/pixie/tests/fs/parent/child/bar.txt | 0 tests/pixie/tests/fs/parent/child/foo.txt | 0 tests/pixie/tests/fs/parent/foo.txt | 0 tests/pixie/tests/test-fs.pxi | 97 ++++++++++ 8 files changed, 379 insertions(+), 31 deletions(-) create mode 100644 pixie/fs.pxi create mode 100644 tests/pixie/tests/fs/parent/bar.txt create mode 100644 tests/pixie/tests/fs/parent/child/bar.txt create mode 100644 tests/pixie/tests/fs/parent/child/foo.txt create mode 100644 tests/pixie/tests/fs/parent/foo.txt create mode 100644 tests/pixie/tests/test-fs.pxi diff --git a/pixie/fs.pxi b/pixie/fs.pxi new file mode 100644 index 00000000..a2235d17 --- /dev/null +++ b/pixie/fs.pxi @@ -0,0 +1,207 @@ +(ns pixie.fs + (require pixie.path :as path) + (require pixie.string :as string)) + + +(defprotocol IFSPath + (path [this] + "Returns the path used to reference the File/Dir/Filesystem Object") + + (rel [this other] + "Returns the path relative to the other path") + + (abs [this] + "Returns the absolute path") + + (exists? [this] + "Returns true if the file exists") + + (basename [this] + "Returns the basename of the Filesystem Object") + + ;; TODO + (permissions [this] + "Returns a string of the octal permissions") + + (mounted? [this] + "Returns true if the directory is a mounted") + + (size [this] + "Returns the size of the file/dir on disk")) + +(defprotocol IFile + (extension [this] + "Returns the extension") + + (extension? [this ext] + "Returns true if file has extension") + + ;; TODO + (touch [this] + "Create the file if it doesn't exist.")) + +(defprotocol IDir + (list [this] + "List files and dirs underneath") + + (empty? [this] + "Returns true if directory is not empty") + + (walk [this] + "Recursively returns all files and directories below") + + (walk-files [this] + "Recursivelt returns all files underneath") + + (walk-dirs [this] + "Recursivley returns all directories underneath")) + +(defn rel-path [a b] + (let [paths-a (string/split (abs a) "/") + paths-b (string/split (abs b) "/") + ;; Get the common root of the two paths and the bits that diverge + [common diff-a diff-b] (loop [ra paths-a rb paths-b common []] + (let [ca (first ra) + cb (first rb)] + (if (and ca cb (= ca cb)) + (recur (rest ra) (rest rb) (conj common ca)) + [common ra rb])))] + (let [level (- (count diff-a) (count diff-b))] + (cond + ;; Same level + (and (zero? (count diff-a)) (zero? (count diff-b))) + "." + + ;; If B diverges by one level and a is a Dir we use ".." + (and (= 1 (count diff-b)) (instance? Dir a)) + ".." + + ;; In all other cases we want to go back to the root, + :else + (apply str (interpose "/" (concat (repeat (count diff-b) "..") diff-a))))))) + + +;; File and Dir are just wrappers around paths. +(deftype File [pathz] + IFSPath + (path [this] pathz) + (rel [this other] + (if (satisfies? IFSPath other) + (rel-path this other) + (throw "Second argument must satisfy IFSPath"))) + + (abs [this] + (path/-abs pathz)) + + (exists? [this] + (path/-exists? pathz)) + + (basename [this] + (last (string/split (abs this) "/"))) + + IFile + ;; TODO: Sort out regex or make strings partitionable. So we can split at + ;; #".". + (extension [this] + (last (string/split (abs this) "."))) + + (extension? [this ext] + (string/ends-with (abs this) ext)) + + IObject + (-hash [this] + (hash (abs this))) + + (-eq [this other] + (if (instance? File other) + (= (abs this) (abs other)) + false)) + + (-str [this] + (str (abs this))) + + (-repr [this] + (str ""))) + +(deftype Dir [pathz] + IFSPath + (path [this] pathz) + + (rel [this other] + (if (satisfies? IFSPath other) + (rel-path this other) + (throw "Second argument must satisfy IFSPath"))) + + (abs [this] + (path/-abs pathz)) + + (exists? [this] + (path/-exists? pathz)) + + (basename [this] + (last (string/split (abs this) "/"))) + + IDir + (list [this] + (path/-list-dir pathz)) + + (walk [this] + (transduce (map #(if (path/-file? %) + (->File %) + (->Dir %))) + conj + pathz)) + + (walk-files [this] + (filter #(instance? File %) + (walk this))) + + (walk-dirs [this] + (filter #(instance? Dir %) + (walk this))) + + IObject + (-hash [this] + (hash (abs this))) + + (-eq [this other] + (if (instance? Dir other) + (= (abs this) (abs other)) + false)) + + (-str [this] + (str (abs this))) + + (-repr [this] + (str ""))) + +;; (deftype Fifo [pathz]) + +(defn file + "Returns a file if the path is a file or does not exist. If a different filesystem object exists at the path an error will be thrown." + [x] + (let [x (path/-path x)] + (cond + (path/-file? x) (->File x) + (not (path/-exists? x)) (->File x) + :else (throw (str "A non-file object exists at path: " x))))) + +(defn dir + "Returns a dir if the path is a dir or does not exist. If a different filesystem object exists at the path an error will be thrown." + [x] + (let [x (path/-path x)] + (cond + (path/-dir? x) (->Dir x) + (not (path/-exists? x)) (->Dir x) + :else (throw (str "A non-dir object exists at path: " x))))) + +(defn fspath + "Returns either a File or Dir if they exist at the path" + [x] + (let [x (path/-path x)] + (cond + (path/-file? x) (->File x) + (path/-dir? x) (->Dir x) + :else (throw (str "No file or directory at path: " x))))) + + diff --git a/pixie/test.pxi b/pixie/test.pxi index 560631c1..6fcab126 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -1,9 +1,9 @@ (ns pixie.test - (require pixie.string :as s)) + (require pixie.string :as s) + (require pixie.fs :as fs)) (def tests (atom {})) - (def ^:dynamic *stats*) (def ^:dynamic *current-test*) @@ -35,7 +35,6 @@ conj @tests)] (println "Running:" (count tests) "tests") - (foreach [test tests] (test))) @@ -44,18 +43,17 @@ (pop-binding-frame!) stats)) - (defn load-all-tests [] (println "Looking for tests...") - (foreach [path @load-paths] - (println "Looking for tests in:" path) - (foreach [[dir desc filename] (pixie.path/file-list path)] - (if (= desc :file) - (if (pixie.string/starts-with filename "test-") - (if (pixie.string/ends-with filename ".pxi") - (let [fullpath (str dir "/" filename)] - (println "Loading" fullpath) - (load-file fullpath)))))))) + (let [dirs (distinct (map fs/dir @load-paths)) + pxi-files (->> dirs + (mapcat fs/walk-files) + (filter #(fs/extension? % "pxi")) + (filter #(s/starts-with (fs/basename %) "test-")) + (distinct))] + (foreach [file pxi-files] + (println "Loading " file) + (load-file (fs/abs file))))) (defmacro assert= [x y] diff --git a/pixie/vm/libs/path.py b/pixie/vm/libs/path.py index f19644e1..ff560301 100644 --- a/pixie/vm/libs/path.py +++ b/pixie/vm/libs/path.py @@ -1,44 +1,90 @@ import pixie.vm.rt as rt +import pixie.vm.util as util from pixie.vm.string import String -from pixie.vm.code import as_var, extend -from pixie.vm.object import Object, Type +from pixie.vm.code import as_var, extend, defprotocol +from pixie.vm.object import Object, Type, affirm import pixie.vm.stdlib as proto from pixie.vm.keyword import keyword +from pixie.vm.primitives import true, false, nil from rpython.rlib.clibffi import get_libc_name +from rpython.rlib.rarithmetic import intmask, r_uint import os -import pixie.vm.rt as rt -class FileList(Object): - _type = Type(u"pixie.path.FileList") +class Path(Object): + _type = Type(u"pixie.path.Path") def type(self): - return FileList._type + return Path._type def __init__(self, top): - self._top = rt.name(top) + self._path = rt.name(top) + + # keyword args don't seem to work nicely. + #def rel_path(self, other): + # "Returns the path relative to other path" + # return rt.wrap(str(os.path.relpath(self._path, start=other._path))) + + def abs_path(self): + "Returns the absolute path" + return rt.wrap(os.path.abspath(str(self._path))) + + # Basename doesn't play well with pypy... + #def basename(self): + # return rt.wrap(rt.name(os.path.basename("a"))) + + def exists(self): + return true if os.path.exists(str(self._path)) else false + + def is_file(self): + return true if os.path.isfile(str(self._path)) else false -KW_DIR = keyword(u"dir") -KW_FILE = keyword(u"file") + def is_dir(self): + return true if os.path.isdir(str(self._path)) else false -@extend(proto._reduce, FileList) +@extend(proto._reduce, Path) def _reduce(self, f, init): - assert isinstance(self, FileList) - for dirpath, dirnames, filenames in os.walk(str(self._top)): + assert isinstance(self, Path) + for dirpath, dirnames, filenames in os.walk(str(self._path)): for dirname in dirnames: - init = f.invoke([init, rt.vector(rt.wrap(dirpath), KW_DIR, rt.wrap(dirname))]) + init = f.invoke([init, Path(rt.wrap(dirpath + "/" + dirname))]) if rt.reduced_QMARK_(init): return rt.deref(init) for filename in filenames: - init = f.invoke([init, rt.vector(rt.wrap(dirpath), KW_FILE, rt.wrap(filename))]) + init = f.invoke([init, Path(rt.wrap(dirpath + "/" + filename))]) if rt.reduced_QMARK_(init): return rt.deref(init) + return init +# I have named prefixed all names with '-' to deal with the +# a namespace issue I was having. +# TODO: remove '-' and update calling functions when issue is fixed. - return init +@as_var("pixie.path", "-path") +def path(path): + return Path(path) + +# TODO: Implement this +#@as_var("pixie.path", "list-dir") +#def list_dir(path): + +@as_var("pixie.path", "-abs") +def _abs(self): + assert isinstance(self, Path) + return self.abs_path() + +@as_var("pixie.path", "-exists?") +def exists_QMARK_(self): + assert isinstance(self, Path) + return self.exists() +@as_var("pixie.path", "-file?") +def file_QMARK_(self): + assert isinstance(self, Path) + return self.is_file() -@as_var("pixie.path", "file-list") -def file_list(path): - return FileList(path) +@as_var("pixie.path", "-dir?") +def dir_QMARK_(self): + assert isinstance(self, Path) + return self.is_dir() diff --git a/tests/pixie/tests/fs/parent/bar.txt b/tests/pixie/tests/fs/parent/bar.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/pixie/tests/fs/parent/child/bar.txt b/tests/pixie/tests/fs/parent/child/bar.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/pixie/tests/fs/parent/child/foo.txt b/tests/pixie/tests/fs/parent/child/foo.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/pixie/tests/fs/parent/foo.txt b/tests/pixie/tests/fs/parent/foo.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/pixie/tests/test-fs.pxi b/tests/pixie/tests/test-fs.pxi new file mode 100644 index 00000000..c56143cd --- /dev/null +++ b/tests/pixie/tests/test-fs.pxi @@ -0,0 +1,97 @@ +(ns pixie.tests.test-fs + (require pixie.test :as t) + (require pixie.fs :as fs)) + +(t/deftest test-file-comparisons + "All these paths are the same" + (let [dir-a "tests/pixie/tests/fs/parent" + dir-b "tests/pixie/tests/fs/../fs/parent" + dir-c "tests/pixie/tests/fs/../fs/parent/../../fs/parent" + file-a (str dir-a "/foo.txt") + file-b (str dir-b "/foo.txt") + file-c (str dir-c "/foo.txt")] + (t/assert= (= (fs/dir dir-a) + (fs/dir dir-b) + (fs/dir dir-c)) + true) + + (t/assert= (= (fs/file file-a) + (fs/file file-b) + (fs/file file-c)) + true) + + (t/assert= (count (distinct [(fs/dir dir-a) + (fs/dir dir-b) + (fs/dir dir-c)])) + 1) + (t/assert= (count (distinct [(fs/file file-a) + (fs/file file-b) + (fs/file file-c)])) + 1))) +(comment + (t/deftest test-walking + (let [dir-a "tests/pixie/tests/fs"] + (t/assert= (set (fs/walk (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/dir (str dir-a "/parent/child")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) + + (t/assert= (set (fs/walk-files (fs/dir dir-a))) + #{(fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) + + (t/assert= (set (fs/walk-dirs (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/dir (str dir-a "/parent/child"))})))) + +(t/deftest test-rel? + (let [dir-a (fs/dir "tests/pixie/tests/fs") + dir-b (fs/dir "tests/pixie/tests/fs/parent") + dir-c (fs/dir "tests/pixie/tests/fs/parent/child") + file-a (fs/file "tests/pixie/tests/fs/parent/foo.txt") + file-b (fs/file "tests/pixie/tests/fs/parent/bar.txt") + file-c (fs/file "tests/pixie/tests/fs/parent/child/foo.txt")] + ;; Test directory-directory comparisons + (t/assert= (fs/rel dir-a dir-a) ".") + (t/assert= (fs/rel dir-a dir-b) "..") + (t/assert= (fs/rel dir-a dir-c) "../..") + (t/assert= (fs/rel dir-c dir-b) "child") + (t/assert= (fs/rel dir-c dir-a) "parent/child") + ;; Test file-file comparisons + (t/assert= (fs/rel file-a file-a) ".") + (t/assert= (fs/rel file-a file-b) "../foo.txt") + (t/assert= (fs/rel file-a file-c) "../../foo.txt") + (t/assert= (fs/rel file-c file-a) "../child/foo.txt") + (t/assert= (fs/rel file-c file-b) "../child/foo.txt") + ;; Test file-directory comparisons + (t/assert= (fs/rel file-a dir-a) "parent/foo.txt") + (t/assert= (fs/rel file-a dir-b) "foo.txt") + (t/assert= (fs/rel file-a dir-c) "../foo.txt"))) + +(t/deftest test-basename? + (let [dir-a (fs/dir "tests/pixie/tests/fs") + dir-b (fs/dir "tests/pixie/tests/fs/parent") + dir-c (fs/dir "tests/pixie/tests/fs/parent/child") + file-a (fs/file "tests/pixie/tests/fs/parent/foo.txt") + file-b (fs/file "tests/pixie/tests/fs/parent/bar.txt") + file-c (fs/file "tests/pixie/tests/fs/parent/child/foo.txt")] + ;; Test directory-directory comparisons + (t/assert= (fs/basename dir-a) "fs") + (t/assert= (fs/basename dir-b) "parent") + (t/assert= (fs/basename dir-c) "child") + (t/assert= (fs/basename file-a) "foo.txt") + (t/assert= (fs/basename file-b) "bar.txt") + (t/assert= (fs/basename file-c) "foo.txt"))) + +(t/deftest test-exists? + (let [real-dir (fs/dir "tests/pixie/tests/fs/parent") + fake-dir (fs/dir "tests/pixie/tests/fs/parent/fake-dir") + fake-file (fs/dir "tests/pixie/tests/fs/parent/fake-file")] + (t/assert= (fs/exists? real-dir) true) + (t/assert= (fs/exists? fake-dir) false) + (t/assert= (fs/exists? fake-file) false))) From f5686f893d834a39f8265b4a6c7b97d9ff7b7c2d Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 7 Jan 2015 01:02:27 +1100 Subject: [PATCH 464/909] Added ex-msg to stdlib this allows us to get the message out of an exception. Useful for testing exceptions. --- pixie/vm/stdlib.py | 6 ++++++ tests/pixie/tests/test-stdlib.pxi | 6 ++++++ 2 files changed, 12 insertions(+) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 768229c8..a3984799 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -651,3 +651,9 @@ def _merge__args(args): def _str(self): assert isinstance(self, RuntimeException) return rt.wrap(self.__repr__()) + +@as_var("ex-msg") +def ex_msg(e): + assert isinstance(e, RuntimeException) + return e._data + diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index dc958839..975aa9e7 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -277,3 +277,9 @@ [[1 :a] [1 :b] [1 :c] [2 :a] [2 :b] [2 :c] [3 :a] [3 :b] [3 :c]])) + +(t/deftest test-ex-msg + (try + (throw "This is an exception") + (catch e + (t/assert= (ex-msg e) "This is an exception")))) \ No newline at end of file From 7d63a985fce838b4a9d5098cb62d79c25b89ef45 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 7 Jan 2015 02:37:50 +1100 Subject: [PATCH 465/909] Strings are iterable --- pixie/stdlib.pxi | 5 +++++ tests/pixie/tests/test-stdlib.pxi | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4768171c..546ce14c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1335,6 +1335,11 @@ The new value is thus `(apply f current-value-of-atom args)`." (dotimes [x (count v)] (yield (nth v x))))) +(extend -iterator String + (fn [v] + (dotimes [x (count v)] + (yield (nth v x))))) + (defmacro and {:doc "Check if the given expressions return truthy values, returning the last, or false." :examples [["(and true false)" nil false] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index dc958839..6a0a44ef 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -27,8 +27,8 @@ (t/assert= (str {}) "{}") (t/assert= (str {:a 1}) "{:a 1}") (t/assert= (str (type 3)) "") - - (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]")) + (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]") + (t/assert= (seq (map identity "iterable")) '(\i \t \e \r \a \b \l \e))) (t/deftest test-repr (t/assert= (-repr nil) "nil") From 1a68e321033dff653cbf566a625f415b1dc1b38a Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 8 Jan 2015 04:04:04 +1100 Subject: [PATCH 466/909] Range can be turned into a lazy-seq --- pixie/stdlib.pxi | 12 +++++++----- tests/pixie/tests/test-stdlib.pxi | 10 +++++++++- 2 files changed, 16 insertions(+), 6 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 546ce14c..9d70741e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1529,7 +1529,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (* -1 x) x)) -(deftype Range [:start :stop :step] +(deftype Range [start stop step] IReduce (-reduce [self f init] (loop [i start @@ -1563,10 +1563,12 @@ For more information, see http://clojure.org/special_forms#binding-forms"} val (+ start (* idx step))] (if (cmp val stop) val - nil)))) - - - + nil))) + ISeqable + (-seq [self] + (when (or (and (> step 0) (< start stop)) + (and (< step 0) (> start stop))) + (cons start (lazy-seq* #(range (+ start step) stop step)))))) (defn range {:doc "Returns a range of numbers." diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 251cd2c9..9c051b22 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -282,4 +282,12 @@ (try (throw "This is an exception") (catch e - (t/assert= (ex-msg e) "This is an exception")))) \ No newline at end of file + (t/assert= (ex-msg e) "This is an exception")))) + +(t/deftest test-range + (t/assert= (= (-seq (range 10)) + (-seq (-iterator (range 10)) + (reduce conj nil (range 10)) + '(0 1 2 3 4 5 6 7 8 9))) + true)) + \ No newline at end of file From bc029775ccf95a1772c4ba19f87113e9f1f621b3 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 8 Jan 2015 21:24:45 +1100 Subject: [PATCH 467/909] Added arity to pixie.code.Code for better runtime errors --- pixie/vm/code.py | 11 ++++++++--- pixie/vm/compiler.py | 8 +++----- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 32679227..04659775 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -187,13 +187,14 @@ def invoke_with(self, args, this_fn): class Code(BaseCode): """Interpreted code block. Contains consts and """ _type = object.Type(u"pixie.stdlib.Code") - __immutable_fields__ = ["_consts[*]", "_bytecode", "_stack_size", "_meta"] + __immutable_fields__ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta"] def type(self): return Code._type - def __init__(self, name, bytecode, consts, stack_size, debug_points, meta=nil): + def __init__(self, name, arity, bytecode, consts, stack_size, debug_points, meta=nil): BaseCode.__init__(self) + self._arity = arity self._bytecode = bytecode self._consts = consts self._name = name @@ -202,7 +203,7 @@ def __init__(self, name, bytecode, consts, stack_size, debug_points, meta=nil): self._meta = meta def with_meta(self, meta): - return Code(self._name, self._bytecode, self._consts, self._stack_size, self._debug_points, meta) + return Code(self._name, self._arity, self._bytecode, self._consts, self._stack_size, self._debug_points, meta=meta) def get_debug_points(self): return self._debug_points @@ -217,6 +218,10 @@ def invoke_with(self, args, this_fn): ex._ex._trace.append(object.PixieCodeInfo(self._name)) raise + @elidable_promote() + def get_arity(self): + return self._arity + @elidable_promote() def get_consts(self): return self._consts diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 195d0e78..13f292e7 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -73,7 +73,7 @@ def __init__(self, name, argc, parent_ctx): locals[x] = Closure(locals[x], parent_ctx) else: locals = {} - + self.argc = argc self.bytecode = [] self.consts = [] self.locals = [locals] @@ -110,7 +110,7 @@ def pop_recur_point(self): self.recur_points.pop() def to_code(self, required_args=-1): - return code.Code(self.name, self.bytecode, clone(self.consts), self._max_sp + 1, self.debug_points) + return code.Code(self.name, self.argc, self.bytecode, clone(self.consts), self._max_sp + 1, self.debug_points) def push_arg(self, idx): self.bytecode.append(code.ARG) @@ -497,8 +497,7 @@ def compile_fn(form, ctx): ctx.sub_sp(len(arities)) else: - compile_fn_body(name, rt.first(form), rt.next(form), ctx) - + res = compile_fn_body(name, rt.first(form), rt.next(form), ctx) if rt.meta(name) is not nil: compile_meta(rt.meta(name), ctx) @@ -534,7 +533,6 @@ def compile_fn_body(name, args, body, ctx): body = rt.next(body) if body is not nil: new_ctx.pop() - new_ctx.bytecode.append(code.RETURN) closed_overs = new_ctx.closed_overs if len(closed_overs) == 0: From 3a3ccb282ac18f50eda7c6394834efb2b1f44eae Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 8 Jan 2015 22:05:52 +1100 Subject: [PATCH 468/909] Runtime-errors for pixie.code.Code arity are improved. --- pixie/vm/interpreter.py | 5 ++-- tests/pixie/tests/test-fns.pxi | 42 ++++++++++++++++++++++++++++++++++ 2 files changed, 45 insertions(+), 2 deletions(-) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 6397fbe1..c3826b26 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -100,8 +100,9 @@ def push_nth(self, delta): def push_arg(self, idx): if not 0 <= idx < len(self.args): - runtime_error(u"Invalid argument " + unicode(str(idx)) + u" function takes " - + unicode(str(len(self.args))) + u" args") + runtime_error(u"Invalid number of arguments " + unicode(str(idx)) + + u" for function '" + unicode(str(self.code_obj._name)) + u"'. Expected " + + unicode(str(self.code_obj.get_arity()))) self.push(self.args[r_uint(idx)]) diff --git a/tests/pixie/tests/test-fns.pxi b/tests/pixie/tests/test-fns.pxi index e487ba89..3f736858 100644 --- a/tests/pixie/tests/test-fns.pxi +++ b/tests/pixie/tests/test-fns.pxi @@ -13,3 +13,45 @@ (t/assert= (#(+ %1 %3) 3 'ignored 4) 7) (t/assert= (#(- %3 %1) 3 'ignored 4) 1) (t/assert= (#(apply + %1 %2 %&) 1 2 3 4 5) (+ 1 2 3 4 5))) + +;; Note these tests are for functions of type 'Code'. +(t/deftest test-code-arity-errors + (let [arity-0 (fn arity-0 []) + arity-1 (fn arity-1 [a]) + arity-2 (fn arity-2 [a b])] + (try + (arity-0 :foo) + (catch e + (t/assert= + (ex-msg e) + "Invalid number of arguments 1 for function 'arity-0'. Expected 0"))) + (try + (arity-0 :foo :bar) + (catch e + (t/assert= + (ex-msg e) + "Invalid number of arguments 2 for function 'arity-0'. Expected 0"))) + (try + (arity-1) + (catch e + (t/assert= + (ex-msg e) + "Invalid number of arguments 0 for function 'arity-1'. Expected 1"))) + (try + (arity-1 :foo :bar) + (catch e + (t/assert= + (ex-msg e) + "Invalid number of arguments 2 for function 'arity-1'. Expected 1"))) + (try + (arity-2) + (catch e + (t/assert= + (ex-msg e) + "Invalid number of arguments 0 for function 'arity-2'. Expected 2"))) + (try + (arity-2 :foo) + (catch e + (t/assert= + (ex-msg e) + "Invalid number of arguments 1 for function 'arity-2'. Expected 2"))))) From 369903de6d303de59783bd27dfd0d3987a4e0fbc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 8 Jan 2015 15:39:46 -0700 Subject: [PATCH 469/909] start of bytecode file support --- pixie/vm/code.py | 17 +++ pixie/vm/interpreter.py | 2 + pixie/vm/libs/pxic/__init__.py | 1 + pixie/vm/libs/pxic/reader.py | 151 ++++++++++++++++++++++ pixie/vm/libs/pxic/tags.py | 32 +++++ pixie/vm/libs/pxic/writer.py | 227 +++++++++++++++++++++++++++++++++ pixie/vm/persistent_list.py | 9 +- pixie/vm/rt.py | 2 +- pixie/vm/stdlib.py | 62 ++++++++- target.py | 50 +++++--- 10 files changed, 526 insertions(+), 27 deletions(-) create mode 100644 pixie/vm/libs/pxic/__init__.py create mode 100644 pixie/vm/libs/pxic/reader.py create mode 100644 pixie/vm/libs/pxic/tags.py create mode 100644 pixie/vm/libs/pxic/writer.py diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 32679227..38866058 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -407,6 +407,9 @@ def invoke_with(self, args, this_fn): def invoke(self, args): return self.deref().invoke(args) + def __repr__(self): + return "" + class bindings(py_object): def __init__(self, *args): @@ -896,3 +899,17 @@ def with_fn(fn): fn._returns = type return fn return with_fn + + + +class bindings(py_object): + def __init__(self, *args): + self._args = list(args) + + def __enter__(self): + _dynamic_vars.push_binding_frame() + for x in range(0, len(self._args), 2): + self._args[x].set_value(self._args[x + 1]) + + def __exit__(self, exc_type, exc_val, exc_tb): + _dynamic_vars.pop_binding_frame() diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 6397fbe1..cd74ce5e 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -190,9 +190,11 @@ def interpret(code_obj=None, args=[], self_obj = None, frame=None): frame=frame, is_continuation=frame._is_continuation) inst = frame.get_inst() + #print code.BYTECODES[inst], frame.consts, frame.ip if inst == code.LOAD_CONST: arg = frame.get_inst() + #print "load const", arg frame.push_const(arg) continue diff --git a/pixie/vm/libs/pxic/__init__.py b/pixie/vm/libs/pxic/__init__.py new file mode 100644 index 00000000..f87606f0 --- /dev/null +++ b/pixie/vm/libs/pxic/__init__.py @@ -0,0 +1 @@ +__author__ = 'tim' diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py new file mode 100644 index 00000000..8cb6ba2c --- /dev/null +++ b/pixie/vm/libs/pxic/reader.py @@ -0,0 +1,151 @@ +from pixie.vm.libs.pxic.tags import * +from pixie.vm.object import runtime_error +from rpython.rlib.runicode import unicode_encode_utf_8, str_decode_utf_8 +from pixie.vm.string import String +from pixie.vm.keyword import Keyword, keyword +from pixie.vm.symbol import Symbol, symbol +from pixie.vm.numbers import Integer, Float +from pixie.vm.code import Code, Var, NativeFn, Namespace, intern_var +import pixie.vm.code as code +from pixie.vm.primitives import nil, true, false +from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP +from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR +from pixie.vm.persistent_list import create_from_list +from pixie.vm.reader import LinePromise +import pixie.vm.rt as rt + + +class Reader(object): + def __init__(self, rdr): + self._rdr = rdr + self._obj_cache = {} + + def read(self, num=1): + return self._rdr.read(num) + + def read_cached(self): + obj = read_obj(self) + self._obj_cache[len(self._obj_cache)] = obj + return obj + + def read_cached_obj(self): + idx = read_raw_integer(self) + return self._obj_cache[idx] + + +def read_tag(rdr): + tag = rdr.read() + return ord(tag) + +def read_raw_integer(rdr): + return ord(rdr.read()) | (ord(rdr.read()) << 8) | (ord(rdr.read()) << 16) | (ord(rdr.read()) << 24) + +def read_raw_string(rdr): + sz = read_raw_integer(rdr) + return unicode(rdr.read(sz)) + +def read_code(rdr): + sz = read_raw_integer(rdr) + bytecode = [0] * sz + for x in range(sz): + bytecode[x] = read_raw_integer(rdr) + + sz = read_raw_integer(rdr) + consts = [None] * sz + for x in range(sz): + consts[x] = read_obj(rdr) + + stack_size = read_raw_integer(rdr) + + nm = read_raw_string(rdr) + + return Code(nm, bytecode, consts, stack_size, {}) + + +def read_var(rdr): + ns = read_raw_string(rdr) + nm = read_raw_string(rdr) + return intern_var(ns, nm) + +def read_map(rdr): + cnt = read_raw_integer(rdr) + acc = EMPTY_MAP + for x in range(cnt): + acc = rt._assoc(acc, read_obj(rdr), read_obj(rdr)) + + return acc + +def read_vector(rdr): + cnt = read_raw_integer(rdr) + acc = EMPTY_VECTOR + for x in range(cnt): + acc = rt._conj(acc, read_obj(rdr)) + + return acc + +def read_seq(rdr): + cnt = read_raw_integer(rdr) + lst = [None] * cnt + for x in range(cnt): + lst[x] = read_obj(rdr) + + return create_from_list(lst) + +def read_float(rdr): + return Float(float(read_raw_string(rdr))) + +def read_namespace(rdr): + nm = read_raw_string(rdr) + return code._ns_registry.find_or_make(nm) + +def read_obj(rdr): + tag = read_tag(rdr) + + if tag == INT: + return Integer(read_raw_integer(rdr)) + elif tag == CODE: + return read_code(rdr) + elif tag == NIL: + return nil + elif tag == VAR: + return read_var(rdr) + elif tag == STRING: + return String(read_raw_string(rdr)) + elif tag == KEYWORD: + return keyword(read_raw_string(rdr)) + elif tag == SYMBOL: + return symbol(read_raw_string(rdr)) + elif tag == LINE_PROMISE: + lp = LinePromise() + lp._str = read_raw_string(rdr) + return rdr + elif tag == MAP: + return read_map(rdr) + elif tag == TRUE: + return true + elif tag == FALSE: + return false + elif tag == NIL: + return nil + elif tag == VECTOR: + return read_vector(rdr) + elif tag == SEQ: + return read_seq(rdr) + elif tag == FLOAT: + return read_float(rdr) + elif tag == NAMESPACE: + return read_namespace(rdr) + elif tag == INT_STRING: + return Integer(int(read_raw_string(rdr))) + + + elif tag == NEW_CACHED_OBJ: + return rdr.read_cached() + elif tag == CACHED_OBJ: + return rdr.read_cached_obj() + + elif tag == EOF: + from pixie.vm.reader import eof + return eof + else: + runtime_error(u"No dispatch for bytecode: " + unicode(tag_name[tag])) diff --git a/pixie/vm/libs/pxic/tags.py b/pixie/vm/libs/pxic/tags.py new file mode 100644 index 00000000..38cbaff4 --- /dev/null +++ b/pixie/vm/libs/pxic/tags.py @@ -0,0 +1,32 @@ + +tag_name = ["INT", + "FLOAT", + "INT_STRING", + "STRING", + "CODE", + "TRUE", + "FALSE", + "NIL", + "VAR", + "MAP", + "VECTOR", + "SEQ", + "KEYWORD", + "SYMBOL", + "CACHED_OBJ", + "NEW_CACHED_OBJ", + "LINE_PROMISE", + "NAMESPACE", + "EOF"] + +tags = {} + + +SMALL_INT_START = 128 +SMALL_INT_END = 255 +SMALL_INT_MAX = SMALL_INT_END - SMALL_INT_END + +for idx, nm in enumerate(tag_name): + globals()[nm] = idx + tags[nm] = idx + diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py new file mode 100644 index 00000000..fe8545e1 --- /dev/null +++ b/pixie/vm/libs/pxic/writer.py @@ -0,0 +1,227 @@ +from pixie.vm.libs.pxic.tags import INT, STRING, INT_STRING, CODE, NIL, VAR, MAP, KEYWORD, CACHED_OBJ, NEW_CACHED_OBJ, \ + LINE_PROMISE, TRUE, FALSE, SYMBOL, VECTOR, SEQ, FLOAT, NAMESPACE, SMALL_INT_END, SMALL_INT_MAX, SMALL_INT_START, EOF +from pixie.vm.object import runtime_error +from rpython.rlib.runicode import unicode_encode_utf_8 +from pixie.vm.string import String +from pixie.vm.keyword import Keyword +from pixie.vm.symbol import Symbol +from pixie.vm.numbers import Integer, Float +from pixie.vm.code import Code, Var, NativeFn, Namespace +from pixie.vm.primitives import nil, true, false +from pixie.vm.reader import LinePromise +import pixie.vm.rt as rt + +MAX_INT32 = 2 << 32 + +class Writer(object): + def __init__(self, wtr, with_cache=False): + self._wtr = wtr + self._obj_cache = {} + self._with_cache = with_cache + + def write(self, s): + self._wtr.write(s) + + def flush(self): + self._wtr.flush() + + def write_cached_obj(self, o, wfn): + if self._with_cache: + idx = self._obj_cache.get(o, -1) + if idx == -1: + idx = len(self._obj_cache) + self._obj_cache[o] = idx + write_tag(NEW_CACHED_OBJ, self) + wfn(o, self) + else: + write_tag(CACHED_OBJ, self) + write_int_raw(idx, self) + else: + return wfn(o, self) + + def write_object(self, o): + write_object(o, self) + + def finish(self): + write_tag(EOF, self) + self._wtr.flush() + + + +def write_tag(tag, wtr): + assert tag <= 0xFF + wtr.write(chr(tag)) + +def write_int_raw(i, wtr): + #if 0 <= i <= SMALL_INT_MAX: + # wtr.write(chr((i & 0xFF) + SMALL_INT_START)) + if 0 <= i <= MAX_INT32: + wtr.write(chr(i & 0xFF)) + wtr.write(chr((i >> 8) & 0xFF)) + wtr.write(chr((i >> 16) & 0xFF)) + wtr.write(chr((i >> 24) & 0xFF)) + else: + runtime_error(u"Raw int must be less than MAX_INT32, got: " + unicode(str(i))) + +def write_string_raw(s, wtr): + assert len(s) <= MAX_INT32 + write_int_raw(len(s), wtr) + wtr.write(s) + +def write_int(i, wtr): + if 0 <= i <= MAX_INT32: + wtr.write(chr(INT)) + write_int_raw(i, wtr) + else: + wtr.write(chr(INT_STRING)) + write_string_raw(unicode(str(i)), wtr) + +def write_float(f, wtr): + write_tag(FLOAT, wtr) + write_string_raw(str(f), wtr) + +def write_string(s, wtr): + write_tag(STRING, wtr) + write_string_raw(s, wtr) + +def write_code(c, wtr): + assert isinstance(c, Code) + wtr.write(chr(CODE)) + + write_int_raw(len(c._bytecode), wtr) + for i in c._bytecode: + write_int_raw(i, wtr) + + write_int_raw(len(c._consts), wtr) + for const in c._consts: + write_object(const, wtr) + + write_int_raw(c._stack_size, wtr) + + + write_string_raw(c._name, wtr) + + +class WriteParirFn(NativeFn): + def __init__(self, wtr): + self._wtr = wtr + + def invoke(self, args): + kv = args[1] + + write_object(rt._key(kv), self._wtr) + write_object(rt._val(kv), self._wtr) + + return nil + + +def write_map(mp, wtr): + write_tag(MAP, wtr) + write_int_raw(rt.count(mp), wtr) + + rt._reduce(mp, WriteParirFn(wtr), nil) + +class WriteItem(NativeFn): + def __init__(self, wtr): + self._wtr = wtr + + def invoke(self, args): + itm = args[0] + + write_object(itm, self._wtr) + + return nil + + +def write_vector(vec, wtr): + write_tag(VECTOR, wtr) + write_int_raw(rt.count(vec), wtr) + + rt._reduce(vec, WriteItem(wtr), nil) + +def write_seq(s, wtr): + write_tag(SEQ, wtr) + write_int_raw(rt.count(s), wtr) + + s = rt.seq(s) + + while s is not nil: + write_object(rt.first(s), wtr) + s = rt.next(s) + + + + +# def __init__(self, name, bytecode, consts, stack_size, debug_points, meta=nil): +# BaseCode.__init__(self) +# self._bytecode = bytecode +# self._consts = consts +# self._name = name +# self._stack_size = stack_size +# self._debug_points = debug_points +# self._meta = meta + +def write_var(var, wtr): + assert isinstance(var, Var) + write_tag(VAR, wtr) + write_string_raw(var._ns, wtr) + write_string_raw(var._name, wtr) + + +def write_keyword(kw, wtr): + assert isinstance(kw, Keyword) + write_tag(KEYWORD, wtr) + write_string_raw(kw._str, wtr) + +def write_symbol(sym, wtr): + assert isinstance(sym, Symbol) + write_tag(SYMBOL, wtr) + write_string_raw(sym._str, wtr) + +def write_line_promise(o, wtr): + write_tag(LINE_PROMISE, wtr) + o.finalize() + write_string_raw(o._str, wtr) + +def write_namespace(o, wtr): + assert isinstance(o, Namespace) + write_tag(NAMESPACE, wtr) + write_string_raw(o._name, wtr) + + +def write_object(obj, wtr): + wtr.flush() + if isinstance(obj, String): + write_string(rt.name(obj), wtr) + elif isinstance(obj, Integer): + write_int(obj.int_val(), wtr) + elif isinstance(obj, Float): + write_float(obj.float_val(), wtr) + elif isinstance(obj, Code): + write_code(obj, wtr) + elif obj is nil: + wtr.write(chr(NIL)) + elif isinstance(obj, Var): + #wtr.write_cached_obj(obj, write_var) + write_var(obj, wtr) + elif rt.satisfies_QMARK_(rt.IMap.deref(), obj): + write_map(obj, wtr) + elif rt.satisfies_QMARK_(rt.IVector.deref(), obj): + write_vector(obj, wtr) + elif rt.satisfies_QMARK_(rt.ISeq.deref(), obj): + write_seq(obj, wtr) + elif isinstance(obj, Keyword): + wtr.write_cached_obj(obj, write_keyword) + elif isinstance(obj, LinePromise): + wtr.write_cached_obj(obj, write_line_promise) + elif obj is true: + write_tag(TRUE, wtr) + elif obj is false: + write_tag(FALSE, wtr) + elif isinstance(obj, Symbol): + write_symbol(obj, wtr) + elif isinstance(obj, Namespace): + wtr.write_cached_obj(obj, write_namespace) + else: + runtime_error(u"Object is not supported by pxic writer: " + rt.name(rt.str(obj.type()))) + diff --git a/pixie/vm/persistent_list.py b/pixie/vm/persistent_list.py index e15ca3a3..85ce81db 100644 --- a/pixie/vm/persistent_list.py +++ b/pixie/vm/persistent_list.py @@ -70,13 +70,16 @@ def count(self): @as_var("list") def list__args(args): - if len(args) == 0: + return create_from_list(args) + +def create_from_list(lst): + if len(lst) == 0: return EmptyList() - i = r_uint(len(args)) + i = r_uint(len(lst)) acc = nil while i > 0: - acc = PersistentList(args[i - 1], acc, len(args) - i + 1, nil) + acc = PersistentList(lst[i - 1], acc, len(lst) - i + 1, nil) i -= 1 return acc diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 6eb52544..9b422283 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -142,7 +142,7 @@ def reinit(): # # stacklet.with_stacklets(run_load_stdlib) - init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec"] + init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file"] for x in init_fns: globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 768229c8..aca246a0 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -389,40 +389,98 @@ def load_ns(filename): rt.load_file(rt.wrap(f)) return nil +PXIC_WRITER = intern_var(u"pixie.stdlib", u"*pxic-writer*") +PXIC_WRITER.set_root(nil) +PXIC_WRITER.set_dynamic() + + @as_var("load-file") def load_file(filename): from pixie.vm.string import String from pixie.vm.util import unicode_from_utf8 import pixie.vm.reader as reader + import pixie.vm.libs.pxic.writer as pxic_writer import os.path as path + affirm(isinstance(filename, String), u"filename must be a string") filename = str(rt.name(filename)) affirm(path.isfile(filename), unicode(filename) + u" does not exist") + if path.isfile(filename + u"c"): + load_pxic_file(filename + u"c") + return nil + f = open(filename) data = f.read() f.close() + + if data.startswith("#!"): newline_pos = data.find("\n") if newline_pos > 0: data = data[newline_pos:] - rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) + pxic_f = open(filename + "c", "wb") + wtr = pxic_writer.Writer(pxic_f) + with code.bindings(PXIC_WRITER, wtr): + rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) + wtr.finish() + pxic_f.close() + return nil +def load_pxic_file(filename): + f = open(filename) + from pixie.vm.libs.pxic.reader import Reader, read_obj + from pixie.vm.reader import eof + import sys + + if not we_are_translated(): + print "Loading precompiled file while interpreted, this may take time" + while True: + if not we_are_translated(): + sys.stdout.write(".") + sys.stdout.flush() + o = read_obj(Reader(f)) + if o is eof: + break + o.invoke([]) + + if not we_are_translated(): + print "done" + + @as_var("load-reader") def load_reader(rdr): import pixie.vm.reader as reader import pixie.vm.compiler as compiler + import sys + + if not we_are_translated(): + print "Loading file while interpreted, this may take time" + + pxic_writer = PXIC_WRITER.deref() with compiler.with_ns(u"user"): while True: + if not we_are_translated(): + sys.stdout.write(".") + sys.stdout.flush() form = reader.read(rdr, False) if form is reader.eof: return nil - compiler.compile(form).invoke([]) + compiled = compiler.compile(form) + + if pxic_writer is not nil: + pxic_writer.write_object(compiled) + + compiled.invoke([]) + + if not we_are_translated(): + print "done" + return nil @as_var("the-ns") diff --git a/target.py b/target.py index fb041241..a82d8805 100644 --- a/target.py +++ b/target.py @@ -177,27 +177,35 @@ def run_load_stdlib(): return import pixie.vm.compiler as compiler import pixie.vm.reader as reader - f = open(rpath.rjoin(str(rt.name(load_path.deref())), "pixie/stdlib.pxi")) - data = f.read() - f.close() - rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pxi") - result = nil - - if not we_are_translated(): - print "Loading stdlib while interpreted, this will take some time..." - - with compiler.with_ns(u"pixie.stdlib"): - while True: - if not we_are_translated(): - sys.stdout.write(".") - sys.stdout.flush() - form = reader.read(rdr, False) - if form is reader.eof: - break - result = compiler.compile(form).invoke([]) - - if not we_are_translated(): - print "done" + + rt.load_file(rt.wrap(u"pixie/stdlib.pxi")) + + #f = open(rpath.rjoin(str(rt.name(load_path.deref())), "pixie/stdlib.pxi")) + #data = f.read() + #f.close() + # rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pxi") + # result = nil + # + # if not we_are_translated(): + # print "Loading stdlib while interpreted, this will take some time..." + # + # wf = open("stdlib.pxic", "wb") + # from pixie.vm.libs.pxic.writer import write_object, Writer + # + # with compiler.with_ns(u"pixie.stdlib"): + # while True: + # if not we_are_translated(): + # sys.stdout.write(".") + # sys.stdout.flush() + # form = reader.read(rdr, False) + # if form is reader.eof: + # break + # result = compiler.compile(form) + # write_object(result, Writer(wf)) + # result.invoke([]) + # + # if not we_are_translated(): + # print "done" stdlib_loaded.set_true() From 5bbacca675321649122f6d125c0c3c5c16d48d63 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 8 Jan 2015 21:11:24 -0700 Subject: [PATCH 470/909] should translate now --- pixie/vm/code.py | 4 ---- pixie/vm/libs/pxic/reader.py | 19 +++++++++++-------- pixie/vm/libs/pxic/writer.py | 23 +++++++++++++++++++---- pixie/vm/stdlib.py | 8 ++++---- 4 files changed, 34 insertions(+), 20 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 38866058..bbd0d001 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -407,10 +407,6 @@ def invoke_with(self, args, this_fn): def invoke(self, args): return self.deref().invoke(args) - def __repr__(self): - return "" - - class bindings(py_object): def __init__(self, *args): self._args = list(args) diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index 8cb6ba2c..420e8be3 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -12,6 +12,7 @@ from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR from pixie.vm.persistent_list import create_from_list from pixie.vm.reader import LinePromise +from rpython.rlib.rarithmetic import r_uint, intmask import pixie.vm.rt as rt @@ -20,8 +21,8 @@ def __init__(self, rdr): self._rdr = rdr self._obj_cache = {} - def read(self, num=1): - return self._rdr.read(num) + def read(self, num=r_uint(1)): + return self._rdr.read(intmask(num)) def read_cached(self): obj = read_obj(self) @@ -35,10 +36,10 @@ def read_cached_obj(self): def read_tag(rdr): tag = rdr.read() - return ord(tag) + return ord(tag[0]) def read_raw_integer(rdr): - return ord(rdr.read()) | (ord(rdr.read()) << 8) | (ord(rdr.read()) << 16) | (ord(rdr.read()) << 24) + return r_uint(ord(rdr.read()[0]) | (ord(rdr.read()[0]) << 8) | (ord(rdr.read()[0]) << 16) | (ord(rdr.read()[0]) << 24)) def read_raw_string(rdr): sz = read_raw_integer(rdr) @@ -46,7 +47,7 @@ def read_raw_string(rdr): def read_code(rdr): sz = read_raw_integer(rdr) - bytecode = [0] * sz + bytecode = [r_uint(0)] * sz for x in range(sz): bytecode[x] = read_raw_integer(rdr) @@ -92,7 +93,7 @@ def read_seq(rdr): return create_from_list(lst) def read_float(rdr): - return Float(float(read_raw_string(rdr))) + return Float(float(str(read_raw_string(rdr)))) def read_namespace(rdr): nm = read_raw_string(rdr) @@ -102,7 +103,7 @@ def read_obj(rdr): tag = read_tag(rdr) if tag == INT: - return Integer(read_raw_integer(rdr)) + return Integer(intmask(read_raw_integer(rdr))) elif tag == CODE: return read_code(rdr) elif tag == NIL: @@ -118,7 +119,7 @@ def read_obj(rdr): elif tag == LINE_PROMISE: lp = LinePromise() lp._str = read_raw_string(rdr) - return rdr + return lp elif tag == MAP: return read_map(rdr) elif tag == TRUE: @@ -149,3 +150,5 @@ def read_obj(rdr): return eof else: runtime_error(u"No dispatch for bytecode: " + unicode(tag_name[tag])) + + return nil diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index fe8545e1..2ce3fb2d 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -1,6 +1,6 @@ from pixie.vm.libs.pxic.tags import INT, STRING, INT_STRING, CODE, NIL, VAR, MAP, KEYWORD, CACHED_OBJ, NEW_CACHED_OBJ, \ LINE_PROMISE, TRUE, FALSE, SYMBOL, VECTOR, SEQ, FLOAT, NAMESPACE, SMALL_INT_END, SMALL_INT_MAX, SMALL_INT_START, EOF -from pixie.vm.object import runtime_error +from pixie.vm.object import runtime_error, Object, Type from rpython.rlib.runicode import unicode_encode_utf_8 from pixie.vm.string import String from pixie.vm.keyword import Keyword @@ -9,9 +9,11 @@ from pixie.vm.code import Code, Var, NativeFn, Namespace from pixie.vm.primitives import nil, true, false from pixie.vm.reader import LinePromise +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import r_uint import pixie.vm.rt as rt -MAX_INT32 = 2 << 32 +MAX_INT32 = r_uint(1 << 32) class Writer(object): def __init__(self, wtr, with_cache=False): @@ -20,6 +22,7 @@ def __init__(self, wtr, with_cache=False): self._with_cache = with_cache def write(self, s): + assert isinstance(s, str) self._wtr.write(s) def flush(self): @@ -46,7 +49,16 @@ def finish(self): write_tag(EOF, self) self._wtr.flush() +class WriterBox(Object): + _type = Type(u"pixie.stdlib.WriterBox") + def type(self): + return WriterBox._type + def __init__(self, wtr): + self._pxic_writer = wtr + + def get_pxic_writer(self): + return self._pxic_writer def write_tag(tag, wtr): assert tag <= 0xFF @@ -63,7 +75,9 @@ def write_int_raw(i, wtr): else: runtime_error(u"Raw int must be less than MAX_INT32, got: " + unicode(str(i))) -def write_string_raw(s, wtr): +def write_string_raw(si, wtr): + assert isinstance(si, unicode) + s = str(si) assert len(s) <= MAX_INT32 write_int_raw(len(s), wtr) wtr.write(s) @@ -78,7 +92,7 @@ def write_int(i, wtr): def write_float(f, wtr): write_tag(FLOAT, wtr) - write_string_raw(str(f), wtr) + write_string_raw(unicode(str(f)), wtr) def write_string(s, wtr): write_tag(STRING, wtr) @@ -179,6 +193,7 @@ def write_symbol(sym, wtr): write_string_raw(sym._str, wtr) def write_line_promise(o, wtr): + assert isinstance(o, LinePromise) write_tag(LINE_PROMISE, wtr) o.finalize() write_string_raw(o._str, wtr) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index aca246a0..44cb42a6 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -407,8 +407,8 @@ def load_file(filename): filename = str(rt.name(filename)) affirm(path.isfile(filename), unicode(filename) + u" does not exist") - if path.isfile(filename + u"c"): - load_pxic_file(filename + u"c") + if path.isfile(filename + "c"): + load_pxic_file(filename + "c") return nil f = open(filename) @@ -424,7 +424,7 @@ def load_file(filename): pxic_f = open(filename + "c", "wb") wtr = pxic_writer.Writer(pxic_f) - with code.bindings(PXIC_WRITER, wtr): + with code.bindings(PXIC_WRITER, pxic_writer.WriterBox(wtr)): rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) wtr.finish() pxic_f.close() @@ -461,7 +461,7 @@ def load_reader(rdr): if not we_are_translated(): print "Loading file while interpreted, this may take time" - pxic_writer = PXIC_WRITER.deref() + pxic_writer = PXIC_WRITER.deref().get_pxic_writer() with compiler.with_ns(u"user"): while True: From 1e5b64dd3b45dbaaad607eeb720692ba81f1fe64 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 9 Jan 2015 09:30:15 -0700 Subject: [PATCH 471/909] fixed bugs related to pxic files --- pixie/ffi-infer.pxi | 25 +++++++++++-------------- pixie/stdlib.pxi | 4 ++-- pixie/vm/compiler.py | 38 +++++++++++--------------------------- pixie/vm/stdlib.py | 28 ++++++++++++++++++++-------- 4 files changed, 44 insertions(+), 51 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 21390f6a..463a60ef 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -122,19 +122,14 @@ return 0; result (read-string (io/run-command cmd-str))] `(do ~@(map generate-code cmds result)))) - -(binding [*config* {:includes ["sys/stat.h"]}] - (run-infer nil - [{:op :struct :name "stat" - :members ["st_size"]}])) - - (defmacro with-config [config & body] - `(binding [*config* ~config - *bodies* (atom []) - *library* (ffi-library ~(str "lib" (:library config) "." pixie.platform/so-ext))] - ~@body - (eval (run-infer *config* @*bodies*)))) + (binding [*config* config + *bodies* (atom []) + *library* (ffi-library (str "lib" (:library config) "." pixie.platform/so-ext))] + (doseq [b body] + (eval b)) + `(let [*library* (ffi-library ~(str "lib" (:library config) "." pixie.platform/so-ext))] + ~(run-infer *config* @*bodies*)))) (defmacro defcfn [nm] (let [name-str (name nm)] @@ -150,16 +145,18 @@ return 0; :members ~(vec (map name members))) )) + + (comment + (with-config {:library "SDL" :cxx-flags ["`sdl2-config --cflags --libs`"] :includes ["SDL.h"] } (defconst SDL_INIT_EVERYTHING) (defcfn SDL_Init) - (defcfn SDL_CreateWindow) - (defconst SDL_WINDOW_SHOWN)) + (defconst SDL_WINDOW_SHOWN)) (f/with-config {:library "c" :cxx-flags ["-lc"] diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4768171c..0eeda17d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1,4 +1,4 @@ -(__ns__ pixie.stdlib) +(in-ns :pixie.stdlib) (def libc (ffi-library pixie.platform/lib-c-name)) (def exit (ffi-fn libc "exit" [CInt] CInt)) @@ -1039,7 +1039,7 @@ Creates new maps if the keys are not present." (refer-ns (this-ns-name) (the-ns (quote ~ns)) (quote ~as-nm)))) (defmacro ns [nm & body] - `(do (__ns__ ~nm) + `(do (in-ns ~(keyword (name nm))) ~@body)) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 195d0e78..e9e52dee 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -441,18 +441,6 @@ def compile_platform_plus(form, ctx): ctx.enable_tail_call() return ctx -def compile_platform_eq(form, ctx): - form = form.next() - - affirm(rt.count(form) == 2, u"TODO: REMOVE") - while form is not nil: - compile_form(form.first(), ctx) - form = form.next() - - ctx.bytecode.append(code.EQ) - ctx.sub_sp(1) - return ctx - def add_args(name, args, ctx): required_args = -1 local_idx = 0 @@ -724,19 +712,6 @@ def compile_loop(form, ctx): def compile_comment(form, ctx): ctx.push_const(nil) -def compile_ns(form, ctx): - affirm(rt.count(form) == 2, u"ns only takes one argument, a symbol") - - nm = rt.first(rt.next(form)) - - affirm(isinstance(nm, symbol.Symbol), u"Namespace name must be a symbol") - - str_name = rt.name(nm) - - NS_VAR.set_value(code._ns_registry.find_or_make(str_name)) - NS_VAR.deref().include_stdlib() - ctx.push_const(nil) - def compile_this_ns(form, ctx): ctx.push_const(NS_VAR.deref()) @@ -762,9 +737,15 @@ def compile_yield(form, ctx): compile_form(arg, ctx) ctx.bytecode.append(code.YIELD) +def compile_in_ns(form, ctx): + affirm(rt.count(form) == 2, u"in-ns requires an argument") + arg = rt.first(rt.next(form)) + NS_VAR.set_value(code._ns_registry.find_or_make(rt.name(arg))) + NS_VAR.deref().include_stdlib() + compile_fn_call(form, ctx) + builtins = {u"fn*": compile_fn, u"if": compile_if, - u"platform=": compile_platform_eq, u"def": compile_def, u"do": compile_do, u"quote": compile_quote, @@ -773,9 +754,9 @@ def compile_yield(form, ctx): u"loop": compile_loop, u"comment": compile_comment, u"var": compile_var, - u"__ns__": compile_ns, u"catch": compile_catch, u"this-ns-name": compile_this_ns, + u"in-ns": compile_in_ns, # yes, this is both a function and a compiler special form. u"yield": compile_yield} def compiler_special(s): @@ -799,6 +780,9 @@ def compile_cons(form, ctx): if special is not None: return special(form, ctx) + return compile_fn_call(form, ctx) + +def compile_fn_call(form, ctx): macro = is_macro_call(form, ctx) if macro: return compile_form(call_macro(macro, form, ctx), ctx) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 44cb42a6..190ca8cb 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -435,18 +435,21 @@ def load_pxic_file(filename): f = open(filename) from pixie.vm.libs.pxic.reader import Reader, read_obj from pixie.vm.reader import eof + import pixie.vm.compiler as compiler import sys if not we_are_translated(): print "Loading precompiled file while interpreted, this may take time" - while True: - if not we_are_translated(): - sys.stdout.write(".") - sys.stdout.flush() - o = read_obj(Reader(f)) - if o is eof: - break - o.invoke([]) + with compiler.with_ns(u"user"): + compiler.NS_VAR.deref().include_stdlib() + while True: + if not we_are_translated(): + sys.stdout.write(".") + sys.stdout.flush() + o = read_obj(Reader(f)) + if o is eof: + break + o.invoke([]) if not we_are_translated(): print "done" @@ -464,6 +467,7 @@ def load_reader(rdr): pxic_writer = PXIC_WRITER.deref().get_pxic_writer() with compiler.with_ns(u"user"): + compiler.NS_VAR.deref().include_stdlib() while True: if not we_are_translated(): sys.stdout.write(".") @@ -489,6 +493,14 @@ def the_ns(ns_name): return code._ns_registry.get(rt.name(ns_name), nil) +@as_var("in-ns") +def the_ns(ns_name): + from pixie.vm.compiler import NS_VAR + NS_VAR.set_value(code._ns_registry.find_or_make(rt.name(ns_name))) + NS_VAR.deref().include_stdlib() + + return nil + @as_var("ns-map") def ns_map(ns): from pixie.vm.code import Namespace From 1c5a79bfc419d4dea8266f2a6d0a2b0e412c95a4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 10 Jan 2015 09:37:54 -0700 Subject: [PATCH 472/909] tests pass when loaded via pxic files --- pixie/stdlib.pxi | 13 +++++++++++++ pixie/vm/code.py | 10 ++++++++-- pixie/vm/libs/pxic/reader.py | 25 +++++++++++++++++++++---- pixie/vm/libs/pxic/tags.py | 1 + pixie/vm/libs/pxic/writer.py | 18 +++++++++++++----- pixie/vm/numbers.py | 13 +++++++++++++ pixie/vm/object.py | 7 +++++-- pixie/vm/persistent_hash_set.py | 6 ++---- pixie/vm/persistent_vector.py | 2 ++ pixie/vm/stdlib.py | 13 +++++++++++-- pixie/vm/string.py | 12 +++++++++++- 11 files changed, 100 insertions(+), 20 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 0eeda17d..763a3eee 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -334,10 +334,18 @@ (fn [v] (transduce ordered-hash-reducing-fn v))) + + +(add-marshall-handlers PersistentHashSet + (fn [obj] (vec obj)) + (fn [obj] (apply hash-set obj))) + (extend -hash PersistentHashMap (fn [v] (transduce ordered-hash-reducing-fn v))) + + (extend -hash EmptyList (fn [v] 5555555)) (extend -hash Bool @@ -677,6 +685,11 @@ there's a value associated with the key. Use `some` for checking for values." [coll key] (-contains-key coll key)) +(defn hash-set [& args] + {:doc "Creates a hash-set from the arguments of the function" + :added "0.1"} + (set args)) + (defn vec {:doc "Converts a reducable collection into a vector using the (optional) transducer." :signatures [[coll] [xform coll]] diff --git a/pixie/vm/code.py b/pixie/vm/code.py index bbd0d001..ba5f6fad 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -379,19 +379,25 @@ def set_dynamic(self): self._dynamic = True self._rev += 1 + def get_dynamic_value(self): return _dynamic_vars.get_var_value(self, self._root) + + @elidable_promote() - def is_dynamic(self, rev): + def _is_dynamic(self, rev): return self._dynamic + def is_dynamic(self): + return self._is_dynamic(self._rev) + @elidable_promote() def get_root(self, rev): return self._root def deref(self): - if self.is_dynamic(self._rev): + if self.is_dynamic(): return self.get_dynamic_value() else: val = self.get_root(self._rev) diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index 420e8be3..be151a1b 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -1,6 +1,6 @@ from pixie.vm.libs.pxic.tags import * -from pixie.vm.object import runtime_error -from rpython.rlib.runicode import unicode_encode_utf_8, str_decode_utf_8 +from pixie.vm.object import runtime_error, get_type_by_name +from rpython.rlib.runicode import str_decode_utf_8 from pixie.vm.string import String from pixie.vm.keyword import Keyword, keyword from pixie.vm.symbol import Symbol, symbol @@ -13,6 +13,7 @@ from pixie.vm.persistent_list import create_from_list from pixie.vm.reader import LinePromise from rpython.rlib.rarithmetic import r_uint, intmask +from pixie.vm.libs.pxic.util import read_handlers import pixie.vm.rt as rt @@ -43,7 +44,9 @@ def read_raw_integer(rdr): def read_raw_string(rdr): sz = read_raw_integer(rdr) - return unicode(rdr.read(sz)) + errors = [] + s, pos = str_decode_utf_8(rdr.read(sz), sz, errors) + return s def read_code(rdr): sz = read_raw_integer(rdr) @@ -66,7 +69,11 @@ def read_code(rdr): def read_var(rdr): ns = read_raw_string(rdr) nm = read_raw_string(rdr) - return intern_var(ns, nm) + is_dynamic = read_tag(rdr) + var = intern_var(ns, nm) + if is_dynamic is TRUE: + var.set_dynamic() + return var def read_map(rdr): cnt = read_raw_integer(rdr) @@ -148,6 +155,16 @@ def read_obj(rdr): elif tag == EOF: from pixie.vm.reader import eof return eof + + elif tag == TAGGED: + tp_name = read_raw_string(rdr) + tp = get_type_by_name(tp_name) + handler = read_handlers.get(tp, None) + if handler is None: + runtime_error(u"No type handler for " + tp_name) + + obj = read_obj(rdr) + return handler.invoke([obj]) else: runtime_error(u"No dispatch for bytecode: " + unicode(tag_name[tag])) diff --git a/pixie/vm/libs/pxic/tags.py b/pixie/vm/libs/pxic/tags.py index 38cbaff4..ac8b6334 100644 --- a/pixie/vm/libs/pxic/tags.py +++ b/pixie/vm/libs/pxic/tags.py @@ -17,6 +17,7 @@ "NEW_CACHED_OBJ", "LINE_PROMISE", "NAMESPACE", + "TAGGED", "EOF"] tags = {} diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index 2ce3fb2d..644045f3 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -1,5 +1,4 @@ -from pixie.vm.libs.pxic.tags import INT, STRING, INT_STRING, CODE, NIL, VAR, MAP, KEYWORD, CACHED_OBJ, NEW_CACHED_OBJ, \ - LINE_PROMISE, TRUE, FALSE, SYMBOL, VECTOR, SEQ, FLOAT, NAMESPACE, SMALL_INT_END, SMALL_INT_MAX, SMALL_INT_START, EOF +from pixie.vm.libs.pxic.tags import * from pixie.vm.object import runtime_error, Object, Type from rpython.rlib.runicode import unicode_encode_utf_8 from pixie.vm.string import String @@ -77,7 +76,8 @@ def write_int_raw(i, wtr): def write_string_raw(si, wtr): assert isinstance(si, unicode) - s = str(si) + errors = [] + s = unicode_encode_utf_8(si, len(si), errors) assert len(s) <= MAX_INT32 write_int_raw(len(s), wtr) wtr.write(s) @@ -140,7 +140,7 @@ def __init__(self, wtr): self._wtr = wtr def invoke(self, args): - itm = args[0] + itm = args[1] write_object(itm, self._wtr) @@ -180,6 +180,7 @@ def write_var(var, wtr): write_tag(VAR, wtr) write_string_raw(var._ns, wtr) write_string_raw(var._name, wtr) + write_tag(TRUE if var.is_dynamic() else FALSE, wtr) def write_keyword(kw, wtr): @@ -238,5 +239,12 @@ def write_object(obj, wtr): elif isinstance(obj, Namespace): wtr.write_cached_obj(obj, write_namespace) else: - runtime_error(u"Object is not supported by pxic writer: " + rt.name(rt.str(obj.type()))) + from pixie.vm.libs.pxic.util import write_handlers + handler = write_handlers.get(obj.type(), None) + if handler is None: + runtime_error(u"Object is not supported by pxic writer: " + rt.name(rt.str(obj.type()))) + else: + write_tag(TAGGED, wtr) + write_string_raw(obj.type().name(), wtr) + write_object(handler.invoke([obj]), wtr) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 624c4dc3..863a854f 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -5,6 +5,7 @@ from rpython.rlib.rbigint import rbigint import rpython.rlib.jit as jit from pixie.vm.code import DoublePolymorphicFn, extend, Protocol, as_var, wrap_fn +from pixie.vm.libs.pxic.util import add_marshall_handlers import pixie.vm.rt as rt import math @@ -68,6 +69,7 @@ class Ratio(Number): _immutable_fields_ = ["_numerator", "_denominator"] def __init__(self, numerator, denominator): + assert numerator is not None and denominator is not None self._numerator = numerator self._denominator = denominator @@ -80,6 +82,17 @@ def denominator(self): def type(self): return Ratio._type +@wrap_fn +def ratio_write(obj): + assert isinstance(obj, Ratio) + return rt.vector(rt.wrap(obj.numerator()), rt.wrap(obj.denominator())) + +@wrap_fn +def ratio_read(obj): + return Ratio(rt.nth(obj, rt.wrap(0)).int_val(), rt.nth(obj, rt.wrap(1)).int_val()) + +add_marshall_handlers(Ratio._type, ratio_write, ratio_read) + IMath = as_var("IMath")(Protocol(u"IMath")) _add = as_var("-add")(DoublePolymorphicFn(u"-add", IMath)) _sub = as_var("-sub")(DoublePolymorphicFn(u"-sub", IMath)) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 4a78cbec..c19cfab0 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -14,11 +14,11 @@ def invoke(self, args): return stdlib.invoke_other(self, args) def int_val(self): - affirm(False, u"Expected Number") + affirm(False, u"Expected Number, not " + self.type().name()) return 0 def r_uint_val(self): - affirm(False, u"Expected Number") + affirm(False, u"Expected Number, not " + self.type().name()) return 0 def promote(self): @@ -58,6 +58,9 @@ def get_by_name(self, nm, default=None): _type_registry = TypeRegistry() +def get_type_by_name(nm): + return _type_registry.get_by_name(nm) + class Type(Object): def __init__(self, name, parent = None): assert isinstance(name, unicode), u"Type names must be unicode" diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 7ae23443..1170b4bd 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -5,11 +5,12 @@ from pixie.vm.numbers import Integer import pixie.vm.persistent_hash_map as persistent_hash_map import pixie.vm.stdlib as proto -from pixie.vm.code import extend, as_var, intern_var +from pixie.vm.code import extend, as_var, intern_var, wrap_fn from rpython.rlib.rarithmetic import r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt from pixie.vm.iterator import MapIterator +from pixie.vm.libs.pxic.util import add_marshall_handlers VAR_KEY = intern_var(u"pixie.stdlib", u"key") @@ -39,9 +40,6 @@ def with_meta(self, meta): def iter(self): return MapIterator(VAR_KEY.deref(), self._map.iter()) - - - EMPTY = PersistentHashSet(nil, persistent_hash_map.EMPTY) @as_var("set") diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 0f070df9..743e2fb3 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -535,6 +535,8 @@ def _reduce(self, f, init): @as_var("vector") def vector__args(args): + if len(args) < 32: + return PersistentVector(nil, r_uint(len(args)), r_uint(5), EMPTY_NODE, args) acc = rt._transient(EMPTY) for x in range(len(args)): acc = rt._conj_BANG_(acc, args[x]) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 190ca8cb..4a5ff376 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -464,7 +464,11 @@ def load_reader(rdr): if not we_are_translated(): print "Loading file while interpreted, this may take time" - pxic_writer = PXIC_WRITER.deref().get_pxic_writer() + val = PXIC_WRITER.deref() + if val is nil: + pxic_writer = None + else: + pxic_writer = val.get_pxic_writer() with compiler.with_ns(u"user"): compiler.NS_VAR.deref().include_stdlib() @@ -477,7 +481,7 @@ def load_reader(rdr): return nil compiled = compiler.compile(form) - if pxic_writer is not nil: + if pxic_writer is not None: pxic_writer.write_object(compiled) compiled.invoke([]) @@ -577,6 +581,11 @@ def type_by_name(nm): runtime_error(u"type name must be string") return _type_registry.get_by_name(nm._str, nil) +@as_var("add-marshall-handlers") +def _add_marshall_handlers(tp, write, read): + from pixie.vm.libs.pxic.util import add_marshall_handlers + add_marshall_handlers(tp, write, read) + return nil @as_var("deref") diff --git a/pixie/vm/string.py b/pixie/vm/string.py index 88d756b8..f7687633 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -1,11 +1,12 @@ import pixie.vm.rt as rt from pixie.vm.object import Object, Type, affirm -from pixie.vm.code import extend, as_var +from pixie.vm.code import extend, as_var, wrap_fn from pixie.vm.primitives import nil, true, false from pixie.vm.numbers import Integer, _add import pixie.vm.stdlib as proto import pixie.vm.util as util from rpython.rlib.rarithmetic import intmask, r_uint +from pixie.vm.libs.pxic.util import add_marshall_handlers class String(Object): _type = Type(u"pixie.stdlib.String") @@ -77,7 +78,16 @@ def __init__(self, i): def char_val(self): return self._char_val +@wrap_fn +def write_char(obj): + assert isinstance(obj, Character) + return rt.wrap(obj._char_val) +@wrap_fn +def read_char(obj): + return Character(obj.int_val()) + +add_marshall_handlers(Character._type, write_char, read_char) @extend(proto._str, Character) def _str(self): From e88c7824da2cc07857d91871b8ddb36e876d3d51 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 10 Jan 2015 09:40:45 -0700 Subject: [PATCH 473/909] disable a non-working optimization --- pixie/vm/persistent_vector.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 743e2fb3..0f070df9 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -535,8 +535,6 @@ def _reduce(self, f, init): @as_var("vector") def vector__args(args): - if len(args) < 32: - return PersistentVector(nil, r_uint(len(args)), r_uint(5), EMPTY_NODE, args) acc = rt._transient(EMPTY) for x in range(len(args)): acc = rt._conj_BANG_(acc, args[x]) From 0e84b7058707a2396922e62591b830859c1f215f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 10 Jan 2015 12:41:11 -0700 Subject: [PATCH 474/909] fixed translation errors --- pixie/vm/libs/pxic/reader.py | 2 +- pixie/vm/libs/pxic/writer.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index be151a1b..ba8a3f83 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -44,7 +44,7 @@ def read_raw_integer(rdr): def read_raw_string(rdr): sz = read_raw_integer(rdr) - errors = [] + errors = "?" s, pos = str_decode_utf_8(rdr.read(sz), sz, errors) return s diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index 644045f3..f87b6868 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -76,7 +76,7 @@ def write_int_raw(i, wtr): def write_string_raw(si, wtr): assert isinstance(si, unicode) - errors = [] + errors = "?" s = unicode_encode_utf_8(si, len(si), errors) assert len(s) <= MAX_INT32 write_int_raw(len(s), wtr) From 6ee07268a29083802e22944bd81ffea40ee4b8c2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 10 Jan 2015 15:49:29 -0700 Subject: [PATCH 475/909] upload missing util file --- pixie/vm/libs/pxic/util.py | 9 +++++++++ 1 file changed, 9 insertions(+) create mode 100644 pixie/vm/libs/pxic/util.py diff --git a/pixie/vm/libs/pxic/util.py b/pixie/vm/libs/pxic/util.py new file mode 100644 index 00000000..01f6e5c1 --- /dev/null +++ b/pixie/vm/libs/pxic/util.py @@ -0,0 +1,9 @@ + + +read_handlers = {} +write_handlers = {} + +def add_marshall_handlers(tp, write, read): + read_handlers[tp] = read + write_handlers[tp] = write + From fec0d5d2335b56a98ca37f770a406d49cefd8e06 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 12 Jan 2015 06:52:15 -0700 Subject: [PATCH 476/909] add -c option to compile files, removed compilation of pxic files by default --- pixie/vm/rt.py | 2 +- pixie/vm/stdlib.py | 32 +++++++++++++++++++++-------- target.py | 51 ++++++++++++++++------------------------------ 3 files changed, 42 insertions(+), 43 deletions(-) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 9b422283..5ee322e3 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -142,7 +142,7 @@ def reinit(): # # stacklet.with_stacklets(run_load_stdlib) - init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file"] + init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file", u"compile-file"] for x in init_fns: globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 4a5ff376..b8cd6e2a 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -393,24 +393,33 @@ def load_ns(filename): PXIC_WRITER.set_root(nil) PXIC_WRITER.set_dynamic() - @as_var("load-file") def load_file(filename): + return _load_file(filename, False) + +@as_var("compile-file") +def compile_file(filename): + return _load_file(filename, True) + +def _load_file(filename, compile=False): from pixie.vm.string import String from pixie.vm.util import unicode_from_utf8 import pixie.vm.reader as reader import pixie.vm.libs.pxic.writer as pxic_writer import os.path as path + from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR + import os affirm(isinstance(filename, String), u"filename must be a string") filename = str(rt.name(filename)) - affirm(path.isfile(filename), unicode(filename) + u" does not exist") - if path.isfile(filename + "c"): + if path.isfile(filename + "c") and not compile: load_pxic_file(filename + "c") return nil + affirm(path.isfile(filename), unicode(filename) + u" does not exist") + f = open(filename) data = f.read() f.close() @@ -422,12 +431,17 @@ def load_file(filename): if newline_pos > 0: data = data[newline_pos:] - pxic_f = open(filename + "c", "wb") - wtr = pxic_writer.Writer(pxic_f) - with code.bindings(PXIC_WRITER, pxic_writer.WriterBox(wtr)): - rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) - wtr.finish() - pxic_f.close() + if compile: + pxic_f = open(filename + "c", "wb") + wtr = pxic_writer.Writer(pxic_f) + with code.bindings(PXIC_WRITER, pxic_writer.WriterBox(wtr)): + rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) + wtr.finish() + pxic_f.close() + else: + with code.bindings(PXIC_WRITER, nil): + rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) + return nil diff --git a/target.py b/target.py index a82d8805..767ae7c8 100644 --- a/target.py +++ b/target.py @@ -175,38 +175,9 @@ def run_load_stdlib(): global stdlib_loaded if stdlib_loaded.is_true(): return - import pixie.vm.compiler as compiler - import pixie.vm.reader as reader rt.load_file(rt.wrap(u"pixie/stdlib.pxi")) - #f = open(rpath.rjoin(str(rt.name(load_path.deref())), "pixie/stdlib.pxi")) - #data = f.read() - #f.close() - # rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pxi") - # result = nil - # - # if not we_are_translated(): - # print "Loading stdlib while interpreted, this will take some time..." - # - # wf = open("stdlib.pxic", "wb") - # from pixie.vm.libs.pxic.writer import write_object, Writer - # - # with compiler.with_ns(u"pixie.stdlib"): - # while True: - # if not we_are_translated(): - # sys.stdout.write(".") - # sys.stdout.flush() - # form = reader.read(rdr, False) - # if form is reader.eof: - # break - # result = compiler.compile(form) - # write_object(result, Writer(wf)) - # result.invoke([]) - # - # if not we_are_translated(): - # print "done" - stdlib_loaded.set_true() def load_stdlib(): @@ -214,6 +185,7 @@ def load_stdlib(): def entry_point(args): interactive = True + exit = False script_args = [] init_load_path(args[0]) @@ -234,6 +206,7 @@ def entry_point(args): print " -v, --version print the version number" print " -e, --eval= evaluate the given expression" print " -l, --load-path= add to pixie.stdlib/load-paths" + print " -c, --compile= compile to a .pxic file" return 0 elif arg == '-e' or arg == '--eval': i += 1 @@ -252,6 +225,17 @@ def entry_point(args): else: print "Expected argument for " + arg return 1 + + elif arg == "-c" or arg == "--compile": + i += 1 + if i < len(args): + path = args[i] + print "Compiling ", path + rt.compile_file(rt.wrap(path)) + exit = True + else: + print "Expected argument for " + arg + return 1 else: print "Unknown option " + arg return 1 @@ -262,10 +246,11 @@ def entry_point(args): i += 1 - if interactive: - ReplFn(args).invoke([]) - else: - BatchModeFn(script_args).invoke([]) + if not exit: + if interactive: + ReplFn(args).invoke([]) + else: + BatchModeFn(script_args).invoke([]) return 0 From 1cd31690cb4431da35d0be3614505fd0c5a9383a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 12 Jan 2015 16:31:10 -0700 Subject: [PATCH 477/909] fix translation errors from merging, enable caching, support debug points --- pixie/vm/libs/pxic/reader.py | 30 ++++++++++++++++++++++++++---- pixie/vm/libs/pxic/tags.py | 1 + pixie/vm/libs/pxic/writer.py | 24 ++++++++++++++++++++++-- pixie/vm/object.py | 3 +++ pixie/vm/stdlib.py | 28 ++++++++++++++++++++++++++-- 5 files changed, 78 insertions(+), 8 deletions(-) diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index ba8a3f83..466c7680 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -25,9 +25,11 @@ def __init__(self, rdr): def read(self, num=r_uint(1)): return self._rdr.read(intmask(num)) - def read_cached(self): + def read_and_cache(self): + idx = len(self._obj_cache) + self._obj_cache[idx] = None # To match cache size growth of writer obj = read_obj(self) - self._obj_cache[len(self._obj_cache)] = obj + self._obj_cache[idx] = obj return obj def read_cached_obj(self): @@ -63,7 +65,15 @@ def read_code(rdr): nm = read_raw_string(rdr) - return Code(nm, bytecode, consts, stack_size, {}) + arity = read_raw_integer(rdr) + + dps = read_raw_integer(rdr) + debug_points = {} + for x in range(dps): + idx = read_raw_integer(rdr) + debug_points[idx] = read_obj(rdr) + + return Code(nm, arity, bytecode, consts, stack_size, debug_points) def read_var(rdr): @@ -106,6 +116,15 @@ def read_namespace(rdr): nm = read_raw_string(rdr) return code._ns_registry.find_or_make(nm) +def read_interpreter_code_info(rdr): + from pixie.vm.object import InterpreterCodeInfo + line = read_obj(rdr) + line_number = read_raw_integer(rdr) + column_number = read_raw_integer(rdr) + file = read_raw_string(rdr) + + return InterpreterCodeInfo(line, intmask(line_number), intmask(column_number), file) + def read_obj(rdr): tag = read_tag(rdr) @@ -148,7 +167,7 @@ def read_obj(rdr): elif tag == NEW_CACHED_OBJ: - return rdr.read_cached() + return rdr.read_and_cache() elif tag == CACHED_OBJ: return rdr.read_cached_obj() @@ -156,6 +175,9 @@ def read_obj(rdr): from pixie.vm.reader import eof return eof + elif tag == CODE_INFO: + return read_interpreter_code_info(rdr) + elif tag == TAGGED: tp_name = read_raw_string(rdr) tp = get_type_by_name(tp_name) diff --git a/pixie/vm/libs/pxic/tags.py b/pixie/vm/libs/pxic/tags.py index ac8b6334..713584de 100644 --- a/pixie/vm/libs/pxic/tags.py +++ b/pixie/vm/libs/pxic/tags.py @@ -18,6 +18,7 @@ "LINE_PROMISE", "NAMESPACE", "TAGGED", + "CODE_INFO", "EOF"] tags = {} diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index f87b6868..7c2992b3 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -1,5 +1,5 @@ from pixie.vm.libs.pxic.tags import * -from pixie.vm.object import runtime_error, Object, Type +from pixie.vm.object import runtime_error, Object, Type, InterpreterCodeInfo from rpython.rlib.runicode import unicode_encode_utf_8 from pixie.vm.string import String from pixie.vm.keyword import Keyword @@ -37,7 +37,7 @@ def write_cached_obj(self, o, wfn): wfn(o, self) else: write_tag(CACHED_OBJ, self) - write_int_raw(idx, self) + write_int_raw(r_uint(idx), self) else: return wfn(o, self) @@ -115,6 +115,14 @@ def write_code(c, wtr): write_string_raw(c._name, wtr) + write_int_raw(c._arity, wtr) + + write_int_raw(len(c._debug_points), wtr) + for k, v in c._debug_points.iteritems(): + write_int_raw(k, wtr) + write_object(v, wtr) + + class WriteParirFn(NativeFn): def __init__(self, wtr): @@ -204,6 +212,16 @@ def write_namespace(o, wtr): write_tag(NAMESPACE, wtr) write_string_raw(o._name, wtr) +def write_interpreter_code_info(obj, wtr): + line, line_number, column_number, file = obj.interpreter_code_info_state() + + write_tag(CODE_INFO, wtr) + + write_object(line, wtr) + write_int_raw(r_uint(line_number), wtr) + write_int_raw(r_uint(column_number), wtr) + write_string_raw(file, wtr) + def write_object(obj, wtr): wtr.flush() @@ -238,6 +256,8 @@ def write_object(obj, wtr): write_symbol(obj, wtr) elif isinstance(obj, Namespace): wtr.write_cached_obj(obj, write_namespace) + elif isinstance(obj, InterpreterCodeInfo): + wtr.write_cached_obj(obj, write_interpreter_code_info) else: from pixie.vm.libs.pxic.util import write_handlers handler = write_handlers.get(obj.type(), None) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index c19cfab0..e6c39441 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -173,6 +173,9 @@ def __repr__(self): + self._line.__repr__() + u"\n" \ + self.pad_chars() + u"^" + def interpreter_code_info_state(self): + return self._line, self._line_number, self._column_number, self._file + class NativeCodeInfo(ErrorInfo): def __init__(self, name): self._name = name diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index c9eda2c2..282cdd1e 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -433,7 +433,7 @@ def _load_file(filename, compile=False): if compile: pxic_f = open(filename + "c", "wb") - wtr = pxic_writer.Writer(pxic_f) + wtr = pxic_writer.Writer(pxic_f, True) with code.bindings(PXIC_WRITER, pxic_writer.WriterBox(wtr)): rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) wtr.finish() @@ -456,11 +456,12 @@ def load_pxic_file(filename): print "Loading precompiled file while interpreted, this may take time" with compiler.with_ns(u"user"): compiler.NS_VAR.deref().include_stdlib() + rdr = Reader(f) while True: if not we_are_translated(): sys.stdout.write(".") sys.stdout.flush() - o = read_obj(Reader(f)) + o = read_obj(rdr) if o is eof: break o.invoke([]) @@ -724,6 +725,29 @@ def _ici(meta): col_number.int_val() if col_number is not nil else 0, rt.name(file) if file is not nil else u" Date: Wed, 14 Jan 2015 08:55:48 -0600 Subject: [PATCH 478/909] cache raw strings --- Makefile | 6 ++++++ pixie/vm/libs/pxic/reader.py | 15 +++++++++++---- pixie/vm/libs/pxic/tags.py | 2 ++ pixie/vm/libs/pxic/writer.py | 29 +++++++++++++++++++++++------ 4 files changed, 42 insertions(+), 10 deletions(-) diff --git a/Makefile b/Makefile index fd995992..e658fbdd 100644 --- a/Makefile +++ b/Makefile @@ -56,3 +56,9 @@ run_built_tests: pixie-vm run_interpreted_tests: target.py PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.pxi + +compile_tests: + find "tests" -name "*.pxi" | xargs -L1 ./pixie-vm -l "tests" -c + +clean_pxic: + find * -name "*.pxic" | xargs rm diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index 466c7680..b4ec9cbb 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -21,6 +21,7 @@ class Reader(object): def __init__(self, rdr): self._rdr = rdr self._obj_cache = {} + self._str_cache = {} def read(self, num=r_uint(1)): return self._rdr.read(intmask(num)) @@ -32,6 +33,15 @@ def read_and_cache(self): self._obj_cache[idx] = obj return obj + def read_cached_string(self): + sz = read_raw_integer(self) + if sz >= MAX_STRING_SIZE: + return self._str_cache[sz - MAX_STRING_SIZE] + else: + s, pos = str_decode_utf_8(self.read(sz), sz, "?") + self._str_cache[len(self._str_cache)] = s + return s + def read_cached_obj(self): idx = read_raw_integer(self) return self._obj_cache[idx] @@ -45,10 +55,7 @@ def read_raw_integer(rdr): return r_uint(ord(rdr.read()[0]) | (ord(rdr.read()[0]) << 8) | (ord(rdr.read()[0]) << 16) | (ord(rdr.read()[0]) << 24)) def read_raw_string(rdr): - sz = read_raw_integer(rdr) - errors = "?" - s, pos = str_decode_utf_8(rdr.read(sz), sz, errors) - return s + return rdr.read_cached_string() def read_code(rdr): sz = read_raw_integer(rdr) diff --git a/pixie/vm/libs/pxic/tags.py b/pixie/vm/libs/pxic/tags.py index 713584de..abe92ac2 100644 --- a/pixie/vm/libs/pxic/tags.py +++ b/pixie/vm/libs/pxic/tags.py @@ -28,6 +28,8 @@ SMALL_INT_END = 255 SMALL_INT_MAX = SMALL_INT_END - SMALL_INT_END +MAX_STRING_SIZE = 1 << 30 + for idx, nm in enumerate(tag_name): globals()[nm] = idx tags[nm] = idx diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index 7c2992b3..eb251f9f 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -18,6 +18,7 @@ class Writer(object): def __init__(self, wtr, with_cache=False): self._wtr = wtr self._obj_cache = {} + self._string_cache = {} self._with_cache = with_cache def write(self, s): @@ -41,6 +42,27 @@ def write_cached_obj(self, o, wfn): else: return wfn(o, self) + def write_raw_cached_string(self, si): + assert isinstance(si, unicode) + if self._with_cache: + idx = self._string_cache.get(si, -1) + if idx == -1: + idx = len(self._string_cache) + self._string_cache[si] = idx + s = unicode_encode_utf_8(si, len(si), "?") + write_int_raw(len(s), self) + assert len(s) <= MAX_STRING_SIZE + self.write(s) + else: + write_int_raw(r_uint(MAX_STRING_SIZE + idx), self) + else: + errors = "?" + s = unicode_encode_utf_8(si, len(si), errors) + assert len(s) <= MAX_INT32 + write_int_raw(len(s), self) + self.write(s) + + def write_object(self, o): write_object(o, self) @@ -75,12 +97,7 @@ def write_int_raw(i, wtr): runtime_error(u"Raw int must be less than MAX_INT32, got: " + unicode(str(i))) def write_string_raw(si, wtr): - assert isinstance(si, unicode) - errors = "?" - s = unicode_encode_utf_8(si, len(si), errors) - assert len(s) <= MAX_INT32 - write_int_raw(len(s), wtr) - wtr.write(s) + wtr.write_raw_cached_string(si) def write_int(i, wtr): if 0 <= i <= MAX_INT32: From b2cde735d258874e3194852bfc19fc38bfeeb4b6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 16 Jan 2015 09:21:33 -0500 Subject: [PATCH 479/909] fixed issue #159 --- pixie/vm/rt.py | 3 +- target.py | 129 +++++++++++++++++++++++++------------------------ 2 files changed, 68 insertions(+), 64 deletions(-) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 5ee322e3..7eb65b33 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -142,7 +142,8 @@ def reinit(): # # stacklet.with_stacklets(run_load_stdlib) - init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file", u"compile-file"] + init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file", u"compile-file", + u"load-ns"] for x in init_fns: globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) diff --git a/target.py b/target.py index 767ae7c8..c9f46461 100644 --- a/target.py +++ b/target.py @@ -176,7 +176,7 @@ def run_load_stdlib(): if stdlib_loaded.is_true(): return - rt.load_file(rt.wrap(u"pixie/stdlib.pxi")) + rt.load_ns(rt.wrap(u"pixie/stdlib.pxi")) stdlib_loaded.set_true() @@ -184,73 +184,76 @@ def load_stdlib(): run_load_stdlib.invoke([]) def entry_point(args): - interactive = True - exit = False - script_args = [] - - init_load_path(args[0]) - load_stdlib() - add_to_load_paths(".") - - i = 1 - while i < len(args): - arg = args[i] - - if arg.startswith('-') and arg != '-': - if arg == '-v' or arg == '--version': - print "Pixie 0.1" - return 0 - elif arg == '-h' or arg == '--help': - print args[0] + " [] []" - print " -h, --help print this help" - print " -v, --version print the version number" - print " -e, --eval= evaluate the given expression" - print " -l, --load-path= add to pixie.stdlib/load-paths" - print " -c, --compile= compile to a .pxic file" - return 0 - elif arg == '-e' or arg == '--eval': - i += 1 - if i < len(args): - expr = args[i] - EvalFn(expr).invoke([]) + try: + interactive = True + exit = False + script_args = [] + + init_load_path(args[0]) + load_stdlib() + add_to_load_paths(".") + + i = 1 + while i < len(args): + arg = args[i] + + if arg.startswith('-') and arg != '-': + if arg == '-v' or arg == '--version': + print "Pixie 0.1" return 0 + elif arg == '-h' or arg == '--help': + print args[0] + " [] []" + print " -h, --help print this help" + print " -v, --version print the version number" + print " -e, --eval= evaluate the given expression" + print " -l, --load-path= add to pixie.stdlib/load-paths" + print " -c, --compile= compile to a .pxic file" + return 0 + elif arg == '-e' or arg == '--eval': + i += 1 + if i < len(args): + expr = args[i] + EvalFn(expr).invoke([]) + return 0 + else: + print "Expected argument for " + arg + return 1 + elif arg == '-l' or arg == '--load-path': + i += 1 + if i < len(args): + path = args[i] + add_to_load_paths(path) + else: + print "Expected argument for " + arg + return 1 + + elif arg == "-c" or arg == "--compile": + i += 1 + if i < len(args): + path = args[i] + print "Compiling ", path + rt.compile_file(rt.wrap(path)) + exit = True + else: + print "Expected argument for " + arg + return 1 else: - print "Expected argument for " + arg - return 1 - elif arg == '-l' or arg == '--load-path': - i += 1 - if i < len(args): - path = args[i] - add_to_load_paths(path) - else: - print "Expected argument for " + arg + print "Unknown option " + arg return 1 + else: + interactive = False + script_args = args[i:] + break - elif arg == "-c" or arg == "--compile": - i += 1 - if i < len(args): - path = args[i] - print "Compiling ", path - rt.compile_file(rt.wrap(path)) - exit = True - else: - print "Expected argument for " + arg - return 1 + i += 1 + + if not exit: + if interactive: + ReplFn(args).invoke([]) else: - print "Unknown option " + arg - return 1 - else: - interactive = False - script_args = args[i:] - break - - i += 1 - - if not exit: - if interactive: - ReplFn(args).invoke([]) - else: - BatchModeFn(script_args).invoke([]) + BatchModeFn(script_args).invoke([]) + except WrappedException as we: + print we._ex.__repr__() return 0 From c9d132b64428929e405e434ea3b76a78bb4be9b7 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 16 Jan 2015 10:47:37 -0500 Subject: [PATCH 480/909] update Makefile a bit --- Makefile | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/Makefile b/Makefile index e658fbdd..6161e96a 100644 --- a/Makefile +++ b/Makefile @@ -15,10 +15,7 @@ help: @echo "make run_interactive - run without compiling (slow)" @echo "make build_with_jit - build with jit enabled" @echo "make build_no_jit - build without jit" - @echo "make build_preload_with_jit - build with jit enabled and preload the stdlib" - @echo " (this means that pixie-vm will run as a standalone binary," - @echo " without having to load 'stdlib.pxi' and friends.)" - @echo "make build_preload_no_jit - build without jit and preload the stdlib" + @echo "make fetch_externals - download and unpack external deps" build_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py From 0c602e3750c56943eda3b44c43a0525e31424aa9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 19 Jan 2015 21:56:50 -0700 Subject: [PATCH 481/909] callbacks and the start of libuv stuff --- pixie/ffi-infer.pxi | 12 +++- pixie/stdlib.pxi | 15 ++++ pixie/vm/libs/ffi.py | 165 ++++++++++++++++++++++++++++++++++++++++--- pixie/vm/uv.pxi | 11 +++ 4 files changed, 193 insertions(+), 10 deletions(-) create mode 100644 pixie/vm/uv.pxi diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 463a60ef..5b1d06db 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -107,7 +107,12 @@ return 0; `[~(keyword name) ~(edn-to-ctype type) ~offset]) members infered-members)]))) - +(defmethod generate-code :type + [{:keys [name members]} {:keys [size infered-members]}] + `(def ~(symbol name) + (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] + `[~(keyword name) ~(edn-to-ctype type) ~offset]) + members infered-members)]))) (defn run-infer [config cmds] (io/spit "/tmp/tmp.cpp" (str (start-string) @@ -144,6 +149,11 @@ return 0; :name ~(name nm) :members ~(vec (map name members))) )) +(defmacro defctype [nm members] + `(swap! *bodies* conj (assoc {:op :type} + :name ~(name nm) + :members ~(vec (map name members))) )) + diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 2c4c48e1..680fed23 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1,9 +1,24 @@ (in-ns :pixie.stdlib) (def libc (ffi-library pixie.platform/lib-c-name)) + + (def exit (ffi-fn libc "exit" [CInt] CInt)) (def puts (ffi-fn libc "puts" [CCharP] CInt)) + + (def qsort-cb (ffi-callback [CVoidP CVoidP] CInt)) + (def qsort (ffi-fn libc "qsort" [CVoidP CInt CInt qsort-cb] CInt)) + + (def buf (buffer 3)) + (pixie.ffi/pack! buf 0 CUInt8 2) + (pixie.ffi/pack! buf 1 CUInt8 1) + (pixie.ffi/pack! buf 2 CUInt8 0) + (qsort buf 3 1 (fn* [x y] (puts (str (pixie.ffi/unpack x 0 CUInt8) (pixie.ffi/unpack y 0 CUInt8))) + 1)) + + (puts (str "DONE " (pixie.ffi/unpack buf 0 CUInt8))) + (def sh (ffi-fn libc "system" [CCharP] CInt)) (def printf (ffi-fn libc "printf" [CCharP] CInt :variadic? true)) (def getenv (ffi-fn libc "getenv" [CCharP] CCharP)) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 81830010..1ae4eb80 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -103,7 +103,7 @@ def __init__(self, lib, name, arg_types, ret_type, is_variadic): def thaw(self): #if not self._is_inited: self._f_ptr = self._lib.get_fn_ptr(self._name) - CifDescrBuilder(self._arg_types, self._ret_type).rawallocate(self) + self._cd = CifDescrBuilder(self._arg_types, self._ret_type).rawallocate() def _cleanup_(self): self._rev += 1 @@ -114,7 +114,7 @@ def _cleanup_(self): @jit.unroll_safe def prep_exb(self, args): size = jit.promote(self._cd.exchange_size) - exb = lltype.malloc(rffi.CCHARP.TO, size, flavor="raw") + exb = rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, size, flavor="raw")) tokens = [None] * len(args) for i, tp in enumerate(self._arg_types): @@ -231,6 +231,9 @@ def set_used_size(self, size): def buffer(self): return self._buffer + def raw_data(self): + return rffi.cast(rffi.VOIDP, self._buffer) + def count(self): return self._used_size @@ -419,7 +422,22 @@ def __init__(self, raw_data): self._raw_data = raw_data def raw_data(self): - return self._raw_data + return rffi.cast(rffi.VOIDP, self._raw_data) + +@as_var(u"pixie.ffi", u"unpack") +def unpack(ptr, offset, tp): + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + affirm(isinstance(tp, CType), u"Packing type must be a CType") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + return tp.ffi_get_value(ptr) + +@as_var(u"pixie.ffi", u"pack!") +def pack(ptr, offset, tp, val): + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + affirm(isinstance(tp, CType), u"Packing type must be a CType") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + tp.ffi_set_value(ptr, val) + return nil class CStructType(object.Type): @@ -457,6 +475,130 @@ def invoke(self, args): return CStruct(self, rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, self._size, flavor="raw"))) +registered_callbacks = {} + +class CCallback(object.Object): + _type = object.Type(u"pixie.ffi.CCallback") + + def __init__(self, cft, raw_closure, id, fn): + self._fn = fn + self._cft = cft + self._raw_closure = raw_closure + self._is_invoked = False + self._unique_id = id + self._auto_delete = False + + def ll_invoke(self, llargs, llres): + cft = self._cft + assert isinstance(cft, CFunctionType) + + args = [None] * len(cft._arg_types) + for i, tp in enumerate(cft._arg_types): + args[i] = tp.ffi_get_value(llargs[i]) + + self._is_invoked = True + retval = self._fn.invoke(args) + cft._ret_type.ffi_set_value(llres, retval) + if self._auto_delete: + self.cleanup() + + def cleanup(self): + del registered_callbacks[self._unique_id] + clibffi.closureHeap.free(self._raw_closure) + + def set_auto_delete(self): + self._auto_delete = True + + +@as_var(u"ffi-callback") +def ffi_callback(args, ret_type): + args_w = [None] * rt.count(args) + + for x in range(rt.count(args)): + arg = rt.nth(args, rt.wrap(x)) + if not isinstance(arg, object.Type): + runtime_error(u"Expected type, got " + rt.name(rt.str(arg))) + args_w[x] = arg + + if not isinstance(ret_type, object.Type): + runtime_error(u"Expected type, got " + rt.name(rt.str(ret_type))) + + return CFunctionType(args_w, ret_type) + +# Callback Code + +@jit.jit_callback("CFFI") +def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): + "" + cb = registered_callbacks[rffi.cast(rffi.INT_real, ll_userdata)] + print "HEY GOT IT", cb.ll_invoke(ll_args, ll_res) + print "..." + +class FunctionTypeNameGenerator(py_object): + def __init__(self): + self._idx = 0 + def next(self): + self._idx += 1 + return unicode("CFunctionType" + str(self._idx)) + +name_gen = FunctionTypeNameGenerator() + +class CFunctionType(object.Type): + base_type = object.Type(u"pixie.ffi.CType") + _immutable_fields_ = ["arg_types", "ret-type"] + + def __init__(self, arg_types, ret_type): + object.Type.__init__(self, name_gen.next(), CStructType.base_type) + self._arg_types = arg_types + self._ret_type = ret_type + self._cd = CifDescrBuilder(self._arg_types, self._ret_type).rawallocate() + + def ffi_get_value(self, ptr): + runtime_error(u"Cannot get a callback value via FFI") + + def ffi_set_value(self, ptr, val): + print "Setting callback" + raw_closure = rffi.cast(rffi.VOIDP, clibffi.closureHeap.alloc()) + + unique_id = len(registered_callbacks) + + cb = CCallback(self, raw_closure, unique_id, val) + registered_callbacks[unique_id] = cb + + res = clibffi.c_ffi_prep_closure(rffi.cast(clibffi.FFI_CLOSUREP, raw_closure), self._cd.cif, + invoke_callback, + rffi.cast(rffi.VOIDP, unique_id)) + + casted = rffi.cast(rffi.VOIDPP, ptr) + casted[0] = raw_closure + + if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: + registered_callbacks[unique_id] = None + runtime_error(u"libffi failed to build this callback") + + return CCallbackToken(cb) + + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer + + +class CCallbackToken(Token): + def __init__(self, cb): + self._cb = cb + + def finalize_token(self): + cb = self._cb + assert isinstance(cb, CCallback) + if cb._is_invoked: + cb.cleanup() + else: + cb.set_auto_delete() + + class CStruct(object.Object): _immutable_fields_ = ["_type", "_buffer"] @@ -468,7 +610,7 @@ def type(self): return self._type def raw_data(self): - return self._buffer + return rffi.cast(rffi.VOIDP, self._buffer) def val_at(self, k, not_found): (tp, offset) = self._type.get_desc(k) @@ -477,7 +619,7 @@ def val_at(self, k, not_found): return not_found offset = rffi.ptradd(self._buffer, offset) - return tp.ffi_get_value(rffi.cast(rffi.CCHARP, offset)) + return tp.ffi_get_value(rffi.cast(rffi.VOIDP, offset)) def set_val(self, k, v): (tp, offset) = self._type.get_desc(k) @@ -486,7 +628,7 @@ def set_val(self, k, v): runtime_error(u"Invalid field name: " + rt.name(rt.str(tp))) offset = rffi.ptradd(self._buffer, offset) - tp.ffi_set_value(rffi.cast(rffi.CCHARP, offset), v) + tp.ffi_set_value(rffi.cast(rffi.VOIDP, offset), v) return nil @@ -535,6 +677,9 @@ def val_at(self, k, not_found): def set_(self, k, val): return self.set_val(k, val) + + + import sys BIG_ENDIAN = sys.byteorder == 'big' @@ -599,7 +744,7 @@ def fb_build_exchange(self, cif_descr): nargs = len(self.fargs) # first, enough room for an array of 'nargs' pointers - exchange_offset = rffi.sizeof(rffi.CCHARP) * nargs + exchange_offset = rffi.sizeof(rffi.VOIDP) * nargs exchange_offset = self.align_arg(exchange_offset) cif_descr.exchange_result = exchange_offset cif_descr.exchange_result_libffi = exchange_offset @@ -635,7 +780,7 @@ def fb_extra_fields(self, cif_descr): cif_descr.atypes = self.atypes @jit.dont_look_inside - def rawallocate(self, ctypefunc): + def rawallocate(self): # compute the total size needed in the CIF_DESCRIPTION buffer self.nb_bytes = 0 @@ -653,7 +798,7 @@ def rawallocate(self, ctypefunc): flavor='raw') # the buffer is automatically managed from the W_CTypeFunc instance - ctypefunc._cd = rawmem + # ctypefunc._cd = rawmem # call again fb_build() to really build the libffi data structures self.bufferp = rffi.cast(rffi.CCHARP, rawmem) @@ -672,6 +817,8 @@ def rawallocate(self, ctypefunc): if res != clibffi.FFI_OK: runtime_error(u"libffi failed to build function type") + return rawmem + def get_item(self, nm): (tp, offset) = self._type.get_desc(nm) ptr = rffi.ptradd(self._buffer, offset) diff --git a/pixie/vm/uv.pxi b/pixie/vm/uv.pxi new file mode 100644 index 00000000..f1e1c12e --- /dev/null +++ b/pixie/vm/uv.pxi @@ -0,0 +1,11 @@ +(ns pixie.uv + (require pixie.ffi-infer :as f)) + +(f/with-config {:library "uv" + :includes ["uv.h"]} + (f/defconst UV_RUN_DEFAULT) + (f/defconst UV_RUN_ONCE) + (f/defconst UV_RUN_NOWAIT) + + (f/defcstruct uv_loop_t []) + ) \ No newline at end of file From 228b020a3c0fbbde0ed3f83c9d0f34824d64a2ba Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 19 Jan 2015 22:06:32 -0700 Subject: [PATCH 482/909] fix translation error --- pixie/vm/threads.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index 3abab66f..2b9fe462 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -4,7 +4,7 @@ from pixie.vm.code import wrap_fn, as_var from rpython.rlib.objectmodel import invoke_around_extcall -from rpython.rlib.rposix import get_errno, set_errno +from rpython.rlib.rposix import get_saved_errno, set_saved_errno class Bootstrapper(object): @@ -72,11 +72,11 @@ def before_external_call(): before_external_call._dont_reach_me_in_del_ = True def after_external_call(): - e = get_errno() + e = get_saved_errno() rgil.gil_acquire() rthread.gc_thread_run() after_thread_switch() - set_errno(e) + set_saved_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True From 270706d98bcfed5e54cdfdd6f47f76499ebb508c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 20 Jan 2015 06:23:17 -0700 Subject: [PATCH 483/909] remove warning filtering --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 6161e96a..40914050 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ build_preload_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target_preload.py build: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) 2>&1 >/dev/null | grep -v 'WARNING' + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) fetch_externals: $(EXTERNALS)/pypy From b6ce2f70a80550dfd04b80029e9888b4cfb17532 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 20 Jan 2015 08:15:16 -0700 Subject: [PATCH 484/909] some code cleanup --- pixie/vm/threads.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index 2b9fe462..09eb44f7 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -32,7 +32,7 @@ def release(self): self._lock.release() - def __cleanup__(self): + def _cleanup_(self): self._lock = None self._is_inited = False @@ -72,11 +72,9 @@ def before_external_call(): before_external_call._dont_reach_me_in_del_ = True def after_external_call(): - e = get_saved_errno() rgil.gil_acquire() rthread.gc_thread_run() after_thread_switch() - set_saved_errno(e) after_external_call._gctransformer_hint_cannot_collect_ = True after_external_call._dont_reach_me_in_del_ = True From b9dcf716b730308c746c569d603fa8bf9f96973a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 20 Jan 2015 17:40:30 -0700 Subject: [PATCH 485/909] use shadowstack on all platforms --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 40914050..76c78d6e 100644 --- a/Makefile +++ b/Makefile @@ -5,7 +5,7 @@ EXTERNALS=../externals PYTHON ?= pypy PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy -COMMON_BUILD_OPTS?=--thread --no-shared +COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack JIT_OPTS?=--opt=jit TARGET_OPTS?=target.py From 26a0a7ca947b459288f59bcd8f732a33977e1aae Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 22 Jan 2015 21:06:17 -0700 Subject: [PATCH 486/909] a bit of dog-fooding via implementing libuv support via ffi --- pixie/PixieChecker.hpp | 117 ++++++++++++++++++++++++++++------------- pixie/ffi-infer.pxi | 40 ++++++++++++-- pixie/vm/libs/ffi.py | 4 +- 3 files changed, 118 insertions(+), 43 deletions(-) diff --git a/pixie/PixieChecker.hpp b/pixie/PixieChecker.hpp index 27e2a48d..2a406a87 100644 --- a/pixie/PixieChecker.hpp +++ b/pixie/PixieChecker.hpp @@ -12,9 +12,10 @@ namespace PixieChecker { -template -struct GenericChecker; - + + +template +std::string GetType(); // Function Checker @@ -33,9 +34,9 @@ struct FunctionTyper<0, T> static std::string getType() { return "{:type :function :arity 0 :returns " + - GenericChecker::result_type>::getType() + + GetType::result_type>() + " :arguments []}"; - + } }; @@ -46,46 +47,94 @@ struct FunctionTyper<0, T> static std::string getType() { return "{:type :function :arity 1 :returns " + - GenericChecker::result_type>::getType() + + GetType::result_type>() + " :arguments [" + - GenericChecker::arg1_type>::getType() + " " + + GetType::arg1_type>() + " " + " ]}"; } }; - + template struct FunctionTyper<2, T> { static std::string getType() { return "{:type :function :arity 2 :returns " + - GenericChecker::result_type>::getType() + + GetType::result_type>() + " :arguments [" + - GenericChecker::arg1_type>::getType() + " " + - GenericChecker::arg2_type>::getType() + " " + - + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + " ]}"; } }; - - + + template struct FunctionTyper<3, T> { static std::string getType() { return "{:type :function :arity 3 :returns " + - GenericChecker::result_type>::getType() + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<4, T> + { + static std::string getType() + { + return "{:type :function :arity 4 :returns " + + GetType::result_type>() + " :arguments [" + - GenericChecker::arg1_type>::getType() + " " + - GenericChecker::arg2_type>::getType() + " " + - GenericChecker::arg3_type>::getType() + " " + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + " ]}"; } }; + // End Function Typer - + +// Is Enum + template + struct GenericCheckerIsEnum + { + static std::string getType() + { + return "Shouldn't happen in enum checker"; + } + }; + + template + struct GenericCheckerIsEnum + { + static std::string getType() + { + return GetType(); + } + }; + + template + struct GenericCheckerIsEnum + { + static std::string getType() + { + return "{:type :unknown}"; + } + }; +// End Is Enum + + + // Is Void template struct GenericCheckerIsVoid @@ -95,7 +144,7 @@ struct FunctionTyper<0, T> return "Shouldn't happen in void checker"; } }; - + template struct GenericCheckerIsVoid { @@ -104,13 +153,13 @@ struct FunctionTyper<0, T> return "{:type :void}"; } }; - + template struct GenericCheckerIsVoid { static std::string getType() { - return "{:type :unknown}"; + return GenericCheckerIsEnum::type, T>::getType(); } }; // End Is Void @@ -131,7 +180,7 @@ struct GenericCheckerIsPointer { static std::string getType() { - return "{:type :pointer :of-type " + GenericChecker::type>::getType() + "}"; + return "{:type :pointer :of-type " + GetType::type>() + "}"; } }; @@ -245,31 +294,25 @@ struct GenericCheckerIsInt // End is Int - // Generic Typer -template -struct GenericChecker +template +std::string GetType() { - static std::string getType() - { - return GenericCheckerIsInt::type, T>::getType(); - } -}; + return GenericCheckerIsInt::type, T>::getType(); +} // End Generic Typer - + template void DumpValue(T t) { - std::cout << GenericChecker::getType() << std::endl; + std::cout << GetType() << std::endl; } - + template void DumpType() { - std::cout << GenericChecker::getType() << std::endl; + std::cout << GetType() << std::endl; } } - - diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 5b1d06db..aadf026f 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -26,7 +26,7 @@ [{:keys [name]}] (str "PixieChecker::DumpType(); \n")) -(defmethod emit-infer-code :struct +(defmethod emit-infer-code :raw-struct [{:keys [name members]}] (str "std::cout << \"{:size \" << sizeof(struct " name ")" " << \" :infered-members [\" << " @@ -37,6 +37,17 @@ members)) "\"]}\" << std::endl;")) +(defmethod emit-infer-code :struct + [{:keys [name members]}] + (str "std::cout << \"{:size \" << sizeof(" name ")" + " << \" :infered-members [\" << " + (apply str + (map (fn [member] + (str "\"{:type \"; PixieChecker::DumpValue((new (" name "))->" member "); " + " std::cout << \":offset \" << offsetof(" name ", " member ") << \" }\" << \n ")) + members)) + "\"]}\" << std::endl;")) + (defn start-string [] (str (apply str (map (fn [i] @@ -59,10 +70,14 @@ return 0; (defmulti edn-to-ctype :type) +(defn callback-type [{:keys [arguments returns]}] + `(ffi-callback ~@(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns))) + (defmethod edn-to-ctype :pointer [{:keys [of-type] :as ptr}] (cond (= of-type {:signed? true :size 1 :type :int}) 'pixie.stdlib/CCharP + (= (:type of-type) :function) (callback-type of-type) :else 'pixie.stdlib/CVoidP)) (defmethod edn-to-ctype :float @@ -71,6 +86,9 @@ return 0; (= size 8) 'pixie.stdlib/CDouble :else (assert False "unknown type"))) +(defmethod edn-to-ctype :void + [_] + `pixie.stdlib/CVoid) (def int-types {[8 true] 'pixie.stdlib/CInt8 [8 false] 'pixie.stdlib/CUInt8 @@ -95,11 +113,20 @@ return 0; [{:keys [name]} {:keys [value type]}] `(def ~(symbol name) ~value)) + + (defmethod generate-code :function [{:keys [name]} {:keys [type arguments returns]}] (assert (= type :function) (str name " is not infered to be a function")) `(def ~(symbol name) (ffi-fn *library* ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) +(defmethod generate-code :raw-struct + [{:keys [name members]} {:keys [size infered-members]}] + `(def ~(symbol name) + (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] + `[~(keyword name) ~(edn-to-ctype type) ~offset]) + members infered-members)]))) + (defmethod generate-code :struct [{:keys [name members]} {:keys [size infered-members]}] `(def ~(symbol name) @@ -133,8 +160,10 @@ return 0; *library* (ffi-library (str "lib" (:library config) "." pixie.platform/so-ext))] (doseq [b body] (eval b)) - `(let [*library* (ffi-library ~(str "lib" (:library config) "." pixie.platform/so-ext))] - ~(run-infer *config* @*bodies*)))) + (let [result `(let [*library* (ffi-library ~(str "lib" (:library config) "." pixie.platform/so-ext))] + ~(run-infer *config* @*bodies*))] + (println result) + result))) (defmacro defcfn [nm] (let [name-str (name nm)] @@ -149,6 +178,11 @@ return 0; :name ~(name nm) :members ~(vec (map name members))) )) +(defmacro defc-raw-struct [nm members] + `(swap! *bodies* conj (assoc {:op :raw-struct} + :name ~(name nm) + :members ~(vec (map name members))) )) + (defmacro defctype [nm members] `(swap! *bodies* conj (assoc {:op :type} :name ~(name nm) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 1ae4eb80..27247628 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -529,10 +529,8 @@ def ffi_callback(args, ret_type): @jit.jit_callback("CFFI") def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): - "" cb = registered_callbacks[rffi.cast(rffi.INT_real, ll_userdata)] - print "HEY GOT IT", cb.ll_invoke(ll_args, ll_res) - print "..." + cb.ll_invoke(ll_args, ll_res) class FunctionTypeNameGenerator(py_object): def __init__(self): From 92790490dc6cc18f896bac2b0d8af33b65c68933 Mon Sep 17 00:00:00 2001 From: Akos Gyimesi Date: Sun, 25 Jan 2015 13:55:36 +0100 Subject: [PATCH 487/909] fix crash on syntax errors --- pixie/vm/reader.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 8816d66e..b2482241 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -284,7 +284,7 @@ def invoke(self, rdr, ch): try: v = rdr.read() except EOFError: - raise Exception("unmatched quote") + return throw_syntax_error_with_data(rdr, u"umatched quote") if v == "\"": return rt.wrap(u"".join(acc)) @@ -303,9 +303,9 @@ def invoke(self, rdr, ch): elif v == "t": acc.append("\t") else: - raise Exception("unhandled escape character") + throw_syntax_error_with_data(rdr, u"unhandled escape character: " + v) except EOFError: - raise Exception("eof after escape character") + throw_syntax_error_with_data(rdr, u"eof after escape character") else: acc.append(v) @@ -416,7 +416,7 @@ def syntax_quote(form): elif is_unquote(form): ret = rt.first(rt.next(form)) elif is_unquote_splicing(form): - raise Exception("Unquote splicing not used inside list") + return runtime_error(u"Unquote splicing not used inside list") elif rt.vector_QMARK_(form) is true: ret = rt.list(APPLY, CONCAT, SyntaxQuoteReader.expand_list(form)) elif form is not nil and rt.seq_QMARK_(form) is true: @@ -555,9 +555,9 @@ def invoke(self, rdr, ch): class DispatchReader(ReaderHandler): def invoke(self, rdr, ch): ch = rdr.read() - handler = dispatch_handlers[ch] + handler = dispatch_handlers.get(ch, None) if handler is None: - raise Exception("unknown dispatch #" + ch) + return throw_syntax_error_with_data(rdr, u"unknown dispatch #" + ch) return handler.invoke(rdr, ch) class LineCommentReader(ReaderHandler): From fd791d6e0023b5b18ed4f2ebe41cb4d6122671db Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 23 Jan 2015 13:34:07 +0000 Subject: [PATCH 488/909] Rename the-ns to in-ns for consistency --- pixie/vm/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 282cdd1e..fe2cb9c5 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -513,7 +513,7 @@ def the_ns(ns_name): return code._ns_registry.get(rt.name(ns_name), nil) @as_var("in-ns") -def the_ns(ns_name): +def in_ns(ns_name): from pixie.vm.compiler import NS_VAR NS_VAR.set_value(code._ns_registry.find_or_make(rt.name(ns_name))) NS_VAR.deref().include_stdlib() From 6ca4b8033ba60b4d1e52f0cc2cd9d185b26f08ae Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 30 Jan 2015 10:04:14 +0000 Subject: [PATCH 489/909] Add tests for pop and push on PersistentVector --- tests/pixie/tests/collections/test-vectors.pxi | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/tests/pixie/tests/collections/test-vectors.pxi b/tests/pixie/tests/collections/test-vectors.pxi index 9ef2914d..e7d6545f 100644 --- a/tests/pixie/tests/collections/test-vectors.pxi +++ b/tests/pixie/tests/collections/test-vectors.pxi @@ -44,6 +44,19 @@ (t/assert= [1 2] (persistent! (conj! (transient [1]) 2))) (t/assert= [1 2 3] (persistent! (conj! (transient [1]) 2 3)))) +(t/deftest vector-push + (t/assert= [1] (push [] 1)) + (t/assert= [1 2] (push [1] 2)) + (t/assert= [0 1 2 3 4] (reduce push [] (range 5)))) + +(t/deftest vector-pop + (t/assert= [] (pop [1])) + (t/assert= [1] (pop [1 2])) + (t/assert= [1 2] (pop [1 2 3])) + (try (pop []) + (catch e + (t/assert= "Can't pop an empty vector" (ex-msg e))))) + (t/deftest vector-push! (t/assert= [1] (persistent! (push! (transient []) 1))) (t/assert= [1 2] (persistent! (push! (transient [1]) 2)))) @@ -58,4 +71,4 @@ (t/assert= 1 (count (transient [1]))) (t/assert= 2 (count (transient [1 2]))) (t/assert= 1 (count (pop! (transient [1 2])))) - (t/assert= 100 (count (reduce conj! (transient []) (range 0 100))))) \ No newline at end of file + (t/assert= 100 (count (reduce conj! (transient []) (range 0 100))))) From a966c0c60b6d16668da078632b1638283c7a1858 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 30 Jan 2015 10:36:08 +0000 Subject: [PATCH 490/909] Add empty TransientVector pop test --- tests/pixie/tests/collections/test-vectors.pxi | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/tests/pixie/tests/collections/test-vectors.pxi b/tests/pixie/tests/collections/test-vectors.pxi index e7d6545f..e6e6f26a 100644 --- a/tests/pixie/tests/collections/test-vectors.pxi +++ b/tests/pixie/tests/collections/test-vectors.pxi @@ -64,7 +64,10 @@ (t/deftest vector-pop! (t/assert= [] (persistent! (pop! (transient [1])))) (t/assert= [1] (persistent! (pop! (transient [1 2])))) - (t/assert= [1 2] (persistent! (pop! (transient [1 2 3]))))) + (t/assert= [1 2] (persistent! (pop! (transient [1 2 3])))) + (try (persistent! (pop! (transient []))) + (catch e + (t/assert= "Can't pop an empty vector" (ex-msg e))))) (t/deftest transient-vector-count (t/assert= 0 (count (transient []))) From 3dacfc5b4b809ecbcdbdb3c27841015f7bb5d49f Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 30 Jan 2015 10:38:05 +0000 Subject: [PATCH 491/909] Add pop and push to stdlib --- pixie/stdlib.pxi | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 2c4c48e1..15c3c94e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -83,6 +83,20 @@ ([coll item & items] (reduce -disj! (-disj! coll item) items))))) +(def pop + (fn ^{:doc "Pops elements off a stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + pop + ([coll] (-pop coll)))) + +(def push + (fn ^{:doc "Push an element on to a stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + push + ([coll x] (-push coll x)))) + (def pop! (fn ^{:doc "Pops elements off a transient stack." :signatures [[] [coll] [coll item] [coll item & args]] From 21091d06159a085f469d0ef6a4b31387b1c22eb4 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 30 Jan 2015 10:38:20 +0000 Subject: [PATCH 492/909] Delete dead code Not needed because of assertion. Fix typo in assertion error --- pixie/vm/persistent_vector.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 0f070df9..c36a0a2b 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -350,11 +350,7 @@ def nth(self, i, not_found=nil): def pop(self): self.ensure_editable() - affirm(self._cnt != 0, u"Can't pop and empty vector") - - if self._cnt == 0: - self._cnt = 0 - return self + affirm(self._cnt != 0, u"Can't pop an empty vector") i = self._cnt - 1 From 9b052403591ee6553770d21888ee7af95fc0efd0 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 30 Jan 2015 15:48:06 +0000 Subject: [PATCH 493/909] Fixes #166 --- pixie/vm/stdlib.py | 18 +++++++++--------- tests/pixie/tests/test-stdlib.pxi | 11 ++++++++++- 2 files changed, 19 insertions(+), 10 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 282cdd1e..3ad6742a 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -513,7 +513,7 @@ def the_ns(ns_name): return code._ns_registry.get(rt.name(ns_name), nil) @as_var("in-ns") -def the_ns(ns_name): +def in_ns(ns_name): from pixie.vm.compiler import NS_VAR NS_VAR.set_value(code._ns_registry.find_or_make(rt.name(ns_name))) NS_VAR.deref().include_stdlib() @@ -523,22 +523,22 @@ def the_ns(ns_name): @as_var("ns-map") def ns_map(ns): from pixie.vm.code import Namespace - assert isinstance(ns, Namespace) from pixie.vm.symbol import Symbol - - affirm(isinstance(ns, code.Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") + affirm(isinstance(ns, Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") if isinstance(ns, Symbol): ns = rt.the_ns(ns) if ns is nil: return nil - m = rt.hashmap() - for name in ns._registry: - var = ns._registry.get(name, nil) - m = rt.assoc(m, rt.symbol(rt.wrap(name)), var) + if isinstance(ns, Namespace): + m = rt.hashmap() + for name in ns._registry: + var = ns._registry.get(name, nil) + m = rt.assoc(m, rt.symbol(rt.wrap(name)), var) + return m - return m + return nil @as_var("refer-ns") def refer(ns, refer, alias): diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 9c051b22..5ffe36cb 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -290,4 +290,13 @@ (reduce conj nil (range 10)) '(0 1 2 3 4 5 6 7 8 9))) true)) - \ No newline at end of file + +(t/deftest test-ns + ;; Create a namespace called foo + (in-ns :foo) + (def bar :humbug) + (defn baz [x y] (+ x y)) + ;; Back into the text namespace + (in-ns :pixie.tests.test-stdlib) + (t/assert= (set (keys (ns-map 'foo))) + #{'bar 'baz})) From 78af0471b5e22ce64e6cf4ece30752e6371bd61b Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 2 Feb 2015 16:02:48 +0100 Subject: [PATCH 494/909] remove unnecessary whitespace --- pixie/stdlib.pxi | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 2c4c48e1..dbcd165e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1,25 +1,24 @@ (in-ns :pixie.stdlib) - (def libc (ffi-library pixie.platform/lib-c-name)) - (def exit (ffi-fn libc "exit" [CInt] CInt)) - (def puts (ffi-fn libc "puts" [CCharP] CInt)) - - (def sh (ffi-fn libc "system" [CCharP] CInt)) - (def printf (ffi-fn libc "printf" [CCharP] CInt :variadic? true)) - (def getenv (ffi-fn libc "getenv" [CCharP] CCharP)) - - (def libedit (ffi-library (str "libedit." pixie.platform/so-ext))) - (def readline (ffi-fn libedit "readline" [CCharP] CCharP)) - (def rand (ffi-fn libc "rand" [] CInt)) - (def srand (ffi-fn libc "srand" [CInt] CInt)) - (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) - (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) - - - (def libm (ffi-library (str "libm." pixie.platform/so-ext))) - (def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) - (def floor (ffi-fn libm "floor" [CDouble] CDouble)) - (def lround (ffi-fn libm "lround" [CDouble] CInt)) +(def libc (ffi-library pixie.platform/lib-c-name)) +(def exit (ffi-fn libc "exit" [CInt] CInt)) +(def puts (ffi-fn libc "puts" [CCharP] CInt)) + +(def sh (ffi-fn libc "system" [CCharP] CInt)) +(def printf (ffi-fn libc "printf" [CCharP] CInt :variadic? true)) +(def getenv (ffi-fn libc "getenv" [CCharP] CCharP)) + +(def libedit (ffi-library (str "libedit." pixie.platform/so-ext))) +(def readline (ffi-fn libedit "readline" [CCharP] CCharP)) +(def rand (ffi-fn libc "rand" [] CInt)) +(def srand (ffi-fn libc "srand" [CInt] CInt)) +(def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) +(def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) + +(def libm (ffi-library (str "libm." pixie.platform/so-ext))) +(def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) +(def floor (ffi-fn libm "floor" [CDouble] CDouble)) +(def lround (ffi-fn libm "lround" [CDouble] CInt)) (def reset! -reset!) From 112b92535c86e30cc3302f3cdcdd28dc1ed6a399 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 3 Feb 2015 10:29:28 -0500 Subject: [PATCH 495/909] Added python or pypy to prereqs. Also changed default to python since the externals include pypy --- Makefile | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 76c78d6e..58ca8c6d 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=../externals -PYTHON ?= pypy +PYTHON ?= python PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack diff --git a/README.md b/README.md index 0a56143b..3c1f4a19 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ Some planned and implemented features: ## Dependencies +* python or pypy to build * [libffi-dev](https://sourceware.org/libffi/) * [libedit-dev](http://thrysoee.dk/editline/) From 8998eaf610457fbd802ca1522d8714ace9b0cef7 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 5 Feb 2015 20:23:37 +0000 Subject: [PATCH 496/909] Cleaned up a bunch of dead imports --- pixie/vm/array.py | 5 ++--- pixie/vm/compiler.py | 9 +++------ pixie/vm/cons.py | 4 ++-- pixie/vm/custom_types.py | 4 ---- pixie/vm/interpreter.py | 7 +++---- pixie/vm/iterator.py | 4 ++-- pixie/vm/keyword.py | 6 +++--- pixie/vm/lazy_seq.py | 4 ++-- pixie/vm/libs/ffi.py | 9 ++++----- pixie/vm/libs/libedit.py | 1 - pixie/vm/libs/path.py | 11 +++-------- pixie/vm/libs/platform.py | 1 - pixie/vm/libs/string.py | 8 ++------ pixie/vm/map_entry.py | 7 ------- pixie/vm/numbers.py | 2 +- pixie/vm/persistent_hash_map.py | 1 - pixie/vm/persistent_hash_set.py | 7 +------ pixie/vm/persistent_list.py | 3 +-- pixie/vm/reader.py | 5 ++--- pixie/vm/reduced.py | 3 +-- pixie/vm/stdlib.py | 5 ++--- pixie/vm/symbol.py | 3 +-- pixie/vm/test/test_compile.py | 11 +++-------- pixie/vm/test/test_hashmaps.py | 4 +--- pixie/vm/test/test_vars.py | 3 +-- pixie/vm/test/test_vectors.py | 5 ++--- pixie/vm/threads.py | 4 +--- pixie/vm/util.py | 4 +--- 28 files changed, 44 insertions(+), 96 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index b8db8401..e0162011 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -6,9 +6,8 @@ from pixie.vm.primitives import nil import pixie.vm.stdlib as proto import rpython.rlib.jit as jit -from rpython.rtyper.lltypesystem import lltype, rffi +from rpython.rtyper.lltypesystem import lltype from rpython.rlib.rarithmetic import intmask -from rpython.rlib.rarithmetic import build_int UNROLL_IF_SMALLER_THAN = 8 @@ -217,4 +216,4 @@ def _nth(self, idx): @extend(proto._count, ByteArray) def _count(self): assert isinstance(self, ByteArray) - return rt.wrap(intmask(self._cnt)) \ No newline at end of file + return rt.wrap(intmask(self._cnt)) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index da2ee174..c4fef5e6 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -1,20 +1,17 @@ -from pixie.vm.object import Object, _type_registry, affirm, InterpreterCodeInfo -from pixie.vm.primitives import nil, true, false, Bool +from pixie.vm.object import affirm +from pixie.vm.primitives import nil, true, Bool from pixie.vm.persistent_vector import EMPTY, PersistentVector from pixie.vm.persistent_hash_set import PersistentHashSet import pixie.vm.numbers as numbers -from pixie.vm.cons import cons, Cons import pixie.vm.symbol as symbol import pixie.vm.code as code from pixie.vm.keyword import Keyword, keyword from pixie.vm.string import Character, String from pixie.vm.atom import Atom -import pixie.vm.stdlib as proto -from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import r_uint, intmask from pixie.vm.persistent_list import EmptyList import pixie.vm.rt as rt -from pixie.vm.util import * NS_VAR = code.intern_var(u"pixie.stdlib", u"*ns*") NS_VAR.set_dynamic() diff --git a/pixie/vm/cons.py b/pixie/vm/cons.py index 3be07e04..e9f0eb92 100644 --- a/pixie/vm/cons.py +++ b/pixie/vm/cons.py @@ -1,5 +1,5 @@ import pixie.vm.object as object -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import nil import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var @@ -65,4 +65,4 @@ def count(self): return cnt def cons(head, tail=nil): - return Cons(head, tail, nil) \ No newline at end of file + return Cons(head, tail, nil) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index cf8f985d..7a1757b9 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -1,11 +1,7 @@ from pixie.vm.object import Object, Type, affirm -from pixie.vm.primitives import nil, true, false import rpython.rlib.jit as jit -from pixie.vm.numbers import Integer from rpython.rlib.rarithmetic import r_uint from pixie.vm.code import as_var -from pixie.vm.symbol import Symbol -from pixie.vm.string import String from pixie.vm.keyword import Keyword import pixie.vm.rt as rt diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 2bd09e5c..7cc5da9e 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -1,12 +1,11 @@ from pixie.vm.object import Object, affirm, WrappedException, Type, runtime_error import pixie.vm.code as code import pixie.vm.numbers as numbers -from pixie.vm.primitives import nil, true, false -from rpython.rlib.rarithmetic import r_uint, intmask -from rpython.rlib.jit import JitDriver, promote, elidable, elidable_promote, hint, unroll_safe +from pixie.vm.primitives import nil, false +from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.jit import JitDriver, promote, elidable, hint, unroll_safe import rpython.rlib.jit as jit import rpython.rlib.debug as debug -import pixie.vm.rt as rt def get_location(ip, sp, is_continuation, bc, base_code): return code.BYTECODES[bc[ip]] + " in " + str(base_code._name) diff --git a/pixie/vm/iterator.py b/pixie/vm/iterator.py index 2d0b3e8b..73327125 100644 --- a/pixie/vm/iterator.py +++ b/pixie/vm/iterator.py @@ -1,6 +1,6 @@ from pixie.vm.object import Object, Type, runtime_error -from pixie.vm.code import extend, as_var -from pixie.vm.primitives import true, false, nil +from pixie.vm.code import extend +from pixie.vm.primitives import true, nil import pixie.vm.stdlib as proto import pixie.vm.rt as rt diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index 866757ba..b1a8f90f 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -1,5 +1,5 @@ -from pixie.vm.object import Object, Type, affirm -from pixie.vm.primitives import nil, true, false +from pixie.vm.object import Object, Type +from pixie.vm.primitives import nil from pixie.vm.string import String import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var @@ -77,4 +77,4 @@ def _keyword(s): if not isinstance(s, String): from pixie.vm.object import runtime_error runtime_error(u"Symbol name must be a string") - return keyword(s._str) \ No newline at end of file + return keyword(s._str) diff --git a/pixie/vm/lazy_seq.py b/pixie/vm/lazy_seq.py index 0d6af515..7d052585 100644 --- a/pixie/vm/lazy_seq.py +++ b/pixie/vm/lazy_seq.py @@ -1,5 +1,5 @@ import pixie.vm.object as object -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import nil import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var import pixie.vm.rt as rt @@ -56,4 +56,4 @@ def _seq(self): @as_var("lazy-seq*") def lazy_seq(f): - return LazySeq(f) \ No newline at end of file + return LazySeq(f) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 81830010..b55c1f25 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -2,7 +2,6 @@ import rpython.rlib.rdynload as dynload import pixie.vm.object as object from pixie.vm.object import runtime_error -import pixie.vm.code as code from pixie.vm.keyword import Keyword import pixie.vm.stdlib as proto from pixie.vm.code import as_var, affirm, extend @@ -11,15 +10,15 @@ from pixie.vm.primitives import nil, true, false from pixie.vm.numbers import Integer, Float from pixie.vm.string import String -from pixie.vm.keyword import Keyword, keyword +from pixie.vm.keyword import Keyword from pixie.vm.util import unicode_to_utf8 from rpython.rlib import clibffi -from rpython.rlib.jit_libffi import jit_ffi_prep_cif, jit_ffi_call, CIF_DESCRIPTION, CIF_DESCRIPTION_P, \ - FFI_TYPE, FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG +from rpython.rlib.jit_libffi import jit_ffi_call, CIF_DESCRIPTION, CIF_DESCRIPTION_P, \ + FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG import rpython.rlib.jit_libffi as jit_libffi from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated import rpython.rlib.jit as jit -from rpython.rlib.rarithmetic import intmask, r_uint +from rpython.rlib.rarithmetic import intmask """ diff --git a/pixie/vm/libs/libedit.py b/pixie/vm/libs/libedit.py index c32580f2..a5a20203 100644 --- a/pixie/vm/libs/libedit.py +++ b/pixie/vm/libs/libedit.py @@ -3,7 +3,6 @@ from pixie.vm.util import unicode_from_utf8 from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.lltypesystem.lloperation import llop from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo diff --git a/pixie/vm/libs/path.py b/pixie/vm/libs/path.py index ff560301..854bedce 100644 --- a/pixie/vm/libs/path.py +++ b/pixie/vm/libs/path.py @@ -1,13 +1,8 @@ import pixie.vm.rt as rt -import pixie.vm.util as util -from pixie.vm.string import String -from pixie.vm.code import as_var, extend, defprotocol -from pixie.vm.object import Object, Type, affirm +from pixie.vm.code import as_var, extend +from pixie.vm.object import Object, Type import pixie.vm.stdlib as proto -from pixie.vm.keyword import keyword -from pixie.vm.primitives import true, false, nil -from rpython.rlib.clibffi import get_libc_name -from rpython.rlib.rarithmetic import intmask, r_uint +from pixie.vm.primitives import true, false import os class Path(Object): diff --git a/pixie/vm/libs/platform.py b/pixie/vm/libs/platform.py index 0b7843a8..2db4cf6c 100644 --- a/pixie/vm/libs/platform.py +++ b/pixie/vm/libs/platform.py @@ -1,5 +1,4 @@ from rpython.translator.platform import platform -import pixie.vm.rt as rt from pixie.vm.string import String from pixie.vm.code import as_var from rpython.rlib.clibffi import get_libc_name diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 4772118d..13977765 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -1,14 +1,10 @@ import pixie.vm.rt as rt from pixie.vm.string import String -from pixie.vm.code import as_var, extend, intern_var, wrap_fn, MultiArityFn -from pixie.vm.object import affirm, runtime_error, Object, Type -import pixie.vm.stdlib as proto -from pixie.vm.keyword import keyword +from pixie.vm.code import as_var, intern_var, wrap_fn, MultiArityFn +from pixie.vm.object import affirm, runtime_error from pixie.vm.numbers import Integer -from rpython.rlib.clibffi import get_libc_name from rpython.rlib.unicodedata import unicodedb_6_2_0 as unicodedb import rpython.rlib.rstring as rstring -import os import pixie.vm.rt as rt diff --git a/pixie/vm/map_entry.py b/pixie/vm/map_entry.py index 53fea618..543fb7d3 100644 --- a/pixie/vm/map_entry.py +++ b/pixie/vm/map_entry.py @@ -1,14 +1,7 @@ py_object = object import pixie.vm.object as object -from pixie.vm.object import affirm -from pixie.vm.primitives import nil, true, false -from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint, intmask -import rpython.rlib.jit as jit -import pixie.vm.rt as rt - class MapEntry(object.Object): _type = object.Type(u"pixie.stdlib.MapEntry") diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 863a854f..fe06eafd 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -1,6 +1,6 @@ import pixie.vm.object as object from pixie.vm.object import affirm -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import true, false from rpython.rlib.rarithmetic import r_uint from rpython.rlib.rbigint import rbigint import rpython.rlib.jit as jit diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index adcccb0b..b2dc1563 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -2,7 +2,6 @@ import pixie.vm.object as object from pixie.vm.object import affirm from pixie.vm.primitives import nil, true, false -from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var from rpython.rlib.rarithmetic import r_int, r_uint, intmask diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 1170b4bd..2cebe830 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -1,16 +1,11 @@ py_object = object import pixie.vm.object as object -from pixie.vm.object import affirm from pixie.vm.primitives import nil, true, false -from pixie.vm.numbers import Integer import pixie.vm.persistent_hash_map as persistent_hash_map import pixie.vm.stdlib as proto -from pixie.vm.code import extend, as_var, intern_var, wrap_fn -from rpython.rlib.rarithmetic import r_uint, intmask -import rpython.rlib.jit as jit +from pixie.vm.code import extend, as_var, intern_var import pixie.vm.rt as rt from pixie.vm.iterator import MapIterator -from pixie.vm.libs.pxic.util import add_marshall_handlers VAR_KEY = intern_var(u"pixie.stdlib", u"key") diff --git a/pixie/vm/persistent_list.py b/pixie/vm/persistent_list.py index 85ce81db..91f76da8 100644 --- a/pixie/vm/persistent_list.py +++ b/pixie/vm/persistent_list.py @@ -1,8 +1,7 @@ import pixie.vm.object as object -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import nil import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var -from pixie.vm.numbers import Integer from rpython.rlib.rarithmetic import r_uint, intmask import pixie.vm.rt as rt diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index b2482241..ab7ac74b 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -10,12 +10,11 @@ import pixie.vm.rt as rt from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR from pixie.vm.libs.libedit import _readline -from pixie.vm.string import Character, String -from pixie.vm.code import wrap_fn, extend +from pixie.vm.string import Character +from pixie.vm.code import wrap_fn from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP from pixie.vm.persistent_hash_set import EMPTY as EMPTY_SET from pixie.vm.persistent_list import EmptyList -import pixie.vm.stdlib as proto import pixie.vm.compiler as compiler from rpython.rlib.rbigint import rbigint diff --git a/pixie/vm/reduced.py b/pixie/vm/reduced.py index 023cb406..d1d703ea 100644 --- a/pixie/vm/reduced.py +++ b/pixie/vm/reduced.py @@ -1,7 +1,6 @@ -import pixie.vm.rt as rt import pixie.vm.object as object from pixie.vm.code import extend, as_var, returns -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import true, false import pixie.vm.stdlib as proto diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 3ad6742a..4ad2c0b6 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,10 +1,9 @@ # -*- coding: utf-8 -*- -from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ +from pixie.vm.object import Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ runtime_error from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ - resize_list, list_copy, returns, get_var_if_defined, intern_var + list_copy, returns, intern_var import pixie.vm.code as code -from types import MethodType from pixie.vm.primitives import true, false, nil import pixie.vm.numbers as numbers import rpython.rlib.jit as jit diff --git a/pixie/vm/symbol.py b/pixie/vm/symbol.py index 59d31ec4..1fb9bbc3 100644 --- a/pixie/vm/symbol.py +++ b/pixie/vm/symbol.py @@ -1,5 +1,4 @@ import pixie.vm.object as object -from pixie.vm.object import affirm from pixie.vm.primitives import nil, true, false import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var @@ -96,4 +95,4 @@ def _meta(self): @extend(proto._with_meta, Symbol) def _with_meta(self, meta): assert isinstance(self, Symbol) - return self.with_meta(meta) \ No newline at end of file + return self.with_meta(meta) diff --git a/pixie/vm/test/test_compile.py b/pixie/vm/test/test_compile.py index 410f846a..eb73801d 100644 --- a/pixie/vm/test/test_compile.py +++ b/pixie/vm/test/test_compile.py @@ -1,17 +1,12 @@ from pixie.vm.reader import read, StringReader, eof -from pixie.vm.object import Object, Type +from pixie.vm.object import Type from pixie.vm.cons import Cons from pixie.vm.numbers import Integer -from pixie.vm.symbol import symbol, Symbol -from pixie.vm.keyword import Keyword -from pixie.vm.compiler import compile_form, compile, with_ns +from pixie.vm.symbol import Symbol +from pixie.vm.compiler import compile, with_ns from pixie.vm.interpreter import interpret from pixie.vm.code import Code, Var from pixie.vm.primitives import nil, true, false -from pixie.vm.custom_types import CustomTypeInstance -import unittest - -import pixie.vm.libs.libedit def read_code(s): with with_ns(u"user"): diff --git a/pixie/vm/test/test_hashmaps.py b/pixie/vm/test/test_hashmaps.py index 14053462..26fb0306 100644 --- a/pixie/vm/test/test_hashmaps.py +++ b/pixie/vm/test/test_hashmaps.py @@ -1,6 +1,4 @@ -import unittest import pixie.vm.rt as rt -from pixie.vm.numbers import Integer from pixie.vm.primitives import nil rt.init() @@ -11,4 +9,4 @@ def test_hashmap_create(): val = rt._val_at(acc, rt.wrap(1), nil) - assert val.int_val() == 2 \ No newline at end of file + assert val.int_val() == 2 diff --git a/pixie/vm/test/test_vars.py b/pixie/vm/test/test_vars.py index 902cb0f2..2f52a631 100644 --- a/pixie/vm/test/test_vars.py +++ b/pixie/vm/test/test_vars.py @@ -1,4 +1,3 @@ -import unittest from pixie.vm.code import intern_var, get_var_if_defined @@ -7,4 +6,4 @@ def test_intern(): assert intern_var(u"foo2", u"bar") is not intern_var(u"foo", u"bar") assert get_var_if_defined(u"foo", u"bar") - assert get_var_if_defined(u"foo2", u"bar") \ No newline at end of file + assert get_var_if_defined(u"foo2", u"bar") diff --git a/pixie/vm/test/test_vectors.py b/pixie/vm/test/test_vectors.py index c9456a0a..b5dc4bff 100644 --- a/pixie/vm/test/test_vectors.py +++ b/pixie/vm/test/test_vectors.py @@ -1,5 +1,4 @@ -import unittest -from pixie.vm.persistent_vector import PersistentVector, EMPTY +from pixie.vm.persistent_vector import EMPTY def test_vector_conj(): @@ -8,4 +7,4 @@ def test_vector_conj(): for x in range(1200): acc = acc.conj(x) for y in range(x): - assert acc.nth(y) == y, "Error at: " + str(x) + " and " + str(y) \ No newline at end of file + assert acc.nth(y) == y, "Error at: " + str(x) + " and " + str(y) diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index 09eb44f7..6d8404b9 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -1,11 +1,9 @@ import rpython.rlib.rthread as rthread from pixie.vm.primitives import nil import rpython.rlib.rgil as rgil -from pixie.vm.code import wrap_fn, as_var +from pixie.vm.code import as_var from rpython.rlib.objectmodel import invoke_around_extcall -from rpython.rlib.rposix import get_saved_errno, set_saved_errno - class Bootstrapper(object): def __init__(self): diff --git a/pixie/vm/util.py b/pixie/vm/util.py index da4314a2..931ba731 100644 --- a/pixie/vm/util.py +++ b/pixie/vm/util.py @@ -1,4 +1,4 @@ -from rpython.rlib.rarithmetic import r_uint, LONG_BIT, intmask, LONG_MASK +from rpython.rlib.rarithmetic import r_uint, LONG_BIT, intmask from rpython.rlib.runicode import str_decode_utf_8, unicode_encode_utf_8 from pixie.vm.object import affirm @@ -72,10 +72,8 @@ def mix_coll_hash(hash, count): from pixie.vm.object import Object, Type -import pixie.vm.code as code from pixie.vm.code import as_var import pixie.vm.rt as rt -import pixie.vm.numbers as numbers class HashingState(Object): _type = Type(u"pixie.stdlib.HashingState") From 4c1923cfb2ea15e3d2571c6ad668605d3d11b003 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 6 Feb 2015 21:15:34 +0000 Subject: [PATCH 497/909] Fixes #151 Namespaced symbols were being resolved locally. --- pixie/vm/compiler.py | 30 +++++++++++++++++++----------- 1 file changed, 19 insertions(+), 11 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index c4fef5e6..dc36e169 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -364,22 +364,30 @@ def compile_form(form, ctx): if isinstance(form, symbol.Symbol): name = rt.name(form) + ns = rt.namespace(form) + loc = resolve_local(ctx, name) - if loc is None: - var = resolve_var(ctx, form) + var = resolve_var(ctx, form) - if var is None: - var = NS_VAR.deref().intern_or_make(name) + if var is None and loc: + loc.emit(ctx) + return - ctx.push_const(var) + if var and loc and ns is None: + loc.emit(ctx) + return + + if var is None: + name = rt.name(form) + var = NS_VAR.deref().intern_or_make(name) - meta = rt.meta(form) - if meta is not nil: - ctx.debug_points[len(ctx.bytecode)] = rt.interpreter_code_info(meta) + ctx.push_const(var) - ctx.bytecode.append(code.DEREF_VAR) - return - loc.emit(ctx) + meta = rt.meta(form) + if meta is not nil: + ctx.debug_points[len(ctx.bytecode)] = rt.interpreter_code_info(meta) + + ctx.bytecode.append(code.DEREF_VAR) return if isinstance(form, Bool) or form is nil: From 3d1018131fde673e3642c425bf92171f5a77d3f1 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 9 Feb 2015 17:22:32 +0100 Subject: [PATCH 498/909] make ffi-infer work on linux --- pixie/ffi-infer.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 463a60ef..e1de0912 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -24,7 +24,7 @@ (defmethod emit-infer-code :function [{:keys [name]}] - (str "PixieChecker::DumpType(); \n")) + (str "PixieChecker::DumpType<__typeof__(" name ")>(); \n")) (defmethod emit-infer-code :struct [{:keys [name members]}] @@ -114,7 +114,7 @@ return 0; (apply str (map emit-infer-code cmds)) (end-string))) - (let [cmd-str (str "c++ -arch x86_64 /tmp/tmp.cpp -I" + (let [cmd-str (str "c++ -m64 -std=c++11 /tmp/tmp.cpp -I" (first @load-paths) (apply str " " (interpose " " (:cxx-flags *config*))) " -o /tmp/a.out && /tmp/a.out") From 88504587a791b8f6e1191bc8e055fe906d469320 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 9 Feb 2015 17:26:44 +0100 Subject: [PATCH 499/909] print compiler cmdline with a newline --- pixie/ffi-infer.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index e1de0912..247e5173 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -118,7 +118,7 @@ return 0; (first @load-paths) (apply str " " (interpose " " (:cxx-flags *config*))) " -o /tmp/a.out && /tmp/a.out") - _ (print cmd-str) + _ (println cmd-str) result (read-string (io/run-command cmd-str))] `(do ~@(map generate-code cmds result)))) From c06d81bd5a8303da769bd1ef9c542f5debffb8ff Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 9 Feb 2015 17:36:20 +0100 Subject: [PATCH 500/909] fix ffi-infer when importing from libc this happens on some systems where `libc.so` is not the library, but a text file/link script. we already do the same in the stdlib. --- pixie/ffi-infer.pxi | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 247e5173..a4443c12 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -122,13 +122,18 @@ return 0; result (read-string (io/run-command cmd-str))] `(do ~@(map generate-code cmds result)))) +(defn full-lib-name [library-name] + (if (= library-name "c") + pixie.platform/lib-c-name + (str "lib" library-name "." pixie.platform/so-ext))) + (defmacro with-config [config & body] (binding [*config* config *bodies* (atom []) - *library* (ffi-library (str "lib" (:library config) "." pixie.platform/so-ext))] + *library* (ffi-library (full-lib-name (:library config)))] (doseq [b body] (eval b)) - `(let [*library* (ffi-library ~(str "lib" (:library config) "." pixie.platform/so-ext))] + `(let [*library* (ffi-library ~(full-lib-name (:library config)))] ~(run-infer *config* @*bodies*)))) (defmacro defcfn [nm] From 26267435d0a8a4ce8ffbe6b6abd1844c4db75304 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 10 Feb 2015 19:40:31 -0500 Subject: [PATCH 501/909] cinch-functional-programmers - Added penultimate --- pixie/stdlib.pxi | 7 +++++++ tests/pixie/tests/test-stdlib.pxi | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ec2ebff3..04d7cf61 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -838,6 +838,13 @@ If further arguments are passed, invokes the method named by symbol, passing the (recur (conj res (first coll)) (next coll)) (seq res)))) +(defn penultimate + {:doc "Returns the second to last element of the collection, or nil if none." + :signatures [[coll]] + :added "0.1"} + [coll] + (last (butlast coll) )) + (defn complement {:doc "Given a function, return a new function which takes the same arguments but returns the opposite truth value"} diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 5ffe36cb..fa7feb63 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -90,6 +90,18 @@ (t/assert= (butlast l) res) (t/assert= (butlast r) res))) +(t/deftest test-penultimate + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (t/assert= (penultimate nil) nil) + (t/assert= (penultimate []) nil) + (t/assert= (penultimate (range 0 0)) nil) + (t/assert= (penultimate v) 4) + (t/assert= (penultimate l) 4) + (t/assert= (penultimate r) 4) + (t/assert= (penultimate [2]) nil))) + (t/deftest test-empty? (t/assert= (empty? []) true) (t/assert= (empty? '()) true) From 1a4254e1ac9b20c53a9e816745b176c48c9c359a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 11 Feb 2015 17:14:08 -0700 Subject: [PATCH 502/909] make ffi-infer use platform flags from rpython --- pixie/PixieChecker.hpp | 13 +++++++++---- pixie/ffi-infer.pxi | 18 +++++++++++------- pixie/vm/libs/platform.py | 12 ++++++++++++ tests/pixie/tests/test-ffi.pxi | 6 +++++- 4 files changed, 37 insertions(+), 12 deletions(-) diff --git a/pixie/PixieChecker.hpp b/pixie/PixieChecker.hpp index 27e2a48d..64403dbf 100644 --- a/pixie/PixieChecker.hpp +++ b/pixie/PixieChecker.hpp @@ -15,7 +15,13 @@ namespace PixieChecker { template struct GenericChecker; - +template < typename T > std::string to_string( const T& n ) +{ + std::ostringstream stm ; + stm << n ; + return stm.str() ; +} + // Function Checker template @@ -197,7 +203,7 @@ struct GenericCheckerIsFloat { static std::string getType() { - return "{:type :float :size " + std::to_string(sizeof(T)) + "}"; + return "{:type :float :size " + to_string(sizeof(T)) + "}"; } }; @@ -228,7 +234,7 @@ struct GenericCheckerIsInt { static std::string getType() { - return "{:type :int :size " + std::to_string(sizeof(T)) + + return "{:type :int :size " + to_string(sizeof(T)) + " :signed? " + (boost::is_signed::value ? "true" : "false") + "}"; } @@ -272,4 +278,3 @@ void DumpType() } - diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index a4443c12..453fe22c 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -98,7 +98,8 @@ return 0; (defmethod generate-code :function [{:keys [name]} {:keys [type arguments returns]}] (assert (= type :function) (str name " is not infered to be a function")) - `(def ~(symbol name) (ffi-fn *library* ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) + `(def ~(symbol name) + (ffi-fn *library* ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) (defmethod generate-code :struct [{:keys [name members]} {:keys [size infered-members]}] @@ -114,10 +115,12 @@ return 0; (apply str (map emit-infer-code cmds)) (end-string))) - (let [cmd-str (str "c++ -m64 -std=c++11 /tmp/tmp.cpp -I" - (first @load-paths) - (apply str " " (interpose " " (:cxx-flags *config*))) - " -o /tmp/a.out && /tmp/a.out") + (let [cmd-str (str "c++ " + (apply str (interpose " " pixie.platform/c-flags)) + " -std=c++11 /tmp/tmp.cpp -I" + (first @load-paths) + (apply str " " (interpose " " (:cxx-flags *config*))) + " -o /tmp/a.out && /tmp/a.out") _ (println cmd-str) result (read-string (io/run-command cmd-str))] `(do ~@(map generate-code cmds result)))) @@ -133,8 +136,9 @@ return 0; *library* (ffi-library (full-lib-name (:library config)))] (doseq [b body] (eval b)) - `(let [*library* (ffi-library ~(full-lib-name (:library config)))] - ~(run-infer *config* @*bodies*)))) + `(binding [*library* (ffi-library ~(full-lib-name (:library config)))] + (println "ll " *library*) + ~(run-infer *config* @*bodies*)))) (defmacro defcfn [nm] (let [name-str (name nm)] diff --git a/pixie/vm/libs/platform.py b/pixie/vm/libs/platform.py index 2db4cf6c..ea3ec32a 100644 --- a/pixie/vm/libs/platform.py +++ b/pixie/vm/libs/platform.py @@ -1,4 +1,5 @@ from rpython.translator.platform import platform +from pixie.vm.persistent_list import create_from_list from pixie.vm.string import String from pixie.vm.code import as_var from rpython.rlib.clibffi import get_libc_name @@ -9,3 +10,14 @@ as_var("pixie.platform", "name")(String(unicode(platform.name))) as_var("pixie.platform", "so-ext")(String(unicode(platform.so_ext))) as_var("pixie.platform", "lib-c-name")(String(unicode(get_libc_name()))) + +c_flags = [] +for itm in platform.cflags: + c_flags.append(String(unicode(itm))) + +as_var("pixie.platform", "c-flags")(create_from_list(c_flags)) + +link_flags = [] +for itm in platform.link_flags: + c_flags.append(String(unicode(itm))) +as_var("pixie.platform", "link-flags")(create_from_list(link_flags)) diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index fb9cd872..d7722885 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -1,5 +1,6 @@ (ns pixie.tests.test-ffi - (require pixie.test :as t)) + (require pixie.test :as t) + (require pixie.math :as m)) @@ -31,3 +32,6 @@ (sscanf-* "string" "fmt") (sscanf-* "string" "fmt" "optional arg1" "optional arg2") (t/assert true))) + +(t/deftest test-ffi-infer + (t/assert= 0.5 (m/asin (m/sin 0.5)))) From 1e327577d8417c93af3ee5692f168f46caf3d7e5 Mon Sep 17 00:00:00 2001 From: Michael Trotter Date: Wed, 11 Feb 2015 20:39:14 -0700 Subject: [PATCH 503/909] Fixed a typo (UnmachedX -> UnmatchedX) --- pixie/vm/reader.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index ab7ac74b..f0d98588 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -223,7 +223,7 @@ def invoke(self, rdr, ch): rdr.unread(ch) lst.append(read(rdr, True)) -class UnmachedListReader(ReaderHandler): +class UnmatchedListReader(ReaderHandler): def invoke(self, rdr, ch): throw_syntax_error_with_data(rdr, u"Unmatched list close ')'") @@ -239,7 +239,7 @@ def invoke(self, rdr, ch): rdr.unread(ch) acc = rt.conj(acc, read(rdr, True)) -class UnmachedVectorReader(ReaderHandler): +class UnmatchedVectorReader(ReaderHandler): def invoke(self, rdr, ch): throw_syntax_error_with_data(rdr, u"Unmatched vector close ']'") @@ -258,7 +258,7 @@ def invoke(self, rdr, ch): acc = rt._assoc(acc, k, v) return acc -class UnmachedMapReader(ReaderHandler): +class UnmatchedMapReader(ReaderHandler): def invoke(self, rdr, ch): affirm(False, u"Unmatched Map brace ") @@ -284,7 +284,7 @@ def invoke(self, rdr, ch): v = rdr.read() except EOFError: return throw_syntax_error_with_data(rdr, u"umatched quote") - + if v == "\"": return rt.wrap(u"".join(acc)) elif v == "\\": @@ -575,11 +575,11 @@ def skip_line(self, rdr): return handlers = {u"(": ListReader(), - u")": UnmachedListReader(), + u")": UnmatchedListReader(), u"[": VectorReader(), - u"]": UnmachedVectorReader(), + u"]": UnmatchedVectorReader(), u"{": MapReader(), - u"}": UnmachedMapReader(), + u"}": UnmatchedMapReader(), u"'": QuoteReader(), u":": KeywordReader(), u"\"": LiteralStringReader(), From e2c73df7852af971a1346b62a3708a063b3a40f3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 12 Feb 2015 13:18:13 -0700 Subject: [PATCH 504/909] try slightly different build flags --- pixie/ffi-infer.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 453fe22c..d0025323 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -117,7 +117,7 @@ return 0; (end-string))) (let [cmd-str (str "c++ " (apply str (interpose " " pixie.platform/c-flags)) - " -std=c++11 /tmp/tmp.cpp -I" + " -std=gnu++11 /tmp/tmp.cpp -I" (first @load-paths) (apply str " " (interpose " " (:cxx-flags *config*))) " -o /tmp/a.out && /tmp/a.out") From ee6b152ead4da021559fed33ed480361fe113fe7 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 12 Feb 2015 15:17:25 -0700 Subject: [PATCH 505/909] see if this fixes the travis failures --- pixie/ffi-infer.pxi | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index d0025323..e59e7f8e 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -117,7 +117,7 @@ return 0; (end-string))) (let [cmd-str (str "c++ " (apply str (interpose " " pixie.platform/c-flags)) - " -std=gnu++11 /tmp/tmp.cpp -I" + " /tmp/tmp.cpp -I" (first @load-paths) (apply str " " (interpose " " (:cxx-flags *config*))) " -o /tmp/a.out && /tmp/a.out") @@ -137,7 +137,6 @@ return 0; (doseq [b body] (eval b)) `(binding [*library* (ffi-library ~(full-lib-name (:library config)))] - (println "ll " *library*) ~(run-infer *config* @*bodies*)))) (defmacro defcfn [nm] From a7b95c1d7b1a0973c136644016f50724f3da61a0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 12 Feb 2015 15:43:50 -0700 Subject: [PATCH 506/909] add boost to deps to install --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index ffd0496f..2825107d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ script: - make run_built_tests before_install: - - sudo apt-get install libffi-dev libedit-dev + - sudo apt-get install libffi-dev libedit-dev libboost-all-dev notifications: irc: "chat.freenode.net#pixie-lang" From e9305d0bb5b91c0adb74083d5e79f2c2a07319eb Mon Sep 17 00:00:00 2001 From: Alexis Lee Date: Fri, 13 Feb 2015 11:21:26 +0000 Subject: [PATCH 507/909] Fix Gilardi scenario for -e This didn't work, because -e would only evaluate the first form provided: px -e '(ns c (require pixie.string :as s)) (println (s/capitalize "yay"))' Now -e works more similarly to script files, so this works fine. --- target.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/target.py b/target.py index c9f46461..a91a940b 100644 --- a/target.py +++ b/target.py @@ -154,7 +154,7 @@ def inner_invoke(self, args): with with_ns(u"user"): NS_VAR.deref().include_stdlib() - interpret(compile(read(StringReader(unicode_from_utf8(self._expr)), True))) + rt.load_reader(StringReader(unicode_from_utf8(self._expr))) class IsPreloadFlag(object): From 64541baf9e348e9983061505099c2bfff57506c3 Mon Sep 17 00:00:00 2001 From: zachcp Date: Thu, 12 Feb 2015 20:21:56 -0500 Subject: [PATCH 508/909] add while, take-while, drop-while tests for while take-while and srop-while return the test --- pixie/stdlib.pxi | 57 +++++++++++++++++++++++++++++++ tests/pixie/tests/test-stdlib.pxi | 30 ++++++++++++++++ 2 files changed, 87 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 04d7cf61..66417f84 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1455,6 +1455,63 @@ The new value is thus `(apply f current-value-of-atom args)`." (recur (dec n) (next s)) s))) +(defmacro while + {:doc "Repeatedly executes body while test expression is true. Presumes + some side-effect will cause test to become false/nil. Returns nil" + :added "0.1"} + [test & body] + `(loop [] + (when ~test + ~@body + (recur)))) + +(defn take-while + {:doc "Returns a lazy sequence of successive items from coll while + (pred item) returns true. pred must be free of side-effects. + Returns a transducer when no collection is provided." + :added "0.1"} + ([pred] + (fn [rf] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (if (pred input) + (rf result input) + (reduced result)))))) + ([pred coll] + (lazy-seq + (when-let [s (seq coll)] + (when (pred (first s)) + (cons (first s) (take-while pred (rest s)))))))) + + +(defn drop-while + {:doc "Returns a lazy sequence of the items in coll starting from the + first item for which (pred item) returns logical false. Returns a + stateful transducer when no collection is provided." + :added "0.1"} + ([pred] + (fn [rf] + (let [dv (atom true)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [drop? @dv] + (if (and drop? (pred input)) + result + (do + (reset! dv nil) + (rf result input))))))))) + ([pred coll] + (let [step (fn [pred coll] + (let [s (seq coll)] + (if (and s (pred (first s))) + (recur pred (rest s)) + s)))] + (lazy-seq (step pred coll))))) + (defn partition {:doc "Separates the collection into collections of size n, starting at the beginning, with an optional step size. diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index fa7feb63..eb4aed2f 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -312,3 +312,33 @@ (in-ns :pixie.tests.test-stdlib) (t/assert= (set (keys (ns-map 'foo))) #{'bar 'baz})) + +(t/deftest test-while + (t/assert= (while (pos? 0) true ) nil) + (t/assert= (while (pos? 0) false) nil) + (t/assert= 0 (let [x (atom 10) + cnt (atom 0)] + (while (pos? @x) + (do (swap! x dec) + (swap! cnt inc))) + @x)) + (t/assert= 10 (let [x (atom 10) + cnt (atom 0)] + (while (pos? @x) + (do (swap! x dec) + (swap! cnt inc))) + @cnt))) + +(t/deftest test-take-while + (t/assert= (take-while pos? [1 2 3 -1]) [1 2 3]) + (t/assert= (take-while pos? [-1 2]) ()) + (t/assert= (transduce (take-while even?) conj [2 4 6 7 8]) [2 4 6]) + (t/assert= (transduce (take-while even?) conj [0 2] [1 4 6]) [0 2]) + (t/assert= (transduce (take-while even?) conj [1 3] [2 4 6 7 8]) [1 3 2 4 6])) + +(t/deftest test-drop-while + (t/assert= (drop-while pos? [1 2 3 -1]) [-1]) + (t/assert= (drop-while pos? [-1 2]) [-1 2]) + (t/assert= (transduce (drop-while even?) conj [2 4 6 7 8]) [7 8]) + (t/assert= (transduce (drop-while even?) conj [0 2] [1 4 6]) [0 2 1 4 6]) + (t/assert= (transduce (drop-while even?) conj [0 2] [2 4 6 7 8]) [0 2 7 8])) From 5712ca9a95555a2ae02e7cb06db59170a866370c Mon Sep 17 00:00:00 2001 From: zach powers Date: Fri, 13 Feb 2015 12:44:09 -0500 Subject: [PATCH 509/909] change drop-while to be nested if loops in order to avoid multiple atom lookups once the drop condition has been triggered --- pixie/stdlib.pxi | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 66417f84..7af45d17 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1499,11 +1499,13 @@ The new value is thus `(apply f current-value-of-atom args)`." ([result] (rf result)) ([result input] (let [drop? @dv] - (if (and drop? (pred input)) - result - (do - (reset! dv nil) - (rf result input))))))))) + (if drop? + (if (pred input) + result + (do + (reset! dv nil) + (rf result input))) + (rf result input)))))))) ([pred coll] (let [step (fn [pred coll] (let [s (seq coll)] From 9e82e6c5460d016b3d65d1951d7f1317d167d1a7 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 15 Feb 2015 15:14:38 -0500 Subject: [PATCH 510/909] update docs for libboost --- README.md | 1 + 1 file changed, 1 insertion(+) diff --git a/README.md b/README.md index 3c1f4a19..16b829fa 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Some planned and implemented features: * python or pypy to build * [libffi-dev](https://sourceware.org/libffi/) * [libedit-dev](http://thrysoee.dk/editline/) +* [libboost-all-dev](http://www.boost.org/) (`brew install boost` for Mac) ## Building From 3a8052c76c0be349386af7e5087ce42771683eb2 Mon Sep 17 00:00:00 2001 From: Rob Biedenharn Date: Mon, 16 Feb 2015 11:19:33 -0500 Subject: [PATCH 511/909] WIP: in-bounds calls work; OOB fails --- pixie/stdlib.pxi | 26 ++++++++ tests/pixie/tests/test-ith.pxi | 108 +++++++++++++++++++++++++++++++++ 2 files changed, 134 insertions(+) create mode 100644 tests/pixie/tests/test-ith.pxi diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7af45d17..36224e96 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -845,6 +845,32 @@ If further arguments are passed, invokes the method named by symbol, passing the [coll] (last (butlast coll) )) +(defn ith + {:doc "Returns the ith element of the collection, negative values count from the end." + :signatures [[coll i]] + :added "0.1"} + [coll i] + (if (nil? coll) + nil + (let [len (count coll)] + (if (and (< i len) (>= i (- len))) + (ith-safe coll i) + (throw "Index out of bounds") + ) + ) + ) + ) + +(comment "No bounds checking on this one") +(defn ith-safe + {:doc "Returns the ith element of the collection, negative values count from the end." + :signatures [[coll i]] + :added "0.1"} + [coll i] + (if (neg? i) + (let [len (count coll)] (nth coll (+ i len))) + (nth coll i))) + (defn complement {:doc "Given a function, return a new function which takes the same arguments but returns the opposite truth value"} diff --git a/tests/pixie/tests/test-ith.pxi b/tests/pixie/tests/test-ith.pxi new file mode 100644 index 00000000..e9731a2f --- /dev/null +++ b/tests/pixie/tests/test-ith.pxi @@ -0,0 +1,108 @@ +(ns pixie.tests.test-ith + (require pixie.test :as t)) + +(t/deftest test-ith + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (t/assert= (ith [1 2 3 4 5] 0) 1) + (t/assert= (ith v 0) 1) + (t/assert= (ith v 0) (nth v 0)) + )) + +(t/deftest test-ith-nil + (t/assert= (ith nil 0) nil) + (t/assert= (ith nil 1) nil) + (t/assert= (ith nil -1) nil) + ) +(comment +(t/deftest test-ith-empty-always-oob + (t/assert= "Index out of bounds" (try (ith [] 0) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith [] 1) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith [] -1) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith '() 0) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith '() 1) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith '() -1) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith (range 0 0) 0) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith (range 0 0) 1) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith (range 0 0) -1) (catch e (ex-msg e)))) + ) + +(t/deftest test-ith-out-of-bounds + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (comment ">= 5 s/b oob") + (t/assert= "Index out of bounds" (try (ith v 5) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith l 5) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith r 5) (catch e (ex-msg e)))) + (comment "< -5 s/b oob") + (t/assert= "Index out of bounds" (try (ith v -6) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith l -6) (catch e (ex-msg e)))) + (t/assert= "Index out of bounds" (try (ith r -6) (catch e (ex-msg e)))) + ) + ) +) +(t/deftest test-ith-explicit + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + + (t/assert= (ith v 1) (nth v 1)) + (t/assert= (ith l 1) (nth l 1)) + (t/assert= (ith r 1) (nth r 1)) + + (t/assert= (ith r 0) (nth r 0)) + (t/assert= (ith l 0) (nth l 0)) + (t/assert= (ith v 0) (nth v 0)) + + (t/assert= (ith v 2) (nth v 2)) + (t/assert= (ith l 2) (nth l 2)) + (t/assert= (ith r 2) (nth r 2)) + + (t/assert= (ith v 3) (nth v 3)) + (t/assert= (ith l 3) (nth l 3)) + (t/assert= (ith r 3) (nth r 3)) + + (t/assert= (ith v 4) (nth v 4)) + (t/assert= (ith l 4) (nth l 4)) + (t/assert= (ith r 4) (nth r 4)) + + (t/assert= (ith v -5) (nth v 0)) + (t/assert= (ith l -5) (nth l 0)) + (t/assert= (ith r -5) (nth r 0)) + + (t/assert= (ith v -4) (nth v 1)) + (t/assert= (ith l -4) (nth l 1)) + (t/assert= (ith r -4) (nth r 1)) + + (t/assert= (ith v -3) (nth v 2)) + (t/assert= (ith l -3) (nth l 2)) + (t/assert= (ith r -3) (nth r 2)) + + (t/assert= (ith v -2) (nth v 3)) + (t/assert= (ith l -2) (nth l 3)) + (t/assert= (ith r -2) (nth r 3)) + + (t/assert= (ith v -1) (nth v 4)) + (t/assert= (ith l -1) (nth l 4)) + (t/assert= (ith r -1) (nth r 4)) + + )) + +(t/deftest test-ith-doseq + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (doseq [i (range 0 5)] + (t/assert= (ith v i) (nth v i)) + (t/assert= (ith l i) (nth l i)) + (t/assert= (ith r i) (nth r i)) + ) + (doseq [i (range -5 0)] + (t/assert= (ith v i) (nth v (+ 5 i))) + (t/assert= (ith l i) (nth l (+ 5 i))) + (t/assert= (ith r i) (nth r (+ 5 i))) + ) + )) + From 7244870bb6edb95788e94ff80855fa9a6755c851 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 17 Feb 2015 16:14:49 -0500 Subject: [PATCH 512/909] nth-not-found added --- pixie/vm/array.py | 19 +++++++++++++++++++ pixie/vm/libs/ffi.py | 5 +++++ pixie/vm/persistent_vector.py | 5 +++++ pixie/vm/stdlib.py | 6 +++++- pixie/vm/string.py | 8 ++++++++ 5 files changed, 42 insertions(+), 1 deletion(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index e0162011..ef01067b 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -50,6 +50,15 @@ def _nth(self, idx): else: return nil +@extend(proto._nth_not_found, Array) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, Array) + ival = idx.int_val() + if ival < len(self._list): + return self._list[ival] + else: + return not_found + @extend(proto._reduce, Array) def reduce(self, f, init): assert isinstance(self, Array) @@ -213,6 +222,16 @@ def _nth(self, idx): return nil +@extend(proto._nth_not_found, ByteArray) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, ByteArray) + affirm(isinstance(idx, Integer), u"Index must be an integer") + ival = idx.r_uint_val() + if 0 <= ival < self._cnt: + return rt.wrap(ord(self._buffer[ival])) + + return not_found + @extend(proto._count, ByteArray) def _count(self): assert isinstance(self, ByteArray) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index b55c1f25..48d4d301 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -245,6 +245,11 @@ def capacity(self): def _nth(self, idx): return rt.wrap(ord(self.nth_char(idx.int_val()))) +@extend(proto._nth_not_found, Buffer) +def _nth_not_found(self, idx, not_found): + return rt.wrap(ord(self.nth_char(idx.int_val()))) + + @extend(proto._count, Buffer) def _count(self): return rt.wrap(intmask(self.count())) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index c36a0a2b..e70bad43 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -425,6 +425,11 @@ def _nth(self, idx): assert isinstance(self, PersistentVector) return self.nth(idx.int_val()) +@extend(proto._nth_not_found, PersistentVector) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, PersistentVector) + return self.nth(idx.int_val(), not_found) + @extend(proto._val_at, PersistentVector) def _val_at(self, key, not_found): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 4ad2c0b6..56b584ed 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -16,7 +16,7 @@ defprotocol("pixie.stdlib", "ICounted", ["-count"]) -defprotocol("pixie.stdlib", "IIndexed", ["-nth"]) +defprotocol("pixie.stdlib", "IIndexed", ["-nth", "-nth-not-found"]) defprotocol("pixie.stdlib", "IPersistentCollection", ["-conj", "-disj"]) @@ -290,6 +290,10 @@ def conj(a, b): def nth(a, b): return rt._nth(a, b) +@as_var("nth-not-found") +def nth_not_found(a, b, c): + return rt._nth_not_found(a, b, c) + @as_var("str") def str__args(args): diff --git a/pixie/vm/string.py b/pixie/vm/string.py index f7687633..a012761b 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -57,6 +57,14 @@ def _nth(self, idx): return Character(ord(self._str[i])) raise IndexError() +@extend(proto._nth_not_found, String) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, String) + i = idx.int_val() + if 0 <= i < len(self._str): + return Character(ord(self._str[i])) + return not_found + @extend(proto._eq, String) def _eq(self, v): assert isinstance(self, String) From 1c13d178be6b308a5dff3b6218a01ea27270ad3a Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 17 Feb 2015 16:21:41 -0500 Subject: [PATCH 513/909] nth with not-found mutli-method --- pixie/stdlib.pxi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7af45d17..ebe6778e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -624,6 +624,14 @@ returns true" [n] (= (rem n 2) 1)) +(defn nth + {:doc "Returns the element at the idx. If the index is not found it will return an error. + However, if you specify a not-found parameter, it will substitue that instead" + :signatures [[coll idx] [coll idx not-found]] + :added "0.1"} + ([coll idx] (-nth coll idx)) + ([coll idx not-found] (-nth-not-found coll idx not-found))) + (defn first {:doc "Returns the first item in coll, if coll implements IIndexed nth will be used to retreive the item from the collection." From 87034a235e4052c09be6421bf152563bc95c5c7c Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 17 Feb 2015 17:00:50 -0500 Subject: [PATCH 514/909] tests pass - but still needs some work --- pixie/stdlib.pxi | 49 ++++++++++++++++++++++------------- pixie/vm/array.py | 4 +-- pixie/vm/persistent_vector.py | 7 +++-- pixie/vm/string.py | 2 +- 4 files changed, 39 insertions(+), 23 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ebe6778e..6aba398d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -245,7 +245,7 @@ @init (if (-eq i max) init - (recur (f init (nth coll i)) (+ i 1)))))))) + (recur (f init (nth coll i nil)) (+ i 1)))))))) (def rest (fn ^{:doc "Returns the elements after the first element, or () if there are no more elements." :signatures [[coll]] @@ -639,7 +639,7 @@ returns true" :added "0.1"} [coll] (if (satisfies? IIndexed coll) - (nth coll 0) + (nth coll 0 nil) (-first coll))) (defn second @@ -649,7 +649,7 @@ returns true" :added "0.1"} [coll] (if (satisfies? IIndexed coll) - (nth coll 1) + (nth coll 1 nil) (first (next coll)))) (defn third @@ -659,7 +659,7 @@ returns true" :added "0.1"} [coll] (if (satisfies? IIndexed coll) - (nth coll 2) + (nth coll 2 nil) (first (next (next coll))))) (defn fourth @@ -669,7 +669,7 @@ returns true" :added "0.1"} [coll] (if (satisfies? IIndexed coll) - (nth coll 3) + (nth coll 3 nil) (first (next (next (next coll)))))) (defn assoc @@ -881,6 +881,9 @@ Stops if it finds such an element." (extend -nth MapEntry (fn map-entry-nth [self idx] (cond (= idx 0) (-key self) (= idx 1) (-val self)))) +(extend -nth-not-found MapEntry (fn map-entry-nth [self idx not-found] + (cond (= idx 0) (-key self) + (= idx 1) (-val self)))) (extend -reduce MapEntry indexed-reduce) @@ -956,7 +959,7 @@ Stops if it finds such an element." (instance? PersistentVector x) (if (= (count x) 2) - (assoc coll (nth x 0) (nth x 1)) + (assoc coll (nth x 0 nil) (nth x 1 nil)) (throw "Vector arg to map conj must be a pair")) (satisfies? ISeqable x) @@ -1334,11 +1337,11 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro foreach [binding & body] (assert (= 2 (count binding)) "binding and collection required") `(reduce - (fn [_ ~(nth binding 0)] + (fn [_ ~(nth binding 0 nil)] ~@body nil) nil - ~(nth binding 1))) + ~(nth binding 1 nil))) (defmacro iterate [binding & body] (assert (= 2 (count binding)) "binding and collection required") @@ -1358,8 +1361,8 @@ The new value is thus `(apply f current-value-of-atom args)`." :signatures [[[i n] & body]] :added "0.1"} [bind & body] - (let [b (nth bind 0)] - `(let [max# ~(nth bind 1)] + (let [b (nth bind 0 nil)] + `(let [max# ~(nth bind 1 nil)] (loop [~b 0] (if (= ~b max#) nil @@ -1369,17 +1372,17 @@ The new value is thus `(apply f current-value-of-atom args)`." (extend -iterator PersistentVector (fn [v] (dotimes [x (count v)] - (yield (nth v x))))) + (yield (nth v x nil))))) (extend -iterator Array (fn [v] (dotimes [x (count v)] - (yield (nth v x))))) + (yield (nth v x nil))))) (extend -iterator String (fn [v] (dotimes [x (count v)] - (yield (nth v x))))) + (yield (nth v x nil))))) (defmacro and {:doc "Check if the given expressions return truthy values, returning the last, or false." @@ -1410,8 +1413,8 @@ The new value is thus `(apply f current-value-of-atom args)`." `(if (not ~test) (do ~@body))) (defmacro when-let [binding & body] - (let [bind (nth binding 0) - test (nth binding 1)] + (let [bind (nth binding 0 nil) + test (nth binding 1 nil)] `(let [tmp# ~test] (when tmp# (let [~bind tmp#] @@ -1420,8 +1423,8 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro if-let ([binding then] `(if-let ~binding ~then nil)) ([binding then else] - (let [bind (nth binding 0) - test (nth binding 1)] + (let [bind (nth binding 0 nil) + test (nth binding 1 nil)] `(let [tmp# ~test] (if tmp# (let [~bind tmp#] @@ -1561,7 +1564,7 @@ not enough elements were present." (= binding :as) (reduce conj res (destructure (second bindings) expr)) :else (recur (next bindings) (inc i) - (reduce conj res (destructure (first bindings) `(nth ~expr ~i)))))) + (reduce conj res (destructure (first bindings) `(nth ~expr ~i nil)))))) res))) (defn destructure-map [binding-map expr] @@ -1620,6 +1623,10 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (if (and (pos? n) s) (recur (next s) (dec n)) (first s)))) +(extend -nth-not-found ISeq (fn [s n not-found] + (if (and (pos? n) s) + (recur (next s) (dec n) not-found) + (first s)))) (defn abs {:doc "Returns the absolute value of x." @@ -1664,6 +1671,12 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (if (cmp val stop) val nil))) + (-nth-not-found [self idx not-found] + (let [cmp (if (< start stop) < >) + val (+ start (* idx step))] + (if (cmp val stop) + val + not-found))) ISeqable (-seq [self] (when (or (and (> step 0) (< start stop)) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index ef01067b..265001f7 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -48,7 +48,7 @@ def _nth(self, idx): if ival < len(self._list): return self._list[ival] else: - return nil + affirm(False, u"Index out of Range") @extend(proto._nth_not_found, Array) def _nth_not_found(self, idx, not_found): @@ -220,7 +220,7 @@ def _nth(self, idx): if 0 <= ival < self._cnt: return rt.wrap(ord(self._buffer[ival])) - return nil + return affirm(False, u"Index out of Range") @extend(proto._nth_not_found, ByteArray) def _nth_not_found(self, idx, not_found): diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index e70bad43..26517cd4 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -64,12 +64,15 @@ def array_for(self, i): affirm(False, u"Index out of Range") - def nth(self, i, not_found=nil): + def nth(self, i, not_found=None): if 0 <= i < self._cnt: node = self.array_for(r_uint(i)) return node[i & 0x01f] - return not_found + if not_found is None: + affirm(False, u"Index out of Range") + else: + return not_found def conj(self, val): assert self._cnt < r_uint(0xFFFFFFFF) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index a012761b..c9be8844 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -55,7 +55,7 @@ def _nth(self, idx): i = idx.int_val() if 0 <= i < len(self._str): return Character(ord(self._str[i])) - raise IndexError() + affirm(False, u"Index out of Range") @extend(proto._nth_not_found, String) def _nth_not_found(self, idx, not_found): From b510a6071f1b1b0ecc35239706bb027e975343bc Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 17 Feb 2015 17:06:40 -0500 Subject: [PATCH 515/909] closer --- pixie/stdlib.pxi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6aba398d..c2e3fc20 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -883,7 +883,8 @@ Stops if it finds such an element." (= idx 1) (-val self)))) (extend -nth-not-found MapEntry (fn map-entry-nth [self idx not-found] (cond (= idx 0) (-key self) - (= idx 1) (-val self)))) + (= idx 1) (-val self) + :else not-found))) (extend -reduce MapEntry indexed-reduce) @@ -1626,7 +1627,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (extend -nth-not-found ISeq (fn [s n not-found] (if (and (pos? n) s) (recur (next s) (dec n) not-found) - (first s)))) + (or (first s) not-found)))) (defn abs {:doc "Returns the absolute value of x." From 9924da89946a7a656a1921680d8339646ae1248c Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 17 Feb 2015 17:35:50 -0500 Subject: [PATCH 516/909] add tests for nth --- tests/pixie/tests/test-stdlib.pxi | 38 +++++++++++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index eb4aed2f..23220adc 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -52,6 +52,44 @@ (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]")) +(t/deftest test-nth + ;; works if the index is found + (t/assert= (nth [1 2 3] 1) 2) + (t/assert= (nth '(1 2 3) 1) 2) + (t/assert= (nth (make-array 3) 2) nil) + (t/assert= (nth (range 4) 1) 1) + (t/assert= (nth "hithere" 1) \i) + + ;; throws error for bad index + (try + (nth [1 2 3] 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth '(1 2 3) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (make-array 3) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (range 4) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth "hithere" 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + + ;; if not-found is specified, uses that for out of range + (t/assert= (nth [1 2 3] 99 :default) :default) + (t/assert= (nth '(1 2 3) 99 :default) :default) + (t/assert= (nth (make-array 3) 99 :default) :default) + (t/assert= (nth (range 4) 99 :default) :default) + (t/assert= (nth "hithere" 99 :default) :default) + + (t/assert= (nth [1 2 3] 1 :default) 2) + (t/assert= (nth '(1 2 3) 1 :default) 2) + (t/assert= (nth (make-array 3) 2 :default) nil) + (t/assert= (nth (range 4) 1 :default) 1) + (t/assert= (nth "hithere" 1 :deafult) \i)) + (t/deftest test-first (t/assert= (first []) nil) (t/assert= (first '()) nil) From 66276e71bd8aafbf6e4deface9d2abebbc2b23d6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Feb 2015 16:48:36 -0700 Subject: [PATCH 517/909] start of uv lib support, and updates to ffi-infer to support it --- pixie/ffi-infer.pxi | 16 +++++++++++++++- pixie/vm/libs/ffi.py | 21 ++++++++++++++++++++- pixie/vm/uv.pxi | 11 ----------- 3 files changed, 35 insertions(+), 13 deletions(-) delete mode 100644 pixie/vm/uv.pxi diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 7e8ea9ad..a2254ef8 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -48,6 +48,10 @@ members)) "\"]}\" << std::endl;")) +(defmethod emit-infer-code :callback + [{:keys [name]}] + (str "PixieChecker::DumpType<__typeof__(" name ")>();")) + (defn start-string [] (str (apply str (map (fn [i] @@ -71,7 +75,7 @@ return 0; (defmulti edn-to-ctype :type) (defn callback-type [{:keys [arguments returns]}] - `(ffi-callback ~@(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns))) + `(ffi-callback ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns))) (defmethod edn-to-ctype :pointer [{:keys [of-type] :as ptr}] @@ -142,6 +146,12 @@ return 0; `[~(keyword name) ~(edn-to-ctype type) ~offset]) members infered-members)]))) +(defmethod generate-code :callback + [{:keys [name]} {:keys [of-type]}] + `(def ~(symbol name) + ~(callback-type of-type))) + + (defn run-infer [config cmds] (io/spit "/tmp/tmp.cpp" (str (start-string) (apply str (map emit-infer-code @@ -194,6 +204,10 @@ return 0; :name ~(name nm) :members ~(vec (map name members))) )) +(defmacro defccallback [nm] + `(swap! *bodies* conj (assoc {:op :callback} + :name ~(name nm)))) + diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 8da4b21d..264015ab 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -363,6 +363,24 @@ def finalize_token(self): +class CVoid(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CVoid") + + def ffi_get_value(self, ptr): + return nil + + def ffi_set_value(self, ptr, val): + runtime_error(u"Can't encode a Void") + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer + +cvoid = CVoid() + class CVoidP(CType): def __init__(self): CType.__init__(self, u"pixie.stdlib.CVoidP") @@ -484,7 +502,8 @@ def ll_invoke(self, llargs, llres): self._is_invoked = True retval = self._fn.invoke(args) - cft._ret_type.ffi_set_value(llres, retval) + if cft._ret_type is not cvoid: + cft._ret_type.ffi_set_value(llres, retval) def cleanup(self): diff --git a/pixie/vm/uv.pxi b/pixie/vm/uv.pxi deleted file mode 100644 index f1e1c12e..00000000 --- a/pixie/vm/uv.pxi +++ /dev/null @@ -1,11 +0,0 @@ -(ns pixie.uv - (require pixie.ffi-infer :as f)) - -(f/with-config {:library "uv" - :includes ["uv.h"]} - (f/defconst UV_RUN_DEFAULT) - (f/defconst UV_RUN_ONCE) - (f/defconst UV_RUN_NOWAIT) - - (f/defcstruct uv_loop_t []) - ) \ No newline at end of file From f4cd070ac8379311646bd4b2e917505d534f7322 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Feb 2015 16:51:42 -0700 Subject: [PATCH 518/909] update deps in README and travis.yml to include libuv --- .travis.yml | 2 +- README.md | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 2825107d..3a1a725a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ script: - make run_built_tests before_install: - - sudo apt-get install libffi-dev libedit-dev libboost-all-dev + - sudo apt-get install libffi-dev libedit-dev libboost-all-dev libuv-dev notifications: irc: "chat.freenode.net#pixie-lang" diff --git a/README.md b/README.md index 16b829fa..5961026c 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,7 @@ Some planned and implemented features: * python or pypy to build * [libffi-dev](https://sourceware.org/libffi/) * [libedit-dev](http://thrysoee.dk/editline/) +* [libuv-dev](https://github.com/joyent/libuv) Version 1.0 or higher * [libboost-all-dev](http://www.boost.org/) (`brew install boost` for Mac) ## Building From a119da16631a1000540e3c0adf0453cb2a0220cd Mon Sep 17 00:00:00 2001 From: gigasquid Date: Thu, 19 Feb 2015 21:41:58 -0500 Subject: [PATCH 519/909] fixed weirdness with defining function before and macro was defined. Also cleanup up tests and got rid of penultimate and safe-ith --- pixie/stdlib.pxi | 47 ++++++-------------- tests/pixie/tests/test-ith.pxi | 71 +++---------------------------- tests/pixie/tests/test-stdlib.pxi | 12 ------ 3 files changed, 21 insertions(+), 109 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 36224e96..3455c8fe 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -838,39 +838,6 @@ If further arguments are passed, invokes the method named by symbol, passing the (recur (conj res (first coll)) (next coll)) (seq res)))) -(defn penultimate - {:doc "Returns the second to last element of the collection, or nil if none." - :signatures [[coll]] - :added "0.1"} - [coll] - (last (butlast coll) )) - -(defn ith - {:doc "Returns the ith element of the collection, negative values count from the end." - :signatures [[coll i]] - :added "0.1"} - [coll i] - (if (nil? coll) - nil - (let [len (count coll)] - (if (and (< i len) (>= i (- len))) - (ith-safe coll i) - (throw "Index out of bounds") - ) - ) - ) - ) - -(comment "No bounds checking on this one") -(defn ith-safe - {:doc "Returns the ith element of the collection, negative values count from the end." - :signatures [[coll i]] - :added "0.1"} - [coll i] - (if (neg? i) - (let [len (count coll)] (nth coll (+ i len))) - (nth coll i))) - (defn complement {:doc "Given a function, return a new function which takes the same arguments but returns the opposite truth value"} @@ -1464,6 +1431,20 @@ The new value is thus `(apply f current-value-of-atom args)`." (recur (dec n) (next xs)) xs))) +(defn ith + {:doc "Returns the ith element of the collection, negative values count from the end. + If an index is out of bounds, will throw an Index out of bounds exception" + :signatures [[coll i]] + :added "0.1"} + [coll i] + (if coll + (let [len (count coll) + idx (if (neg? i) (+ i len) i) + in-bounds? (and (< idx len) (>= idx 0))] + (if in-bounds? + (nth coll idx) + (throw "Index out of bounds"))))) + (defn take {:doc "Takes n elements from the collection, or fewer, if not enough." :added "0.1"} diff --git a/tests/pixie/tests/test-ith.pxi b/tests/pixie/tests/test-ith.pxi index e9731a2f..f47f00fd 100644 --- a/tests/pixie/tests/test-ith.pxi +++ b/tests/pixie/tests/test-ith.pxi @@ -7,15 +7,13 @@ r (range 1 6)] (t/assert= (ith [1 2 3 4 5] 0) 1) (t/assert= (ith v 0) 1) - (t/assert= (ith v 0) (nth v 0)) - )) + (t/assert= (ith v 0) (nth v 0)))) (t/deftest test-ith-nil (t/assert= (ith nil 0) nil) (t/assert= (ith nil 1) nil) - (t/assert= (ith nil -1) nil) - ) -(comment + (t/assert= (ith nil -1) nil)) + (t/deftest test-ith-empty-always-oob (t/assert= "Index out of bounds" (try (ith [] 0) (catch e (ex-msg e)))) (t/assert= "Index out of bounds" (try (ith [] 1) (catch e (ex-msg e)))) @@ -25,70 +23,18 @@ (t/assert= "Index out of bounds" (try (ith '() -1) (catch e (ex-msg e)))) (t/assert= "Index out of bounds" (try (ith (range 0 0) 0) (catch e (ex-msg e)))) (t/assert= "Index out of bounds" (try (ith (range 0 0) 1) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith (range 0 0) -1) (catch e (ex-msg e)))) - ) + (t/assert= "Index out of bounds" (try (ith (range 0 0) -1) (catch e (ex-msg e))))) (t/deftest test-ith-out-of-bounds (let [v [1 2 3 4 5] l '(1 2 3 4 5) r (range 1 6)] - (comment ">= 5 s/b oob") (t/assert= "Index out of bounds" (try (ith v 5) (catch e (ex-msg e)))) (t/assert= "Index out of bounds" (try (ith l 5) (catch e (ex-msg e)))) (t/assert= "Index out of bounds" (try (ith r 5) (catch e (ex-msg e)))) - (comment "< -5 s/b oob") (t/assert= "Index out of bounds" (try (ith v -6) (catch e (ex-msg e)))) (t/assert= "Index out of bounds" (try (ith l -6) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith r -6) (catch e (ex-msg e)))) - ) - ) -) -(t/deftest test-ith-explicit - (let [v [1 2 3 4 5] - l '(1 2 3 4 5) - r (range 1 6)] - - (t/assert= (ith v 1) (nth v 1)) - (t/assert= (ith l 1) (nth l 1)) - (t/assert= (ith r 1) (nth r 1)) - - (t/assert= (ith r 0) (nth r 0)) - (t/assert= (ith l 0) (nth l 0)) - (t/assert= (ith v 0) (nth v 0)) - - (t/assert= (ith v 2) (nth v 2)) - (t/assert= (ith l 2) (nth l 2)) - (t/assert= (ith r 2) (nth r 2)) - - (t/assert= (ith v 3) (nth v 3)) - (t/assert= (ith l 3) (nth l 3)) - (t/assert= (ith r 3) (nth r 3)) - - (t/assert= (ith v 4) (nth v 4)) - (t/assert= (ith l 4) (nth l 4)) - (t/assert= (ith r 4) (nth r 4)) - - (t/assert= (ith v -5) (nth v 0)) - (t/assert= (ith l -5) (nth l 0)) - (t/assert= (ith r -5) (nth r 0)) - - (t/assert= (ith v -4) (nth v 1)) - (t/assert= (ith l -4) (nth l 1)) - (t/assert= (ith r -4) (nth r 1)) - - (t/assert= (ith v -3) (nth v 2)) - (t/assert= (ith l -3) (nth l 2)) - (t/assert= (ith r -3) (nth r 2)) - - (t/assert= (ith v -2) (nth v 3)) - (t/assert= (ith l -2) (nth l 3)) - (t/assert= (ith r -2) (nth r 3)) - - (t/assert= (ith v -1) (nth v 4)) - (t/assert= (ith l -1) (nth l 4)) - (t/assert= (ith r -1) (nth r 4)) - - )) + (t/assert= "Index out of bounds" (try (ith r -6) (catch e (ex-msg e)))))) (t/deftest test-ith-doseq (let [v [1 2 3 4 5] @@ -97,12 +43,9 @@ (doseq [i (range 0 5)] (t/assert= (ith v i) (nth v i)) (t/assert= (ith l i) (nth l i)) - (t/assert= (ith r i) (nth r i)) - ) + (t/assert= (ith r i) (nth r i))) (doseq [i (range -5 0)] (t/assert= (ith v i) (nth v (+ 5 i))) (t/assert= (ith l i) (nth l (+ 5 i))) - (t/assert= (ith r i) (nth r (+ 5 i))) - ) - )) + (t/assert= (ith r i) (nth r (+ 5 i)))))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index eb4aed2f..24b113fa 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -90,18 +90,6 @@ (t/assert= (butlast l) res) (t/assert= (butlast r) res))) -(t/deftest test-penultimate - (let [v [1 2 3 4 5] - l '(1 2 3 4 5) - r (range 1 6)] - (t/assert= (penultimate nil) nil) - (t/assert= (penultimate []) nil) - (t/assert= (penultimate (range 0 0)) nil) - (t/assert= (penultimate v) 4) - (t/assert= (penultimate l) 4) - (t/assert= (penultimate r) 4) - (t/assert= (penultimate [2]) nil))) - (t/deftest test-empty? (t/assert= (empty? []) true) (t/assert= (empty? '()) true) From 1a35480b91e076f778da525f19e1f423180d4471 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 20 Feb 2015 16:58:32 -0700 Subject: [PATCH 520/909] add support for searching for libs/includes on load-paths --- Makefile | 17 +++++++++++++---- README.md | 2 +- pixie/ffi-infer.pxi | 6 ++++-- pixie/vm/libs/ffi.py | 43 ++++++++++++++++++++++++++++--------------- 4 files changed, 46 insertions(+), 22 deletions(-) diff --git a/Makefile b/Makefile index 58ca8c6d..3562d53a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=../externals -PYTHON ?= python +PYTHON ?= pypy PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack @@ -17,6 +17,9 @@ help: @echo "make build_no_jit - build without jit" @echo "make fetch_externals - download and unpack external deps" +find_externals_name: + @PYTHONPATH=$(PYTHONPATH) $(PYTHON) find_externals_name.py + build_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py @@ -32,7 +35,13 @@ build_preload_no_jit: fetch_externals build: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) -fetch_externals: $(EXTERNALS)/pypy +fetch_externals: $(EXTERNALS)/pypy $(EXTERNALS)/binaries + +$(EXTERNALS)/binaries: + mkdir $(EXTERNALS); \ + curl -L `make find_externals_name` > $(EXTERNALS)/externals.tar.bz2 + cd $(EXTERNALS) && tar -jxf externals.tar.bz2 + $(EXTERNALS)/pypy: mkdir $(EXTERNALS); \ @@ -46,10 +55,10 @@ run: ./pixie-vm run_interactive: - PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py + @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py -l ../externals/externals/lib run_built_tests: pixie-vm - ./pixie-vm run-tests.pxi + ./pixie-vm -l ../externals/externals/lib -l ../externals/externals/include run-tests.pxi run_interpreted_tests: target.py PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.pxi diff --git a/README.md b/README.md index 5961026c..55c4380c 100644 --- a/README.md +++ b/README.md @@ -22,7 +22,7 @@ Some planned and implemented features: * python or pypy to build * [libffi-dev](https://sourceware.org/libffi/) * [libedit-dev](http://thrysoee.dk/editline/) -* [libuv-dev](https://github.com/joyent/libuv) Version 1.0 or higher +* [libuv-dev](https://github.com/libuv/libuv) Version 1.0 or higher * [libboost-all-dev](http://www.boost.org/) (`brew install boost` for Mac) ## Building diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index a2254ef8..b797814b 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -157,10 +157,12 @@ return 0; (apply str (map emit-infer-code cmds)) (end-string))) + (println @load-paths) (let [cmd-str (str "c++ " (apply str (interpose " " pixie.platform/c-flags)) - " /tmp/tmp.cpp -I" - (first @load-paths) + " /tmp/tmp.cpp " + (apply str (map (fn [x] ( str " -I " x " ")) + @load-paths)) (apply str " " (interpose " " (:cxx-flags *config*))) " -o /tmp/a.out && /tmp/a.out") _ (println cmd-str) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 264015ab..f8b93f3c 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -27,11 +27,6 @@ This code gets a bit interesting. We use the RPython rlib module jit_libffi to do the interfacing, you can find good docs in that module. -The problem is we can't serialize/translate function pointers (makes sense), so we make use of the _cleanup_ method -hook to clean out all unsupported fields. We then lazy re-load these values as needed. This allows us to specifiy FFI -functions inside of the stdlib (and other places) allowing for fast interpreter boot times. - - """ @@ -50,20 +45,38 @@ def __init__(self, nm): assert isinstance(nm, unicode) self._name = nm self._is_inited = False - self.thaw() + self.load_lib() - def thaw(self): + def load_lib(self): if not self._is_inited: - s = rffi.str2charp(str(self._name)) + load_paths = rt.deref(rt.deref(rt.load_paths)) + + for x in range(rt.count(load_paths)): + s = rffi.str2charp(str(rt.name(rt.nth(load_paths, rt.wrap(x)))) + "/" + str(self._name)) + try: + self._dyn_lib = dynload.dlopen(s) + self._is_inited = True + except dynload.DLOpenError as ex: + continue + finally: + rffi.free_charp(s) + break + + if not self._is_inited: + s = rffi.str2charp(str(self._name)) + try: + self._dyn_lib = dynload.dlopen(s) + self._is_inited = True + except dynload.DLOpenError as ex: + raise object.WrappedException(object.RuntimeException(rt.wrap(u"Couldn't Load Library: " + self._name))) + finally: + rffi.free_charp(s) + + + + - try: - self._dyn_lib = dynload.dlopen(s) - except dynload.DLOpenError as ex: - raise object.WrappedException(object.RuntimeException(rt.wrap(ex.msg))) - finally: - rffi.free_charp(s) - self._is_inited = True def get_fn_ptr(self, nm): From e8bd3ddb73b58860e22817cece1e05f39d63e248 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 20 Feb 2015 17:02:17 -0700 Subject: [PATCH 521/909] remove libuv-dev dep in travis file --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 3a1a725a..2825107d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,7 +15,7 @@ script: - make run_built_tests before_install: - - sudo apt-get install libffi-dev libedit-dev libboost-all-dev libuv-dev + - sudo apt-get install libffi-dev libedit-dev libboost-all-dev notifications: irc: "chat.freenode.net#pixie-lang" From 4150dec2a1aeded7cb35f92baed7f01aefbff5de Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 20 Feb 2015 17:03:56 -0700 Subject: [PATCH 522/909] add missing externals file --- find_externals_name.py | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 find_externals_name.py diff --git a/find_externals_name.py b/find_externals_name.py new file mode 100644 index 00000000..7f9b88d4 --- /dev/null +++ b/find_externals_name.py @@ -0,0 +1,3 @@ +from rpython.translator.platform import platform + +print "https://github.com/pixie-lang/external-deps/releases/download/1.0/externals-"+platform.name+".tar.bz2" \ No newline at end of file From e2fa275e426e6e35e03f513eb853bb79350d902d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 20 Feb 2015 22:07:23 -0700 Subject: [PATCH 523/909] rework makefile to pull externals --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 3562d53a..7931f035 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,8 @@ fetch_externals: $(EXTERNALS)/pypy $(EXTERNALS)/binaries $(EXTERNALS)/binaries: mkdir $(EXTERNALS); \ - curl -L `make find_externals_name` > $(EXTERNALS)/externals.tar.bz2 + echo https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 + curl -L https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 > $(EXTERNALS)/externals.tar.bz2 cd $(EXTERNALS) && tar -jxf externals.tar.bz2 From d0717e2ac71edcecb3d715c601d7770d23bc8a64 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 20 Feb 2015 22:30:57 -0700 Subject: [PATCH 524/909] tests and basic uv support --- pixie/uv.pxi | 38 +++++++++++++++++++++++++++++++++++ tests/pixie/tests/test-uv.pxi | 20 ++++++++++++++++++ 2 files changed, 58 insertions(+) create mode 100644 pixie/uv.pxi create mode 100644 tests/pixie/tests/test-uv.pxi diff --git a/pixie/uv.pxi b/pixie/uv.pxi new file mode 100644 index 00000000..e6f03f15 --- /dev/null +++ b/pixie/uv.pxi @@ -0,0 +1,38 @@ +(ns pixie.uv + (require pixie.ffi-infer :as f)) + +(f/with-config {:library "uv" + :includes ["uv.h"]} + (f/defconst UV_RUN_DEFAULT) + (f/defconst UV_RUN_ONCE) + (f/defconst UV_RUN_NOWAIT) + + (f/defcstruct uv_loop_t []) + + (f/defcfn uv_loop_init) + (f/defcfn uv_loop_close) + (f/defcfn uv_default_loop) + + (f/defcfn uv_run) + (f/defcfn uv_loop_alive) + (f/defcfn uv_stop) + (f/defcfn uv_loop_size) + (f/defcfn uv_backend_fd) + (f/defcfn uv_backend_timeout) + (f/defcfn uv_now) + (f/defcfn uv_update_time) + (f/defcfn uv_walk) + + (f/defccallback uv_read_cb) + + + ;; Timer + + (f/defcstruct uv_timer_t []) + (f/defccallback uv_timer_cb) + (f/defcfn uv_timer_init) + (f/defcfn uv_timer_start) + (f/defcfn uv_timer_stop) + (f/defcfn uv_timer_again) + (f/defcfn uv_timer_set_repeat) + (f/defcfn uv_timer_get_repeat)) diff --git a/tests/pixie/tests/test-uv.pxi b/tests/pixie/tests/test-uv.pxi new file mode 100644 index 00000000..3998ab37 --- /dev/null +++ b/tests/pixie/tests/test-uv.pxi @@ -0,0 +1,20 @@ +(ns pixie.test-uv + (require pixie.uv :as uv) + (require pixie.test :as t)) + + +(t/deftest timer-tests + (let [cb (atom nil) + result (atom false) + loop (uv/uv_loop_t) + timer (uv/uv_timer_t)] + (reset! cb (ffi-prep-callback uv/uv_timer_cb + (fn [handle] + (reset! result true) + (uv/uv_timer_stop timer) + (-dispose! @cb)))) + (uv/uv_loop_init loop) + (uv/uv_timer_init loop timer) + (uv/uv_timer_start timer @cb 10 0) + (uv/uv_run loop uv/UV_RUN_ONCE) + (t/assert @result))) From 592adcef3bfb0c7a67991d2913973344c0727a06 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 22 Feb 2015 14:22:13 -0500 Subject: [PATCH 525/909] =?UTF-8?q?Refactored=20ith=20so=20that=20it=20rel?= =?UTF-8?q?ies=20on=20nth=20out=20of=20bounds=20checking=20-=20also=20adde?= =?UTF-8?q?d=20nth=E2=80=99s=20not-found=20functionality=20-=20fixed=20bug?= =?UTF-8?q?=20in=20nth=20not=20throwing=20Index=20out=20of=20Range=20for?= =?UTF-8?q?=20empty=20ranges=20and=20lists?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pixie/stdlib.pxi | 35 +++++++++++++++++++------------ tests/pixie/tests/test-ith.pxi | 30 +++++++++++++------------- tests/pixie/tests/test-stdlib.pxi | 9 ++++++++ 3 files changed, 46 insertions(+), 28 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8e13060e..3e335a91 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1445,17 +1445,20 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn ith {:doc "Returns the ith element of the collection, negative values count from the end. - If an index is out of bounds, will throw an Index out of bounds exception" - :signatures [[coll i]] - :added "0.1"} - [coll i] - (if coll - (let [len (count coll) - idx (if (neg? i) (+ i len) i) - in-bounds? (and (< idx len) (>= idx 0))] - (if in-bounds? - (nth coll idx) - (throw "Index out of bounds"))))) + If an index is out of bounds, will throw an Index out of Range exception. + However, if you specify a not-found parameter, it will substitute that instead" + :signatures [[coll i] [coll idx not-found]] + :added "0.1"} + ([coll i] + (if coll + (let [len (count coll) + idx (if (neg? i) (+ i len) i)] + (nth coll idx)))) + ([coll i not-found] + (if coll + (let [len (count coll) + idx (if (neg? i) (+ i len) i)] + (nth coll idx not-found))))) (defn take {:doc "Takes n elements from the collection, or fewer, if not enough." @@ -1628,9 +1631,13 @@ For more information, see http://clojure.org/special_forms#binding-forms"} ~@body))) (extend -nth ISeq (fn [s n] + (when (empty? s) + (throw "Index out of Range")) (if (and (pos? n) s) (recur (next s) (dec n)) - (first s)))) + (if (zero? n) + (first s) + (throw "Index out of Range"))))) (extend -nth-not-found ISeq (fn [s n not-found] (if (and (pos? n) s) (recur (next s) (dec n) not-found) @@ -1674,11 +1681,13 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (abs (quot (- start stop) step)))) IIndexed (-nth [self idx] + (when (or (= start stop 0) (neg? idx)) + (throw "Index out of Range")) (let [cmp (if (< start stop) < >) val (+ start (* idx step))] (if (cmp val stop) val - nil))) + (throw "Index out of Range")))) (-nth-not-found [self idx not-found] (let [cmp (if (< start stop) < >) val (+ start (* idx step))] diff --git a/tests/pixie/tests/test-ith.pxi b/tests/pixie/tests/test-ith.pxi index f47f00fd..0591165c 100644 --- a/tests/pixie/tests/test-ith.pxi +++ b/tests/pixie/tests/test-ith.pxi @@ -15,26 +15,26 @@ (t/assert= (ith nil -1) nil)) (t/deftest test-ith-empty-always-oob - (t/assert= "Index out of bounds" (try (ith [] 0) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith [] 1) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith [] -1) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith '() 0) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith '() 1) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith '() -1) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith (range 0 0) 0) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith (range 0 0) 1) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith (range 0 0) -1) (catch e (ex-msg e))))) + (t/assert= "Index out of Range" (try (ith [] 0) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith [] 1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith [] -1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith '() 0) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith '() 1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith '() -1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith (range 0 0) 0) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith (range 0 0) 1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith (range 0 0) -1) (catch e (ex-msg e))))) (t/deftest test-ith-out-of-bounds (let [v [1 2 3 4 5] l '(1 2 3 4 5) r (range 1 6)] - (t/assert= "Index out of bounds" (try (ith v 5) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith l 5) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith r 5) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith v -6) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith l -6) (catch e (ex-msg e)))) - (t/assert= "Index out of bounds" (try (ith r -6) (catch e (ex-msg e)))))) + (t/assert= "Index out of Range" (try (ith v 5) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith l 5) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith r 5) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith v -6) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith l -6) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith r -6) (catch e (ex-msg e)))))) (t/deftest test-ith-doseq (let [v [1 2 3 4 5] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 2e8f8583..23d48153 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -76,6 +76,15 @@ (try (nth "hithere" 99) (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth [] 0) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth '() 0) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (range 0 0) 0) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) ;; if not-found is specified, uses that for out of range (t/assert= (nth [1 2 3] 99 :default) :default) From 586736ee1067e17274b2832b38fd42bdcfb649a0 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sun, 22 Feb 2015 14:26:53 -0500 Subject: [PATCH 526/909] dry up ith a bit --- pixie/stdlib.pxi | 10 ++++------ 1 file changed, 4 insertions(+), 6 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 3e335a91..fad9afc2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1450,14 +1450,12 @@ The new value is thus `(apply f current-value-of-atom args)`." :signatures [[coll i] [coll idx not-found]] :added "0.1"} ([coll i] - (if coll - (let [len (count coll) - idx (if (neg? i) (+ i len) i)] + (when coll + (let [idx (if (neg? i) (+ i (count coll)) i)] (nth coll idx)))) ([coll i not-found] - (if coll - (let [len (count coll) - idx (if (neg? i) (+ i len) i)] + (when coll + (let [idx (if (neg? i) (+ i (count coll)) i)] (nth coll idx not-found))))) (defn take From 9316fdc3ed243f988d5d1eb816290e9668c239dc Mon Sep 17 00:00:00 2001 From: gigasquid Date: Mon, 23 Feb 2015 08:15:19 -0500 Subject: [PATCH 527/909] Taking out the special build note for Macs since the issue is fixed --- README.md | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/README.md b/README.md index 16b829fa..02f0bc06 100644 --- a/README.md +++ b/README.md @@ -29,16 +29,6 @@ Some planned and implemented features: make build_with_jit ./pixie-vm -## Special Note for Macs - -If you are having trouble building or running the interpreter on Mac, check out this issue -[https://github.com/pixie-lang/pixie/issues/49](https://github.com/pixie-lang/pixie/issues/49). -In particular, try this: - -``` -PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' make build_with_jit -PKG_CONFIG_PATH='/usr/local/Cellar/libffi/3.0.13/lib/pkgconfig' make run_interactive -``` ## Running the tests From 2024c5657fc54869291a10987ef52f7ba2c78ad9 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 23 Feb 2015 06:24:27 -0700 Subject: [PATCH 528/909] switch back to python --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 7931f035..97dde21b 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=../externals -PYTHON ?= pypy +PYTHON ?= python PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack From d5ecb21cf78c47a9c39ae0980d687730f583ed8d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 23 Feb 2015 15:30:56 -0700 Subject: [PATCH 529/909] more sane exernals support --- .gitignore | 2 ++ Makefile | 20 ++++++++++---------- pixie/ffi-infer.pxi | 8 ++++++++ pixie/preload.pxi | 21 --------------------- pixie/vm/libs/pxic/writer.py | 2 +- 5 files changed, 21 insertions(+), 32 deletions(-) delete mode 100644 pixie/preload.pxi diff --git a/.gitignore b/.gitignore index 0e4825b4..391f400b 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,5 @@ externals pixie-vm .idea +lib +include diff --git a/Makefile b/Makefile index 97dde21b..c7a1e635 100644 --- a/Makefile +++ b/Makefile @@ -5,6 +5,7 @@ EXTERNALS=../externals PYTHON ?= python PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy + COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack JIT_OPTS?=--opt=jit TARGET_OPTS?=target.py @@ -17,9 +18,6 @@ help: @echo "make build_no_jit - build without jit" @echo "make fetch_externals - download and unpack external deps" -find_externals_name: - @PYTHONPATH=$(PYTHONPATH) $(PYTHON) find_externals_name.py - build_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py @@ -35,13 +33,12 @@ build_preload_no_jit: fetch_externals build: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) -fetch_externals: $(EXTERNALS)/pypy $(EXTERNALS)/binaries +fetch_externals: $(EXTERNALS)/pypy libs -$(EXTERNALS)/binaries: - mkdir $(EXTERNALS); \ +libs: echo https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 - curl -L https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 > $(EXTERNALS)/externals.tar.bz2 - cd $(EXTERNALS) && tar -jxf externals.tar.bz2 + curl -L https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 > /tmp/externals.tar.bz2 + tar -jxf /tmp/externals.tar.bz2 --strip-components=2 $(EXTERNALS)/pypy: @@ -56,10 +53,10 @@ run: ./pixie-vm run_interactive: - @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py -l ../externals/externals/lib + @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run_built_tests: pixie-vm - ./pixie-vm -l ../externals/externals/lib -l ../externals/externals/include run-tests.pxi + ./pixie-vm run-tests.pxi run_interpreted_tests: target.py PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.pxi @@ -67,5 +64,8 @@ run_interpreted_tests: target.py compile_tests: find "tests" -name "*.pxi" | xargs -L1 ./pixie-vm -l "tests" -c +compile_src: + find * -name "*.pxi" | grep "^pixie/" | xargs -L1 ./pixie-vm $(EXTERNALS_FLAGS) -c + clean_pxic: find * -name "*.pxic" | xargs rm diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index b797814b..56df06ac 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -1,6 +1,14 @@ (ns pixie.ffi-infer (require pixie.io :as io)) +(defn -add-rel-path [rel] + (println (first @load-paths) "/" rel) + (swap! load-paths conj (str (first @load-paths) "/" rel))) + +(-add-rel-path "lib") +(-add-rel-path "include") + + (def *config* nil) (set-dynamic! (var *config*)) (def *bodies* nil) diff --git a/pixie/preload.pxi b/pixie/preload.pxi deleted file mode 100644 index 55abe180..00000000 --- a/pixie/preload.pxi +++ /dev/null @@ -1,21 +0,0 @@ -(ns pixie.preload - (require pixie.string :as s)) - -(defn load-all-pxi-files - "Only used by preloading, loads all found .pxi files" - [] - (println "Looking for pxi files...") - (foreach [path @load-paths] - (println "Looking for files in:" path) - (foreach [desc (pixie.path/file-list path)] - (if (= (nth desc 1) :file) - (let [filename (nth desc 2)] - (if (and (pixie.string/ends-with filename ".pxi") - (not (= filename "preload.pxi")) - (not (= filename "stdlib.pxi"))) - (if (pixie.string/starts-with (nth desc 0) "./pixie") - (let [fullpath (str (nth desc 0) "/" filename)] - (println "Loading" fullpath) - (load-file fullpath))))))))) - -(load-all-pxi-files) \ No newline at end of file diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index eb251f9f..9b3e806d 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -12,7 +12,7 @@ from rpython.rlib.rarithmetic import r_uint import pixie.vm.rt as rt -MAX_INT32 = r_uint(1 << 32) +MAX_INT32 = r_uint(1 << 31) class Writer(object): def __init__(self, wtr, with_cache=False): From 43acf949098eec1e65118025607980505fce6166 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 23 Feb 2015 15:32:05 -0700 Subject: [PATCH 530/909] more default paths, useful for when pixie is installed --- pixie/ffi-infer.pxi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 56df06ac..49578c20 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -2,11 +2,12 @@ (require pixie.io :as io)) (defn -add-rel-path [rel] - (println (first @load-paths) "/" rel) (swap! load-paths conj (str (first @load-paths) "/" rel))) (-add-rel-path "lib") (-add-rel-path "include") +(-add-rel-path "../lib") +(-add-rel-path "../include") (def *config* nil) From 991951fcc359d72117255ed0589c19c55c526f6b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 23 Feb 2015 16:15:47 -0700 Subject: [PATCH 531/909] only download libs once --- Makefile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index c7a1e635..bb5cda71 100644 --- a/Makefile +++ b/Makefile @@ -33,9 +33,9 @@ build_preload_no_jit: fetch_externals build: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) -fetch_externals: $(EXTERNALS)/pypy libs +fetch_externals: $(EXTERNALS)/pypy ./lib -libs: +lib: echo https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 curl -L https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 > /tmp/externals.tar.bz2 tar -jxf /tmp/externals.tar.bz2 --strip-components=2 From 7a69e13ca90bf101b314e3fc2eb815ce253e675b Mon Sep 17 00:00:00 2001 From: gigasquid Date: Mon, 23 Feb 2015 19:48:48 -0500 Subject: [PATCH 532/909] print and seq working --- pixie/vm/stdlib.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 26a2680a..f54f8e56 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -774,8 +774,23 @@ def _str(self): assert isinstance(self, RuntimeException) return rt.wrap(self.__repr__()) +@extend(_seq, RuntimeException) +def _seq(self): + import pixie.vm.persistent_vector as vector + assert isinstance(self, RuntimeException) + trace = vector.EMPTY + trace = rt.conj(trace, self._data) + for x in self._trace: + trace = rt.conj(trace, rt.wrap(x.__repr__())) + return rt._seq(trace) + @as_var("ex-msg") def ex_msg(e): assert isinstance(e, RuntimeException) return e._data +@as_var("ex-pr") +def ex_pr(e): + assert isinstance(e, RuntimeException) + print e.__repr__() + return nil From cceb54fe34946fcea17763726d84f0dc72bd3004 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Mon, 23 Feb 2015 20:02:52 -0500 Subject: [PATCH 533/909] pst and trace --- pixie/stdlib.pxi | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 400a66eb..39b3cddf 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2100,7 +2100,6 @@ Expands to calls to `extend-type`." ([sb] (str sb)) ([sb item] (conj! sb item))) - (defmacro using [bindings & body] (let [pairs (partition 2 bindings) names (map first pairs)] @@ -2110,3 +2109,16 @@ Expands to calls to `extend-type`." `(-dispose! ~nm)) names) result#))) +(defn pst + {:doc "Prints the trace of a Runtime Exception if given, or the last Runtime Exception in *e" + :signatures [[] [e]] + :added "0.1"} + ([] (pst *e)) + ([e] (when *e (ex-pr *e)))) + +(defn trace + {:doc "Returns a seq of the trace of a Runtime Exception or the last Runtime Exception in *e" + :signatures [[] [e]] + :added "0.1"} + ([] (trace *e)) + ([e] (seq e))) From ae5900fe2016e7004c36bc48616468964c7699e9 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Mon, 23 Feb 2015 20:16:36 -0500 Subject: [PATCH 534/909] test for trace --- tests/pixie/tests/test-stdlib.pxi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 23d48153..6e8e6485 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -377,3 +377,11 @@ (t/assert= (transduce (drop-while even?) conj [2 4 6 7 8]) [7 8]) (t/assert= (transduce (drop-while even?) conj [0 2] [1 4 6]) [0 2 1 4 6]) (t/assert= (transduce (drop-while even?) conj [0 2] [2 4 6 7 8]) [0 2 7 8])) + + +(t/deftest test-trace + (try + (/ 0 0) + (catch e + (t/assert= (first (trace e)) "Divide by zero") + (t/assert= (second (trace e)) "in internal function _div\n")))) From 320216031d83bb5bb262bd1651bd37942119dbb5 Mon Sep 17 00:00:00 2001 From: Kyle Gann Date: Mon, 23 Feb 2015 20:31:48 -0500 Subject: [PATCH 535/909] implement pixie.string/blank? --- pixie/string.pxi | 14 ++++++++++++++ tests/pixie/tests/test-strings.pxi | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/pixie/string.pxi b/pixie/string.pxi index 574de008..5c212fcd 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -49,3 +49,17 @@ (nil? s) res (nil? (next s)) (str res (first s)) :else (recur (next s) (str res (first s) seperator)))))) + +(defn blank? + "True if s is nil, empty, or contains only whitespace." + [s] + (if s + (let [white #{\space \newline \tab \backspace \formfeed \return} + length (count s)] + (loop [index 0] + (if (= length index) + true + (if (white (nth s index)) + (recur (inc index)) + false)))) + true)) diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 2f88965b..11dc17c6 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -134,3 +134,10 @@ (t/deftest test-unicode (t/assert= "hâllo" "hâllo")) + +(t/deftest test-blank? + (t/assert= (s/blank? nil) true) + (t/assert= (s/blank? "") true) + (t/assert= (s/blank? " ") true) + (t/assert= (s/blank? " \t \n \r ") true) + (t/assert= (s/blank? " foo ") false)) From 57e9842aca453407e91faa259d2f24dcdd39b689 Mon Sep 17 00:00:00 2001 From: Kyle Gann Date: Mon, 23 Feb 2015 20:39:08 -0500 Subject: [PATCH 536/909] rename pixie.string/starts-with and pixie.string/ends-with --- pixie/fs.pxi | 2 +- pixie/stdlib.pxi | 2 +- pixie/string.pxi | 4 ++-- pixie/test.pxi | 2 +- tests/pixie/tests/test-strings.pxi | 28 ++++++++++++++-------------- 5 files changed, 19 insertions(+), 19 deletions(-) diff --git a/pixie/fs.pxi b/pixie/fs.pxi index a2235d17..7be65fda 100644 --- a/pixie/fs.pxi +++ b/pixie/fs.pxi @@ -106,7 +106,7 @@ (last (string/split (abs this) "."))) (extension? [this ext] - (string/ends-with (abs this) ext)) + (string/ends-with? (abs this) ext)) IObject (-hash [this] diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 400a66eb..85a42d77 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1850,7 +1850,7 @@ Supported filters: :only same as refer user => (refer 'pixie.string :refer :all) -user => (refer 'pixie.string :only '(index-of starts-with ends-with)) +user => (refer 'pixie.string :only '(index-of starts-with? ends-with?)) user => (refer 'pixie.string :rename '{index-of find}) user => (refer 'pixie.string :exclude '(substring))" :added "0.1"} diff --git a/pixie/string.pxi b/pixie/string.pxi index 5c212fcd..48b2f8dd 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -6,8 +6,8 @@ (def index-of si/index-of) (def split si/split) -(def ends-with si/ends-with) -(def starts-with si/starts-with) +(def ends-with? si/ends-with) +(def starts-with? si/starts-with) (def trim si/trim) (def triml si/triml) diff --git a/pixie/test.pxi b/pixie/test.pxi index 6fcab126..41c14914 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -49,7 +49,7 @@ pxi-files (->> dirs (mapcat fs/walk-files) (filter #(fs/extension? % "pxi")) - (filter #(s/starts-with (fs/basename %) "test-")) + (filter #(s/starts-with? (fs/basename %) "test-")) (distinct))] (foreach [file pxi-files] (println "Loading " file) diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 11dc17c6..248d8ca2 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -2,25 +2,25 @@ (require pixie.test :as t) (require pixie.string :as s)) -(t/deftest test-starts-with +(t/deftest test-starts-with? (let [s "heyhohuh"] - (t/assert= (s/starts-with s "") true) - (t/assert= (s/starts-with s "hey") true) - (t/assert= (s/starts-with s "heyho") true) - (t/assert= (s/starts-with s s) true) + (t/assert= (s/starts-with? s "") true) + (t/assert= (s/starts-with? s "hey") true) + (t/assert= (s/starts-with? s "heyho") true) + (t/assert= (s/starts-with? s s) true) - (t/assert= (s/starts-with s "ho") false) - (t/assert= (s/starts-with s "foo") false))) + (t/assert= (s/starts-with? s "ho") false) + (t/assert= (s/starts-with? s "foo") false))) -(t/deftest test-ends-with +(t/deftest test-ends-with? (let [s "heyhohuh"] - (t/assert= (s/ends-with s "") true) - (t/assert= (s/ends-with s "huh") true) - (t/assert= (s/ends-with s "hohuh") true) - (t/assert= (s/ends-with s s) true) + (t/assert= (s/ends-with? s "") true) + (t/assert= (s/ends-with? s "huh") true) + (t/assert= (s/ends-with? s "hohuh") true) + (t/assert= (s/ends-with? s s) true) - (t/assert= (s/ends-with s "hey") false) - (t/assert= (s/ends-with s "foo") false))) + (t/assert= (s/ends-with? s "hey") false) + (t/assert= (s/ends-with? s "foo") false))) (t/deftest test-split (let [s "hey,ho,huh"] From 6be5e5679442d6d2769e5a844a8a626252d3edad Mon Sep 17 00:00:00 2001 From: gigasquid Date: Mon, 23 Feb 2015 21:18:19 -0500 Subject: [PATCH 537/909] fix *e --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 39b3cddf..3eab081e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2114,7 +2114,7 @@ Expands to calls to `extend-type`." :signatures [[] [e]] :added "0.1"} ([] (pst *e)) - ([e] (when *e (ex-pr *e)))) + ([e] (when e (ex-pr e)))) (defn trace {:doc "Returns a seq of the trace of a Runtime Exception or the last Runtime Exception in *e" From 2a51e157c53a77cc8ae45687373bb7c3dfc54221 Mon Sep 17 00:00:00 2001 From: Kyle Gann Date: Tue, 24 Feb 2015 00:01:20 -0500 Subject: [PATCH 538/909] implement pixie.io/line-seq and pixie.io/read-line --- pixie/io.pxi | 20 ++++++++++++++++++++ tests/pixie/tests/test-io.pxi | 15 +++++++++++++-- tests/pixie/tests/test-io.txt | 1 + 3 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 51e6d3e7..bbc252b2 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -12,6 +12,7 @@ (defprotocol IInputStream (read-byte [this] "Read a single character") + (read-line [this] "Read a single line") (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) (defprotocol IOutputStream @@ -33,6 +34,18 @@ read-count)) (read-byte [this] (fgetc buffer)) + (read-line [this] + (let [line-feed (into #{} (map int [\newline \return])) + buf (buffer 1)] + (loop [acc []] + (let [len (read this buf 1)] + (cond + (and (pos? len) (not (line-feed (first buf)))) + (recur (conj acc (first buf))) + + (zero? len) nil + + :else (apply str (map char acc))))))) IClosable (close [this] (fclose fp)) @@ -58,6 +71,13 @@ (assert (string? filename) "Filename must be a string") (->FileStream (fopen filename "r"))) +(defn line-seq + "Returns the lines of text from file-stream as a lazy sequence of strings. + file-stream must implement IInputStream - specifically read-line." + [file-stream] + (when-let [line (read-line file-stream)] + (cons line (lazy-seq (line-seq file-stream))))) + (deftype FileOutputStream [fp] IOutputStream (write-byte [this val] diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 4c81ab0a..36ffa040 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -7,14 +7,25 @@ (t/assert= (transduce (map identity) count-rf f) - 78))) + 91))) (t/deftest test-process-reduction (let [f (io/run-command "ls tests/pixie/tests/test-io.txt")] (t/assert= f "tests/pixie/tests/test-io.txt\n"))) +(t/deftest test-read-line + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (io/read-line f) + (t/assert= (io/read-line f) "Second line.") + (t/assert= (io/read-line f) nil))) + +(t/deftest test-line-seq + (let [f (io/open-read "tests/pixie/tests/test-io.txt") + s (io/line-seq f)] + (t/assert= (last s) "Second line."))) + (comment (t/deftest test-slurp-spit (let [val (vec (range 128))] (t/assert= val (read-string (io/slurp "test.tmp" (io/spit "test.tmp" val)))))) -) \ No newline at end of file +) diff --git a/tests/pixie/tests/test-io.txt b/tests/pixie/tests/test-io.txt index c5187b04..ce2e9b4d 100644 --- a/tests/pixie/tests/test-io.txt +++ b/tests/pixie/tests/test-io.txt @@ -1 +1,2 @@ This is a test file used in testing the io routines. Please do not remove it. +Second line. From 1849c62b2271a608f07020f10dfde1a023782275 Mon Sep 17 00:00:00 2001 From: Kyle Gann Date: Tue, 24 Feb 2015 08:25:47 -0500 Subject: [PATCH 539/909] Remove read-line from protocol and implement as function of IInputStream --- pixie/io.pxi | 41 +++++++++++++++++++++-------------------- 1 file changed, 21 insertions(+), 20 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index bbc252b2..46524dbf 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -12,7 +12,6 @@ (defprotocol IInputStream (read-byte [this] "Read a single character") - (read-line [this] "Read a single line") (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) (defprotocol IOutputStream @@ -34,18 +33,6 @@ read-count)) (read-byte [this] (fgetc buffer)) - (read-line [this] - (let [line-feed (into #{} (map int [\newline \return])) - buf (buffer 1)] - (loop [acc []] - (let [len (read this buf 1)] - (cond - (and (pos? len) (not (line-feed (first buf)))) - (recur (conj acc (first buf))) - - (zero? len) nil - - :else (apply str (map char acc))))))) IClosable (close [this] (fclose fp)) @@ -62,8 +49,6 @@ @result)) acc)))))) - - (defn open-read {:doc "Open a file for reading, returning a IInputStream" :added "0.1"} @@ -71,12 +56,28 @@ (assert (string? filename) "Filename must be a string") (->FileStream (fopen filename "r"))) +(defn read-line + "Read one line from input-stream for each invocation. + nil when all lines have been read" + [input-stream] + (let [line-feed (into #{} (map int [\newline \return])) + buf (buffer 1)] + (loop [acc []] + (let [len (read input-stream buf 1)] + (cond + (and (pos? len) (not (line-feed (first buf)))) + (recur (conj acc (first buf))) + + (and (zero? len) (empty? acc)) nil + + :else (apply str (map char acc))))))) + (defn line-seq - "Returns the lines of text from file-stream as a lazy sequence of strings. - file-stream must implement IInputStream - specifically read-line." - [file-stream] - (when-let [line (read-line file-stream)] - (cons line (lazy-seq (line-seq file-stream))))) + "Returns the lines of text from input-stream as a lazy sequence of strings. + input-stream must implement IInputStream" + [input-stream] + (when-let [line (read-line input-stream)] + (cons line (lazy-seq (line-seq input-stream))))) (deftype FileOutputStream [fp] IOutputStream From 71f706cc05f2ccc545020c316c189a4f28219f1a Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 24 Feb 2015 19:27:33 -0500 Subject: [PATCH 540/909] map trace format --- pixie/vm/object.py | 25 +++++++++++++++++++++++-- pixie/vm/stdlib.py | 15 +++++++++++++-- 2 files changed, 36 insertions(+), 4 deletions(-) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index e6c39441..4491501f 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -122,7 +122,6 @@ def __repr__(self): s.extend([u"RuntimeException: " + rt.name(rt.str(self._data)) + u"\n"]) - return u"".join(s) class WrappedException(Exception): @@ -176,6 +175,14 @@ def __repr__(self): def interpreter_code_info_state(self): return self._line, self._line_number, self._column_number, self._file + def trace_map(self): + tm = {u"type" : u"interpreter"} + tm[u"line"] = self._line.__repr__() + tm[u"line_number"] = unicode(str(self._line_number)) + tm[u"column_number"] = unicode(str(self._column_number)) + tm[u"file"] = self._file + return tm + class NativeCodeInfo(ErrorInfo): def __init__(self, name): self._name = name @@ -183,6 +190,11 @@ def __init__(self, name): def __repr__(self): return u"in internal function " + self._name + u"\n" + def trace_map(self): + tm = {u"type" : u"native"} + tm[u"name"] = self._name + return tm + class PolymorphicCodeInfo(ErrorInfo): def __init__(self, name, tp): self._name = name @@ -193,7 +205,11 @@ def __repr__(self): assert isinstance(tp, Type) return u"in polymorphic function " + self._name + u" dispatching on " + tp._name + u"\n" - + def trace_map(self): + tm = {u"type" : u"polymorphic"} + tm[u"name"] = self._name + tm[u"tp"] = self._tp + return tm class PixieCodeInfo(ErrorInfo): def __init__(self, name): @@ -201,3 +217,8 @@ def __init__(self, name): def __repr__(self): return u"in pixie function " + self._name + u"\n" + + def trace_map(self): + tm = {u"type" : u"pixie"} + tm[u"name"] = self._name + return tm diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index f54f8e56..7cffe5a4 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -777,11 +777,22 @@ def _str(self): @extend(_seq, RuntimeException) def _seq(self): import pixie.vm.persistent_vector as vector + import pixie.vm.persistent_hash_map as hmap + from pixie.vm.keyword import keyword assert isinstance(self, RuntimeException) trace = vector.EMPTY - trace = rt.conj(trace, self._data) + trace_element = hmap.EMPTY + trace_element = rt.assoc(trace_element, keyword(u"type"), keyword(u"runtime")) + trace_element = rt.assoc(trace_element, keyword(u"data"), rt.wrap(self._data)) + trace = rt.conj(trace, trace_element) for x in self._trace: - trace = rt.conj(trace, rt.wrap(x.__repr__())) + trace_element = hmap.EMPTY + tmap = x.trace_map() + for key in tmap: + trace_element = rt.assoc(trace_element, keyword(key), rt.wrap(tmap[key])) + + trace = rt.conj(trace, trace_element) + return rt._seq(trace) @as_var("ex-msg") From ec432ccc15fd4459c8a3055d3df3463ea4d5c09d Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 24 Feb 2015 20:45:41 -0500 Subject: [PATCH 541/909] fix for hit and tests --- pixie/vm/object.py | 4 +++- tests/pixie/tests/test-stdlib.pxi | 4 ++-- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 4491501f..22f1c414 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -206,9 +206,11 @@ def __repr__(self): return u"in polymorphic function " + self._name + u" dispatching on " + tp._name + u"\n" def trace_map(self): + tp = self._tp + assert isinstance(tp, Type) tm = {u"type" : u"polymorphic"} tm[u"name"] = self._name - tm[u"tp"] = self._tp + tm[u"tp"] = tp._name return tm class PixieCodeInfo(ErrorInfo): diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 6e8e6485..3092d844 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -383,5 +383,5 @@ (try (/ 0 0) (catch e - (t/assert= (first (trace e)) "Divide by zero") - (t/assert= (second (trace e)) "in internal function _div\n")))) + (t/assert= (first (trace e)) {:data "Divide by zero", :type :runtime}) + (t/assert= (second (trace e)) {:type "native", :name "_div"} )))) From f73eb94bc165db2045c2065a80fe634efc0d4d6a Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 24 Feb 2015 21:10:20 -0500 Subject: [PATCH 542/909] unkeywording first type --- pixie/vm/stdlib.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 7cffe5a4..04bb2397 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -782,7 +782,7 @@ def _seq(self): assert isinstance(self, RuntimeException) trace = vector.EMPTY trace_element = hmap.EMPTY - trace_element = rt.assoc(trace_element, keyword(u"type"), keyword(u"runtime")) + trace_element = rt.assoc(trace_element, keyword(u"type"), u"runtime") trace_element = rt.assoc(trace_element, keyword(u"data"), rt.wrap(self._data)) trace = rt.conj(trace, trace_element) for x in self._trace: From cbba3bbb7477550cbc70e9f9e28de02587b1b77a Mon Sep 17 00:00:00 2001 From: gigasquid Date: Tue, 24 Feb 2015 21:16:49 -0500 Subject: [PATCH 543/909] change type val to string to make it uniform --- pixie/vm/stdlib.py | 2 +- tests/pixie/tests/test-stdlib.pxi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 04bb2397..e5bdbd3f 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -782,7 +782,7 @@ def _seq(self): assert isinstance(self, RuntimeException) trace = vector.EMPTY trace_element = hmap.EMPTY - trace_element = rt.assoc(trace_element, keyword(u"type"), u"runtime") + trace_element = rt.assoc(trace_element, keyword(u"type"), rt.wrap(u"runtime")) trace_element = rt.assoc(trace_element, keyword(u"data"), rt.wrap(self._data)) trace = rt.conj(trace, trace_element) for x in self._trace: diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 3092d844..012cfd81 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -383,5 +383,5 @@ (try (/ 0 0) (catch e - (t/assert= (first (trace e)) {:data "Divide by zero", :type :runtime}) + (t/assert= (first (trace e)) {:data "Divide by zero", :type "runtime"}) (t/assert= (second (trace e)) {:type "native", :name "_div"} )))) From 301c09288a753f5fd9b0e61ee387e8791848dfa3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 25 Feb 2015 18:39:46 +0100 Subject: [PATCH 544/909] return nil from print, println, pr and prn the previous return value (how many characters were printed) was an implementation detail and possibly a bit confusing. returning nil is also what clojure does, although very likely for different reasons. --- pixie/stdlib.pxi | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 85a42d77..eb69fc64 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1217,13 +1217,15 @@ and implements IAssociative, ILookup and IObject." {:doc "Prints the arguments, seperated by spaces." :added "0.1"} [& args] - (printf (transduce (interpose " ") str args))) + (printf (transduce (interpose " ") str args)) + nil) (defn println {:doc "Prints the arguments, separated by spaces, with a newline at the end." :added "0.1"} [& args] - (puts (transduce (interpose " ") str args))) + (puts (transduce (interpose " ") str args)) + nil) (defn pr-str {:doc "Formats the arguments using -repr, separated by spaces, returning a string." @@ -1235,13 +1237,15 @@ and implements IAssociative, ILookup and IObject." {:doc "Prints the arguments using -repr, separated by spaces." :added "0.1"} [& args] - (printf (apply pr-str args))) + (printf (apply pr-str args)) + nil) (defn prn {:doc "Prints the arguments using -repr, separated by spaces, with a newline at the end." :added "0.1"} [& args] - (puts (apply pr-str args))) + (puts (apply pr-str args)) + nil) (defn repeat ([x] From b6391d9d579f863cd151769abb830511ca4f0f23 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 25 Feb 2015 17:05:25 -0700 Subject: [PATCH 545/909] first commit of re-enabling stacklets --- Makefile | 4 +-- pixie/stacklets.pxi | 81 ++++++++++++++++++++++++++++++++++++++++++++ pixie/stdlib.pxi | 5 ++- pixie/vm/rt.py | 1 + pixie/vm/stacklet.py | 56 ++++++++++++++++++++++++++++++ target.py | 3 ++ 6 files changed, 147 insertions(+), 3 deletions(-) create mode 100644 pixie/stacklets.pxi create mode 100644 pixie/vm/stacklet.py diff --git a/Makefile b/Makefile index bb5cda71..e8f1c0a4 100644 --- a/Makefile +++ b/Makefile @@ -2,11 +2,11 @@ all: help EXTERNALS=../externals -PYTHON ?= python +PYTHON ?= pypy PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy -COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack +COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack --continuation JIT_OPTS?=--opt=jit TARGET_OPTS?=target.py diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi new file mode 100644 index 00000000..e8d75da4 --- /dev/null +++ b/pixie/stacklets.pxi @@ -0,0 +1,81 @@ +(ns pixie.stacklets + (require pixie.uv :as uv)) + +(def stacklet-loop-h (atom nil)) + +(defn -spawn-thread [fn] + (let [[h val] (@stacklet-loop-h [:spawn fn])] + (reset! stacklet-loop-h h) + (println h val) + val)) + +(defn enqueue [q itm] + ; TODO: Rewrite this crappy impl + (vec (conj (seq q) itm))) + +(defn dequeue [q] + [(ith q -1) + (pop q)]) + + +(defn yield-control [] + (let [[h] (@stacklet-loop-h [:yield nil])] + (reset! stacklet-loop-h h) + nil)) + + +(defmacro spawn [& body] + `(let [f (fn [h# _] + (reset! stacklet-loop-h h#) + (try + (println "spawn" h#) + ;(reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (println "returning " result#) + (@stacklet-loop-h [:spawn-end result#])) + (catch e (println e)))) + [h# val#] (@stacklet-loop-h [:spawn f])] + (reset! stacklet-loop-h h#) + val#)) + +(defn -with-stacklets [fn] + (let [[h [op arg]] ((new-stacklet fn) nil)] + (println h op arg) + (loop [op op + arg arg + this_h h + tasks []] + (println "in loop" op arg this_h tasks) + (cond + (= op :spawn) (let [wh (new-stacklet arg) + [h [op arg]] (this_h nil)] + (recur op arg h (enqueue tasks wh))) + (= op :yield) (let [tasks (enqueue tasks this_h) + [task tasks] (dequeue tasks) + [h [op arg]] (task nil)] + (recur op arg h tasks)) + (= op :spawn-end) (if (empty? tasks) + arg + (let [[task tasks] (dequeue tasks) + [h [op arg]] (task nil)] + (recur op arg h tasks))) + :else (assert false (str "Unkown command " op " " arg)))))) + + +(defmacro with-stacklets [& body] + `(-with-stacklets + (fn [h# _] + (try + (println h# _) + (reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (@stacklet-loop-h [:spawn-end result#])) + (catch e + (println e)))))) + +(with-stacklets (spawn (dotimes [x 10] + (yield-control) + (println x))) + (spawn (dotimes [x 10] + (yield-control) + (println x)))(yield-control)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index eb69fc64..30ab4933 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1630,7 +1630,10 @@ All these forms can be combined and nested, in the example below: For more information, see http://clojure.org/special_forms#binding-forms"} [bindings & body] - (let* [destructured-bindings (transduce (map #(apply destructure %1)) + (let* [destructured-bindings (transduce (map (fn [args] + (assert (= 2 (count args)) (str "Bindings must be in pairs, not " args + " " (meta (first args)))) + (apply destructure args))) concat [] (partition 2 bindings))] diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 7eb65b33..d9ff37ed 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -68,6 +68,7 @@ def wrapper(*args): import pixie.vm.libs.string import pixie.vm.threads import pixie.vm.string_builder + import pixie.vm.stacklet numbers.init() diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py new file mode 100644 index 00000000..da42cfc0 --- /dev/null +++ b/pixie/vm/stacklet.py @@ -0,0 +1,56 @@ +import rpython.rlib.rstacklet as rstacklet +from pixie.vm.object import Object, Type, affirm +from pixie.vm.code import as_var +import pixie.vm.rt as rt + + +class GlobalState(object): + def __init__(self): + self._is_inited = False + self._val = None + + +global_state = GlobalState() + + +def init(): + if not global_state._is_inited: + global_state._th = rstacklet.StackletThread(rt.__config__) + global_state._is_inited = True + + +class StackletHandle(Object): + _type = Type(u"StackletHandle") + def __init__(self, h): + self._stacklet_handle = h + self._used = False + + def type(self): + return self._type + + def invoke(self, args): + affirm(not self._used, u"Can only call a given stacklet handle once.") + affirm(len(args) == 1, u"Only one arg should be handed to a stacklet handle") + self._used = True + global_state._val = args[0] + new_h = StackletHandle(global_state._th.switch(self._stacklet_handle)) + val = global_state._val + global_state._val = None + return rt.vector(new_h, val) + +def new_handler(h, _): + fn = global_state._val + global_state._val = None + h = global_state._th.switch(h) + val = global_state._val + fn.invoke([StackletHandle(h), val]) + affirm(False, u"TODO: What do we do now?") + return h + + + +@as_var("new-stacklet") +def new_stacklet(fn): + global_state._val = fn + h = global_state._th.new(new_handler) + return StackletHandle(h) \ No newline at end of file diff --git a/target.py b/target.py index a91a940b..c1726876 100644 --- a/target.py +++ b/target.py @@ -185,6 +185,9 @@ def load_stdlib(): def entry_point(args): try: + import pixie.vm.stacklet + pixie.vm.stacklet.init() + interactive = True exit = False script_args = [] From 3783b23bc392cd97249233a8ee3bea9cb40507b6 Mon Sep 17 00:00:00 2001 From: gigasquid Date: Thu, 26 Feb 2015 09:28:20 -0500 Subject: [PATCH 546/909] Return ints for line numbers and refactor --- pixie/stdlib.pxi | 2 +- pixie/vm/object.py | 40 +++++++++++++++++++++---------- pixie/vm/rt.py | 2 +- pixie/vm/stdlib.py | 14 ++++------- tests/pixie/tests/test-stdlib.pxi | 4 ++-- 5 files changed, 36 insertions(+), 26 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 3eab081e..e7db6f32 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2114,7 +2114,7 @@ Expands to calls to `extend-type`." :signatures [[] [e]] :added "0.1"} ([] (pst *e)) - ([e] (when e (ex-pr e)))) + ([e] (when e (print (str e))))) (defn trace {:doc "Returns a seq of the trace of a Runtime Exception or the last Runtime Exception in *e" diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 22f1c414..defec587 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -176,11 +176,15 @@ def interpreter_code_info_state(self): return self._line, self._line_number, self._column_number, self._file def trace_map(self): - tm = {u"type" : u"interpreter"} - tm[u"line"] = self._line.__repr__() - tm[u"line_number"] = unicode(str(self._line_number)) - tm[u"column_number"] = unicode(str(self._column_number)) - tm[u"file"] = self._file + from pixie.vm.string import String + from pixie.vm.numbers import Integer + from pixie.vm.keyword import keyword + + tm = {keyword(u"type") : keyword(u"interpreter")} + tm[keyword(u"line")] = String(self._line.__repr__()) + tm[keyword(u"line-number")] = Integer(self._line_number) + tm[keyword(u"column-number")] = Integer(self._column_number) + tm[keyword(u"file")] = String(self._file) return tm class NativeCodeInfo(ErrorInfo): @@ -191,8 +195,12 @@ def __repr__(self): return u"in internal function " + self._name + u"\n" def trace_map(self): - tm = {u"type" : u"native"} - tm[u"name"] = self._name + from pixie.vm.string import String + from pixie.vm.numbers import Integer + from pixie.vm.keyword import keyword + + tm = {keyword(u"type") : keyword(u"native")} + tm[keyword(u"name")] = String(self._name) return tm class PolymorphicCodeInfo(ErrorInfo): @@ -206,11 +214,15 @@ def __repr__(self): return u"in polymorphic function " + self._name + u" dispatching on " + tp._name + u"\n" def trace_map(self): + from pixie.vm.string import String + from pixie.vm.numbers import Integer + from pixie.vm.keyword import keyword + tp = self._tp assert isinstance(tp, Type) - tm = {u"type" : u"polymorphic"} - tm[u"name"] = self._name - tm[u"tp"] = tp._name + tm = {keyword(u"type") : keyword(u"polymorphic")} + tm[keyword(u"name")] = String(self._name) + tm[keyword(u"tp")] = String(tp._name) return tm class PixieCodeInfo(ErrorInfo): @@ -221,6 +233,10 @@ def __repr__(self): return u"in pixie function " + self._name + u"\n" def trace_map(self): - tm = {u"type" : u"pixie"} - tm[u"name"] = self._name + from pixie.vm.string import String + from pixie.vm.numbers import Integer + from pixie.vm.keyword import keyword + + tm = {keyword(u"type") : keyword(u"pixie")} + tm[keyword(u"name")] = String(self._name) return tm diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 7eb65b33..20cc157c 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -143,7 +143,7 @@ def reinit(): # stacklet.with_stacklets(run_load_stdlib) init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file", u"compile-file", - u"load-ns"] + u"load-ns", u"hashmap"] for x in init_fns: globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index e5bdbd3f..ba6f186b 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -781,15 +781,15 @@ def _seq(self): from pixie.vm.keyword import keyword assert isinstance(self, RuntimeException) trace = vector.EMPTY - trace_element = hmap.EMPTY - trace_element = rt.assoc(trace_element, keyword(u"type"), rt.wrap(u"runtime")) + trace_element = rt.hashmap(keyword(u"type"), keyword(u"runtime")) trace_element = rt.assoc(trace_element, keyword(u"data"), rt.wrap(self._data)) trace = rt.conj(trace, trace_element) for x in self._trace: - trace_element = hmap.EMPTY tmap = x.trace_map() + trace_element = hmap.EMPTY for key in tmap: - trace_element = rt.assoc(trace_element, keyword(key), rt.wrap(tmap[key])) + val = tmap[key] + trace_element = rt.assoc(trace_element, key, val) trace = rt.conj(trace, trace_element) @@ -799,9 +799,3 @@ def _seq(self): def ex_msg(e): assert isinstance(e, RuntimeException) return e._data - -@as_var("ex-pr") -def ex_pr(e): - assert isinstance(e, RuntimeException) - print e.__repr__() - return nil diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 012cfd81..84fc2858 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -383,5 +383,5 @@ (try (/ 0 0) (catch e - (t/assert= (first (trace e)) {:data "Divide by zero", :type "runtime"}) - (t/assert= (second (trace e)) {:type "native", :name "_div"} )))) + (t/assert= (first (trace e)) {:type :runtime :data "Divide by zero"}) + (t/assert= (second (trace e)) {:type :native :name "_div"} )))) From 358cb86dc21d57714c0c719e227e02047888784d Mon Sep 17 00:00:00 2001 From: Kyle Gann Date: Thu, 26 Feb 2015 15:58:29 -0500 Subject: [PATCH 547/909] implement pixie.stdlib/tree-seq --- pixie/stdlib.pxi | 13 +++++++++++++ tests/pixie/tests/test-stdlib.pxi | 10 +++++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 113be1af..bd1246fd 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2126,3 +2126,16 @@ Expands to calls to `extend-type`." :added "0.1"} ([] (trace *e)) ([e] (seq e))) + +(defn tree-seq + "Returns a lazy sequence of the nodes in a tree via a depth-first walk. + branch? - fn of node that should true when node has children + children - fn of node that should return a sequence of children (called if branch? true) + root - root node of the tree" + [branch? children root] + (let [walk (fn walk [node] + (lazy-seq + (cons node + (when (branch? node) + (mapcat walk (children node))))))] + (walk root))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 84fc2858..4ed20017 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -378,10 +378,18 @@ (t/assert= (transduce (drop-while even?) conj [0 2] [1 4 6]) [0 2 1 4 6]) (t/assert= (transduce (drop-while even?) conj [0 2] [2 4 6 7 8]) [0 2 7 8])) - (t/deftest test-trace (try (/ 0 0) (catch e (t/assert= (first (trace e)) {:type :runtime :data "Divide by zero"}) (t/assert= (second (trace e)) {:type :native :name "_div"} )))) + +(t/deftest test-tree-seq + (t/assert= (vec (filter string? + (tree-seq map? + :ch + {:ch [{:ch ["a" "b"]} + {:ch ["c" "d"]} + {:ch [{:ch ["e" {:ch ["f"]}]}]}]}))) + ["a" "b" "c" "d" "e" "f"])) From 1ac69ec5687e796301b997d4c58bcfbb8bcaafcc Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 26 Feb 2015 22:27:59 +0000 Subject: [PATCH 548/909] Fixes namespacing of keywords in reader --- pixie/vm/keyword.py | 4 +++- pixie/vm/reader.py | 2 +- tests/pixie/tests/test-keywords.pxi | 14 ++++++++++++++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index b1a8f90f..e1c14568 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -49,7 +49,9 @@ def intern(self, nm): _kw_cache = KeywordCache() -def keyword(nm): +def keyword(nm, ns=None): + if ns: + nm = u"/".join([ns, nm]) return _kw_cache.intern(nm) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index f0d98588..ac63e472 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -272,7 +272,7 @@ def invoke(self, rdr, ch): itm = read(rdr, True) affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") - return keyword(rt.name(itm)) + return keyword(rt.name(itm), rt.namespace(itm)) class LiteralStringReader(ReaderHandler): def invoke(self, rdr, ch): diff --git a/tests/pixie/tests/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi index 5f336b73..af6e4334 100644 --- a/tests/pixie/tests/test-keywords.pxi +++ b/tests/pixie/tests/test-keywords.pxi @@ -8,3 +8,17 @@ (t/assert= (:c m) 3) (t/assert= (:d m) nil))) + +(t/deftest keyword-namespace + (t/assert= (namespace :foo/bar) "foo") + (t/assert= (namespace :cat/dog) "cat")) + + +(t/deftest keyword-equality + (t/assert= :foo/bar :foo/bar) + (t/assert= (not= :foo/bar :cat/bar) true) + (t/assert= (not= :foo/cat :foo/dog) true)) + +(t/deftest string-to-keyword + (t/assert= (keyword "foo") :foo) + (t/assert= (keyword "foo/bar") :foo/bar)) From 5b155ccc3c853dce27ebdc244036614cc4e4df40 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 26 Feb 2015 15:56:09 -0700 Subject: [PATCH 549/909] more stacklet support of libuv and fix a bug with ffi callbacks --- pixie/stacklets.pxi | 75 ++++++++++++++++++++++++++++++++------------ pixie/vm/libs/ffi.py | 2 +- 2 files changed, 56 insertions(+), 21 deletions(-) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index e8d75da4..91578687 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -11,11 +11,12 @@ (defn enqueue [q itm] ; TODO: Rewrite this crappy impl - (vec (conj (seq q) itm))) + (swap! q (fn [q] (vec (conj (seq q) itm))))) (defn dequeue [q] - [(ith q -1) - (pop q)]) + (let [itm (ith @q -1)] + (swap! q pop) + itm)) (defn yield-control [] @@ -38,28 +39,60 @@ (reset! stacklet-loop-h h#) val#)) +(defn sleep [ms] + (let [[h] (@stacklet-loop-h [:sleep ms])] + (reset! stacklet-loop-h h) + nil)) + +(defmulti async-fn (fn [f args k tasks] f)) + +(defmethod async-fn :sleep + ([f args k tasks] + (let [cb (atom nil) + timer (uv/uv_timer_t)] + (reset! cb (ffi-prep-callback uv/uv_timer_cb + (fn [handle] + (enqueue tasks k) + (uv/uv_timer_stop timer) + (-dispose! @cb)))) + (uv/uv_timer_init (uv/uv_default_loop) timer) + (uv/uv_timer_start timer @cb args 0)))) + (defn -with-stacklets [fn] - (let [[h [op arg]] ((new-stacklet fn) nil)] + (let [[h [op arg]] ((new-stacklet fn) nil) + tasks (atom [])] (println h op arg) (loop [op op arg arg - this_h h - tasks []] + this_h h] (println "in loop" op arg this_h tasks) (cond + ; (= 0 (count tasks)) (uv/uv_run_loop (uv/uv_default_loop)) (= op :spawn) (let [wh (new-stacklet arg) [h [op arg]] (this_h nil)] - (recur op arg h (enqueue tasks wh))) - (= op :yield) (let [tasks (enqueue tasks this_h) - [task tasks] (dequeue tasks) + (enqueue tasks wh) + (println @tasks) + (recur op arg h)) + (= op :yield) (let [_ (enqueue tasks this_h) + task (dequeue tasks) [h [op arg]] (task nil)] - (recur op arg h tasks)) - (= op :spawn-end) (if (empty? tasks) - arg - (let [[task tasks] (dequeue tasks) - [h [op arg]] (task nil)] - (recur op arg h tasks))) - :else (assert false (str "Unkown command " op " " arg)))))) + (recur op arg h)) + (= op :spawn-end) (do (when (empty? @tasks) + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT)) + (if (empty? @tasks) + :done + (let [task (dequeue tasks) + [h [op arg]] (task nil)] + (recur op arg h)))) + :else (do (async-fn op arg this_h tasks) + (when (empty? @tasks) + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT)) + (if (empty? @tasks) + :done + (let [task (dequeue tasks) + [h [op arg]] (task nil)] + (recur op arg h)))) + :else (assert false (str "Unknown command " op " " arg)))))) (defmacro with-stacklets [& body] @@ -74,8 +107,10 @@ (println e)))))) (with-stacklets (spawn (dotimes [x 10] - (yield-control) - (println x))) + (sleep 100) + + (println "<- " x))) (spawn (dotimes [x 10] - (yield-control) - (println x)))(yield-control)) + (sleep 100) + (println "-> " x))) + (yield-control)) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 15e13167..33d43d8e 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -554,7 +554,7 @@ def ffi_prep_callback(tp, f): affirm(isinstance(tp, CFunctionType), u"First argument to ffi-prep-callback must be a CFunctionType") raw_closure = rffi.cast(rffi.VOIDP, clibffi.closureHeap.alloc()) - unique_id = len(registered_callbacks) + unique_id = rffi.cast(lltype.Signed, raw_closure) res = clibffi.c_ffi_prep_closure(rffi.cast(clibffi.FFI_CLOSUREP, raw_closure), tp.get_cd().cif, invoke_callback, From c769b86de4dc037996d81cd1efb952aea9d1c876 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 26 Feb 2015 22:27:04 -0700 Subject: [PATCH 550/909] start of async io support for pixie --- pixie/PixieChecker.hpp | 123 ++++++++++++++++++++++++++++++++++++++++- pixie/stacklets.pxi | 59 +++++++++++++------- pixie/uv.pxi | 109 +++++++++++++++++++++++++++++++++++- 3 files changed, 267 insertions(+), 24 deletions(-) diff --git a/pixie/PixieChecker.hpp b/pixie/PixieChecker.hpp index 4c5f0571..7266cbea 100644 --- a/pixie/PixieChecker.hpp +++ b/pixie/PixieChecker.hpp @@ -24,7 +24,7 @@ template < typename T > std::string to_string( const T& n ) stm << n ; return stm.str() ; } - + // Function Checker template @@ -92,7 +92,7 @@ struct FunctionTyper<0, T> " ]}"; } }; - + template struct FunctionTyper<4, T> { @@ -108,7 +108,124 @@ struct FunctionTyper<0, T> " ]}"; } }; - + + template + struct FunctionTyper<5, T> + { + static std::string getType() + { + return "{:type :function :arity 5 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<6, T> + { + static std::string getType() + { + return "{:type :function :arity 6 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<7, T> + { + static std::string getType() + { + return "{:type :function :arity 7 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<8, T> + { + static std::string getType() + { + return "{:type :function :arity 8 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + GetType::arg8_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<9, T> + { + static std::string getType() + { + return "{:type :function :arity 9 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + GetType::arg8_type>() + " " + + GetType::arg9_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<10, T> + { + static std::string getType() + { + return "{:type :function :arity 10 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + GetType::arg8_type>() + " " + + GetType::arg9_type>() + " " + + GetType::arg10_type>() + " " + + " ]}"; + } + }; + // End Function Typer diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 91578687..cac6e968 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -52,12 +52,39 @@ timer (uv/uv_timer_t)] (reset! cb (ffi-prep-callback uv/uv_timer_cb (fn [handle] - (enqueue tasks k) + (enqueue tasks [k nil]) (uv/uv_timer_stop timer) (-dispose! @cb)))) (uv/uv_timer_init (uv/uv_default_loop) timer) (uv/uv_timer_start timer @cb args 0)))) + +(defmacro defuvfsfn [nm args return] + (let [kw (keyword (str "pixie.uv/" (name nm)))] + `(do (defn ~nm ~args + (let [[h# result#] (@stacklet-loop-h [~kw ~args])] + (reset! stacklet-loop-h h#) + result#)) + (defmethod async-fn ~kw + [f# ~args k# tasks#] + (let [cb# (atom nil)] + (reset! cb# (ffi-prep-callback uv/uv_fs_cb + (fn [req#] + (enqueue tasks# [k# (~return req#)]) + (uv/uv_fs_req_cleanup req#) + (-dispose! @cb#)))) + (~(symbol (str "pixie.uv/uv_fs_" (name nm))) + (uv/uv_default_loop) + (uv/uv_fs_t) + ~@args + @cb#)))))) + +((var defuvfsfn) 'open '[path flags mode] :result) + +(defuvfsfn open [path flags mode] :result) + +(keyword "foo/bar") + (defn -with-stacklets [fn] (let [[h [op arg]] ((new-stacklet fn) nil) tasks (atom [])] @@ -70,27 +97,26 @@ ; (= 0 (count tasks)) (uv/uv_run_loop (uv/uv_default_loop)) (= op :spawn) (let [wh (new-stacklet arg) [h [op arg]] (this_h nil)] - (enqueue tasks wh) - (println @tasks) + (enqueue tasks [wh nil]) (recur op arg h)) - (= op :yield) (let [_ (enqueue tasks this_h) - task (dequeue tasks) - [h [op arg]] (task nil)] + (= op :yield) (let [_ (enqueue tasks [this_h nil]) + [task val] (dequeue tasks) + [h [op arg]] (task val)] (recur op arg h)) (= op :spawn-end) (do (when (empty? @tasks) - (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT)) + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_ONCE)) (if (empty? @tasks) :done - (let [task (dequeue tasks) - [h [op arg]] (task nil)] + (let [[task val] (dequeue tasks) + [h [op arg]] (task val)] (recur op arg h)))) :else (do (async-fn op arg this_h tasks) (when (empty? @tasks) - (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT)) + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_ONCE)) (if (empty? @tasks) :done - (let [task (dequeue tasks) - [h [op arg]] (task nil)] + (let [[task val] (dequeue tasks) + [h [op arg]] (task val)] (recur op arg h)))) :else (assert false (str "Unknown command " op " " arg)))))) @@ -106,11 +132,4 @@ (catch e (println e)))))) -(with-stacklets (spawn (dotimes [x 10] - (sleep 100) - - (println "<- " x))) - (spawn (dotimes [x 10] - (sleep 100) - (println "-> " x))) - (yield-control)) +(with-stacklets (open "/tmp/foo-bar-baz" uv/O_WRONLY uv/O_CREAT)) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index e6f03f15..94fefb0a 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -35,4 +35,111 @@ (f/defcfn uv_timer_stop) (f/defcfn uv_timer_again) (f/defcfn uv_timer_set_repeat) - (f/defcfn uv_timer_get_repeat)) + (f/defcfn uv_timer_get_repeat) + + ;; Filesystem + + (f/defcstruct uv_fs_t [:loop + :fs_type + :path + :result + :ptr]) + (f/defcstruct uv_timespec_t [:tv_sec + :tv_nsec]) + (f/defcstruct uv_stat_t [:st_dev + :st_mode + :st_nlink + :st_uid + :st_gid + :st_rdev + :st_ino + :st_size + :st_blksize + :st_blocks + :st_flags + :st_gen]) + + (f/defconst UV_FS_UNKNOWN) + (f/defconst UV_FS_CUSTOM) + (f/defconst UV_FS_OPEN) + (f/defconst UV_FS_CLOSE) + (f/defconst UV_FS_READ) + (f/defconst UV_FS_WRITE) + (f/defconst UV_FS_SENDFILE) + (f/defconst UV_FS_STAT) + (f/defconst UV_FS_LSTAT) + (f/defconst UV_FS_FSTAT) + (f/defconst UV_FS_FTRUNCATE) + (f/defconst UV_FS_UTIME) + (f/defconst UV_FS_FUTIME) + (f/defconst UV_FS_ACCESS) + (f/defconst UV_FS_CHMOD) + (f/defconst UV_FS_FCHMOD) + (f/defconst UV_FS_FSYNC) + (f/defconst UV_FS_FDATASYNC) + (f/defconst UV_FS_UNLINK) + (f/defconst UV_FS_RMDIR) + (f/defconst UV_FS_MKDIR) + (f/defconst UV_FS_MKDTEMP) + (f/defconst UV_FS_RENAME) + (f/defconst UV_FS_SCANDIR) + (f/defconst UV_FS_LINK) + (f/defconst UV_FS_SYMLINK) + (f/defconst UV_FS_READLINK) + (f/defconst UV_FS_CHOWN) + (f/defconst UV_FS_FCHOWN) + + (f/defconst UV_DIRENT_UNKNOWN) + (f/defconst UV_DIRENT_FILE) + (f/defconst UV_DIRENT_DIR) + (f/defconst UV_DIRENT_LINK) + (f/defconst UV_DIRENT_FIFO) + (f/defconst UV_DIRENT_SOCKET) + (f/defconst UV_DIRENT_CHAR) + (f/defconst UV_DIRENT_BLOCK) + + (f/defcstruct uv_dirent_t [:name + :type]) + + (f/defcfn uv_fs_req_cleanup) + (f/defcfn uv_fs_close) + (f/defcfn uv_fs_open) + + (f/defccallback uv_fs_cb) + + (f/defcfn uv_fs_unlink) + (f/defcfn uv_fs_write) + (f/defcfn uv_fs_read) + (f/defcfn uv_fs_mkdir) + (f/defcfn uv_fs_mkdtemp) + (f/defcfn uv_fs_rmdir) + (f/defcfn uv_fs_scandir) + (f/defcfn uv_fs_scandir_next) + (f/defcfn uv_fs_stat) + (f/defcfn uv_fs_fstat) + (f/defcfn uv_fs_lstat) + (f/defcfn uv_fs_rename) + (f/defcfn uv_fs_fsync) + (f/defcfn uv_fs_fdatasync) + (f/defcfn uv_fs_ftruncate) + (f/defcfn uv_fs_sendfile) + (f/defcfn uv_fs_access) + (f/defcfn uv_fs_chmod) + (f/defcfn uv_fs_fchmod) + (f/defcfn uv_fs_utime) + (f/defcfn uv_fs_futime) + (f/defcfn uv_fs_link) + (f/defcfn uv_fs_symlink) + (f/defcfn uv_fs_readlink) + (f/defcfn uv_fs_chown) + (f/defcfn uv_fs_fchown) + + (f/defconst O_RDONLY) + (f/defconst O_WRONLY) + (f/defconst O_RDWR) + + (f/defconst O_APPEND) + (f/defconst O_ASYNC) + (f/defconst O_CREAT) + + ) From 05bd97e4cb1cf33101571b2693176633e7b38508 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 27 Feb 2015 20:52:09 -0700 Subject: [PATCH 551/909] stacklets play nice with libuv now. Main loop refactored to be much cleaner --- pixie/stacklets.pxi | 166 +++++++++++++++++++++++++++----------------- pixie/uv.pxi | 30 +++++++- 2 files changed, 129 insertions(+), 67 deletions(-) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index cac6e968..4869c296 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -1,13 +1,23 @@ (ns pixie.stacklets (require pixie.uv :as uv)) +;; LibUV seems to act up when we invoke a stacklet from inside a callbac +;; so we compensate by simply storing the stacklets in a task queue +;; and calling them later outside of the libuv loop. + (def stacklet-loop-h (atom nil)) -(defn -spawn-thread [fn] - (let [[h val] (@stacklet-loop-h [:spawn fn])] - (reset! stacklet-loop-h h) - (println h val) - val)) +(def thread-count (atom 0)) + +(defmulti async-fn (fn [f args k] f)) + +(defmethod async-fn :spawn-end + [_ _ _] + (swap! thread-count dec) + (when (= @thread-count 0) + (uv/uv_stop (uv/uv_default_loop)))) + +(def tasks (atom [])) (defn enqueue [q itm] ; TODO: Rewrite this crappy impl @@ -18,41 +28,60 @@ (swap! q pop) itm)) +(comment + (defn run-and-process [k args] + + (let [[h [op args]] (k args)] + (async-fn op args h)))) + +(defn -run-and-process [k args] + + (let [[h [op args]] (k args)] + (async-fn op args h))) + +(defn run-and-process [k args] + (swap! tasks conj [k args])) + +;; Yield (defn yield-control [] (let [[h] (@stacklet-loop-h [:yield nil])] (reset! stacklet-loop-h h) nil)) +(def close_cb (ffi-prep-callback uv/uv_close_cb + (fn [handle] + (pixie.ffi/free handle) + ))) -(defmacro spawn [& body] - `(let [f (fn [h# _] - (reset! stacklet-loop-h h#) - (try - (println "spawn" h#) - ;(reset! stacklet-loop-h h#) - (let [result# (do ~@body)] - (println "returning " result#) - (@stacklet-loop-h [:spawn-end result#])) - (catch e (println e)))) - [h# val#] (@stacklet-loop-h [:spawn f])] - (reset! stacklet-loop-h h#) - val#)) +(def tasks (atom [])) + +(defmethod async-fn :yield + [_ args k] + + (let [a (uv/uv_async_t) + cb (atom nil)] + (reset! cb (ffi-prep-callback uv/uv_async_cb + (fn [handle] + (uv/uv_close a close_cb) + (run-and-process k nil)))) + (uv/uv_async_init (uv/uv_default_loop) a @cb) + (uv/uv_async_send a))) + +;;; Sleep (defn sleep [ms] (let [[h] (@stacklet-loop-h [:sleep ms])] (reset! stacklet-loop-h h) nil)) -(defmulti async-fn (fn [f args k tasks] f)) - (defmethod async-fn :sleep - ([f args k tasks] + ([f args k] (let [cb (atom nil) timer (uv/uv_timer_t)] (reset! cb (ffi-prep-callback uv/uv_timer_cb (fn [handle] - (enqueue tasks [k nil]) + (run-and-process k nil) (uv/uv_timer_stop timer) (-dispose! @cb)))) (uv/uv_timer_init (uv/uv_default_loop) timer) @@ -70,66 +99,73 @@ (let [cb# (atom nil)] (reset! cb# (ffi-prep-callback uv/uv_fs_cb (fn [req#] - (enqueue tasks# [k# (~return req#)]) - (uv/uv_fs_req_cleanup req#) - (-dispose! @cb#)))) + (try + (enqueue tasks# [k# (~return (pixie.ffi/cast req# uv/uv_fs_t))]) + (uv/uv_fs_req_cleanup req#) + (-dispose! @cb#) + (catch e (println e)))))) (~(symbol (str "pixie.uv/uv_fs_" (name nm))) (uv/uv_default_loop) (uv/uv_fs_t) ~@args @cb#)))))) +(comment + ((var defuvfsfn) 'open '[path flags mode] :result) -((var defuvfsfn) 'open '[path flags mode] :result) - -(defuvfsfn open [path flags mode] :result) - -(keyword "foo/bar") + (defuvfsfn open [path flags mode] :result) + (defuvfsfn read [file bufs nbufs offset] :result) + (defuvfsfn close [file] :result)) (defn -with-stacklets [fn] - (let [[h [op arg]] ((new-stacklet fn) nil) - tasks (atom [])] - (println h op arg) - (loop [op op - arg arg - this_h h] - (println "in loop" op arg this_h tasks) - (cond - ; (= 0 (count tasks)) (uv/uv_run_loop (uv/uv_default_loop)) - (= op :spawn) (let [wh (new-stacklet arg) - [h [op arg]] (this_h nil)] - (enqueue tasks [wh nil]) - (recur op arg h)) - (= op :yield) (let [_ (enqueue tasks [this_h nil]) - [task val] (dequeue tasks) - [h [op arg]] (task val)] - (recur op arg h)) - (= op :spawn-end) (do (when (empty? @tasks) - (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_ONCE)) - (if (empty? @tasks) - :done - (let [[task val] (dequeue tasks) - [h [op arg]] (task val)] - (recur op arg h)))) - :else (do (async-fn op arg this_h tasks) - (when (empty? @tasks) - (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_ONCE)) - (if (empty? @tasks) - :done - (let [[task val] (dequeue tasks) - [h [op arg]] (task val)] - (recur op arg h)))) - :else (assert false (str "Unknown command " op " " arg)))))) + (let [[h [op arg]] ((new-stacklet fn) nil)] + (swap! thread-count inc) + (async-fn op arg h) + (loop [] + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT) + (let [tks @tasks] + (reset! tasks []) + (when (not (empty? tks)) + (doseq [[k args] tks] + (-run-and-process k args)) + (recur)))))) (defmacro with-stacklets [& body] `(-with-stacklets (fn [h# _] (try - (println h# _) (reset! stacklet-loop-h h#) (let [result# (do ~@body)] (@stacklet-loop-h [:spawn-end result#])) (catch e (println e)))))) -(with-stacklets (open "/tmp/foo-bar-baz" uv/O_WRONLY uv/O_CREAT)) + + +(with-stacklets (dotimes [x 10000] + (yield-control) + (println x))) + +(comment + (defn run-later [f] + (let [a (uv/uv_async_t) + cb (atom nil)] + (println "start yield") + (reset! cb (ffi-prep-callback uv/uv_async_cb + (fn [handle] + (println "process yield") + (f) + (uv/uv_close a close_cb) + (-dispose! @cb) + (println "done process yield")))) + (uv/uv_async_init (uv/uv_default_loop) a @cb) + (uv/uv_async_send a))) + + (defn cfn [x] + (fn [] + (println x) + (if (pos? x) + (run-later (cfn (dec x)))))) + + (do (run-later (cfn 10000)) + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT))) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 94fefb0a..85079189 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -7,6 +7,10 @@ (f/defconst UV_RUN_ONCE) (f/defconst UV_RUN_NOWAIT) + (f/defcfn uv_close) + (f/defccallback uv_close_cb) + + (f/defcstruct uv_loop_t []) (f/defcfn uv_loop_init) @@ -23,6 +27,7 @@ (f/defcfn uv_update_time) (f/defcfn uv_walk) + (f/defccallback uv_read_cb) @@ -37,6 +42,7 @@ (f/defcfn uv_timer_set_repeat) (f/defcfn uv_timer_get_repeat) + ;; Filesystem (f/defcstruct uv_fs_t [:loop @@ -57,7 +63,15 @@ :st_blksize :st_blocks :st_flags - :st_gen]) + :st_gen + :st_atim.tv_sec + :st_atim.tv_nsec + :st_mtim.tv_sec + :st_mtim.tv_nsec + :st_ctim.tv_sec + :st_ctim.tv_nsec + :st_birthtim.tv_sec + :st_birthtim.tv_nsec]) (f/defconst UV_FS_UNKNOWN) (f/defconst UV_FS_CUSTOM) @@ -142,4 +156,16 @@ (f/defconst O_ASYNC) (f/defconst O_CREAT) - ) + (f/defconst S_IRUSR) + + + ; ERRNO + (f/defconst UV_E2BIG) + (f/defconst UV_EACCES) + + + ; async + (f/defcstruct uv_async_t []) + (f/defccallback uv_async_cb) + (f/defcfn uv_async_init) + (f/defcfn uv_async_send)) From 7f54b8eb386c26c793ef4413607542fd07dd6d47 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 28 Feb 2015 11:54:35 +0000 Subject: [PATCH 552/909] Fix typo --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index bc5a9c12..39110fb4 100644 --- a/README.md +++ b/README.md @@ -62,7 +62,7 @@ However there are a few features of pixie that although may not be uncommon, are * Pixie implements its own virtual machine. It does not run on the JVM, CLR or Python VM. It implements its own bytecode, has its own GC and JIT. And it's small. Currently the interpreter, JIT, GC, and stdlib clock in at about 5.5MB once compiled down to an executable. -* The JIT makes some things fast. Very fast. Code like the following compiles down to a loop with 6 CPU instructions. While this may not be too impressive for any language that uses a tracing jit, it is faily unique for a language as young as Pixie. +* The JIT makes some things fast. Very fast. Code like the following compiles down to a loop with 6 CPU instructions. While this may not be too impressive for any language that uses a tracing jit, it is fairly unique for a language as young as Pixie. ```clojure From 27bda32b8e7c671417a59b6f82574edd4a79bb1d Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 28 Feb 2015 11:10:04 +0000 Subject: [PATCH 553/909] added an identity hash method to Object --- pixie/vm/object.py | 5 +++++ pixie/vm/stdlib.py | 8 ++++++-- tests/pixie/tests/test-object.pxi | 6 ++++++ 3 files changed, 17 insertions(+), 2 deletions(-) create mode 100644 tests/pixie/tests/test-object.pxi diff --git a/pixie/vm/object.py b/pixie/vm/object.py index defec587..2abf674e 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -1,3 +1,4 @@ +from rpython.rlib.objectmodel import compute_identity_hash import rpython.rlib.jit as jit class Object(object): @@ -21,6 +22,10 @@ def r_uint_val(self): affirm(False, u"Expected Number, not " + self.type().name()) return 0 + def hash(self): + import pixie.vm.rt as rt + return rt.wrap(compute_identity_hash(self)) + def promote(self): return self diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index ba6f186b..d4c6cfb1 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -105,8 +105,14 @@ def default_str(x): assert isinstance(tp, Type) return rt.wrap(u"") +def default_hash(x): + tp = x.type() + assert isinstance(tp, Type) + return x.hash() + _str.set_default_fn(wrap_fn(default_str)) _repr.set_default_fn(wrap_fn(default_str)) +_hash.set_default_fn(wrap_fn(default_hash)) _meta.set_default_fn(wrap_fn(lambda x: nil)) @@ -248,8 +254,6 @@ def _(self): def __hash(x): return rt._hash(x) - - _count_driver = jit.JitDriver(name="pixie.stdlib.count", greens=["tp"], reds="auto") diff --git a/tests/pixie/tests/test-object.pxi b/tests/pixie/tests/test-object.pxi new file mode 100644 index 00000000..810470b0 --- /dev/null +++ b/tests/pixie/tests/test-object.pxi @@ -0,0 +1,6 @@ +(ns pixie.tests.test-object + (require pixie.test :as t)) + +(t/deftest test-hash + (t/assert= (hash (var foo)) (hash (var foo))) + (t/assert (not= (hash (var foo)) (hash (var bar))))) From 582f29540da1018f1e8f95d7adc90b1f0a819e5a Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 28 Feb 2015 23:10:36 +0000 Subject: [PATCH 554/909] remove dead code --- pixie/vm/stdlib.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index d4c6cfb1..07fdb380 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -106,8 +106,6 @@ def default_str(x): return rt.wrap(u"") def default_hash(x): - tp = x.type() - assert isinstance(tp, Type) return x.hash() _str.set_default_fn(wrap_fn(default_str)) From 41482f2dae09a1b12a7f03e7478d3247a44c190e Mon Sep 17 00:00:00 2001 From: gigasquid Date: Sat, 28 Feb 2015 18:13:04 -0500 Subject: [PATCH 555/909] pull NS_VAR into rt --- pixie/vm/reader.py | 19 +++++++++++++++---- pixie/vm/rt.py | 3 +++ tests/pixie/tests/test-keywords.pxi | 3 ++- 3 files changed, 20 insertions(+), 5 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index ac63e472..abc7eadf 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -71,7 +71,7 @@ def __init__(self): def read(self): if self._string_reader is None: - result = _readline(str(rt.name(compiler.NS_VAR.deref())) + " => ") + result = _readline(str(rt.name(rt.ns.deref())) + " => ") if result == u"": raise EOFError() self._string_reader = StringReader(result) @@ -269,10 +269,21 @@ def invoke(self, rdr, ch): class KeywordReader(ReaderHandler): def invoke(self, rdr, ch): - itm = read(rdr, True) - affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") + nms = u"" + ch = rdr.read() + if ch == u":": + itm = read(rdr, True) + nms = rt.ns.deref().name() + else: + rdr.unread(ch) + itm = read(rdr, True) - return keyword(rt.name(itm), rt.namespace(itm)) + affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") + if nms: + affirm(rt.namespace(itm) is None, u"Kewyword cannot have two namespaces") + return keyword(rt.name(itm), nms) + else: + return keyword(rt.name(itm), rt.namespace(itm)) class LiteralStringReader(ReaderHandler): def invoke(self, rdr, ch): diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 20cc157c..342708c3 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -12,6 +12,7 @@ def init(): from pixie.vm.primitives import nil, true, false from pixie.vm.string import String from pixie.vm.object import Object + from pixie.vm.compiler import NS_VAR _type_registry.set_registry(code._ns_registry) @@ -151,6 +152,8 @@ def reinit(): for x in init_vars: globals()[py_str(code.munge(x))] = code.intern_var(u"pixie.stdlib", x) + globals()[py_str(code.munge(u"ns"))] = NS_VAR + globals()["__inited__"] = True diff --git a/tests/pixie/tests/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi index af6e4334..5156d27a 100644 --- a/tests/pixie/tests/test-keywords.pxi +++ b/tests/pixie/tests/test-keywords.pxi @@ -11,7 +11,8 @@ (t/deftest keyword-namespace (t/assert= (namespace :foo/bar) "foo") - (t/assert= (namespace :cat/dog) "cat")) + (t/assert= (namespace :cat/dog) "cat") + (t/assert= (namespace ::foo) "pixie.tests.test-keywords")) (t/deftest keyword-equality From a4114bef26524b5a7aa3c84ab5597e7d728e077a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 28 Feb 2015 21:44:51 -0700 Subject: [PATCH 556/909] compile tests before running to validate pxic code --- .travis.yml | 2 ++ pixie/stdlib.pxi | 5 +++++ 2 files changed, 7 insertions(+) diff --git a/.travis.yml b/.travis.yml index 2825107d..9d48c020 100644 --- a/.travis.yml +++ b/.travis.yml @@ -12,6 +12,8 @@ matrix: script: - make PYTHON=python build + - make compile_src + - make compile_tests - make run_built_tests before_install: diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index eb69fc64..1c73507c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -337,6 +337,11 @@ (fn [v] (apply str "(" (conj (transduce (comp (map -repr) (interpose " ")) conj v) ")")))) +(extend -hash PersistentList + (fn [v] + (transduce ordered-hash-reducing-fn v))) + + (extend -str LazySeq (fn [v] (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) From 0055edb20d2fbf920b51d2c489ad586606bcc6d6 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 1 Mar 2015 09:49:32 +0000 Subject: [PATCH 557/909] Added group-by Added frequencies --- pixie/stdlib.pxi | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index f9f82b3b..8983fa6c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1548,6 +1548,25 @@ The new value is thus `(apply f current-value-of-atom args)`." s)))] (lazy-seq (step pred coll))))) +;; TODO: use a transient map in the future +(defn group-by [f coll] + {:doc "Groups the collection into a map keyed by the result of applying f on each element. The value at each key is a vector of elements in order of appearance." + :examples [["(group-by even? [1 2 3 4 5])" nil {false [1 3 5] true [2 4]}] + ["(group-by (partial apply +) [[1 2 3][2 4][1 2]]" nil {6 [[1 2 3] [2 4]] 3 [[1 2]]}]] + :signatures [[f coll]] + :added "0.1"} + (reduce (fn [res elem] + (update-in res [(f elem)] (fnil conj []) elem)) + {} + coll)) + +;; TODO: use a transient map in the future +(defn frequencies [coll] + (reduce (fn [res elem] + (update-in res [elem] (fnil inc 0))) + {} + coll) + (defn partition {:doc "Separates the collection into collections of size n, starting at the beginning, with an optional step size. From 10919b18f850b5d4282dce129f8520db48c9ddcf Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 1 Mar 2015 14:40:26 +0000 Subject: [PATCH 558/909] name is not a method of namespace --- pixie/vm/reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index abc7eadf..3049ae2f 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -273,7 +273,7 @@ def invoke(self, rdr, ch): ch = rdr.read() if ch == u":": itm = read(rdr, True) - nms = rt.ns.deref().name() + nms = rt.name(rt.ns.deref()) else: rdr.unread(ch) itm = read(rdr, True) From ea7d63208067488e2d349b26b1c55bd75dff90f5 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 1 Mar 2015 14:18:15 +0000 Subject: [PATCH 559/909] Added tests for group-by and frequencies --- pixie/stdlib.pxi | 4 +++- tests/pixie/tests/test-stdlib.pxi | 16 ++++++++++++++++ 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8983fa6c..fe3db177 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1562,10 +1562,12 @@ The new value is thus `(apply f current-value-of-atom args)`." ;; TODO: use a transient map in the future (defn frequencies [coll] + {:doc "Returns a map with distinct elements as keys and the number of occurences as values" + :added "0.1"} (reduce (fn [res elem] (update-in res [elem] (fnil inc 0))) {} - coll) + coll)) (defn partition {:doc "Separates the collection into collections of size n, starting at the beginning, with an optional step size. diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 4ed20017..2439d837 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -393,3 +393,19 @@ {:ch ["c" "d"]} {:ch [{:ch ["e" {:ch ["f"]}]}]}]}))) ["a" "b" "c" "d" "e" "f"])) + +(t/deftest test-group-by + (t/assert= (group-by :age [{:name "banjo" :age 3} + {:name "mary" :age 3} + {:name "boris" :age 7}]) + {3 [{:name "banjo" :age 3} + {:name "mary" :age 3}] + 7 [{:name "boris" :age 7}]}) + (t/assert= (group-by even? (range 1 5)) + {true [2 4] + false [1 3]})) + + +(t/deftest test-frequencies + (t/assert= (frequencies [1 2 3 4 3 2 1]) + {1 2, 2 2, 3 2, 4 1})) From 0400ef40503337953a0e2dd7f4238c26011a6854 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 4 Mar 2015 16:09:15 -0700 Subject: [PATCH 560/909] latest work --- Makefile | 7 ++++- pixie/stacklets.pxi | 71 +++++++++++++++++++++++++++++++++------------ pixie/vm/threads.py | 35 ++++++++++++++++++++++ 3 files changed, 93 insertions(+), 20 deletions(-) diff --git a/Makefile b/Makefile index e8f1c0a4..37645acb 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=../externals -PYTHON ?= pypy +PYTHON ?= python PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy @@ -52,9 +52,14 @@ $(EXTERNALS)/pypy: run: ./pixie-vm + run_interactive: @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py +run_interactive_stacklets: + @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py pixie/stacklets.pxi + + run_built_tests: pixie-vm ./pixie-vm run-tests.pxi diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 4869c296..abf22e2b 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -58,15 +58,7 @@ (defmethod async-fn :yield [_ args k] - - (let [a (uv/uv_async_t) - cb (atom nil)] - (reset! cb (ffi-prep-callback uv/uv_async_cb - (fn [handle] - (uv/uv_close a close_cb) - (run-and-process k nil)))) - (uv/uv_async_init (uv/uv_default_loop) a @cb) - (uv/uv_async_send a))) + (add-item task-queue [k args])) ;;; Sleep @@ -116,18 +108,53 @@ (defuvfsfn read [file bufs nbufs offset] :result) (defuvfsfn close [file] :result)) + +(defprotocol IBlockingQueue + (add-item [this item]) + (remove-item [this])) + +(deftype BlockingQueue [items lock locked] + IBlockingQueue + (add-item [this item] + (enqueue items item) + (when @locked + (-release-lock lock) + (reset! locked false)) + (-yield-thread)) + (remove-item [this] + (when (empty? @items) + (reset! locked true) + (-acquire-lock lock true)) + (dequeue items))) + +(defn blocking-queue [] + (let [l (-create-lock)] + (-acquire-lock l true) + (->BlockingQueue (atom []) l (atom true)))) + +(def task-queue (blocking-queue)) + + + + (defn -with-stacklets [fn] (let [[h [op arg]] ((new-stacklet fn) nil)] (swap! thread-count inc) (async-fn op arg h) (loop [] - (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT) - (let [tks @tasks] - (reset! tasks []) - (when (not (empty? tks)) - (doseq [[k args] tks] - (-run-and-process k args)) - (recur)))))) + (let [[k args] (remove-item task-queue)] + (-run-and-process k args) + (recur))))) + +(defn -with-stacklets [fn] + (let [new-s (new-stacklet fn) + [h [op arg]] (new-s nil)] + (loop [h h + op op + arg arg] + (if (not (= op :spawn-end)) + (let [[h [op arg]] (h nil)] + (recur h op arg)))))) (defmacro with-stacklets [& body] @@ -141,10 +168,16 @@ (println e)))))) - (with-stacklets (dotimes [x 10000] - (yield-control) - (println x))) + (yield-control) + (println x))) + +(comment + + + (dotimes [t 33] + (-thread (fn [] (dotimes [x 10000] + (println t x)))))) (comment (defn run-later [f] diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index 6d8404b9..d34b47b8 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -1,7 +1,10 @@ +from pixie.vm.object import Object, Type +from pixie.vm.primitives import true import rpython.rlib.rthread as rthread from pixie.vm.primitives import nil import rpython.rlib.rgil as rgil from pixie.vm.code import as_var +import pixie.vm.rt as rt from rpython.rlib.objectmodel import invoke_around_extcall @@ -55,6 +58,38 @@ def yield_thread(): do_yield_thread() return nil +# Locks + +class Lock(Object): + _type = Type(u"pixie.stdlib.Lock") + def __init__(self, ll_lock): + self._ll_lock = ll_lock + + def type(self): + return Lock._type + + +@as_var("-create-lock") +def _create_lock(): + return Lock(rthread.allocate_lock()) + +@as_var("-acquire-lock") +def _acquire_lock(self, no_wait): + assert isinstance(self, Lock) + return rt.wrap(self._ll_lock.acquire(no_wait == true)) + +@as_var("-acquire-lock-timed") +def _acquire_lock(self, ms): + assert isinstance(self, Lock) + return rt.wrap(self._ll_lock.acquire(ms.int_val())) + +@as_var("-release-lock") +def _release_lock(self): + assert isinstance(self, Lock) + return rt.wrap(self._ll_lock.release()) + + + ## From PYPY From 71174a5c248b91f1dbada1f028adfb1332b3561c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 4 Mar 2015 23:53:23 +0000 Subject: [PATCH 561/909] Fixes Issue #211 Turns out my tests were rather poor... --- pixie/vm/code.py | 9 +++++++-- pixie/vm/interpreter.py | 8 ++------ 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 0ec2344b..729ec4ea 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -1,6 +1,6 @@ py_object = object import pixie.vm.object as object -from pixie.vm.object import affirm +from pixie.vm.object import affirm, runtime_error from pixie.vm.primitives import nil, false from rpython.rlib.rarithmetic import r_uint from rpython.rlib.jit import elidable_promote, promote @@ -209,7 +209,12 @@ def get_debug_points(self): return self._debug_points def invoke(self, args): - return self.invoke_with(args, self) + if len(args) == self.get_arity(): + return self.invoke_with(args, self) + else: + runtime_error(u"Invalid number of arguments " + unicode(len(args)) + + u" for function '" + unicode(str(self._name)) + u"'. Expected " + + unicode(str(self.get_arity()))) def invoke_with(self, args, this_fn): try: diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 7cc5da9e..9e80107a 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -98,12 +98,8 @@ def push_nth(self, delta): self.push(self.nth(delta)) def push_arg(self, idx): - if not 0 <= idx < len(self.args): - runtime_error(u"Invalid number of arguments " + unicode(str(idx)) - + u" for function '" + unicode(str(self.code_obj._name)) + u"'. Expected " - + unicode(str(self.code_obj.get_arity()))) - - self.push(self.args[r_uint(idx)]) + if 0 <= idx < len(self.args): + self.push(self.args[r_uint(idx)]) @unroll_safe def push_n(self, args, argc): From 2fcf979c380b07192c6f0d0c8192fbc442d2eb8c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 4 Mar 2015 23:55:15 +0000 Subject: [PATCH 562/909] Runtime arg error caught some stuff... more --- tests/pixie/tests/test-stdlib.pxi | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 2439d837..c0c0b711 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -311,9 +311,9 @@ (t/assert= (merge-with identity {} {:a 1, :b 2}) {:a 1, :b 2}) (t/assert= (merge-with identity {:a 1} {:b 2}) {:a 1, :b 2}) - (t/assert= (merge-with #(identity %1) {:a 1} {:a 2}) {:a 1}) - (t/assert= (merge-with #(identity %1) {:a 1} {:a 2} {:a 3}) {:a 1}) - (t/assert= (merge-with #(identity %2) {:a 1} {:a 2}) {:a 2}) + (t/assert= (merge-with (fn [a b] a) {:a 1} {:a 2}) {:a 1}) + (t/assert= (merge-with (fn [a b] a) {:a 1} {:a 2} {:a 3}) {:a 1}) + (t/assert= (merge-with (fn [a b] b) {:a 1} {:a 2}) {:a 2}) (t/assert= (merge-with + {:a 21} {:a 21}) {:a 42}) (t/assert= (merge-with + {:a 21} {:a 21, :b 1}) {:a 42, :b 1})) @@ -333,9 +333,8 @@ (t/deftest test-range (t/assert= (= (-seq (range 10)) - (-seq (-iterator (range 10)) - (reduce conj nil (range 10)) - '(0 1 2 3 4 5 6 7 8 9))) + (-seq (-iterator (range 10))) + '(0 1 2 3 4 5 6 7 8 9)) true)) (t/deftest test-ns From e1c869ec1d643aaefcfb00ece5582a858532bf55 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 4 Mar 2015 21:14:19 -0700 Subject: [PATCH 563/909] well it seems stacklets work much better on Linux --- pixie/stacklets.pxi | 172 +++++++++++++++++++++++++------------------- pixie/stdlib.pxi | 5 ++ 2 files changed, 105 insertions(+), 72 deletions(-) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index abf22e2b..f6d9144d 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -39,16 +39,35 @@ (let [[h [op args]] (k args)] (async-fn op args h))) -(defn run-and-process [k args] - (swap! tasks conj [k args])) +(defn run-and-process [k] + (let [[h f] (k nil)] + (f h))) ;; Yield -(defn yield-control [] - (let [[h] (@stacklet-loop-h [:yield nil])] +(defn switch-back [f] + (let [[h] (@stacklet-loop-h f)] (reset! stacklet-loop-h h) nil)) +(defn -run-later [f] + (let [a (uv/uv_async_t) + cb (atom nil)] + (reset! cb (ffi-prep-callback uv/uv_async_cb + (fn [handle] + (try + (uv/uv_close a close_cb) + (-dispose! @cb) + (f) + (catch ex (println ex)))))) + (uv/uv_async_init (uv/uv_default_loop) a @cb) + (uv/uv_async_send a))) + + +(defn yield-control [] + (switch-back (fn [k] + (-run-later (partial run-and-process k))))) + (def close_cb (ffi-prep-callback uv/uv_close_cb (fn [handle] (pixie.ffi/free handle) @@ -67,17 +86,36 @@ (reset! stacklet-loop-h h) nil)) -(defmethod async-fn :sleep - ([f args k] - (let [cb (atom nil) - timer (uv/uv_timer_t)] - (reset! cb (ffi-prep-callback uv/uv_timer_cb - (fn [handle] - (run-and-process k nil) - (uv/uv_timer_stop timer) - (-dispose! @cb)))) - (uv/uv_timer_init (uv/uv_default_loop) timer) - (uv/uv_timer_start timer @cb args 0)))) +(defn sleep [ms] + (let [f (fn [k] + (let [cb (atom nil) + timer (uv/uv_timer_t)] + (reset! cb (ffi-prep-callback uv/uv_timer_cb + (fn [handle] + (try + (run-and-process k) + (uv/uv_timer_stop timer) + (-dispose! @cb) + (catch ex + (println ex)))))) + (uv/uv_timer_init (uv/uv_default_loop) timer) + (uv/uv_timer_start timer @cb ms 0)))] + (switch-back f))) + +(defn -spawn [start-fn] + (switch-back (fn [k] + (-run-later (fn [] + (run-and-process (new-stacklet start-fn)))) + (-run-later (partial run-and-process k))))) + +(defmacro spawn [& body] + `(-spawn (fn [h# _] + (try + (reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (switch-back (fn [_] nil))) + (catch e + (println e)))))) (defmacro defuvfsfn [nm args return] @@ -101,61 +139,12 @@ (uv/uv_fs_t) ~@args @cb#)))))) -(comment - ((var defuvfsfn) 'open '[path flags mode] :result) - - (defuvfsfn open [path flags mode] :result) - (defuvfsfn read [file bufs nbufs offset] :result) - (defuvfsfn close [file] :result)) - - -(defprotocol IBlockingQueue - (add-item [this item]) - (remove-item [this])) - -(deftype BlockingQueue [items lock locked] - IBlockingQueue - (add-item [this item] - (enqueue items item) - (when @locked - (-release-lock lock) - (reset! locked false)) - (-yield-thread)) - (remove-item [this] - (when (empty? @items) - (reset! locked true) - (-acquire-lock lock true)) - (dequeue items))) - -(defn blocking-queue [] - (let [l (-create-lock)] - (-acquire-lock l true) - (->BlockingQueue (atom []) l (atom true)))) - -(def task-queue (blocking-queue)) - - (defn -with-stacklets [fn] - (let [[h [op arg]] ((new-stacklet fn) nil)] - (swap! thread-count inc) - (async-fn op arg h) - (loop [] - (let [[k args] (remove-item task-queue)] - (-run-and-process k args) - (recur))))) - -(defn -with-stacklets [fn] - (let [new-s (new-stacklet fn) - [h [op arg]] (new-s nil)] - (loop [h h - op op - arg arg] - (if (not (= op :spawn-end)) - (let [[h [op arg]] (h nil)] - (recur h op arg)))))) - + (let [[h f] ((new-stacklet fn) nil)] + (f h) + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT))) (defmacro with-stacklets [& body] `(-with-stacklets @@ -163,14 +152,13 @@ (try (reset! stacklet-loop-h h#) (let [result# (do ~@body)] - (@stacklet-loop-h [:spawn-end result#])) + (switch-back (fn [_] nil))) (catch e (println e)))))) - -(with-stacklets (dotimes [x 10000] - (yield-control) - (println x))) +(with-stacklets + (dotimes [x (* 1024 10)] + (spawn 1))) (comment @@ -202,3 +190,43 @@ (do (run-later (cfn 10000)) (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT))) + + +(comment + ((var defuvfsfn) 'open '[path flags mode] :result) + + (defuvfsfn open [path flags mode] :result) + (defuvfsfn read [file bufs nbufs offset] :result) + (defuvfsfn close [file] :result)) + + +(comment + (defprotocol IBlockingQueue + (add-item [this item]) + (remove-item [this])) + + (deftype BlockingQueue [items lock locked] + IBlockingQueue + (add-item [this item] + (enqueue items item) + (when @locked + (-release-lock lock) + (reset! locked false)) + (-yield-thread)) + (remove-item [this] + (when (empty? @items) + (reset! locked true) + (-acquire-lock lock true)) + (dequeue items))) + + (defn blocking-queue [] + (let [l (-create-lock)] + (-acquire-lock l true) + (->BlockingQueue (atom []) l (atom true)))) + + (def task-queue (blocking-queue)) + + + + +) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 30ab4933..cb1a8beb 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2117,3 +2117,8 @@ Expands to calls to `extend-type`." `(-dispose! ~nm)) names) result#))) + + +(defn partial [f & args] + (fn [& args2] + (apply f (-> args vec (into args2))))) From 4d0d836f620898c5eb387d553f72938adba147dc Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 5 Mar 2015 09:23:43 +0000 Subject: [PATCH 564/909] Sets the _name in BaseCode for MultiArityFns Passing an invalid number of args to a MultiArityFn should throw and error now which includes the name of the function. --- pixie/vm/code.py | 9 +++++---- pixie/vm/compiler.py | 2 +- pixie/vm/interpreter.py | 3 +-- pixie/vm/libs/string.py | 4 ++-- tests/pixie/tests/test-fns.pxi | 3 ++- 5 files changed, 11 insertions(+), 10 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 729ec4ea..7b87df5c 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -130,15 +130,16 @@ class MultiArityFn(BaseCode): def type(self): return MultiArityFn._type - def __init__(self, arities, required_arity=0, rest_fn=None, meta=nil): + def __init__(self, name, arities, required_arity=0, rest_fn=None, meta=nil): BaseCode.__init__(self) + self._name = name self._arities = arities self._required_arity = required_arity self._rest_fn = rest_fn self._meta = meta def with_meta(self, meta): - return MultiArityFn(self._arities, self._required_arity, self._rest_fn, meta) + return MultiArityFn(self._name, self._arities, self._required_arity, self._rest_fn, meta) @elidable_promote() def get_fn(self, arity): @@ -155,7 +156,7 @@ def get_fn(self, arity): if self._rest_fn: acc.append(u" or more") - affirm(False, u"Wrong number of args to fn: got " + unicode(str(arity)) + u" expected " + u",".join(acc)) + runtime_error(u"Wrong number of args to fn " + unicode(self._name) + " got " + unicode(str(arity)) + u" expected " + u",".join(acc)) def invoke(self, args): return self.invoke_with(args, self) @@ -212,7 +213,7 @@ def invoke(self, args): if len(args) == self.get_arity(): return self.invoke_with(args, self) else: - runtime_error(u"Invalid number of arguments " + unicode(len(args)) + runtime_error(u"Invalid number of arguments " + unicode(str(len(args))) + u" for function '" + unicode(str(self._name)) + u"'. Expected " + unicode(str(self.get_arity()))) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index dc36e169..afd06a1a 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -33,7 +33,7 @@ def gensym2(prefix): return rt.symbol(rt.str(prefix, i)) gensym = code.intern_var(u"pixie.stdlib", u"gensym") -gensym.set_root(code.MultiArityFn({0: code.wrap_fn(gensym1), 1: code.wrap_fn(gensym2)})) +gensym.set_root(code.MultiArityFn(u"gensym", {0: code.wrap_fn(gensym1), 1: code.wrap_fn(gensym2)})) class with_ns(object): def __init__(self, nm, include_stdlib=False): diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 9e80107a..a424e785 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -145,8 +145,7 @@ def make_multi_arity(frame, argc): else: fn = frame.pop() d[a] = fn - - return code.MultiArityFn(d, required_arity, rest_fn) + return code.MultiArityFn(fn._name, d, required_arity, rest_fn) class ShallowContinuation(Object): _type = Type(u"pixie.stdlib.ShallowContinuation") diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 13977765..967046c8 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -47,7 +47,7 @@ def index_of4(a, sep, start, end): runtime_error(u"Third and fourth argument must be non-negative integers") index_of = intern_var(u"pixie.string.internal", u"index-of") -index_of.set_root(MultiArityFn({2: wrap_fn(index_of2), 3: wrap_fn(index_of3), 4: wrap_fn(index_of4)}, +index_of.set_root(MultiArityFn(u"index-of", {2: wrap_fn(index_of2), 3: wrap_fn(index_of3), 4: wrap_fn(index_of4)}, required_arity = 2)) def substring2(a, start): @@ -64,7 +64,7 @@ def substring3(a, start, end): runtime_error(u"Second and third argument must be non-negative integers") substring = intern_var(u"pixie.string.internal", u"substring") -substring.set_root(MultiArityFn({2: wrap_fn(substring2), 3: wrap_fn(substring3)}, +substring.set_root(MultiArityFn(u"substring", {2: wrap_fn(substring2), 3: wrap_fn(substring3)}, required_arity = 2)) @as_var("pixie.string.internal", "upper-case") diff --git a/tests/pixie/tests/test-fns.pxi b/tests/pixie/tests/test-fns.pxi index 3f736858..a42bf20c 100644 --- a/tests/pixie/tests/test-fns.pxi +++ b/tests/pixie/tests/test-fns.pxi @@ -18,7 +18,8 @@ (t/deftest test-code-arity-errors (let [arity-0 (fn arity-0 []) arity-1 (fn arity-1 [a]) - arity-2 (fn arity-2 [a b])] + arity-2 (fn arity-2 [a b]) + multi-arity (fn arity-0-or-1)] (try (arity-0 :foo) (catch e From 757c5a246702042a81d1141ff4baaff278ea983e Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 5 Mar 2015 09:46:10 +0000 Subject: [PATCH 565/909] Proper errors for MultiArityFns with VariadicFns --- pixie/vm/code.py | 17 +++++++++++-- pixie/vm/interpreter.py | 4 +++- tests/pixie/tests/test-fns.pxi | 44 ++++++++++++++++++++++++++++++++-- 3 files changed, 60 insertions(+), 5 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 7b87df5c..ed17035f 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -101,6 +101,9 @@ def meta(self): def with_meta(self, meta): assert false, "not implemented" + def name(self): + return self._name + def set_macro(self): self._is_macro = True @@ -154,9 +157,9 @@ def get_fn(self, arity): acc.append(unicode(str(x))) if self._rest_fn: - acc.append(u" or more") + acc.append(unicode(str(self._rest_fn.required_arity())) + u" or more") - runtime_error(u"Wrong number of args to fn " + unicode(self._name) + " got " + unicode(str(arity)) + u" expected " + u",".join(acc)) + runtime_error(u"Wrong number of arguments " + unicode(str(arity)) + u" for function '" + unicode(self._name) + u"'. Expected " + u",".join(acc)) def invoke(self, args): return self.invoke_with(args, self) @@ -260,6 +263,12 @@ def __init__(self, code, required_arity, meta=nil): def with_meta(self, meta): return VariadicCode(self._code, self._required_arity, meta) + + def name(self): + return None + + def required_arity(self): + return self._required_arity def invoke(self, args): return self.invoke_with(args, self) @@ -298,6 +307,10 @@ def __init__(self, code, closed_overs, meta=nil): def with_meta(self, meta): return Closure(self._code, self._closed_overs, meta) + + def name(self): + return None + def invoke(self, args): return self.invoke_with(args, self) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index a424e785..bbc96324 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -136,6 +136,7 @@ def make_multi_arity(frame, argc): d = {} required_arity = 0 rest_fn = None + fn_name = None for i in range(argc): a = frame.get_inst() if a & 256: @@ -144,8 +145,9 @@ def make_multi_arity(frame, argc): rest_fn = frame.pop() else: fn = frame.pop() + fn_name = fn.name() d[a] = fn - return code.MultiArityFn(fn._name, d, required_arity, rest_fn) + return code.MultiArityFn(fn_name, d, required_arity, rest_fn) class ShallowContinuation(Object): _type = Type(u"pixie.stdlib.ShallowContinuation") diff --git a/tests/pixie/tests/test-fns.pxi b/tests/pixie/tests/test-fns.pxi index a42bf20c..b5f2e8e2 100644 --- a/tests/pixie/tests/test-fns.pxi +++ b/tests/pixie/tests/test-fns.pxi @@ -19,7 +19,10 @@ (let [arity-0 (fn arity-0 []) arity-1 (fn arity-1 [a]) arity-2 (fn arity-2 [a b]) - multi-arity (fn arity-0-or-1)] + arity-0-or-1 (fn arity-0-or-1 ([]) ([a])) + arity-1-or-3 (fn arity-1-or-3 ([a]) ([a b c])) + arity-0-or-1-or-3-or-more + (fn arity-0-or-1-or-3-or-more ([]) ([a]) ([a b c & more]))] (try (arity-0 :foo) (catch e @@ -55,4 +58,41 @@ (catch e (t/assert= (ex-msg e) - "Invalid number of arguments 1 for function 'arity-2'. Expected 2"))))) + "Invalid number of arguments 1 for function 'arity-2'. Expected 2"))) + + (try + (arity-0-or-1 :foo :bar) + (catch e + (t/assert= + (ex-msg e) + "Wrong number of arguments 2 for function 'arity-0-or-1'. Expected 1,0"))) + + (try + (arity-0-or-1 :foo :bar :baz) + (catch e + (t/assert= + (ex-msg e) + "Wrong number of arguments 3 for function 'arity-0-or-1'. Expected 1,0"))) + + (try + (arity-1-or-3 :foo :bar) + (catch e + (t/assert= + (ex-msg e) + "Wrong number of arguments 2 for function 'arity-1-or-3'. Expected 3,1"))) + + (try + (arity-1-or-3) + (catch e + (t/assert= + (ex-msg e) + "Wrong number of arguments 0 for function 'arity-1-or-3'. Expected 3,1"))) + + (try + (arity-0-or-1-or-3-or-more) + (catch e + (t/assert= + (ex-msg e) + "Wrong number of arguments 0 for function 'arity-0-or-1-or-3-or-more'. Expected 0, 1, 3 or more"))) + + )) From 7191e20433cbb8ab5a234c1999af03723976c806 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 5 Mar 2015 21:37:02 +0000 Subject: [PATCH 566/909] add a assert-throws? function --- pixie/test.pxi | 12 ++++ tests/pixie/tests/test-fns.pxi | 106 ++++++++++----------------------- 2 files changed, 45 insertions(+), 73 deletions(-) diff --git a/pixie/test.pxi b/pixie/test.pxi index 41c14914..cf77ccf2 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -61,10 +61,22 @@ yr# ~y] (assert (= xr# yr#) (str (show '~x xr#) " != " (show '~y yr#))))) +(defmacro assert-throws? [klass msg body] + `(try + ~body + (assert false (str "Expected a " ~klass " exception: " ~msg)) + (catch e# + (assert (= (type e#) ~klass) + (str "Expected exception of class " ~klass " but got " (type e#))) + (assert (= (ex-msg e#) ~msg) + (str "Expected message: " ~msg " but got " (ex-msg e#)))))) + (defmacro assert [x] `(let [x# ~x] (assert x# (str '~x " is " x#)))) + + (defn show ([orig res] (if (= orig res) diff --git a/tests/pixie/tests/test-fns.pxi b/tests/pixie/tests/test-fns.pxi index b5f2e8e2..12f74bc9 100644 --- a/tests/pixie/tests/test-fns.pxi +++ b/tests/pixie/tests/test-fns.pxi @@ -23,76 +23,36 @@ arity-1-or-3 (fn arity-1-or-3 ([a]) ([a b c])) arity-0-or-1-or-3-or-more (fn arity-0-or-1-or-3-or-more ([]) ([a]) ([a b c & more]))] - (try - (arity-0 :foo) - (catch e - (t/assert= - (ex-msg e) - "Invalid number of arguments 1 for function 'arity-0'. Expected 0"))) - (try - (arity-0 :foo :bar) - (catch e - (t/assert= - (ex-msg e) - "Invalid number of arguments 2 for function 'arity-0'. Expected 0"))) - (try - (arity-1) - (catch e - (t/assert= - (ex-msg e) - "Invalid number of arguments 0 for function 'arity-1'. Expected 1"))) - (try - (arity-1 :foo :bar) - (catch e - (t/assert= - (ex-msg e) - "Invalid number of arguments 2 for function 'arity-1'. Expected 1"))) - (try - (arity-2) - (catch e - (t/assert= - (ex-msg e) - "Invalid number of arguments 0 for function 'arity-2'. Expected 2"))) - (try - (arity-2 :foo) - (catch e - (t/assert= - (ex-msg e) - "Invalid number of arguments 1 for function 'arity-2'. Expected 2"))) - - (try - (arity-0-or-1 :foo :bar) - (catch e - (t/assert= - (ex-msg e) - "Wrong number of arguments 2 for function 'arity-0-or-1'. Expected 1,0"))) - - (try - (arity-0-or-1 :foo :bar :baz) - (catch e - (t/assert= - (ex-msg e) - "Wrong number of arguments 3 for function 'arity-0-or-1'. Expected 1,0"))) - - (try - (arity-1-or-3 :foo :bar) - (catch e - (t/assert= - (ex-msg e) - "Wrong number of arguments 2 for function 'arity-1-or-3'. Expected 3,1"))) - - (try - (arity-1-or-3) - (catch e - (t/assert= - (ex-msg e) - "Wrong number of arguments 0 for function 'arity-1-or-3'. Expected 3,1"))) - - (try - (arity-0-or-1-or-3-or-more) - (catch e - (t/assert= - (ex-msg e) - "Wrong number of arguments 0 for function 'arity-0-or-1-or-3-or-more'. Expected 0, 1, 3 or more"))) - - )) + (t/assert-throws? RuntimeException + "Invalid number of arguments 1 for function 'arity-0'. Expected 0" + (arity-0 :foo)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 2 for function 'arity-0'. Expected 0" + (arity-0 :foo :bar)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 0 for function 'arity-1'. Expected 1" + (arity-1)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 2 for function 'arity-1'. Expected 1" + (arity-1 :foo :bar)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 0 for function 'arity-2'. Expected 2" + (arity-2)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 1 for function 'arity-2'. Expected 2" + (arity-2 :foo)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 2 for function 'arity-0-or-1'. Expected 1,0" + (arity-0-or-1 :foo :bar)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 3 for function 'arity-0-or-1'. Expected 1,0" + (arity-0-or-1 :foo :bar :baz)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 2 for function 'arity-1-or-3'. Expected 3,1" + (arity-1-or-3 :foo :bar)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 0 for function 'arity-1-or-3'. Expected 3,1" + (arity-1-or-3)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 2 for function 'arity-0-or-1-or-3-or-more'. Expected 1,0,3 or more" + (arity-0-or-1-or-3-or-more :foo :bar)))) From df8214b84163e53ba20a5a5f4bb5842ae0d38608 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 6 Mar 2015 15:43:32 -0700 Subject: [PATCH 567/909] more work on libuv ffi, it's getting there --- pixie/ffi-infer.pxi | 41 ++++++++++++++++---------- pixie/stacklets.pxi | 70 ++++++++++++++++++++++++-------------------- pixie/uv.pxi | 16 ++++++++-- pixie/vm/libs/ffi.py | 29 ++++++++++++++---- 4 files changed, 100 insertions(+), 56 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 49578c20..cea11bc6 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -81,26 +81,30 @@ return 0; ;; To Ctype conversion -(defmulti edn-to-ctype :type) +(defmulti edn-to-ctype (fn [n _] + (:type n))) -(defn callback-type [{:keys [arguments returns]}] - `(ffi-callback ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns))) +(defn callback-type [{:keys [arguments returns]} in-struct?] + `(ffi-callback ~(vec (map (fn [x] (edn-to-ctype x in-struct?)) + arguments)) + ~(edn-to-ctype returns in-struct?))) (defmethod edn-to-ctype :pointer - [{:keys [of-type] :as ptr}] + [{:keys [of-type] :as ptr} in-struct?] (cond - (= of-type {:signed? true :size 1 :type :int}) 'pixie.stdlib/CCharP - (= (:type of-type) :function) (callback-type of-type) + (and (= of-type {:signed? true :size 1 :type :int}) + (not in-struct?)) 'pixie.stdlib/CCharP + (= (:type of-type) :function) (callback-type of-type in-struct?) :else 'pixie.stdlib/CVoidP)) (defmethod edn-to-ctype :float - [{:keys [size]}] + [{:keys [size]} _] (cond (= size 8) 'pixie.stdlib/CDouble :else (assert False "unknown type"))) (defmethod edn-to-ctype :void - [_] + [_ _] `pixie.stdlib/CVoid) (def int-types {[8 true] 'pixie.stdlib/CInt8 @@ -113,7 +117,7 @@ return 0; [64 false] 'pixie.stdlib/CUInt64}) (defmethod edn-to-ctype :int - [{:keys [size signed?] :as tp}] + [{:keys [size signed?] :as tp} _] (let [tp-found (get int-types [(* 8 size) signed?])] (assert tp-found (str "No type found for " tp)) tp-found)) @@ -132,33 +136,36 @@ return 0; [{:keys [name]} {:keys [type arguments returns]}] (assert (= type :function) (str name " is not infered to be a function")) `(def ~(symbol name) - (ffi-fn *library* ~name ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns)))) + (ffi-fn *library* ~name + ~(vec (map (fn [x] (edn-to-ctype x false)) + arguments)) + ~(edn-to-ctype returns false)))) (defmethod generate-code :raw-struct [{:keys [name members]} {:keys [size infered-members]}] `(def ~(symbol name) (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] - `[~(keyword name) ~(edn-to-ctype type) ~offset]) + `[~(keyword name) ~(edn-to-ctype type true) ~offset]) members infered-members)]))) (defmethod generate-code :struct [{:keys [name members]} {:keys [size infered-members]}] `(def ~(symbol name) (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] - `[~(keyword name) ~(edn-to-ctype type) ~offset]) + `[~(keyword name) ~(edn-to-ctype type true) ~offset]) members infered-members)]))) (defmethod generate-code :type [{:keys [name members]} {:keys [size infered-members]}] `(def ~(symbol name) (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] - `[~(keyword name) ~(edn-to-ctype type) ~offset]) + `[~(keyword name) ~(edn-to-ctype type true) ~offset]) members infered-members)]))) (defmethod generate-code :callback [{:keys [name]} {:keys [of-type]}] `(def ~(symbol name) - ~(callback-type of-type))) + ~(callback-type of-type false))) (defn run-infer [config cmds] @@ -175,8 +182,10 @@ return 0; (apply str " " (interpose " " (:cxx-flags *config*))) " -o /tmp/a.out && /tmp/a.out") _ (println cmd-str) - result (read-string (io/run-command cmd-str))] - `(do ~@(map generate-code cmds result)))) + result (read-string (io/run-command cmd-str)) + gen (vec (map generate-code cmds result))] + (println gen) + `(do ~@gen))) (defn full-lib-name [library-name] (if (= library-name "c") diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index f6d9144d..b36bf6c9 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -39,16 +39,19 @@ (let [[h [op args]] (k args)] (async-fn op args h))) -(defn run-and-process [k] - (let [[h f] (k nil)] - (f h))) +(defn run-and-process + ([k] + (run-and-process k nil)) + ([k val] + (let [[h f] (k val)] + (f h)))) ;; Yield (defn switch-back [f] - (let [[h] (@stacklet-loop-h f)] + (let [[h val] (@stacklet-loop-h f)] (reset! stacklet-loop-h h) - nil)) + val)) (defn -run-later [f] (let [a (uv/uv_async_t) @@ -81,11 +84,6 @@ ;;; Sleep -(defn sleep [ms] - (let [[h] (@stacklet-loop-h [:sleep ms])] - (reset! stacklet-loop-h h) - nil)) - (defn sleep [ms] (let [f (fn [k] (let [cb (atom nil) @@ -102,6 +100,7 @@ (uv/uv_timer_start timer @cb ms 0)))] (switch-back f))) +;; Spawn (defn -spawn [start-fn] (switch-back (fn [k] (-run-later (fn [] @@ -119,26 +118,26 @@ (defmacro defuvfsfn [nm args return] - (let [kw (keyword (str "pixie.uv/" (name nm)))] - `(do (defn ~nm ~args - (let [[h# result#] (@stacklet-loop-h [~kw ~args])] - (reset! stacklet-loop-h h#) - result#)) - (defmethod async-fn ~kw - [f# ~args k# tasks#] - (let [cb# (atom nil)] - (reset! cb# (ffi-prep-callback uv/uv_fs_cb - (fn [req#] - (try - (enqueue tasks# [k# (~return (pixie.ffi/cast req# uv/uv_fs_t))]) - (uv/uv_fs_req_cleanup req#) - (-dispose! @cb#) - (catch e (println e)))))) - (~(symbol (str "pixie.uv/uv_fs_" (name nm))) - (uv/uv_default_loop) - (uv/uv_fs_t) - ~@args - @cb#)))))) + `(defn ~nm ~args + (let [f (fn [k#] + (let [cb# (atom nil)] + (reset! cb# (ffi-prep-callback uv/uv_fs_cb + (fn [req#] + (try + (run-and-process k# (~return (pixie.ffi/cast req# uv/uv_fs_t))) + (uv/uv_fs_req_cleanup req#) + (-dispose! @cb#) + (catch e (println e)))))) + (~(symbol (str "pixie.uv/uv_" (name nm))) + (uv/uv_default_loop) + (uv/uv_fs_t) + ~@args + @cb#)))] + (switch-back f)))) + +(defuvfsfn fs_open [path flags mode] :result) +(defuvfsfn fs_read [file bufs nbufs offset] :result) +(defuvfsfn fs_close [file] :result) (defn -with-stacklets [fn] @@ -157,8 +156,15 @@ (println e)))))) (with-stacklets - (dotimes [x (* 1024 10)] - (spawn 1))) + (let [f (fs_open "/tmp/foo.txt" uv/O_RDONLY uv/S_IRUSR) + b (uv/new-fs-buf 1024)] + (println (type (:base b))) + (let [err (fs_read f b 1 0)] + (println err) + (assert (pos? err))) + (dotimes [x 100] + (puts (str (char (pixie.ffi/unpack (:base b) x CUInt8))))) + (println "done"))) (comment diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 85079189..877274c5 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -1,7 +1,7 @@ (ns pixie.uv (require pixie.ffi-infer :as f)) -(f/with-config {:library "uv" +(f/with-config {:library "uv" :includes ["uv.h"]} (f/defconst UV_RUN_DEFAULT) (f/defconst UV_RUN_ONCE) @@ -115,6 +115,8 @@ (f/defcstruct uv_dirent_t [:name :type]) + (f/defcstruct uv_buf_t [:base :len]) + (f/defcfn uv_fs_req_cleanup) (f/defcfn uv_fs_close) (f/defcfn uv_fs_open) @@ -163,9 +165,19 @@ (f/defconst UV_E2BIG) (f/defconst UV_EACCES) + (f/defcfn uv_err_name) ; async (f/defcstruct uv_async_t []) (f/defccallback uv_async_cb) (f/defcfn uv_async_init) - (f/defcfn uv_async_send)) + (f/defcfn uv_async_send) + ) + + +(defn new-fs-buf [size] + (let [b (buffer size) + bt (uv_buf_t)] + (pixie.ffi/set! bt :base b) + (pixie.ffi/set! bt :len size) + bt)) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 33d43d8e..fdb17b1b 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -359,11 +359,28 @@ def ffi_get_value(self, ptr): return String(unicode(rffi.charp2str(casted[0]))) def ffi_set_value(self, ptr, val): - pnt = rffi.cast(rffi.CCHARPP, ptr) - utf8 = unicode_to_utf8(rt.name(val)) - raw = rffi.str2charp(utf8) - pnt[0] = raw - return CCharPToken(raw) + if isinstance(val, String): + pnt = rffi.cast(rffi.CCHARPP, ptr) + utf8 = unicode_to_utf8(rt.name(val)) + raw = rffi.str2charp(utf8) + pnt[0] = raw + return CCharPToken(raw) + elif isinstance(val, Buffer): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = val.buffer() + elif isinstance(val, VoidP): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = val.raw_data() + elif val is nil: + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = rffi.cast(rffi.VOIDP, 0) + elif isinstance(val, CStruct): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) + else: + print val + affirm(False, u"Cannot encode this type") + def ffi_size(self): return rffi.sizeof(rffi.CCHARP) @@ -642,7 +659,7 @@ def set_val(self, k, v): (tp, offset) = self._type.get_desc(k) if tp is None: - runtime_error(u"Invalid field name: " + rt.name(rt.str(tp))) + runtime_error(u"Invalid field name: " + rt.name(rt.str(k))) offset = rffi.ptradd(self._buffer, offset) tp.ffi_set_value(rffi.cast(rffi.VOIDP, offset), v) From e5bb1211477a165534f93faf806768104ad39516 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 6 Mar 2015 16:37:04 -0700 Subject: [PATCH 568/909] implemented promises --- pixie/stacklets.pxi | 122 +++++++++++++++++++++----------------------- 1 file changed, 59 insertions(+), 63 deletions(-) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index b36bf6c9..a2babc2a 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -7,48 +7,18 @@ (def stacklet-loop-h (atom nil)) -(def thread-count (atom 0)) -(defmulti async-fn (fn [f args k] f)) -(defmethod async-fn :spawn-end - [_ _ _] - (swap! thread-count dec) - (when (= @thread-count 0) - (uv/uv_stop (uv/uv_default_loop)))) - -(def tasks (atom [])) - -(defn enqueue [q itm] - ; TODO: Rewrite this crappy impl - (swap! q (fn [q] (vec (conj (seq q) itm))))) - -(defn dequeue [q] - (let [itm (ith @q -1)] - (swap! q pop) - itm)) - -(comment - (defn run-and-process [k args] - - (let [[h [op args]] (k args)] - (async-fn op args h)))) - -(defn -run-and-process [k args] - - (let [[h [op args]] (k args)] - (async-fn op args h))) +;; Yield (defn run-and-process ([k] (run-and-process k nil)) ([k val] - (let [[h f] (k val)] - (f h)))) - -;; Yield + (let [[h f] (k val)] + (f h)))) -(defn switch-back [f] +(defn call-cc [f] (let [[h val] (@stacklet-loop-h f)] (reset! stacklet-loop-h h) val)) @@ -68,20 +38,11 @@ (defn yield-control [] - (switch-back (fn [k] - (-run-later (partial run-and-process k))))) + (call-cc (fn [k] + (-run-later (partial run-and-process k))))) (def close_cb (ffi-prep-callback uv/uv_close_cb - (fn [handle] - (pixie.ffi/free handle) - ))) - -(def tasks (atom [])) - -(defmethod async-fn :yield - [_ args k] - (add-item task-queue [k args])) - + pixie.ffi/free)) ;;; Sleep (defn sleep [ms] @@ -98,21 +59,21 @@ (println ex)))))) (uv/uv_timer_init (uv/uv_default_loop) timer) (uv/uv_timer_start timer @cb ms 0)))] - (switch-back f))) + (call-cc f))) ;; Spawn (defn -spawn [start-fn] - (switch-back (fn [k] - (-run-later (fn [] - (run-and-process (new-stacklet start-fn)))) - (-run-later (partial run-and-process k))))) + (call-cc (fn [k] + (-run-later (fn [] + (run-and-process (new-stacklet start-fn)))) + (-run-later (partial run-and-process k))))) (defmacro spawn [& body] `(-spawn (fn [h# _] (try (reset! stacklet-loop-h h#) (let [result# (do ~@body)] - (switch-back (fn [_] nil))) + (call-cc (fn [_] nil))) (catch e (println e)))))) @@ -133,7 +94,7 @@ (uv/uv_fs_t) ~@args @cb#)))] - (switch-back f)))) + (call-cc f)))) (defuvfsfn fs_open [path flags mode] :result) (defuvfsfn fs_read [file bufs nbufs offset] :result) @@ -151,20 +112,55 @@ (try (reset! stacklet-loop-h h#) (let [result# (do ~@body)] - (switch-back (fn [_] nil))) + (call-cc (fn [_] nil))) (catch e (println e)))))) + + +(deftype Promise [val pending-callbacks delivered?] + IDeref + (-deref [self] + (if delivered? + val + (do + (call-cc (fn [k] + (swap! pending-callbacks conj + (fn [v] + (-run-later (partial run-and-process k v))))))))) + IFn + (-invoke [self v] + (assert (not delivered?) "Can only deliver a promise once") + (set-field! self :val v) + (println @pending-callbacks) + (doseq [f @pending-callbacks] + (f v)) + (reset! pending-callbacks nil) + nil)) + +(defn promise [] + (->Promise nil (atom []) false)) + (with-stacklets - (let [f (fs_open "/tmp/foo.txt" uv/O_RDONLY uv/S_IRUSR) - b (uv/new-fs-buf 1024)] - (println (type (:base b))) - (let [err (fs_read f b 1 0)] - (println err) - (assert (pos? err))) - (dotimes [x 100] - (puts (str (char (pixie.ffi/unpack (:base b) x CUInt8))))) - (println "done"))) + (let [p (promise)] + (spawn @p) + (spawn @p) + (spawn (p 42)) + @p)) + +(comment + + + (with-stacklets + (let [f (fs_open "/tmp/foo.txt" uv/O_RDONLY uv/S_IRUSR) + b (uv/new-fs-buf 1024)] + (println (type (:base b))) + (let [err (fs_read f b 1 0)] + (println err) + (assert (pos? err))) + (dotimes [x 100] + (puts (str (char (pixie.ffi/unpack (:base b) x CUInt8))))) + (println "done")))) (comment From 38271ac70f9f5b058d8a61087269b6bf592ad4cf Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sun, 8 Mar 2015 16:23:39 -0600 Subject: [PATCH 569/909] move blocking io code into ffi-blocking --- pixie/ffi-infer.pxi | 3 +- pixie/io.pxi | 140 ---------------------------------- pixie/stdlib.pxi | 23 ++++-- tests/pixie/tests/test-io.pxi | 2 +- 4 files changed, 17 insertions(+), 151 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index cea11bc6..9a2d54b3 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -1,5 +1,5 @@ (ns pixie.ffi-infer - (require pixie.io :as io)) + (require pixie.io-blocking :as io)) (defn -add-rel-path [rel] (swap! load-paths conj (str (first @load-paths) "/" rel))) @@ -184,7 +184,6 @@ return 0; _ (println cmd-str) result (read-string (io/run-command cmd-str)) gen (vec (map generate-code cmds result))] - (println gen) `(do ~@gen))) (defn full-lib-name [library-name] diff --git a/pixie/io.pxi b/pixie/io.pxi index 46524dbf..efb24575 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -9,7 +9,6 @@ (def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) (def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) - (defprotocol IInputStream (read-byte [this] "Read a single character") (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) @@ -20,142 +19,3 @@ (defprotocol IClosable (close [this] "Closes the stream")) - -(def DEFAULT-BUFFER-SIZE 1024) - -(deftype FileStream [fp] - IInputStream - (read [this buffer len] - (assert (<= (buffer-capacity buffer) len) - "Not enough capacity in the buffer") - (let [read-count (fread buffer 1 len fp)] - (set-buffer-count! buffer read-count) - read-count)) - (read-byte [this] - (fgetc buffer)) - IClosable - (close [this] - (fclose fp)) - IReduce - (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) - -(defn open-read - {:doc "Open a file for reading, returning a IInputStream" - :added "0.1"} - [filename] - (assert (string? filename) "Filename must be a string") - (->FileStream (fopen filename "r"))) - -(defn read-line - "Read one line from input-stream for each invocation. - nil when all lines have been read" - [input-stream] - (let [line-feed (into #{} (map int [\newline \return])) - buf (buffer 1)] - (loop [acc []] - (let [len (read input-stream buf 1)] - (cond - (and (pos? len) (not (line-feed (first buf)))) - (recur (conj acc (first buf))) - - (and (zero? len) (empty? acc)) nil - - :else (apply str (map char acc))))))) - -(defn line-seq - "Returns the lines of text from input-stream as a lazy sequence of strings. - input-stream must implement IInputStream" - [input-stream] - (when-let [line (read-line input-stream)] - (cons line (lazy-seq (line-seq input-stream))))) - -(deftype FileOutputStream [fp] - IOutputStream - (write-byte [this val] - (assert (integer? val) "Value must be a int") - (fputc val fp)) - (write [this buffer] - (fwrite buffer 1 (count buffer) fp)) - IClosable - (close [this] - (fclose fp))) - -(defn file-output-rf [filename] - (let [fp (->FileOutputStream (fopen filename "w"))] - (fn ([] 0) - ([cnt] (close fp) cnt) - ([cnt chr] - (assert (integer? chr)) - (let [written (write-byte fp chr)] - (if (= written 0) - (reduced cnt) - (+ cnt written))))))) - - -(defn spit [filename val] - (transduce (map int) - (file-output-rf filename) - (str val))) - -(defn slurp [filename] - (let [c (->FileStream (fopen filename "r")) - result (transduce - (map char) - string-builder - c)] - (close c) - result)) - -(deftype ProcessInputStream [fp] - IInputStream - (read [this buffer len] - (assert (<= (buffer-capacity buffer) len) - "Not enough capacity in the buffer") - (let [read-count (fread buffer 1 len fp)] - (set-buffer-count! buffer read-count) - read-count)) - (read-byte [this] - (fgetc fp)) - IClosable - (close [this] - (pclose fp)) - IReduce - (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) - - -(defn popen-read - {:doc "Open a file for reading, returning a IInputStream" - :added "0.1"} - [command] - (assert (string? command) "Command must be a string") - (->ProcessInputStream (popen command "r"))) - - -(defn run-command [command] - (let [c (->ProcessInputStream (popen command "r")) - result (transduce - (map char) - string-builder - c)] - (close c) - result)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index cb1a8beb..0ac534da 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1075,12 +1075,6 @@ Creates new maps if the keys are not present." (pop-binding-frame!) ret)))) -(defmacro require [ns kw as-nm] - (assert (= kw :as) "Require expects :as as the second argument") - `(do (load-ns (quote ~ns)) - (assert (the-ns (quote ~ns)) (str "Couldn't find the namespace " (quote ~ns) " after loading the file")) - (refer-ns (this-ns-name) (the-ns (quote ~ns)) (quote ~as-nm)))) - (defmacro ns [nm & body] `(do (in-ns ~(keyword (name nm))) ~@body)) @@ -1137,7 +1131,7 @@ Creates new maps if the keys are not present." (protocol? @(resolve-in *ns* body)) [@(resolve-in *ns* body) (second res) (conj (third res) body)] - :else (throw (str "can only extend protocols or Object, not " body))) + :else (throw (str "can only extend protocols or Object, not " body " of type " (type body)))) (seq? body) (let [proto (first res) tbs (second res) pbs (third res)] (if (protocol? proto) [proto tbs (conj pbs body)] @@ -1869,9 +1863,11 @@ user => (refer 'pixie.string :exclude '(substring))" exclude (set (:exclude filters)) refers (if (= :all (:refer filters)) (keys nsmap) - (or (:refer filters) (:only filters) (keys nsmap)))] + (or (:refer filters) (:only filters)))] (when (and refers (not (satisfies? ISeqable refers))) (throw ":only/:refer must be a collection of symbols")) + (when-let [as (:as filters)] + (refer-ns *ns* ns-sym as)) (loop [syms (seq refers)] (if (not syms) nil @@ -1885,6 +1881,17 @@ user => (refer 'pixie.string :exclude '(substring))" (recur (next syms))))) nil)) + + +(defmacro require [ns & args] + `(do (load-ns (quote ~ns)) + (assert (the-ns (quote ~ns)) + (str "Couldn't find the namespace " (quote ~ns) " after loading the file")) + + (apply refer (quote [~ns ~@args])))) + + + (extend -iterator ISeq (fn [s] (loop [s s] (when s diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 36ffa040..86a238b8 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -1,6 +1,6 @@ (ns pixie.tests.test-io (require pixie.test :as t) - (require pixie.io :as io)) + (require pixie.io-blocking :as io)) (t/deftest test-file-reduction (let [f (io/open-read "tests/pixie/tests/test-io.txt")] From 249463e1c53baa51e97c0d83196de10118eeeab0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 9 Mar 2015 07:06:10 -0600 Subject: [PATCH 570/909] async slurp implemented --- pixie/io-blocking.pxi | 153 +++++++++++++++++++++++++++++++++ pixie/io.pxi | 194 +++++++++++++++++++++++++++++++++++++----- pixie/stacklets.pxi | 36 ++------ pixie/streams.pxi | 21 +++++ target.py | 10 ++- 5 files changed, 363 insertions(+), 51 deletions(-) create mode 100644 pixie/io-blocking.pxi create mode 100644 pixie/streams.pxi diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi new file mode 100644 index 00000000..f5565373 --- /dev/null +++ b/pixie/io-blocking.pxi @@ -0,0 +1,153 @@ +(ns pixie.io-blocking + (require pixie.streams :as st :refer :all)) + + +(def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) +(def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) +(def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) +(def fputc (ffi-fn libc "fputc" [CInt CVoidP] CInt)) +(def fwrite (ffi-fn libc "fwrite" [CVoidP CInt CInt CVoidP] CInt)) +(def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) +(def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) +(def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) + + + +(def DEFAULT-BUFFER-SIZE 1024) + +(deftype FileStream [fp] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [read-count (fread buffer 1 len fp)] + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (fgetc buffer)) + IClosable + (close [this] + (fclose fp)) + IReduce + (-reduce [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc)))))) + +(defn open-read + {:doc "Open a file for reading, returning a IInputStream" + :added "0.1"} + [filename] + (assert (string? filename) "Filename must be a string") + (->FileStream (fopen filename "r"))) + +(defn read-line + "Read one line from input-stream for each invocation. + nil when all lines have been read" + [input-stream] + (let [line-feed (into #{} (map int [\newline \return])) + buf (buffer 1)] + (loop [acc []] + (let [len (read input-stream buf 1)] + (cond + (and (pos? len) (not (line-feed (first buf)))) + (recur (conj acc (first buf))) + + (and (zero? len) (empty? acc)) nil + + :else (apply str (map char acc))))))) + +(defn line-seq + "Returns the lines of text from input-stream as a lazy sequence of strings. + input-stream must implement IInputStream" + [input-stream] + (when-let [line (read-line input-stream)] + (cons line (lazy-seq (line-seq input-stream))))) + +(deftype FileOutputStream [fp] + IOutputStream + (write-byte [this val] + (assert (integer? val) "Value must be a int") + (fputc val fp)) + (write [this buffer] + (fwrite buffer 1 (count buffer) fp)) + IClosable + (close [this] + (fclose fp))) + +(defn file-output-rf [filename] + (let [fp (->FileOutputStream (fopen filename "w"))] + (fn ([] 0) + ([cnt] (close fp) cnt) + ([cnt chr] + (assert (integer? chr)) + (let [written (write-byte fp chr)] + (if (= written 0) + (reduced cnt) + (+ cnt written))))))) + + +(defn spit [filename val] + (transduce (map int) + (file-output-rf filename) + (str val))) + +(defn slurp [filename] + (let [c (->FileStream (fopen filename "r")) + result (transduce + (map char) + string-builder + c)] + (close c) + result)) + +(deftype ProcessInputStream [fp] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [read-count (fread buffer 1 len fp)] + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (fgetc fp)) + IClosable + (close [this] + (pclose fp)) + IReduce + (-reduce [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc)))))) + + +(defn popen-read + {:doc "Open a file for reading, returning a IInputStream" + :added "0.1"} + [command] + (assert (string? command) "Command must be a string") + (->ProcessInputStream (popen command "r"))) + + +(defn run-command [command] + (let [c (->ProcessInputStream (popen command "r")) + result (transduce + (map char) + string-builder + c)] + (close c) + result)) diff --git a/pixie/io.pxi b/pixie/io.pxi index efb24575..c9e3ae8f 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -1,21 +1,173 @@ -(ns pixie.io) - -(def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) -(def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) -(def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) -(def fputc (ffi-fn libc "fputc" [CInt CVoidP] CInt)) -(def fwrite (ffi-fn libc "fwrite" [CVoidP CInt CInt CVoidP] CInt)) -(def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) -(def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) -(def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) - -(defprotocol IInputStream - (read-byte [this] "Read a single character") - (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) - -(defprotocol IOutputStream - (write-byte [this byte]) - (write [this buffer])) - -(defprotocol IClosable - (close [this] "Closes the stream")) +(ns pixie.io + (require pixie.streams :as st :refer :all) + (require pixie.uv :as uv) + (require pixie.stacklets :as st)) + +(defmacro defuvfsfn [nm args return] + `(defn ~nm ~args + (let [f (fn [k#] + (let [cb# (atom nil)] + (reset! cb# (ffi-prep-callback uv/uv_fs_cb + (fn [req#] + (try + (st/run-and-process k# (~return (pixie.ffi/cast req# uv/uv_fs_t))) + (uv/uv_fs_req_cleanup req#) + (-dispose! @cb#) + (catch e (println e)))))) + (~(symbol (str "pixie.uv/uv_" (name nm))) + (uv/uv_default_loop) + (uv/uv_fs_t) + ~@args + @cb#)))] + (st/call-cc f)))) + +(defuvfsfn fs_open [path flags mode] :result) +(defuvfsfn fs_read [file bufs nbufs offset] :result) +(defuvfsfn fs_close [file] :result) + + +(def DEFAULT-BUFFER-SIZE 1024) + +(deftype FileStream [fp offset uvbuf] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [_ (pixie.ffi/set! uvbuf :base buffer) + _ (pixie.ffi/set! uvbuf :len (buffer-capacity buffer)) + read-count (fs_read fp uvbuf 1 offset)] + (assert (not (neg? read-count)) "Read Error") + (set-field! this :offset (+ offset read-count)) + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (assert false "Does not support read-byte, wrap in a buffering reader")) + IClosable + (close [this] + (pixie.ffi/free uvbuf) + (fs_close fp)) + IReduce + (-reduce [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc)))))) + +(defn open-read + {:doc "Open a file for reading, returning a IInputStream" + :added "0.1"} + [filename] + (assert (string? filename) "Filename must be a string") + (->FileStream (fs_open filename uv/O_RDONLY 0) 0 (uv/uv_buf_t))) + +(defn read-line + "Read one line from input-stream for each invocation. + nil when all lines have been read" + [input-stream] + (let [line-feed (into #{} (map int [\newline \return])) + buf (buffer 1)] + (loop [acc []] + (let [len (read input-stream buf 1)] + (cond + (and (pos? len) (not (line-feed (first buf)))) + (recur (conj acc (first buf))) + + (and (zero? len) (empty? acc)) nil + + :else (apply str (map char acc))))))) + +(defn line-seq + "Returns the lines of text from input-stream as a lazy sequence of strings. + input-stream must implement IInputStream" + [input-stream] + (when-let [line (read-line input-stream)] + (cons line (lazy-seq (line-seq input-stream))))) + +(deftype FileOutputStream [fp] + IOutputStream + (write-byte [this val] + (assert (integer? val) "Value must be a int") + (fputc val fp)) + (write [this buffer] + (fwrite buffer 1 (count buffer) fp)) + IClosable + (close [this] + (fclose fp))) + +(defn file-output-rf [filename] + (let [fp (->FileOutputStream (fopen filename "w"))] + (fn ([] 0) + ([cnt] (close fp) cnt) + ([cnt chr] + (assert (integer? chr)) + (let [written (write-byte fp chr)] + (if (= written 0) + (reduced cnt) + (+ cnt written))))))) + + +(defn spit [filename val] + (transduce (map int) + (file-output-rf filename) + (str val))) + +(defn slurp [filename] + (let [c (open-read filename) + result (transduce + (map char) + string-builder + c)] + (close c) + result)) + +(println (slurp "/tmp/a.txt")) + +(deftype ProcessInputStream [fp] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [read-count (fread buffer 1 len fp)] + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (fgetc fp)) + IClosable + (close [this] + (pclose fp)) + IReduce + (-reduce [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc)))))) + + +(defn popen-read + {:doc "Open a file for reading, returning a IInputStream" + :added "0.1"} + [command] + (assert (string? command) "Command must be a string") + (->ProcessInputStream (popen command "r"))) + + +(defn run-command [command] + (let [c (->ProcessInputStream (popen command "r")) + result (transduce + (map char) + string-builder + c)] + (close c) + result)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index a2babc2a..ab101bb2 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -78,27 +78,6 @@ (println e)))))) -(defmacro defuvfsfn [nm args return] - `(defn ~nm ~args - (let [f (fn [k#] - (let [cb# (atom nil)] - (reset! cb# (ffi-prep-callback uv/uv_fs_cb - (fn [req#] - (try - (run-and-process k# (~return (pixie.ffi/cast req# uv/uv_fs_t))) - (uv/uv_fs_req_cleanup req#) - (-dispose! @cb#) - (catch e (println e)))))) - (~(symbol (str "pixie.uv/uv_" (name nm))) - (uv/uv_default_loop) - (uv/uv_fs_t) - ~@args - @cb#)))] - (call-cc f)))) - -(defuvfsfn fs_open [path flags mode] :result) -(defuvfsfn fs_read [file bufs nbufs offset] :result) -(defuvfsfn fs_close [file] :result) (defn -with-stacklets [fn] @@ -116,6 +95,9 @@ (catch e (println e)))))) +(defn run-with-stacklets [f] + (with-stacklets + (f))) (deftype Promise [val pending-callbacks delivered?] @@ -141,12 +123,12 @@ (defn promise [] (->Promise nil (atom []) false)) -(with-stacklets - (let [p (promise)] - (spawn @p) - (spawn @p) - (spawn (p 42)) - @p)) +(comment (with-stacklets + (let [p (promise)] + (spawn @p) + (spawn @p) + (spawn (p 42)) + @p))) (comment diff --git a/pixie/streams.pxi b/pixie/streams.pxi new file mode 100644 index 00000000..b037be76 --- /dev/null +++ b/pixie/streams.pxi @@ -0,0 +1,21 @@ +(ns pixie.streams) + +(def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) +(def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) +(def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) +(def fputc (ffi-fn libc "fputc" [CInt CVoidP] CInt)) +(def fwrite (ffi-fn libc "fwrite" [CVoidP CInt CInt CVoidP] CInt)) +(def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) +(def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) +(def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) + +(defprotocol IInputStream + (read-byte [this] "Read a single character") + (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) + +(defprotocol IOutputStream + (write-byte [this byte]) + (write [this buffer])) + +(defprotocol IClosable + (close [this] "Closes the stream")) diff --git a/target.py b/target.py index c1726876..a1219674 100644 --- a/target.py +++ b/target.py @@ -177,12 +177,16 @@ def run_load_stdlib(): return rt.load_ns(rt.wrap(u"pixie/stdlib.pxi")) + rt.load_ns(rt.wrap(u"pixie/stacklets.pxi")) stdlib_loaded.set_true() def load_stdlib(): run_load_stdlib.invoke([]) +from pixie.vm.code import intern_var +run_with_stacklets = intern_var(u"pixie.stacklets", u"run-with-stacklets") + def entry_point(args): try: import pixie.vm.stacklet @@ -216,7 +220,7 @@ def entry_point(args): i += 1 if i < len(args): expr = args[i] - EvalFn(expr).invoke([]) + run_with_stacklets.invoke([EvalFn(expr)]) return 0 else: print "Expected argument for " + arg @@ -252,9 +256,9 @@ def entry_point(args): if not exit: if interactive: - ReplFn(args).invoke([]) + run_with_stacklets.invoke([ReplFn(args)]) else: - BatchModeFn(script_args).invoke([]) + run_with_stacklets.invoke([BatchModeFn(script_args)]) except WrappedException as we: print we._ex.__repr__() From 9490bb14b10404ffc2f135353949f394e0ffd34d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 9 Mar 2015 15:52:32 -0600 Subject: [PATCH 571/909] updated spit to use async io --- pixie/io.pxi | 63 +++++++++++++++++++++++++++++------ tests/pixie/tests/test-io.pxi | 5 ++- 2 files changed, 54 insertions(+), 14 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index c9e3ae8f..b537d04f 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -23,6 +23,7 @@ (defuvfsfn fs_open [path flags mode] :result) (defuvfsfn fs_read [file bufs nbufs offset] :result) +(defuvfsfn fs_write [file bufs nbufs offset] :result) (defuvfsfn fs_close [file] :result) @@ -59,6 +60,7 @@ @result)) acc)))))) + (defn open-read {:doc "Open a file for reading, returning a IInputStream" :added "0.1"} @@ -66,6 +68,7 @@ (assert (string? filename) "Filename must be a string") (->FileStream (fs_open filename uv/O_RDONLY 0) 0 (uv/uv_buf_t))) + (defn read-line "Read one line from input-stream for each invocation. nil when all lines have been read" @@ -89,27 +92,65 @@ (when-let [line (read-line input-stream)] (cons line (lazy-seq (line-seq input-stream))))) -(deftype FileOutputStream [fp] +(deftype FileOutputStream [fp offset uvbuf] IOutputStream (write-byte [this val] - (assert (integer? val) "Value must be a int") - (fputc val fp)) + (assert false)) (write [this buffer] - (fwrite buffer 1 (count buffer) fp)) + (let [_ (pixie.ffi/set! uvbuf :base buffer) + _ (pixie.ffi/set! uvbuf :len (count buffer)) + write-count (fs_write fp uvbuf 1 offset)] + (when (neg? write-count) + (throw (uv/uv_err_name read-count))) + (assert (= write-count (count buffer)) (str "Write error!" write-count " " (count buffer))) + (set-field! this :offset (+ offset write-count)) + (set-buffer-count! buffer write-count) + write-count)) IClosable (close [this] (fclose fp))) +(deftype BufferedOutputStream [downstream idx buffer] + IOutputStream + (write-byte [this val] + (pixie.ffi/pack! buffer idx CUInt8 val) + (set-field! this :idx (inc idx)) + (when (= idx (buffer-capacity buffer)) + (write downstream buffer) + (set-field this :idx 0))) + IClosable + (close [this] + (set-buffer-count! buffer idx) + (write downstream buffer))) + +(defn buffered-output-stream [downstream size] + (->BufferedOutputStream downstream 0 (buffer size))) + + +(defn throw-on-error [result] + (when (neg? result) + (throw (uv/uv_err_name result))) + result) + +(defn open-write + {:doc "Open a file for reading, returning a IInputStream" + :added "0.1"} + [filename] + (assert (string? filename) "Filename must be a string") + (->FileOutputStream (throw-on-error (fs_open filename uv/O_WRONLY 0)) + 0 + (uv/uv_buf_t))) + + (defn file-output-rf [filename] - (let [fp (->FileOutputStream (fopen filename "w"))] + (let [fp (buffered-output-stream (open-write filename) + DEFAULT-BUFFER-SIZE)] (fn ([] 0) - ([cnt] (close fp) cnt) - ([cnt chr] + ([_] (close fp) nil) + ([_ chr] (assert (integer? chr)) - (let [written (write-byte fp chr)] - (if (= written 0) - (reduced cnt) - (+ cnt written))))))) + (write-byte fp chr) + nil)))) (defn spit [filename val] diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 86a238b8..e3f80fa3 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -1,6 +1,6 @@ (ns pixie.tests.test-io (require pixie.test :as t) - (require pixie.io-blocking :as io)) + (require pixie.io :as io)) (t/deftest test-file-reduction (let [f (io/open-read "tests/pixie/tests/test-io.txt")] @@ -24,8 +24,7 @@ s (io/line-seq f)] (t/assert= (last s) "Second line."))) -(comment + (t/deftest test-slurp-spit (let [val (vec (range 128))] (t/assert= val (read-string (io/slurp "test.tmp" (io/spit "test.tmp" val)))))) -) From 37d50af42ea69a53ef8e5790e17add180f8c8804 Mon Sep 17 00:00:00 2001 From: Igor Vuk Date: Mon, 9 Mar 2015 23:48:58 +0100 Subject: [PATCH 572/909] Fix typos in README.md, remove trailing spaces --- README.md | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/README.md b/README.md index 39110fb4..0f587afb 100644 --- a/README.md +++ b/README.md @@ -51,25 +51,25 @@ Pixie now comes with a build tool called [dust](https://github.com/pixie-lang/du ### So this is written in Python? -It's actually written in the RPython, the same language PyPy is written in. `make build_with_jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm` this executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... +It's actually written in the RPython, the same language PyPy is written in. `make build_with_jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm`. This executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... ### What's this bit about "magical powers"? -First of all, the word "magic" is in quotes as it's partly a play on words, pixies are small, light and often considered to have magical powers. +First of all, the word "magic" is in quotes as it's partly a play on words, pixies are small, light and often considered to have magical powers. -However there are a few features of pixie that although may not be uncommon, are perhaps unexpected from a lisp. +However there are a few features of pixie that although may not be uncommon, are perhaps unexpected from a lisp. -* Pixie implements its own virtual machine. It does not run on the JVM, CLR or Python VM. It implements its own bytecode, has its own GC and JIT. And it's small. Currently the interpreter, JIT, GC, and stdlib clock in at about 5.5MB once compiled down to an executable. +* Pixie implements its own virtual machine. It does not run on the JVM, CLR or Python VM. It implements its own bytecode, has its own GC and JIT. And it's small. Currently the interpreter, JIT, GC, and stdlib clock in at about 5.5MB once compiled down to an executable. -* The JIT makes some things fast. Very fast. Code like the following compiles down to a loop with 6 CPU instructions. While this may not be too impressive for any language that uses a tracing jit, it is fairly unique for a language as young as Pixie. +* The JIT makes some things fast. Very fast. Code like the following compiles down to a loop with 6 CPU instructions. While this may not be too impressive for any language that uses a tracing jit, it is fairly unique for a language as young as Pixie. ```clojure (comment - This code adds up to 10000 from 0 via calling a function that takes a variable number of arguments. + This code adds up to 10000 from 0 via calling a function that takes a variable number of arguments. That function then reduces over the argument list to add up all given arguments.) - + (defn add-fn [& args] (reduce -add 0 args)) @@ -77,12 +77,12 @@ However there are a few features of pixie that although may not be uncommon, are (if (eq x 10000) x (recur (add-fn x 1)))) - + ``` - - -* Math system is fully polymorphic. Math primitives (+,-, etc.) are built off of polymorphic functions that dispatch on the types of the first two arguments. This allows the math system to be extended to complex numbers, matricies, etc. The performance penalty of such a polymorphic call is completely removed by the RPython generated JIT. + + +* Math system is fully polymorphic. Math primitives (+,-, etc.) are built off of polymorphic functions that dispatch on the types of the first two arguments. This allows the math system to be extended to complex numbers, matrices, etc. The performance penalty of such a polymorphic call is completely removed by the RPython generated JIT. (Planned "magical" Features) @@ -90,21 +90,21 @@ However there are a few features of pixie that although may not be uncommon, are * STM for parallelism. Once STM gets merged into the mainline branch of PyPy, we'll adopt it pretty quickly. -* CSP for concurrency. We already have stacklets, it's not that hard to use them for CSP style concurrency as well. +* CSP for concurrency. We already have stacklets, it's not that hard to use them for CSP style concurrency as well. ## Where do the devs hangout? Mostly on FreeNode at `#pixie-lang` stop by and say "hello". ## Contributing -We have a very open contribution process. If you have a feature you'd like to implement, submit a PR or file an issue and we'll see what we can do. Most PRs are either rejected (if there is a techincal flaw) or accepted within a day, so send an improvement our way and see what happens. +We have a very open contribution process. If you have a feature you'd like to implement, submit a PR or file an issue and we'll see what we can do. Most PRs are either rejected (if there is a technical flaw) or accepted within a day, so send an improvement our way and see what happens. ## Implementation Notes Although parts of the language may be very close to Clojure (they are both lisps after all), language parity is not a design goal. We will take the features from Clojure or other languages that are suitable to our needs, and feel free to reject those that aren't. Therefore this should not be considered a "Clojure Dialect", but instead a "Clojure inspired lisp". ## Disclaimer -This project is the personal work of Timothy Baldridge and contributors. It is not supported by any entity, including Timothy's employer, or any employers of any other contributors. +This project is the personal work of Timothy Baldridge and contributors. It is not supported by any entity, including Timothy's employer, or any employers of any other contributors. ## Copying From 22686acb95852721fcc137b735bd1ef49925d4d1 Mon Sep 17 00:00:00 2001 From: Igor Vuk Date: Mon, 9 Mar 2015 23:50:46 +0100 Subject: [PATCH 573/909] Fix typo: example -> examples --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0f587afb..2cda23c1 100644 --- a/README.md +++ b/README.md @@ -38,7 +38,7 @@ Some planned and implemented features: ## Examples -There are example in the /examples directory. +There are examples in the /examples directory. Try out "Hello World" with: ./examples/hello-world.pxi From 4db301f9a713139ccfb7968dcb93f5d70a26bded Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 9 Mar 2015 21:43:26 -0600 Subject: [PATCH 574/909] more work on spit --- pixie/io.pxi | 26 +++++++++++++++----------- pixie/vm/libs/ffi.py | 5 +++++ tests/pixie/tests/test-io.pxi | 2 +- 3 files changed, 21 insertions(+), 12 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index b537d04f..2f20749e 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -1,7 +1,8 @@ (ns pixie.io (require pixie.streams :as st :refer :all) (require pixie.uv :as uv) - (require pixie.stacklets :as st)) + (require pixie.stacklets :as st) + (require pixie.ffi :as ffi)) (defmacro defuvfsfn [nm args return] `(defn ~nm ~args @@ -97,15 +98,18 @@ (write-byte [this val] (assert false)) (write [this buffer] - (let [_ (pixie.ffi/set! uvbuf :base buffer) - _ (pixie.ffi/set! uvbuf :len (count buffer)) - write-count (fs_write fp uvbuf 1 offset)] - (when (neg? write-count) - (throw (uv/uv_err_name read-count))) - (assert (= write-count (count buffer)) (str "Write error!" write-count " " (count buffer))) - (set-field! this :offset (+ offset write-count)) - (set-buffer-count! buffer write-count) - write-count)) + (loop [buffer-offset 0] + (let [_ (pixie.ffi/set! uvbuf :base (ffi/ptr-add buffer buffer-offset)) + _ (pixie.ffi/set! uvbuf :len (- (count buffer) buffer-offset)) + write-count (fs_write fp uvbuf 1 (get-field this :offset))] + (println "offset " offset) + (when (neg? write-count) + (throw (uv/uv_err_name read-count))) + (assert (= write-count (count buffer)) (str "Write error!" write-count " " (count buffer))) + (set-field! this :offset (+ (get-field this :offset) write-count)) + (if (< (+ buffer-offset write-count) (count buffer)) + (recur (+ buffer-offset write-count)) + write-count)))) IClosable (close [this] (fclose fp))) @@ -117,7 +121,7 @@ (set-field! this :idx (inc idx)) (when (= idx (buffer-capacity buffer)) (write downstream buffer) - (set-field this :idx 0))) + (set-field! this :idx 0))) IClosable (close [this] (set-buffer-count! buffer idx) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index fdb17b1b..48ff27db 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -473,6 +473,11 @@ def pack(ptr, offset, tp, val): tp.ffi_set_value(ptr, val) return nil +@as_var(u"pixie.ffi", u"ptr-add") +def pack(ptr, offset): + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + return VoidP(ptr) class CStructType(object.Type): base_type = object.Type(u"pixie.ffi.CStruct") diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index e3f80fa3..09684e76 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -26,5 +26,5 @@ (t/deftest test-slurp-spit - (let [val (vec (range 128))] + (let [val (vec (range 1280))] (t/assert= val (read-string (io/slurp "test.tmp" (io/spit "test.tmp" val)))))) From 2cc2469f72e638032c097590c7262091497f953e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 9 Mar 2015 22:34:43 -0600 Subject: [PATCH 575/909] fixed mutable fields --- pixie/stdlib.pxi | 17 ++++++++++------ pixie/vm/compiler.py | 31 ++++++++++++++++++++++++++++- tests/pixie/tests/test-compiler.pxi | 15 ++++++++++++++ 3 files changed, 56 insertions(+), 7 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index fe3db177..4746ebac 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1128,12 +1128,17 @@ Creates new maps if the keys are not present." (seq? args)) "protocol override must have arguments") self-arg (first args) _ (assert (symbol? self-arg) "protocol override must have at least one `self' argument") - field-lets (transduce (comp (map (fn [f] - [(symbol (name f)) (list 'get-field self-arg f)])) - cat) - conj fields) - rest (next (next body))] - `(fn ~fn-name ~args (let ~field-lets ~@rest)))) + + rest (next (next body)) + body (reduce + (fn [body f] + `[(local-macro [~(symbol (name f)) + (get-field ~self-arg ~(keyword (name f)))] + ~@body)]) + rest + fields)] + (println body) + `(fn ~fn-name ~args ~@body))) bodies (reduce (fn [res body] (cond diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index afd06a1a..b17b3b4f 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -245,6 +245,13 @@ def emit(self, ctx): ctx.bytecode.append(self.idx) ctx.add_sp(1) +class LocalMacro(LocalType): + def __init__(self, form): + self._from = form + + def emit(self, ctx): + compile_form(self._from, ctx) + @@ -747,6 +754,27 @@ def compile_in_ns(form, ctx): NS_VAR.deref().include_stdlib() compile_fn_call(form, ctx) +def compile_local_macro(form, ctx): + form = rt.next(form) + binding = rt.first(form) + body = rt.next(form) + + sym = rt.nth(binding, rt.wrap(0)) + bind_form = rt.nth(binding, rt.wrap(1)) + + ctx.add_local(rt.name(sym), LocalMacro(bind_form)) + + while True: + compile_form(rt.first(body), ctx) + body = rt.next(body) + + if body is nil: + break + else: + ctx.pop() + + ctx.pop_locals() + builtins = {u"fn*": compile_fn, u"if": compile_if, u"def": compile_def, @@ -760,7 +788,8 @@ def compile_in_ns(form, ctx): u"catch": compile_catch, u"this-ns-name": compile_this_ns, u"in-ns": compile_in_ns, # yes, this is both a function and a compiler special form. - u"yield": compile_yield} + u"yield": compile_yield, + u"local-macro": compile_local_macro} def compiler_special(s): if isinstance(s, symbol.Symbol): diff --git a/tests/pixie/tests/test-compiler.pxi b/tests/pixie/tests/test-compiler.pxi index 052bafe2..944343d4 100644 --- a/tests/pixie/tests/test-compiler.pxi +++ b/tests/pixie/tests/test-compiler.pxi @@ -20,3 +20,18 @@ (t/deftest test-lists (t/assert= (vec '()) []) (t/assert= (vec '()) ())) + + +(defprotocol IMutable + (mutate! [this])) + +(deftype Foo [x] + IMutable + (mutate! [this] + (let [xold x] + (set-field! this :x 42) + (t/assert (not (= xold x))) + (t/assert (= x 42))))) + +(t/deftest test-deftype-mutables + (mutate! (->Foo 0))) From 7be556ee6b04437151745eeaf8c24227d1274a2e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 10 Mar 2015 06:04:40 -0600 Subject: [PATCH 576/909] how about we don't check in debug code --- pixie/stdlib.pxi | 1 - 1 file changed, 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4746ebac..1cfb420c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1137,7 +1137,6 @@ Creates new maps if the keys are not present." ~@body)]) rest fields)] - (println body) `(fn ~fn-name ~args ~@body))) bodies (reduce (fn [res body] From 301b7efaf2c5d83f333f9c38164207c2b9c8faf6 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 10 Mar 2015 06:36:10 -0600 Subject: [PATCH 577/909] fix async version of spit --- pixie/io.pxi | 11 +++++++---- pixie/uv.pxi | 2 ++ tests/pixie/tests/test-io.pxi | 3 ++- 3 files changed, 11 insertions(+), 5 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 2f20749e..4d197916 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -101,12 +101,12 @@ (loop [buffer-offset 0] (let [_ (pixie.ffi/set! uvbuf :base (ffi/ptr-add buffer buffer-offset)) _ (pixie.ffi/set! uvbuf :len (- (count buffer) buffer-offset)) - write-count (fs_write fp uvbuf 1 (get-field this :offset))] - (println "offset " offset) + write-count (fs_write fp uvbuf 1 offset)] + (println "offset " offset write-count buffer-offset (count buffer)) (when (neg? write-count) (throw (uv/uv_err_name read-count))) (assert (= write-count (count buffer)) (str "Write error!" write-count " " (count buffer))) - (set-field! this :offset (+ (get-field this :offset) write-count)) + (set-field! this :offset (+ offset write-count)) (if (< (+ buffer-offset write-count) (count buffer)) (recur (+ buffer-offset write-count)) write-count)))) @@ -120,6 +120,7 @@ (pixie.ffi/pack! buffer idx CUInt8 val) (set-field! this :idx (inc idx)) (when (= idx (buffer-capacity buffer)) + (set-buffer-count! buffer (buffer-capacity buffer)) (write downstream buffer) (set-field! this :idx 0))) IClosable @@ -141,7 +142,9 @@ :added "0.1"} [filename] (assert (string? filename) "Filename must be a string") - (->FileOutputStream (throw-on-error (fs_open filename uv/O_WRONLY 0)) + (->FileOutputStream (throw-on-error (fs_open filename + (bit-or uv/O_WRONLY uv/O_CREAT) + uv/S_IRWXU)) 0 (uv/uv_buf_t))) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 877274c5..90bb127f 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -159,6 +159,8 @@ (f/defconst O_CREAT) (f/defconst S_IRUSR) + (f/defconst S_IRWXU) + (f/defconst S_IWUSR) ; ERRNO diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 09684e76..ec4dee02 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -27,4 +27,5 @@ (t/deftest test-slurp-spit (let [val (vec (range 1280))] - (t/assert= val (read-string (io/slurp "test.tmp" (io/spit "test.tmp" val)))))) + (io/spit "test.tmp" val) + (t/assert= val (read-string (io/slurp "test.tmp"))))) From 62cbd153368a45977845d040b44545b67550a54e Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 10 Mar 2015 06:36:52 -0600 Subject: [PATCH 578/909] fix async version of spit --- pixie/io.pxi | 2 -- 1 file changed, 2 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 4d197916..228f8944 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -102,10 +102,8 @@ (let [_ (pixie.ffi/set! uvbuf :base (ffi/ptr-add buffer buffer-offset)) _ (pixie.ffi/set! uvbuf :len (- (count buffer) buffer-offset)) write-count (fs_write fp uvbuf 1 offset)] - (println "offset " offset write-count buffer-offset (count buffer)) (when (neg? write-count) (throw (uv/uv_err_name read-count))) - (assert (= write-count (count buffer)) (str "Write error!" write-count " " (count buffer))) (set-field! this :offset (+ offset write-count)) (if (< (+ buffer-offset write-count) (count buffer)) (recur (+ buffer-offset write-count)) From 76538053a0ada7ea14f1d170c621f984032a7532 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 10 Mar 2015 16:35:25 -0600 Subject: [PATCH 579/909] fixed compilation error with stacklets --- pixie/io.pxi | 2 -- pixie/stacklets.pxi | 8 ++++---- pixie/vm/stdlib.py | 4 ++++ target.py | 11 ++++++++++- tests/pixie/tests/test-io.pxi | 1 - 5 files changed, 18 insertions(+), 8 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 228f8944..cc76046c 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -172,8 +172,6 @@ (close c) result)) -(println (slurp "/tmp/a.txt")) - (deftype ProcessInputStream [fp] IInputStream (read [this buffer len] diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index ab101bb2..0f60d989 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -1,11 +1,11 @@ (ns pixie.stacklets (require pixie.uv :as uv)) -;; LibUV seems to act up when we invoke a stacklet from inside a callbac -;; so we compensate by simply storing the stacklets in a task queue -;; and calling them later outside of the libuv loop. +;; If we don't do this, compiling this file doesn't work since the def will clear out +;; the existing value. -(def stacklet-loop-h (atom nil)) +(if (undefined? (var stacklet-loop-h)) + (def stacklet-loop-h (atom nil))) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 07fdb380..b139cb3f 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -363,6 +363,10 @@ def eval(form): val = interpret(compile(form)) return val +@as_var("undefined?") +def is_undefined(var): + return rt.wrap(not var.is_defined()) + @as_var("load-ns") def load_ns(filename): import pixie.vm.string as string diff --git a/target.py b/target.py index a1219674..d7f7ca3d 100644 --- a/target.py +++ b/target.py @@ -156,6 +156,15 @@ def inner_invoke(self, args): rt.load_reader(StringReader(unicode_from_utf8(self._expr))) +class CompileFileFn(NativeFn): + def __init__(self, filename): + self._filename = filename + + def inner_invoke(self, args): + import pixie.vm.rt as rt + + rt.compile_file(rt.wrap(self._filename)) + class IsPreloadFlag(object): def __init__(self): @@ -239,7 +248,7 @@ def entry_point(args): if i < len(args): path = args[i] print "Compiling ", path - rt.compile_file(rt.wrap(path)) + run_with_stacklets.invoke([CompileFileFn(path)]) exit = True else: print "Expected argument for " + arg diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index ec4dee02..09de53e3 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -24,7 +24,6 @@ s (io/line-seq f)] (t/assert= (last s) "Second line."))) - (t/deftest test-slurp-spit (let [val (vec (range 1280))] (io/spit "test.tmp" val) From 9d99cdd479bf026d7d7b01e4d7b60905d3810825 Mon Sep 17 00:00:00 2001 From: Peter Monks Date: Tue, 10 Mar 2015 15:35:47 -0700 Subject: [PATCH 580/909] Working on issue #218. Code works (with fully qualified paths), but the tests fail (due to fully qualified paths). --- pixie/fs.pxi | 38 ++++++++++++++-------------- pixie/vm/libs/path.py | 17 ++++++++----- tests/pixie/tests/test-fs.pxi | 47 ++++++++++++++++++++--------------- 3 files changed, 57 insertions(+), 45 deletions(-) diff --git a/pixie/fs.pxi b/pixie/fs.pxi index 7be65fda..a7745a7a 100644 --- a/pixie/fs.pxi +++ b/pixie/fs.pxi @@ -18,11 +18,11 @@ (basename [this] "Returns the basename of the Filesystem Object") - + ;; TODO (permissions [this] "Returns a string of the octal permissions") - + (mounted? [this] "Returns true if the directory is a mounted") @@ -30,12 +30,12 @@ "Returns the size of the file/dir on disk")) (defprotocol IFile - (extension [this] + (extension [this] "Returns the extension") - - (extension? [this ext] + + (extension? [this ext] "Returns true if file has extension") - + ;; TODO (touch [this] "Create the file if it doesn't exist.")) @@ -71,7 +71,7 @@ ;; Same level (and (zero? (count diff-a)) (zero? (count diff-b))) "." - + ;; If B diverges by one level and a is a Dir we use ".." (and (= 1 (count diff-b)) (instance? Dir a)) ".." @@ -92,16 +92,16 @@ (abs [this] (path/-abs pathz)) - + (exists? [this] (path/-exists? pathz)) (basename [this] (last (string/split (abs this) "/"))) - IFile + IFile ;; TODO: Sort out regex or make strings partitionable. So we can split at - ;; #".". + ;; #".". (extension [this] (last (string/split (abs this) "."))) @@ -113,11 +113,11 @@ (hash (abs this))) (-eq [this other] - (if (instance? File other) + (if (instance? File other) (= (abs this) (abs other)) false)) - (-str [this] + (-str [this] (str (abs this))) (-repr [this] @@ -140,10 +140,10 @@ (basename [this] (last (string/split (abs this) "/"))) - + IDir (list [this] - (path/-list-dir pathz)) + (vec (map fspath (path/-list-dir pathz)))) (walk [this] (transduce (map #(if (path/-file? %) @@ -165,11 +165,11 @@ (hash (abs this))) (-eq [this other] - (if (instance? Dir other) + (if (instance? Dir other) (= (abs this) (abs other)) false)) - (-str [this] + (-str [this] (str (abs this))) (-repr [this] @@ -177,11 +177,11 @@ ;; (deftype Fifo [pathz]) -(defn file +(defn file "Returns a file if the path is a file or does not exist. If a different filesystem object exists at the path an error will be thrown." [x] (let [x (path/-path x)] - (cond + (cond (path/-file? x) (->File x) (not (path/-exists? x)) (->File x) :else (throw (str "A non-file object exists at path: " x))))) @@ -190,7 +190,7 @@ "Returns a dir if the path is a dir or does not exist. If a different filesystem object exists at the path an error will be thrown." [x] (let [x (path/-path x)] - (cond + (cond (path/-dir? x) (->Dir x) (not (path/-exists? x)) (->Dir x) :else (throw (str "A non-dir object exists at path: " x))))) diff --git a/pixie/vm/libs/path.py b/pixie/vm/libs/path.py index 854bedce..8afd0ef2 100644 --- a/pixie/vm/libs/path.py +++ b/pixie/vm/libs/path.py @@ -18,7 +18,7 @@ def __init__(self, top): #def rel_path(self, other): # "Returns the path relative to other path" # return rt.wrap(str(os.path.relpath(self._path, start=other._path))) - + def abs_path(self): "Returns the absolute path" return rt.wrap(os.path.abspath(str(self._path))) @@ -28,8 +28,8 @@ def abs_path(self): # return rt.wrap(rt.name(os.path.basename("a"))) def exists(self): - return true if os.path.exists(str(self._path)) else false - + return true if os.path.exists(str(self._path)) else false + def is_file(self): return true if os.path.isfile(str(self._path)) else false @@ -61,9 +61,14 @@ def path(path): return Path(path) # TODO: Implement this -#@as_var("pixie.path", "list-dir") -#def list_dir(path): - +@as_var("pixie.path", "-list-dir") +def list_dir(self): + assert isinstance(self, Path) + result = rt.vector() + for item in os.listdir(str(self._path)): + result = rt.conj(result, rt.wrap(str(self._path) + "/" + str(item))) + return result + @as_var("pixie.path", "-abs") def _abs(self): assert isinstance(self, Path) diff --git a/tests/pixie/tests/test-fs.pxi b/tests/pixie/tests/test-fs.pxi index c56143cd..b4a14e8a 100644 --- a/tests/pixie/tests/test-fs.pxi +++ b/tests/pixie/tests/test-fs.pxi @@ -14,7 +14,7 @@ (fs/dir dir-b) (fs/dir dir-c)) true) - + (t/assert= (= (fs/file file-a) (fs/file file-b) (fs/file file-c)) @@ -28,26 +28,33 @@ (fs/file file-b) (fs/file file-c)])) 1))) -(comment - (t/deftest test-walking - (let [dir-a "tests/pixie/tests/fs"] - (t/assert= (set (fs/walk (fs/dir dir-a))) - #{(fs/dir (str dir-a "/parent")) - (fs/file (str dir-a "/parent/foo.txt")) - (fs/file (str dir-a "/parent/bar.txt")) - (fs/dir (str dir-a "/parent/child")) - (fs/file (str dir-a "/parent/child/foo.txt")) - (fs/file (str dir-a "/parent/child/bar.txt"))}) - (t/assert= (set (fs/walk-files (fs/dir dir-a))) - #{(fs/file (str dir-a "/parent/foo.txt")) - (fs/file (str dir-a "/parent/bar.txt")) - (fs/file (str dir-a "/parent/child/foo.txt")) - (fs/file (str dir-a "/parent/child/bar.txt"))}) +(t/deftest test-walking + (let [dir-a "tests/pixie/tests/fs"] + (t/assert= (set (fs/walk (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/dir (str dir-a "/parent/child")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) + + (t/assert= (set (fs/walk-files (fs/dir dir-a))) + #{(fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) + + (t/assert= (set (fs/walk-dirs (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/dir (str dir-a "/parent/child"))}))) - (t/assert= (set (fs/walk-dirs (fs/dir dir-a))) - #{(fs/dir (str dir-a "/parent")) - (fs/dir (str dir-a "/parent/child"))})))) +(t/deftest test-list + (let [dir-a "tests/pixie/tests/fs/parent"] + (t/assert= (set (fs/list (fs/dir dir-a))) + #{(fs/file (str dir-a "foo.txt")) + (fs/file (str dir-a "bar.txt")) + (fs/dir (str dir-a "child"))}))) (t/deftest test-rel? (let [dir-a (fs/dir "tests/pixie/tests/fs") @@ -91,7 +98,7 @@ (t/deftest test-exists? (let [real-dir (fs/dir "tests/pixie/tests/fs/parent") fake-dir (fs/dir "tests/pixie/tests/fs/parent/fake-dir") - fake-file (fs/dir "tests/pixie/tests/fs/parent/fake-file")] + fake-file (fs/dir "tests/pixie/tests/fs/parent/fake-file")] (t/assert= (fs/exists? real-dir) true) (t/assert= (fs/exists? fake-dir) false) (t/assert= (fs/exists? fake-file) false))) From f2b7fd2a3d1ef7071a06916f9e3253dcf56b1fca Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 10 Mar 2015 17:25:37 -0600 Subject: [PATCH 581/909] remove old code --- pixie/stacklets.pxi | 92 --------------------------------------------- 1 file changed, 92 deletions(-) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 0f60d989..ff96a3dc 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -122,95 +122,3 @@ (defn promise [] (->Promise nil (atom []) false)) - -(comment (with-stacklets - (let [p (promise)] - (spawn @p) - (spawn @p) - (spawn (p 42)) - @p))) - -(comment - - - (with-stacklets - (let [f (fs_open "/tmp/foo.txt" uv/O_RDONLY uv/S_IRUSR) - b (uv/new-fs-buf 1024)] - (println (type (:base b))) - (let [err (fs_read f b 1 0)] - (println err) - (assert (pos? err))) - (dotimes [x 100] - (puts (str (char (pixie.ffi/unpack (:base b) x CUInt8))))) - (println "done")))) - -(comment - - - (dotimes [t 33] - (-thread (fn [] (dotimes [x 10000] - (println t x)))))) - -(comment - (defn run-later [f] - (let [a (uv/uv_async_t) - cb (atom nil)] - (println "start yield") - (reset! cb (ffi-prep-callback uv/uv_async_cb - (fn [handle] - (println "process yield") - (f) - (uv/uv_close a close_cb) - (-dispose! @cb) - (println "done process yield")))) - (uv/uv_async_init (uv/uv_default_loop) a @cb) - (uv/uv_async_send a))) - - (defn cfn [x] - (fn [] - (println x) - (if (pos? x) - (run-later (cfn (dec x)))))) - - (do (run-later (cfn 10000)) - (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT))) - - -(comment - ((var defuvfsfn) 'open '[path flags mode] :result) - - (defuvfsfn open [path flags mode] :result) - (defuvfsfn read [file bufs nbufs offset] :result) - (defuvfsfn close [file] :result)) - - -(comment - (defprotocol IBlockingQueue - (add-item [this item]) - (remove-item [this])) - - (deftype BlockingQueue [items lock locked] - IBlockingQueue - (add-item [this item] - (enqueue items item) - (when @locked - (-release-lock lock) - (reset! locked false)) - (-yield-thread)) - (remove-item [this] - (when (empty? @items) - (reset! locked true) - (-acquire-lock lock true)) - (dequeue items))) - - (defn blocking-queue [] - (let [l (-create-lock)] - (-acquire-lock l true) - (->BlockingQueue (atom []) l (atom true)))) - - (def task-queue (blocking-queue)) - - - - -) From a0a26b8426911609b31ab64e4d0be5802c74d46f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 11 Mar 2015 06:11:29 -0600 Subject: [PATCH 582/909] deftype performance boost for immutable fields #215 --- benchmarks/deftype_fields.pxi | 16 +++++++++++++ pixie/stdlib.pxi | 18 +++++--------- pixie/vm/custom_types.py | 44 ++++++++++++++++++++++++++++++----- 3 files changed, 60 insertions(+), 18 deletions(-) create mode 100644 benchmarks/deftype_fields.pxi diff --git a/benchmarks/deftype_fields.pxi b/benchmarks/deftype_fields.pxi new file mode 100644 index 00000000..fbee851b --- /dev/null +++ b/benchmarks/deftype_fields.pxi @@ -0,0 +1,16 @@ +;; Before Immutable Opt: 3.9 sec +;; After 3.2 sec + +(defprotocol IAdder + (add-them [this])) + +(deftype Adder [a b] + IAdder + (add-them [this] + (set-field! this :b (+ a b)) + b)) + + +(def adder (->Adder 1 0)) +(dotimes [x (* 1024 1024 1024)] + (assert (= (inc x) (add-them adder)))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1cfb420c..9b95d973 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1159,18 +1159,12 @@ Creates new maps if the keys are not present." type-decl `(def ~nm (create-type ~(keyword (name nm)) ~all-fields)) inst (gensym) ctor `(defn ~ctor-name ~field-syms - (let [~inst (new ~nm)] - ~@(transduce - (map (fn [field] - `(set-field! ~inst ~field ~(symbol (name field))))) - conj - fields) - ~@(transduce - (map (fn [type-body] - `(set-field! ~inst ~(keyword (name (first type-body))) ~(mk-body type-body)))) - conj - type-bodies) - ~inst)) + (new ~nm + ~@field-syms + ~@(transduce (map (fn [type-body] + (mk-body type-body))) + conj + type-bodies))) proto-bodies (transduce (map (fn [body] (cond diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 7a1757b9..0434ba36 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -6,39 +6,64 @@ import pixie.vm.rt as rt class CustomType(Type): - __immutable_fields__ = ["_slots"] + __immutable_fields__ = ["_slots", "_rev?"] def __init__(self, name, slots): Type.__init__(self, name) self._slots = slots + self._mutable_slots = {} + self._rev = 0 @jit.elidable_promote() def get_slot_idx(self, nm): return self._slots[nm] + def set_mutable(self, nm): + if not self.is_mutable(nm): + self._rev += 1 + self._mutable_slots[nm] = nm + + + @jit.elidable_promote() + def _is_mutable(self, nm, rev): + return nm in self._mutable_slots + + def is_mutable(self, nm): + return self._is_mutable(nm, self._rev) + @jit.elidable_promote() def get_num_slots(self): return len(self._slots) class CustomTypeInstance(Object): __immutable_fields__ = ["_type"] - def __init__(self, type): + def __init__(self, type, fields): affirm(isinstance(type, CustomType), u"Can't create a instance of a non custom type") self._custom_type = type - self._fields = [None] * self._custom_type.get_num_slots() + self._fields = fields def type(self): return self._custom_type def set_field(self, name, val): + self._custom_type.set_mutable(name) idx = self._custom_type.get_slot_idx(name) self._fields[idx] = val return self - def get_field(self, name): + @jit.elidable_promote() + def get_field_immutable(self, name): idx = self._custom_type.get_slot_idx(name) return self._fields[idx] + + def get_field(self, name): + if self._custom_type.is_mutable(name): + idx = self._custom_type.get_slot_idx(name) + return self._fields[idx] + else: + return self.get_field_immutable(name) + def set_field_by_idx(self, idx, val): affirm(isinstance(idx, r_uint), u"idx must be a r_uint") self._fields[idx] = val @@ -60,9 +85,16 @@ def create_type(type_name, fields): return CustomType(rt.name(type_name), acc) @as_var("new") -def _new(tp): +def _new__args(args): + affirm(len(args) >= 1, u"new takes at least one parameter") + tp = args[0] affirm(isinstance(tp, CustomType), u"Can only create a new instance of a custom type") - return CustomTypeInstance(tp) + cnt = len(args) - 1 + affirm(cnt - 1 != tp.get_num_slots(), u"Wrong number of initializer fields to custom type") + arr = [None] * cnt + for x in range(cnt): + arr[x] = args[x + 1] + return CustomTypeInstance(tp, arr) @as_var("set-field!") def set_field(inst, field, val): From d26ed0b71365493394746c287b20b0c70685a76c Mon Sep 17 00:00:00 2001 From: Peter Monks Date: Wed, 11 Mar 2015 11:19:24 -0700 Subject: [PATCH 583/909] Fixed issue #218 --- tests/pixie/tests/test-fs.pxi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/pixie/tests/test-fs.pxi b/tests/pixie/tests/test-fs.pxi index b4a14e8a..e5edaffb 100644 --- a/tests/pixie/tests/test-fs.pxi +++ b/tests/pixie/tests/test-fs.pxi @@ -52,9 +52,9 @@ (t/deftest test-list (let [dir-a "tests/pixie/tests/fs/parent"] (t/assert= (set (fs/list (fs/dir dir-a))) - #{(fs/file (str dir-a "foo.txt")) - (fs/file (str dir-a "bar.txt")) - (fs/dir (str dir-a "child"))}))) + #{(fs/file (str dir-a "/foo.txt")) + (fs/file (str dir-a "/bar.txt")) + (fs/dir (str dir-a "/child"))}))) (t/deftest test-rel? (let [dir-a (fs/dir "tests/pixie/tests/fs") From 3c0f3b5d4ecde799421018b9684584a92b3ddf15 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 11 Mar 2015 21:50:44 -0600 Subject: [PATCH 584/909] implement doc string support for native functions --- pixie/stdlib.pxi | 2 ++ pixie/vm/code.py | 7 ++++--- pixie/vm/stdlib.py | 8 ++++++++ 3 files changed, 14 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 9b95d973..067e52c9 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1273,11 +1273,13 @@ and implements IAssociative, ILookup and IObject." {:doc "Returns the documentation of the given value." :added "0.1"} [v] + (let [vr (resolve v) x (if vr @vr) doc (get (meta x) :doc) has-doc? (if doc true (get (meta x) :signatures))] (cond + (satisfies? IDoc x) (-doc x) has-doc? (let [sigs (get (meta x) :signatures) examples (get (meta x) :examples) indent (fn [s] diff --git a/pixie/vm/code.py b/pixie/vm/code.py index ed17035f..9e3ad263 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -172,7 +172,7 @@ class NativeFn(BaseCode): """Wrapper for a native function""" _type = object.Type(u"pixie.stdlib.NativeFn") - def __init__(self): + def __init__(self, doc=None): BaseCode.__init__(self) def type(self): @@ -807,11 +807,12 @@ def assert_type(x, tp): def wrap_fn(fn, tp=object.Object): """Converts a native Python function into a pixie function.""" + docstring = unicode(fn.__doc__) if fn.__doc__ else u"" def as_native_fn(f): - return type("W" + fn.__name__, (NativeFn,), {"inner_invoke": f})() + return type("W" + fn.__name__, (NativeFn,), {"inner_invoke": f, "_doc": docstring})() def as_variadic_fn(f): - return type("W" + fn.__name__[:len("__args")], (NativeFn,), {"inner_invoke": f})() + return type("W" + fn.__name__[:len("__args")], (NativeFn,), {"inner_invoke": f, "_doc": docstring})() code = fn.func_code if fn.__name__.endswith("__args"): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 07fdb380..4cfe5d05 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -43,6 +43,8 @@ defprotocol("pixie.stdlib", "IFn", ["-invoke"]) +defprotocol("pixie.stdlib", "IDoc", ["-doc"]) + IVector = as_var("pixie.stdlib", "IVector")(Protocol(u"IVector")) IMap = as_var("pixie.stdlib", "IMap")(Protocol(u"IMap")) @@ -799,5 +801,11 @@ def _seq(self): @as_var("ex-msg") def ex_msg(e): + """Returns the message contained in an exception""" assert isinstance(e, RuntimeException) return e._data + +@extend(_doc, code.NativeFn._type) +def _doc(self): + assert isinstance(self, code.NativeFn) + return rt.wrap(self._doc) \ No newline at end of file From 4016b85b2aa1b297be9090c358d49ad239e2e60d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 12 Mar 2015 06:43:40 -0600 Subject: [PATCH 585/909] split the stream apis, remove ICloseable in favor of IDisposable --- pixie/io-blocking.pxi | 22 ++++++++------- pixie/io.pxi | 66 ++++++------------------------------------- pixie/stdlib.pxi | 6 ++++ pixie/streams.pxi | 18 ++++-------- 4 files changed, 32 insertions(+), 80 deletions(-) diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index f5565373..425120b1 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -25,8 +25,8 @@ read-count)) (read-byte [this] (fgetc buffer)) - IClosable - (close [this] + IDisposable + (-dispose! [this] (fclose fp)) IReduce (-reduce [this f init] @@ -72,20 +72,21 @@ (cons line (lazy-seq (line-seq input-stream))))) (deftype FileOutputStream [fp] - IOutputStream + IByteOutputStream (write-byte [this val] (assert (integer? val) "Value must be a int") (fputc val fp)) + IOutputStream (write [this buffer] (fwrite buffer 1 (count buffer) fp)) - IClosable - (close [this] + IDisposable + (-dispose! [this] (fclose fp))) (defn file-output-rf [filename] (let [fp (->FileOutputStream (fopen filename "w"))] (fn ([] 0) - ([cnt] (close fp) cnt) + ([cnt] (dispose! fp) nil) ([cnt chr] (assert (integer? chr)) (let [written (write-byte fp chr)] @@ -105,7 +106,7 @@ (map char) string-builder c)] - (close c) + (dispose! c) result)) (deftype ProcessInputStream [fp] @@ -116,10 +117,11 @@ (let [read-count (fread buffer 1 len fp)] (set-buffer-count! buffer read-count) read-count)) + IByteInputStream (read-byte [this] (fgetc fp)) - IClosable - (close [this] + IDisposable + (-dispose! [this] (pclose fp)) IReduce (-reduce [this f init] @@ -149,5 +151,5 @@ (map char) string-builder c)] - (close c) + (dispose! c) result)) diff --git a/pixie/io.pxi b/pixie/io.pxi index cc76046c..013a7bd7 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -42,10 +42,8 @@ (set-field! this :offset (+ offset read-count)) (set-buffer-count! buffer read-count) read-count)) - (read-byte [this] - (assert false "Does not support read-byte, wrap in a buffering reader")) - IClosable - (close [this] + IDisposable + (-dispose! [this] (pixie.ffi/free uvbuf) (fs_close fp)) IReduce @@ -95,8 +93,6 @@ (deftype FileOutputStream [fp offset uvbuf] IOutputStream - (write-byte [this val] - (assert false)) (write [this buffer] (loop [buffer-offset 0] (let [_ (pixie.ffi/set! uvbuf :base (ffi/ptr-add buffer buffer-offset)) @@ -108,12 +104,12 @@ (if (< (+ buffer-offset write-count) (count buffer)) (recur (+ buffer-offset write-count)) write-count)))) - IClosable - (close [this] + IDisposable + (-dispose! [this] (fclose fp))) (deftype BufferedOutputStream [downstream idx buffer] - IOutputStream + IByteOutputStream (write-byte [this val] (pixie.ffi/pack! buffer idx CUInt8 val) (set-field! this :idx (inc idx)) @@ -121,8 +117,8 @@ (set-buffer-count! buffer (buffer-capacity buffer)) (write downstream buffer) (set-field! this :idx 0))) - IClosable - (close [this] + IDisposable + (-dispose! [this] (set-buffer-count! buffer idx) (write downstream buffer))) @@ -151,7 +147,7 @@ (let [fp (buffered-output-stream (open-write filename) DEFAULT-BUFFER-SIZE)] (fn ([] 0) - ([_] (close fp) nil) + ([_] (dispose! fp)) ([_ chr] (assert (integer? chr)) (write-byte fp chr) @@ -169,49 +165,5 @@ (map char) string-builder c)] - (close c) - result)) - -(deftype ProcessInputStream [fp] - IInputStream - (read [this buffer len] - (assert (<= (buffer-capacity buffer) len) - "Not enough capacity in the buffer") - (let [read-count (fread buffer 1 len fp)] - (set-buffer-count! buffer read-count) - read-count)) - (read-byte [this] - (fgetc fp)) - IClosable - (close [this] - (pclose fp)) - IReduce - (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) - - -(defn popen-read - {:doc "Open a file for reading, returning a IInputStream" - :added "0.1"} - [command] - (assert (string? command) "Command must be a string") - (->ProcessInputStream (popen command "r"))) - - -(defn run-command [command] - (let [c (->ProcessInputStream (popen command "r")) - result (transduce - (map char) - string-builder - c)] - (close c) + (dispose! c) result)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index c4ea4fbb..41380fb2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2144,6 +2144,12 @@ Expands to calls to `extend-type`." ([sb] (str sb)) ([sb item] (conj! sb item))) +(defn dispose! + "Finalizes use of the object by cleaning up resources used by the object" + [x] + (-dispose! x) + nil) + (defmacro using [bindings & body] (let [pairs (partition 2 bindings) names (map first pairs)] diff --git a/pixie/streams.pxi b/pixie/streams.pxi index b037be76..05b4ad44 100644 --- a/pixie/streams.pxi +++ b/pixie/streams.pxi @@ -1,21 +1,13 @@ (ns pixie.streams) -(def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) -(def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) -(def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) -(def fputc (ffi-fn libc "fputc" [CInt CVoidP] CInt)) -(def fwrite (ffi-fn libc "fwrite" [CVoidP CInt CInt CVoidP] CInt)) -(def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) -(def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) -(def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) - (defprotocol IInputStream - (read-byte [this] "Read a single character") (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) (defprotocol IOutputStream - (write-byte [this byte]) (write [this buffer])) -(defprotocol IClosable - (close [this] "Closes the stream")) +(defprotocol IByteInputStream + (read-byte [this])) + +(defprotocol IByteOutputStream + (write-byte [this byte])) From 5a5d0cb6dce478725693f424a48aeb5d15bccde0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 13 Mar 2015 23:22:47 -0600 Subject: [PATCH 586/909] started work on source compilation --- pixie/ffi-infer.pxi | 27 +++++++++++++++++++++++++++ pixie/io.pxi | 21 +++++++++++++++++++++ pixie/uv.pxi | 20 ++++++++++++++++++++ pixie/vm/libs/ffi.py | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 103 insertions(+) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 9a2d54b3..8056b57e 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -228,6 +228,33 @@ return 0; :name ~(name nm)))) +(defn compile-library [{:keys [prefix includes]} & source] + (let [c-name (str "/tmp/" prefix ".c") + source-header (apply str (map (fn [i] + (str "#include \"" i "\"\n")) + includes)) + source (apply str source-header (interpose "\n\n" source)) + lib-name (str "/tmp/" prefix "-" (hash source) "." pixie.platform/so-ext) + cmd (str "cc -dynamic-lang " + c-name + (apply str (map (fn [x] ( str " -I " x " ")) + @load-paths)) + + " -o " + lib-name)] + (io/spit c-name source) + (println cmd) + (io/run-command cmd))) + + + +(compile-library + {:prefix "foo" + :includes ["uv.h"]} + "int foo(int bar) + { + return 42; + }") (comment diff --git a/pixie/io.pxi b/pixie/io.pxi index 013a7bd7..ed8378ed 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -167,3 +167,24 @@ c)] (dispose! c) result)) + + +(defn tcp-server [ip port on-connection] + (assert (string? ip) "Ip should be a string") + (assert (integer? port) "Port should be a int") + (let [server (uv/uv_tcp_t) + bind-addr (uv/sockaddr_in) + _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) + on-new-connetion (atom nil)] + (reset! (ffi-prep-callback + uv/uv_connection_cb + (fn [server status] + (when (not (= status -1)) + (println "Got Client!!!!!!!"))))) + (uv/uv_tcp_init (uv/uv_default_loop) server) + (uv/uv_tcp_bind server bind_addr) + (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) + (st/yield-control))) + + +(tcp-server "0.0.0.0" 4242 nil) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 90bb127f..3cab69ae 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -174,6 +174,20 @@ (f/defccallback uv_async_cb) (f/defcfn uv_async_init) (f/defcfn uv_async_send) + + + ; TCP Networking + (f/defcstruct uv_tcp_t []) + (f/defc-raw-struct sockaddr_in) + (f/defcfn uv_tcp_init) + (f/defcfn uv_ip4_addr) + (f/defcfn uv_tcp_bind) + (f/defcfn uv_listen) + (f/defcfn uv_accept) + (f/defcfn uv_read_start) + + (f/defccallback uv_connection_cb) + ) @@ -183,3 +197,9 @@ (pixie.ffi/set! bt :base b) (pixie.ffi/set! bt :len size) bt)) + + +(defn throw-on-error [result] + (if (neg? result) + (throw (str "UV Error: " (uv/uv_err_name result))) + result)) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 48ff27db..94bd917f 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -238,6 +238,15 @@ def nth_char(self, idx): def capacity(self): return self._size + def free_data(self): + lltype.free(self._buffer, flavor="raw") + + + +@extend(proto._dispose_BANG_, Buffer) +def _dispose_voidp(self): + self.free_data() + @extend(proto._nth, Buffer) def _nth(self, idx): @@ -458,6 +467,24 @@ def __init__(self, raw_data): def raw_data(self): return rffi.cast(rffi.VOIDP, self._raw_data) + def free_data(self): + lltype.free(self._raw_data, flavor="raw") + +@extend(proto._dispose_BANG_, cvoidp) +def _dispose_voidp(self): + self.free_data() + + + +@as_var(u"pixie.ffi", u"prep-string") +def prep_string(s): + """Takes a Pixie string and returns a VoidP to that string. The string should be freed via dispose!, otherwise + memory leaks could result.""" + affirm(isinstance(s, String), u"Can only prep strings with prep-string") + utf8 = unicode_to_utf8(rt.name(s)) + raw = rffi.str2charp(utf8) + return VoidP(rffi.cast(rffi.VOIDP, raw)) + @as_var(u"pixie.ffi", u"unpack") def unpack(ptr, offset, tp): affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") @@ -718,6 +745,14 @@ def set_(self, k, val): +@as_var("pixie.ffi", "prep-ffi-call") +def prep_ffi_call__args(args): + fn = args[0] + affirm(isinstance(fn, CFunctionType), u"First arg must be a FFI function") + + + + import sys From 0ec3044c720c524eb2a509ccb97204a58ad30033 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 13 Mar 2015 23:45:56 -0600 Subject: [PATCH 587/909] ffi doc strings and cleanup --- pixie/ffi-infer.pxi | 3 ++- pixie/vm/libs/ffi.py | 30 ++++++++++++++++++++---------- tests/pixie/tests/test-ffi.pxi | 4 ++-- tests/pixie/tests/test-uv.pxi | 5 +++-- 4 files changed, 27 insertions(+), 15 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 49578c20..b0bf7e86 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -1,4 +1,5 @@ (ns pixie.ffi-infer + (require pixie.io :as io)) (defn -add-rel-path [rel] @@ -84,7 +85,7 @@ return 0; (defmulti edn-to-ctype :type) (defn callback-type [{:keys [arguments returns]}] - `(ffi-callback ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns))) + `(pixie.ffi/ffi-callback ~(vec (map edn-to-ctype arguments)) ~(edn-to-ctype returns))) (defmethod edn-to-ctype :pointer [{:keys [of-type] :as ptr}] diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 15e13167..f4836dc0 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -443,6 +443,8 @@ def raw_data(self): @as_var(u"pixie.ffi", u"unpack") def unpack(ptr, offset, tp): + """(unpack ptr offset tp) + Reads a value of type tp from offset of ptr.""" affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") affirm(isinstance(tp, CType), u"Packing type must be a CType") ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) @@ -450,6 +452,8 @@ def unpack(ptr, offset, tp): @as_var(u"pixie.ffi", u"pack!") def pack(ptr, offset, tp, val): + """(pack! ptr offset tp val) + Writes val at offset of ptr with the format tp""" affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") affirm(isinstance(tp, CType), u"Packing type must be a CType") ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) @@ -534,8 +538,11 @@ def _dispose(self): -@as_var(u"ffi-callback") +@as_var(u"pixie.ffi", u"ffi-callback") def ffi_callback(args, ret_type): + """(ffi-callback args ret-type) + Creates a ffi callback type. Args is a vector of CType args. Ret-type is the CType return + type of the callback. Returns a ffi callback type that can be used with ffi-prep-callback.""" args_w = [None] * rt.count(args) for x in range(rt.count(args)): @@ -549,8 +556,12 @@ def ffi_callback(args, ret_type): return CFunctionType(args_w, ret_type) -@as_var(u"ffi-prep-callback") +@as_var(u"pixie.ffi", u"ffi-prep-callback") def ffi_prep_callback(tp, f): + """(ffi-prep-callback callback-tp fn) + Prepares a Pixie function for use as a c callback. callback-tp is a ffi callback type, + fn is a pixie function (can be a closure, native fn or any object that implements -invoke. + Returns a function pointer that can be passed to c and invoked as a callback.""" affirm(isinstance(tp, CFunctionType), u"First argument to ffi-prep-callback must be a CFunctionType") raw_closure = rffi.cast(rffi.VOIDP, clibffi.closureHeap.alloc()) @@ -652,6 +663,9 @@ def set_val(self, k, v): @as_var("pixie.ffi", "c-struct") def c_struct(name, size, spec): + """(c-struct name size spec) + Creates a CStruct named name, of size size, with the given spec. Spec is a vector + of vectors. Each row of the format [field-name type offset]""" d = {} for x in range(rt.count(spec)): row = rt.nth(spec, rt.wrap(x)) @@ -669,6 +683,8 @@ def c_struct(name, size, spec): @as_var("pixie.ffi", "cast") def c_cast(frm, to): + """(cast from to) + Converts a VoidP to a CStruct. From is either a VoidP or a CStruct, to is a CStruct type.""" if not isinstance(to, CStructType): runtime_error(u"Expected a CStruct type to cast to, got " + rt.name(rt.str(to))) @@ -677,14 +693,6 @@ def c_cast(frm, to): return to.cast_to(frm) -@as_var("pixie.ffi", "free") -def c_free(frm): - if not isinstance(frm, CStruct) and not isinstance(frm, VoidP): - runtime_error(u"Can only free CStructs or CVoidP") - - lltype.free(frm.raw_data(), flavor="raw") - - return nil @extend(proto._val_at, CStructType.base_type) def val_at(self, k, not_found): @@ -692,6 +700,8 @@ def val_at(self, k, not_found): @as_var("pixie.ffi", "set!") def set_(self, k, val): + """(set! ptr k val) + Sets a field k of struct ptr to value val""" return self.set_val(k, val) diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index 71502c8e..8ea60649 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -39,11 +39,11 @@ (t/deftest test-ffi-callbacks (let [MAX 255 - qsort-cb (ffi-callback [CVoidP CVoidP] CInt) + qsort-cb (pixie.ffi/ffi-callback [CVoidP CVoidP] CInt) qsort (ffi-fn libc "qsort" [CVoidP CInt CInt qsort-cb] CInt) buf (buffer MAX)] - (using [cb (ffi-prep-callback qsort-cb (fn [x y] + (using [cb (pixie.ffi/ffi-prep-callback qsort-cb (fn [x y] (if (> (pixie.ffi/unpack x 0 CUInt8) (pixie.ffi/unpack y 0 CUInt8)) -1 diff --git a/tests/pixie/tests/test-uv.pxi b/tests/pixie/tests/test-uv.pxi index 3998ab37..4167955e 100644 --- a/tests/pixie/tests/test-uv.pxi +++ b/tests/pixie/tests/test-uv.pxi @@ -1,6 +1,7 @@ (ns pixie.test-uv (require pixie.uv :as uv) - (require pixie.test :as t)) + (require pixie.test :as t) + (require pixie.ffi :as ffi)) (t/deftest timer-tests @@ -8,7 +9,7 @@ result (atom false) loop (uv/uv_loop_t) timer (uv/uv_timer_t)] - (reset! cb (ffi-prep-callback uv/uv_timer_cb + (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb (fn [handle] (reset! result true) (uv/uv_timer_stop timer) From 54794eb8339f3da378d782a18ea2f8932f6a74e3 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sat, 14 Mar 2015 11:10:33 +0000 Subject: [PATCH 588/909] Fix line comment bug and improve reader tests Previously, code like (1 2 3 ;foo ) would fail to parse, because LineCommentReader would attempt to return the next form, which was `)`, causing an error. To fix this I changed the semantics of `read` to allow it to not return a form at all (signalled by returning the reader itself, as in https://github.com/clojure/tools.reader), for situations like `;`. Unsure if this is the best approach to fixing, open to discussion. Another option could be to treat comments as whitespace. Also, the reader tests had some cases where they wouldn't verify the input properly, so I fixed those as well. --- pixie/vm/reader.py | 34 +++++++++++++++++++++------------- pixie/vm/test/test_reader.py | 20 ++++++++++++++++++-- 2 files changed, 39 insertions(+), 15 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 3049ae2f..fedb558b 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -221,7 +221,9 @@ def invoke(self, rdr, ch): return acc rdr.unread(ch) - lst.append(read(rdr, True)) + itm = read(rdr, True, always_return_form=False) + if itm != rdr: + lst.append(itm) class UnmatchedListReader(ReaderHandler): def invoke(self, rdr, ch): @@ -237,7 +239,9 @@ def invoke(self, rdr, ch): return acc rdr.unread(ch) - acc = rt.conj(acc, read(rdr, True)) + itm = read(rdr, True, always_return_form=False) + if itm != rdr: + acc = rt.conj(acc, itm) class UnmatchedVectorReader(ReaderHandler): def invoke(self, rdr, ch): @@ -253,9 +257,14 @@ def invoke(self, rdr, ch): return acc rdr.unread(ch) - k = read(rdr, True) - v = read(rdr, False) - acc = rt._assoc(acc, k, v) + itm = read(rdr, True, always_return_form=False) + if itm != rdr: + k = itm + itm = rdr + while itm == rdr: + itm = read(rdr, False, always_return_form=False) + v = itm + acc = rt._assoc(acc, k, v) return acc class UnmatchedMapReader(ReaderHandler): @@ -573,17 +582,13 @@ def invoke(self, rdr, ch): class LineCommentReader(ReaderHandler): def invoke(self, rdr, ch): self.skip_line(rdr) - return read(rdr, True) + return rdr def skip_line(self, rdr): while True: ch = rdr.read() if ch == u"\n": return - elif ch == u"\r": - ch2 = rdr.read() - if ch2 == u"\n": - return handlers = {u"(": ListReader(), u")": UnmatchedListReader(), @@ -723,7 +728,7 @@ def throw_syntax_error_with_data(rdr, txt): -def read(rdr, error_on_eof): +def read(rdr, error_on_eof, always_return_form=True): try: eat_whitespace(rdr) except EOFError as ex: @@ -742,6 +747,8 @@ def read(rdr, error_on_eof): macro = handlers.get(ch, None) if macro is not None: itm = macro.invoke(rdr, ch) + if always_return_form and itm == rdr: + return read(rdr, error_on_eof, always_return_form=always_return_form) elif is_digit(ch): itm = read_number(rdr, ch) @@ -759,8 +766,9 @@ def read(rdr, error_on_eof): else: itm = read_symbol(rdr, ch) - if rt.has_meta_QMARK_(itm): - itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) + if itm != rdr: + if rt.has_meta_QMARK_(itm): + itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) return itm diff --git a/pixie/vm/test/test_reader.py b/pixie/vm/test/test_reader.py index 33a08152..19539f0c 100644 --- a/pixie/vm/test/test_reader.py +++ b/pixie/vm/test/test_reader.py @@ -4,6 +4,8 @@ from pixie.vm.numbers import Integer from pixie.vm.symbol import symbol, Symbol from pixie.vm.persistent_vector import PersistentVector +from pixie.vm.persistent_hash_map import PersistentHashMap +from pixie.vm.primitives import nil import pixie.vm.rt as rt import unittest @@ -17,7 +19,14 @@ u"(platform+ 1 2)": (symbol(u"platform+"), 1, 2), u"[42 43 44]": [42, 43, 44], u"(1 2 ; 7 8 9\n3)": (1, 2, 3,), - u"(1 2 ; 7 8 9\r\n3)": (1, 2, 3,)} + u"(1 2 ; 7 8 9\r\n3)": (1, 2, 3,), + u"(1 2 ; 7 8 9\r\n)": (1, 2,), + u"(1 2 ; 7 8 9\n)": (1, 2,), + u"[1 2 ; 7 8 9\n]": [1, 2,], + u"(1 2; 7 8 9\n)": (1, 2,), + u";foo\n(1 2; 7 8 9\n)": (1, 2,), + u"{\"foo\" 1 ;\"bar\" 2\n\"baz\" 3}": {"foo": 1, "baz": 3}, + u"{\"foo\" ; \"bar\" 2\n1 \"baz\" 3}": {"foo": 1, "baz": 3}} class TestReader(unittest.TestCase): def _compare(self, frm, to): @@ -27,6 +36,7 @@ def _compare(self, frm, to): for x in to: self._compare(frm.first(), x) frm = frm.next() + assert frm == nil elif isinstance(to, int): assert isinstance(frm, Integer) assert frm._int_val == to @@ -38,9 +48,15 @@ def _compare(self, frm, to): elif isinstance(to, list): assert isinstance(frm, PersistentVector) - for x in range(len(to)): + for x in range(max(len(to), rt._count(frm)._int_val)): self._compare(rt.nth(frm, rt.wrap(x)), to[x]) + elif isinstance(to, dict): + assert isinstance(frm, PersistentHashMap) + for key in dict.keys(to): + self._compare(frm.val_at(rt.wrap(key), ""), to[key]) + + assert rt._count(frm)._int_val == len(dict.keys(to)) else: raise Exception("Don't know how to handle " + str(type(to))) From ddbc00dd1e523eba0e6a7111202f48ce489fe839 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sat, 14 Mar 2015 14:59:00 +0000 Subject: [PATCH 589/909] Allow macro characters as character literals Previously ``` (def a \)) ``` would fail to parse. This commit fixes that. --- pixie/vm/reader.py | 2 +- pixie/vm/test/test_reader.py | 12 +++++++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index fedb558b..2ba86059 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -329,7 +329,7 @@ def invoke(self, rdr, ch): acc.append(v) def read_token(rdr): - acc = u"" + acc = rdr.read() while True: ch = rdr.read() if is_whitespace(ch) or is_terminating_macro(ch): diff --git a/pixie/vm/test/test_reader.py b/pixie/vm/test/test_reader.py index 19539f0c..96e8d458 100644 --- a/pixie/vm/test/test_reader.py +++ b/pixie/vm/test/test_reader.py @@ -3,6 +3,7 @@ from pixie.vm.cons import Cons from pixie.vm.numbers import Integer from pixie.vm.symbol import symbol, Symbol +from pixie.vm.string import Character from pixie.vm.persistent_vector import PersistentVector from pixie.vm.persistent_hash_map import PersistentHashMap from pixie.vm.primitives import nil @@ -26,7 +27,11 @@ u"(1 2; 7 8 9\n)": (1, 2,), u";foo\n(1 2; 7 8 9\n)": (1, 2,), u"{\"foo\" 1 ;\"bar\" 2\n\"baz\" 3}": {"foo": 1, "baz": 3}, - u"{\"foo\" ; \"bar\" 2\n1 \"baz\" 3}": {"foo": 1, "baz": 3}} + u"{\"foo\" ; \"bar\" 2\n1 \"baz\" 3}": {"foo": 1, "baz": 3}, + u"(\\a)": (Character(ord("a")),), + u"(\\))": (Character(ord(")")),), + u"(\\()": (Character(ord("(")),), + u"(\\;)": (Character(ord(";")),)} class TestReader(unittest.TestCase): def _compare(self, frm, to): @@ -57,6 +62,11 @@ def _compare(self, frm, to): self._compare(frm.val_at(rt.wrap(key), ""), to[key]) assert rt._count(frm)._int_val == len(dict.keys(to)) + + elif isinstance(to, Character): + assert isinstance(frm, Character) + assert to._char_val == frm._char_val + else: raise Exception("Don't know how to handle " + str(type(to))) From 9dcf7ffcbb1759ee4076385c6fd3f5df35325886 Mon Sep 17 00:00:00 2001 From: Peter Monks Date: Sat, 14 Mar 2015 08:04:21 -0700 Subject: [PATCH 590/909] Commented out pixie.fs tests that work locally but segfault on travis-ci. --- tests/pixie/tests/test-fs.pxi | 49 +++++++++++++++++++---------------- 1 file changed, 26 insertions(+), 23 deletions(-) diff --git a/tests/pixie/tests/test-fs.pxi b/tests/pixie/tests/test-fs.pxi index e5edaffb..5a8ba3b2 100644 --- a/tests/pixie/tests/test-fs.pxi +++ b/tests/pixie/tests/test-fs.pxi @@ -29,32 +29,35 @@ (fs/file file-c)])) 1))) -(t/deftest test-walking - (let [dir-a "tests/pixie/tests/fs"] - (t/assert= (set (fs/walk (fs/dir dir-a))) - #{(fs/dir (str dir-a "/parent")) - (fs/file (str dir-a "/parent/foo.txt")) - (fs/file (str dir-a "/parent/bar.txt")) - (fs/dir (str dir-a "/parent/child")) - (fs/file (str dir-a "/parent/child/foo.txt")) - (fs/file (str dir-a "/parent/child/bar.txt"))}) +(comment + ; Although these pass locally, they don't appear to work on travis-ci (segfault) + (t/deftest test-walking + (let [dir-a "tests/pixie/tests/fs"] + (t/assert= (set (fs/walk (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/dir (str dir-a "/parent/child")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) - (t/assert= (set (fs/walk-files (fs/dir dir-a))) - #{(fs/file (str dir-a "/parent/foo.txt")) - (fs/file (str dir-a "/parent/bar.txt")) - (fs/file (str dir-a "/parent/child/foo.txt")) - (fs/file (str dir-a "/parent/child/bar.txt"))}) + (t/assert= (set (fs/walk-files (fs/dir dir-a))) + #{(fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) - (t/assert= (set (fs/walk-dirs (fs/dir dir-a))) - #{(fs/dir (str dir-a "/parent")) - (fs/dir (str dir-a "/parent/child"))}))) + (t/assert= (set (fs/walk-dirs (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/dir (str dir-a "/parent/child"))}))) -(t/deftest test-list - (let [dir-a "tests/pixie/tests/fs/parent"] - (t/assert= (set (fs/list (fs/dir dir-a))) - #{(fs/file (str dir-a "/foo.txt")) - (fs/file (str dir-a "/bar.txt")) - (fs/dir (str dir-a "/child"))}))) + (t/deftest test-list + (let [dir-a "tests/pixie/tests/fs/parent"] + (t/assert= (set (fs/list (fs/dir dir-a))) + #{(fs/file (str dir-a "/foo.txt")) + (fs/file (str dir-a "/bar.txt")) + (fs/dir (str dir-a "/child"))}))) +) (t/deftest test-rel? (let [dir-a (fs/dir "tests/pixie/tests/fs") From 58a12d4b30c3afdb5d0bcf9e9a4b19652fd27c79 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 14 Mar 2015 15:56:18 -0600 Subject: [PATCH 591/909] added a bit of code to support multiple threads --- pixie/io.pxi | 17 ++++++++++++++--- pixie/stacklets.pxi | 24 ++++++++++++++++++++++++ pixie/uv.pxi | 4 ++-- pixie/vm/libs/libedit.py | 2 +- 4 files changed, 41 insertions(+), 6 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index ed8378ed..5664180c 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -2,7 +2,8 @@ (require pixie.streams :as st :refer :all) (require pixie.uv :as uv) (require pixie.stacklets :as st) - (require pixie.ffi :as ffi)) + (require pixie.ffi :as ffi) + (require pixie.ffi-infer :as ffi-infer)) (defmacro defuvfsfn [nm args return] `(defn ~nm ~args @@ -176,15 +177,25 @@ bind-addr (uv/sockaddr_in) _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) on-new-connetion (atom nil)] - (reset! (ffi-prep-callback + (reset! on-new-connetion + (ffi-prep-callback uv/uv_connection_cb (fn [server status] (when (not (= status -1)) (println "Got Client!!!!!!!"))))) (uv/uv_tcp_init (uv/uv_default_loop) server) - (uv/uv_tcp_bind server bind_addr) + (uv/uv_tcp_bind server bind-addr 0) (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) (st/yield-control))) +(st/apply-blocking println "FROM OTHER THREAD <---!!!!!") + + (tcp-server "0.0.0.0" 4242 nil) +(comment + (defmacro make-readline-async [] + `(let [libname ~(ffi-infer/compile-library {:prefix "pixie.io.readline" + :includes ["uv.h" "editline/readline.h"]})])) + + (ffi-infer/compile-library)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index ff96a3dc..19dc3ca6 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -122,3 +122,27 @@ (defn promise [] (->Promise nil (atom []) false)) + +(defprotocol IThreadPool + (-execute [this work-fn])) + + +;; Super basic Thread Pool, yes, this should be improved + +(deftype ThreadPool [] + IThreadPool + (-execute [this work-fn] + (-thread (fn [] (work-fn))))) + +(def basic-thread-pool (->ThreadPool)) + +(defn -run-in-other-thread [work-fn] + (-execute basic-thread-pool work-fn)) + + +(defn apply-blocking [f & args] + (call-cc (fn [k] + (-run-in-other-thread + (fn [] + (let [result (apply f args)] + (-run-later (fn [] (run-and-process k result))))))))) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 3cab69ae..d984d52a 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -178,7 +178,7 @@ ; TCP Networking (f/defcstruct uv_tcp_t []) - (f/defc-raw-struct sockaddr_in) + (f/defc-raw-struct sockaddr_in []) (f/defcfn uv_tcp_init) (f/defcfn uv_ip4_addr) (f/defcfn uv_tcp_bind) @@ -201,5 +201,5 @@ (defn throw-on-error [result] (if (neg? result) - (throw (str "UV Error: " (uv/uv_err_name result))) + (throw (str "UV Error: " (uv_err_name result))) result)) diff --git a/pixie/vm/libs/libedit.py b/pixie/vm/libs/libedit.py index a5a20203..090eff3f 100644 --- a/pixie/vm/libs/libedit.py +++ b/pixie/vm/libs/libedit.py @@ -14,7 +14,7 @@ libraries=["edit"]) def llexternal(*args, **kwargs): - return rffi.llexternal(*args, compilation_info=compilation_info, **kwargs) + return rffi.llexternal(*args, compilation_info=compilation_info, releasegil=True, **kwargs) __readline = llexternal('readline', [rffi.CCHARP], rffi.CCHARP) From f6f5c56955fc324f0bb586f34a22dbc97f4baa8a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 14 Mar 2015 17:10:39 -0600 Subject: [PATCH 592/909] async repl is getting closer --- pixie/io.pxi | 6 +++--- pixie/repl.pxi | 20 ++++++++++++++++++++ pixie/stacklets.pxi | 4 +++- pixie/vm/reader.py | 44 ++++++++++++++++++++++++++++++++++++++++++++ pixie/vm/rt.py | 2 ++ 5 files changed, 72 insertions(+), 4 deletions(-) create mode 100644 pixie/repl.pxi diff --git a/pixie/io.pxi b/pixie/io.pxi index 5664180c..48ad51f8 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -188,11 +188,11 @@ (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) (st/yield-control))) - -(st/apply-blocking println "FROM OTHER THREAD <---!!!!!") +(comment + (st/apply-blocking println "FROM OTHER THREAD <---!!!!!") -(tcp-server "0.0.0.0" 4242 nil) + (tcp-server "0.0.0.0" 4242 nil)) (comment (defmacro make-readline-async [] `(let [libname ~(ffi-infer/compile-library {:prefix "pixie.io.readline" diff --git a/pixie/repl.pxi b/pixie/repl.pxi new file mode 100644 index 00000000..f00e79c2 --- /dev/null +++ b/pixie/repl.pxi @@ -0,0 +1,20 @@ +(ns pixie.repl + (require pixie.stacklets :as st) + (require pixie.io :as io) + (require pixie.ffi-infer :as f)) + +(f/with-config {:library "edit" + :includes ["editline/readline.h"]} + (f/defcfn readline)) + + +(defn run-repl [] + (let [rdr (reader-fn (fn [] (str (st/apply-blocking readline "user->>>") "\n")))] + (loop [x 1] + (when (< x 3) + + (let [form (read rdr false)] + (println (eval form)) + (recur (inc x))))))) + +(run-repl) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 19dc3ca6..a912e7c0 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -83,7 +83,9 @@ (defn -with-stacklets [fn] (let [[h f] ((new-stacklet fn) nil)] (f h) - (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT))) + (loop [] + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT) + (recur)))) (defmacro with-stacklets [& body] `(-with-stacklets diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 3049ae2f..cb91cfd4 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -2,6 +2,7 @@ import pixie.vm.object as object from pixie.vm.object import affirm, runtime_error import pixie.vm.code as code +from pixie.vm.code import as_var from pixie.vm.primitives import nil, true, false import pixie.vm.numbers as numbers from pixie.vm.cons import cons @@ -89,6 +90,40 @@ def unread(self, ch): assert self._string_reader is not None self._string_reader.unread(ch) + +class UserSpaceReader(PlatformReader): + def __init__(self, reader_fn): + self._string_reader = None + self._reader_fn = reader_fn + + + def read(self): + if self._string_reader is None: + result = rt.name(self._reader_fn.invoke([])) + if result == u"": + raise EOFError() + self._string_reader = StringReader(result) + + try: + return self._string_reader.read() + except EOFError: + self._string_reader = None + return self.read() + + def reset_line(self): + self._string_reader = None + + def unread(self, ch): + assert self._string_reader is not None + self._string_reader.unread(ch) + +@as_var(u"reader-fn") +def reader_fn(fn): + """(reader-fn f) + Creates a new reader that can be passed to read, that will call the given f when a new string + of input is needed.""" + return MetaDataReader(UserSpaceReader(fn)) + class LinePromise(object.Object): _type = object.Type(u"pixie.stdlib.LinePromise") def type(self): @@ -764,5 +799,14 @@ def read(rdr, error_on_eof): return itm +@as_var("read") +def _read_(rdr, error_on_eof): + """(read rdr error-on-eof) + Reads a single form from the input reader. If error-on-eof is true, an error will be thrown if eof is reached + and a valid form has not been parsed. Else will return eof""" + assert isinstance(rdr, PlatformReader) + return read(rdr, rt.is_true(error_on_eof)) + + diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index ce6dbe9c..bfc835a6 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -157,6 +157,8 @@ def reinit(): globals()["__inited__"] = True + globals()["is_true"] = lambda x: False if x is false or x is nil or x is None else True + From 195559e6f2d33565bca05e4ef8d706dc023291fc Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sun, 15 Mar 2015 10:17:58 +0000 Subject: [PATCH 593/909] A couple of small doc and typo fixes --- pixie/io.pxi | 2 +- pixie/stdlib.pxi | 10 +++++----- pixie/string.pxi | 6 +++--- 3 files changed, 9 insertions(+), 9 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 46524dbf..6c38dcdb 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -144,7 +144,7 @@ (defn popen-read - {:doc "Open a file for reading, returning a IInputStream" + {:doc "Open a process for reading, returning a IInputStream" :added "0.1"} [command] (assert (string? command) "Command must be a string") diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 067e52c9..68cef62e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -636,14 +636,14 @@ returns true" (defn nth {:doc "Returns the element at the idx. If the index is not found it will return an error. - However, if you specify a not-found parameter, it will substitue that instead" + However, if you specify a not-found parameter, it will substitute that instead" :signatures [[coll idx] [coll idx not-found]] :added "0.1"} ([coll idx] (-nth coll idx)) ([coll idx not-found] (-nth-not-found coll idx not-found))) (defn first - {:doc "Returns the first item in coll, if coll implements IIndexed nth will be used to retreive + {:doc "Returns the first item in coll, if coll implements IIndexed nth will be used to retrieve the item from the collection." :signatures [[coll]] :added "0.1"} @@ -653,7 +653,7 @@ returns true" (-first coll))) (defn second - {:doc "Returns the second item in coll, if coll implements IIndexed nth will be used to retreive + {:doc "Returns the second item in coll, if coll implements IIndexed nth will be used to retrieve the item from the collection." :signatures [[coll]] :added "0.1"} @@ -663,7 +663,7 @@ returns true" (first (next coll)))) (defn third - {:doc "Returns the third item in coll, if coll implements IIndexed nth will be used to retreive + {:doc "Returns the third item in coll, if coll implements IIndexed nth will be used to retrieve the item from the collection." :signatures [[coll]] :added "0.1"} @@ -673,7 +673,7 @@ returns true" (first (next (next coll))))) (defn fourth - {:doc "Returns the fourth item in coll, if coll implements IIndexed nth will be used to retreive + {:doc "Returns the fourth item in coll, if coll implements IIndexed nth will be used to retrieve the item from the collection." :signatures [[coll]] :added "0.1"} diff --git a/pixie/string.pxi b/pixie/string.pxi index 48b2f8dd..496d3363 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -37,18 +37,18 @@ (str (substring s 0 i) r (substring s (+ i (count x))))))) (defn join - {:doc "Join the elements of the collection using an optional seperator." + {:doc "Join the elements of the collection using an optional separator" :examples [["(require pixie.string :as s)"] ["(s/join [1 2 3])" nil "123"] ["(s/join \", \" [1 2 3])" nil "1, 2, 3"]]} ([coll] (join "" coll)) - ([seperator coll] + ([separator coll] (loop [s (seq coll) res ""] (cond (nil? s) res (nil? (next s)) (str res (first s)) - :else (recur (next s) (str res (first s) seperator)))))) + :else (recur (next s) (str res (first s) separator)))))) (defn blank? "True if s is nil, empty, or contains only whitespace." From 517637ca5630e34e36105ecde3a93144b44b7025 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 15 Mar 2015 16:01:50 +0000 Subject: [PATCH 594/909] Sort arities when creating arity errors Before they just got spat out in order of definition. Uses rpthons listsort to do sorting. --- pixie/vm/code.py | 27 ++++++++++++++++++++++++--- tests/pixie/tests/test-fns.pxi | 12 +++++++----- 2 files changed, 31 insertions(+), 8 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 9e3ad263..c8c4f456 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -3,6 +3,7 @@ from pixie.vm.object import affirm, runtime_error from pixie.vm.primitives import nil, false from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.listsort import TimSort from rpython.rlib.jit import elidable_promote, promote import rpython.rlib.jit as jit import pixie.vm.rt as rt @@ -124,6 +125,21 @@ def stack_size(self): def invoke_with(self, args, this_fn): return self.invoke(args) +def join_last(words, sep): + """ + Joins by commas and uses 'sep' on last word. + + Eg. join_last(['dog', 'cat', 'rat'] , 'and') = 'dog, cat and rat' + """ + if len(words) == 1: + return words[0] + else: + if len(words) == 2: + s = words[0] + u" " + sep + u" " + words[1] + else: + s = u", ".join(words[0:-1]) + s += u" " + sep + u" " + words[-1] + return s class MultiArityFn(BaseCode): _type = object.Type(u"pixie.stdlib.MultiArityFn") @@ -153,13 +169,18 @@ def get_fn(self, arity): return self._rest_fn acc = [] - for x in self._arities: + sorted = TimSort(self.get_arities()) + sorted.sort() + for x in sorted.list: acc.append(unicode(str(x))) if self._rest_fn: - acc.append(unicode(str(self._rest_fn.required_arity())) + u" or more") + acc.append(unicode(str(self._rest_fn.required_arity())) + u"+") + + runtime_error(u"Wrong number of arguments " + unicode(str(arity)) + u" for function '" + unicode(self._name) + u"'. Expected " + join_last(acc, u"or")) - runtime_error(u"Wrong number of arguments " + unicode(str(arity)) + u" for function '" + unicode(self._name) + u"'. Expected " + u",".join(acc)) + def get_arities(self): + return self._arities.keys() def invoke(self, args): return self.invoke_with(args, self) diff --git a/tests/pixie/tests/test-fns.pxi b/tests/pixie/tests/test-fns.pxi index 12f74bc9..23915d0c 100644 --- a/tests/pixie/tests/test-fns.pxi +++ b/tests/pixie/tests/test-fns.pxi @@ -42,17 +42,19 @@ "Invalid number of arguments 1 for function 'arity-2'. Expected 2" (arity-2 :foo)) (t/assert-throws? RuntimeException - "Wrong number of arguments 2 for function 'arity-0-or-1'. Expected 1,0" + "Wrong number of arguments 2 for function 'arity-0-or-1'. Expected 0 or 1" (arity-0-or-1 :foo :bar)) (t/assert-throws? RuntimeException - "Wrong number of arguments 3 for function 'arity-0-or-1'. Expected 1,0" + "Wrong number of arguments 3 for function 'arity-0-or-1'. Expected 0 or 1" (arity-0-or-1 :foo :bar :baz)) (t/assert-throws? RuntimeException - "Wrong number of arguments 2 for function 'arity-1-or-3'. Expected 3,1" + "Wrong number of arguments 2 for function 'arity-1-or-3'. Expected 1 or 3" (arity-1-or-3 :foo :bar)) (t/assert-throws? RuntimeException - "Wrong number of arguments 0 for function 'arity-1-or-3'. Expected 3,1" + "Wrong number of arguments 0 for function 'arity-1-or-3'. Expected 1 or 3" (arity-1-or-3)) (t/assert-throws? RuntimeException - "Wrong number of arguments 2 for function 'arity-0-or-1-or-3-or-more'. Expected 1,0,3 or more" + "Wrong number of arguments 2 for function 'arity-0-or-1-or-3-or-more'. Expected 0, 1 or 3+" (arity-0-or-1-or-3-or-more :foo :bar)))) + +(t/deftest test-code-arities) From 3b515123b110207ff4ac25e5e7daec793ed1a51a Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 15 Mar 2015 16:03:21 +0000 Subject: [PATCH 595/909] Catch EOFErrors while reading things Throws better errors when trying to parse lists, vectors, maps, sets and strings --- pixie/vm/reader.py | 66 ++++++++++++++++------------- tests/pixie/tests/test-readeval.pxi | 40 +++++++++++++++++ 2 files changed, 77 insertions(+), 29 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 2ba86059..803f3359 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -210,8 +210,12 @@ class ListReader(ReaderHandler): def invoke(self, rdr, ch): lst = [] while True: - eat_whitespace(rdr) + try: + eat_whitespace(rdr) + except EOFError: + throw_syntax_error_with_data(rdr, u"Unmatched list open '('") ch = rdr.read() + if ch == u")": if len(lst) == 0: return EmptyList() @@ -219,11 +223,10 @@ def invoke(self, rdr, ch): for x in range(len(lst) - 1, -1, -1): acc = cons(lst[x], acc) return acc - + rdr.unread(ch) - itm = read(rdr, True, always_return_form=False) - if itm != rdr: - lst.append(itm) + + lst.append(read(rdr, True)) class UnmatchedListReader(ReaderHandler): def invoke(self, rdr, ch): @@ -233,15 +236,17 @@ class VectorReader(ReaderHandler): def invoke(self, rdr, ch): acc = EMPTY_VECTOR while True: - eat_whitespace(rdr) + try: + eat_whitespace(rdr) + except EOFError: + throw_syntax_error_with_data(rdr, u"Unmatched vector open '['") + ch = rdr.read() if ch == u"]": return acc rdr.unread(ch) - itm = read(rdr, True, always_return_form=False) - if itm != rdr: - acc = rt.conj(acc, itm) + acc = rt.conj(acc, read(rdr, True)) class UnmatchedVectorReader(ReaderHandler): def invoke(self, rdr, ch): @@ -251,25 +256,24 @@ class MapReader(ReaderHandler): def invoke(self, rdr, ch): acc = EMPTY_MAP while True: - eat_whitespace(rdr) + try: + eat_whitespace(rdr) + except EOFError: + throw_syntax_error_with_data(rdr, u"Unmatched map open '{'") + ch = rdr.read() if ch == u"}": return acc rdr.unread(ch) - itm = read(rdr, True, always_return_form=False) - if itm != rdr: - k = itm - itm = rdr - while itm == rdr: - itm = read(rdr, False, always_return_form=False) - v = itm - acc = rt._assoc(acc, k, v) + k = read(rdr, True) + v = read(rdr, False) + acc = rt._assoc(acc, k, v) return acc class UnmatchedMapReader(ReaderHandler): def invoke(self, rdr, ch): - affirm(False, u"Unmatched Map brace ") + affirm(False, u"Unmatched Map brace '}'") class QuoteReader(ReaderHandler): def invoke(self, rdr, ch): @@ -303,7 +307,7 @@ def invoke(self, rdr, ch): try: v = rdr.read() except EOFError: - return throw_syntax_error_with_data(rdr, u"umatched quote") + return throw_syntax_error_with_data(rdr, u"Unmatched string quote '\"'") if v == "\"": return rt.wrap(u"".join(acc)) @@ -329,7 +333,7 @@ def invoke(self, rdr, ch): acc.append(v) def read_token(rdr): - acc = rdr.read() + acc = u"" while True: ch = rdr.read() if is_whitespace(ch) or is_terminating_macro(ch): @@ -558,7 +562,10 @@ class SetReader(ReaderHandler): def invoke(self, rdr, ch): acc = EMPTY_SET while True: - eat_whitespace(rdr) + try: + eat_whitespace(rdr) + except EOFError: + throw_syntax_error_with_data(rdr, u"Unmatched set open '#{'") ch = rdr.read() if ch == u"}": return acc @@ -582,13 +589,17 @@ def invoke(self, rdr, ch): class LineCommentReader(ReaderHandler): def invoke(self, rdr, ch): self.skip_line(rdr) - return rdr + return read(rdr, True) def skip_line(self, rdr): while True: ch = rdr.read() if ch == u"\n": return + elif ch == u"\r": + ch2 = rdr.read() + if ch2 == u"\n": + return handlers = {u"(": ListReader(), u")": UnmatchedListReader(), @@ -728,7 +739,7 @@ def throw_syntax_error_with_data(rdr, txt): -def read(rdr, error_on_eof, always_return_form=True): +def read(rdr, error_on_eof): try: eat_whitespace(rdr) except EOFError as ex: @@ -747,8 +758,6 @@ def read(rdr, error_on_eof, always_return_form=True): macro = handlers.get(ch, None) if macro is not None: itm = macro.invoke(rdr, ch) - if always_return_form and itm == rdr: - return read(rdr, error_on_eof, always_return_form=always_return_form) elif is_digit(ch): itm = read_number(rdr, ch) @@ -766,9 +775,8 @@ def read(rdr, error_on_eof, always_return_form=True): else: itm = read_symbol(rdr, ch) - if itm != rdr: - if rt.has_meta_QMARK_(itm): - itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) + if rt.has_meta_QMARK_(itm): + itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) return itm diff --git a/tests/pixie/tests/test-readeval.pxi b/tests/pixie/tests/test-readeval.pxi index 01be1a3d..dbafe99f 100644 --- a/tests/pixie/tests/test-readeval.pxi +++ b/tests/pixie/tests/test-readeval.pxi @@ -15,3 +15,43 @@ (t/assert= (read-string "false") false) (t/assert= (read-string "true") true) (t/assert= (read-string "(foo (bar (baz)))") '(foo (bar (baz))))) + +(t/deftest test-list-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched list open '('" + (read-string "(")) + (t/assert-throws? RuntimeException + "Unmatched list open '('" + (read-string "((foo bar)"))) + +(t/deftest test-vector-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched vector open '['" + (read-string "[")) + (t/assert-throws? RuntimeException + "Unmatched vector open '['" + (read-string "[[foo bar]"))) + +(t/deftest test-map-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched map open '{'" + (read-string "{")) + (t/assert-throws? RuntimeException + "Unmatched map open '{'" + (read-string "{foo {a b}"))) + +(t/deftest test-set-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched set open '#{'" + (read-string "#{")) + (t/assert-throws? RuntimeException + "Unmatched set open '#{'" + (read-string "#{foo #{a}"))) + +(t/deftest test-string-unclosed-fail + (t/assert-throws? RuntimeException + "Unmatched string quote '\"'" + (read-string "\"")) + (t/assert-throws? RuntimeException + "Unmatched string quote '\"'" + (read-string "\"foo"))) From 4257329b617b8d769099cb9d01748b5450ea86b7 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 15 Mar 2015 23:17:27 +0000 Subject: [PATCH 596/909] Make errors work in compiled mode --- pixie/vm/primitives.py | 5 ++++- pixie/vm/reader.py | 41 ++++++++++++++++++++++------------------- pixie/vm/stdlib.py | 4 ++-- 3 files changed, 28 insertions(+), 22 deletions(-) diff --git a/pixie/vm/primitives.py b/pixie/vm/primitives.py index acb37963..6bab30aa 100644 --- a/pixie/vm/primitives.py +++ b/pixie/vm/primitives.py @@ -4,6 +4,9 @@ class Nil(object.Object): _type = object.Type(u"pixie.stdlib.Nil") + def __repr__(self): + return u"nil" + def type(self): return Nil._type @@ -19,4 +22,4 @@ def type(self): true = Bool() -false = Bool() \ No newline at end of file +false = Bool() diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 803f3359..daac8913 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -41,7 +41,7 @@ class PlatformReader(object.Object): def read(self): assert False - def unread(self, ch): + def unread(self): pass def reset_line(self): @@ -61,7 +61,7 @@ def read(self): self._idx += 1 return ch - def unread(self, ch): + def unread(self): self._idx -= 1 class PromptReader(PlatformReader): @@ -85,9 +85,9 @@ def read(self): def reset_line(self): self._string_reader = None - def unread(self, ch): + def unread(self): assert self._string_reader is not None - self._string_reader.unread(ch) + self._string_reader.unread() class LinePromise(object.Object): _type = object.Type(u"pixie.stdlib.LinePromise") @@ -168,7 +168,7 @@ def get_metadata(self): FILE_KW, rt.wrap(self._filename)) - def unread(self, ch): + def unread(self): affirm(not self._has_unread, u"Can't unread twice") self._has_unread = True self._prev_chr = self._cur_ch @@ -199,7 +199,7 @@ def eat_whitespace(rdr): ch = rdr.read() if is_whitespace(ch): continue - rdr.unread(ch) + rdr.unread() return class ReaderHandler(py_object): @@ -213,6 +213,7 @@ def invoke(self, rdr, ch): try: eat_whitespace(rdr) except EOFError: + rdr.unread() throw_syntax_error_with_data(rdr, u"Unmatched list open '('") ch = rdr.read() @@ -224,7 +225,7 @@ def invoke(self, rdr, ch): acc = cons(lst[x], acc) return acc - rdr.unread(ch) + rdr.unread() lst.append(read(rdr, True)) @@ -239,13 +240,14 @@ def invoke(self, rdr, ch): try: eat_whitespace(rdr) except EOFError: + rdr throw_syntax_error_with_data(rdr, u"Unmatched vector open '['") ch = rdr.read() if ch == u"]": return acc - rdr.unread(ch) + rdr.unread() acc = rt.conj(acc, read(rdr, True)) class UnmatchedVectorReader(ReaderHandler): @@ -265,7 +267,7 @@ def invoke(self, rdr, ch): if ch == u"}": return acc - rdr.unread(ch) + rdr.unread() k = read(rdr, True) v = read(rdr, False) acc = rt._assoc(acc, k, v) @@ -288,7 +290,7 @@ def invoke(self, rdr, ch): itm = read(rdr, True) nms = rt.name(rt.ns.deref()) else: - rdr.unread(ch) + rdr.unread() itm = read(rdr, True) affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") @@ -337,7 +339,7 @@ def read_token(rdr): while True: ch = rdr.read() if is_whitespace(ch) or is_terminating_macro(ch): - rdr.unread(ch) + rdr.unread() return acc acc += ch @@ -469,7 +471,7 @@ def invoke(self, rdr, ch): if ch == "@": sym = UNQUOTE_SPLICING else: - rdr.unread(ch) + rdr.unread() form = read(rdr, True) return rt.list(sym, form) @@ -496,7 +498,7 @@ def invoke(self, rdr, ch): return read_symbol(rdr, ch) ch = rdr.read() - rdr.unread(ch) + rdr.unread() if is_whitespace(ch) or is_terminating_macro(ch): return ArgReader.register_next_arg(1) @@ -536,7 +538,7 @@ def invoke(self, rdr, ch): try: ARG_ENV.set_value(rt.assoc(EMPTY_MAP, ARG_MAX, rt.wrap(-1))) - rdr.unread(ch) + rdr.unread() form = read(rdr, True) args = EMPTY_VECTOR @@ -570,7 +572,7 @@ def invoke(self, rdr, ch): if ch == u"}": return acc - rdr.unread(ch) + rdr.unread() acc = acc.conj(read(rdr, True)) dispatch_handlers = { @@ -683,7 +685,7 @@ def read_number(rdr, ch): while True: ch = rdr.read() if is_whitespace(ch) or ch in handlers: - rdr.unread(ch) + rdr.unread() break acc.append(ch) except EOFError: @@ -701,7 +703,7 @@ def read_symbol(rdr, ch): while True: ch = rdr.read() if is_whitespace(ch) or is_terminating_macro(ch): - rdr.unread(ch) + rdr.unread() break acc.append(ch) except EOFError: @@ -756,6 +758,7 @@ def read(rdr, error_on_eof): meta = nil macro = handlers.get(ch, None) + itm = nil if macro is not None: itm = macro.invoke(rdr, ch) @@ -765,11 +768,11 @@ def read(rdr, error_on_eof): elif ch == u"-": ch2 = rdr.read() if is_digit(ch2): - rdr.unread(ch2) + rdr.unread() itm = read_number(rdr, ch) else: - rdr.unread(ch2) + rdr.unread() itm = read_symbol(rdr, ch) else: diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 4cfe5d05..0d84d625 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -732,7 +732,7 @@ def _ici(meta): return InterpreterCodeInfo(line, line_number.int_val() if line_number is not nil else 0, col_number.int_val() if col_number is not nil else 0, - rt.name(file) if file is not nil else u"") # @wrap_fn @@ -808,4 +808,4 @@ def ex_msg(e): @extend(_doc, code.NativeFn._type) def _doc(self): assert isinstance(self, code.NativeFn) - return rt.wrap(self._doc) \ No newline at end of file + return rt.wrap(self._doc) From 8b0034845e4e819f0811cd940399d341a3e16e0c Mon Sep 17 00:00:00 2001 From: Arsene Rei Date: Mon, 16 Mar 2015 21:26:17 -0700 Subject: [PATCH 597/909] Add clean Makefile task --- Makefile | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Makefile b/Makefile index bb5cda71..59a30e03 100644 --- a/Makefile +++ b/Makefile @@ -69,3 +69,9 @@ compile_src: clean_pxic: find * -name "*.pxic" | xargs rm + +clean: clean_pxic + rm -rf ./lib + rm -rf ./include + rm -f ./pixie-vm + rm -f ./*.pyc From 0eaff7da98d01e06e6349f2355c2720103c24066 Mon Sep 17 00:00:00 2001 From: Arsene Rei Date: Mon, 16 Mar 2015 21:27:07 -0700 Subject: [PATCH 598/909] Move externals build dir into project --- Makefile | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 59a30e03..46511f39 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ all: help -EXTERNALS=../externals +EXTERNALS=externals PYTHON ?= python PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy @@ -73,5 +73,6 @@ clean_pxic: clean: clean_pxic rm -rf ./lib rm -rf ./include + rm -rf ./externals rm -f ./pixie-vm rm -f ./*.pyc From c956093ae4c80487a1ded77c368260c9cb587275 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 17 Mar 2015 06:03:33 -0600 Subject: [PATCH 599/909] async repl, bug fixes, etc --- Makefile | 4 ++ pixie/ffi-infer.pxi | 8 ---- pixie/io.pxi | 39 +++++++++------- pixie/repl.pxi | 27 +++++++----- pixie/stacklets.pxi | 38 +++++++++++----- pixie/vm/reader.py | 55 ++++++++++++++++------- pixie/vm/test/test_compile.py | 2 +- pixie/vm/test/test_reader.py | 2 +- target.py | 83 ++++++++++++++++++++--------------- 9 files changed, 157 insertions(+), 101 deletions(-) diff --git a/Makefile b/Makefile index 37645acb..16cdf85b 100644 --- a/Makefile +++ b/Makefile @@ -24,6 +24,10 @@ build_with_jit: fetch_externals build_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target.py +compile_basics: + @echo -e "\e[31mWARNING: Compiling core libs. If you want to modify one of these files delete the .pxic files first\e[0m" + ./pixie-vm -c pixie/uv.pxi -c pixie/io.pxi -c pixie/stacklets.pxi -c pixie/stdlib.pxi + build_preload_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 8056b57e..e6964524 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -248,14 +248,6 @@ return 0; -(compile-library - {:prefix "foo" - :includes ["uv.h"]} - "int foo(int bar) - { - return 42; - }") - (comment diff --git a/pixie/io.pxi b/pixie/io.pxi index 48ad51f8..483c9f9f 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -1,5 +1,6 @@ (ns pixie.io (require pixie.streams :as st :refer :all) + (require pixie.io-blocking :as io-blocking) (require pixie.uv :as uv) (require pixie.stacklets :as st) (require pixie.ffi :as ffi) @@ -169,24 +170,28 @@ (dispose! c) result)) +(defn run-command [command] + (st/apply-blocking io-blocking/run-command command)) -(defn tcp-server [ip port on-connection] - (assert (string? ip) "Ip should be a string") - (assert (integer? port) "Port should be a int") - (let [server (uv/uv_tcp_t) - bind-addr (uv/sockaddr_in) - _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) - on-new-connetion (atom nil)] - (reset! on-new-connetion - (ffi-prep-callback - uv/uv_connection_cb - (fn [server status] - (when (not (= status -1)) - (println "Got Client!!!!!!!"))))) - (uv/uv_tcp_init (uv/uv_default_loop) server) - (uv/uv_tcp_bind server bind-addr 0) - (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) - (st/yield-control))) +(comment + + (defn tcp-server [ip port on-connection] + (assert (string? ip) "Ip should be a string") + (assert (integer? port) "Port should be a int") + (let [server (uv/uv_tcp_t) + bind-addr (uv/sockaddr_in) + _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) + on-new-connetion (atom nil)] + (reset! on-new-connetion + (ffi-prep-callback + uv/uv_connection_cb + (fn [server status] + (when (not (= status -1)) + (println "Got Client!!!!!!!"))))) + (uv/uv_tcp_init (uv/uv_default_loop) server) + (uv/uv_tcp_bind server bind-addr 0) + (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) + (st/yield-control)))) (comment (st/apply-blocking println "FROM OTHER THREAD <---!!!!!") diff --git a/pixie/repl.pxi b/pixie/repl.pxi index f00e79c2..c9c8e9a4 100644 --- a/pixie/repl.pxi +++ b/pixie/repl.pxi @@ -8,13 +8,20 @@ (f/defcfn readline)) -(defn run-repl [] - (let [rdr (reader-fn (fn [] (str (st/apply-blocking readline "user->>>") "\n")))] - (loop [x 1] - (when (< x 3) - - (let [form (read rdr false)] - (println (eval form)) - (recur (inc x))))))) - -(run-repl) +(defn repl [] + (let [rdr (reader-fn (fn [] + (let [prompt (if (= 0 pixie.stdlib/*reading-form*) + (str (name pixie.stdlib/*ns*) " => ") + "") + line (st/apply-blocking readline prompt)] + (if line + (str line "\n") + ""))))] + (loop [] + (try (let [form (read rdr false)] + (if (= form eof) + (exit 0) + (println (eval form)))) + (catch ex + (println "ERROR: \n" ex))) + (recur)))) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index a912e7c0..e122c91b 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -4,8 +4,12 @@ ;; If we don't do this, compiling this file doesn't work since the def will clear out ;; the existing value. -(if (undefined? (var stacklet-loop-h)) - (def stacklet-loop-h (atom nil))) +(when (undefined? (var stacklet-loop-h)) + (def stacklet-loop-h (atom nil)) + (def running-threads (atom 0)) + (def main-loop-running? (atom false)) + (def main-loop-lock (-create-lock)) + (-acquire-lock main-loop-lock true)) @@ -34,7 +38,11 @@ (f) (catch ex (println ex)))))) (uv/uv_async_init (uv/uv_default_loop) a @cb) - (uv/uv_async_send a))) + (uv/uv_async_send a) + (when (not @main-loop-running?) + (reset! main-loop-running? true) + (-release-lock main-loop-lock)) + nil)) (defn yield-control [] @@ -53,7 +61,7 @@ (fn [handle] (try (run-and-process k) - (uv/uv_timer_stop timer) + (uv/uv_timer_stop timer) (-dispose! @cb) (catch ex (println ex)))))) @@ -70,22 +78,29 @@ (defmacro spawn [& body] `(-spawn (fn [h# _] - (try - (reset! stacklet-loop-h h#) - (let [result# (do ~@body)] - (call-cc (fn [_] nil))) - (catch e - (println e)))))) + (try + (swap! running-threads inc) + (reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (swap! running-threads dec) + (call-cc (fn [_] nil))) + (catch e + (println e)))))) (defn -with-stacklets [fn] + (swap! running-threads inc) + (reset! main-loop-running? true) (let [[h f] ((new-stacklet fn) nil)] (f h) (loop [] (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT) - (recur)))) + (when (> @running-threads 0) + (reset! main-loop-running? false) + (-acquire-lock main-loop-lock true) + (recur))))) (defmacro with-stacklets [& body] `(-with-stacklets @@ -93,6 +108,7 @@ (try (reset! stacklet-loop-h h#) (let [result# (do ~@body)] + (swap! running-threads dec) (call-cc (fn [_] nil))) (catch e (println e)))))) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index cb91cfd4..ef5cbc1a 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -21,6 +21,10 @@ from rpython.rlib.rbigint import rbigint from rpython.rlib.rsre import rsre_re as re +READING_FORM_VAR = code.intern_var(u"pixie.stdlib", u"*reading-form*") +READING_FORM_VAR.set_dynamic() +READING_FORM_VAR.set_root(false) + LINE_NUMBER_KW = keyword(u"line-number") COLUMN_NUMBER_KW = keyword(u"column-number") LINE_KW = keyword(u"line") @@ -100,6 +104,7 @@ def __init__(self, reader_fn): def read(self): if self._string_reader is None: result = rt.name(self._reader_fn.invoke([])) + code._dynamic_vars.set_var_value(READING_FORM_VAR, rt.wrap(READING_FORM_VAR.deref().int_val() + 1)) if result == u"": raise EOFError() self._string_reader = StringReader(result) @@ -256,7 +261,7 @@ def invoke(self, rdr, ch): return acc rdr.unread(ch) - lst.append(read(rdr, True)) + lst.append(read_inner(rdr, True)) class UnmatchedListReader(ReaderHandler): def invoke(self, rdr, ch): @@ -272,7 +277,7 @@ def invoke(self, rdr, ch): return acc rdr.unread(ch) - acc = rt.conj(acc, read(rdr, True)) + acc = rt.conj(acc, read_inner(rdr, True)) class UnmatchedVectorReader(ReaderHandler): def invoke(self, rdr, ch): @@ -288,8 +293,8 @@ def invoke(self, rdr, ch): return acc rdr.unread(ch) - k = read(rdr, True) - v = read(rdr, False) + k = read_inner(rdr, True) + v = read_inner(rdr, False) acc = rt._assoc(acc, k, v) return acc @@ -299,7 +304,7 @@ def invoke(self, rdr, ch): class QuoteReader(ReaderHandler): def invoke(self, rdr, ch): - itm = read(rdr, True) + itm = read_inner(rdr, True) return cons(symbol(u"quote"), cons(itm)) class KeywordReader(ReaderHandler): @@ -307,11 +312,11 @@ def invoke(self, rdr, ch): nms = u"" ch = rdr.read() if ch == u":": - itm = read(rdr, True) + itm = read_inner(rdr, True) nms = rt.name(rt.ns.deref()) else: rdr.unread(ch) - itm = read(rdr, True) + itm = read_inner(rdr, True) affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") if nms: @@ -407,7 +412,7 @@ def invoke(self, rdr, ch): class DerefReader(ReaderHandler): def invoke(self, rdr, ch): - return rt.cons(symbol(u"-deref"), rt.cons(read(rdr, True), nil)) + return rt.cons(symbol(u"-deref"), rt.cons(read_inner(rdr, True), nil)) QUOTE = symbol(u"quote") @@ -430,7 +435,7 @@ def is_unquote_splicing(form): class SyntaxQuoteReader(ReaderHandler): def invoke(self, rdr, ch): - form = read(rdr, True) + form = read_inner(rdr, True) with code.bindings(GEN_SYM_ENV, EMPTY_MAP): result = self.syntax_quote(form) @@ -493,13 +498,13 @@ def invoke(self, rdr, ch): else: rdr.unread(ch) - form = read(rdr, True) + form = read_inner(rdr, True) return rt.list(sym, form) class MetaReader(ReaderHandler): def invoke(self, rdr, ch): - meta = read(rdr, True) - obj = read(rdr, True) + meta = read_inner(rdr, True) + obj = read_inner(rdr, True) if isinstance(meta, Keyword): meta = rt.hashmap(meta, true) @@ -522,7 +527,7 @@ def invoke(self, rdr, ch): if is_whitespace(ch) or is_terminating_macro(ch): return ArgReader.register_next_arg(1) - n = read(rdr, True) + n = read_inner(rdr, True) if rt.eq(n, ARG_AMP): return ArgReader.register_next_arg(-1) if not isinstance(n, numbers.Integer): @@ -559,7 +564,7 @@ def invoke(self, rdr, ch): ARG_ENV.set_value(rt.assoc(EMPTY_MAP, ARG_MAX, rt.wrap(-1))) rdr.unread(ch) - form = read(rdr, True) + form = read_inner(rdr, True) args = EMPTY_VECTOR percent_args = ARG_ENV.deref() @@ -590,7 +595,7 @@ def invoke(self, rdr, ch): return acc rdr.unread(ch) - acc = acc.conj(read(rdr, True)) + acc = acc.conj(read_inner(rdr, True)) dispatch_handlers = { u"{": SetReader(), @@ -608,7 +613,7 @@ def invoke(self, rdr, ch): class LineCommentReader(ReaderHandler): def invoke(self, rdr, ch): self.skip_line(rdr) - return read(rdr, True) + return read_inner(rdr, True) def skip_line(self, rdr): while True: @@ -738,9 +743,13 @@ def read_symbol(rdr, ch): class EOF(object.Object): _type = object.Type(u"EOF") + def type(self): + return EOF._type + eof = EOF() +code.intern_var(u"pixie.stdlib", u"eof").set_root(eof) @@ -758,7 +767,7 @@ def throw_syntax_error_with_data(rdr, txt): -def read(rdr, error_on_eof): +def read_inner(rdr, error_on_eof): try: eat_whitespace(rdr) except EOFError as ex: @@ -799,6 +808,16 @@ def read(rdr, error_on_eof): return itm + +def read(rdr, error_on_eof): + code._dynamic_vars.push_binding_frame() + code._dynamic_vars.set_var_value(READING_FORM_VAR, rt.wrap(0)) + try: + form = read_inner(rdr, error_on_eof) + return form + finally: + code._dynamic_vars.pop_binding_frame() + @as_var("read") def _read_(rdr, error_on_eof): """(read rdr error-on-eof) @@ -810,3 +829,5 @@ def _read_(rdr, error_on_eof): + + diff --git a/pixie/vm/test/test_compile.py b/pixie/vm/test/test_compile.py index eb73801d..6793e333 100644 --- a/pixie/vm/test/test_compile.py +++ b/pixie/vm/test/test_compile.py @@ -1,4 +1,4 @@ -from pixie.vm.reader import read, StringReader, eof +from pixie.vm.reader import read_inner, StringReader, eof from pixie.vm.object import Type from pixie.vm.cons import Cons from pixie.vm.numbers import Integer diff --git a/pixie/vm/test/test_reader.py b/pixie/vm/test/test_reader.py index 33a08152..d8f17d94 100644 --- a/pixie/vm/test/test_reader.py +++ b/pixie/vm/test/test_reader.py @@ -1,4 +1,4 @@ -from pixie.vm.reader import read, StringReader +from pixie.vm.reader import read_inner, StringReader from pixie.vm.object import Object from pixie.vm.cons import Cons from pixie.vm.numbers import Integer diff --git a/target.py b/target.py index d7f7ca3d..5bd33e85 100644 --- a/target.py +++ b/target.py @@ -1,5 +1,5 @@ from pixie.vm.compiler import compile, with_ns, NS_VAR -from pixie.vm.reader import StringReader, read, eof, PromptReader, MetaDataReader +from pixie.vm.reader import StringReader, read_inner, eof, PromptReader, MetaDataReader from pixie.vm.interpreter import interpret from rpython.jit.codewriter.policy import JitPolicy from rpython.rlib.jit import JitHookInterface, Counters @@ -54,44 +54,55 @@ def __init__(self, args): self._argv = args def inner_invoke(self, args): - from pixie.vm.keyword import keyword import pixie.vm.rt as rt - from pixie.vm.string import String - import pixie.vm.persistent_vector as vector - - print "Pixie 0.1 - Interactive REPL" - print "(" + platform.name + ", " + platform.cc + ")" - print ":exit-repl or Ctrl-D to quit" - print "----------------------------" - - with with_ns(u"user"): - NS_VAR.deref().include_stdlib() - - acc = vector.EMPTY - for x in self._argv: - acc = rt.conj(acc, rt.wrap(x)) - - PROGRAM_ARGUMENTS.set_root(acc) + from pixie.vm.code import intern_var + rt.load_ns(rt.wrap(u"pixie/repl.pxi")) - rdr = MetaDataReader(PromptReader()) + repl = intern_var(u"pixie.repl", u"repl") with with_ns(u"user"): - while True: - try: - val = read(rdr, False) - if val is eof: - break - val = interpret(compile(val)) - self.set_recent_vars(val) - except WrappedException as ex: - print "Error: ", ex._ex.__repr__() - rdr.reset_line() - self.set_error_var(ex._ex) - continue - if val is keyword(u"exit-repl"): - break - val = rt._repr(val) - assert isinstance(val, String), "str should always return a string" - print unicode_to_utf8(val._str) + repl.invoke([]) + + + + # def inner_invoke(self, args): + # from pixie.vm.keyword import keyword + # import pixie.vm.rt as rt + # from pixie.vm.string import String + # import pixie.vm.persistent_vector as vector + # + # print "Pixie 0.1 - Interactive REPL" + # print "(" + platform.name + ", " + platform.cc + ")" + # print ":exit-repl or Ctrl-D to quit" + # print "----------------------------" + # + # with with_ns(u"user"): + # NS_VAR.deref().include_stdlib() + # + # acc = vector.EMPTY + # for x in self._argv: + # acc = rt.conj(acc, rt.wrap(x)) + # + # PROGRAM_ARGUMENTS.set_root(acc) + # + # rdr = MetaDataReader(PromptReader()) + # with with_ns(u"user"): + # while True: + # try: + # val = read(rdr, False) + # if val is eof: + # break + # val = interpret(compile(val)) + # self.set_recent_vars(val) + # except WrappedException as ex: + # print "Error: ", ex._ex.__repr__() + # rdr.reset_line() + # self.set_error_var(ex._ex) + # continue + # if val is keyword(u"exit-repl"): + # break + # val = rt._repr(val) + # assert isinstance(val, String), "str should always return a string" + # print unicode_to_utf8(val._str) def set_recent_vars(self, val): if rt.eq(val, STAR_1.deref()): From 97b06baf2bb8cb676aec489868416bfc41e23f67 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 17 Mar 2015 06:41:59 -0600 Subject: [PATCH 600/909] tests pass, probably about time to merge --- pixie/io-blocking.pxi | 4 ++-- pixie/io.pxi | 6 +++--- pixie/stacklets.pxi | 11 ++++++----- pixie/vm/libs/ffi.py | 14 ++++++++++++-- pixie/vm/reader.py | 4 ++-- 5 files changed, 25 insertions(+), 14 deletions(-) diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index 18ded14a..326d5aca 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -119,8 +119,8 @@ read-count)) (read-byte [this] (fgetc fp)) - IClosable - (close [this] + IDisposable + (-dispose! [this] (pclose fp)) IReduce (-reduce [this f init] diff --git a/pixie/io.pxi b/pixie/io.pxi index 483c9f9f..045e14b5 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -10,7 +10,7 @@ `(defn ~nm ~args (let [f (fn [k#] (let [cb# (atom nil)] - (reset! cb# (ffi-prep-callback uv/uv_fs_cb + (reset! cb# (ffi/ffi-prep-callback uv/uv_fs_cb (fn [req#] (try (st/run-and-process k# (~return (pixie.ffi/cast req# uv/uv_fs_t))) @@ -46,7 +46,7 @@ read-count)) IDisposable (-dispose! [this] - (pixie.ffi/free uvbuf) + (dispose! uvbuf) (fs_close fp)) IReduce (-reduce [this f init] @@ -183,7 +183,7 @@ _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) on-new-connetion (atom nil)] (reset! on-new-connetion - (ffi-prep-callback + (ffi/ffi-prep-callback uv/uv_connection_cb (fn [server status] (when (not (= status -1)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index e122c91b..ac570593 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -1,5 +1,6 @@ (ns pixie.stacklets - (require pixie.uv :as uv)) + (require pixie.uv :as uv) + (require pixie.ffi :as ffi)) ;; If we don't do this, compiling this file doesn't work since the def will clear out ;; the existing value. @@ -30,7 +31,7 @@ (defn -run-later [f] (let [a (uv/uv_async_t) cb (atom nil)] - (reset! cb (ffi-prep-callback uv/uv_async_cb + (reset! cb (ffi/ffi-prep-callback uv/uv_async_cb (fn [handle] (try (uv/uv_close a close_cb) @@ -49,15 +50,15 @@ (call-cc (fn [k] (-run-later (partial run-and-process k))))) -(def close_cb (ffi-prep-callback uv/uv_close_cb - pixie.ffi/free)) +(def close_cb (ffi/ffi-prep-callback uv/uv_close_cb + pixie.ffi/dispose!)) ;;; Sleep (defn sleep [ms] (let [f (fn [k] (let [cb (atom nil) timer (uv/uv_timer_t)] - (reset! cb (ffi-prep-callback uv/uv_timer_cb + (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb (fn [handle] (try (run-and-process k) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index f56a0754..ead55775 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -4,7 +4,7 @@ from pixie.vm.object import runtime_error from pixie.vm.keyword import Keyword import pixie.vm.stdlib as proto -from pixie.vm.code import as_var, affirm, extend +from pixie.vm.code import as_var, affirm, extend, wrap_fn import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype, llmemory from pixie.vm.primitives import nil, true, false @@ -709,6 +709,14 @@ def set_val(self, k, v): return nil + def free_data(self): + lltype.free(self._buffer, flavor="raw") + +@wrap_fn +def _dispose_cstruct(self): + self.free_data() + + @as_var("pixie.ffi", "c-struct") def c_struct(name, size, spec): @@ -728,7 +736,9 @@ def c_struct(name, size, spec): d[nm] = (tp, offset.int_val()) - return CStructType(rt.name(name), size.int_val(), d) + tp = CStructType(rt.name(name), size.int_val(), d) + proto._dispose_BANG_.extend(tp, _dispose_cstruct) + return tp @as_var("pixie.ffi", "cast") def c_cast(frm, to): diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index df67d7fb..4c61f3c7 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -118,9 +118,9 @@ def read(self): def reset_line(self): self._string_reader = None - def unread(self, ch): + def unread(self): assert self._string_reader is not None - self._string_reader.unread(ch) + self._string_reader.unread() @as_var(u"reader-fn") def reader_fn(fn): From 004fa83861ea996aed5301c1118fbfd9c2e88938 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 16 Mar 2015 23:51:56 +0000 Subject: [PATCH 601/909] Not delete all of @justinj's comment reading fixes --- pixie/vm/reader.py | 43 +++++++++++++++++------------ tests/pixie/tests/test-readeval.pxi | 3 ++ 2 files changed, 28 insertions(+), 18 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index daac8913..8aa53bd2 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -213,7 +213,6 @@ def invoke(self, rdr, ch): try: eat_whitespace(rdr) except EOFError: - rdr.unread() throw_syntax_error_with_data(rdr, u"Unmatched list open '('") ch = rdr.read() @@ -226,8 +225,9 @@ def invoke(self, rdr, ch): return acc rdr.unread() - - lst.append(read(rdr, True)) + itm = read(rdr, True, always_return_form=False) + if itm != rdr: + lst.append(itm) class UnmatchedListReader(ReaderHandler): def invoke(self, rdr, ch): @@ -248,7 +248,10 @@ def invoke(self, rdr, ch): return acc rdr.unread() - acc = rt.conj(acc, read(rdr, True)) + itm = read(rdr, True, always_return_form=False) + if itm != rdr: + acc = rt.conj(acc, itm) + class UnmatchedVectorReader(ReaderHandler): def invoke(self, rdr, ch): @@ -268,9 +271,15 @@ def invoke(self, rdr, ch): return acc rdr.unread() - k = read(rdr, True) - v = read(rdr, False) - acc = rt._assoc(acc, k, v) + itm = read(rdr, True, always_return_form=False) + if itm != rdr: + k = itm + itm = rdr + while itm == rdr: + itm = read(rdr, False, always_return_form=False) + v = itm + acc = rt._assoc(acc, k, v) + return acc class UnmatchedMapReader(ReaderHandler): @@ -335,7 +344,7 @@ def invoke(self, rdr, ch): acc.append(v) def read_token(rdr): - acc = u"" + acc = rdr.read() while True: ch = rdr.read() if is_whitespace(ch) or is_terminating_macro(ch): @@ -591,17 +600,13 @@ def invoke(self, rdr, ch): class LineCommentReader(ReaderHandler): def invoke(self, rdr, ch): self.skip_line(rdr) - return read(rdr, True) + return rdr def skip_line(self, rdr): while True: ch = rdr.read() if ch == u"\n": return - elif ch == u"\r": - ch2 = rdr.read() - if ch2 == u"\n": - return handlers = {u"(": ListReader(), u")": UnmatchedListReader(), @@ -740,8 +745,7 @@ def throw_syntax_error_with_data(rdr, txt): raise object.WrappedException(err) - -def read(rdr, error_on_eof): +def read(rdr, error_on_eof, always_return_form=True): try: eat_whitespace(rdr) except EOFError as ex: @@ -758,9 +762,11 @@ def read(rdr, error_on_eof): meta = nil macro = handlers.get(ch, None) - itm = nil if macro is not None: itm = macro.invoke(rdr, ch) + if always_return_form and itm == rdr: + return read(rdr, error_on_eof, always_return_form=always_return_form) + elif is_digit(ch): itm = read_number(rdr, ch) @@ -778,8 +784,9 @@ def read(rdr, error_on_eof): else: itm = read_symbol(rdr, ch) - if rt.has_meta_QMARK_(itm): - itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) + if itm != rdr: + if rt.has_meta_QMARK_(itm): + itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) return itm diff --git a/tests/pixie/tests/test-readeval.pxi b/tests/pixie/tests/test-readeval.pxi index dbafe99f..196ce3af 100644 --- a/tests/pixie/tests/test-readeval.pxi +++ b/tests/pixie/tests/test-readeval.pxi @@ -55,3 +55,6 @@ (t/assert-throws? RuntimeException "Unmatched string quote '\"'" (read-string "\"foo"))) + +(t/deftest test-comments-in-forms + (t/assert= (read-string "(foo ; a comment\n )") '(foo))) From ece8c167ebbe0e14e9520e635c16e38701d84b40 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 17 Mar 2015 16:47:27 -0600 Subject: [PATCH 602/909] adds a async namespace. Includes futures and promises --- pixie/async.pxi | 34 ++++++++++++++++++++++++++++++++ pixie/stacklets.pxi | 25 ----------------------- tests/pixie/tests/test-async.pxi | 15 ++++++++++++++ 3 files changed, 49 insertions(+), 25 deletions(-) create mode 100644 pixie/async.pxi create mode 100644 tests/pixie/tests/test-async.pxi diff --git a/pixie/async.pxi b/pixie/async.pxi new file mode 100644 index 00000000..11ccb268 --- /dev/null +++ b/pixie/async.pxi @@ -0,0 +1,34 @@ +(ns pixie.async + (require pixie.stacklets :as st)) + + +(deftype Promise [val pending-callbacks delivered?] + IDeref + (-deref [self] + (println "waiting... " delivered?) + (if delivered? + val + (do + (st/call-cc (fn [k] + (swap! pending-callbacks conj + (fn [v] + (st/-run-later (partial st/run-and-process k v))))))))) + IFn + (-invoke [self v] + (println "delivering....") + (assert (not delivered?) "Can only deliver a promise once") + (set-field! self :val v) + (set-field! self :delivered? true) + (println @pending-callbacks) + (doseq [f @pending-callbacks] + (f v)) + (reset! pending-callbacks nil) + nil)) + +(defn promise [] + (->Promise nil (atom []) false)) + +(defmacro future [& body] + `(let [p# (promise)] + (st/spawn (p# (do ~@body))) + p#)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index ac570593..5673afcf 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -118,34 +118,9 @@ (with-stacklets (f))) - -(deftype Promise [val pending-callbacks delivered?] - IDeref - (-deref [self] - (if delivered? - val - (do - (call-cc (fn [k] - (swap! pending-callbacks conj - (fn [v] - (-run-later (partial run-and-process k v))))))))) - IFn - (-invoke [self v] - (assert (not delivered?) "Can only deliver a promise once") - (set-field! self :val v) - (println @pending-callbacks) - (doseq [f @pending-callbacks] - (f v)) - (reset! pending-callbacks nil) - nil)) - -(defn promise [] - (->Promise nil (atom []) false)) - (defprotocol IThreadPool (-execute [this work-fn])) - ;; Super basic Thread Pool, yes, this should be improved (deftype ThreadPool [] diff --git a/tests/pixie/tests/test-async.pxi b/tests/pixie/tests/test-async.pxi new file mode 100644 index 00000000..7d2055e2 --- /dev/null +++ b/tests/pixie/tests/test-async.pxi @@ -0,0 +1,15 @@ +(ns pixie.tests.test-async + (require pixie.stacklets :as st) + (require pixie.async :as async :refer :all) + (require pixie.test :as t :refer :all)) + + +(deftest test-future-deref + (let [f (future 42)] + (assert= @f 42))) + +(deftest test-future-deref-chain + (let [f1 (future 42) + f2 (future @f1) + f3 (future @f2)] + (assert= @f3 42))) From fe85dc1b42ed36d39027abd033056dada6c86700 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 17 Mar 2015 16:49:15 -0600 Subject: [PATCH 603/909] remove debug printlns --- pixie/async.pxi | 3 --- 1 file changed, 3 deletions(-) diff --git a/pixie/async.pxi b/pixie/async.pxi index 11ccb268..2907936a 100644 --- a/pixie/async.pxi +++ b/pixie/async.pxi @@ -5,7 +5,6 @@ (deftype Promise [val pending-callbacks delivered?] IDeref (-deref [self] - (println "waiting... " delivered?) (if delivered? val (do @@ -15,11 +14,9 @@ (st/-run-later (partial st/run-and-process k v))))))))) IFn (-invoke [self v] - (println "delivering....") (assert (not delivered?) "Can only deliver a promise once") (set-field! self :val v) (set-field! self :delivered? true) - (println @pending-callbacks) (doseq [f @pending-callbacks] (f v)) (reset! pending-callbacks nil) From bcd8e978d6b9ae37e345823cf11655875e2041ca Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Tue, 17 Mar 2015 21:59:50 +0000 Subject: [PATCH 604/909] Add more comments tests and fix case I missed I fixed the comment problem for lists, vectors, and maps, but missed sets, so I've fixed that now. --- pixie/vm/reader.py | 4 +++- tests/pixie/tests/test-readeval.pxi | 7 ++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 9e587fee..97be5baa 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -623,7 +623,9 @@ def invoke(self, rdr, ch): return acc rdr.unread() - acc = acc.conj(read_inner(rdr, True)) + itm = read_inner(rdr, True, always_return_form=False) + if itm != rdr: + acc = acc.conj(itm) dispatch_handlers = { u"{": SetReader(), diff --git a/tests/pixie/tests/test-readeval.pxi b/tests/pixie/tests/test-readeval.pxi index 196ce3af..4344bff6 100644 --- a/tests/pixie/tests/test-readeval.pxi +++ b/tests/pixie/tests/test-readeval.pxi @@ -14,6 +14,7 @@ (t/assert= (read-string "\"fo\\\\o\"") "fo\\o") (t/assert= (read-string "false") false) (t/assert= (read-string "true") true) + (t/assert= (read-string "#{1 2 3}") #{1 2 3}) (t/assert= (read-string "(foo (bar (baz)))") '(foo (bar (baz))))) (t/deftest test-list-unclosed-list-fail @@ -57,4 +58,8 @@ (read-string "\"foo"))) (t/deftest test-comments-in-forms - (t/assert= (read-string "(foo ; a comment\n )") '(foo))) + (t/assert= (read-string "(foo ; a comment\n )") '(foo)) + (t/assert= (read-string "[foo ; a comment\n ]") '[foo]) + (t/assert= (read-string "{:foo :bar ; a comment\n }") '{:foo :bar}) + (t/assert= (read-string "#{:foo ; a comment\n }") '#{:foo}) + (t/assert= (read-string "{:foo ; a comment\n :bar }") '{:foo :bar})) From e600b341fe89026a29ff172443e785a22adc0591 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Mar 2015 15:27:52 -0600 Subject: [PATCH 605/909] fixed dynamic vars for stacklets --- pixie/stacklets.pxi | 24 ++++++++++++--------- pixie/vm/code.py | 37 ++++++++++++++++++++++++++------ pixie/vm/reader.py | 4 ++-- pixie/vm/rt.py | 9 +++++--- pixie/vm/stdlib.py | 14 ++++++++++++ tests/pixie/tests/test-async.pxi | 14 ++++++++++++ 6 files changed, 80 insertions(+), 22 deletions(-) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 5673afcf..ec03ae31 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -24,8 +24,10 @@ (f h)))) (defn call-cc [f] - (let [[h val] (@stacklet-loop-h f)] + (let [frames (-get-current-var-frames nil) + [h val] (@stacklet-loop-h f)] (reset! stacklet-loop-h h) + (-set-current-var-frames nil frames) val)) (defn -run-later [f] @@ -78,15 +80,17 @@ (-run-later (partial run-and-process k))))) (defmacro spawn [& body] - `(-spawn (fn [h# _] - (try - (swap! running-threads inc) - (reset! stacklet-loop-h h#) - (let [result# (do ~@body)] - (swap! running-threads dec) - (call-cc (fn [_] nil))) - (catch e - (println e)))))) + `(let [frames (-get-current-var-frames nil)] + (-spawn (fn [h# _] + (-set-current-var-frames nil frames) + (try + (swap! running-threads inc) + (reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (swap! running-threads dec) + (call-cc (fn [_] nil))) + (catch e + (println e))))))) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index c8c4f456..852b6f79 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -5,6 +5,7 @@ from rpython.rlib.rarithmetic import r_uint from rpython.rlib.listsort import TimSort from rpython.rlib.jit import elidable_promote, promote +from rpython.rlib.objectmodel import we_are_translated import rpython.rlib.jit as jit import pixie.vm.rt as rt @@ -377,21 +378,32 @@ def type(self): class DynamicVars(py_object): def __init__(self): - self._vars = [{}] + self._vars = rt.cons(rt.hashmap(), nil) def push_binding_frame(self): - self._vars.append(self._vars[-1].copy()) + self._vars = rt.cons(rt.first(self._vars), self._vars) def pop_binding_frame(self): - self._vars.pop() + self._vars = rt.next(self._vars) + + def current_frame(self): + return rt.first(self._vars) + + def get_current_frames(self): + return self._vars + + def set_current_frames(self, vars): + self._vars = vars def get_var_value(self, var, not_found): - return self._vars[-1].get(var, not_found) + return rt._val_at(self.current_frame(), var, not_found) def set_var_value(self, var, val): - self._vars[-1][var] = val + cur_frame = self.current_frame() + self.pop_binding_frame() + self._vars = rt.cons(rt._assoc(cur_frame, var, val), self._vars) + -_dynamic_vars = DynamicVars() class Var(BaseCode): @@ -443,7 +455,14 @@ def get_root(self, rev): def deref(self): if self.is_dynamic(): - return self.get_dynamic_value() + if we_are_translated(): + return self.get_dynamic_value() + else: + ## NOT RPYTHON + if globals().has_key("_dynamic_vars"): + return self.get_dynamic_value() + else: + return self.get_root(self._rev) else: val = self.get_root(self._rev) affirm(val is not undefined, u"Var " + self._name + u" is undefined") @@ -961,3 +980,7 @@ def __enter__(self): def __exit__(self, exc_type, exc_val, exc_tb): _dynamic_vars.pop_binding_frame() + + +def init(): + globals()["_dynamic_vars"] = DynamicVars() diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 9e587fee..b80bd77b 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -32,13 +32,13 @@ GEN_SYM_ENV = code.intern_var(u"pixie.stdlib.reader", u"*gen-sym-env*") GEN_SYM_ENV.set_dynamic() -GEN_SYM_ENV.set_value(EMPTY_MAP) +GEN_SYM_ENV.set_root(EMPTY_MAP) ARG_AMP = symbol(u"&") ARG_MAX = keyword(u"max-arg") ARG_ENV = code.intern_var(u"pixie.stdlib.reader", u"*arg-env*") ARG_ENV.set_dynamic() -ARG_ENV.set_value(nil) +ARG_ENV.set_root(nil) class PlatformReader(object.Object): _type = object.Type(u"PlatformReader") diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index bfc835a6..794a4c3b 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -49,6 +49,7 @@ def wrapper(*args): import sys #sys.setrecursionlimit(10000) # Yeah we blow the stack sometimes, we promise it's not a bug + import pixie.vm.code as code import pixie.vm.numbers as numbers import pixie.vm.bits import pixie.vm.interpreter @@ -71,8 +72,6 @@ def wrapper(*args): import pixie.vm.string_builder import pixie.vm.stacklet - numbers.init() - @specialize.argtype(0) def wrap(x): if isinstance(x, bool): @@ -145,7 +144,7 @@ def reinit(): # stacklet.with_stacklets(run_load_stdlib) init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file", u"compile-file", - u"load-ns", u"hashmap"] + u"load-ns", u"hashmap", u"cons", u"-assoc", u"-val-at"] for x in init_fns: globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) @@ -159,6 +158,10 @@ def reinit(): globals()["is_true"] = lambda x: False if x is false or x is nil or x is None else True + numbers.init() + code.init() + + diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index a4c319ba..560578a2 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -813,3 +813,17 @@ def ex_msg(e): def _doc(self): assert isinstance(self, code.NativeFn) return rt.wrap(self._doc) + + +@as_var("-get-current-var-frames") +def _get_current_var_frames(self): + """(-get-current-var-frames) + Returns the current var frames, will be a cons list of hash maps containing mappings of vars to dynamic values""" + return code._dynamic_vars.get_current_frames() + +@as_var("-set-current-var-frames") +def _set_current_var_frames(self, frames): + """(-set-current-var-frames frames) + Sets the current var frames. Frames should be a cons list of hashmaps containing mappings of vars to dynamic + values. Setting this value to anything but this data format will cause undefined errors.""" + code._dynamic_vars.set_current_frames(frames) \ No newline at end of file diff --git a/tests/pixie/tests/test-async.pxi b/tests/pixie/tests/test-async.pxi index 7d2055e2..d5764af6 100644 --- a/tests/pixie/tests/test-async.pxi +++ b/tests/pixie/tests/test-async.pxi @@ -13,3 +13,17 @@ f2 (future @f1) f3 (future @f2)] (assert= @f3 42))) + +(def *some-var* 0) +(set-dynamic! (var *some-var*)) + +(deftest test-dynamic-var-propagation + (set! (var *some-var*) 0) + (assert= *some-var* 0) + (let [fr @(future (do (println "running") + (let [old-val *some-var*] + (set! (var *some-var*) 42) + [old-val *some-var*])))] + + (assert= fr [0 42]) + (assert= *some-var* 0))) From d40c9759bf69e6c9cb4b23bb17984165255dd265 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Mar 2015 16:07:46 -0600 Subject: [PATCH 606/909] added ring buffer and tests --- pixie/buffers.pxi | 79 ++++++++++++++++++++++++++++++ tests/pixie/tests/test-buffers.pxi | 25 ++++++++++ 2 files changed, 104 insertions(+) create mode 100644 pixie/buffers.pxi create mode 100644 tests/pixie/tests/test-buffers.pxi diff --git a/pixie/buffers.pxi b/pixie/buffers.pxi new file mode 100644 index 00000000..757bc8b0 --- /dev/null +++ b/pixie/buffers.pxi @@ -0,0 +1,79 @@ +(ns pixie.buffers) + +(defn acopy [src src-start dest dest-start len] + (loop [cnt 0] + (when (< cnt len) + (aset dest + (+ dest-start cnt) + (aget src (+ src-start cnt))) + (recur (inc cnt))))) + + +(defprotocol IMutableBuffer + (remove! [this]) + (add! [this]) + (full? [this])) + +(defprotocol IResizableMutableBuffer + (add-unbounded! [this val]) + (resize! [this new-size])) + +(deftype RingBuffer [head tail length arr] + IMutableBuffer + (remove! [this] + (when-not (zero? length) + (let [x (aget arr tail)] + (aset arr tail nil) + (set-field! this :tail (int (rem (inc tail) (alength arr)))) + (set-field! this :length (dec length)) + x))) + (add! [this x] + (assert (< length (alength arr))) + (aset arr head x) + (set-field! this :head (int (rem (inc head) (alength arr)))) + (set-field! this :length (inc length)) + nil) + + (full? [this] + (= length (alength arr))) + + + IResizableMutableBuffer + (resize! [this new-size] + (let [new-arr (make-array new-size)] + (cond + (< tail head) + (do (acopy arr tail new-arr 0 length) + (set-field! this :tail 0) + (set-field! this :head length) + (set-field! this :arr new-arr)) + + (> tail head) + (do (acopy arr tail new-arr 0 (- (alength arr) tail)) + (acopy arr 0 new-arr (- (alength arr) tail) head) + (set-field! this :tail 0) + (set-field! this :head length) + (set-field! this :arr new-arr)) + + + (full? this) + (do (acopy arr tail new-arr 0 length) + (set-field! this :tail 0) + (set-field! this :head length) + (set-field! this :arr new-arr)) + + + :else + (do (set-field! this :tail 0) + (set-field! this :head 0) + (set-field! this :arr new-arr))))) + + (add-unbounded! [this val] + (when (full? this) + (resize! this (* 2 length))) + (add! this val))) + + +(defn ring-buffer [size] + (assert (> size 0) "Can't create a ring buffer of size <= 0") + (->RingBuffer 0 0 0 (make-array size))) diff --git a/tests/pixie/tests/test-buffers.pxi b/tests/pixie/tests/test-buffers.pxi new file mode 100644 index 00000000..382ffc8f --- /dev/null +++ b/tests/pixie/tests/test-buffers.pxi @@ -0,0 +1,25 @@ +(ns pixie.tests.test-buffers + (require pixie.test :refer :all) + (require pixie.buffers :refer :all)) + + +(deftest test-adding-and-removing-from-buffer + (let [buffer (ring-buffer 10)] + (dotimes [x 100] + (add! buffer x) + (assert= x (remove! buffer))))) + +(deftest test-adding-multiple-items + (let [buffer (ring-buffer 10)] + (dotimes [x 10] + (add! buffer x)) + (dotimes [x 10] + (assert= x (remove! buffer))))) + +(deftest test-adding-multiple-items-with-resize + (dotimes [y 100] + (let [buffer (ring-buffer 2)] + (dotimes [x y] + (add-unbounded! buffer x)) + (dotimes [x y] + (assert= x (remove! buffer)))))) From 641d49ae5d2ede0c04909c9a8fde47a87a06179a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Mar 2015 16:40:07 -0600 Subject: [PATCH 607/909] implemented a few other buffer types --- pixie/buffers.pxi | 54 +++++++++++++++++++++++++++++- pixie/vm/stdlib.py | 2 +- tests/pixie/tests/test-buffers.pxi | 10 ++++++ 3 files changed, 64 insertions(+), 2 deletions(-) diff --git a/pixie/buffers.pxi b/pixie/buffers.pxi index 757bc8b0..76ee9fb7 100644 --- a/pixie/buffers.pxi +++ b/pixie/buffers.pxi @@ -71,9 +71,61 @@ (add-unbounded! [this val] (when (full? this) (resize! this (* 2 length))) - (add! this val))) + (add! this val)) + + ICounted + (-count [this] + length)) (defn ring-buffer [size] (assert (> size 0) "Can't create a ring buffer of size <= 0") (->RingBuffer 0 0 0 (make-array size))) + + +(defn fixed-buffer [size] + (ring-buffer size)) + + +(deftype DroppingBuffer [buf] + IMutableBuffer + (full? [this] + false) + (remove! [this] + (remove! buf)) + (add! [this val] + (when-not (full? buf) + (add! buf val))) + + ICounted + (-count [this] + (count buf))) + +(defn dropping-buffer [size] + (->DroppingBuffer (ring-buffer size))) + + +(deftype SlidingBuffer [buf] + IMutableBuffer + (full? [this] + false) + (remove! [this] + (remove! buf)) + (add! [this val] + (when (full? buf) + (remove! buf)) + (add! buf val)) + + ICounted + (-count [this] + (count buf))) + + +(extend -reduce IMutableBuffer + (fn [buf f acc] + (loop [acc acc] + (if (reduced? acc) + @acc + (if (> 0 (count buf)) + (recur (f acc (remove! buf))) + acc))))) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index a4c319ba..9f6923c4 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -668,7 +668,7 @@ def _try_catch(main_fn, catch_fn, final): if not we_are_translated(): print "Python Error Info: ", ex.__dict__, ex raise - ex = RuntimeException(rt.wrap(u"Some error")) + ex = RuntimeException(rt.wrap(u"Some error: " + unicode(str(ex)))) else: ex = RuntimeException(nil) return catch_fn.invoke([ex]) diff --git a/tests/pixie/tests/test-buffers.pxi b/tests/pixie/tests/test-buffers.pxi index 382ffc8f..55f78ada 100644 --- a/tests/pixie/tests/test-buffers.pxi +++ b/tests/pixie/tests/test-buffers.pxi @@ -23,3 +23,13 @@ (add-unbounded! buffer x)) (dotimes [x y] (assert= x (remove! buffer)))))) + + +(def drain-buffer (partial into [])) + +(deftest test-dropping-buffer + (let [buf (dropping-buffer 4)] + (dotimes [x 5] + (println (count buf) "x" x) + (add! buf x)) + (assert= [1 2 3 4] (drain-buffer buf)))) From e2d5f4b0ab7bd4c78c68c4814548121e43be65cc Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Mar 2015 16:49:45 -0600 Subject: [PATCH 608/909] fixes #234 --- pixie/vm/custom_types.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 0434ba36..25a5e7e7 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -1,4 +1,4 @@ -from pixie.vm.object import Object, Type, affirm +from pixie.vm.object import Object, Type, affirm, runtime_error import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import r_uint from pixie.vm.code import as_var @@ -16,7 +16,10 @@ def __init__(self, name, slots): @jit.elidable_promote() def get_slot_idx(self, nm): - return self._slots[nm] + val = self._slots.get(nm, -1) + if val == -1: + runtime_error(u"Invalid field named " + rt.name(rt.str(nm)) + u" on type " + rt.name(rt.str(self))) + return val def set_mutable(self, nm): if not self.is_mutable(nm): From 7f879128b5d276873049482f507f066446880e28 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Mar 2015 17:14:07 -0600 Subject: [PATCH 609/909] added deftype fix for shadowing protocol method names, fixed failing buffer tests --- pixie/buffers.pxi | 2 +- pixie/stdlib.pxi | 2 +- tests/pixie/tests/test-buffers.pxi | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/buffers.pxi b/pixie/buffers.pxi index 76ee9fb7..4813d624 100644 --- a/pixie/buffers.pxi +++ b/pixie/buffers.pxi @@ -126,6 +126,6 @@ (loop [acc acc] (if (reduced? acc) @acc - (if (> 0 (count buf)) + (if (pos? (count buf)) (recur (f acc (remove! buf))) acc))))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 791f53d4..4165876e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1131,7 +1131,7 @@ Creates new maps if the keys are not present." ~@body)]) rest fields)] - `(fn ~fn-name ~args ~@body))) + `(fn ~(symbol (str fn-name "_" nm)) ~args ~@body))) bodies (reduce (fn [res body] (cond diff --git a/tests/pixie/tests/test-buffers.pxi b/tests/pixie/tests/test-buffers.pxi index 55f78ada..ea4abf4c 100644 --- a/tests/pixie/tests/test-buffers.pxi +++ b/tests/pixie/tests/test-buffers.pxi @@ -32,4 +32,4 @@ (dotimes [x 5] (println (count buf) "x" x) (add! buf x)) - (assert= [1 2 3 4] (drain-buffer buf)))) + (assert= [0 1 2 3] (drain-buffer buf)))) From 9e05111d13aa084a568d33b50824350ca0235554 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Mar 2015 20:44:20 -0600 Subject: [PATCH 610/909] fix jit error with custom type errors --- pixie/vm/custom_types.py | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 25a5e7e7..1fd5c4ab 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -16,10 +16,7 @@ def __init__(self, name, slots): @jit.elidable_promote() def get_slot_idx(self, nm): - val = self._slots.get(nm, -1) - if val == -1: - runtime_error(u"Invalid field named " + rt.name(rt.str(nm)) + u" on type " + rt.name(rt.str(self))) - return val + return self._slots.get(nm, -1) def set_mutable(self, nm): if not self.is_mutable(nm): @@ -51,18 +48,27 @@ def type(self): def set_field(self, name, val): self._custom_type.set_mutable(name) idx = self._custom_type.get_slot_idx(name) + if idx == -1: + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) + self._fields[idx] = val return self @jit.elidable_promote() def get_field_immutable(self, name): idx = self._custom_type.get_slot_idx(name) + if idx == -1: + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) + return self._fields[idx] def get_field(self, name): if self._custom_type.is_mutable(name): idx = self._custom_type.get_slot_idx(name) + if idx == -1: + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) + return self._fields[idx] else: return self.get_field_immutable(name) From 3ab8ee4b17132ec1b15aad7c2c6b19ac819d052b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 18 Mar 2015 21:16:46 -0600 Subject: [PATCH 611/909] one more fix for the jit --- pixie/vm/custom_types.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 1fd5c4ab..0a91145d 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -55,23 +55,19 @@ def set_field(self, name, val): return self @jit.elidable_promote() - def get_field_immutable(self, name): - idx = self._custom_type.get_slot_idx(name) - if idx == -1: - runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) - + def get_field_immutable(self, idx): return self._fields[idx] def get_field(self, name): - if self._custom_type.is_mutable(name): - idx = self._custom_type.get_slot_idx(name) - if idx == -1: - runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) + idx = self._custom_type.get_slot_idx(name) + if idx == -1: + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) + if self._custom_type.is_mutable(name): return self._fields[idx] else: - return self.get_field_immutable(name) + return self.get_field_immutable(idx) def set_field_by_idx(self, idx, val): affirm(isinstance(idx, r_uint), u"idx must be a r_uint") From 5401318d3724d2c46c8b80409b1de04ac7ee1b68 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 19 Mar 2015 06:05:19 -0600 Subject: [PATCH 612/909] add tests and finish up sliding buffers --- pixie/buffers.pxi | 3 +++ tests/pixie/tests/test-buffers.pxi | 11 +++++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/pixie/buffers.pxi b/pixie/buffers.pxi index 4813d624..cc3c4ce6 100644 --- a/pixie/buffers.pxi +++ b/pixie/buffers.pxi @@ -120,6 +120,9 @@ (-count [this] (count buf))) +(defn sliding-buffer [size] + (->SlidingBuffer (ring-buffer size))) + (extend -reduce IMutableBuffer (fn [buf f acc] diff --git a/tests/pixie/tests/test-buffers.pxi b/tests/pixie/tests/test-buffers.pxi index ea4abf4c..92a99c94 100644 --- a/tests/pixie/tests/test-buffers.pxi +++ b/tests/pixie/tests/test-buffers.pxi @@ -30,6 +30,13 @@ (deftest test-dropping-buffer (let [buf (dropping-buffer 4)] (dotimes [x 5] - (println (count buf) "x" x) (add! buf x)) - (assert= [0 1 2 3] (drain-buffer buf)))) + (assert= [0 1 2 3] (drain-buffer buf)) + (assert (not (full? buf))))) + +(deftest test-sliding-buffer + (let [buf (sliding-buffer 4)] + (dotimes [x 5] + (add! buf x)) + (assert= [1 2 3 4] (drain-buffer buf)) + (assert (not (full? buf))))) From 09dc2bff560afb967ff9c4d069fe507276f5220d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 19 Mar 2015 07:01:51 -0600 Subject: [PATCH 613/909] added the basics of channels --- pixie/buffers.pxi | 12 +++ pixie/channels.pxi | 121 ++++++++++++++++++++++++++++ pixie/stdlib.pxi | 4 + tests/pixie/tests/test-channels.pxi | 42 ++++++++++ 4 files changed, 179 insertions(+) create mode 100644 pixie/channels.pxi create mode 100644 tests/pixie/tests/test-channels.pxi diff --git a/pixie/buffers.pxi b/pixie/buffers.pxi index cc3c4ce6..5e6009db 100644 --- a/pixie/buffers.pxi +++ b/pixie/buffers.pxi @@ -123,6 +123,18 @@ (defn sliding-buffer [size] (->SlidingBuffer (ring-buffer size))) +(defn empty-buffer? [buf] + (= (count buf) 0)) + + +(deftype NullBuffer [] + IMutableBuffer + (full? [this] + true) + ICounted + (-count [this] 0)) + +(def null-buffer (->NullBuffer)) (extend -reduce IMutableBuffer (fn [buf f acc] diff --git a/pixie/channels.pxi b/pixie/channels.pxi new file mode 100644 index 00000000..65934678 --- /dev/null +++ b/pixie/channels.pxi @@ -0,0 +1,121 @@ +(ns pixie.channels + (require pixie.stacklets :as st) + (require pixie.buffers :as b)) + +(defprotocol ICancelable + (-canceled? [this] "Determines if a request (such as a callback) that can be canceled")) + +(defprotocol IReadPort + (-take! [this cfn] "Take a value from this port passing it to a cancellable function")) + +(defprotocol IWritePort + (-put! [this itm cfn] "Write a value to this port passing true if the write succeeds and the + callback isn't canceled")) + +(deftype OpCell [val cfn] + IIndexed + (-nth [this idx] + (cond + (= idx 0) val + (= idx 1) cfn + :else (throw "Index out of range"))) + (-nth-not-found [this idx not-found] + (cond + (= idx 0) val + (= idx 1) cfn + :else not-found)) + ICounted + (-count [this] + 2) + ICancelable + (-canceled? [this] + (canceled? cfn))) + +(defn canceled? [this] + (-canceled? this)) + + +(defn -move-puts-to-buffer [puts buffer] + (loop [] + (if (or (b/full? buffer) + (b/empty-buffer? puts)) + nil + (let [[val cfn] (b/remove! puts)] + (if (cancelled? cfn) + (recur) + (do (st/-run-later (partial cfn true)) + (b/add! buffer val) + (recur))))))) + +(defn -get-non-canceled! [buffer] + (loop [] + (if (b/empty-buffer? buffer) + nil + (let [v (b/remove! buffer)] + (if (canceled? v) + (recur) + v))))) + + +(deftype MultiReaderWriterChannel [puts takes buffer closed? ops-since-last-clean] + IReadPort + (-take! [this cfn] + (if (canceled? cfn) + false + (if closed? + (do (-run-later (partial cfn nil)) + false) + (if (not (b/empty-buffer? buffer)) + (do (st/-run-later (partial cfn (b/remove! buffer))) + (-move-puts-to-buffer puts buffer)) + + (if-let [[v pcfn] (-get-non-canceled! puts)] + (do (st/-run-later (partial pcfn true)) + (st/-run-later (partial cfn v)) + true) + (do (set-field! this :ops-since-last-clean (inc ops-since-last-clean)) + (b/add-unbounded! takes cfn) + true)))))) + (-put! [this val cfn] + (if (or (canceled? cfn)) + false + (if closed? + (do (-run-later (partial cfn false)) + false) + (if-let [tfn (-get-non-canceled! takes)] + (do (st/-run-later (partial tfn val)) + (st/-run-later (partial cfn true)) + true) + (if (not (b/full? buffer)) + (do (b/add! buffer val) + (st/-run-later (partial cfn true)) + true) + (do (b/add-unbounded! puts (->OpCell val cfn)) + (set-field! this :ops-since-last-clean (inc ops-since-last-clean)) + true))))))) + +(defn chan + ([] + (chan 0)) + ([size-or-buffer] + (if (= 0 size-or-buffer) + (->MultiReaderWriterChannel (b/ring-buffer 8) + (b/ring-buffer 8) + b/null-buffer + false + 0) + (if (integer? size-or-buffer) + (->MultiReaderWriterChannel (b/ring-buffer 8) + (b/ring-buffer 8) + (b/fixed-buffer size-or-buffer) + false + 0) + (->MultiReaderWriterChannel (b/ring-buffer 8) + (b/ring-buffer 8) + size-or-buffer + false + 0))))) + + +(extend -canceled? IFn + (fn [this] false)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4165876e..d306cc86 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2186,3 +2186,7 @@ Expands to calls to `extend-type`." (when (branch? node) (mapcat walk (children node))))))] (walk root))) + +(defn mapv + ([f col] + (transduce (map f) conj col))) diff --git a/tests/pixie/tests/test-channels.pxi b/tests/pixie/tests/test-channels.pxi new file mode 100644 index 00000000..f314af1c --- /dev/null +++ b/tests/pixie/tests/test-channels.pxi @@ -0,0 +1,42 @@ +(ns pixie.tests.test-channels + (require pixie.test :refer :all) + (require pixie.channels :refer :all) + (require pixie.async :refer :all)) + + +(deftest simple-read-and-write + (let [takep (promise) + putp (promise) + c (chan)] + (assert (-put! c 42 putp)) + (assert (-take! c takep)) + + (assert= @takep 42) + (assert= @putp true))) + +(deftest simple-take-before-put + (let [takep (promise) + putp (promise) + c (chan)] + (assert (-take! c takep)) + (assert (-put! c 42 putp)) + + (assert= @takep 42) + (assert= @putp true))) + +(deftest simple-take-before-put-with-buffers + (let [c (chan 2) + takesp (mapv (fn [_] (promise)) + (range 10)) + putsp (mapv (fn [_] (promise)) + (range 10))] + (doseq [p takesp] + (assert (-take! c p))) + (dotimes [x 10] + (assert (-put! c x (nth putsp x)))) + + (doseq [p putsp] + (assert= @p true)) + + (dotimes [x 10] + (assert= @(nth takesp x) x)))) From 5152bbf6a0fb21f77e5198e9515e503a84503dfa Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 19 Mar 2015 16:13:06 -0600 Subject: [PATCH 614/909] added go blocks cps operations and fixed closing of channels --- pixie/channels.pxi | 25 +++++++++++++++++---- pixie/csp.pxi | 35 +++++++++++++++++++++++++++++ tests/pixie/tests/test-channels.pxi | 30 +++++++++++++++++++++++++ tests/pixie/tests/test-csp.pxi | 15 +++++++++++++ 4 files changed, 101 insertions(+), 4 deletions(-) create mode 100644 pixie/csp.pxi create mode 100644 tests/pixie/tests/test-csp.pxi diff --git a/pixie/channels.pxi b/pixie/channels.pxi index 65934678..22324955 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -12,6 +12,10 @@ (-put! [this itm cfn] "Write a value to this port passing true if the write succeeds and the callback isn't canceled")) +(defprotocol ICloseable + (-close! [this] "Closes the channel, future writes will be rejected, future reads will + drain the channel before returning nil.")) + (deftype OpCell [val cfn] IIndexed (-nth [this idx] @@ -62,8 +66,10 @@ (-take! [this cfn] (if (canceled? cfn) false - (if closed? - (do (-run-later (partial cfn nil)) + (if (and closed? + (b/empty-buffer? buffer) + (b/empty-buffer? puts)) + (do (st/-run-later (partial cfn nil)) false) (if (not (b/empty-buffer? buffer)) (do (st/-run-later (partial cfn (b/remove! buffer))) @@ -76,11 +82,12 @@ (do (set-field! this :ops-since-last-clean (inc ops-since-last-clean)) (b/add-unbounded! takes cfn) true)))))) + IWritePort (-put! [this val cfn] (if (or (canceled? cfn)) false (if closed? - (do (-run-later (partial cfn false)) + (do (st/-run-later (partial cfn false)) false) (if-let [tfn (-get-non-canceled! takes)] (do (st/-run-later (partial tfn val)) @@ -92,9 +99,19 @@ true) (do (b/add-unbounded! puts (->OpCell val cfn)) (set-field! this :ops-since-last-clean (inc ops-since-last-clean)) - true))))))) + true)))))) + ICloseable + (-close! [this] + (set-field! this :closed? true) + (when (not (b/empty-buffer? takes)) + (loop [] + (when-let [tfn (-get-non-canceled! takes)] + (st/-run-later (partial tfn nil)) + (recur)))))) (defn chan + "Creates a CSP channel with the given buffer. If an integer is provided as the argument + creates a channel with a fixed buffer of that size. " ([] (chan 0)) ([size-or-buffer] diff --git a/pixie/csp.pxi b/pixie/csp.pxi new file mode 100644 index 00000000..c1e12152 --- /dev/null +++ b/pixie/csp.pxi @@ -0,0 +1,35 @@ +(ns pixie.csp + (require pixie.stacklets :as st) + (require pixie.buffers :as b) + (require pixie.channels :as chans)) + +(def chan chans/chan) + +(def -null-callback (fn [_] nil)) + +(defn put! + "Puts the value into the channel, calling the optional callback when the operation has + completed." + ([c v] + (chans/-put! c v -null-callback)) + ([c v f] + (chans/-put! c v f))) + +(defn take! + "Takes a value from a channel, calling the provided callback when completed" + ([c f] + (chans/-take! c f))) + +(defn >! [c v] + (st/call-cc (fn [k] + (chans/-put! c v (partial st/run-and-process k))))) + +(defn ! c) (range 10)))] + (assert= (map Date: Thu, 19 Mar 2015 16:24:02 -0600 Subject: [PATCH 615/909] not sure if this is a good idea yet, but I extended reduce to IReadPorts. --- pixie/csp.pxi | 20 +++++++++++++++++++- tests/pixie/tests/test-csp.pxi | 13 +++++++------ 2 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pixie/csp.pxi b/pixie/csp.pxi index c1e12152..8dfb4db0 100644 --- a/pixie/csp.pxi +++ b/pixie/csp.pxi @@ -5,6 +5,12 @@ (def chan chans/chan) +(defn close! + "Closes the channel, future writes will be rejected, future reads will + drain the channel before returning nil." + [c] + (chans/-close! c)) + (def -null-callback (fn [_] nil)) (defn put! @@ -31,5 +37,17 @@ (defmacro go [& body] `(let [ret-chan# (chans/chan 1)] - (st/spawn (put! ret-chan# (do ~@body))) + (st/spawn (put! ret-chan# (do ~@body)) + (close! ret-chan#)) ret-chan#)) + + +(extend -reduce chans/IReadPort + (fn [c f init] + (loop [acc init] + (if (reduced? acc) + @acc + (let [v (! c) (range 10)))] - (assert= (map ! c) (range 10)) + (close! c))] + (assert= (vec c) (range 10)) + (assert= ( Date: Thu, 19 Mar 2015 16:06:18 -0700 Subject: [PATCH 616/909] Added pixie/*.pxic to .gitignore --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 391f400b..e7206d68 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ pixie-vm .idea lib include +pixie/*.pxic From 548a1b2e7eab8cb56589dad45189fe06be7a38c5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 19 Mar 2015 22:07:26 -0600 Subject: [PATCH 617/909] added commit and cancel and basic alt handlers --- pixie/channels.pxi | 37 ++++++++++++++++++++++++----- tests/pixie/tests/test-channels.pxi | 17 ++++++++++++- 2 files changed, 47 insertions(+), 7 deletions(-) diff --git a/pixie/channels.pxi b/pixie/channels.pxi index 22324955..de275303 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -3,7 +3,8 @@ (require pixie.buffers :as b)) (defprotocol ICancelable - (-canceled? [this] "Determines if a request (such as a callback) that can be canceled")) + (-canceled? [this] "Determines if a request (such as a callback) that can be canceled") + (-commit! [this])) (defprotocol IReadPort (-take! [this cfn] "Take a value from this port passing it to a cancellable function")) @@ -69,14 +70,18 @@ (if (and closed? (b/empty-buffer? buffer) (b/empty-buffer? puts)) - (do (st/-run-later (partial cfn nil)) + (do (-commit! cfn) + (st/-run-later (partial cfn nil)) false) (if (not (b/empty-buffer? buffer)) - (do (st/-run-later (partial cfn (b/remove! buffer))) + (do (-commit! cfn) + (st/-run-later (partial cfn (b/remove! buffer))) (-move-puts-to-buffer puts buffer)) (if-let [[v pcfn] (-get-non-canceled! puts)] - (do (st/-run-later (partial pcfn true)) + (do (-commit! pcfn) + (-commit! cfn) + (st/-run-later (partial pcfn true)) (st/-run-later (partial cfn v)) true) (do (set-field! this :ops-since-last-clean (inc ops-since-last-clean)) @@ -87,14 +92,18 @@ (if (or (canceled? cfn)) false (if closed? - (do (st/-run-later (partial cfn false)) + (do (-commit! cfn) + (st/-run-later (partial cfn false)) false) (if-let [tfn (-get-non-canceled! takes)] - (do (st/-run-later (partial tfn val)) + (do (-commit! cfn) + (-commit! tfn) + (st/-run-later (partial tfn val)) (st/-run-later (partial cfn true)) true) (if (not (b/full? buffer)) (do (b/add! buffer val) + (-commit! cfn) (st/-run-later (partial cfn true)) true) (do (b/add-unbounded! puts (->OpCell val cfn)) @@ -106,6 +115,7 @@ (when (not (b/empty-buffer? takes)) (loop [] (when-let [tfn (-get-non-canceled! takes)] + (-commit! tfn) (st/-run-later (partial tfn nil)) (recur)))))) @@ -133,6 +143,21 @@ false 0))))) +(deftype AltHandler [atm f] + ICancelable + (-canceled? [this] + @atm) + (-commit! [this] + (reset! atm true)) + IFn + (-invoke [this & args] + (apply f args))) + +(defn alt-handlers [fns] + (mapv (partial ->AltHandler (atom false)) fns)) (extend -canceled? IFn (fn [this] false)) + +(extend -commit! IFn + (fn [this] nil)) diff --git a/tests/pixie/tests/test-channels.pxi b/tests/pixie/tests/test-channels.pxi index 8607a64b..9f35fe61 100644 --- a/tests/pixie/tests/test-channels.pxi +++ b/tests/pixie/tests/test-channels.pxi @@ -1,7 +1,8 @@ (ns pixie.tests.test-channels (require pixie.test :refer :all) (require pixie.channels :refer :all) - (require pixie.async :refer :all)) + (require pixie.async :refer :all) + (require pixie.stacklets :as st)) (deftest simple-read-and-write @@ -70,3 +71,17 @@ (-close! c) (assert= (mapv (partial -take! c) tps) [true true false]) (assert= (mapv deref tps) [0 1 nil]))) + +(deftest alt-handlers-only-invoke-one-callback + (let [completed (atom 0) + c (chan 5) + [f1 f2] (alt-handlers + [(fn [_] (swap! completed inc)) + (fn [_] (swap! completed inc))])] + (-take! c f1) + (-take! c f2) + (-put! c 1 (fn [_])) + (-put! c 2 (fn [_])) + (st/yield-control) + + (assert= @completed 1))) From 0a371b88c975d77b964e2acce8f3cde7adebde19 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 20 Mar 2015 15:26:23 -0600 Subject: [PATCH 618/909] adds ffi-voidp --- pixie/vm/libs/ffi.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index ead55775..1a30a535 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -196,6 +196,13 @@ def _ffi_fn__args(args): f = FFIFn(lib, rt.name(nm), new_args, ret_type, is_variadic) return f +@as_var("ffi-voidp") +def _ffi_voidp(lib, nm): + affirm(isinstance(lib, ExternalLib), u"First argument to ffi-voidp should be an external library") + name = rt.name(nm) + return VoidP(lib.get_fn_ptr(name)) + + From 398fc4a87603b0e71cf28755da8d382b84dca9d4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 20 Mar 2015 15:46:13 -0600 Subject: [PATCH 619/909] add defglobal to ffi-infer --- pixie/ffi-infer.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 822f55ae..b2db256b 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -62,6 +62,10 @@ [{:keys [name]}] (str "PixieChecker::DumpType<__typeof__(" name ")>();")) +(defmethod emit-infer-code :global + [_] + (str "std::cout <<\"[]\" << std::endl;")) + (defn start-string [] (str (apply str (map (fn [i] @@ -80,6 +84,7 @@ return 0; }") + ;; To Ctype conversion (defmulti edn-to-ctype (fn [n _] @@ -131,6 +136,10 @@ return 0; [{:keys [name]} {:keys [value type]}] `(def ~(symbol name) ~value)) +(defmethod generate-code :global + [{:keys [name]} _] + `(def ~(symbol name) (ffi-voidp *library* ~(str name)))) + (defmethod generate-code :function @@ -228,6 +237,10 @@ return 0; `(swap! *bodies* conj (assoc {:op :callback} :name ~(name nm)))) +(defmacro defglobal [nm] + `(swap! *bodies* conj (assoc {:op :global} + :name ~(name nm)))) + (defn compile-library [{:keys [prefix includes]} & source] (let [c-name (str "/tmp/" prefix ".c") From cc69dd0777b929147ccd01309d1c9227e54401c1 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 21 Mar 2015 16:05:28 -0600 Subject: [PATCH 620/909] implemented alts! I think it's time to merge --- pixie/channels.pxi | 20 ++++++++++++++++++++ pixie/csp.pxi | 9 +++++++++ tests/pixie/tests/test-csp.pxi | 18 ++++++++++++++++++ 3 files changed, 47 insertions(+) diff --git a/pixie/channels.pxi b/pixie/channels.pxi index de275303..f6a89418 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -161,3 +161,23 @@ (extend -commit! IFn (fn [this] nil)) + +(defn alts! [ops k options] + (let [handler-atom (atom false)] + (reduce + (fn [_ op] + (if (vector? op) + (let [[c val] op + f (fn [v] + (st/-run-later (partial k [c v])))] + (-put! c val (->AltHandler handler-atom f))) + (let [c op + f (fn [v] + (st/-run-later (partial k [c v])))] + (-take! c (->AltHandler handler-atom f))))) + nil + ops) + (when (and (:default options) + (not @handler-atom)) + (reset! handler-atom true) + (st/-run-later (partial k [:default (:default options)]))))) diff --git a/pixie/csp.pxi b/pixie/csp.pxi index 8dfb4db0..5cd9ff71 100644 --- a/pixie/csp.pxi +++ b/pixie/csp.pxi @@ -51,3 +51,12 @@ (if (nil? v) acc (recur (f acc v)))))))) + + +(defn alts! + ([ops] + (st/call-cc (fn [k] + (chans/alts! ops (partial st/run-and-process k) {})))) + ([ops & opts] + (st/call-cc (fn [k] + (chans/alts! ops (partial st/run-and-process k) (apply hashmap opts)))))) diff --git a/tests/pixie/tests/test-csp.pxi b/tests/pixie/tests/test-csp.pxi index 1fc49306..ae65a572 100644 --- a/tests/pixie/tests/test-csp.pxi +++ b/tests/pixie/tests/test-csp.pxi @@ -14,3 +14,21 @@ (close! c))] (assert= (vec c) (range 10)) (assert= (! c1 1) + (>! c2 2) + (println (hash-set [c1 1] [c2 2])) + (assert (not (= c1 c2))) + (assert= (! c 1) + (assert= (alts! [c] :default 42) [c 1]))) From 8652879c40b75565655fdcdca1c152c32390e1d2 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sat, 21 Mar 2015 20:24:17 +0000 Subject: [PATCH 621/909] Implement map without using iterators, fixes #242 The fact that map was implemented with iterators caused the following to happen: ``` user => (def sqr #(* % %)) user => (def sqrs (map sqr [1 2 3])) user => sqrs user => (vec sqrs) [1 4 9] user => (vec sqrs) [] ``` This commit implements map differently and adds tests to ensure that it behaves as intended. The implementation is pretty similar to the CLJS one, there are just a few things that aren't available (since `map` is defined earlier in the pixie stdlib than the CLJS core). --- pixie/stdlib.pxi | 40 ++++++++++++------------------- tests/pixie/tests/test-stdlib.pxi | 8 +++++++ 2 files changed, 23 insertions(+), 25 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 791f53d4..4e426ad4 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -144,32 +144,22 @@ ([result] (xf result)) ([result item] (xf result (f item)))))) ([f coll] - (let [i (iterator coll)] - (loop [] - (if (at-end? i) - nil - (do (yield (f (current i))) - (move-next! i) - (recur)))))) + (lazy-seq* + (fn [] + (let [s (seq coll)] + (if s + (cons (f (first s)) + (map f (rest s))) + nil))))) ([f & colls] - (let [its (vec (map iterator colls))] - (loop [] - (let [args (if (at-end? (first its)) - nil - (reduce - (fn [acc i] - (if (at-end? i) - (reduced nil) - (let [new-acc (conj acc (current i))] - (move-next! i) - new-acc))) - [] - its))] - - (if args - (do (yield (apply f args)) - (recur))))))))) - + (let [step (fn step [cs] + (lazy-seq* + (fn [] + (let [ss (map seq cs)] + (if (every? identity ss) + (cons (map first ss) (step (map rest ss))) + nil)))))] + (map (fn [args] (apply f args)) (step colls)))))) (def reduce (fn [rf init col] (-reduce col rf init))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index c0c0b711..0f01e5fa 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -6,6 +6,14 @@ (doseq [v vs] (t/assert= (identity v) v)))) +(t/deftest test-map + (t/assert= (map inc [1 2 3]) [2 3 4]) + (t/assert= (map + [1 2 3] [4 5 6]) [5 7 9]) + (t/assert= (map + [1 2 3] [4 5 6] [7 8 9]) [12 15 18]) + (let [value (map identity [1 2 3])] + (t/assert= (seq value) [1 2 3]) + (t/assert= (seq value) [1 2 3]))) + (t/deftest test-mapcat (t/assert= (mapcat identity []) []) (t/assert= (mapcat first [[[1 2]] [[3] [:not :present]] [[4 5 6]]]) [1 2 3 4 5 6])) From 6ae8a30ece855255994e7de4fb077ecb0636158f Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Sun, 22 Mar 2015 00:24:20 +0100 Subject: [PATCH 622/909] Make *1, *2 and *3 work. --- pixie/repl.pxi | 4 +++- pixie/stdlib.pxi | 5 +++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/pixie/repl.pxi b/pixie/repl.pxi index c9c8e9a4..38b94fde 100644 --- a/pixie/repl.pxi +++ b/pixie/repl.pxi @@ -21,7 +21,9 @@ (try (let [form (read rdr false)] (if (= form eof) (exit 0) - (println (eval form)))) + (let [x (eval form)] + (pixie.stdlib/-push-history x) + (println x)))) (catch ex (println "ERROR: \n" ex))) (recur)))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 791f53d4..c47f061f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2186,3 +2186,8 @@ Expands to calls to `extend-type`." (when (branch? node) (mapcat walk (children node))))))] (walk root))) + +(defn -push-history [x] + (def *3 *2) + (def *2 *1) + (def *1 x)) From 533921af91b2b86fabe800bca1dee1683db030ab Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Sun, 22 Mar 2015 00:26:11 +0100 Subject: [PATCH 623/909] Make *e work. --- pixie/repl.pxi | 3 ++- pixie/stdlib.pxi | 3 +++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pixie/repl.pxi b/pixie/repl.pxi index 38b94fde..06298e7a 100644 --- a/pixie/repl.pxi +++ b/pixie/repl.pxi @@ -25,5 +25,6 @@ (pixie.stdlib/-push-history x) (println x)))) (catch ex - (println "ERROR: \n" ex))) + (pixie.stdlib/-set-*e ex) + (println "ERROR: \n" ex))) (recur)))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index c47f061f..427aebd0 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2191,3 +2191,6 @@ Expands to calls to `extend-type`." (def *3 *2) (def *2 *1) (def *1 x)) + +(defn -set-*e [e] + (def *e e)) From dc3f22a996121396ff8a1224fa3a1c83e07f1519 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 21 Mar 2015 21:45:45 -0600 Subject: [PATCH 624/909] a bit of cleanup --- pixie/channels.pxi | 2 +- pixie/csp.pxi | 2 +- pixie/stdlib.pxi | 1 + tests/pixie/tests/test-csp.pxi | 1 - 4 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/channels.pxi b/pixie/channels.pxi index f6a89418..1b48ecf8 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -177,7 +177,7 @@ (-take! c (->AltHandler handler-atom f))))) nil ops) - (when (and (:default options) + (when (and (contains? options :default) (not @handler-atom)) (reset! handler-atom true) (st/-run-later (partial k [:default (:default options)]))))) diff --git a/pixie/csp.pxi b/pixie/csp.pxi index 5cd9ff71..b20fa08c 100644 --- a/pixie/csp.pxi +++ b/pixie/csp.pxi @@ -56,7 +56,7 @@ (defn alts! ([ops] (st/call-cc (fn [k] - (chans/alts! ops (partial st/run-and-process k) {})))) + (chans/alts! ops (partial st/run-and-process k) nil)))) ([ops & opts] (st/call-cc (fn [k] (chans/alts! ops (partial st/run-and-process k) (apply hashmap opts)))))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d306cc86..8bfa4690 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -292,6 +292,7 @@ (extend -with-meta Nil (fn [self _] nil)) (extend -at-end? Nil (fn [_] true)) (extend -deref Nil (fn [_] nil)) +(extend -contains-key Nil (fn [_ _] false)) (extend -hash Integer hash-int) diff --git a/tests/pixie/tests/test-csp.pxi b/tests/pixie/tests/test-csp.pxi index ae65a572..0b4cc396 100644 --- a/tests/pixie/tests/test-csp.pxi +++ b/tests/pixie/tests/test-csp.pxi @@ -23,7 +23,6 @@ (alts! [c1 c2])))] (>! c1 1) (>! c2 2) - (println (hash-set [c1 1] [c2 2])) (assert (not (= c1 c2))) (assert= ( Date: Sat, 21 Mar 2015 23:32:10 -0600 Subject: [PATCH 625/909] optimizations for lazy seqs, partial, and immutable_fields --- pixie/stdlib.pxi | 25 +++++++++++-------------- pixie/vm/array.py | 4 ++-- pixie/vm/code.py | 13 +++++++------ pixie/vm/custom_types.py | 4 ++-- pixie/vm/keyword.py | 2 +- pixie/vm/lazy_seq.py | 31 +++++++++++++++++++------------ pixie/vm/stdlib.py | 39 ++++++++++++++++++++++++++++++++++++++- pixie/vm/symbol.py | 2 +- 8 files changed, 81 insertions(+), 39 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 791f53d4..ad5426e9 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -241,16 +241,17 @@ (seq (next coll))) init))))) -(def indexed-reduce (fn indexed-reduce - [coll f init] - (let [max (count coll)] - (loop [init init - i 0] - (if (reduced? init) - @init - (if (-eq i max) - init - (recur (f init (nth coll i nil)) (+ i 1)))))))) +(def indexed-reduce + (fn indexed-reduce + [coll f init] + (let [max (count coll)] + (loop [init init + i 0] + (if (reduced? init) + @init + (if (-eq i max) + init + (recur (f init (nth coll i nil)) (+ i 1)))))))) (def rest (fn ^{:doc "Returns the elements after the first element, or () if there are no more elements." :signatures [[coll]] @@ -2156,10 +2157,6 @@ Expands to calls to `extend-type`." names) result#))) -(defn partial [f & args] - (fn [& args2] - (apply f (-> args vec (into args2))))) - (defn pst {:doc "Prints the trace of a Runtime Exception if given, or the last Runtime Exception in *e" :signatures [[] [e]] diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 265001f7..597ea5e7 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -13,7 +13,7 @@ class Array(object.Object): _type = object.Type(u"pixie.stdlib.Array") - __immutable_fields__ = ["_list[*]"] + _immutable_fields_ = ["_list[*]"] def type(self): return Array._type @@ -76,7 +76,7 @@ def _seq(self): class ArraySeq(object.Object): _type = object.Type(u"pixie.stdlib.ArraySeq") - __immutable_fields__ = ["_idx", "_w_array"] + _immutable_fields_ = ["_idx", "_w_array"] def __init__(self, idx, array): self._idx = idx diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 852b6f79..548cb48a 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -91,6 +91,7 @@ def slice_from_start(from_list, count, extra=r_uint(0)): class BaseCode(object.Object): + _immutable_fields_ = ["_meta"] def __init__(self): assert isinstance(self, BaseCode) self._name = u"unknown" @@ -213,7 +214,7 @@ def invoke_with(self, args, this_fn): class Code(BaseCode): """Interpreted code block. Contains consts and """ _type = object.Type(u"pixie.stdlib.Code") - __immutable_fields__ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta"] + _immutable_fields_ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta"] def type(self): return Code._type @@ -271,7 +272,7 @@ def get_base_code(self): class VariadicCode(BaseCode): - __immutable_fields__ = ["_required_arity", "_code", "_meta"] + _immutable_fields_ = ["_required_arity", "_code", "_meta"] _type = object.Type(u"pixie.stdlib.VariadicCode") def type(self): @@ -314,7 +315,7 @@ def invoke_with(self, args, self_fn): class Closure(BaseCode): _type = object.Type(u"pixie.stdlib.Closure") - __immutable_fields__ = ["_closed_overs[*]", "_code", "_meta"] + _immutable_fields_ = ["_closed_overs[*]", "_code", "_meta"] def type(self): return Closure._type @@ -632,7 +633,7 @@ def invoke(self, args): class Protocol(object.Object): _type = object.Type(u"pixie.stdlib.Protocol") - __immutable_fields__ = ["_rev?"] + _immutable_fields_ = ["_rev?"] def type(self): return Protocol._type @@ -664,7 +665,7 @@ class PolymorphicFn(BaseCode): def type(self): return PolymorphicFn._type - __immutable_fields__ = ["_rev?"] + _immutable_fields_ = ["_rev?"] def __init__(self, name, protocol): BaseCode.__init__(self) @@ -738,7 +739,7 @@ class DoublePolymorphicFn(BaseCode): def type(self): return DefaultProtocolFn._type - __immutable_fields__ = ["_rev?"] + _immutable_fields_ = ["_rev?"] def __init__(self, name, protocol): BaseCode.__init__(self) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 0a91145d..0bf431ae 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -6,7 +6,7 @@ import pixie.vm.rt as rt class CustomType(Type): - __immutable_fields__ = ["_slots", "_rev?"] + _immutable_fields_ = ["_slots", "_rev?"] def __init__(self, name, slots): Type.__init__(self, name) @@ -36,7 +36,7 @@ def get_num_slots(self): return len(self._slots) class CustomTypeInstance(Object): - __immutable_fields__ = ["_type"] + _immutable_fields_ = ["_type"] def __init__(self, type, fields): affirm(isinstance(type, CustomType), u"Can't create a instance of a non custom type") self._custom_type = type diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index e1c14568..58538418 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -10,7 +10,7 @@ class Keyword(Object): _type = Type(u"pixie.stdlib.Keyword") - __immutable_fields__ = ["_hash"] + _immutable_fields_ = ["_hash"] def __init__(self, name): self._str = name self._w_name = None diff --git a/pixie/vm/lazy_seq.py b/pixie/vm/lazy_seq.py index 7d052585..b95c3c94 100644 --- a/pixie/vm/lazy_seq.py +++ b/pixie/vm/lazy_seq.py @@ -3,6 +3,7 @@ import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var import pixie.vm.rt as rt +import rpython.rlib.jit as jit class LazySeq(object.Object): @@ -16,6 +17,7 @@ def __init__(self, fn, meta=nil): self._meta = meta self._s = nil + @jit.jit_callback("lazy_seq_sval") def sval(self): if self._fn is None: return self._s @@ -24,6 +26,21 @@ def sval(self): self._fn = None return self._s + @jit.dont_look_inside + def lazy_seq_seq(self): + self.sval() + if self._s is not nil: + ls = self._s + while True: + if isinstance(ls, LazySeq): + ls = ls.sval() + continue + else: + self._s = ls + return rt.seq(self._s) + else: + return nil + @extend(proto._first, LazySeq) @@ -41,18 +58,8 @@ def _next(self): @extend(proto._seq, LazySeq) def _seq(self): assert isinstance(self, LazySeq) - self.sval() - if self._s is not nil: - ls = self._s - while True: - if isinstance(ls, LazySeq): - ls = ls.sval() - continue - else: - self._s = ls - return rt.seq(self._s) - else: - return nil + return self.lazy_seq_seq() + @as_var("lazy-seq*") def lazy_seq(f): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 560578a2..4bc34c62 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -815,6 +815,42 @@ def _doc(self): return rt.wrap(self._doc) +class PartialFunction(code.NativeFn): + _immutable_fields_ = ["_partial_f", "_partial_args"] + def __init__(self, f, args): + self._partial_f = f + self._partial_args = args + + @jit.unroll_safe + def invoke(self, args): + new_args = [None] * (len(args) + len(self._partial_args)) + + for x in range(len(self._partial_args)): + new_args[x] = self._partial_args[x] + + plen = len(self._partial_args) + + for x in range(len(args)): + new_args[plen + x] = args[x] + + + return self._partial_f.invoke(new_args) + +@as_var("partial") +@jit.unroll_safe +def _partial__args(args): + """(partial f & args) + Creates a function that is a partial application of f. Thus ((partial + 1) 2) == 3""" + + f = args[0] + + new_args = [None] * (len(args) - 1) + + for x in range(len(new_args)): + new_args[x] = args[x + 1] + + return PartialFunction(f, new_args) + @as_var("-get-current-var-frames") def _get_current_var_frames(self): """(-get-current-var-frames) @@ -826,4 +862,5 @@ def _set_current_var_frames(self, frames): """(-set-current-var-frames frames) Sets the current var frames. Frames should be a cons list of hashmaps containing mappings of vars to dynamic values. Setting this value to anything but this data format will cause undefined errors.""" - code._dynamic_vars.set_current_frames(frames) \ No newline at end of file + code._dynamic_vars.set_current_frames(frames) + diff --git a/pixie/vm/symbol.py b/pixie/vm/symbol.py index 1fb9bbc3..6b0f2bfc 100644 --- a/pixie/vm/symbol.py +++ b/pixie/vm/symbol.py @@ -10,7 +10,7 @@ class Symbol(object.Object): _type = object.Type(u"pixie.stdlib.Symbol") - __immutable_fields__ = ["_hash"] + _immutable_fields_ = ["_hash"] def __init__(self, s, meta=nil): #assert isinstance(s, unicode) From 0a37ab3fe2e0396447e06f76dc264a05aa76ed6c Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sun, 22 Mar 2015 10:36:19 +0100 Subject: [PATCH 626/909] print stuff in the repl using `prn`, not `println` you'll want this for strings, for example. --- pixie/repl.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/repl.pxi b/pixie/repl.pxi index 06298e7a..d10b6a44 100644 --- a/pixie/repl.pxi +++ b/pixie/repl.pxi @@ -23,7 +23,7 @@ (exit 0) (let [x (eval form)] (pixie.stdlib/-push-history x) - (println x)))) + (prn x)))) (catch ex (pixie.stdlib/-set-*e ex) (println "ERROR: \n" ex))) From 9b8425aeb1e25462625dd99eb38e5de311981d66 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sun, 22 Mar 2015 11:29:01 +0000 Subject: [PATCH 627/909] Improve assert-throws? macro Since `assert-throws?` throws an exception inside the try regardless of if the body throws, if the body *doesn't* throw, you get a sort of strange error message: ``` RuntimeException: Assert failed Expected message: exception! but got Assert failed Expected a exception: exception! ``` I fixed this, improved the messages a bit, and also added some extra arities for just checking that an expression throws, and also just checking the class of the exception, instead of always the class and the message. --- pixie/test.pxi | 28 +++++++++++++++++++--------- 1 file changed, 19 insertions(+), 9 deletions(-) diff --git a/pixie/test.pxi b/pixie/test.pxi index cf77ccf2..26c3927b 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -61,15 +61,25 @@ yr# ~y] (assert (= xr# yr#) (str (show '~x xr#) " != " (show '~y yr#))))) -(defmacro assert-throws? [klass msg body] - `(try - ~body - (assert false (str "Expected a " ~klass " exception: " ~msg)) - (catch e# - (assert (= (type e#) ~klass) - (str "Expected exception of class " ~klass " but got " (type e#))) - (assert (= (ex-msg e#) ~msg) - (str "Expected message: " ~msg " but got " (ex-msg e#)))))) +(defmacro assert-throws? + ([body] + `(let [exn# (try (do ~body nil) (catch e# e#))] + (assert (not (nil? exn#)) + (str "Expected " (pr-str (quote ~body)) " to throw an exception")) + exn#)) + ([klass body] + `(let [exn# (assert-throws? ~body)] + (assert (= (type exn#) ~klass) + (str "Expected " (pr-str (quote ~body)) + " to throw exception of class " (pr-str ~klass) + " but got " (pr-str (type exn#)))) + exn#)) + ([klass msg body] + `(let [exn# (assert-throws? ~klass ~body)] + (assert (= (ex-msg exn#) ~msg) + (str "Expected " (pr-str (quote ~body)) + " to throw exception with message " (pr-str ~msg) + " but got " (pr-str (ex-msg exn#))))))) (defmacro assert [x] `(let [x# ~x] From 71d65fab4e63576b71d4cd4bf21576f3d980d2c0 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 22 Mar 2015 13:19:44 +0000 Subject: [PATCH 628/909] added set predicate also deleted unused import --- pixie/stdlib.pxi | 1 + pixie/vm/stdlib.py | 3 +-- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 791f53d4..9699d8c0 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -811,6 +811,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn keyword? [v] (instance? Keyword v)) (defn list? [v] (instance? PersistentList v)) +(defn set? [v] (instance? PersistentHashSet v)) (defn map? [v] (satisfies? IMap v)) (defn fn? [v] (satisfies? IFn v)) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 560578a2..f7e4fd98 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -420,7 +420,6 @@ def _load_file(filename, compile=False): import pixie.vm.reader as reader import pixie.vm.libs.pxic.writer as pxic_writer import os.path as path - from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR import os @@ -826,4 +825,4 @@ def _set_current_var_frames(self, frames): """(-set-current-var-frames frames) Sets the current var frames. Frames should be a cons list of hashmaps containing mappings of vars to dynamic values. Setting this value to anything but this data format will cause undefined errors.""" - code._dynamic_vars.set_current_frames(frames) \ No newline at end of file + code._dynamic_vars.set_current_frames(frames) From 0a76f8ef71b078bed08c9e0c248c6c63e90807b2 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 23 Mar 2015 16:52:04 -0600 Subject: [PATCH 629/909] fix segfault --- pixie/vm/stdlib.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index f2735f75..79bff639 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -817,6 +817,7 @@ def _doc(self): class PartialFunction(code.NativeFn): _immutable_fields_ = ["_partial_f", "_partial_args"] def __init__(self, f, args): + code.NativeFn.__init__(self) self._partial_f = f self._partial_args = args @@ -835,6 +836,7 @@ def invoke(self, args): return self._partial_f.invoke(new_args) + @as_var("partial") @jit.unroll_safe def _partial__args(args): From cfb7379dfe7df20ab7fe8f83e3743391b909411d Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 23 Mar 2015 21:22:56 -0600 Subject: [PATCH 630/909] fixed bad immutable flags --- pixie/vm/array.py | 2 +- pixie/vm/keyword.py | 1 - pixie/vm/symbol.py | 2 -- 3 files changed, 1 insertion(+), 4 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 597ea5e7..3bde8642 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -13,7 +13,7 @@ class Array(object.Object): _type = object.Type(u"pixie.stdlib.Array") - _immutable_fields_ = ["_list[*]"] + _immutable_fields_ = ["_list"] def type(self): return Array._type diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index 58538418..f6ce6dbc 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -10,7 +10,6 @@ class Keyword(Object): _type = Type(u"pixie.stdlib.Keyword") - _immutable_fields_ = ["_hash"] def __init__(self, name): self._str = name self._w_name = None diff --git a/pixie/vm/symbol.py b/pixie/vm/symbol.py index 6b0f2bfc..390805d0 100644 --- a/pixie/vm/symbol.py +++ b/pixie/vm/symbol.py @@ -10,8 +10,6 @@ class Symbol(object.Object): _type = object.Type(u"pixie.stdlib.Symbol") - _immutable_fields_ = ["_hash"] - def __init__(self, s, meta=nil): #assert isinstance(s, unicode) self._str = s From d2435370361ce5804bba2fa39ef185e5d9a85c21 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 23 Mar 2015 22:28:15 -0600 Subject: [PATCH 631/909] add specialization for ints --- pixie/vm/custom_types.py | 48 +++++++++++++++++++++++++++++++++++----- 1 file changed, 43 insertions(+), 5 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 0bf431ae..2e9776d7 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -2,6 +2,7 @@ import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import r_uint from pixie.vm.code import as_var +from pixie.vm.numbers import Integer from pixie.vm.keyword import Keyword import pixie.vm.rt as rt @@ -46,12 +47,15 @@ def type(self): return self._custom_type def set_field(self, name, val): - self._custom_type.set_mutable(name) idx = self._custom_type.get_slot_idx(name) if idx == -1: runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) - self._fields[idx] = val + old_val = self._fields[idx] + if isinstance(old_val, AbstractMutableCell): + old_val.set_mutable_cell_value(self._custom_type, self._fields, name, idx, val) + else: + self._fields[idx] = val return self @jit.elidable_promote() @@ -65,9 +69,14 @@ def get_field(self, name): runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) if self._custom_type.is_mutable(name): - return self._fields[idx] + value = self._fields[idx] + else: + value = self.get_field_immutable(idx) + + if isinstance(value, AbstractMutableCell): + return value.get_mutable_cell_value() else: - return self.get_field_immutable(idx) + return value def set_field_by_idx(self, idx, val): affirm(isinstance(idx, r_uint), u"idx must be a r_uint") @@ -98,7 +107,10 @@ def _new__args(args): affirm(cnt - 1 != tp.get_num_slots(), u"Wrong number of initializer fields to custom type") arr = [None] * cnt for x in range(cnt): - arr[x] = args[x + 1] + val = args[x + 1] + if isinstance(val, Integer): + val = IntegerMutableCell(val.int_val()) + arr[x] = val return CustomTypeInstance(tp, arr) @as_var("set-field!") @@ -114,3 +126,29 @@ def get_field(inst, field): affirm(isinstance(field, Keyword), u"Field must be a keyword") return inst.get_field(field) + + +class AbstractMutableCell(Object): + _type = Type(u"pixie.stdlib.AbstractMutableCell") + def type(self): + return self._type + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + pass + + def get_mutable_cell_value(self): + pass + +class IntegerMutableCell(AbstractMutableCell): + def __init__(self, int_val): + self._mutable_integer_val = int_val + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + if not isinstance(value, Integer): + ct.set_mutable(nm) + fields[idx] = value + else: + self._mutable_integer_val = value.int_val() + + def get_mutable_cell_value(self): + return rt.wrap(self._mutable_integer_val) \ No newline at end of file From 0f2f45d1fc0648d1e8bcc7a842958b6c87735516 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 24 Mar 2015 06:47:53 -0600 Subject: [PATCH 632/909] specialize deftypes for ints and floats --- pixie/vm/custom_types.py | 29 ++++++++++++++++++++++++++--- 1 file changed, 26 insertions(+), 3 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 2e9776d7..05008d2a 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -2,7 +2,7 @@ import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import r_uint from pixie.vm.code import as_var -from pixie.vm.numbers import Integer +from pixie.vm.numbers import Integer, Float from pixie.vm.keyword import Keyword import pixie.vm.rt as rt @@ -110,6 +110,9 @@ def _new__args(args): val = args[x + 1] if isinstance(val, Integer): val = IntegerMutableCell(val.int_val()) + elif isinstance(val, Float): + val = FloatMutableCell(val.float_val()) + arr[x] = val return CustomTypeInstance(tp, arr) @@ -146,9 +149,29 @@ def __init__(self, int_val): def set_mutable_cell_value(self, ct, fields, nm, idx, value): if not isinstance(value, Integer): ct.set_mutable(nm) - fields[idx] = value + if isinstance(value, Float): + fields[idx] = FloatMutableCell(value.float_val()) + else: + fields[idx] = value else: self._mutable_integer_val = value.int_val() def get_mutable_cell_value(self): - return rt.wrap(self._mutable_integer_val) \ No newline at end of file + return rt.wrap(self._mutable_integer_val) + +class FloatMutableCell(AbstractMutableCell): + def __init__(self, float_val): + self._mutable_float_val = float_val + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + if not isinstance(value, Float): + ct.set_mutable(nm) + if isinstance(value, Integer): + fields[idx] = IntegerMutableCell(value.int_val()) + else: + fields[idx] = value + else: + self._mutable_float_val = value.float_val() + + def get_mutable_cell_value(self): + return rt.wrap(self._mutable_float_val) \ No newline at end of file From 74c5e4b30b7bd7c7ee0c856bcf3278ceee003dd1 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 22:38:13 +0100 Subject: [PATCH 633/909] Add some useful constants to string.pxi --- pixie/string.pxi | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pixie/string.pxi b/pixie/string.pxi index 496d3363..8ab30b34 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -17,6 +17,17 @@ (def lower-case si/lower-case) (def upper-case si/upper-case) +; TODO: There should be locale-aware variants of these values +(def lower "abcdefghijklmnopqrstuvwxyz") +(def upper "ABCDEFGHIJKLMNOPQRSTUVWXYZ") +(def digits "0123456789") +(def punctuation "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") +(def whitespace (str \space \newline \tab \backspace \formfeed \return)) +(def letters (str lower upper)) +(def printable (str letters digits punctuation whitespace)) +(def hexdigits "0123456789abcdefABCDEF") +(def octdigits "012345678") + (defn replace "Replace all occurrences of x in s with r." [s x r] @@ -54,7 +65,7 @@ "True if s is nil, empty, or contains only whitespace." [s] (if s - (let [white #{\space \newline \tab \backspace \formfeed \return} + (let [white (set whitespace) length (count s)] (loop [index 0] (if (= length index) From 621020f4047937d8d8626f8c6eb9784acf98b761 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 24 Mar 2015 16:04:19 -0600 Subject: [PATCH 634/909] removed jit abort warnings, will re-enable later if needed --- target.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/target.py b/target.py index 5bd33e85..b153fafe 100644 --- a/target.py +++ b/target.py @@ -18,10 +18,12 @@ import rpython.rlib.rpath as rpath import rpython.rlib.rpath as rposix from rpython.rlib.objectmodel import we_are_translated +from rpython.jit.codewriter.policy import log class DebugIFace(JitHookInterface): def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations): - print "Aborted Trace, reason: ", Counters.counter_names[reason], logops, greenkey_repr + # print "Aborted Trace, reason: ", Counters.counter_names[reason], logops, greenkey_repr + pass import sys, pdb From 07b20d883f69f31dcb968b71ae9f8e5a53845c91 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 23:06:55 +0100 Subject: [PATCH 635/909] Let pixie.string/index-of return nil instead of -1 when the substring was not found. This allows more concise code because now you can use if-let, or, etc. instead of having to write an explicit check for -1 every time. --- pixie/string.pxi | 16 +++++++--------- tests/pixie/tests/test-strings.pxi | 8 ++++---- 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pixie/string.pxi b/pixie/string.pxi index 8ab30b34..2ca53b83 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -3,7 +3,7 @@ ; reexport native string functions (def substring si/substring) -(def index-of si/index-of) +(def index-of (comp #(if (not= -1 %) %) si/index-of)) (def split si/split) (def ends-with? si/ends-with) @@ -34,18 +34,16 @@ (let [offset (if (zero? (count x)) (+ 1 (count r)) (count r))] (loop [start 0 s s] - (let [i (index-of s x start)] - (if (neg? i) - s - (recur (+ i offset) (str (substring s 0 i) r (substring s (+ i (count x)))))))))) + (if-let [i (index-of s x start)] + (recur (+ i offset) (str (substring s 0 i) r (substring s (+ i (count x))))) + s)))) (defn replace-first "Replace the first occurrence of x in s with r." [s x r] - (let [i (index-of s x)] - (if (neg? i) - s - (str (substring s 0 i) r (substring s (+ i (count x))))))) + (if-let [i (index-of s x)] + (str (substring s 0 i) r (substring s (+ i (count x)))) + s)) (defn join {:doc "Join the elements of the collection using an optional separator" diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 248d8ca2..27b3f13b 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -31,15 +31,15 @@ (let [s "heyhohuh"] (t/assert= (s/index-of s "hey") 0) (t/assert= (s/index-of s "ho") 3) - (t/assert= (s/index-of s "foo") -1) + (t/assert= (s/index-of s "foo") nil) (t/assert= (s/index-of s "h" 2) 3) (t/assert= (s/index-of s "h" 4) 5) (t/assert= (s/index-of s "hey" 0) 0) - (t/assert= (s/index-of s "hey" 1) -1) + (t/assert= (s/index-of s "hey" 1) nil) - (t/assert= (s/index-of s "h" 0 0) -1) - (t/assert= (s/index-of s "h" 1 2) -1))) + (t/assert= (s/index-of s "h" 0 0) nil) + (t/assert= (s/index-of s "h" 1 2) nil))) (t/deftest test-substring (let [s "heyhohuh"] From 30aac1b6cc4cf40fd74a67c3dc3ead4b5767ccf6 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 23:13:58 +0100 Subject: [PATCH 636/909] Add a macro for string interpolation to pixie.string. --- pixie/string.pxi | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) diff --git a/pixie/string.pxi b/pixie/string.pxi index 2ca53b83..36be0b86 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -72,3 +72,32 @@ (recur (inc index)) false)))) true)) + +(defmacro interp + ; TODO: This might merit special read syntax + {:doc "String interpolation." + :examples [["(require pixie.string :refer [interp])"] + ["(interp \"2 plus 2 is $(+ 2 2)$!\")" nil "2 plus 2 is 4!"] + ["(let [x \"locals\"] (interp \"You can use arbitrary forms; for example $x$\"))" + nil "You can use arbitrary forms; for example locals"] + ["(interp \"$$$$ is the escape for a literal $$\")" + nil "$$ is the escape for a literal $"] + ]} + [txt] + (loop [forms [], txt txt] + (cond + (empty? txt) `(str ~@ forms) + (starts-with? txt "$") + (let [pos (or (index-of txt "$" 1) + (throw "Unmatched $ in interp argument!")) + form-str (subs txt 1 pos) + form (if (empty? form-str) "$" + (read-string form-str)) + rest-str (subs txt (inc pos))] + (recur (conj forms form) rest-str)) + :else + (let [pos (or (index-of txt "$") + (count txt)) + form (subs txt 0 pos) + rest-str (subs txt pos)] + (recur (conj forms form) rest-str))))) From 93007e16ffc029ea617fa861d2293e989f636feb Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 23:22:24 +0100 Subject: [PATCH 637/909] Implement constantly. --- pixie/stdlib.pxi | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e733bee2..bd30be04 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -861,6 +861,12 @@ If further arguments are passed, invokes the method named by symbol, passing the ([x y] (not (f x y))) ([x y & more] (not (apply f x y more)))))) +(defn constantly [x] + {:doc "Return a function that always returns x, no matter what it is called with." + :examples [["(let [f (constantly :me)] [(f 1) (f \"foo\") (f :abc) (f nil)])" + nil [:me :me :me :me]]]} + (fn [& _] x)) + (defn some {:doc "Checks if the predicate is true for any element of the collection. From 6a89ba01d3a4e6ad85989cee09fbc0ef771e5e9e Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 23:22:44 +0100 Subject: [PATCH 638/909] Implement repeatedly. --- pixie/stdlib.pxi | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index bd30be04..bf6a31fe 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1249,6 +1249,17 @@ and implements IAssociative, ILookup and IObject." ([n x] (take n (repeat x)))) +(defn repeatedly + {:doc "Returns a lazy seq that contains the return values of repeated calls to f. + + Yields an infinite seq with one argument. + With two arguments n specifies the number of elements." + :examples [["(into '(:batman!) (repeatedly 8 (fn [] :na)))" + nil (:na :na :na :na :na :na :na :na :batman!)]] + :signatures [[f] [n f]]} + ([f] (lazy-seq (cons (f) (repeatedly f)))) + ([n f] (take n (repeatedly f)))) + (defmacro doseq {:doc "Evaluates all elements of the seq, presumably for side effects. Returns nil." :added "0.1"} From 654829b6d0f7123831d5b8f79eadac8a0a63415d Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 23:25:53 +0100 Subject: [PATCH 639/909] Define the REPL history vars in stdlib.pxi instead of in target.py. --- pixie/stdlib.pxi | 4 ++++ target.py | 61 ------------------------------------------------ 2 files changed, 4 insertions(+), 61 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index bf6a31fe..72f11e10 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2197,10 +2197,14 @@ Expands to calls to `extend-type`." ([f col] (transduce (map f) conj col))) +(def *1) +(def *2) +(def *3) (defn -push-history [x] (def *3 *2) (def *2 *1) (def *1 x)) +(def *e) (defn -set-*e [e] (def *e e)) diff --git a/target.py b/target.py index 5bd33e85..9b908672 100644 --- a/target.py +++ b/target.py @@ -40,15 +40,6 @@ def jitpolicy(driver): LOAD_PATHS.set_root(nil) load_path = Var(u"pixie.stdlib", u"internal-load-path") -STAR_1 = intern_var(u"pixie.stdlib", u"*1") -STAR_1.set_root(nil) -STAR_2 = intern_var(u"pixie.stdlib", u"*2") -STAR_2.set_root(nil) -STAR_3 = intern_var(u"pixie.stdlib", u"*3") -STAR_3.set_root(nil) -STAR_E = intern_var(u"pixie.stdlib", u"*e") -STAR_E.set_root(nil) - class ReplFn(NativeFn): def __init__(self, args): self._argv = args @@ -62,58 +53,6 @@ def inner_invoke(self, args): with with_ns(u"user"): repl.invoke([]) - - - # def inner_invoke(self, args): - # from pixie.vm.keyword import keyword - # import pixie.vm.rt as rt - # from pixie.vm.string import String - # import pixie.vm.persistent_vector as vector - # - # print "Pixie 0.1 - Interactive REPL" - # print "(" + platform.name + ", " + platform.cc + ")" - # print ":exit-repl or Ctrl-D to quit" - # print "----------------------------" - # - # with with_ns(u"user"): - # NS_VAR.deref().include_stdlib() - # - # acc = vector.EMPTY - # for x in self._argv: - # acc = rt.conj(acc, rt.wrap(x)) - # - # PROGRAM_ARGUMENTS.set_root(acc) - # - # rdr = MetaDataReader(PromptReader()) - # with with_ns(u"user"): - # while True: - # try: - # val = read(rdr, False) - # if val is eof: - # break - # val = interpret(compile(val)) - # self.set_recent_vars(val) - # except WrappedException as ex: - # print "Error: ", ex._ex.__repr__() - # rdr.reset_line() - # self.set_error_var(ex._ex) - # continue - # if val is keyword(u"exit-repl"): - # break - # val = rt._repr(val) - # assert isinstance(val, String), "str should always return a string" - # print unicode_to_utf8(val._str) - - def set_recent_vars(self, val): - if rt.eq(val, STAR_1.deref()): - return - STAR_3.set_root(STAR_2.deref()) - STAR_2.set_root(STAR_1.deref()) - STAR_1.set_root(val) - - def set_error_var(self, ex): - STAR_E.set_root(ex) - class BatchModeFn(NativeFn): def __init__(self, args): self._file = args[0] From fe5eefb79bc0ad6ee4a661c1316c5b82277929d9 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 23:30:43 +0100 Subject: [PATCH 640/909] Implement map-invert. --- pixie/stdlib.pxi | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 72f11e10..d5af0017 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1049,6 +1049,15 @@ Creates new maps if the keys are not present." (assoc m k (apply update-inner-f (get m k) f ks))))] (apply update-inner-f m f ks))) +; FIXME: This causes compile_basics to fail on stacklets.pxi for some weird reason. +; (defn map-invert +; {:doc "Return a map where the vals are mapped to the keys." +; :examples [["(map-invert {:a :b, :c :d})" nil {:b :a, :d :c}]]} +; [m] +; (reduce (fn [m* ent] +; (assoc m* (val ent) (key ent))) +; {} m)) + (def subs pixie.string.internal/substring) (defmacro assert From 9977b6e02bd40b464e7215952d241b90dcda8afe Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 24 Mar 2015 23:50:23 +0100 Subject: [PATCH 641/909] Implement condp and case. --- pixie/stdlib.pxi | 18 +++++++++++++++++- 1 file changed, 17 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d5af0017..a8427e73 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -751,7 +751,6 @@ there's a value associated with the key. Use `some` for checking for values." ~then (cond ~@clauses)))) - (defmacro try [& body] (loop [catch nil catch-sym nil @@ -2141,6 +2140,23 @@ Expands to calls to `extend-type`." [~@body])))] `(or (seq ~(gen-loop [] bindings)) '()))) +;; TODO: docs, tests +;; TODO: implement :>> like in Clojure? +(defmacro condp [pred-form expr & clauses] + (let [x (gensym 'expr), pred (gensym 'pred)] + `(let [~x ~expr, ~pred ~pred-form] + (cond ~@(mapcat + (fn [[a b :as clause]] + (if (> (count clause) 1) + `((~pred ~a ~x) ~b) + `(:else ~a))) + (partition 2 clauses)))))) + +;; TODO: Not sure if I like the set idea... +(defmacro case [expr & args] + `(condp #(if (set? %1) (%1 %2) (= %1 %2)) + ~expr ~@args)) + (defmacro use [ns] `(do From a09fdb9650fbfed4a751a18fe425c6569ae68cb2 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 02:32:00 +0100 Subject: [PATCH 642/909] Add a comment that will hopefully prevent others from repeating my latest mistake. --- pixie/stdlib.pxi | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a8427e73..569678b9 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1967,6 +1967,9 @@ user => (refer 'pixie.string :exclude '(substring))" (pred (first coll)) (recur pred (next coll)) :else false)) +; If you want a fn that uses destructuring in its parameter list, place +; it after this definition. If you don't, you will get compile failures +; in unrelated files. (defmacro fn {:doc "Creates a function. From 4c551420181e5d1aa7c9c5a9a16c57f5cc6cd80e Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 00:01:59 +0100 Subject: [PATCH 643/909] Implement flatten. --- pixie/stdlib.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 569678b9..1421d1d8 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2221,6 +2221,19 @@ Expands to calls to `extend-type`." (mapcat walk (children node))))))] (walk root))) +(defn flatten [x] + ; TODO: laziness? + {:doc "Take any nested combination of ISeqable things, and return their contents + as a single, flat sequence. + + Calling this function on something that is not ISeqable returns a seq with that + value as its only element." + :examples [["(flatten [[1 2 [3 4] [5 6]] 7])" nil [1 2 3 4 5 6 7]]]} + (if (not (satisfies? ISeqable x)) [x] + (transduce (comp (map flatten) cat) + conj [] + (seq x)))) + (defn mapv ([f col] (transduce (map f) conj col))) From 8b59da425abeee964c51b6491ce1c75c74d61446 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 00:43:30 +0100 Subject: [PATCH 644/909] Implement partitionf. --- pixie/stdlib.pxi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1421d1d8..58ba6396 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1596,6 +1596,14 @@ not enough elements were present." (when-let [s (seq coll)] (cons (take n s) (partition n step (drop step s)))))) +;; TODO: docs, tests +(defn partitionf [f coll] + (when-let [s (seq coll)] + (lazy-seq + (let [n (f s)] + (cons (take n s) + (partitionf f (drop n coll))))))) + (defn destructure [binding expr] (cond (symbol? binding) [binding expr] From d4d7a05faabe236b71e84290fc96223cefc5d6f0 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 00:44:05 +0100 Subject: [PATCH 645/909] Make partition return a lazy seq. --- pixie/stdlib.pxi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 58ba6396..4ed561d2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1593,8 +1593,9 @@ not enough elements were present." :added "0.1"} ([n coll] (partition n n coll)) ([n step coll] - (when-let [s (seq coll)] - (cons (take n s) (partition n step (drop step s)))))) + (when-let [s (seq coll)] + (lazy-seq + (cons (take n s) (partition n step (drop step s))))))) ;; TODO: docs, tests (defn partitionf [f coll] From 665e78bbd843b15fa690d4262a5c64253ce0e751 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 02:01:34 +0100 Subject: [PATCH 646/909] Make it possible to pass a vector of types/protocols to instance? and satisfies?. --- pixie/vm/stdlib.py | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 79bff639..2e16f608 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -337,16 +337,36 @@ def apply__args(args): @returns(bool) @as_var("instance?") def _instance(c, o): - affirm(isinstance(c, Type), u"c must be a type") + from pixie.vm.persistent_vector import PersistentVector + if isinstance(c, Type): + return true if istypeinstance(o, c) else false + elif isinstance(c, PersistentVector): + for i in range(rt.count(c)): + t = rt.nth(c, rt.wrap(i)) + affirm(isinstance(t, Type), u"every element of c must be a type") + if istypeinstance(o, t): + return true + return false + else: + affirm(False, u"c must be a type or a vector") - return true if istypeinstance(o, c) else false @returns(bool) @as_var("satisfies?") def _satisfies(proto, o): - affirm(isinstance(proto, Protocol), u"proto must be a Protocol") - - return true if proto.satisfies(o.type()) else false + from pixie.vm.persistent_vector import PersistentVector + t = o.type() + if isinstance(proto, Protocol): + return true if proto.satisfies(t) else false + elif isinstance(proto, PersistentVector): + for i in range(rt.count(proto)): + p = rt.nth(proto, rt.wrap(i)) + affirm(isinstance(p, Protocol), u"every element of proto must be a Protocol") + if p.satisfies(t): + return true + return false + else: + affirm(False, u"c must be a Protocol or a vector") import pixie.vm.rt as rt From 80373e8ee5e7ad7240fb5cddf0ce34f801ab61e4 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 02:43:34 +0100 Subject: [PATCH 647/909] Let list? also return true when the argument is a Cons. This means that (list? '(1 2 3)) is now true. --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4ed561d2..40014735 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -801,7 +801,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn string? [v] (instance? String v)) (defn keyword? [v] (instance? Keyword v)) -(defn list? [v] (instance? PersistentList v)) +(defn list? [v] (instance? [PersistentList Cons] v)) (defn set? [v] (instance? PersistentHashSet v)) (defn map? [v] (satisfies? IMap v)) (defn fn? [v] (satisfies? IFn v)) From 2195850eb331e667500ed8066ed2bf80a8cdd750 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 02:45:39 +0100 Subject: [PATCH 648/909] Implement juxt. --- pixie/stdlib.pxi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 40014735..5d5cc79d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2243,6 +2243,11 @@ Expands to calls to `extend-type`." conj [] (seq x)))) +; TODO: tests, docs +(defn juxt [& fns] + (fn [& args] + (mapv #(apply % args) fns))) + (defn mapv ([f col] (transduce (map f) conj col))) From 3b779d9394bc6000e46c939012b9784fd900749e Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 02:46:11 +0100 Subject: [PATCH 649/909] Add a colon to the error message of assert. This makes it clear where the user-supplied message begins. (I had one occasion where I misinterpreted the message, until I had looked at the code.) --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 5d5cc79d..f6ae715d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1067,7 +1067,7 @@ Creates new maps if the keys are not present." ([test msg] `(if ~test nil - (throw (str "Assert failed " ~msg))))) + (throw (str "Assert failed: " ~msg))))) (defmacro resolve {:doc "Resolve the var associated with the symbol in the current namespace." From b3b551869490dddcfb17116aaf8a19c36eb996a8 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:06:44 +0100 Subject: [PATCH 650/909] Have some return the first true value of the predicate instead of just true. --- pixie/stdlib.pxi | 12 +++++------- tests/pixie/tests/test-stdlib.pxi | 5 ++++- 2 files changed, 9 insertions(+), 8 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index f6ae715d..e72f554b 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -867,16 +867,14 @@ If further arguments are passed, invokes the method named by symbol, passing the (fn [& _] x)) (defn some - {:doc "Checks if the predicate is true for any element of the collection. - -Stops if it finds such an element." + {:doc "Returns the first true value of the predicate for the elements of the collection." :signatures [[pred coll]] :added "0.1"} [pred coll] - (cond - (nil? (seq coll)) false - (pred (first coll)) true - :else (recur pred (next coll)))) + (if-let [coll (seq coll)] + (or (pred (first coll)) + (recur pred (next coll))) + false)) (extend -count MapEntry (fn [self] 2)) (extend -nth MapEntry (fn map-entry-nth [self idx] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 0f01e5fa..8a411d97 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -277,7 +277,10 @@ (t/assert= (some even? [2 3 6 8]) true) (t/assert= (some even? [1 3 5 8]) true) (t/assert= (some even? []) false) - (t/assert= (some odd? [2]) false)) + (t/assert= (some odd? [2]) false) + (t/assert= (some #{:x :y} [:a :b :x :y]) :x) + (t/assert= (some #{:x :y} [:a :b :c :y]) :y) + (t/assert= (some #{:x :y} [:a :b :c :d]) false)) (t/deftest test-filter (t/assert= (vec (filter (fn [x] true) [])) []) From e0efdc7654c35d19e16f5775c78bc9c3ca867d84 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:15:04 +0100 Subject: [PATCH 651/909] Uncomment map-invert. --- pixie/stdlib.pxi | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e72f554b..48172688 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1046,15 +1046,6 @@ Creates new maps if the keys are not present." (assoc m k (apply update-inner-f (get m k) f ks))))] (apply update-inner-f m f ks))) -; FIXME: This causes compile_basics to fail on stacklets.pxi for some weird reason. -; (defn map-invert -; {:doc "Return a map where the vals are mapped to the keys." -; :examples [["(map-invert {:a :b, :c :d})" nil {:b :a, :d :c}]]} -; [m] -; (reduce (fn [m* ent] -; (assoc m* (val ent) (key ent))) -; {} m)) - (def subs pixie.string.internal/substring) (defmacro assert @@ -2246,6 +2237,14 @@ Expands to calls to `extend-type`." (fn [& args] (mapv #(apply % args) fns))) +(defn map-invert + {:doc "Return a map where the vals are mapped to the keys." + :examples [["(map-invert {:a :b, :c :d})" nil {:b :a, :d :c}]]} + [m] + (reduce (fn [m* ent] + (assoc m* (val ent) (key ent))) + {} m)) + (defn mapv ([f col] (transduce (map f) conj col))) From c44680e963b193245a5fb0945e481d7c178f2633 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:20:09 +0100 Subject: [PATCH 652/909] Add a docstring for partitionf. --- pixie/stdlib.pxi | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 48172688..b7a77981 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1586,8 +1586,12 @@ not enough elements were present." (lazy-seq (cons (take n s) (partition n step (drop step s))))))) -;; TODO: docs, tests (defn partitionf [f coll] + {:doc "A generalized version of partition. Instead of taking a constant number of elements, + this function calls f with the remaining collection to determine how many elements to + take." + :examples [["(fpartition first [1 :a, 2 :a b, 3 :a :b :c])" + nil ((1 :a) (2 :a :b) (3 :a :b :c))]]} (when-let [s (seq coll)] (lazy-seq (let [n (f s)] From 6c0ee5a975d555586380962aec1c3dfc407b45e0 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:21:57 +0100 Subject: [PATCH 653/909] Add a docstring for juxt. --- pixie/stdlib.pxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index b7a77981..a9746f47 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2236,8 +2236,10 @@ Expands to calls to `extend-type`." conj [] (seq x)))) -; TODO: tests, docs (defn juxt [& fns] + {:doc "Returns a function that applies all fns to its arguments, + and returns a vector of the results." + :examples [["((juxt + - *) 2 3)" nil [5 -1 6]]]} (fn [& args] (mapv #(apply % args) fns))) From 64411dc9575089eed8558d238ed90d30dc2e2861 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:25:17 +0100 Subject: [PATCH 654/909] Implement reverse. --- pixie/stdlib.pxi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a9746f47..c6f56a9f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2145,6 +2145,13 @@ Expands to calls to `extend-type`." [~@body])))] `(or (seq ~(gen-loop [] bindings)) '()))) +(defn reverse + ; TODO: We should probably have a protocol IReversible, so we can e.g. + ; reverse vectors efficiently, etc.. + [coll] + "Returns a collection that contains all the elements of the argument in reverse order." + (into () coll)) + ;; TODO: docs, tests ;; TODO: implement :>> like in Clojure? (defmacro condp [pred-form expr & clauses] From 3992bf3029b180e3e9083b3eff313df23dcf6a87 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:33:34 +0100 Subject: [PATCH 655/909] Docs and tests for condp --- pixie/stdlib.pxi | 10 ++++++++-- tests/pixie/tests/test-stdlib.pxi | 11 +++++++++++ 2 files changed, 19 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index c6f56a9f..112e25f8 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2152,9 +2152,15 @@ Expands to calls to `extend-type`." "Returns a collection that contains all the elements of the argument in reverse order." (into () coll)) -;; TODO: docs, tests ;; TODO: implement :>> like in Clojure? -(defmacro condp [pred-form expr & clauses] +(defmacro condp + "Takes a binary predicate, an expression and a number of two-form clauses. + Calls the predicate on the first value of each clause and the expression. + If the result is truthy returns the second value of the clause. + + If the number of arguments is odd and no clause matches, the last argument is returned. + If the number of arguments is even and no clause matches, nil is returned." + [pred-form expr & clauses] (let [x (gensym 'expr), pred (gensym 'pred)] `(let [~x ~expr, ~pred ~pred-form] (cond ~@(mapcat diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 8a411d97..f7610298 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -419,3 +419,14 @@ (t/deftest test-frequencies (t/assert= (frequencies [1 2 3 4 3 2 1]) {1 2, 2 2, 3 2, 4 1})) + +(t/deftest test-condp + (t/assert= (condp :dont-call-me :dont-use-me) nil) + (let [f (fn [x] + (condp = x + 0 :one + 1 :two + :whatever))] + (t/assert= (f 0) :one) + (t/assert= (f 1) :two) + (t/assert= (f 9) :whatever))) From 57577179697621177ae008de2ae1004901de7db8 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:38:56 +0100 Subject: [PATCH 656/909] Docs and tests for case --- pixie/stdlib.pxi | 13 +++++++++++-- tests/pixie/tests/test-stdlib.pxi | 14 ++++++++++++++ 2 files changed, 25 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 112e25f8..be39e096 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2170,8 +2170,17 @@ Expands to calls to `extend-type`." `(:else ~a))) (partition 2 clauses)))))) -;; TODO: Not sure if I like the set idea... -(defmacro case [expr & args] +(defmacro case + "Takes an expression and a number of two-form clauses. + Checks for each clause if the first part is equal to the expression. + If yes, returns the value of the second part. + + The first part of each clause can also be a set. If that is the case, the clause + matches when the result of the expression is in the set. + + If the number of arguments is odd and no clause matches, the last argument is returned. + If the number of arguments is even and no clause matches, nil is returned." + [expr & args] `(condp #(if (set? %1) (%1 %2) (= %1 %2)) ~expr ~@args)) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index f7610298..2a99da4c 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -430,3 +430,17 @@ (t/assert= (f 0) :one) (t/assert= (f 1) :two) (t/assert= (f 9) :whatever))) + +(t/deftest test-case + (t/assert= (case :unused) nil) + (let [f (fn [x] + (case x + 1 :one + 2 :two + #{3 4} :large + :toolarge))] + (t/assert= (f 1) :one) + (t/assert= (f 2) :two) + (t/assert= (f 3) :large) + (t/assert= (f 4) :large) + (t/assert= (f 9) :toolarge))) From d644aca8c2f217b0dc7b481ed20edca49e73b90d Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 03:39:42 +0100 Subject: [PATCH 657/909] Fix my way of defining the natural numbers being slightly inconsistent with the rest of the universe. --- tests/pixie/tests/test-stdlib.pxi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 2a99da4c..75e71785 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -424,11 +424,11 @@ (t/assert= (condp :dont-call-me :dont-use-me) nil) (let [f (fn [x] (condp = x - 0 :one - 1 :two + 1 :one + 2 :two :whatever))] - (t/assert= (f 0) :one) - (t/assert= (f 1) :two) + (t/assert= (f 1) :one) + (t/assert= (f 2) :two) (t/assert= (f 9) :whatever))) (t/deftest test-case From cd537bed418c01e830c2f86698c0a3ba64047c5c Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 04:51:15 +0100 Subject: [PATCH 658/909] Implement split-at. --- pixie/stdlib.pxi | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index be39e096..53171006 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1490,6 +1490,13 @@ The new value is thus `(apply f current-value-of-atom args)`." (recur (dec n) (next s)) s))) +(defn split-at + {:doc "Returns a vector of the first n elements of the collection, and the remaining elements." + :examples [["(split-at 2 [:a :b :c :d :e])" nil + [(:a :b) (:c :d :e)]]]} + [n coll] + [(take n coll) (drop n coll)]) + (defmacro while {:doc "Repeatedly executes body while test expression is true. Presumes some side-effect will cause test to become false/nil. Returns nil" From f4974404cd0c98afbd718f553c35897ac2146d7d Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 04:54:23 +0100 Subject: [PATCH 659/909] More consistent docstrings --- pixie/stdlib.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 53171006..a313b34c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2254,7 +2254,7 @@ Expands to calls to `extend-type`." (defn flatten [x] ; TODO: laziness? - {:doc "Take any nested combination of ISeqable things, and return their contents + {:doc "Takes any nested combination of ISeqable things, and return their contents as a single, flat sequence. Calling this function on something that is not ISeqable returns a seq with that @@ -2273,7 +2273,7 @@ Expands to calls to `extend-type`." (mapv #(apply % args) fns))) (defn map-invert - {:doc "Return a map where the vals are mapped to the keys." + {:doc "Returns a map where the vals are mapped to the keys." :examples [["(map-invert {:a :b, :c :d})" nil {:b :a, :d :c}]]} [m] (reduce (fn [m* ent] From 0c1ae9139a16fe0eaf7f71094ad7cc9e7b1e21a0 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 04:54:49 +0100 Subject: [PATCH 660/909] Add another example for flatten. --- pixie/stdlib.pxi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a313b34c..c2df95cc 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2259,7 +2259,8 @@ Expands to calls to `extend-type`." Calling this function on something that is not ISeqable returns a seq with that value as its only element." - :examples [["(flatten [[1 2 [3 4] [5 6]] 7])" nil [1 2 3 4 5 6 7]]]} + :examples [["(flatten [[1 2 [3 4] [5 6]] 7])" nil [1 2 3 4 5 6 7]] + ["(flatten :this)" nil [:this]]]} (if (not (satisfies? ISeqable x)) [x] (transduce (comp (map flatten) cat) conj [] From 30f4f2cdab8f42cf6d74fc1991baa381428f5a53 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 25 Mar 2015 11:07:01 +0000 Subject: [PATCH 661/909] fix buffer-size-check --- pixie/io-blocking.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index 326d5aca..5dd57346 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -18,7 +18,7 @@ (deftype FileStream [fp] IInputStream (read [this buffer len] - (assert (<= (buffer-capacity buffer) len) + (assert (>= (buffer-capacity buffer) len) "Not enough capacity in the buffer") (let [read-count (fread buffer 1 len fp)] (set-buffer-count! buffer read-count) From 406a624891b73f374a2d44a547b0f87f2a5b841d Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 12:49:09 +0100 Subject: [PATCH 662/909] Let case and condp throw RuntimeExceptions when no clause matches. --- pixie/stdlib.pxi | 10 +++++----- tests/pixie/tests/test-stdlib.pxi | 8 ++++++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index c2df95cc..e2d6e742 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2166,7 +2166,7 @@ Expands to calls to `extend-type`." If the result is truthy returns the second value of the clause. If the number of arguments is odd and no clause matches, the last argument is returned. - If the number of arguments is even and no clause matches, nil is returned." + If the number of arguments is even and no clause matches, throws an exception." [pred-form expr & clauses] (let [x (gensym 'expr), pred (gensym 'pred)] `(let [~x ~expr, ~pred ~pred-form] @@ -2175,18 +2175,18 @@ Expands to calls to `extend-type`." (if (> (count clause) 1) `((~pred ~a ~x) ~b) `(:else ~a))) - (partition 2 clauses)))))) + (partition 2 clauses)) + :else (throw "No matching clause!"))))) (defmacro case "Takes an expression and a number of two-form clauses. Checks for each clause if the first part is equal to the expression. If yes, returns the value of the second part. - The first part of each clause can also be a set. If that is the case, the clause - matches when the result of the expression is in the set. + The first part of each clause can also be a set. If that is the case, the clause matches when the result of the expression is in the set. If the number of arguments is odd and no clause matches, the last argument is returned. - If the number of arguments is even and no clause matches, nil is returned." + If the number of arguments is even and no clause matches, throws an exception." [expr & args] `(condp #(if (set? %1) (%1 %2) (= %1 %2)) ~expr ~@args)) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 75e71785..3f23d9db 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -421,7 +421,9 @@ {1 2, 2 2, 3 2, 4 1})) (t/deftest test-condp - (t/assert= (condp :dont-call-me :dont-use-me) nil) + (t/assert-throws? RuntimeException + "No matching clause!" + (condp :dont-call-me :dont-use-me)) (let [f (fn [x] (condp = x 1 :one @@ -432,7 +434,9 @@ (t/assert= (f 9) :whatever))) (t/deftest test-case - (t/assert= (case :unused) nil) + (t/assert-throws? RuntimeException + "No matching clause!" + (case :no-matter-what)) (let [f (fn [x] (case x 1 :one From 3fdbcad059b9efa4841483924d1d2ed31b8cab31 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 12:50:33 +0100 Subject: [PATCH 663/909] Fix example for partitionf. --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e2d6e742..8a99daac 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1597,7 +1597,7 @@ not enough elements were present." {:doc "A generalized version of partition. Instead of taking a constant number of elements, this function calls f with the remaining collection to determine how many elements to take." - :examples [["(fpartition first [1 :a, 2 :a b, 3 :a :b :c])" + :examples [["(partitionf first [1 :a, 2 :a b, 3 :a :b :c])" nil ((1 :a) (2 :a :b) (3 :a :b :c))]]} (when-let [s (seq coll)] (lazy-seq From 4ba33b2bdccb1ce3877e972bf036f3a33be84cec Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 12:56:02 +0100 Subject: [PATCH 664/909] Revert the changes to instance? and satisfies?. This reverts commit 665e78bbd843b15fa690d4262a5c64253ce0e751. I will reimplement this in stdlib.pxi. --- pixie/vm/stdlib.py | 30 +++++------------------------- 1 file changed, 5 insertions(+), 25 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 2e16f608..79bff639 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -337,36 +337,16 @@ def apply__args(args): @returns(bool) @as_var("instance?") def _instance(c, o): - from pixie.vm.persistent_vector import PersistentVector - if isinstance(c, Type): - return true if istypeinstance(o, c) else false - elif isinstance(c, PersistentVector): - for i in range(rt.count(c)): - t = rt.nth(c, rt.wrap(i)) - affirm(isinstance(t, Type), u"every element of c must be a type") - if istypeinstance(o, t): - return true - return false - else: - affirm(False, u"c must be a type or a vector") + affirm(isinstance(c, Type), u"c must be a type") + return true if istypeinstance(o, c) else false @returns(bool) @as_var("satisfies?") def _satisfies(proto, o): - from pixie.vm.persistent_vector import PersistentVector - t = o.type() - if isinstance(proto, Protocol): - return true if proto.satisfies(t) else false - elif isinstance(proto, PersistentVector): - for i in range(rt.count(proto)): - p = rt.nth(proto, rt.wrap(i)) - affirm(isinstance(p, Protocol), u"every element of proto must be a Protocol") - if p.satisfies(t): - return true - return false - else: - affirm(False, u"c must be a Protocol or a vector") + affirm(isinstance(proto, Protocol), u"proto must be a Protocol") + + return true if proto.satisfies(o.type()) else false import pixie.vm.rt as rt From b233b136bb929127b2ac9a53125bb2cca2f4c984 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 14:08:55 +0100 Subject: [PATCH 665/909] Let satisfies? and instance? both take seqables as their first argument. This time this is implemented in pixie code. --- pixie/stdlib.pxi | 24 ++++++++++++++++++++++++ pixie/vm/compiler.py | 6 +++--- pixie/vm/libs/pxic/writer.py | 6 +++--- pixie/vm/persistent_vector.py | 2 +- pixie/vm/reader.py | 6 +++--- pixie/vm/stdlib.py | 20 ++++++++++---------- 6 files changed, 44 insertions(+), 20 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8a99daac..1853fe83 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -164,6 +164,30 @@ (def reduce (fn [rf init col] (-reduce col rf init))) +(def instance? (fn ^{:doc "Checks if x is an instance of t. + + When t is seqable, checks if x is an instance of + any of the types contained therein."} + instance? [t x] + (if (-satisfies? ISeqable t) + (let [ts (seq t)] + (if (not ts) false + (or (-instance? (first ts) x) + (instance? (rest ts) x)))) + (-instance? t x)))) + +(def satisfies? (fn ^{:doc "Checks if x satisfies the protocol p. + + When p is seqable, checks if x satisfies all of + the protocols contained therein."} + satisfies? [p x] + (if (-satisfies? ISeqable p) + (let [ps (seq p)] + (if (not ps) true + (and (-satisfies? (first ps) x) + (satisfies? (rest ps) x)))) + (-satisfies? p x)))) + (def into (fn ^{:doc "Add the elements of `from` to the collection `to`." :signatures [[to from]] :added "0.1"} diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index b17b3b4f..c915214a 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -353,7 +353,7 @@ def compile_form(form, ctx): ctx.push_const(nil) return - if rt.satisfies_QMARK_(rt.ISeq.deref(), form) and form is not nil: + if rt._satisfies_QMARK_(rt.ISeq.deref(), form) and form is not nil: form = macroexpand(form) return compile_cons(form, ctx) if isinstance(form, numbers.Integer): @@ -425,7 +425,7 @@ def compile_form(form, ctx): compile_set_literal(form, ctx) return - if rt.satisfies_QMARK_(rt.IMap.deref(), form): + if rt._satisfies_QMARK_(rt.IMap.deref(), form): compile_map_literal(form, ctx) return @@ -480,7 +480,7 @@ def compile_fn(form, ctx): - if rt.satisfies_QMARK_(rt.ISeq.deref(), rt.first(form)): + if rt._satisfies_QMARK_(rt.ISeq.deref(), rt.first(form)): arities = [] while form is not nil: required_arity, argc = compile_fn_body(name, rt.first(rt.first(form)), rt.next(rt.first(form)), ctx) diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index 9b3e806d..ecd8865d 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -255,11 +255,11 @@ def write_object(obj, wtr): elif isinstance(obj, Var): #wtr.write_cached_obj(obj, write_var) write_var(obj, wtr) - elif rt.satisfies_QMARK_(rt.IMap.deref(), obj): + elif rt._satisfies_QMARK_(rt.IMap.deref(), obj): write_map(obj, wtr) - elif rt.satisfies_QMARK_(rt.IVector.deref(), obj): + elif rt._satisfies_QMARK_(rt.IVector.deref(), obj): write_vector(obj, wtr) - elif rt.satisfies_QMARK_(rt.ISeq.deref(), obj): + elif rt._satisfies_QMARK_(rt.ISeq.deref(), obj): write_seq(obj, wtr) elif isinstance(obj, Keyword): wtr.write_cached_obj(obj, write_keyword) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 26517cd4..65aaacfb 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -456,7 +456,7 @@ def _eq(self, obj): return false return true else: - if obj is nil or not rt.satisfies_QMARK_(proto.ISeqable, obj): + if obj is nil or not rt._satisfies_QMARK_(proto.ISeqable, obj): return false seq = rt.seq(obj) for i in range(0, intmask(self._cnt)): diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 4004157d..0e150600 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -449,12 +449,12 @@ def invoke(self, rdr, ch): LIST = symbol(u"list") def is_unquote(form): - return True if rt.satisfies_QMARK_(rt.ISeq.deref(), form) \ + return True if rt._satisfies_QMARK_(rt.ISeq.deref(), form) \ and rt.eq(rt.first(form), UNQUOTE) \ else False def is_unquote_splicing(form): - return True if rt.satisfies_QMARK_(rt.ISeq.deref(), form) \ + return True if rt._satisfies_QMARK_(rt.ISeq.deref(), form) \ and rt.eq(rt.first(form), UNQUOTE_SPLICING) \ else False @@ -537,7 +537,7 @@ def invoke(self, rdr, ch): if isinstance(meta, Symbol): meta = rt.hashmap(keyword(u"tag"), meta) - if rt.satisfies_QMARK_(rt.IMeta.deref(), obj): + if rt._satisfies_QMARK_(rt.IMeta.deref(), obj): return rt.with_meta(obj, rt.merge(meta, rt.meta(obj))) return obj diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 79bff639..6741063b 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -121,7 +121,7 @@ def default_hash(x): @as_var("first") def first(x): - if rt.satisfies_QMARK_(ISeq, x): + if rt._satisfies_QMARK_(ISeq, x): return rt._first(x) seq = rt.seq(x) @@ -131,7 +131,7 @@ def first(x): @as_var("next") def next(x): - if rt.satisfies_QMARK_(ISeq, x): + if rt._satisfies_QMARK_(ISeq, x): return rt.seq(rt._next(x)) seq = rt.seq(x) if seq is nil: @@ -145,7 +145,7 @@ def seq(x): @as_var("seq?") def seq_QMARK_(x): - return true if rt.satisfies_QMARK_(rt.ISeq.deref(), x) else false + return true if rt._satisfies_QMARK_(rt.ISeq.deref(), x) else false @as_var("-seq-eq") def _seq_eq(a, b): @@ -153,7 +153,7 @@ def _seq_eq(a, b): return true if a is nil or b is nil: return false - if not (rt.satisfies_QMARK_(rt.ISeqable.deref(), b) or rt.satisfies_QMARK_(rt.ISeq.deref(), b)): + if not (rt._satisfies_QMARK_(rt.ISeqable.deref(), b) or rt._satisfies_QMARK_(rt.ISeq.deref(), b)): return false a = rt.seq(a) @@ -286,7 +286,7 @@ def __with_meta(a, b): @returns(bool) @as_var("has-meta?") def __has_meta(a): - return true if rt.satisfies_QMARK_(rt.IMeta.deref(), a) else false + return true if rt._satisfies_QMARK_(rt.IMeta.deref(), a) else false @as_var("conj") def conj(a, b): @@ -313,8 +313,8 @@ def str__args(args): @jit.unroll_safe def apply__args(args): last_itm = args[len(args) - 1] - if not (rt.satisfies_QMARK_(rt.IIndexed.deref(), last_itm) and - rt.satisfies_QMARK_(rt.ICounted.deref(), last_itm)): + if not (rt._satisfies_QMARK_(rt.IIndexed.deref(), last_itm) and + rt._satisfies_QMARK_(rt.ICounted.deref(), last_itm)): last_itm = rt.vec(last_itm) fn = args[0] @@ -335,14 +335,14 @@ def apply__args(args): # return nil @returns(bool) -@as_var("instance?") +@as_var("-instance?") def _instance(c, o): affirm(isinstance(c, Type), u"c must be a type") return true if istypeinstance(o, c) else false @returns(bool) -@as_var("satisfies?") +@as_var("-satisfies?") def _satisfies(proto, o): affirm(isinstance(proto, Protocol), u"proto must be a Protocol") @@ -625,7 +625,7 @@ def identical(a, b): @as_var("vector?") def vector_QMARK_(a): - return true if rt.satisfies_QMARK_(rt.IVector.deref(), a) else false + return true if rt._satisfies_QMARK_(rt.IVector.deref(), a) else false @returns(bool) @as_var("eq") From eed57f857c601334ac22b4a5481eeb63561c1680 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 14:14:08 +0100 Subject: [PATCH 666/909] Add :signatures to satisfies? and instance?. --- pixie/stdlib.pxi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1853fe83..773f8c8c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -167,7 +167,8 @@ (def instance? (fn ^{:doc "Checks if x is an instance of t. When t is seqable, checks if x is an instance of - any of the types contained therein."} + any of the types contained therein." + :signatures [[t x]]} instance? [t x] (if (-satisfies? ISeqable t) (let [ts (seq t)] @@ -179,7 +180,8 @@ (def satisfies? (fn ^{:doc "Checks if x satisfies the protocol p. When p is seqable, checks if x satisfies all of - the protocols contained therein."} + the protocols contained therein." + :signatures [[t x]]} satisfies? [p x] (if (-satisfies? ISeqable p) (let [ps (seq p)] From 67d8641eefd5a0c26d01dc31e00abab945dc74c9 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Wed, 25 Mar 2015 14:22:32 +0100 Subject: [PATCH 667/909] Add tests for satisfies? and instance?. --- tests/pixie/tests/test-stdlib.pxi | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 3f23d9db..3e9b50a4 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -448,3 +448,30 @@ (t/assert= (f 3) :large) (t/assert= (f 4) :large) (t/assert= (f 9) :toolarge))) + +(t/deftest test-instance? + (t/assert= (instance? Keyword :a) true) + (t/assert= (instance? Keyword 'a) false) + (t/assert= (instance? [Symbol Keyword] :a) true) + (t/assert= (instance? [Symbol Keyword] 'a) true) + (t/assert= (instance? [Symbol Keyword] 42) false) + (t/assert= (instance? [] :x) false) + (t/assert-throws? RuntimeException + "c must be a type" + (instance? :not-a-type 123)) + (t/assert-throws? RuntimeException + "c must be a type" + (instance? [Keyword :also-not-a-type] 123))) + +(t/deftest test-satisfies? + (t/assert= (satisfies? IIndexed [1 2]) true) + (t/assert= (satisfies? IIndexed '(1 2)) false) + (t/assert= (satisfies? [] :xyz) true) + (t/assert= (satisfies? [ILookup IIndexed] [1 2]) true) + (t/assert= (satisfies? [ILookup IIndexed] {1 2}) false) + (t/assert-throws? RuntimeException + "proto must be a Protocol" + (satisfies? :not-a-proto 123)) + (t/assert-throws? RuntimeException + "proto must be a Protocol" + (satisfies? [IIndexed :also-not-a-proto] [1 2]))) From 2acfbdbedd72d4d31b939968e8be12d1765bf92f Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 25 Mar 2015 16:20:24 -0600 Subject: [PATCH 668/909] fix some ffi bugs --- pixie/vm/libs/ffi.py | 19 ++++++++++++++----- pixie/vm/object.py | 10 ++++++++++ pixie/vm/threads.py | 4 ++-- 3 files changed, 26 insertions(+), 7 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 1a30a535..0d346472 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -394,8 +394,9 @@ def ffi_set_value(self, ptr, val): vpnt = rffi.cast(rffi.VOIDPP, ptr) vpnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) else: - print val - affirm(False, u"Cannot encode this type") + frm_name = rt.name(rt.str(val.type())) + to_name = rt.name(rt.str(self)) + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) def ffi_size(self): @@ -445,7 +446,13 @@ def ffi_get_value(self, ptr): def ffi_set_value(self, ptr, val): pnt = rffi.cast(rffi.VOIDPP, ptr) - if isinstance(val, Buffer): + if isinstance(val, String): + pnt = rffi.cast(rffi.CCHARPP, ptr) + utf8 = unicode_to_utf8(rt.name(val)) + raw = rffi.str2charp(utf8) + pnt[0] = raw + return CCharPToken(raw) + elif isinstance(val, Buffer): pnt[0] = val.buffer() elif isinstance(val, VoidP): pnt[0] = val.raw_data() @@ -454,8 +461,10 @@ def ffi_set_value(self, ptr, val): elif isinstance(val, CStruct): pnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) else: - print val - affirm(False, u"Cannot encode this type") + frm_name = rt.name(rt.str(val.type())) + to_name = rt.name(rt.str(self)) + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) + def ffi_size(self): return rffi.sizeof(rffi.VOIDP) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 2abf674e..32bdd199 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -151,6 +151,16 @@ def runtime_error(msg): import pixie.vm.rt as rt raise WrappedException(RuntimeException(rt.wrap(msg))) +def safe_invoke(f, args): + try: + f.invoke(args) + except Exception as ex: + if isinstance(ex, WrappedException): + print "UNSAFE EXCEPTION", ex._ex.__repr__() + else: + print "UNSAFE EXCEPTION", ex + return None + class ErrorInfo(Object): _type = Type(u"pixie.stdlib.ErrorInfo") def type(self): diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index d34b47b8..5140608b 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -1,4 +1,4 @@ -from pixie.vm.object import Object, Type +from pixie.vm.object import Object, Type, safe_invoke from pixie.vm.primitives import true import rpython.rlib.rthread as rthread from pixie.vm.primitives import nil @@ -42,7 +42,7 @@ def bootstrap(): rthread.gc_thread_start() fn = bootstrapper.fn() bootstrapper.release() - fn.invoke([]) + safe_invoke(fn, []) rthread.gc_thread_die() bootstrapper = Bootstrapper() From 4059f4ca1988ee924d5f41d6c0f352b169ab1b68 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 26 Mar 2015 12:06:09 +0000 Subject: [PATCH 669/909] properly infer functions that return char* on platforms with unsigned chars --- pixie/ffi-infer.pxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index b2db256b..e30a2e0f 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -97,8 +97,10 @@ return 0; (defmethod edn-to-ctype :pointer [{:keys [of-type] :as ptr} in-struct?] + (println ptr in-struct?) (cond - (and (= of-type {:signed? true :size 1 :type :int}) + (and (= (:size of-type) 1) + (= (:type of-type) :int) (not in-struct?)) 'pixie.stdlib/CCharP (= (:type of-type) :function) (callback-type of-type in-struct?) :else 'pixie.stdlib/CVoidP)) From 1d212836e8c3bd29e04944f2d6a6ce9c3b18d896 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 26 Mar 2015 12:08:03 +0000 Subject: [PATCH 670/909] remove debug line --- pixie/ffi-infer.pxi | 1 - 1 file changed, 1 deletion(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index e30a2e0f..5e3b086f 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -97,7 +97,6 @@ return 0; (defmethod edn-to-ctype :pointer [{:keys [of-type] :as ptr} in-struct?] - (println ptr in-struct?) (cond (and (= (:size of-type) 1) (= (:type of-type) :int) From 1838d80528c9cad6dcc1f09424dc5e506e673190 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 26 Mar 2015 17:11:27 +0000 Subject: [PATCH 671/909] add ISeekableStream protocol --- pixie/io-blocking.pxi | 51 +++++++++++++++++++++-------------- pixie/io.pxi | 7 +++++ pixie/streams.pxi | 5 ++++ tests/pixie/tests/test-io.pxi | 11 ++++++++ 4 files changed, 54 insertions(+), 20 deletions(-) diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index 5dd57346..f2ffb41a 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -3,6 +3,9 @@ (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) +(def fseek (ffi-fn libc "fseek" [CVoidP CInt CInt] CInt)) +(def ftell (ffi-fn libc "ftell" [CVoidP] CInt)) +(def -rewind (ffi-fn libc "rewind" [CVoidP] CVoidP)) (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) (def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) (def fputc (ffi-fn libc "fputc" [CInt CVoidP] CInt)) @@ -15,6 +18,18 @@ (def DEFAULT-BUFFER-SIZE 1024) +(defn default-stream-reducer [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) + (deftype FileStream [fp] IInputStream (read [this buffer len] @@ -25,21 +40,17 @@ read-count)) (read-byte [this] (fgetc buffer)) + ISeekableStream + (seek [this pos] + (fseek fp pos 0)) + (rewind [this] + (-rewind fp)) IDisposable (-dispose! [this] (fclose fp)) IReduce (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) + (default-stream-reducer this f init))) (defn open-read {:doc "Open a file for reading, returning a IInputStream" @@ -109,6 +120,15 @@ (dispose! c) result)) +(defn slurp-stream [stream] + (let [c stream + result (transduce + (map char) + string-builder + c)] + (dispose! c) + result)) + (deftype ProcessInputStream [fp] IInputStream (read [this buffer len] @@ -124,16 +144,7 @@ (pclose fp)) IReduce (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) + (default-stream-reducer this f init))) (defn popen-read diff --git a/pixie/io.pxi b/pixie/io.pxi index 045e14b5..6dfed2ff 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -44,6 +44,13 @@ (set-field! this :offset (+ offset read-count)) (set-buffer-count! buffer read-count) read-count)) + ISeekableStream + (position [this] + offset) + (rewind [this] + (set-field! this :offset 0)) + (seek [this pos] + (set-field! this :offset pos)) IDisposable (-dispose! [this] (dispose! uvbuf) diff --git a/pixie/streams.pxi b/pixie/streams.pxi index 05b4ad44..b39fa4f3 100644 --- a/pixie/streams.pxi +++ b/pixie/streams.pxi @@ -11,3 +11,8 @@ (defprotocol IByteOutputStream (write-byte [this byte])) + +(defprotocol ISeekableStream + (position [this]) + (seek [this position]) + (rewind [this])) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 09de53e3..c93e3dd9 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -1,5 +1,6 @@ (ns pixie.tests.test-io (require pixie.test :as t) + (require pixie.streams :as st :refer :all) (require pixie.io :as io)) (t/deftest test-file-reduction @@ -24,6 +25,16 @@ s (io/line-seq f)] (t/assert= (last s) "Second line."))) +(t/deftest test-seek + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (io/read-line f) + (t/assert= (io/read-line f) "Second line.") + (io/rewind f) + (io/read-line f) + (t/assert= (io/read-line f) "Second line.") + (io/seek f (- (position f) 6)) + (t/assert= (io/read-line f) "line."))) + (t/deftest test-slurp-spit (let [val (vec (range 1280))] (io/spit "test.tmp" val) From eca7f7dc56ab86f9729ceb6162b4d2768f54bd7c Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Fri, 27 Mar 2015 11:26:49 +0100 Subject: [PATCH 672/909] Implement macroexpand-1. --- pixie/stdlib.pxi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 773f8c8c..6276dd5d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2311,6 +2311,24 @@ Expands to calls to `extend-type`." ([f col] (transduce (map f) conj col))) +(defn macroexpand-1 [form] + {:doc "If form is a macro call, returns the expanded form. + + Does nothing if not a macro call." + :signatures [[form]] + :examples [["(macroexpand-1 '(when condition this and-this))" + nil `(if condition (do this and-this))] + ["(macroexpand-1 ())" nil ()] + ["(macroexpand-1 [1 2])" nil [1 2]]]} + (if (or (not (list? form)) + (= () form)) + form + (let [[sym & args] form + fvar (resolve sym)] + (if (and fvar (macro? @fvar)) + (apply @fvar args) + form)))) + (def *1) (def *2) (def *3) From 2491d2b43dd869f4066ff5b0da57c2da57234479 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Sat, 28 Mar 2015 15:45:25 -0600 Subject: [PATCH 673/909] added basic utf8 encoder-decoder --- pixie/io.pxi | 18 ++++++++ pixie/streams/utf8.pxi | 55 +++++++++++++++++++++++++ tests/pixie/tests/streams/test-utf8.pxi | 17 ++++++++ 3 files changed, 90 insertions(+) create mode 100644 pixie/streams/utf8.pxi create mode 100644 tests/pixie/tests/streams/test-utf8.pxi diff --git a/pixie/io.pxi b/pixie/io.pxi index 6dfed2ff..e1d43b10 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -131,9 +131,27 @@ (set-buffer-count! buffer idx) (write downstream buffer))) +(deftype BufferedInputStream [upstream idx buffer] + IByteInputStream + (read-byte [this] + (when (= idx (count buffer)) + (set-field! this :idx 0) + (read upstream buffer (buffer-capacity buffer))) + (let [val (nth buffer idx)] + (set-field! this :idx (inc idx)) + val)) + IDisposable + (-dispose! [this] + (dispose! upstream) + (dispose! buffer))) + (defn buffered-output-stream [downstream size] (->BufferedOutputStream downstream 0 (buffer size))) +(defn buffered-input-stream [upstream size] + (let [b (buffer size)] + (set-buffer-count! b size) + (->BufferedInputStream upstream size b))) (defn throw-on-error [result] (when (neg? result) diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi new file mode 100644 index 00000000..baf8bd85 --- /dev/null +++ b/pixie/streams/utf8.pxi @@ -0,0 +1,55 @@ +(ns pixie.streams.utf8 + (require pixie.streams :refer :all)) + +(defprotocol IUTF8OutputStream + (write-char [this char])) + +(defprotocol IUTF8InputStream + (read-char [this])) + +(deftype UTF8OutputStream [out] + IUTF8OutputStream + (write-char [this ch] + (let [ch (int ch)] + (cond + (<= ch 0x7F) (write-byte out ch) + (<= ch 0x7FF) (do (write-byte out (bit-or 0xC0 (bit-shift-right ch 6))) + (write-byte out (bit-or 0x80 (bit-and ch 0x3F)))) + (<= ch 0xFFFF) (do (write-byte out (bit-or 0xE0 (bit-shift-right ch 12))) + (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 6) 0x3F))) + (write-byte out (bit-or 0x80 (bit-and ch 0x3F)))) + (<= ch 0x1FFFFF) (do (write-byte out (bit-or 0xE0 (bit-shift-right ch 18))) + (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 12) 0x3F))) + (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 6) 0x3F))) + (write-byte out (bit-or 0x80 (bit-and ch 0x3F))) )))) + IDisposable + (-dispose! [this] + (dispose! out))) + + +(deftype UTF8InputStream [in] + IUTF8InputStream + (read-char [this] + (let [ch (int (read-byte in)) + [n bytes] (cond + (>= 0x7F ch) [ch 1] + (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2] + (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3] + (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4] + :else (assert false (str "Got bad code " ch)))] + (loop [i (dec bytes) + n n] + (if (pos? i) + (recur (dec i) + (bit-or (bit-shift-left n 6) + (bit-and (read-byte in) 0x3F))) + (char n))))) + IDisposable + (-dispose! [this] + (dispose! in))) + +(defn utf8-input-stream [i] + (->UTF8InputStream i)) + +(defn utf8-output-stream [o] + (->UTF8OutputStream o)) diff --git a/tests/pixie/tests/streams/test-utf8.pxi b/tests/pixie/tests/streams/test-utf8.pxi new file mode 100644 index 00000000..ede7a1c5 --- /dev/null +++ b/tests/pixie/tests/streams/test-utf8.pxi @@ -0,0 +1,17 @@ +(ns pixie.streams.test-utf8 + (require pixie.streams.utf8 :refer :all) + (require pixie.io :as io) + (require pixie.test :refer :all)) + + +(deftest test-writing-ints + (using [os (-> (io/open-write "/tmp/pixie-utf-test.txt") + (io/buffered-output-stream 1024) + utf8-output-stream)] + (dotimes [x 32000] + (write-char os (char x)))) + (using [is (-> (io/open-read "/tmp/pixie-utf-test.txt") + (io/buffered-input-stream 1024) + utf8-input-stream)] + (dotimes [x 32000] + (assert= x (int (read-char is)))))) From 5c5732b0591190d0de220f969372c9d220b80e66 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sun, 29 Mar 2015 16:40:43 +0100 Subject: [PATCH 674/909] Implement -seq for sets the same way as maps Sets were using iterators instead of just building a list (which were mentioned should be removed in #242), and I guess some of the iterator code doesn't work properly for maps, because executing `(seq #{4 7})` causes a segfault without this patch. I'm not really familar with the implementation of iterators over maps, but this is sort of an interesting bug because I've only been able to reproduce it with sets that have #{4 7} as a subset. --- pixie/stdlib.pxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6276dd5d..149f7009 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -959,7 +959,9 @@ If further arguments are passed, invokes the method named by symbol, passing the (fn [v] (transduce cat unordered-hash-reducing-fn v))) -(extend -seq PersistentHashSet (fn [self] (seq (iterator self)))) +(extend -seq PersistentHashSet + (fn [s] + (reduce conj nil s))) (extend -str PersistentHashSet (fn [s] From f140b486cdd85ee47ce8895cf2be0778fcd31950 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 20 Mar 2015 10:55:41 +0000 Subject: [PATCH 675/909] Add basic functions for dealing with environment --- pixie/stdlib.pxi | 10 ++++++++ pixie/vm/libs/env.py | 55 ++++++++++++++++++++++++++++++++++++++++++++ pixie/vm/rt.py | 1 + pixie/vm/stdlib.py | 1 - 4 files changed, 66 insertions(+), 1 deletion(-) create mode 100644 pixie/vm/libs/env.py diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6276dd5d..1db82f15 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2340,3 +2340,13 @@ Expands to calls to `extend-type`." (def *e) (defn -set-*e [e] (def *e e)) + +(extend -str Environment + (fn [v] + (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] + (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) + +(extend -repr Environment + (fn [v] + (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] + (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) diff --git a/pixie/vm/libs/env.py b/pixie/vm/libs/env.py new file mode 100644 index 00000000..25bfb7ca --- /dev/null +++ b/pixie/vm/libs/env.py @@ -0,0 +1,55 @@ +from pixie.vm.code import as_var +from pixie.vm.object import Object, Type, runtime_error +from pixie.vm.primitives import nil +from pixie.vm.string import String +import pixie.vm.stdlib as proto +from pixie.vm.code import extend, as_var +import pixie.vm.rt as rt +import os + +class Environment(Object): + _type = Type(u"pixie.stdlib.Environment") + + def type(self): + return Environment._type + + def val_at(self, key, not_found): + if not isinstance(key, String): + runtime_error(u"Environment variables are strings ") + key_str = str(rt.name(key)) + try: + var = os.environ[key_str] + return rt.wrap(var) + except KeyError: + return not_found + + # TODO: Implement me. + # def dissoc(self): + # def asssoc(self): + + def reduce_vars(self, f, init): + for k, v in os.environ.items(): + init = f.invoke([init, rt.map_entry(rt.wrap(k), rt.wrap(v))]) + if rt.reduced_QMARK_(init): + return init + return init + + +@extend(proto._val_at, Environment) +def _val_at(self, key, not_found): + assert isinstance(self, Environment) + v = self.val_at(key, not_found) + return v + +@extend(proto._reduce, Environment) +def _reduce(self, f, init): + assert isinstance(self, Environment) + val = self.reduce_vars(f, init) + if rt.reduced_QMARK_(val): + return rt.deref(val) + + return val + +@as_var("pixie.stdlib", "env") +def _env(): + return Environment() diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 794a4c3b..37fa0676 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -65,6 +65,7 @@ def wrapper(*args): import pixie.vm.map_entry import pixie.vm.libs.platform import pixie.vm.libs.ffi + import pixie.vm.libs.env import pixie.vm.symbol import pixie.vm.libs.path import pixie.vm.libs.string diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 6741063b..78eb48bd 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -864,4 +864,3 @@ def _set_current_var_frames(self, frames): Sets the current var frames. Frames should be a cons list of hashmaps containing mappings of vars to dynamic values. Setting this value to anything but this data format will cause undefined errors.""" code._dynamic_vars.set_current_frames(frames) - From 8071349815897926ee73f0e8618815f3fc6752c5 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 30 Mar 2015 06:53:23 -0600 Subject: [PATCH 676/909] added error handling to UTF8 streams --- benchmarks/deftype_fields.pxi | 7 ++++-- pixie/math.pxi | 6 ++++- pixie/streams/utf8.pxi | 42 +++++++++++++++++++++++------------ 3 files changed, 38 insertions(+), 17 deletions(-) diff --git a/benchmarks/deftype_fields.pxi b/benchmarks/deftype_fields.pxi index fbee851b..caf84fb8 100644 --- a/benchmarks/deftype_fields.pxi +++ b/benchmarks/deftype_fields.pxi @@ -11,6 +11,9 @@ b)) -(def adder (->Adder 1 0)) -(dotimes [x (* 1024 1024 1024)] + +(def adder (->Adder 1.0 0)) +(println "Starting....") +(dotimes [x (* 1024 1024 1024 20)] (assert (= (inc x) (add-them adder)))) +(println "Ending....") diff --git a/pixie/math.pxi b/pixie/math.pxi index 7c16889d..4b952398 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -23,4 +23,8 @@ (i/defcfn ceil) (i/defcfn fabs) (i/defcfn floor) - (i/defcfn fmod)) + (i/defcfn fmod) + + (i/defconst M_PI)) + +(def pi M_PI) diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi index baf8bd85..055733df 100644 --- a/pixie/streams/utf8.pxi +++ b/pixie/streams/utf8.pxi @@ -2,10 +2,10 @@ (require pixie.streams :refer :all)) (defprotocol IUTF8OutputStream - (write-char [this char])) + (write-char [this char] "Write a single character to the UTF8 stream")) (defprotocol IUTF8InputStream - (read-char [this])) + (read-char [this] "Read a single character from the UTF8 stream")) (deftype UTF8OutputStream [out] IUTF8OutputStream @@ -21,35 +21,49 @@ (<= ch 0x1FFFFF) (do (write-byte out (bit-or 0xE0 (bit-shift-right ch 18))) (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 12) 0x3F))) (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 6) 0x3F))) - (write-byte out (bit-or 0x80 (bit-and ch 0x3F))) )))) + (write-byte out (bit-or 0x80 (bit-and ch 0x3F)))) + :else (assert false (str "Cannot encode a UTF8 character of code " ch))))) IDisposable (-dispose! [this] (dispose! out))) -(deftype UTF8InputStream [in] +(deftype UTF8InputStream [in bad-char] IUTF8InputStream (read-char [this] (let [ch (int (read-byte in)) - [n bytes] (cond - (>= 0x7F ch) [ch 1] - (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2] - (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3] - (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4] - :else (assert false (str "Got bad code " ch)))] + [n bytes error?] (cond + (>= 0x7F ch) [ch 1] + (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2 false] + (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3 false] + (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4 false] + (= 0xF8 (bit-and ch 0xF8)) [(bit-and ch 3) 5 true] + (= 0xFC (bit-and ch 0xFE)) [(bit-and ch 1) 6 true] + :else [n 1 true])] (loop [i (dec bytes) n n] (if (pos? i) (recur (dec i) (bit-or (bit-shift-left n 6) (bit-and (read-byte in) 0x3F))) - (char n))))) + (if error? + (if bad-char + bad-char + (throw (str "Invalid UTF8 character decoded: " n))) + (char n)))))) IDisposable (-dispose! [this] (dispose! in))) -(defn utf8-input-stream [i] - (->UTF8InputStream i)) +(defn utf8-input-stream + "Creates a UTF8 decoder that reads characters from the given IByteInputStream. If a bad character is found + an error will be thrown, unless an optional bad-character marker character is provided." + ([i] + (->UTF8InputStream i nil)) + ([i bad-char] + (->UTF8InputStream i bad-char))) -(defn utf8-output-stream [o] +(defn utf8-output-stream + "Creates a UTF8 encoder that writes characters to the given IByteOutputStream." + [o] (->UTF8OutputStream o)) From 1eb6e0ae00f1ad42290dcbe8c0b1fdb8911933ff Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 30 Mar 2015 06:56:03 -0600 Subject: [PATCH 677/909] remove some accidentally commited files --- benchmarks/deftype_fields.pxi | 7 ++----- pixie/math.pxi | 6 +----- 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/benchmarks/deftype_fields.pxi b/benchmarks/deftype_fields.pxi index caf84fb8..fbee851b 100644 --- a/benchmarks/deftype_fields.pxi +++ b/benchmarks/deftype_fields.pxi @@ -11,9 +11,6 @@ b)) - -(def adder (->Adder 1.0 0)) -(println "Starting....") -(dotimes [x (* 1024 1024 1024 20)] +(def adder (->Adder 1 0)) +(dotimes [x (* 1024 1024 1024)] (assert (= (inc x) (add-them adder)))) -(println "Ending....") diff --git a/pixie/math.pxi b/pixie/math.pxi index 4b952398..7c16889d 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -23,8 +23,4 @@ (i/defcfn ceil) (i/defcfn fabs) (i/defcfn floor) - (i/defcfn fmod) - - (i/defconst M_PI)) - -(def pi M_PI) + (i/defcfn fmod)) From 72e1bfc206cba5e4e170e50995c8d136f4bfe67c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 30 Mar 2015 16:19:29 -0600 Subject: [PATCH 678/909] fix randomly failing tests in JIT mode --- pixie/vm/custom_types.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 05008d2a..0d6d126b 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -55,13 +55,18 @@ def set_field(self, name, val): if isinstance(old_val, AbstractMutableCell): old_val.set_mutable_cell_value(self._custom_type, self._fields, name, idx, val) else: + self._custom_type.set_mutable(name) self._fields[idx] = val return self @jit.elidable_promote() - def get_field_immutable(self, idx): + def _get_field_immutable(self, idx, rev): return self._fields[idx] + def get_field_immutable(self, idx): + tp = self._custom_type + assert isinstance(tp, CustomType) + return self._get_field_immutable(idx, tp._rev) def get_field(self, name): idx = self._custom_type.get_slot_idx(name) @@ -78,11 +83,6 @@ def get_field(self, name): else: return value - def set_field_by_idx(self, idx, val): - affirm(isinstance(idx, r_uint), u"idx must be a r_uint") - self._fields[idx] = val - return self - @as_var("create-type") def create_type(type_name, fields): From aef70b13851f15b1c719cd7a90567fd238a43a9b Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 30 Mar 2015 16:28:12 -0600 Subject: [PATCH 679/909] remove outdated failing timer test --- tests/pixie/tests/test-uv.pxi | 21 --------------------- 1 file changed, 21 deletions(-) delete mode 100644 tests/pixie/tests/test-uv.pxi diff --git a/tests/pixie/tests/test-uv.pxi b/tests/pixie/tests/test-uv.pxi deleted file mode 100644 index 4167955e..00000000 --- a/tests/pixie/tests/test-uv.pxi +++ /dev/null @@ -1,21 +0,0 @@ -(ns pixie.test-uv - (require pixie.uv :as uv) - (require pixie.test :as t) - (require pixie.ffi :as ffi)) - - -(t/deftest timer-tests - (let [cb (atom nil) - result (atom false) - loop (uv/uv_loop_t) - timer (uv/uv_timer_t)] - (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb - (fn [handle] - (reset! result true) - (uv/uv_timer_stop timer) - (-dispose! @cb)))) - (uv/uv_loop_init loop) - (uv/uv_timer_init loop timer) - (uv/uv_timer_start timer @cb 10 0) - (uv/uv_run loop uv/UV_RUN_ONCE) - (t/assert @result))) From 6905dfa3c826495995e16dd223c3e4b794ef9927 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 1 Apr 2015 10:44:20 +0100 Subject: [PATCH 680/909] clojure style :require --- pixie/async.pxi | 2 +- pixie/channels.pxi | 4 ++-- pixie/csp.pxi | 6 +++--- pixie/ffi-infer.pxi | 2 +- pixie/fs.pxi | 4 ++-- pixie/io-blocking.pxi | 2 +- pixie/io.pxi | 12 ++++++------ pixie/math.pxi | 2 +- pixie/repl.pxi | 6 +++--- pixie/stacklets.pxi | 4 ++-- pixie/stdlib.pxi | 19 ++++++++++++++++--- pixie/string.pxi | 2 +- pixie/test.pxi | 4 ++-- pixie/uv.pxi | 2 +- tests/pixie/tests/test-async.pxi | 6 +++--- tests/pixie/tests/test-buffers.pxi | 5 ++--- tests/pixie/tests/test-channels.pxi | 8 ++++---- 17 files changed, 51 insertions(+), 39 deletions(-) diff --git a/pixie/async.pxi b/pixie/async.pxi index 2907936a..ca7862f8 100644 --- a/pixie/async.pxi +++ b/pixie/async.pxi @@ -1,5 +1,5 @@ (ns pixie.async - (require pixie.stacklets :as st)) + (:require [pixie.stacklets :as st])) (deftype Promise [val pending-callbacks delivered?] diff --git a/pixie/channels.pxi b/pixie/channels.pxi index 1b48ecf8..c44cc060 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -1,6 +1,6 @@ (ns pixie.channels - (require pixie.stacklets :as st) - (require pixie.buffers :as b)) + (:require [pixie.stacklets :as st] + [pixie.buffers :as b])) (defprotocol ICancelable (-canceled? [this] "Determines if a request (such as a callback) that can be canceled") diff --git a/pixie/csp.pxi b/pixie/csp.pxi index b20fa08c..94ca90d9 100644 --- a/pixie/csp.pxi +++ b/pixie/csp.pxi @@ -1,7 +1,7 @@ (ns pixie.csp - (require pixie.stacklets :as st) - (require pixie.buffers :as b) - (require pixie.channels :as chans)) + (:require [pixie.stacklets :as st] + [pixie.buffers :as b] + [pixie.channels :as chans])) (def chan chans/chan) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 5e3b086f..f2074f53 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -1,5 +1,5 @@ (ns pixie.ffi-infer - (require pixie.io-blocking :as io)) + (:require [pixie.io-blocking :as io])) (defn -add-rel-path [rel] diff --git a/pixie/fs.pxi b/pixie/fs.pxi index a7745a7a..048e6f63 100644 --- a/pixie/fs.pxi +++ b/pixie/fs.pxi @@ -1,6 +1,6 @@ (ns pixie.fs - (require pixie.path :as path) - (require pixie.string :as string)) + (:require [pixie.path :as path] + [pixie.string :as string])) (defprotocol IFSPath diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index f2ffb41a..9b4f3943 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -1,5 +1,5 @@ (ns pixie.io-blocking - (require pixie.streams :as st :refer :all)) + (:require [pixie.streams :as st :refer :all])) (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) diff --git a/pixie/io.pxi b/pixie/io.pxi index e1d43b10..aa868411 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -1,10 +1,10 @@ (ns pixie.io - (require pixie.streams :as st :refer :all) - (require pixie.io-blocking :as io-blocking) - (require pixie.uv :as uv) - (require pixie.stacklets :as st) - (require pixie.ffi :as ffi) - (require pixie.ffi-infer :as ffi-infer)) + (:require [pixie.streams :as st :refer :all] + [pixie.io-blocking :as io-blocking] + [pixie.uv :as uv] + [pixie.stacklets :as st] + [pixie.ffi :as ffi] + [pixie.ffi-infer :as ffi-infer])) (defmacro defuvfsfn [nm args return] `(defn ~nm ~args diff --git a/pixie/math.pxi b/pixie/math.pxi index 7c16889d..47d45cd0 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -1,5 +1,5 @@ (ns pixie.math - (require pixie.ffi-infer :as i)) + (:require [pixie.ffi-infer :as i])) (i/with-config {:library "m" :cxx-flags ["-lm"] diff --git a/pixie/repl.pxi b/pixie/repl.pxi index d10b6a44..e8432cba 100644 --- a/pixie/repl.pxi +++ b/pixie/repl.pxi @@ -1,7 +1,7 @@ (ns pixie.repl - (require pixie.stacklets :as st) - (require pixie.io :as io) - (require pixie.ffi-infer :as f)) + (:require [pixie.stacklets :as st] + [pixie.io :as io] + [pixie.ffi-infer :as f])) (f/with-config {:library "edit" :includes ["editline/readline.h"]} diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index ec03ae31..c3d2b236 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -1,6 +1,6 @@ (ns pixie.stacklets - (require pixie.uv :as uv) - (require pixie.ffi :as ffi)) + (:require [pixie.uv :as uv] + [pixie.ffi :as ffi])) ;; If we don't do this, compiling this file doesn't work since the def will clear out ;; the existing value. diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a3c6c4f0..34cd0c52 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1105,9 +1105,22 @@ Creates new maps if the keys are not present." ret)))) (defmacro ns [nm & body] - `(do (in-ns ~(keyword (name nm))) - ~@body)) - + (let [bmap (reduce (fn [m b] + (update-in m [(first b)] (fnil conj []) (rest b))) + {} + body) + requires + (do + (assert (>= 1 (count (:require bmap))) + "Only one :require block can be defined per namespace") + (map (fn [r] `(require ~@r)) (first (:require bmap)))) + + old-style-requires + (map (fn [r] `(require ~@r)) + (bmap 'require))] + `(do (in-ns ~(keyword (name nm))) + ~@requires + ~@old-style-requires))) (defn symbol? [x] (identical? Symbol (type x))) diff --git a/pixie/string.pxi b/pixie/string.pxi index 36be0b86..fdbbcbb3 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -1,5 +1,5 @@ (ns pixie.string - (require pixie.string.internal :as si)) + (:require [pixie.string.internal :as si])) ; reexport native string functions (def substring si/substring) diff --git a/pixie/test.pxi b/pixie/test.pxi index 26c3927b..745efa00 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -1,6 +1,6 @@ (ns pixie.test - (require pixie.string :as s) - (require pixie.fs :as fs)) + (:require [pixie.string :as s] + [pixie.fs :as fs])) (def tests (atom {})) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index d984d52a..76417f47 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -1,5 +1,5 @@ (ns pixie.uv - (require pixie.ffi-infer :as f)) + (:require [pixie.ffi-infer :as f])) (f/with-config {:library "uv" :includes ["uv.h"]} diff --git a/tests/pixie/tests/test-async.pxi b/tests/pixie/tests/test-async.pxi index d5764af6..19fa3731 100644 --- a/tests/pixie/tests/test-async.pxi +++ b/tests/pixie/tests/test-async.pxi @@ -1,7 +1,7 @@ (ns pixie.tests.test-async - (require pixie.stacklets :as st) - (require pixie.async :as async :refer :all) - (require pixie.test :as t :refer :all)) + (:require [pixie.stacklets :as st] + [pixie.async :as async :refer :all] + [pixie.test :as t :refer :all])) (deftest test-future-deref diff --git a/tests/pixie/tests/test-buffers.pxi b/tests/pixie/tests/test-buffers.pxi index 92a99c94..1afd5326 100644 --- a/tests/pixie/tests/test-buffers.pxi +++ b/tests/pixie/tests/test-buffers.pxi @@ -1,7 +1,6 @@ (ns pixie.tests.test-buffers - (require pixie.test :refer :all) - (require pixie.buffers :refer :all)) - + (:require [pixie.test :refer :all] + [pixie.buffers :refer :all])) (deftest test-adding-and-removing-from-buffer (let [buffer (ring-buffer 10)] diff --git a/tests/pixie/tests/test-channels.pxi b/tests/pixie/tests/test-channels.pxi index 9f35fe61..0cdebfd0 100644 --- a/tests/pixie/tests/test-channels.pxi +++ b/tests/pixie/tests/test-channels.pxi @@ -1,8 +1,8 @@ (ns pixie.tests.test-channels - (require pixie.test :refer :all) - (require pixie.channels :refer :all) - (require pixie.async :refer :all) - (require pixie.stacklets :as st)) + (:require [pixie.test :refer :all] + [pixie.channels :refer :all] + [pixie.async :refer :all] + [pixie.stacklets :as st])) (deftest simple-read-and-write From 0d6e1aef015a26b972ff99031ba87328b8c8a971 Mon Sep 17 00:00:00 2001 From: Peter Monks Date: Wed, 1 Apr 2015 17:10:10 -0700 Subject: [PATCH 681/909] Added UTF8 test Added temp file generated from tests to .gitignore --- .gitignore | 1 + tests/pixie/tests/test-utf8.pxi | 9 +++++++++ 2 files changed, 10 insertions(+) create mode 100644 tests/pixie/tests/test-utf8.pxi diff --git a/.gitignore b/.gitignore index e7206d68..43e35ec3 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ pixie-vm lib include pixie/*.pxic +test.tmp diff --git a/tests/pixie/tests/test-utf8.pxi b/tests/pixie/tests/test-utf8.pxi new file mode 100644 index 00000000..1c52ef01 --- /dev/null +++ b/tests/pixie/tests/test-utf8.pxi @@ -0,0 +1,9 @@ +(ns pixie.test.test-utf8 + (require pixie.test :as t)) + +(t/deftest test-utf8-string-val + (t/assert= "🍺=👍" "🍺=👍")) + +(t/deftest test-utf8-var-name + (let [🍺 "🍺=👍"] + (t/assert= 🍺 "🍺=👍"))) From 7ba81e3a1299e12bccbe1dacf088e80db40f38d0 Mon Sep 17 00:00:00 2001 From: Michael Trotter Date: Thu, 2 Apr 2015 01:18:37 -0600 Subject: [PATCH 682/909] minor grammer fix in README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 2cda23c1..658d8c59 100644 --- a/README.md +++ b/README.md @@ -51,7 +51,7 @@ Pixie now comes with a build tool called [dust](https://github.com/pixie-lang/du ### So this is written in Python? -It's actually written in the RPython, the same language PyPy is written in. `make build_with_jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm`. This executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... +It's actually written in RPython, the same language PyPy is written in. `make build_with_jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm`. This executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... ### What's this bit about "magical powers"? From b20575edc65d67c8195d226460ec376f72141eb0 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Sat, 4 Apr 2015 21:47:14 +0200 Subject: [PATCH 683/909] Fix some docstrings being in the wrong position. --- pixie/stdlib.pxi | 20 +++++++++++++------- 1 file changed, 13 insertions(+), 7 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 34cd0c52..7df603d9 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1598,21 +1598,23 @@ The new value is thus `(apply f current-value-of-atom args)`." (lazy-seq (step pred coll))))) ;; TODO: use a transient map in the future -(defn group-by [f coll] +(defn group-by {:doc "Groups the collection into a map keyed by the result of applying f on each element. The value at each key is a vector of elements in order of appearance." :examples [["(group-by even? [1 2 3 4 5])" nil {false [1 3 5] true [2 4]}] ["(group-by (partial apply +) [[1 2 3][2 4][1 2]]" nil {6 [[1 2 3] [2 4]] 3 [[1 2]]}]] :signatures [[f coll]] :added "0.1"} + [f coll] (reduce (fn [res elem] (update-in res [(f elem)] (fnil conj []) elem)) {} coll)) ;; TODO: use a transient map in the future -(defn frequencies [coll] +(defn frequencies {:doc "Returns a map with distinct elements as keys and the number of occurences as values" :added "0.1"} + [coll] (reduce (fn [res elem] (update-in res [elem] (fnil inc 0))) {} @@ -1634,17 +1636,18 @@ not enough elements were present." (lazy-seq (cons (take n s) (partition n step (drop step s))))))) -(defn partitionf [f coll] +(defn partitionf {:doc "A generalized version of partition. Instead of taking a constant number of elements, this function calls f with the remaining collection to determine how many elements to take." :examples [["(partitionf first [1 :a, 2 :a b, 3 :a :b :c])" nil ((1 :a) (2 :a :b) (3 :a :b :c))]]} + [f coll] (when-let [s (seq coll)] (lazy-seq (let [n (f s)] (cons (take n s) - (partitionf f (drop n coll))))))) + (partitionf f (drop n s))))))) (defn destructure [binding expr] (cond @@ -2293,7 +2296,7 @@ Expands to calls to `extend-type`." (mapcat walk (children node))))))] (walk root))) -(defn flatten [x] +(defn flatten ; TODO: laziness? {:doc "Takes any nested combination of ISeqable things, and return their contents as a single, flat sequence. @@ -2302,15 +2305,17 @@ Expands to calls to `extend-type`." value as its only element." :examples [["(flatten [[1 2 [3 4] [5 6]] 7])" nil [1 2 3 4 5 6 7]] ["(flatten :this)" nil [:this]]]} + [x] (if (not (satisfies? ISeqable x)) [x] (transduce (comp (map flatten) cat) conj [] (seq x)))) -(defn juxt [& fns] +(defn juxt {:doc "Returns a function that applies all fns to its arguments, and returns a vector of the results." :examples [["((juxt + - *) 2 3)" nil [5 -1 6]]]} + [& fns] (fn [& args] (mapv #(apply % args) fns))) @@ -2326,7 +2331,7 @@ Expands to calls to `extend-type`." ([f col] (transduce (map f) conj col))) -(defn macroexpand-1 [form] +(defn macroexpand-1 {:doc "If form is a macro call, returns the expanded form. Does nothing if not a macro call." @@ -2335,6 +2340,7 @@ Expands to calls to `extend-type`." nil `(if condition (do this and-this))] ["(macroexpand-1 ())" nil ()] ["(macroexpand-1 [1 2])" nil [1 2]]]} + [form] (if (or (not (list? form)) (= () form)) form From 1066bcaaa5df4a50f80ccf0bd03b7ec970cff2f4 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Sat, 4 Apr 2015 21:54:09 +0200 Subject: [PATCH 684/909] Fix a few failing tests. --- pixie/stdlib.pxi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7df603d9..95809f9f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1601,7 +1601,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn group-by {:doc "Groups the collection into a map keyed by the result of applying f on each element. The value at each key is a vector of elements in order of appearance." :examples [["(group-by even? [1 2 3 4 5])" nil {false [1 3 5] true [2 4]}] - ["(group-by (partial apply +) [[1 2 3][2 4][1 2]]" nil {6 [[1 2 3] [2 4]] 3 [[1 2]]}]] + ["(group-by (partial apply +) [[1 2 3] [2 4] [1 2]])" nil {6 [[1 2 3] [2 4]] 3 [[1 2]]}]] :signatures [[f coll]] :added "0.1"} [f coll] @@ -1640,8 +1640,8 @@ not enough elements were present." {:doc "A generalized version of partition. Instead of taking a constant number of elements, this function calls f with the remaining collection to determine how many elements to take." - :examples [["(partitionf first [1 :a, 2 :a b, 3 :a :b :c])" - nil ((1 :a) (2 :a :b) (3 :a :b :c))]]} + :examples [["(partitionf first [2 :a, 3 :a :b, 4 :a :b :c])" + nil ((2 :a) (3 :a :b) (4 :a :b :c))]]} [f coll] (when-let [s (seq coll)] (lazy-seq @@ -2337,7 +2337,7 @@ Expands to calls to `extend-type`." Does nothing if not a macro call." :signatures [[form]] :examples [["(macroexpand-1 '(when condition this and-this))" - nil `(if condition (do this and-this))] + nil (if condition (do this and-this))] ["(macroexpand-1 ())" nil ()] ["(macroexpand-1 [1 2])" nil [1 2]]]} [form] From 073c9a30126de70bbb5581cd4dac0b79065f5dd3 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Sat, 4 Apr 2015 22:09:50 +0200 Subject: [PATCH 685/909] Fix some docstring formatting issues. --- pixie/stdlib.pxi | 37 ++++++++++++++++--------------------- 1 file changed, 16 insertions(+), 21 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 95809f9f..5e66e8e2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2206,11 +2206,11 @@ Expands to calls to `extend-type`." ;; TODO: implement :>> like in Clojure? (defmacro condp "Takes a binary predicate, an expression and a number of two-form clauses. - Calls the predicate on the first value of each clause and the expression. - If the result is truthy returns the second value of the clause. +Calls the predicate on the first value of each clause and the expression. +If the result is truthy returns the second value of the clause. - If the number of arguments is odd and no clause matches, the last argument is returned. - If the number of arguments is even and no clause matches, throws an exception." +If the number of arguments is odd and no clause matches, the last argument is returned. +If the number of arguments is even and no clause matches, throws an exception." [pred-form expr & clauses] (let [x (gensym 'expr), pred (gensym 'pred)] `(let [~x ~expr, ~pred ~pred-form] @@ -2224,13 +2224,13 @@ Expands to calls to `extend-type`." (defmacro case "Takes an expression and a number of two-form clauses. - Checks for each clause if the first part is equal to the expression. - If yes, returns the value of the second part. +Checks for each clause if the first part is equal to the expression. +If yes, returns the value of the second part. - The first part of each clause can also be a set. If that is the case, the clause matches when the result of the expression is in the set. +The first part of each clause can also be a set. If that is the case, the clause matches when the result of the expression is in the set. - If the number of arguments is odd and no clause matches, the last argument is returned. - If the number of arguments is even and no clause matches, throws an exception." +If the number of arguments is odd and no clause matches, the last argument is returned. +If the number of arguments is even and no clause matches, throws an exception." [expr & args] `(condp #(if (set? %1) (%1 %2) (= %1 %2)) ~expr ~@args)) @@ -2285,9 +2285,9 @@ Expands to calls to `extend-type`." (defn tree-seq "Returns a lazy sequence of the nodes in a tree via a depth-first walk. - branch? - fn of node that should true when node has children - children - fn of node that should return a sequence of children (called if branch? true) - root - root node of the tree" +branch? - fn of node that should true when node has children +children - fn of node that should return a sequence of children (called if branch? true) +root - root node of the tree" [branch? children root] (let [walk (fn walk [node] (lazy-seq @@ -2298,11 +2298,9 @@ Expands to calls to `extend-type`." (defn flatten ; TODO: laziness? - {:doc "Takes any nested combination of ISeqable things, and return their contents - as a single, flat sequence. + {:doc "Takes any nested combination of ISeqable things, and return their contents as a single, flat sequence. - Calling this function on something that is not ISeqable returns a seq with that - value as its only element." +Calling this function on something that is not ISeqable returns a seq with that value as its only element." :examples [["(flatten [[1 2 [3 4] [5 6]] 7])" nil [1 2 3 4 5 6 7]] ["(flatten :this)" nil [:this]]]} [x] @@ -2312,8 +2310,7 @@ Expands to calls to `extend-type`." (seq x)))) (defn juxt - {:doc "Returns a function that applies all fns to its arguments, - and returns a vector of the results." + {:doc "Returns a function that applies all fns to its arguments, and returns a vector of the results." :examples [["((juxt + - *) 2 3)" nil [5 -1 6]]]} [& fns] (fn [& args] @@ -2332,9 +2329,7 @@ Expands to calls to `extend-type`." (transduce (map f) conj col))) (defn macroexpand-1 - {:doc "If form is a macro call, returns the expanded form. - - Does nothing if not a macro call." + {:doc "If form is a macro call, returns the expanded form. Does nothing if not a macro call." :signatures [[form]] :examples [["(macroexpand-1 '(when condition this and-this))" nil (if condition (do this and-this))] From 8e9549e7e0f85b9d87a7096bb0beb3aafd4849de Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sat, 4 Apr 2015 23:13:23 +0200 Subject: [PATCH 686/909] ignore all pxics --- .gitignore | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 43e35ec3..d2f96fde 100644 --- a/.gitignore +++ b/.gitignore @@ -4,5 +4,5 @@ pixie-vm .idea lib include -pixie/*.pxic +*.pxic test.tmp From f97295e3324f32792b5a970bbb113e6ce5ed3f30 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Sat, 4 Apr 2015 23:41:45 +0200 Subject: [PATCH 687/909] Fix satisfies? and instance? using `and` and `or` before they are defined. --- pixie/stdlib.pxi | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 34cd0c52..0a1ea972 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -173,8 +173,9 @@ (if (-satisfies? ISeqable t) (let [ts (seq t)] (if (not ts) false - (or (-instance? (first ts) x) - (instance? (rest ts) x)))) + (if (-instance? (first ts) x) + true + (instance? (rest ts) x)))) (-instance? t x)))) (def satisfies? (fn ^{:doc "Checks if x satisfies the protocol p. @@ -186,8 +187,9 @@ (if (-satisfies? ISeqable p) (let [ps (seq p)] (if (not ps) true - (and (-satisfies? (first ps) x) - (satisfies? (rest ps) x)))) + (if (not (-satisfies? (first ps) x)) + false + (satisfies? (rest ps) x)))) (-satisfies? p x)))) (def into (fn ^{:doc "Add the elements of `from` to the collection `to`." From 0d95bb90756c4dbf8e866fc455801fe2e063feab Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 6 Apr 2015 17:34:13 +0200 Subject: [PATCH 688/909] install g++ in the Dockerfile it's a dependency for the FFI stuff. --- Dockerfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index ea969e9a..3da76a00 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM debian:sid # install dependencies RUN apt-get update \ - && apt-get install -y gcc pkg-config make curl bzip2 python2.7 \ + && apt-get install -y gcc g++ pkg-config make curl bzip2 python2.7 \ && apt-get install -y libffi-dev libuv-dev libedit-dev ADD . /usr/src/pixie @@ -12,4 +12,4 @@ RUN cd /usr/src/pixie \ && make PYTHON=python2.7 build_with_jit \ && ln -s /usr/src/pixie/pixie-vm /usr/bin/pxi -ENTRYPOINT ["/usr/bin/pxi"] \ No newline at end of file +ENTRYPOINT ["/usr/bin/pxi"] From 3782dbacf49aac0bfc28c7fd31bfa914c4db39a2 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Mon, 6 Apr 2015 17:58:32 +0200 Subject: [PATCH 689/909] install libboost-dev in the Dockerfile --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 3da76a00..76ca6849 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,7 +2,7 @@ FROM debian:sid # install dependencies RUN apt-get update \ - && apt-get install -y gcc g++ pkg-config make curl bzip2 python2.7 \ + && apt-get install -y gcc g++ libboost-dev pkg-config make curl bzip2 python2.7 \ && apt-get install -y libffi-dev libuv-dev libedit-dev ADD . /usr/src/pixie From 0c9e38b438c1de564aa051e926a492c5f603ab81 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Tue, 7 Apr 2015 20:11:58 +0100 Subject: [PATCH 690/909] remove iterators --- pixie/vm/iterator.py | 76 ---------------- pixie/vm/persistent_hash_map.py | 149 -------------------------------- pixie/vm/persistent_hash_set.py | 8 -- 3 files changed, 233 deletions(-) delete mode 100644 pixie/vm/iterator.py diff --git a/pixie/vm/iterator.py b/pixie/vm/iterator.py deleted file mode 100644 index 73327125..00000000 --- a/pixie/vm/iterator.py +++ /dev/null @@ -1,76 +0,0 @@ -from pixie.vm.object import Object, Type, runtime_error -from pixie.vm.code import extend -from pixie.vm.primitives import true, nil -import pixie.vm.stdlib as proto -import pixie.vm.rt as rt - - -class EmptyIterator(Object): - _type = Type(u"pixie.vm.Iterator") - - def type(self): - return EmptyIterator._type - - - def __init__(self): - pass - - def move_next(self): - runtime_error(u"Empty Iterator") - - def at_end(self): - return True - - -empty_iterator = EmptyIterator() - -@extend(proto._at_end_QMARK_, EmptyIterator) -def _at_end(_): - return true - - -class NativeIterator(Object): - _type = Type(u"pixie.vm.NativeIterator") - - def type(self): - return NativeIterator._type - - def __init__(self): - pass - -@extend(proto._at_end_QMARK_, NativeIterator) -def _at_end(self): - return rt.wrap(self.at_end()) - - -@extend(proto._move_next_BANG_, NativeIterator) -def _move_next(self): - return self.move_next() - - -@extend(proto._current, NativeIterator) -def _current(self): - return self.current() - - -class MapIterator(NativeIterator): - def __init__(self, fn, iter): - self._w_fn = fn - self._w_iter = iter - if not iter.at_end(): - self._w_current = self._w_fn.invoke([self._w_iter.current()]) - - def move_next(self): - self._w_iter.move_next() - - if not self._w_iter.at_end(): - self._w_current = self._w_fn.invoke([self._w_iter.current()]) - else: - self._w_current = nil - - def current(self): - return self._w_current - - def at_end(self): - return self._w_iter.at_end() - diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index b2dc1563..c36dbbdc 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -7,7 +7,6 @@ from rpython.rlib.rarithmetic import r_int, r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt -from pixie.vm.iterator import NativeIterator, empty_iterator MASK_32 = r_uint(0xFFFFFFFF) @@ -59,16 +58,6 @@ def without(self, key): return self return PersistentHashMap(self._cnt - 1, new_root, self._meta) - def iter(self): - if self._root is None: - return empty_iterator - else: - return self._root.iter() - - - - - class INode(object.Object): _type = object.Type(u"pixie.stdlib.INode") @@ -185,9 +174,6 @@ def reduce_inode(self, f, init): return init return init - def iter(self): - return BitmapIndexedNodeIterator(self._array) - def without_inode(self, shift, hash, key): bit = bitpos(hash, shift) if self._bitmap & bit == 0: @@ -216,59 +202,6 @@ def without_inode(self, shift, hash, key): BitmapIndexedNode_EMPTY = BitmapIndexedNode(None, r_uint(0), []) - -class BitmapIndexedNodeIterator(NativeIterator): - def __init__(self, array): - self._array_w = array - self._idx = 0 - self._child_iterator_w = None - self._at_end = False - self._current = nil - self.move_next() - - def move_next(self): - while True: - if self._child_iterator_w is None: - while True: - if self._idx == len(self._array_w): - self._current = nil - self._at_end = True - self._array_w = None - return self - key_or_none = self._array_w[self._idx] - val_or_node = self._array_w[self._idx + 1] - - if key_or_none is not None: - self._idx += 2 - self._current = rt.map_entry(key_or_none, val_or_node) - return self - - elif val_or_node is not None: - iter = val_or_node.iter() - if iter.at_end(): - self._idx += 1 - continue - self._child_iterator_w = iter - self._current = self._child_iterator_w.current() - return self - - self._idx += 2 - else: - self._child_iterator_w.move_next() - if self._child_iterator_w.at_end(): - continue - self._current = self._child_iterator_w.current() - return self - - def at_end(self): - return self._at_end - - def current(self): - return self._current - - - - class ArrayNode(INode): def __init__(self, edit, cnt, array): self._cnt = cnt @@ -345,54 +278,6 @@ def reduce_inode(self, f, init): return init - def iter(self): - return ArrayMapIterator(self._array) - -class ArrayMapIterator(NativeIterator): - def __init__(self, array): - self._array_w = array - self._idx = 0 - self._child_iterator_w = None - self._at_end = False - self._current = nil - self.move_next() - - def move_next(self): - while True: - if self._child_iterator_w is None: - while True: - if self._idx == len(self._array_w): - self._current = nil - self._at_end = True - self._array_w = None - return self - val = self._array_w[self._idx] - if val is not None: - iter = val.iter() - if iter.at_end(): - self._idx += 1 - continue - self._child_iterator_w = iter - self._current = self._child_iterator_w.current() - return self - - self._idx += 1 - else: - self._child_iterator_w.move_next() - if self._child_iterator_w.at_end(): - continue - self._current = self._child_iterator_w.current() - return self - - def at_end(self): - return self._at_end - - def current(self): - return self._current - - - - class HashCollisionNode(INode): def __init__(self, edit, hash, array): self._hash = hash @@ -437,9 +322,6 @@ def reduce_inode(self, f, init): return init return init - def iter(self): - return HashCollisionNodeIterator(self._array) - def find_index(self, key): i = r_int(0) while i < len(self._array): @@ -461,37 +343,6 @@ def without_inode(self, shift, hash, key): return HashCollisionNode(None, self._hash, remove_pair(self._array, r_uint(idx) / 2)) -class HashCollisionNodeIterator(NativeIterator): - def __init__(self, array): - self._w_array = array - self._w_idx = 0 - self._w_current = nil - self.move_next() - - def move_next(self): - while True: - if self._w_idx == len(self._w_array): - self._w_current = nil - self._w_array = None - return self - - key = self._w_array[self._w_idx] - val = self._w_array[self._w_idx + 1] - self._w_idx += 2 - if key is None: - continue - - self._w_current = rt.map_entry(key, val) - return - - def at_end(self): - return self._w_current is None - - def current(self): - return self._w_current - - - def create_node(shift, key1, val1, key2hash, key2, val2): key1hash = rt.hash(key1) & MASK_32 if key1hash == key2hash: diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 2cebe830..9327ea42 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -5,7 +5,6 @@ import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var, intern_var import pixie.vm.rt as rt -from pixie.vm.iterator import MapIterator VAR_KEY = intern_var(u"pixie.stdlib", u"key") @@ -32,9 +31,6 @@ def meta(self): def with_meta(self, meta): return PersistentHashSet(meta, self._map) - def iter(self): - return MapIterator(VAR_KEY.deref(), self._map.iter()) - EMPTY = PersistentHashSet(nil, persistent_hash_map.EMPTY) @as_var("set") @@ -102,7 +98,3 @@ def _meta(self): def _with_meta(self, meta): assert isinstance(self, PersistentHashSet) return self.with_meta(meta) - -@extend(proto._iterator, PersistentHashSet) -def _iterator(self): - return self.iter() From 380340a80df3e4abf30e7528063d51c45fb493ac Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 8 Apr 2015 06:42:09 -0600 Subject: [PATCH 691/909] added basic parser support --- Makefile | 2 +- pixie/parser.pxi | 393 ++++++++++++++++++++++++++++++ pixie/stdlib.pxi | 7 + tests/pixie/tests/test-parser.pxi | 10 + 4 files changed, 411 insertions(+), 1 deletion(-) create mode 100644 pixie/parser.pxi create mode 100644 tests/pixie/tests/test-parser.pxi diff --git a/Makefile b/Makefile index c6a4850a..5cfac9aa 100644 --- a/Makefile +++ b/Makefile @@ -37,7 +37,7 @@ build_preload_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target_preload.py build: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) fetch_externals: $(EXTERNALS)/pypy ./lib diff --git a/pixie/parser.pxi b/pixie/parser.pxi new file mode 100644 index 00000000..d5b8e10c --- /dev/null +++ b/pixie/parser.pxi @@ -0,0 +1,393 @@ +(ns pixie.parser + (require pixie.stdlib :as s)) + + +;; This file contans a small framework for writing generic parsers in Pixie. Although the generated +;; code is probably not the fastest, it is fairly simple, and that simplicity should open the road for +;; future optimizations. The parsers allowed support multiple inheritance and multiple input data types. +;; Backtracking is supported by snapshots taken at key parts of the parsing process. For a string parser +;; these snapshots are simply a integer index into the string being parsed. + +;; Cursors + +(defprotocol ICursor + (next! [this] "Advance to the next element") + (current [this] "Return the current element") + (snapshot [this] "Return a snapshot of the cursor's mutable state") + (rewind! [this snapshot] "Rewind the cursor to a previous snapshot") + (at-end? [this] "Is there more to parse?")) + +(deftype StringCursor [idx s] + ICursor + (next! [this] + (set-field! this :idx (inc idx))) + (current [this] + (when (< idx (count s)) + (nth s idx))) + (snapshot [this] + idx) + (rewind! [this val] + (set-field! this :idx val)) + (at-end? [this] + (= idx (count s)))) + +;; Create a cursor from the given string +(defn string-cursor [s] + (->StringCursor 0 s)) + +;; Mechanics + +(deftype ParseFailure []) + +;; If a parser returns this value, parsing has failed +(def fail (->ParseFailure)) + +(defn failure? + "Returns true if return value from a parser is a parse failure" + [v] + (identical? v fail)) + +(defn parse-if + "Parse and return the current value of the cursor if this predicate succeeds against the cursor. Advances + the cursor to the next element." + [pred] + (fn [cursor] + (if (pred (current cursor)) + (let [value (current cursor)] + (next! cursor) + value) + fail))) + + +(defprotocol IParserGenerator + (to-parser [this] "Convert the current object to a parser")) + +(extend-protocol IParserGenerator + IFn + (to-parser [this] + this) + Character + (to-parser [this] + (parse-if #(= % this)))) + + + + +(defn or + "Defines a parser that succeeds if one of the provided parsers succeeds. Parsers are tried in-order." + ([a] a) + ([a b] + (let [a (to-parser a) + b (to-parser b) + m (atom #{})] + (fn [cursor] + (let [key [cursor (snapshot cursor)]] + (if-let [v (contains? @m key)] + (b cursor) + (let [_ (swap! m conj key) + state (snapshot cursor) + val (a cursor)] + (swap! m disj key) + (if (identical? val fail) + (do (rewind! cursor state) + (b cursor)) + val))))))) + ([a b & more] + (apply or (or a b) more))) + +(defn add-clauses [cursor-sym body [[sym goal] & more]] + (if sym + `(let [~sym (~sym ~cursor-sym)] + (if (identical? ~sym fail) + fail + ~(add-clauses cursor-sym body more))) + body)) + +(defn -parse-args + [args] + (loop [args args + rules [] + return nil] + (let [[arg & rest] args] + (assert (not (= '-> arg)) "invalid position for ->") + (if arg + (if (= '<- arg) + (let [return (first rest)] + (recur (next rest) + rules + return)) + (if (= (first rest) '->) + (let [binding (-> rest next first) + rest (-> rest next next)] + (recur rest + (conj rules [binding arg]) + return)) + (recur rest + (conj rules [(gensym "_") arg]) + return))) + [rules return])))) + +(defmacro and + "Defines a parser that succeeds only if all parsers succeed. Tried in order. Each parser clause can be followed + by a -> to give the parser's output a name. There may also be a single <- followed by any Pixie code that can be used + to post-process the parsed output." + [& args] + (let [[parsed body] (-parse-args args) + cursor-sym (gensym "cursor")] + `(let [~@(mapcat + (fn [[sym parser]] + [sym `(to-parser ~parser)]) + parsed)] + (fn [~cursor-sym] + (let [prev-pos# (snapshot ~cursor-sym) + result# ~(add-clauses cursor-sym body parsed)] + (if (identical? result# fail) + (do (rewind! ~cursor-sym prev-pos#) + fail) + result#)))))) + + +(defprotocol IDeliverable + (-deliver [this val])) + +(deftype PromiseFn [f name] + IDeliverable + (-deliver [this val] + (set-field! this :f val)) + IFn + (-invoke [this val] + (assert f (str "PromiseFN " name " has not been delivered")) + (f val))) + +(defn promise-fn + "Defines a promise that is callable." + [name] + (->PromiseFn nil name)) + +(defmacro parser + "(parser nm inherits & rules) + Defines a new parser named `nm` that inherits from zero or more other parsers defined ion `inherits`. Rules are pairs + of names and rules that will be assigned to those names. Names are inherited from parent parsers in the order they are + defined." + [inherits & rules] + (let [parted (apply merge + (conj (mapv (fn [sym] + (-> sym resolve deref ::forms)) inherits) + (apply hashmap rules))) + rules (apply concat parted) + syms (keys parted)] + `(let [~@(mapcat (fn [s] + `[~s (promise-fn (quote ~s))]) + syms)] + ~@(map (fn [[s goal]] + `(-deliver ~s ~goal)) + parted) + ~(assoc (zipmap (map (comp keyword name) syms) + syms) + ::forms (list 'quote (apply hashmap rules)))))) + +(defmacro defparser + "(defparser nm inherits rules) + Same as parser but assigns the resulting parser to a var with the name nm" + [nm inherits & rules] + `(def ~nm (parser ~inherits ~@rules))) + +;; Common parsers + +(defn char-range + "Defines a parser that parses a numerical range of characters" + [from to] + (parse-if (fn [v] + (when (char? v) + (<= (int from) (int v) (int to)))))) + +(defn one+ + "Defines a parser that succeeds if the given parser succeeds once or more. Will return a vector, but any + reducing function can be provided via rf as well." + ([g] + (one+ g conj)) + ([g rf] + (let [g (to-parser g)] + (fn [cursor] + (loop [acc (rf) + cnt 0] + (let [prev-pos (snapshot cursor) + v (g cursor)] + (if (identical? v fail) + (if (= 0 cnt) + (do (rewind! cursor prev-pos) + fail) + (rf acc)) + (recur (rf acc v) + (inc cnt))))))))) + +(def one+chars #(one+ % string-builder)) + +(defn zero+ + "Defines a parser that succeeds if a given parser succeeds zero or more times. Will return a vector, but + any reducing function can be provided via rf as well." + ([g] + (zero+ g conj)) + ([g rf] + (let [g (to-parser g)] + (fn [cursor] + (loop [acc (rf)] + (let [v (g cursor)] + (if (identical? v fail) + (rf acc) + (recur (rf acc v))))))))) + +(def zero+chars #(zero+ % string-builder)) + +(defn eat + "Eagerly parses as many values as possible until g fails. Discards the result, returns nil." + [g] + (fn [cursor] + (loop [] + (let [prev-pos (snapshot cursor) + v (g cursor)] + (if (identical? v fail) + (do (rewind! cursor prev-pos) + nil) + (recur)))))) + +(defn maybe + "Always succeeds, returns nil when the input did not match the parser." + ([g] + (maybe g nil)) + ([g default] + (let [g (to-parser g)] + (fn [cursor] + (let [v (g cursor)] + (if (failure? v) + default + v)))))) + +(defmacro sequence + [coll arrow body] + (assert (= '<- arrow) "Middle argument to sequence must be a return arrow") + `(and ~@coll ~'<- ~body)) + +(def end + "A parser that only succeeds if there is no more input left to process." + (fn [cursor] + (if (at-end? cursor) + nil + fail))) + +(defn one-of + "Deines a parser that succeeds if the value being parsed is found in v" + [v] + (parse-if (partial contains? v))) + +(def digits (parse-if (set "1234567890"))) + +(def whitespace (parse-if #{\newline \return \space \tab})) + +;; Basic numeric parser. Supports integers (1, 2, 43), decimals (0.1, 1.1, 1000.11) and exponents (1e42, 1E-2) +(defparser NumberParser [] + NUMBER (and (maybe \-) + -> sign + + (or (and + (parse-if (set "123456789")) -> first + (zero+chars digits) -> rest + <- (str first rest)) + (and \0)) + -> integer-digits + + (maybe (and \. + (one+chars digits) -> digits + <- digits)) + -> fraction-digits + + + (maybe (and (parse-if (set "eE")) + (maybe (parse-if (set "-+"))) -> exp-sign + (one+chars digits) -> exp-digits + <- [(s/or exp-sign "") exp-digits])) + -> exp-data + + <- (read-string (str (s/or sign "") + integer-digits + (if fraction-digits (str "." fraction-digits) "") + (if exp-data (apply str "E" exp-data) ""))))) + +(def valid-escape-chars + {\\ \\ + \" \" + \/ \/ + \b \backspace + \f \formfeed + \n \newline + \r \return + \t \tab}) + + +;; Defines a JSON escaped string parser. Supports all the normal \n \f \r stuff as well +;; as \uXXXX unicode characters +(defparser EscapedStringParser [] + CHAR (or (and \\ + (one-of valid-escape-chars) -> char + <- (valid-escape-chars char)) + + (and \\ + \u + digits -> d1 + digits -> d2 + digits -> d3 + digits -> d4 + <- (do + (println [d1 d2 d3 d4]) + (char (read-string (str "0x" d1 d2 d3 d4))))) + + (parse-if #(not= % \"))) + + STRING (and \" + (zero+chars CHAR) -> s + \" + <- s)) + +;; Basic JSON parser +(defparser JSONParser [NumberParser EscapedStringParser] + + NULL (sequence "null" <- nil) + TRUE (sequence "true" <- true) + FALSE (sequence "false" <- false) + ARRAY (and \[ + (eat whitespace) + (zero+ (and ENTRY -> e + (maybe \,) + <- e)) -> items + (eat whitespace) + (eat whitespace) + \] + <- items) + MAP-ENTRY (and (eat whitespace) + STRING -> key + (eat whitespace) + \: + ENTRY -> value + (maybe \,) + <- [key value]) + MAP (and \{ + (zero+ MAP-ENTRY) -> items + (eat whitespace) + \} + <- (apply hashmap (apply concat items))) + ENTRY (and + (eat whitespace) + (or NUMBER MAP STRING NULL TRUE FALSE ARRAY) -> val + (eat whitespace) + <- val) + ENTRY-AT-END (and ENTRY -> e + (eat whitespace) + end + <- e)) + +(defn read-json-string [s] + (let [c (string-cursor s) + result ((:ENTRY-AT-END JSONParser) c)] + (if (failure? result) + (println (current c) (snapshot c)) + result))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6276dd5d..c8a3e506 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -280,6 +280,7 @@ (extend -invoke Code -invoke) (extend -invoke NativeFn -invoke) (extend -invoke VariadicCode -invoke) +(extend -invoke MultiArityFn -invoke) (extend -invoke Closure -invoke) (extend -invoke Var -invoke) (extend -invoke PolymorphicFn -invoke) @@ -2340,3 +2341,9 @@ Expands to calls to `extend-type`." (def *e) (defn -set-*e [e] (def *e e)) + + +(def hash-map hashmap) + +(defn zipmap [a b] + (into {} (map vector a b))) diff --git a/tests/pixie/tests/test-parser.pxi b/tests/pixie/tests/test-parser.pxi new file mode 100644 index 00000000..54f1d580 --- /dev/null +++ b/tests/pixie/tests/test-parser.pxi @@ -0,0 +1,10 @@ +(ns pixie.tests.test-parser + (require pixie.test :refer :all) + (require pixie.parser :refer :all)) + +(deftest test-and + (let [p (parser [] + (and \a \b)) + c (string-cursor "abc")] + (assert= [\a \b] c) + (assert= (current c) \c))) From 6320356ed95ef38aa074d9a21d0f6b21e3f91049 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 8 Apr 2015 20:47:34 +0100 Subject: [PATCH 692/909] add another test to catch iteraters over maps --- tests/pixie/tests/test-stdlib.pxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 3e9b50a4..1b80de5a 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -296,7 +296,9 @@ (t/assert= (seq (filter (fn [x] true) [])) nil) (t/assert= (seq (filter (fn [x] false) [])) nil) (t/assert= (seq (filter (fn [x] true) [1 2 3 4])) '(1 2 3 4)) - (t/assert= (seq (filter (fn [x] false) [1 2 3 4])) nil)) + (t/assert= (seq (filter (fn [x] false) [1 2 3 4])) nil) + (t/assert= (filter (fn [[_ v]] (odd? v)) {:a 1, :b 2, :c 3, :d 4}) + '(1 3))) (t/deftest test-distinct (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) From 774cdfe7ccf591a146ec55f0032b06ed9a6bfb05 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 8 Apr 2015 22:11:27 +0100 Subject: [PATCH 693/909] remove iterators from stdlib --- pixie/stdlib.pxi | 175 ++++++++++-------------------- pixie/vm/stdlib.py | 19 ---- tests/pixie/tests/test-stdlib.pxi | 5 +- 3 files changed, 60 insertions(+), 139 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d5cf4875..c3060254 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -415,7 +415,14 @@ nil (cons (nth self x) (lazy-seq* (fn [] (vector-seq self (+ x 1))))))))) - +(extend -seq String + (fn string-seq + ([self] + (string-seq self 0)) + ([self x] + (if (= x (count self)) + nil + (cons (nth self x) (lazy-seq* (fn [] (string-seq self (+ x 1))))))))) (def concat (fn ^{:doc "Concatenates its arguments." @@ -894,16 +901,6 @@ If further arguments are passed, invokes the method named by symbol, passing the nil [:me :me :me :me]]]} (fn [& _] x)) -(defn some - {:doc "Returns the first true value of the predicate for the elements of the collection." - :signatures [[pred coll]] - :added "0.1"} - [pred coll] - (if-let [coll (seq coll)] - (or (pred (first coll)) - (recur pred (next coll))) - false)) - (extend -count MapEntry (fn [self] 2)) (extend -nth MapEntry (fn map-entry-nth [self idx] (cond (= idx 0) (-key self) @@ -1421,21 +1418,6 @@ The new value is thus `(apply f current-value-of-atom args)`." (do ~@body (recur (inc ~b)))))))) -(extend -iterator PersistentVector - (fn [v] - (dotimes [x (count v)] - (yield (nth v x nil))))) - -(extend -iterator Array - (fn [v] - (dotimes [x (count v)] - (yield (nth v x nil))))) - -(extend -iterator String - (fn [v] - (dotimes [x (count v)] - (yield (nth v x nil))))) - (defmacro and {:doc "Check if the given expressions return truthy values, returning the last, or false." :examples [["(and true false)" nil false] @@ -1483,6 +1465,16 @@ The new value is thus `(apply f current-value-of-atom args)`." ~then) ~else))))) +(defn some + {:doc "Returns the first true value of the predicate for the elements of the collection." + :signatures [[pred coll]] + :added "0.1"} + [pred coll] + (if (seq coll) + (or (pred (first coll)) + (some pred (next coll))) + false)) + (defn nnext {:doc "Equivalent to (next (next coll))" :added "0.1"} @@ -1767,14 +1759,6 @@ For more information, see http://clojure.org/special_forms#binding-forms"} @acc (recur (+ i step) acc))) acc))) - IIterable - (-iterator [self] - (loop [i start] - (when (or (and (> step 0) (< i stop)) - (and (< step 0) (> i stop)) - (and (= step 0))) - (yield i) - (recur (+ i step))))) ICounted (-count [self] (if (or (and (< start stop) (< step 0)) @@ -1816,33 +1800,6 @@ For more information, see http://clojure.org/special_forms#binding-forms"} ([start stop] (->Range start stop 1)) ([start stop step] (->Range start stop step))) -(defn iterator - {:doc "Returns an iterator for the collection." - :added "0.1"} - [coll] - (-iterator coll)) - -(defn move-next! [i] - (-move-next! i) - i) - -(defn at-end? [i] - (-at-end? i)) - -(defn current [i] - (-current i)) - -(defn iterator-seq [i] - (if (at-end? i) - nil - (cons (current i) (lazy-seq (iterator-seq (move-next! i)))))) - -(extend -first IIterator -current) -(extend -iterator IIterator identity) - -(extend -seq IIterator iterator-seq) -(extend -seq IIterable (comp seq iterator)) - (extend -eq ISeqable -seq-eq) (deftype Unknown []) @@ -1863,36 +1820,25 @@ For more information, see http://clojure.org/special_forms#binding-forms"} true self)))) -(extend -reduce ShallowContinuation - (fn [k f init] - (loop [acc init] - (if (reduced? init) - @init - (if (-at-end? k) - acc - (let [acc (f acc (-current k))] - (-move-next! k) - (recur acc))))))) - (defn filter {:doc "Filter the collection for elements matching the predicate." :signatures [[pred] [pred coll]] :added "0.1"} - ([f] (fn [xf] - (fn - ([] (xf)) - ([acc] (xf acc)) - ([acc i] (if (f i) - (xf acc i) - acc))))) - ([f coll] - (let [iter (iterator coll)] - (loop [] - (when (not (at-end? iter)) - (if (f (current iter)) - (yield (current iter))) - (move-next! iter) - (recur)))))) + ([pred] + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (if (pred i) + (xf acc i) + acc))))) + ([pred coll] + (lazy-seq + (when-let [s (seq coll)] + (let [[f & r] s] + (if (pred f) + (cons f (filter pred r)) + (filter pred r))))))) (defn distinct {:doc "Returns the distinct elements in the collection." @@ -1910,33 +1856,35 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (swap! seen conj i) (xf acc i)))))))) ([coll] - (let [iter (iterator coll)] - (loop [acc #{}] - (when (not (at-end? iter)) - (if (contains? acc (current iter)) - (do (move-next! iter) - (recur acc)) - (let [val (current iter)] - (yield val) - (move-next! iter) - (recur (conj acc val))))))))) - + (let [step (fn step [xs seen] + (lazy-seq + ((fn [f seen] + (when-let [s (seq f)] + (let [xs (first s)] + (if (contains? seen xs) + (step (rest s) seen) + (cons xs (step (rest s) (conj seen xs))))))) + xs seen)))] + (step coll #{})))) (defn keep ([f] - (fn [xf] - (fn - ([] (xf)) - ([acc] (xf acc)) - ([acc i] (let [result (f i)] - (if result - (xf acc result) - acc)))))) + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (let [result (f i)] + (if result + (xf acc result) + acc)))))) ([f coll] - (iterate [x coll] - (let [result (f x)] - (if result - (yield result)))))) + (lazy-seq + (when-let [s (seq coll)] + (let [[first & rest] s + result (f first)] + (if result + (cons result (keep f rest)) + (keep f rest))))))) (defn refer {:doc "Refer to the specified vars from a namespace directly. @@ -1991,13 +1939,6 @@ user => (refer 'pixie.string :exclude '(substring))" -(extend -iterator ISeq (fn [s] - (loop [s s] - (when s - (yield (first s)) - (recur (next s)))))) -(extend -at-end? EmptyList (fn [_] true)) - (defn merge-with [f & maps] (cond diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 78eb48bd..1e581eb6 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -57,9 +57,6 @@ defprotocol("pixie.stdlib", "ITransientCollection", ["-conj!"]) defprotocol("pixie.stdlib", "ITransientStack", ["-push!", "-pop!"]) -defprotocol("pixie.stdlib", "IIterable", ["-iterator"]) -defprotocol("pixie.stdlib", "IIterator", ["-current", "-at-end?", "-move-next!"]) - defprotocol("pixie.stdlib", "IDisposable", ["-dispose!"]) @as_var("pixie.stdlib.internal", "-defprotocol") @@ -233,22 +230,6 @@ def __name(self): def __name(_): return nil -@extend(_current, ShallowContinuation) -def _current(self): - assert isinstance(self, ShallowContinuation) - return self._val - -@extend(_at_end_QMARK_, ShallowContinuation) -def _(self): - assert isinstance(self, ShallowContinuation) - return true if self.is_finished() else false - -@extend(_move_next_BANG_, ShallowContinuation) -def _(self): - assert isinstance(self, ShallowContinuation) - self.invoke([nil]) - return self - @returns(r_uint) @as_var("hash") def __hash(x): diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 1b80de5a..dab376dc 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -297,8 +297,8 @@ (t/assert= (seq (filter (fn [x] false) [])) nil) (t/assert= (seq (filter (fn [x] true) [1 2 3 4])) '(1 2 3 4)) (t/assert= (seq (filter (fn [x] false) [1 2 3 4])) nil) - (t/assert= (filter (fn [[_ v]] (odd? v)) {:a 1, :b 2, :c 3, :d 4}) - '(1 3))) + (t/assert= (into {} (filter (fn [[_ v]] (odd? v)) {:a 1, :b 2, :c 3, :d 4})) + {:a 1 :c 3})) (t/deftest test-distinct (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) @@ -346,7 +346,6 @@ (t/deftest test-range (t/assert= (= (-seq (range 10)) - (-seq (-iterator (range 10))) '(0 1 2 3 4 5 6 7 8 9)) true)) From fe9bf9a98e65ed711c98dec973c4ba467b102092 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 8 Apr 2015 20:50:23 -0600 Subject: [PATCH 694/909] fixed a bunch of issues with error messages --- pixie/vm/compiler.py | 5 ++++- pixie/vm/libs/pxic/reader.py | 5 +++-- pixie/vm/libs/pxic/writer.py | 1 - pixie/vm/object.py | 21 +++++++++++++++++++++ pixie/vm/stdlib.py | 30 +++++++++++++++++++++++++----- 5 files changed, 53 insertions(+), 9 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index c915214a..8165b486 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -78,7 +78,10 @@ def __init__(self, name, argc, parent_ctx): self._max_sp = 0 self.can_tail_call = False self.closed_overs = [] - self.name = name + if parent_ctx: + self.name = parent_ctx.name + u"_" + name + else: + self.name = name self.recur_points = [] self.debug_points = {} diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index b4ec9cbb..571cbddc 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -55,7 +55,8 @@ def read_raw_integer(rdr): return r_uint(ord(rdr.read()[0]) | (ord(rdr.read()[0]) << 8) | (ord(rdr.read()[0]) << 16) | (ord(rdr.read()[0]) << 24)) def read_raw_string(rdr): - return rdr.read_cached_string() + s = rdr.read_cached_string() + return s def read_code(rdr): sz = read_raw_integer(rdr) @@ -129,7 +130,6 @@ def read_interpreter_code_info(rdr): line_number = read_raw_integer(rdr) column_number = read_raw_integer(rdr) file = read_raw_string(rdr) - return InterpreterCodeInfo(line, intmask(line_number), intmask(column_number), file) def read_obj(rdr): @@ -151,6 +151,7 @@ def read_obj(rdr): return symbol(read_raw_string(rdr)) elif tag == LINE_PROMISE: lp = LinePromise() + lp._chrs = None lp._str = read_raw_string(rdr) return lp elif tag == MAP: diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index ecd8865d..8f1318ec 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -231,7 +231,6 @@ def write_namespace(o, wtr): def write_interpreter_code_info(obj, wtr): line, line_number, column_number, file = obj.interpreter_code_info_state() - write_tag(CODE_INFO, wtr) write_object(line, wtr) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 32bdd199..ad95b463 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -255,3 +255,24 @@ def trace_map(self): tm = {keyword(u"type") : keyword(u"pixie")} tm[keyword(u"name")] = String(self._name) return tm + +class ExtraCodeInfo(ErrorInfo): + def __init__(self, str): + self._str = str + + def __repr__(self): + return self._str + + def trace_map(self): + import pixie.vm.rt as rt + from pixie.vm.keyword import keyword + + tm = {keyword(u"type"): keyword(u"extra"), + keyword(u"data"): rt.wrap(self._str)} + return tm + + +def add_info(ex, data): + assert isinstance(ex, WrappedException) + ex._ex._trace.append(ExtraCodeInfo(data)) + return ex \ No newline at end of file diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 78eb48bd..f5702385 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from pixie.vm.object import Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ - runtime_error + runtime_error, add_info from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ list_copy, returns, intern_var import pixie.vm.code as code @@ -506,12 +506,32 @@ def load_reader(rdr): form = reader.read(rdr, False) if form is reader.eof: return nil - compiled = compiler.compile(form) - if pxic_writer is not None: - pxic_writer.write_object(compiled) + try: + compiled = compiler.compile(form) + + except WrappedException as ex: + meta = rt.meta(form) + if meta is not nil: + ci = rt.interpreter_code_info(meta) + add_info(ex, ci.__repr__()) + add_info(ex, u"Compiling: " + rt.name(rt.str(form))) + raise ex + + try: + if pxic_writer is not None: + pxic_writer.write_object(compiled) + + compiled.invoke([]) + + except WrappedException as ex: + meta = rt.meta(form) + if meta is not nil: + ci = rt.interpreter_code_info(meta) + add_info(ex, ci.__repr__()) + add_info(ex, u"Running: " + rt.name(rt.str(form))) + raise ex - compiled.invoke([]) if not we_are_translated(): print "done" From ce45c1c04d15c47dd492e69b880b0a7fc1580dac Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 9 Apr 2015 06:36:02 -0600 Subject: [PATCH 695/909] fixed up function names and added a few more debug helpers --- Makefile | 2 +- pixie/stdlib.pxi | 7 +++---- pixie/vm/compiler.py | 18 ++++++++++-------- 3 files changed, 14 insertions(+), 13 deletions(-) diff --git a/Makefile b/Makefile index c6a4850a..a39ea229 100644 --- a/Makefile +++ b/Makefile @@ -28,7 +28,7 @@ build_no_jit: fetch_externals compile_basics: @echo -e "\n\n\n\nWARNING: Compiling core libs. If you want to modify one of these files delete the .pxic files first\n\n\n\n" - ./pixie-vm -c pixie/uv.pxi -c pixie/io.pxi -c pixie/stacklets.pxi -c pixie/stdlib.pxi + ./pixie-vm -c pixie/uv.pxi -c pixie/io.pxi -c pixie/stacklets.pxi -c pixie/stdlib.pxi -c pixie/repl.pxi build_preload_with_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index d5cf4875..185ce1c3 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2034,9 +2034,8 @@ The following two forms are allowed: The params can be destructuring bindings, see `(doc let)` for details."} [& decls] - (let [name (if (symbol? (first decls)) (first decls) nil) + (let [name (if (symbol? (first decls)) [(first decls)] nil) decls (if name (next decls) decls) - name (or name '-fn) decls (cond (vector? (first decls)) (list decls) ;(satisfies? ISeqable (first decls)) decls @@ -2059,8 +2058,8 @@ The params can be destructuring bindings, see `(doc let)` for details."} ~@body))))) decls))] (if (= (count decls) 1) - `(fn* ~name ~(first (first decls)) ~@(next (first decls))) - `(fn* ~name ~@decls)))) + `(fn* ~@name ~(first (first decls)) ~@(next (first decls))) + `(fn* ~@name ~@decls)))) (deftype MultiMethod [dispatch-fn default-val methods] IFn diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 8165b486..f314ac1c 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -78,8 +78,8 @@ def __init__(self, name, argc, parent_ctx): self._max_sp = 0 self.can_tail_call = False self.closed_overs = [] - if parent_ctx: - self.name = parent_ctx.name + u"_" + name + if name == default_fn_name and parent_ctx: + self.name = parent_ctx.name + u"_fn" else: self.name = name self.recur_points = [] @@ -459,7 +459,10 @@ def compile_platform_plus(form, ctx): def add_args(name, args, ctx): required_args = -1 local_idx = 0 - ctx.add_local(name, Self()) + + if name != default_fn_name: + ctx.add_local(name, Self()) + for x in range(rt.count(args)): arg = rt.nth(args, rt.wrap(x)) affirm(isinstance(arg, symbol.Symbol), u"Argument names must be symbols") @@ -471,6 +474,7 @@ def add_args(name, args, ctx): local_idx += 1 return required_args +default_fn_name = u"some_long_name_unlikely_to_be_used" def compile_fn(form, ctx): form = rt.next(form) @@ -478,7 +482,7 @@ def compile_fn(form, ctx): name = rt.first(form) form = rt.next(form) else: - name = symbol.symbol(u"-fn") + name = symbol.symbol(default_fn_name) @@ -511,9 +515,7 @@ def compile_fn_body(name, args, body, ctx): required_args = add_args(rt.name(name), args, new_ctx) bc = 0 - if name is not None: - affirm(isinstance(name, symbol.Symbol), u"Function names must be symbols") - #new_ctx.add_local(name._str, Self()) + affirm(isinstance(name, symbol.Symbol), u"Function names must be symbols") arg_syms = EMPTY for x in range(rt.count(args)): @@ -847,7 +849,7 @@ def compile_fn_call(form, ctx): def compile(form): - ctx = Context(u"main", 0, None) + ctx = Context(u"_toplevel_", 0, None) compile_form(form, ctx) ctx.bytecode.append(code.RETURN) return ctx.to_code() From 8f7368ab1da5598b3002dff2806380efba33612f Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Thu, 9 Apr 2015 20:59:05 +0100 Subject: [PATCH 696/909] Add a few extra uv functions --- pixie/uv.pxi | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 76417f47..210a63bd 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -27,9 +27,18 @@ (f/defcfn uv_update_time) (f/defcfn uv_walk) + (f/defccallback uv_alloc_cb) + (f/defcstruct uv_connect_t [:handle]) + (f/defcstruct uv_stream_t []) + + (f/defcfn uv_read_start) (f/defccallback uv_read_cb) + (f/defcfn uv_write) + (f/defcstruct uv_write_t []) + (f/defccallback uv_write_cb) + ;; Timer @@ -184,9 +193,11 @@ (f/defcfn uv_tcp_bind) (f/defcfn uv_listen) (f/defcfn uv_accept) - (f/defcfn uv_read_start) + (f/defcfn uv_tcp_connect) + (f/defcfn uv_tcp_keepalive) (f/defccallback uv_connection_cb) + (f/defccallback uv_connect_cb) ) From 3597a8191e4ef1399cfe85a9bdbee86096d3f626 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 9 Apr 2015 16:09:40 -0600 Subject: [PATCH 697/909] added tests and data support for add-exception-info --- pixie/vm/object.py | 9 +++++++-- pixie/vm/stdlib.py | 9 ++++++++- tests/pixie/tests/test-errors.pxi | 15 +++++++++++++++ 3 files changed, 30 insertions(+), 3 deletions(-) create mode 100644 tests/pixie/tests/test-errors.pxi diff --git a/pixie/vm/object.py b/pixie/vm/object.py index ad95b463..99123bd3 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -257,8 +257,9 @@ def trace_map(self): return tm class ExtraCodeInfo(ErrorInfo): - def __init__(self, str): + def __init__(self, str, data=None): self._str = str + self._data = data def __repr__(self): return self._str @@ -268,7 +269,11 @@ def trace_map(self): from pixie.vm.keyword import keyword tm = {keyword(u"type"): keyword(u"extra"), - keyword(u"data"): rt.wrap(self._str)} + keyword(u"msg"): rt.wrap(self._str)} + + if self._data: + tm[keyword(u"data")] = self._data + return tm diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 487d9c8c..cd9a283d 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,6 +1,6 @@ # -*- coding: utf-8 -*- from pixie.vm.object import Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ - runtime_error, add_info + runtime_error, add_info, ExtraCodeInfo from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ list_copy, returns, intern_var import pixie.vm.code as code @@ -865,3 +865,10 @@ def _set_current_var_frames(self, frames): Sets the current var frames. Frames should be a cons list of hashmaps containing mappings of vars to dynamic values. Setting this value to anything but this data format will cause undefined errors.""" code._dynamic_vars.set_current_frames(frames) + +@as_var("add-exception-info") +def _add_exception_info(ex, str, data): + affirm(isinstance(ex, RuntimeException), u"First argument must be an exception") + assert isinstance(ex, RuntimeException) + ex._trace.append(ExtraCodeInfo(rt.name(str), data)) + return ex diff --git a/tests/pixie/tests/test-errors.pxi b/tests/pixie/tests/test-errors.pxi new file mode 100644 index 00000000..1ef095b5 --- /dev/null +++ b/tests/pixie/tests/test-errors.pxi @@ -0,0 +1,15 @@ +(ns pixie.test.test-errors + (:require [pixie.test :refer :all])) + +(deftest test-add-exception-info + (try + (try + (+ 1 "foo") + (catch ex + (throw (add-exception-info ex "My Msg" :my-data)))) + (catch ex + (let [filter-fn (fn [mp] + (and (= (:type mp) :extra) + (= (:msg mp) "My Msg") + (= (:data mp) :my-data)))] + (assert= 1 (count (filter filter-fn ex))))))) From d388d1bd25d74dc6e58702a709ebcdc15ea3a883 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Fri, 10 Apr 2015 16:46:25 -0600 Subject: [PATCH 698/909] moved json code into its own namespace --- pixie/parsers/json.pxi | 111 ++++++++++++++++++++++++ pixie/stdlib.pxi | 38 ++++---- pixie/test.pxi | 7 ++ tests/pixie/tests/parsers/test-json.pxi | 11 +++ 4 files changed, 148 insertions(+), 19 deletions(-) create mode 100644 pixie/parsers/json.pxi create mode 100644 tests/pixie/tests/parsers/test-json.pxi diff --git a/pixie/parsers/json.pxi b/pixie/parsers/json.pxi new file mode 100644 index 00000000..c870a8c7 --- /dev/null +++ b/pixie/parsers/json.pxi @@ -0,0 +1,111 @@ +(ns pixie.parsers.json + (:require [pixie.parser :refer :all] + [pixie.stdlib :as std])) + + + +;; Basic numeric parser. Supports integers (1, 2, 43), decimals (0.1, 1.1, 1000.11) and exponents (1e42, 1E-2) +(defparser NumberParser [] + NUMBER (and (maybe \-) + -> sign + + (or (and + (parse-if (set "123456789")) -> first + (zero+chars digits) -> rest + <- (str first rest)) + (and \0)) + -> integer-digits + + (maybe (and \. + (one+chars digits) -> digits + <- digits)) + -> fraction-digits + + + (maybe (and (parse-if (set "eE")) + (maybe (parse-if (set "-+"))) -> exp-sign + (one+chars digits) -> exp-digits + <- [(std/or exp-sign "") exp-digits])) + -> exp-data + + <- (std/read-string (str (std/or sign "") + integer-digits + (if fraction-digits (str "." fraction-digits) "") + (if exp-data (apply str "E" exp-data) ""))))) + +(def valid-escape-chars + {\\ \\ + \" \" + \/ \/ + \b \backspace + \f \formfeed + \n \newline + \r \return + \t \tab}) + + +;; Defines a JSON escaped string parser. Supports all the normal \n \f \r stuff as well +;; as \uXXXX unicode characters +(defparser EscapedStringParser [] + CHAR (or (and \\ + (one-of valid-escape-chars) -> char + <- (valid-escape-chars char)) + + (and \\ + \u + digits -> d1 + digits -> d2 + digits -> d3 + digits -> d4 + <- (char (std/read-string (str "0x" d1 d2 d3 d4)))) + + (parse-if #(not= % \"))) + + STRING (and \" + (zero+chars CHAR) -> s + \" + <- s)) + +;; Basic JSON parser +(defparser JSONParser [NumberParser EscapedStringParser] + + NULL (sequence "null" <- nil) + TRUE (sequence "true" <- true) + FALSE (sequence "false" <- false) + ARRAY (and \[ + (eat whitespace) + (zero+ (and ENTRY -> e + (maybe \,) + <- e)) -> items + (eat whitespace) + (eat whitespace) + \] + <- items) + MAP-ENTRY (and (eat whitespace) + STRING -> key + (eat whitespace) + \: + ENTRY -> value + (maybe \,) + <- [key value]) + MAP (and \{ + (zero+ MAP-ENTRY) -> items + (eat whitespace) + \} + <- (apply hashmap (apply concat items))) + ENTRY (and + (eat whitespace) + (or NUMBER MAP STRING NULL TRUE FALSE ARRAY) -> val + (eat whitespace) + <- val) + ENTRY-AT-END (and ENTRY -> e + (eat whitespace) + end + <- e)) + +(defn read-string [s] + (let [c (string-cursor s) + result ((:ENTRY-AT-END JSONParser) c)] + (if (failure? result) + (println (current c) (snapshot c)) + result))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 5ca0ef2f..900bdbb3 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -310,7 +310,6 @@ (extend -reduce Nil (fn [self f init] init)) (extend -hash Nil (fn [self] 100000)) (extend -with-meta Nil (fn [self _] nil)) -(extend -at-end? Nil (fn [_] true)) (extend -deref Nil (fn [_] nil)) (extend -contains-key Nil (fn [_ _] false)) @@ -398,13 +397,6 @@ 1111111 3333333))) -(def stacklet->lazy-seq - (fn [k] - (if (-at-end? k) - nil - (cons (-current k) - (lazy-seq* (fn [] (stacklet->lazy-seq (-move-next! k)))))))) - (def = -eq) (extend -seq PersistentVector @@ -1393,17 +1385,6 @@ The new value is thus `(apply f current-value-of-atom args)`." nil ~(nth binding 1 nil))) -(defmacro iterate [binding & body] - (assert (= 2 (count binding)) "binding and collection required") - `(let [i# (iterator ~(second binding))] - (loop [] - (if (at-end? i#) - nil - (let [~(first binding) (current i#)] - ~@body - (move-next! i#) - (recur)))))) - (defmacro dotimes {:doc "Execute the expressions in the body n times." @@ -2314,3 +2295,22 @@ Calling this function on something that is not ISeqable returns a seq with that (fn [v] (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) + + +(defn interleave + "Returns a seq of all the items in the input collections interleaved" + ([] ()) + ([c1] (seq c1)) + ([c1 c2] + (lazy-seq + (let [s1 (seq c1) + s2 (seq c2)] + (when (and s1 s2) + (cons (first s1) (cons (first s2) + (interleave (next s1) (next s2)))))))) + ([& colls] + (lazy-seq + (let [ss (map seq colls)] + (when (every? identity ss) + (concat (map first ss) + (apply interleave (map next ss)))))))) diff --git a/pixie/test.pxi b/pixie/test.pxi index 745efa00..2f622977 100644 --- a/pixie/test.pxi +++ b/pixie/test.pxi @@ -92,3 +92,10 @@ (if (= orig res) (pr-str orig) (str (pr-str orig) " = " (pr-str res))))) + + +(defmacro assert-table [pattern expr & vals] + (let [parted (partition (count pattern) vals)] + `(do ~@(for [fact parted] + `(let [~@(interleave pattern fact)] + ~expr))))) diff --git a/tests/pixie/tests/parsers/test-json.pxi b/tests/pixie/tests/parsers/test-json.pxi new file mode 100644 index 00000000..c1d8133d --- /dev/null +++ b/tests/pixie/tests/parsers/test-json.pxi @@ -0,0 +1,11 @@ +(ns pixie.tests.parsers.test-json + (:require [pixie.test :refer :all] + [pixie.parsers.json :as json])) + + + +(deftest test-json-numbers + (assert-table [x y] (assert= (json/read-string x) y) + "1" 1 + "1.0" 1.0 + "0.1" 0.1)) From a2bb0a1b90dffce6e1741dcd70f6fbefc2f34728 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Fri, 10 Apr 2015 23:48:21 +0100 Subject: [PATCH 699/909] Remove some lingering iterator stuff --- pixie/stdlib.pxi | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ee40a1b6..83bda7c0 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -309,7 +309,6 @@ (extend -reduce Nil (fn [self f init] init)) (extend -hash Nil (fn [self] 100000)) (extend -with-meta Nil (fn [self _] nil)) -(extend -at-end? Nil (fn [_] true)) (extend -deref Nil (fn [_] nil)) (extend -contains-key Nil (fn [_ _] false)) @@ -397,13 +396,6 @@ 1111111 3333333))) -(def stacklet->lazy-seq - (fn [k] - (if (-at-end? k) - nil - (cons (-current k) - (lazy-seq* (fn [] (stacklet->lazy-seq (-move-next! k)))))))) - (def = -eq) (extend -seq PersistentVector @@ -1392,18 +1384,6 @@ The new value is thus `(apply f current-value-of-atom args)`." nil ~(nth binding 1 nil))) -(defmacro iterate [binding & body] - (assert (= 2 (count binding)) "binding and collection required") - `(let [i# (iterator ~(second binding))] - (loop [] - (if (at-end? i#) - nil - (let [~(first binding) (current i#)] - ~@body - (move-next! i#) - (recur)))))) - - (defmacro dotimes {:doc "Execute the expressions in the body n times." :examples [["(dotimes [i 3] (println i))" "1\n2\n3\n"]] From 05ca6cc89b52ca07e55e4ac02a8a3d2c7a3fb6df Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Tue, 31 Mar 2015 20:31:55 +0100 Subject: [PATCH 700/909] Implement fs/size and fs/permissions --- pixie/fs.pxi | 60 ++++++++++++++++++++++++++--- tests/pixie/tests/fs/parent/foo.txt | 1 + tests/pixie/tests/test-fs.pxi | 18 +++++++++ 3 files changed, 73 insertions(+), 6 deletions(-) diff --git a/pixie/fs.pxi b/pixie/fs.pxi index 048e6f63..36bec55d 100644 --- a/pixie/fs.pxi +++ b/pixie/fs.pxi @@ -1,7 +1,19 @@ (ns pixie.fs (:require [pixie.path :as path] - [pixie.string :as string])) + [pixie.string :as string] + [pixie.ffi-infer :as f])) +(f/with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["sys/stat.h"]} + (f/defc-raw-struct stat [:st_mode :st_size])) + +(def stat-struct stat) + +(f/with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["sys/stat.h"]} + (f/defcfn stat)) (defprotocol IFSPath (path [this] @@ -19,15 +31,16 @@ (basename [this] "Returns the basename of the Filesystem Object") - ;; TODO (permissions [this] "Returns a string of the octal permissions") - (mounted? [this] - "Returns true if the directory is a mounted") - (size [this] - "Returns the size of the file/dir on disk")) + "Returns the size of the file/dir on disk") + + ;; TODO + + (mounted? [this] + "Returns true if the directory is a mounted")) (defprotocol IFile (extension [this] @@ -80,6 +93,29 @@ :else (apply str (interpose "/" (concat (repeat (count diff-b) "..") diff-a))))))) +(defn- assert-existence [f] + (assert (exists? f) (str "No file or directory at \"" (abs f) "\""))) + +(defn- read-stat-field [path field] + (assert-existence path) + (let [s (stat-struct) + _ (stat (abs path) s) + result (field s)] + (dispose! s) + result)) + +(defn- size-of-path [path] + (read-stat-field path :st_size)) + +(defn- permission-string [n] + (apply str + (for [shift [6 3 0]] + (let [mask (bit-shift-left 7 shift) + masked (bit-and mask n)] + (bit-shift-right masked shift))))) + +(defn- permissions-of-path [path] + (permission-string (read-stat-field path :st_mode))) ;; File and Dir are just wrappers around paths. (deftype File [pathz] @@ -99,6 +135,12 @@ (basename [this] (last (string/split (abs this) "/"))) + (size [this] + (size-of-path this)) + + (permissions [this] + (permissions-of-path this)) + IFile ;; TODO: Sort out regex or make strings partitionable. So we can split at ;; #".". @@ -141,6 +183,12 @@ (basename [this] (last (string/split (abs this) "/"))) + (size [this] + (size-of-path this)) + + (permissions [this] + (permissions-of-path pathz)) + IDir (list [this] (vec (map fspath (path/-list-dir pathz)))) diff --git a/tests/pixie/tests/fs/parent/foo.txt b/tests/pixie/tests/fs/parent/foo.txt index e69de29b..257cc564 100644 --- a/tests/pixie/tests/fs/parent/foo.txt +++ b/tests/pixie/tests/fs/parent/foo.txt @@ -0,0 +1 @@ +foo diff --git a/tests/pixie/tests/test-fs.pxi b/tests/pixie/tests/test-fs.pxi index 5a8ba3b2..c02d664a 100644 --- a/tests/pixie/tests/test-fs.pxi +++ b/tests/pixie/tests/test-fs.pxi @@ -105,3 +105,21 @@ (t/assert= (fs/exists? real-dir) true) (t/assert= (fs/exists? fake-dir) false) (t/assert= (fs/exists? fake-file) false))) + +(t/deftest test-size + (let [file-with-content (fs/file "tests/pixie/tests/fs/parent/foo.txt") + file-without-content (fs/file "tests/pixie/tests/fs/parent/bar.txt") + fake-file (fs/file "tests/pixie/tests/fs/parent/fake-file")] + (t/assert= (fs/size file-with-content) 4) + (t/assert= (fs/size file-without-content) 0) + (t/assert-throws? (fs/size fake-file)))) + +(t/deftest test-permissions + (let [file (fs/file "tests/pixie/tests/fs/parent/foo.txt") + fake-file (fs/file "tests/pixie/tests/fs/parent/fake-file")] + ; because Travis seems to change the permissions of some files... + (let [perms (fs/permissions file)] + (t/assert= (count perms) 3) + (t/assert (instance? String perms)) + (t/assert= (set (seq perms)) #{\6 \4})) + (t/assert-throws? (fs/permissions fake-file)))) From 35ac3a67e6761e3392a1f0a5810ab849f3974bdc Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Fri, 10 Apr 2015 22:48:43 +0100 Subject: [PATCH 701/909] Implement size and permissions using libuv --- pixie/fs.pxi | 29 +++++++---------------------- pixie/io.pxi | 26 ++++---------------------- pixie/uv.pxi | 28 ++++++++++++++++++++++++++-- 3 files changed, 37 insertions(+), 46 deletions(-) diff --git a/pixie/fs.pxi b/pixie/fs.pxi index 36bec55d..7a943857 100644 --- a/pixie/fs.pxi +++ b/pixie/fs.pxi @@ -1,19 +1,7 @@ (ns pixie.fs (:require [pixie.path :as path] [pixie.string :as string] - [pixie.ffi-infer :as f])) - -(f/with-config {:library "c" - :cxx-flags ["-lc"] - :includes ["sys/stat.h"]} - (f/defc-raw-struct stat [:st_mode :st_size])) - -(def stat-struct stat) - -(f/with-config {:library "c" - :cxx-flags ["-lc"] - :includes ["sys/stat.h"]} - (f/defcfn stat)) + [pixie.uv :as uv])) (defprotocol IFSPath (path [this] @@ -96,16 +84,12 @@ (defn- assert-existence [f] (assert (exists? f) (str "No file or directory at \"" (abs f) "\""))) -(defn- read-stat-field [path field] - (assert-existence path) - (let [s (stat-struct) - _ (stat (abs path) s) - result (field s)] - (dispose! s) - result)) +(uv/defuvfsfn fs-size pixie.uv/uv_fs_stat [file] :statbuf.st_size) +(uv/defuvfsfn fs-mode pixie.uv/uv_fs_stat [file] :statbuf.st_mode) (defn- size-of-path [path] - (read-stat-field path :st_size)) + (assert-existence path) + (fs-size (abs path))) (defn- permission-string [n] (apply str @@ -115,7 +99,8 @@ (bit-shift-right masked shift))))) (defn- permissions-of-path [path] - (permission-string (read-stat-field path :st_mode))) + (assert-existence path) + (permission-string (fs-mode (abs path)))) ;; File and Dir are just wrappers around paths. (deftype File [pathz] diff --git a/pixie/io.pxi b/pixie/io.pxi index aa868411..dabeac54 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -6,28 +6,10 @@ [pixie.ffi :as ffi] [pixie.ffi-infer :as ffi-infer])) -(defmacro defuvfsfn [nm args return] - `(defn ~nm ~args - (let [f (fn [k#] - (let [cb# (atom nil)] - (reset! cb# (ffi/ffi-prep-callback uv/uv_fs_cb - (fn [req#] - (try - (st/run-and-process k# (~return (pixie.ffi/cast req# uv/uv_fs_t))) - (uv/uv_fs_req_cleanup req#) - (-dispose! @cb#) - (catch e (println e)))))) - (~(symbol (str "pixie.uv/uv_" (name nm))) - (uv/uv_default_loop) - (uv/uv_fs_t) - ~@args - @cb#)))] - (st/call-cc f)))) - -(defuvfsfn fs_open [path flags mode] :result) -(defuvfsfn fs_read [file bufs nbufs offset] :result) -(defuvfsfn fs_write [file bufs nbufs offset] :result) -(defuvfsfn fs_close [file] :result) +(uv/defuvfsfn fs_open [path flags mode] :result) +(uv/defuvfsfn fs_read [file bufs nbufs offset] :result) +(uv/defuvfsfn fs_write [file bufs nbufs offset] :result) +(uv/defuvfsfn fs_close [file] :result) (def DEFAULT-BUFFER-SIZE 1024) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 210a63bd..792d81a4 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -1,5 +1,6 @@ (ns pixie.uv - (:require [pixie.ffi-infer :as f])) + (:require [pixie.ffi :as ffi] + [pixie.ffi-infer :as f])) (f/with-config {:library "uv" :includes ["uv.h"]} @@ -58,7 +59,9 @@ :fs_type :path :result - :ptr]) + :ptr + :statbuf.st_size + :statbuf.st_mode]) (f/defcstruct uv_timespec_t [:tv_sec :tv_nsec]) (f/defcstruct uv_stat_t [:st_dev @@ -214,3 +217,24 @@ (if (neg? result) (throw (str "UV Error: " (uv_err_name result))) result)) + +(defmacro defuvfsfn + ([nm args return] + (defuvfsfn nm (symbol (str "pixie.uv/uv_" (name nm))) args return)) + ([nm uv-fn args return] + `(defn ~nm ~args + (let [f (fn [k#] + (let [cb# (atom nil)] + (reset! cb# (ffi/ffi-prep-callback uv_fs_cb + (fn [req#] + (try + (pixie.stacklets/run-and-process k# (~return (pixie.ffi/cast req# uv_fs_t))) + (uv_fs_req_cleanup req#) + (-dispose! @cb#) + (catch e (println e)))))) + (~uv-fn + (uv_default_loop) + (uv_fs_t) + ~@args + @cb#)))] + (pixie.stacklets/call-cc f))))) From 8f251d03ad725dd7795adcda60bcbf387ae19dad Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Mon, 13 Apr 2015 19:23:02 +0100 Subject: [PATCH 702/909] Add a way to get the size of a CStructType --- pixie/vm/libs/ffi.py | 11 +++++++++++ tests/pixie/tests/test-ffi.pxi | 3 +++ 2 files changed, 14 insertions(+) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 0d346472..6fbc53a1 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -550,6 +550,9 @@ def get_type(self, nm): return tp + def get_size(self): + return self._size + def cast_to(self, frm): return CStruct(self, frm.raw_data()) @@ -768,6 +771,14 @@ def c_cast(frm, to): return to.cast_to(frm) +@as_var("pixie.ffi", "struct-size") +def struct_size(tp): + """(struct-size tp) + Gives the size of the given CStruct type tp, in bytes.""" + if not isinstance(tp, CStructType): + runtime_error(u"Expected a CStruct type to get the size of, got " + rt.name(rt.str(tp))) + + return rt.wrap(tp.get_size()) @extend(proto._val_at, CStructType.base_type) def val_at(self, k, not_found): diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index 8ea60649..beaaa079 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -55,3 +55,6 @@ (dotimes [x (dec MAX)] (t/assert (> (pixie.ffi/unpack buf x CUInt8) (pixie.ffi/unpack buf (inc x) CUInt8))))))) + +(t/deftest test-size + (t/assert= (pixie.ffi/struct-size (pixie.ffi/c-struct "struct" 1234 [])) 1234)) From 8025418a9416b49a563183a6d3cba77c80fdd938 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 13 Apr 2015 16:00:18 -0600 Subject: [PATCH 703/909] more tests for the json parser --- pixie/parsers/json.pxi | 2 +- tests/pixie/tests/parsers/test-json.pxi | 24 +++++++++++++++++++++++- 2 files changed, 24 insertions(+), 2 deletions(-) diff --git a/pixie/parsers/json.pxi b/pixie/parsers/json.pxi index c870a8c7..56a21989 100644 --- a/pixie/parsers/json.pxi +++ b/pixie/parsers/json.pxi @@ -13,7 +13,7 @@ (parse-if (set "123456789")) -> first (zero+chars digits) -> rest <- (str first rest)) - (and \0)) + (and \0 <- "0")) -> integer-digits (maybe (and \. diff --git a/tests/pixie/tests/parsers/test-json.pxi b/tests/pixie/tests/parsers/test-json.pxi index c1d8133d..3d03979d 100644 --- a/tests/pixie/tests/parsers/test-json.pxi +++ b/tests/pixie/tests/parsers/test-json.pxi @@ -8,4 +8,26 @@ (assert-table [x y] (assert= (json/read-string x) y) "1" 1 "1.0" 1.0 - "0.1" 0.1)) + "0.1" 0.1 + "1.1" 1.1 + "1234.5678" 1234.5678 + + "-1" -1 + "-0.1" -0.1 + "-1.1" -1.1 + "-1234.5678" -1234.5678 + "1e1" 1e1)) + +(deftest test-vectors + (assert-table [x y] (assert= (json/read-string x) y) + "[]" [] + "[null]" [nil] + "[1, 2]" [1 2] + "[1, 1.0, null]" [1 1.0 nil] + "[\"foo\", 42]" ["foo" 42])) + +(deftest test-maps + (assert-table [x y] (assert= (json/read-string x) y) + "{\"foo\": 42}" {"foo", 42} + "{\"foo\": 42, \"bar\":null}" {"foo" 42 + "bar" nil})) From 57ea3d9235aa5548fc166ab8e857490f40f0f569 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 13 Apr 2015 20:48:08 -0600 Subject: [PATCH 704/909] start of tcp support --- pixie/io/tcp.pxi | 41 +++++++++++++++++++++++++++++++ pixie/stacklets.pxi | 13 ++++++++++ tests/pixie/tests/io/test-tcp.pxi | 6 +++++ 3 files changed, 60 insertions(+) create mode 100644 pixie/io/tcp.pxi create mode 100644 tests/pixie/tests/io/test-tcp.pxi diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi new file mode 100644 index 00000000..c32c9d62 --- /dev/null +++ b/pixie/io/tcp.pxi @@ -0,0 +1,41 @@ +(ns pixie.io.tcp + (:require [pixie.stacklets :as st] + [pixie.uv :as uv] + [pixie.ffi :as ffi])) + +(defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb]) + +(defrecord TCPStream [uv-client]) + +(defn launch-tcp-client-from-server [svr] + (assert (instance? TCPServer svr) "Requires a TCPServer as the first argument") + (let [client (uv/uv_tcp_t)] + (uv/uv_tcp_init (uv/uv_default_loop) client) + (if (= 0 (uv/uv_accept (:uv-server svr) client)) + (do (st/spawn-from-non-stacklet #((:on-connect svr) + (->TCPStream client))) + svr) + (do (uv/uv_close client nil) + svr)))) + + + +(defn tcp-server [ip port on-connection] + (assert (string? ip) "Ip should be a string") + (assert (integer? port) "Port should be a int") + + (let [server (uv/uv_tcp_t) + bind-addr (uv/sockaddr_in) + _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) + on-new-connetion (atom nil) + tcp-server (->TCPServer ip port on-connection server bind-addr on-new-connetion)] + (reset! on-new-connetion + (ffi/ffi-prep-callback + uv/uv_connection_cb + (fn [server status] + (launch-tcp-client-from-server tcp-server)))) + (uv/uv_tcp_init (uv/uv_default_loop) server) + (uv/uv_tcp_bind server bind-addr 0) + (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) + (st/yield-control) + tcp-server)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index c3d2b236..378c46cf 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -93,6 +93,19 @@ (println e))))))) +(defn spawn-from-non-stacklet [f] + (let [s (new-stacklet (fn [h _] + (try + (reset! stacklet-loop-h h) + (swap! running-threads inc) + (f) + (swap! running-threads dec) + (call-cc (fn [_] nil)) + (catch e + (println e)))))] + (-run-later + (fn [] + (run-and-process s))))) (defn -with-stacklets [fn] diff --git a/tests/pixie/tests/io/test-tcp.pxi b/tests/pixie/tests/io/test-tcp.pxi new file mode 100644 index 00000000..84a9848d --- /dev/null +++ b/tests/pixie/tests/io/test-tcp.pxi @@ -0,0 +1,6 @@ +(ns pixie.test.io.test-tcp + (:require [pixie.io.tcp :refer :all] + [pixie.async :as async])) + + +(tcp-server "0.0.0.0" 4242 #(println "FOOOOO " %)) From 61bea2f9b49028620881ccefa0e40c700818b656 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 14 Apr 2015 05:57:58 -0600 Subject: [PATCH 705/909] make everything inherit from Object --- pixie/vm/object.py | 13 +++++++++---- tests/pixie/tests/test-object.pxi | 9 +++++++++ 2 files changed, 18 insertions(+), 4 deletions(-) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 99123bd3..e986bbbe 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -67,14 +67,18 @@ def get_type_by_name(nm): return _type_registry.get_by_name(nm) class Type(Object): - def __init__(self, name, parent = None): + def __init__(self, name, parent=None, object_inited=True): assert isinstance(name, unicode), u"Type names must be unicode" _type_registry.register_type(name, self) self._name = name - self._parent = parent - if parent is not None: + + if object_inited: + if parent is None: + parent = Object._type + parent.add_subclass(self) + self._parent = parent self._subclasses = [] def name(self): @@ -89,7 +93,8 @@ def add_subclass(self, tp): def subclasses(self): return self._subclasses -Type._type = Type(u"Type") +Object._type = Type(u"pixie.stdlib.Object", None, False) +Type._type = Type(u"pixie.stdlib.Type") @jit.elidable_promote() def istypeinstance(obj, t): diff --git a/tests/pixie/tests/test-object.pxi b/tests/pixie/tests/test-object.pxi index 810470b0..3a805042 100644 --- a/tests/pixie/tests/test-object.pxi +++ b/tests/pixie/tests/test-object.pxi @@ -4,3 +4,12 @@ (t/deftest test-hash (t/assert= (hash (var foo)) (hash (var foo))) (t/assert (not= (hash (var foo)) (hash (var bar))))) + + +(deftype FooType []) + +(t/deftest test-everything-is-an-object + (t/assert (instance? Object 42)) + (t/assert (instance? Object [])) + (t/assert (instance? Object "Foo")) + (t/assert (instance? Object FooType))) From 64f7cbc1da3521791c29b0dde10e41d6e71d85ff Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 14 Apr 2015 06:20:56 -0600 Subject: [PATCH 706/909] removed some duplicate code, moved parsers.* namespaces to parser.* --- pixie/parser.pxi | 109 +---------------------- pixie/parsers/json.pxi | 111 ------------------------ tests/pixie/tests/parsers/test-json.pxi | 33 ------- 3 files changed, 2 insertions(+), 251 deletions(-) delete mode 100644 pixie/parsers/json.pxi delete mode 100644 tests/pixie/tests/parsers/test-json.pxi diff --git a/pixie/parser.pxi b/pixie/parser.pxi index 7718bd27..e1b12a26 100644 --- a/pixie/parser.pxi +++ b/pixie/parser.pxi @@ -198,7 +198,8 @@ [inherits & rules] (let [parted (apply merge (conj (mapv (fn [sym] - (-> sym resolve deref ::forms)) inherits) + (::forms (deref (resolve-in *ns* sym)))) + inherits) (-parse-parser-args rules))) rules (apply concat parted) syms (keys parted)] @@ -309,109 +310,3 @@ (def digits (parse-if (set "1234567890"))) (def whitespace (parse-if #{\newline \return \space \tab})) - -;; Basic numeric parser. Supports integers (1, 2, 43), decimals (0.1, 1.1, 1000.11) and exponents (1e42, 1E-2) -(defparser NumberParser [] - NUMBER (and (maybe \-) - -> sign - - (or (and - (parse-if (set "123456789")) -> first - (zero+chars digits) -> rest - <- (str first rest)) - (and \0)) - -> integer-digits - - (maybe (and \. - (one+chars digits) -> digits - <- digits)) - -> fraction-digits - - - (maybe (and (parse-if (set "eE")) - (maybe (parse-if (set "-+"))) -> exp-sign - (one+chars digits) -> exp-digits - <- [(s/or exp-sign "") exp-digits])) - -> exp-data - - <- (read-string (str (s/or sign "") - integer-digits - (if fraction-digits (str "." fraction-digits) "") - (if exp-data (apply str "E" exp-data) ""))))) - -(def valid-escape-chars - {\\ \\ - \" \" - \/ \/ - \b \backspace - \f \formfeed - \n \newline - \r \return - \t \tab}) - - -;; Defines a JSON escaped string parser. Supports all the normal \n \f \r stuff as well -;; as \uXXXX unicode characters -(defparser EscapedStringParser [] - CHAR (or (and \\ - (one-of valid-escape-chars) -> char - <- (valid-escape-chars char)) - - (and \\ - \u - digits -> d1 - digits -> d2 - digits -> d3 - digits -> d4 - <- (char (read-string (str "0x" d1 d2 d3 d4)))) - - (parse-if #(not= % \"))) - - STRING (and \" - (zero+chars CHAR) -> s - \" - <- s)) - -;; Basic JSON parser -(defparser JSONParser [NumberParser EscapedStringParser] - - NULL (sequence "null" <- nil) - TRUE (sequence "true" <- true) - FALSE (sequence "false" <- false) - ARRAY (and \[ - (eat whitespace) - (zero+ (and ENTRY -> e - (maybe \,) - <- e)) -> items - (eat whitespace) - (eat whitespace) - \] - <- items) - MAP-ENTRY (and (eat whitespace) - STRING -> key - (eat whitespace) - \: - ENTRY -> value - (maybe \,) - <- [key value]) - MAP (and \{ - (zero+ MAP-ENTRY) -> items - (eat whitespace) - \} - <- (apply hashmap (apply concat items))) - ENTRY (and - (eat whitespace) - (or NUMBER MAP STRING NULL TRUE FALSE ARRAY) -> val - (eat whitespace) - <- val) - ENTRY-AT-END (and ENTRY -> e - (eat whitespace) - end - <- e)) - -(defn read-json-string [s] - (let [c (string-cursor s) - result ((:ENTRY-AT-END JSONParser) c)] - (if (failure? result) - (println (current c) (snapshot c)) - result))) diff --git a/pixie/parsers/json.pxi b/pixie/parsers/json.pxi deleted file mode 100644 index 56a21989..00000000 --- a/pixie/parsers/json.pxi +++ /dev/null @@ -1,111 +0,0 @@ -(ns pixie.parsers.json - (:require [pixie.parser :refer :all] - [pixie.stdlib :as std])) - - - -;; Basic numeric parser. Supports integers (1, 2, 43), decimals (0.1, 1.1, 1000.11) and exponents (1e42, 1E-2) -(defparser NumberParser [] - NUMBER (and (maybe \-) - -> sign - - (or (and - (parse-if (set "123456789")) -> first - (zero+chars digits) -> rest - <- (str first rest)) - (and \0 <- "0")) - -> integer-digits - - (maybe (and \. - (one+chars digits) -> digits - <- digits)) - -> fraction-digits - - - (maybe (and (parse-if (set "eE")) - (maybe (parse-if (set "-+"))) -> exp-sign - (one+chars digits) -> exp-digits - <- [(std/or exp-sign "") exp-digits])) - -> exp-data - - <- (std/read-string (str (std/or sign "") - integer-digits - (if fraction-digits (str "." fraction-digits) "") - (if exp-data (apply str "E" exp-data) ""))))) - -(def valid-escape-chars - {\\ \\ - \" \" - \/ \/ - \b \backspace - \f \formfeed - \n \newline - \r \return - \t \tab}) - - -;; Defines a JSON escaped string parser. Supports all the normal \n \f \r stuff as well -;; as \uXXXX unicode characters -(defparser EscapedStringParser [] - CHAR (or (and \\ - (one-of valid-escape-chars) -> char - <- (valid-escape-chars char)) - - (and \\ - \u - digits -> d1 - digits -> d2 - digits -> d3 - digits -> d4 - <- (char (std/read-string (str "0x" d1 d2 d3 d4)))) - - (parse-if #(not= % \"))) - - STRING (and \" - (zero+chars CHAR) -> s - \" - <- s)) - -;; Basic JSON parser -(defparser JSONParser [NumberParser EscapedStringParser] - - NULL (sequence "null" <- nil) - TRUE (sequence "true" <- true) - FALSE (sequence "false" <- false) - ARRAY (and \[ - (eat whitespace) - (zero+ (and ENTRY -> e - (maybe \,) - <- e)) -> items - (eat whitespace) - (eat whitespace) - \] - <- items) - MAP-ENTRY (and (eat whitespace) - STRING -> key - (eat whitespace) - \: - ENTRY -> value - (maybe \,) - <- [key value]) - MAP (and \{ - (zero+ MAP-ENTRY) -> items - (eat whitespace) - \} - <- (apply hashmap (apply concat items))) - ENTRY (and - (eat whitespace) - (or NUMBER MAP STRING NULL TRUE FALSE ARRAY) -> val - (eat whitespace) - <- val) - ENTRY-AT-END (and ENTRY -> e - (eat whitespace) - end - <- e)) - -(defn read-string [s] - (let [c (string-cursor s) - result ((:ENTRY-AT-END JSONParser) c)] - (if (failure? result) - (println (current c) (snapshot c)) - result))) diff --git a/tests/pixie/tests/parsers/test-json.pxi b/tests/pixie/tests/parsers/test-json.pxi deleted file mode 100644 index 3d03979d..00000000 --- a/tests/pixie/tests/parsers/test-json.pxi +++ /dev/null @@ -1,33 +0,0 @@ -(ns pixie.tests.parsers.test-json - (:require [pixie.test :refer :all] - [pixie.parsers.json :as json])) - - - -(deftest test-json-numbers - (assert-table [x y] (assert= (json/read-string x) y) - "1" 1 - "1.0" 1.0 - "0.1" 0.1 - "1.1" 1.1 - "1234.5678" 1234.5678 - - "-1" -1 - "-0.1" -0.1 - "-1.1" -1.1 - "-1234.5678" -1234.5678 - "1e1" 1e1)) - -(deftest test-vectors - (assert-table [x y] (assert= (json/read-string x) y) - "[]" [] - "[null]" [nil] - "[1, 2]" [1 2] - "[1, 1.0, null]" [1 1.0 nil] - "[\"foo\", 42]" ["foo" 42])) - -(deftest test-maps - (assert-table [x y] (assert= (json/read-string x) y) - "{\"foo\": 42}" {"foo", 42} - "{\"foo\": 42, \"bar\":null}" {"foo" 42 - "bar" nil})) From e976478213aab3ed3e5a6672e6cea6fb948434b4 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 14 Apr 2015 22:12:23 -0600 Subject: [PATCH 707/909] tcp server works, needs quite a bit of cleanup --- pixie/io/tcp.pxi | 59 +++++++++++++++++++++++++++++-- pixie/stdlib.pxi | 41 +++++++++++++++++++-- pixie/uv.pxi | 5 ++- tests/pixie/tests/io/test-tcp.pxi | 11 +++++- 4 files changed, 110 insertions(+), 6 deletions(-) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index c32c9d62..081aa480 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -1,11 +1,66 @@ (ns pixie.io.tcp (:require [pixie.stacklets :as st] + [pixie.streams :refer [IInputStream read IOutputStream write]] [pixie.uv :as uv] [pixie.ffi :as ffi])) (defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb]) -(defrecord TCPStream [uv-client]) +(defn -prep-uv-buffer-fn [buf read-bytes] + (ffi/ffi-prep-callback + uv/uv_alloc_cb + (fn [handle suggested-size uv-buf] + (try + (let [casted (ffi/cast uv-buf uv/uv_buf_t)] + (println "Alloc " handle suggested-size buf read-bytes) + (ffi/set! casted :base buf) + (ffi/set! casted :len (min suggested-size + (buffer-capacity buf) + read-bytes))) + (catch ex (println ex)))))) + +(deftype TCPStream [uv-client uv-write-buf] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [alloc-cb (-prep-uv-buffer-fn buffer len) + read-cb (atom nil)] + (st/call-cc (fn [k] + (reset! read-cb (ffi/ffi-prep-callback + uv/uv_read_cb + (fn [stream nread uv-buf] + (println "-<<< nread <-- " nread) + (set-buffer-count! buffer nread) + (try + (dispose! alloc-cb) + (dispose! @read-cb) + ;(dispose! uv-buf) + (uv/uv_read_stop stream) + (st/run-and-process k nread) + (catch ex + (println ex)))))) + (uv/uv_read_start uv-client alloc-cb @read-cb))))) + IOutputStream + (write [this buffer] + (let [write-cb (atom nil) + uv_write (uv/uv_write_t)] + (println "writing " (count buffer)) + (ffi/set! uv-write-buf :base buffer) + (ffi/set! uv-write-buf :len (count buffer)) + (st/call-cc + (fn [k] + (reset! write-cb (ffi/ffi-prep-callback + uv/uv_write_cb + (fn [req status] + (println status "<<-- status") + (try + (dispose! @write-cb) + ;(uv/uv_close uv_write st/close_cb) + (st/run-and-process k status) + (catch ex + (println ex)))))) + (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb)))))) (defn launch-tcp-client-from-server [svr] (assert (instance? TCPServer svr) "Requires a TCPServer as the first argument") @@ -13,7 +68,7 @@ (uv/uv_tcp_init (uv/uv_default_loop) client) (if (= 0 (uv/uv_accept (:uv-server svr) client)) (do (st/spawn-from-non-stacklet #((:on-connect svr) - (->TCPStream client))) + (->TCPStream client (uv/uv_buf_t)))) svr) (do (uv/uv_close client nil) svr)))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 83bda7c0..ef4683f2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -282,6 +282,7 @@ (extend -invoke Code -invoke) (extend -invoke NativeFn -invoke) (extend -invoke VariadicCode -invoke) +(extend -invoke MultiArityFn -invoke) (extend -invoke Closure -invoke) (extend -invoke Var -invoke) (extend -invoke PolymorphicFn -invoke) @@ -1804,7 +1805,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} {:doc "Filter the collection for elements matching the predicate." :signatures [[pred] [pred coll]] :added "0.1"} - ([pred] + ([pred] (fn [xf] (fn ([] (xf)) @@ -1858,7 +1859,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (xf acc result) acc)))))) ([f coll] - (lazy-seq + (lazy-seq (when-let [s (seq coll)] (let [[first & rest] s result (f first)] @@ -2279,6 +2280,11 @@ Calling this function on something that is not ISeqable returns a seq with that (defn -set-*e [e] (def *e e)) +(def hash-map hashmap) + +(defn zipmap [a b] + (into {} (map vector a b))) + (extend -str Environment (fn [v] (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] @@ -2288,3 +2294,34 @@ Calling this function on something that is not ISeqable returns a seq with that (fn [v] (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) + +(defn interleave + "Returns a seq of all the items in the input collections interleaved" + ([] ()) + ([c1] (seq c1)) + ([c1 c2] + (lazy-seq + (let [s1 (seq c1) + s2 (seq c2)] + (when (and s1 s2) + (cons (first s1) (cons (first s2) + (interleave (next s1) (next s2)))))))) + ([& colls] + (lazy-seq + (let [ss (map seq colls)] + (when (every? identity ss) + (concat (map first ss) + (apply interleave (map next ss)))))))) + + +(defn min + "Returns the smallest of all the arguments to this function. Assumes arguments are numeric" + ([x] x) + ([x y] (if (< x y) x y)) + ([x y & zs] (apply min (min x y) zs))) + +(defn max + "Returns the largest of all the arguments to this function. Assumes arguments are numeric" + ([x] x) + ([x y] (if (> x y) x y)) + ([x y & zs] (apply min (min x y) zs))) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 792d81a4..474901bb 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -198,11 +198,14 @@ (f/defcfn uv_accept) (f/defcfn uv_tcp_connect) (f/defcfn uv_tcp_keepalive) + (f/defcfn uv_read_start) + (f/defcfn uv_read_stop) (f/defccallback uv_connection_cb) (f/defccallback uv_connect_cb) - ) + (f/defccallback uv_alloc_cb) + (f/defccallback uv_read_cb)) (defn new-fs-buf [size] diff --git a/tests/pixie/tests/io/test-tcp.pxi b/tests/pixie/tests/io/test-tcp.pxi index 84a9848d..fadb3f64 100644 --- a/tests/pixie/tests/io/test-tcp.pxi +++ b/tests/pixie/tests/io/test-tcp.pxi @@ -1,6 +1,15 @@ (ns pixie.test.io.test-tcp (:require [pixie.io.tcp :refer :all] + [pixie.io :refer [read write]] + [pixie.stacklets :as st] [pixie.async :as async])) +(defn on-client [conn] + (let [b (buffer 1024)] + (read conn b 1024) + (write conn b) + (dotimes [x 1000] + (st/yield-control)) + (println "Done Writing.."))) -(tcp-server "0.0.0.0" 4242 #(println "FOOOOO " %)) +(tcp-server "0.0.0.0" 4242 on-client) From 0cccf33e7dda91adedff80df03d8e7af50c1d142 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 16 Apr 2015 15:43:32 -0600 Subject: [PATCH 708/909] implemented a basic tcp server and tests --- pixie/io.pxi | 43 ++++++++++++----------------- pixie/io/tcp.pxi | 45 ++++++++++++++++++++++++++----- pixie/stacklets.pxi | 11 +++++++- pixie/stdlib.pxi | 2 +- pixie/streams.pxi | 3 +++ pixie/uv.pxi | 2 ++ tests/pixie/tests/io/test-tcp.pxi | 45 ++++++++++++++++++++++++------- 7 files changed, 106 insertions(+), 45 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index dabeac54..20dd2d64 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -111,6 +111,11 @@ IDisposable (-dispose! [this] (set-buffer-count! buffer idx) + (write downstream buffer)) + IFlushableStream + (flush [this] + (set-buffer-count! buffer idx) + (set-field! this :idx 0) (write downstream buffer))) (deftype BufferedInputStream [upstream idx buffer] @@ -124,16 +129,21 @@ val)) IDisposable (-dispose! [this] - (dispose! upstream) (dispose! buffer))) -(defn buffered-output-stream [downstream size] - (->BufferedOutputStream downstream 0 (buffer size))) +(defn buffered-output-stream + ([downstream] + (buffered-output-stream downstream DEFAULT-BUFFER-SIZE)) + ([downstream size] + (->BufferedOutputStream downstream 0 (buffer size)))) -(defn buffered-input-stream [upstream size] - (let [b (buffer size)] - (set-buffer-count! b size) - (->BufferedInputStream upstream size b))) +(defn buffered-input-stream + ([upstream] + (buffered-input-stream upstream DEFAULT-BUFFER-SIZE)) + ([upstream size] + (let [b (buffer size)] + (set-buffer-count! b size) + (->BufferedInputStream upstream size b)))) (defn throw-on-error [result] (when (neg? result) @@ -180,25 +190,6 @@ (defn run-command [command] (st/apply-blocking io-blocking/run-command command)) -(comment - - (defn tcp-server [ip port on-connection] - (assert (string? ip) "Ip should be a string") - (assert (integer? port) "Port should be a int") - (let [server (uv/uv_tcp_t) - bind-addr (uv/sockaddr_in) - _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) - on-new-connetion (atom nil)] - (reset! on-new-connetion - (ffi/ffi-prep-callback - uv/uv_connection_cb - (fn [server status] - (when (not (= status -1)) - (println "Got Client!!!!!!!"))))) - (uv/uv_tcp_init (uv/uv_default_loop) server) - (uv/uv_tcp_bind server bind-addr 0) - (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) - (st/yield-control)))) (comment (st/apply-blocking println "FROM OTHER THREAD <---!!!!!") diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index 081aa480..95614f7e 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -4,7 +4,12 @@ [pixie.uv :as uv] [pixie.ffi :as ffi])) -(defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb]) +(defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb] + IDisposable + (-dispose! [this] + (uv/uv_close uv-server st/close_cb) + (dispose! @on-connection-cb) + (dispose! bind-addr))) (defn -prep-uv-buffer-fn [buf read-bytes] (ffi/ffi-prep-callback @@ -12,7 +17,6 @@ (fn [handle suggested-size uv-buf] (try (let [casted (ffi/cast uv-buf uv/uv_buf_t)] - (println "Alloc " handle suggested-size buf read-bytes) (ffi/set! casted :base buf) (ffi/set! casted :len (min suggested-size (buffer-capacity buf) @@ -30,14 +34,15 @@ (reset! read-cb (ffi/ffi-prep-callback uv/uv_read_cb (fn [stream nread uv-buf] - (println "-<<< nread <-- " nread) (set-buffer-count! buffer nread) (try (dispose! alloc-cb) (dispose! @read-cb) ;(dispose! uv-buf) (uv/uv_read_stop stream) - (st/run-and-process k nread) + (st/run-and-process k (or + (st/exception-on-uv-error nread) + nread)) (catch ex (println ex)))))) (uv/uv_read_start uv-client alloc-cb @read-cb))))) @@ -45,7 +50,6 @@ (write [this buffer] (let [write-cb (atom nil) uv_write (uv/uv_write_t)] - (println "writing " (count buffer)) (ffi/set! uv-write-buf :base buffer) (ffi/set! uv-write-buf :len (count buffer)) (st/call-cc @@ -53,14 +57,17 @@ (reset! write-cb (ffi/ffi-prep-callback uv/uv_write_cb (fn [req status] - (println status "<<-- status") (try (dispose! @write-cb) ;(uv/uv_close uv_write st/close_cb) (st/run-and-process k status) (catch ex (println ex)))))) - (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb)))))) + (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) + IDisposable + (-dispose! [this] + (dispose! uv-write-buf) + (uv/uv_close uv-client st/close_cb))) (defn launch-tcp-client-from-server [svr] (assert (instance? TCPServer svr) "Requires a TCPServer as the first argument") @@ -94,3 +101,27 @@ (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) (st/yield-control) tcp-server)) + + +(defn tcp-client [ip port] + (let [client-addr (uv/sockaddr_in) + uv-connect (uv/uv_connect_t) + client (uv/uv_tcp_t) + cb (atom nil)] + (uv/throw-on-error (uv/uv_ip4_addr ip port client-addr)) + (uv/uv_tcp_init (uv/uv_default_loop) client) + (st/call-cc (fn [k] + (reset! cb (ffi/ffi-prep-callback + uv/uv_connect_cb + (fn [_ status] + (try + (dispose! @cb) + (dispose! uv-connect) + (dispose! client-addr) + (st/run-and-process k (or (st/exception-on-uv-error status) + (->TCPStream client (uv/uv_buf_t)))) + (catch ex + (println ex)))))) + (uv/uv_tcp_connect uv-connect client client-addr @cb))) + + )) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 378c46cf..30e0ada3 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -16,6 +16,8 @@ ;; Yield +(defrecord ThrowException [ex]) + (defn run-and-process ([k] (run-and-process k nil)) @@ -23,12 +25,19 @@ (let [[h f] (k val)] (f h)))) +(defn exception-on-uv-error [result] + (when (neg? result) + (->ThrowException (str "UV Error: " (uv/uv_err_name result))))) + + (defn call-cc [f] (let [frames (-get-current-var-frames nil) [h val] (@stacklet-loop-h f)] (reset! stacklet-loop-h h) (-set-current-var-frames nil frames) - val)) + (if (instance? ThrowException val) + (throw (:ex val)) + val))) (defn -run-later [f] (let [a (uv/uv_async_t) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ef4683f2..3478b185 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2324,4 +2324,4 @@ Calling this function on something that is not ISeqable returns a seq with that "Returns the largest of all the arguments to this function. Assumes arguments are numeric" ([x] x) ([x y] (if (> x y) x y)) - ([x y & zs] (apply min (min x y) zs))) + ([x y & zs] (apply max (max x y) zs))) diff --git a/pixie/streams.pxi b/pixie/streams.pxi index b39fa4f3..1bdf358f 100644 --- a/pixie/streams.pxi +++ b/pixie/streams.pxi @@ -1,5 +1,8 @@ (ns pixie.streams) +(defprotocol IFlushableStream + (flush [this] "Flushes all buffers in this stream and applies writes to any parent streams")) + (defprotocol IInputStream (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 474901bb..67f93cdd 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -124,6 +124,8 @@ (f/defconst UV_DIRENT_CHAR) (f/defconst UV_DIRENT_BLOCK) + (f/defconst UV_EOF) + (f/defcstruct uv_dirent_t [:name :type]) diff --git a/tests/pixie/tests/io/test-tcp.pxi b/tests/pixie/tests/io/test-tcp.pxi index fadb3f64..a86439a2 100644 --- a/tests/pixie/tests/io/test-tcp.pxi +++ b/tests/pixie/tests/io/test-tcp.pxi @@ -1,15 +1,40 @@ (ns pixie.test.io.test-tcp (:require [pixie.io.tcp :refer :all] - [pixie.io :refer [read write]] + [pixie.io :refer [buffered-input-stream buffered-output-stream read-byte write-byte]] + [pixie.streams :refer :all] [pixie.stacklets :as st] - [pixie.async :as async])) + [pixie.async :as async] + [pixie.uv :as uv] + [pixie.test :refer :all])) -(defn on-client [conn] - (let [b (buffer 1024)] - (read conn b 1024) - (write conn b) - (dotimes [x 1000] - (st/yield-control)) - (println "Done Writing.."))) +(deftest test-echo-server + (let [client-done (async/promise) + on-client (fn on-client [conn] + (let [in (buffered-input-stream conn) + out (buffered-output-stream conn)] + (try + (loop [] + (let [val (read-byte in)] + (write-byte out val) + (flush out) + (recur))) + (catch ex + (dispose! in) + (dispose! out) -(tcp-server "0.0.0.0" 4242 on-client) + (dispose! conn) + (client-done true))))) + + server (tcp-server "0.0.0.0" 4242 on-client)] + + (let [client-stream (tcp-client "127.0.0.1" 4242) + in (buffered-input-stream client-stream) + out (buffered-output-stream client-stream)] + + (dotimes [x 255] + (write-byte out x) + (flush out) + (assert= x (read-byte in))) + (dispose! client-stream) + (assert @client-done) + (dispose! server)))) From fb414d9c20877247cec29499dc635c232d052aa0 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 16 Apr 2015 15:44:02 -0600 Subject: [PATCH 709/909] move json parser, don't delete it --- pixie/parser/json.pxi | 111 +++++++++++++++++++++++++ tests/pixie/tests/parser/test-json.pxi | 33 ++++++++ 2 files changed, 144 insertions(+) create mode 100644 pixie/parser/json.pxi create mode 100644 tests/pixie/tests/parser/test-json.pxi diff --git a/pixie/parser/json.pxi b/pixie/parser/json.pxi new file mode 100644 index 00000000..321d71c1 --- /dev/null +++ b/pixie/parser/json.pxi @@ -0,0 +1,111 @@ +(ns pixie.parser.json + (:require [pixie.parser :refer :all] + [pixie.stdlib :as std])) + + + +;; Basic numeric parser. Supports integers (1, 2, 43), decimals (0.1, 1.1, 1000.11) and exponents (1e42, 1E-2) +(defparser NumberParser [] + NUMBER (and (maybe \-) + -> sign + + (or (and + (parse-if (set "123456789")) -> first + (zero+chars digits) -> rest + <- (str first rest)) + (and \0 <- "0")) + -> integer-digits + + (maybe (and \. + (one+chars digits) -> digits + <- digits)) + -> fraction-digits + + + (maybe (and (parse-if (set "eE")) + (maybe (parse-if (set "-+"))) -> exp-sign + (one+chars digits) -> exp-digits + <- [(std/or exp-sign "") exp-digits])) + -> exp-data + + <- (std/read-string (str (std/or sign "") + integer-digits + (if fraction-digits (str "." fraction-digits) "") + (if exp-data (apply str "E" exp-data) ""))))) + +(def valid-escape-chars + {\\ \\ + \" \" + \/ \/ + \b \backspace + \f \formfeed + \n \newline + \r \return + \t \tab}) + + +;; Defines a JSON escaped string parser. Supports all the normal \n \f \r stuff as well +;; as \uXXXX unicode characters +(defparser EscapedStringParser [] + CHAR (or (and \\ + (one-of valid-escape-chars) -> char + <- (valid-escape-chars char)) + + (and \\ + \u + digits -> d1 + digits -> d2 + digits -> d3 + digits -> d4 + <- (char (std/read-string (str "0x" d1 d2 d3 d4)))) + + (parse-if #(not= % \"))) + + STRING (and \" + (zero+chars CHAR) -> s + \" + <- s)) + +;; Basic JSON parser +(defparser JSONParser [NumberParser EscapedStringParser] + + NULL (sequence "null" <- nil) + TRUE (sequence "true" <- true) + FALSE (sequence "false" <- false) + ARRAY (and \[ + (eat whitespace) + (zero+ (and ENTRY -> e + (maybe \,) + <- e)) -> items + (eat whitespace) + (eat whitespace) + \] + <- items) + MAP-ENTRY (and (eat whitespace) + STRING -> key + (eat whitespace) + \: + ENTRY -> value + (maybe \,) + <- [key value]) + MAP (and \{ + (zero+ MAP-ENTRY) -> items + (eat whitespace) + \} + <- (apply hashmap (apply concat items))) + ENTRY (and + (eat whitespace) + (or NUMBER MAP STRING NULL TRUE FALSE ARRAY) -> val + (eat whitespace) + <- val) + ENTRY-AT-END (and ENTRY -> e + (eat whitespace) + end + <- e)) + +(defn read-string [s] + (let [c (string-cursor s) + result ((:ENTRY-AT-END JSONParser) c)] + (if (failure? result) + (println (current c) (snapshot c)) + result))) diff --git a/tests/pixie/tests/parser/test-json.pxi b/tests/pixie/tests/parser/test-json.pxi new file mode 100644 index 00000000..e79ef6cd --- /dev/null +++ b/tests/pixie/tests/parser/test-json.pxi @@ -0,0 +1,33 @@ +(ns pixie.tests.parser.test-json + (:require [pixie.test :refer :all] + [pixie.parser.json :as json])) + + + +(deftest test-json-numbers + (assert-table [x y] (assert= (json/read-string x) y) + "1" 1 + "1.0" 1.0 + "0.1" 0.1 + "1.1" 1.1 + "1234.5678" 1234.5678 + + "-1" -1 + "-0.1" -0.1 + "-1.1" -1.1 + "-1234.5678" -1234.5678 + "1e1" 1e1)) + +(deftest test-vectors + (assert-table [x y] (assert= (json/read-string x) y) + "[]" [] + "[null]" [nil] + "[1, 2]" [1 2] + "[1, 1.0, null]" [1 1.0 nil] + "[\"foo\", 42]" ["foo" 42])) + +(deftest test-maps + (assert-table [x y] (assert= (json/read-string x) y) + "{\"foo\": 42}" {"foo", 42} + "{\"foo\": 42, \"bar\":null}" {"foo" 42 + "bar" nil})) From 5905f573cdfd2d6356473604e2b5e2b01a5ce582 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 16 Apr 2015 16:27:04 -0600 Subject: [PATCH 710/909] added a few comments in tcp.pxi --- pixie/io/tcp.pxi | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index 95614f7e..45ab1d3a 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -82,7 +82,10 @@ -(defn tcp-server [ip port on-connection] +(defn tcp-server + "Creates a TCP server on the given ip (as a string) and port (as an integer). Returns a TCPServer that can be + shutdown with dispose!. on-connection is a function that will be passed a TCPStream for each connecting client." + [ip port on-connection] (assert (string? ip) "Ip should be a string") (assert (integer? port) "Port should be a int") @@ -103,7 +106,9 @@ tcp-server)) -(defn tcp-client [ip port] +(defn tcp-client + "Creates a TCP connection to the given ip (as a string) and port (an integer). Will return a TCPStream" + [ip port] (let [client-addr (uv/sockaddr_in) uv-connect (uv/uv_connect_t) client (uv/uv_tcp_t) @@ -122,6 +127,4 @@ (->TCPStream client (uv/uv_buf_t)))) (catch ex (println ex)))))) - (uv/uv_tcp_connect uv-connect client client-addr @cb))) - - )) + (uv/uv_tcp_connect uv-connect client client-addr @cb))))) From bfe04685f9351b251b0e22ffac93affe73495efa Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 17 Apr 2015 08:46:36 +0100 Subject: [PATCH 711/909] Fix typo --- pixie/io/tcp.pxi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index 45ab1d3a..fb136bb1 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -92,16 +92,16 @@ (let [server (uv/uv_tcp_t) bind-addr (uv/sockaddr_in) _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) - on-new-connetion (atom nil) - tcp-server (->TCPServer ip port on-connection server bind-addr on-new-connetion)] - (reset! on-new-connetion + on-new-connection (atom nil) + tcp-server (->TCPServer ip port on-connection server bind-addr on-new-connection)] + (reset! on-new-connection (ffi/ffi-prep-callback uv/uv_connection_cb (fn [server status] (launch-tcp-client-from-server tcp-server)))) (uv/uv_tcp_init (uv/uv_default_loop) server) (uv/uv_tcp_bind server bind-addr 0) - (uv/throw-on-error (uv/uv_listen server 128 @on-new-connetion)) + (uv/throw-on-error (uv/uv_listen server 128 @on-new-connection)) (st/yield-control) tcp-server)) From d7470199b94406b9492181f238fb2004359d89c4 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 17 Apr 2015 13:58:32 +0100 Subject: [PATCH 712/909] basic tty io support --- examples/tty-io-test.pxi | 12 +++++++ pixie/io.pxi | 16 ++++++++- pixie/io/tcp.pxi | 14 +------- pixie/io/tty.pxi | 73 ++++++++++++++++++++++++++++++++++++++++ pixie/system.pxi | 13 +++++++ pixie/uv.pxi | 20 +++++++++++ 6 files changed, 134 insertions(+), 14 deletions(-) create mode 100644 examples/tty-io-test.pxi create mode 100644 pixie/io/tty.pxi create mode 100644 pixie/system.pxi diff --git a/examples/tty-io-test.pxi b/examples/tty-io-test.pxi new file mode 100644 index 00000000..ec8fe209 --- /dev/null +++ b/examples/tty-io-test.pxi @@ -0,0 +1,12 @@ +(ns io-test + (:require [pixie.io :as io] + [pixie.system :as sys] + [pixie.io.tty :as tty])) + +(io/write-stream tty/stdout "This is on STDOUT\n") +(io/write-stream tty/stderr "This is on STDERR\n") + +(loop [] + (let [input (io/read-line tty/stdin)] + (io/write-stream tty/stdout (str "You typed: " input "\n")) + (recur))) diff --git a/pixie/io.pxi b/pixie/io.pxi index 20dd2d64..afe0e733 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -151,7 +151,7 @@ result) (defn open-write - {:doc "Open a file for reading, returning a IInputStream" + {:doc "Open a file for writing, returning a IOutputStream" :added "0.1"} [filename] (assert (string? filename) "Filename must be a string") @@ -172,6 +172,20 @@ (write-byte fp chr) nil)))) +(defn stream-output-rf [output-stream] + (let [fp (buffered-output-stream output-stream + DEFAULT-BUFFER-SIZE)] + (fn ([] 0) + ([_] (dispose! fp)) + ([_ chr] + (assert (integer? chr)) + (write-byte fp chr) + nil)))) + +(defn write-stream [output-stream val] + (transduce (map int) + (stream-output-rf output-stream) + (str val))) (defn spit [filename val] (transduce (map int) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index fb136bb1..2ca440f4 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -11,24 +11,12 @@ (dispose! @on-connection-cb) (dispose! bind-addr))) -(defn -prep-uv-buffer-fn [buf read-bytes] - (ffi/ffi-prep-callback - uv/uv_alloc_cb - (fn [handle suggested-size uv-buf] - (try - (let [casted (ffi/cast uv-buf uv/uv_buf_t)] - (ffi/set! casted :base buf) - (ffi/set! casted :len (min suggested-size - (buffer-capacity buf) - read-bytes))) - (catch ex (println ex)))))) - (deftype TCPStream [uv-client uv-write-buf] IInputStream (read [this buffer len] (assert (<= (buffer-capacity buffer) len) "Not enough capacity in the buffer") - (let [alloc-cb (-prep-uv-buffer-fn buffer len) + (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) read-cb (atom nil)] (st/call-cc (fn [k] (reset! read-cb (ffi/ffi-prep-callback diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi new file mode 100644 index 00000000..f84d10d0 --- /dev/null +++ b/pixie/io/tty.pxi @@ -0,0 +1,73 @@ +(ns pixie.io.tty + (:require [pixie.stacklets :as st] + [pixie.streams :refer [IInputStream read IOutputStream write]] + [pixie.uv :as uv] + [pixie.io :as io] + [pixie.system :as sys] + [pixie.ffi :as ffi])) + +(deftype TTYInputStream [uv-client uv-write-buf] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) + read-cb (atom nil)] + (st/call-cc (fn [k] + (reset! read-cb (ffi/ffi-prep-callback + uv/uv_read_cb + (fn [stream nread uv-buf] + (set-buffer-count! buffer nread) + (try + (dispose! alloc-cb) + (dispose! @read-cb) + ;(dispose! uv-buf) + (uv/uv_read_stop stream) + (st/run-and-process k (or + (st/exception-on-uv-error nread) + nread)) + (catch ex + (println ex)))))) + (uv/uv_read_start uv-client alloc-cb @read-cb)))))) + +(deftype TTYOutputStream [uv-client uv-write-buf] + IOutputStream + (write [this buffer] + (let [write-cb (atom nil) + uv_write (uv/uv_write_t)] + (ffi/set! uv-write-buf :base buffer) + (ffi/set! uv-write-buf :len (count buffer)) + (st/call-cc + (fn [k] + (reset! write-cb (ffi/ffi-prep-callback + uv/uv_write_cb + (fn [req status] + (try + (dispose! @write-cb) + (st/run-and-process k status) + (catch ex + (println ex)))))) + (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) + + IDisposable + (-dispose! [this] + (dispose! uv-write-buf) + (uv/uv_close uv-client st/close_cb))) + +(defn tty-output-stream [fd] + ;(assert (= uv/UV_TTY (uv/uv_guess_handle fd)) "fd is not a TTY") + (let [buf (uv/uv_buf_t) + tty (uv/uv_tty_t)] + (uv/uv_tty_init (uv/uv_default_loop) tty fd 0) + (->TTYOutputStream tty buf))) + +(defn tty-input-stream [fd] + ;(assert (= uv/UV_TTY (uv/uv_guess_handle fd)) "fd is not a TTY") + (let [buf (uv/uv_buf_t) + tty (uv/uv_tty_t)] + (uv/uv_tty_init (uv/uv_default_loop) tty fd 0) + (->TTYInputStream tty buf))) + +(def stdin (tty-input-stream sys/stdin)) +(def stdout (tty-output-stream sys/stdout)) +(def stderr (tty-output-stream sys/stderr)) diff --git a/pixie/system.pxi b/pixie/system.pxi new file mode 100644 index 00000000..21306122 --- /dev/null +++ b/pixie/system.pxi @@ -0,0 +1,13 @@ +(ns pixie.system + (:require [pixie.ffi-infer :as i])) + +(i/with-config {:library "c" :imports ["stdio.h"]} + (i/defconst STDIN_FILENO) + (i/defconst STDOUT_FILENO) + (i/defconst STDERR_FILENO)) + +(def fdopen (ffi-fn libc "fdopen" [CInt CCharP] CVoidP)) + +(def stdin STDIN_FILENO) +(def stderr STDOUT_FILENO) +(def stdout STDERR_FILENO) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 67f93cdd..d49da8f7 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -189,6 +189,14 @@ (f/defcfn uv_async_init) (f/defcfn uv_async_send) + ; TTY + (f/defcfn uv_tty_init) + (f/defconst UV_TTY_MODE_NORMAL) + (f/defcfn uv_guess_handle) + ;(f/defcfn uv_tty_set_mode) + (f/defcstruct uv_tty_t []) + (f/defconst UV_TTY) + ; TCP Networking (f/defcstruct uv_tcp_t []) @@ -243,3 +251,15 @@ ~@args @cb#)))] (pixie.stacklets/call-cc f))))) + +(defn -prep-uv-buffer-fn [buf read-bytes] + (ffi/ffi-prep-callback + uv_alloc_cb + (fn [handle suggested-size uv-buf] + (try + (let [casted (ffi/cast uv-buf uv_buf_t)] + (ffi/set! casted :base buf) + (ffi/set! casted :len (min suggested-size + (buffer-capacity buf) + read-bytes))) + (catch ex (println ex)))))) From 4c3093fe3709c6bd1c8c0eee3964385e697a242d Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 19 Apr 2015 13:00:20 +0100 Subject: [PATCH 713/909] make slurp/spit accept filenames or streams. * Use utf8 streams for slurp and spit. * Also make the tty example a bit cooler. Note: EOF file errors occur when slurping in IInputStreams but seems to nearly work --- examples/tty-io-test.pxi | 27 ++++++++++++--- pixie/io.pxi | 72 +++++++++++++++++++--------------------- pixie/io/tty.pxi | 30 ++++++++++++++--- pixie/streams/utf8.pxi | 9 +++++ 4 files changed, 90 insertions(+), 48 deletions(-) diff --git a/examples/tty-io-test.pxi b/examples/tty-io-test.pxi index ec8fe209..0fadfa3a 100644 --- a/examples/tty-io-test.pxi +++ b/examples/tty-io-test.pxi @@ -3,10 +3,27 @@ [pixie.system :as sys] [pixie.io.tty :as tty])) -(io/write-stream tty/stdout "This is on STDOUT\n") -(io/write-stream tty/stderr "This is on STDERR\n") +(def history (atom [])) +(defn depth [d] + (apply str (take d (repeat " ")))) + +(defn nested-trace + "Prints a stack trace with each level indented slightly" + [e] + (loop [d 0 traces (trace e)] + (io/spit tty/stdout (str (depth d) (pr-str (first traces)) "\n")) + (if (seq traces) + (recur (inc d) (rest traces))))) + +(io/spit tty/stdout "TTY Demo REPL\n") (loop [] - (let [input (io/read-line tty/stdin)] - (io/write-stream tty/stdout (str "You typed: " input "\n")) - (recur))) + (let [command-number (count @history)] + (io/spit tty/stdout (str "[ " command-number " ] < " )) + (let [input (io/read-line tty/stdin) + res (try (eval (read-string input)) + (catch e + (nested-trace e)))] + (io/spit tty/stdout (str "[ " command-number " ] > " res "\n")) + (swap! history conj input) + (recur)))) diff --git a/pixie/io.pxi b/pixie/io.pxi index afe0e733..9929018e 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -1,5 +1,6 @@ (ns pixie.io (:require [pixie.streams :as st :refer :all] + [pixie.streams.utf8 :as utf8] [pixie.io-blocking :as io-blocking] [pixie.uv :as uv] [pixie.stacklets :as st] @@ -161,45 +162,40 @@ 0 (uv/uv_buf_t))) - -(defn file-output-rf [filename] - (let [fp (buffered-output-stream (open-write filename) - DEFAULT-BUFFER-SIZE)] - (fn ([] 0) - ([_] (dispose! fp)) - ([_ chr] - (assert (integer? chr)) - (write-byte fp chr) - nil)))) - -(defn stream-output-rf [output-stream] - (let [fp (buffered-output-stream output-stream - DEFAULT-BUFFER-SIZE)] - (fn ([] 0) - ([_] (dispose! fp)) - ([_ chr] - (assert (integer? chr)) - (write-byte fp chr) - nil)))) - -(defn write-stream [output-stream val] - (transduce (map int) - (stream-output-rf output-stream) - (str val))) - -(defn spit [filename val] - (transduce (map int) - (file-output-rf filename) - (str val))) - -(defn slurp [filename] - (let [c (open-read filename) +(defn spit + "Writes the content to output. Output must be a file or an IOutputStream." + [output content] + (cond + (string? output) + (transduce (map identity) + (-> output + open-write + buffered-output-stream + utf8/utf8-output-stream-rf) + (str content)) + + (satisfies? IOutputStream output) + (transduce (map identity) + (-> output + buffered-output-stream + utf8/utf8-output-stream-rf) + (str content)) + + :else (throw "Expected a string or IOutputStream"))) + +(defn slurp + "Reads in the contents of input. Input must be a filename or an IInputStream" + [input] + (let [stream (cond + (string? input) (open-read input) + (satisfies? IInputStream input) input + :else (throw "Expected a string or an IInputStream")) result (transduce - (map char) - string-builder - c)] - (dispose! c) - result)) + (map char) + string-builder + stream)] + (dispose! stream) + result)) (defn run-command [command] (st/apply-blocking io-blocking/run-command command)) diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi index f84d10d0..24a909cd 100644 --- a/pixie/io/tty.pxi +++ b/pixie/io/tty.pxi @@ -6,18 +6,20 @@ [pixie.system :as sys] [pixie.ffi :as ffi])) +(def DEFAULT-BUFFER-SIZE 1024) + (deftype TTYInputStream [uv-client uv-write-buf] IInputStream - (read [this buffer len] - (assert (<= (buffer-capacity buffer) len) + (read [this buf len] + (assert (<= (buffer-capacity buf) len) "Not enough capacity in the buffer") - (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) + (let [alloc-cb (uv/-prep-uv-buffer-fn buf len) read-cb (atom nil)] (st/call-cc (fn [k] (reset! read-cb (ffi/ffi-prep-callback uv/uv_read_cb (fn [stream nread uv-buf] - (set-buffer-count! buffer nread) + (set-buffer-count! buf nread) (try (dispose! alloc-cb) (dispose! @read-cb) @@ -28,7 +30,25 @@ nread)) (catch ex (println ex)))))) - (uv/uv_read_start uv-client alloc-cb @read-cb)))))) + (uv/uv_read_start uv-client alloc-cb @read-cb))))) + + IDisposable + (-dispose! [this] + (dispose! uvbuf) + (fs_close fp)) + + IReduce + (-reduce [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc)))))) (deftype TTYOutputStream [uv-client uv-write-buf] IOutputStream diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi index 055733df..bc9b4044 100644 --- a/pixie/streams/utf8.pxi +++ b/pixie/streams/utf8.pxi @@ -67,3 +67,12 @@ "Creates a UTF8 encoder that writes characters to the given IByteOutputStream." [o] (->UTF8OutputStream o)) + +(defn utf8-output-stream-rf [output-stream] + (let [fp (utf8-output-stream output-stream)] + (fn ([] 0) + ([_] (dispose! fp)) + ([_ chr] + (assert (char? chr)) + (write-char fp chr) + nil)))) From c1a23c8660300264f279caef75c58bd0e1fe5e49 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 19 Apr 2015 15:55:42 -0400 Subject: [PATCH 714/909] 3 arity into --- pixie/fs.pxi | 2 +- pixie/stdlib.pxi | 13 ++++++++----- tests/pixie/tests/test-stdlib.pxi | 4 ++++ 3 files changed, 13 insertions(+), 6 deletions(-) diff --git a/pixie/fs.pxi b/pixie/fs.pxi index 7a943857..f3e73960 100644 --- a/pixie/fs.pxi +++ b/pixie/fs.pxi @@ -52,7 +52,7 @@ "Recursively returns all files and directories below") (walk-files [this] - "Recursivelt returns all files underneath") + "Recursively returns all files underneath") (walk-dirs [this] "Recursivley returns all directories underneath")) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 48110302..a2770d6c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -195,11 +195,14 @@ (def into (fn ^{:doc "Add the elements of `from` to the collection `to`." :signatures [[to from]] :added "0.1"} - into - [to from] - (if (satisfies? IToTransient to) - (persistent! (reduce conj! (transient to) from)) - (reduce conj to from)))) + ([to from] + (if (satisfies? IToTransient to) + (persistent! (reduce conj! (transient to) from)) + (reduce conj to from))) + ([to xform from] + (if (satisfies? IToTransient to) + (transduce xform conj! (transient to) from) + (transduce xform conj to from))))) (def interpose (fn ^{:doc "Returns a transducer that inserts `val` in between elements of a collection." diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 49bec527..6e35566b 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -338,6 +338,10 @@ [2 :a] [2 :b] [2 :c] [3 :a] [3 :b] [3 :c]])) +(t/deftest test-into + (t/assert= [1 3] (into [] (comp (map inc) (filter odd?)) (range 3))) + (t/assert= {:a 1 :b 2} (into {} [[:a 1] [:b 2]]))) + (t/deftest test-ex-msg (try (throw "This is an exception") From 77db40c29b985dcaa514fc2820a5cbe0d89c87c5 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 19 Apr 2015 21:17:43 -0400 Subject: [PATCH 715/909] string representations for Range --- pixie/stdlib.pxi | 7 +++++++ tests/pixie/tests/test-stdlib.pxi | 6 ++++-- 2 files changed, 11 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 48110302..215b5527 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1768,6 +1768,13 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (and (< step 0) (> start stop))) (cons start (lazy-seq* #(range (+ start step) stop step)))))) +(extend -str Range + (fn [v] + (-str (seq v)))) +(extend -repr Range + (fn [v] + (-repr (seq v)))) + (defn range {:doc "Returns a range of numbers." :examples [["(seq (range 3))" nil (0 1 2)] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 49bec527..b1ab65da 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -36,7 +36,8 @@ (t/assert= (str {:a 1}) "{:a 1}") (t/assert= (str (type 3)) "") (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]") - (t/assert= (seq (map identity "iterable")) '(\i \t \e \r \a \b \l \e))) + (t/assert= (seq (map identity "iterable")) '(\i \t \e \r \a \b \l \e)) + (t/assert= (str (range 3)) "(0 1 2)")) (t/deftest test-repr (t/assert= (-repr nil) "nil") @@ -58,7 +59,8 @@ (t/assert= (-repr {:a 1}) "{:a 1}") (t/assert= (-repr (type 3)) "pixie.stdlib.Integer") - (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]")) + (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]") + (t/assert= (-repr (range 3)) "(0 1 2)")) (t/deftest test-nth ;; works if the index is found From ba08757e2eeb395b4af64db298c15a852c462961 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 20 Apr 2015 06:53:11 -0600 Subject: [PATCH 716/909] Require that exceptions support filterable data, and update catch to allow filtering on these exceptions --- pixie/stdlib.pxi | 107 ++++++++++++++++++++---------- pixie/vm/code.py | 6 +- pixie/vm/custom_types.py | 6 +- pixie/vm/interpreter.py | 6 +- pixie/vm/libs/ffi.py | 3 +- pixie/vm/object.py | 19 ++++-- pixie/vm/persistent_vector.py | 3 +- pixie/vm/reader.py | 5 +- pixie/vm/stdlib.py | 27 +++++++- tests/pixie/tests/test-stdlib.pxi | 34 ++++++++-- 10 files changed, 158 insertions(+), 58 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a2770d6c..d8783a10 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1,6 +1,5 @@ (in-ns :pixie.stdlib) - (def libc (ffi-library pixie.platform/lib-c-name)) @@ -795,19 +794,37 @@ there's a value associated with the key. Use `some` for checking for values." (let [head (first form)] (cond (= head 'catch) (if catch - (throw "Can only have one catch clause per try") + (throw [:pixie.stdlib/SyntaxException + "Can only have one catch clause per try"]) (recur (next (next form)) (first (next form)) body-items finally (next body))) (= head 'finally) (if finally - (throw "Can only have one finally clause per try") + (throw [:pixie.stdlib/SyntaxException + "Can only have one finally clause per try"]) (recur catch catch-sym body-items (next form) (next body))) :else (recur catch catch-sym (conj body-items form) finally (next body))))) - `(-try-catch - (fn [] ~@body-items) - ~(if catch - `(fn [~catch-sym] ~@catch) - `(fn [] nil)) - - (fn [] ~@finally)))))) + (let [catch-data (cond + (keyword? catch-sym) (let [sym (first catch)] + (assert (symbol? sym) (str "Invalid catch binding form" + catch)) + [`[(if (= ~catch-sym (ex-data ~sym)) + (do ~@(next catch)) + (throw ~sym))] + sym]) + (seq? catch-sym) (let [sym (first catch)] + (assert (symbol? sym) (str "Invalid catch binding form" + catch)) + [`[(if ~catch-sym + (do ~@(next catch)) + (throw ~sym))] + sym]) + :else [catch catch-sym])] + `(-try-catch + (fn [] ~@body-items) + ~(if catch + `(fn [~(nth catch-data 1)] ~@(nth catch-data 0)) + `(fn [] nil)) + + (fn [] ~@finally))))))) (defn . {:doc "Access the field named by the symbol. @@ -846,7 +863,9 @@ If further arguments are passed, invokes the method named by symbol, passing the [x] (cond (number? x) (+ x 0.0) - :else (throw (str "Can't convert a value of type " (type x) " to a Float")))) + :else (throw + [:pixie.stdlib/ConversionException + (str "Can't convert a value of type " (type x) " to a Float")]))) (defn int {:doc "Converts a number to an integer." @@ -857,7 +876,9 @@ If further arguments are passed, invokes the method named by symbol, passing the (float? x) (lround (floor x)) (ratio? x) (int (/ (float (numerator x)) (float (denominator x)))) (char? x) (+ x 0) - :else (throw (str "Can't convert a value of type " (type x) " to an Integer")))) + :else (throw + [:pixie.stdlib/ConversionException + (throw (str "Can't convert a value of type " (type x) " to an Integer"))]))) (defn last {:doc "Returns the last element of the collection, or nil if none." @@ -883,13 +904,12 @@ If further arguments are passed, invokes the method named by symbol, passing the {:doc "Given a function, return a new function which takes the same arguments but returns the opposite truth value"} [f] - (if (not (fn? f)) - (throw "Complement must be passed a function") - (fn - ([] (not (f))) - ([x] (not (f x))) - ([x y] (not (f x y))) - ([x y & more] (not (apply f x y more)))))) + (assert (fn? f) "Complement must be passed a function") + (fn + ([] (not (f))) + ([x] (not (f x))) + ([x y] (not (f x y))) + ([x y & more] (not (apply f x y more))))) (defn constantly [x] {:doc "Return a function that always returns x, no matter what it is called with." @@ -983,13 +1003,17 @@ If further arguments are passed, invokes the method named by symbol, passing the (instance? PersistentVector x) (if (= (count x) 2) (assoc coll (nth x 0 nil) (nth x 1 nil)) - (throw "Vector arg to map conj must be a pair")) + (throw + [:pixie.stdlib/InvalidArgumentException + "Vector arg to map conj must be a pair"])) (satisfies? ISeqable x) (reduce conj coll (-seq x)) :else - (throw (str (type x) " cannot be conjed to a map"))))) + (throw + [:pixie.stdlib/InvalidArgumentException + (str (type x) " cannot be conjed to a map")])))) (extend -conj Cons (fn [coll x] @@ -1075,11 +1099,13 @@ Creates new maps if the keys are not present." ([test] `(if ~test nil - (throw "Assert failed"))) + (throw [:pixie.stdlib/AssertionException + "Assert failed"]))) ([test msg] `(if ~test nil - (throw (str "Assert failed: " ~msg))))) + (throw [:pixie.stdlib/AssertionException + (str "Assert failed: " ~msg)])))) (defmacro resolve {:doc "Resolve the var associated with the symbol in the current namespace." @@ -1172,7 +1198,9 @@ Creates new maps if the keys are not present." (protocol? @(resolve-in *ns* body)) [@(resolve-in *ns* body) (second res) (conj (third res) body)] - :else (throw (str "can only extend protocols or Object, not " body " of type " (type body)))) + :else (throw + [:pixie.stdlib/AssertionException + (str "can only extend protocols or Object, not " body " of type " (type body))])) (seq? body) (let [proto (first res) tbs (second res) pbs (third res)] (if (protocol? proto) [proto tbs (conj pbs body)] @@ -1222,7 +1250,8 @@ and implements IAssociative, ILookup and IObject." `(-contains-key [self k] (contains? ~(set fields) k)) `(-dissoc [self k] - (throw "dissoc is not supported on defrecords")) + (throw [:pixie.stdlib/NotImplementedException + "dissoc is not supported on defrecords"])) 'ILookup `(-val-at [self k not-found] (if (contains? ~(set fields) k) @@ -1237,7 +1266,7 @@ and implements IAssociative, ILookup and IObject." `(= (. self ~field) (. other ~field))) fields))) `(-hash [self] - (throw "not implemented"))] + (throw [:pixie.stdlib/NotImplementedException "not implemented"]))] deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] `(do ~type-from-map ~deftype-decl))) @@ -1636,7 +1665,8 @@ not enough elements were present." (map? binding) (let [name (gensym "map__")] (reduce conj [name expr] (destructure-map binding name))) - :else (throw (str "unsupported binding form: " binding)))) + :else (throw [:pixie.stdlib/AssertionException + (str "unsupported binding form: " binding)]))) (defn destructure-vector [binding-vector expr] (loop [bindings (seq binding-vector) @@ -1711,12 +1741,15 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (extend -nth ISeq (fn [s n] (when (empty? s) - (throw "Index out of Range")) + (throw + [:pixie.stdlib/OutOfRangeException + "Index out of Range"])) (if (and (pos? n) s) (recur (next s) (dec n)) (if (zero? n) (first s) - (throw "Index out of Range"))))) + (throw [:pixie.stdlib/OutOfRangeException + "Index out of Range"]))))) (extend -nth-not-found ISeq (fn [s n not-found] (if (and (pos? n) s) (recur (next s) (dec n) not-found) @@ -1753,12 +1786,12 @@ For more information, see http://clojure.org/special_forms#binding-forms"} IIndexed (-nth [self idx] (when (or (= start stop 0) (neg? idx)) - (throw "Index out of Range")) + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"])) (let [cmp (if (< start stop) < >) val (+ start (* idx step))] (if (cmp val stop) val - (throw "Index out of Range")))) + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"])))) (-nth-not-found [self idx not-found] (let [cmp (if (< start stop) < >) val (+ start (* idx step))] @@ -1887,7 +1920,8 @@ user => (refer 'pixie.string :rename '{index-of find}) user => (refer 'pixie.string :exclude '(substring))" :added "0.1"} [ns-sym & filters] - (let [ns (or (the-ns ns-sym) (throw (str "No such namespace: " ns-sym))) + (let [ns (or (the-ns ns-sym) (throw [:pixie.stdlib/NamespaceNotFoundException + (str "No such namespace: " ns-sym)])) filters (apply hashmap filters) nsmap (ns-map ns) rename (or (:rename filters) {}) @@ -1896,7 +1930,8 @@ user => (refer 'pixie.string :exclude '(substring))" (keys nsmap) (or (:refer filters) (:only filters)))] (when (and refers (not (satisfies? ISeqable refers))) - (throw ":only/:refer must be a collection of symbols")) + (throw [:pixie.stdlib/InvalidArgumentException + ":only/:refer must be a collection of symbols"])) (when-let [as (:as filters)] (refer-ns *ns* ns-sym as)) (loop [syms (seq refers)] @@ -1907,7 +1942,8 @@ user => (refer 'pixie.string :exclude '(substring))" (when-not (exclude sym) (let [v (nsmap sym)] (when-not v - (throw (str sym "does not exist"))) + (throw [:pixie.stdlib/SymbolNotFoundException + (str sym "does not exist")])) (refer-symbol *ns* (or (rename sym) sym) v)))) (recur (next syms))))) nil)) @@ -2146,7 +2182,8 @@ If the number of arguments is even and no clause matches, throws an exception." `((~pred ~a ~x) ~b) `(:else ~a))) (partition 2 clauses)) - :else (throw "No matching clause!"))))) + :else (throw [:pixie.stdlib/MissingClauseException + "No matching clause!"]))))) (defmacro case "Takes an expression and a number of two-form clauses. diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 548cb48a..b1ac4caa 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -179,7 +179,8 @@ def get_fn(self, arity): if self._rest_fn: acc.append(unicode(str(self._rest_fn.required_arity())) + u"+") - runtime_error(u"Wrong number of arguments " + unicode(str(arity)) + u" for function '" + unicode(self._name) + u"'. Expected " + join_last(acc, u"or")) + runtime_error(u"Wrong number of arguments " + unicode(str(arity)) + u" for function '" + unicode(self._name) + u"'. Expected " + join_last(acc, u"or"), + u"pixie.stdlib/InvalidArityException") def get_arities(self): return self._arities.keys() @@ -241,7 +242,8 @@ def invoke(self, args): else: runtime_error(u"Invalid number of arguments " + unicode(str(len(args))) + u" for function '" + unicode(str(self._name)) + u"'. Expected " - + unicode(str(self.get_arity()))) + + unicode(str(self.get_arity())), + u":pixie.stdlib/InvalidArityException") def invoke_with(self, args, this_fn): try: diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 0d6d126b..081e57fc 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -49,7 +49,8 @@ def type(self): def set_field(self, name, val): idx = self._custom_type.get_slot_idx(name) if idx == -1: - runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type())), + u"pixie.stdlib/InvalidFieldException") old_val = self._fields[idx] if isinstance(old_val, AbstractMutableCell): @@ -71,7 +72,8 @@ def get_field_immutable(self, idx): def get_field(self, name): idx = self._custom_type.get_slot_idx(name) if idx == -1: - runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type()))) + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type())), + u"pixie.stdlib/InvalidFieldException") if self._custom_type.is_mutable(name): value = self._fields[idx] diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index bbc96324..83e873df 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -81,7 +81,8 @@ def pop(self): self.sp -= 1 if not 0 <= self.sp < len(self.stack): - runtime_error(u"Stack out of range: " + unicode(str(self.sp))) + runtime_error(u"Stack out of range: " + unicode(str(self.sp)), + u"pixie.vm.interpreter/InterpreterError") v = self.stack[self.sp] self.stack[self.sp] = None @@ -91,7 +92,8 @@ def pop(self): def nth(self, delta): affirm(delta >= 0, u"Invalid nth value, (compiler error)") if not self.sp - 1 >= delta: - runtime_error(u"Interpreter nth out of range: " + unicode(str(self.sp - 1)) + u", " + unicode(str(delta))) + runtime_error(u"Interpreter nth out of range: " + unicode(str(self.sp - 1)) + u", " + unicode(str(delta)), + u"pixie.vm.interpreter/InterpreterError") return self.stack[self.sp - delta - 1] def push_nth(self, delta): diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 6fbc53a1..b8b92512 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -68,7 +68,8 @@ def load_lib(self): self._dyn_lib = dynload.dlopen(s) self._is_inited = True except dynload.DLOpenError as ex: - raise object.WrappedException(object.RuntimeException(rt.wrap(u"Couldn't Load Library: " + self._name))) + raise object.runtime_error(u"Couldn't Load Library: " + self._name, + u"pixie.stdlib/LibraryNotFoundException") finally: rffi.free_charp(s) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index e986bbbe..4c39f162 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -114,7 +114,10 @@ def istypeinstance(obj, t): class RuntimeException(Object): _type = Type(u"pixie.stdlib.RuntimeException") - def __init__(self, data): + def __init__(self, msg, data): + assert data is not None + assert msg is not None + self._msg = msg self._data = data self._trace = [] @@ -129,8 +132,8 @@ def __repr__(self): for x in trace: s.append(x.__repr__()) s.append(u"\n") - - s.extend([u"RuntimeException: " + rt.name(rt.str(self._data)) + u"\n"]) + s.extend([u"RuntimeException: " + rt.name(rt.str(self._data)) + u" " + + rt.name(rt.str(self._msg)) + u"\n"]) return u"".join(s) @@ -150,11 +153,15 @@ def affirm(val, msg): assert isinstance(msg, unicode) if not val: import pixie.vm.rt as rt - raise WrappedException(RuntimeException(rt.wrap(msg))) + from pixie.vm.keyword import keyword + raise WrappedException(RuntimeException(rt.wrap(msg), keyword(u"pixie.stdlib/AssertionException"))) -def runtime_error(msg): +def runtime_error(msg, data=None): import pixie.vm.rt as rt - raise WrappedException(RuntimeException(rt.wrap(msg))) + from pixie.vm.keyword import keyword + if data is None: + data = u"pixie.stdlib/AssertionException" + raise WrappedException(RuntimeException(rt.wrap(msg), keyword(data))) def safe_invoke(f, args): try: diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 65aaacfb..a0e1ae55 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -182,7 +182,8 @@ def assoc_at(self, idx, val): if idx == self._cnt: return self.conj(val) else: - object.runtime_error(u"index out of range") + object.runtime_error(u"index out of range", + u"pixie.stdlib/OutOfRangeException") def do_assoc(lvl, node, idx, val): diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 0e150600..6173859b 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -787,7 +787,7 @@ def throw_syntax_error_with_data(rdr, txt): meta = nil data = rt.interpreter_code_info(meta) - err = object.RuntimeException(rt.wrap(txt)) + err = object.runtime_error(txt) err._trace.append(data) raise object.WrappedException(err) @@ -797,7 +797,8 @@ def read_inner(rdr, error_on_eof, always_return_form=True): eat_whitespace(rdr) except EOFError as ex: if error_on_eof: - runtime_error(u"Unexpected EOF while reading") + runtime_error(u"Unexpected EOF while reading", + u"pixie.stdlib/EOFWhileReadingException") return eof diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index cd9a283d..05c309c3 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -659,6 +659,7 @@ def namespace(s): @as_var("-try-catch") def _try_catch(main_fn, catch_fn, final): + from pixie.vm.keyword import keyword try: return main_fn.invoke([]) except Exception as ex: @@ -668,9 +669,9 @@ def _try_catch(main_fn, catch_fn, final): if not we_are_translated(): print "Python Error Info: ", ex.__dict__, ex raise - ex = RuntimeException(rt.wrap(u"Some error: " + unicode(str(ex)))) + ex = RuntimeException(rt.wrap(u"Internal error: " + unicode(str(ex))), keyword(u"pixie.stdlib/InternalError")) else: - ex = RuntimeException(nil) + ex = RuntimeException(u"No available message", keyword(u"pixie.stdlib/UnknownInternalError")) return catch_fn.invoke([ex]) else: return catch_fn.invoke([ex._ex]) @@ -680,9 +681,19 @@ def _try_catch(main_fn, catch_fn, final): @as_var("throw") def _throw(ex): + from pixie.vm.keyword import keyword if isinstance(ex, RuntimeException): raise WrappedException(ex) - raise WrappedException(RuntimeException(ex)) + if rt._satisfies_QMARK_(IVector, ex): + data = rt.nth(ex, rt.wrap(0)) + msg = rt.nth(ex, rt.wrap(1)) + elif rt._satisfies_QMARK_(ILookup, ex): + data = rt._val_at(ex, keyword(u"data"), nil) + msg = rt._val_at(ex, keyword(u"msg"), nil) + else: + affirm(False, u"Can only throw vectors, maps and exceptions") + return nil + raise WrappedException(RuntimeException(msg, data)) @as_var("resolve-in") def _var(ns, nm): @@ -791,6 +802,7 @@ def _seq(self): trace = vector.EMPTY trace_element = rt.hashmap(keyword(u"type"), keyword(u"runtime")) trace_element = rt.assoc(trace_element, keyword(u"data"), rt.wrap(self._data)) + trace_element = rt.assoc(trace_element, keyword(u"msg"), rt.wrap(self._msg)) trace = rt.conj(trace, trace_element) for x in self._trace: tmap = x.trace_map() @@ -806,9 +818,18 @@ def _seq(self): @as_var("ex-msg") def ex_msg(e): """Returns the message contained in an exception""" + affirm(isinstance(e, RuntimeException), u"Argument must be a RuntimeException") + assert isinstance(e, RuntimeException) + return e._msg + +@as_var("ex-data") +def ex_data(e): + """Returns the data contained in an exception""" + affirm(isinstance(e, RuntimeException), u"Argument must be a RuntimeException") assert isinstance(e, RuntimeException) return e._data + @extend(_doc, code.NativeFn._type) def _doc(self): assert isinstance(self, code.NativeFn) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 6e35566b..37f038ee 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -344,9 +344,33 @@ (t/deftest test-ex-msg (try - (throw "This is an exception") - (catch e - (t/assert= (ex-msg e) "This is an exception")))) + (throw [::something "This is an exception"]) + (catch e + (t/assert= (ex-msg e) "This is an exception") + (t/assert= (ex-data e) ::something)))) + +(t/deftest test-ex-filtering + (let [f (fn [val] + (try + (try + (throw [val "Some failure"]) + (catch ::catch-this ex + :found)) + (catch ex + :not-found)))] + (t/assert= (f ::catch-this) :found) + (t/assert= (f :something-else) :not-found)) + + (let [f (fn [val] + (try + (try + (throw [val "Some failure"]) + (catch (= ::catch-this (ex-data ex)) ex + :found)) + (catch ex + :not-found)))] + (t/assert= (f ::catch-this) :found) + (t/assert= (f :something-else) :not-found))) (t/deftest test-range (t/assert= (= (-seq (range 10)) @@ -397,7 +421,9 @@ (try (/ 0 0) (catch e - (t/assert= (first (trace e)) {:type :runtime :data "Divide by zero"}) + (t/assert= (first (trace e)) {:type :runtime + :data :pixie.stdlib/AssertionException + :msg "Divide by zero"}) (t/assert= (second (trace e)) {:type :native :name "_div"} )))) (t/deftest test-tree-seq From abe9791453b42879ec43d39499405cd8d512915f Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Mon, 20 Apr 2015 10:29:18 -0400 Subject: [PATCH 717/909] Make remainder return Integer if both arguments are integers Only extend the rem operation to use math.fmod if one or both of the two arguments is not an integer. --- pixie/vm/numbers.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index fe06eafd..6f1d029e 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -135,7 +135,8 @@ def define_num_ops(): continue extend_num_op(op, c1, c2, conv1, sym, conv2) extend_num_op("_quot", c1, c2, conv1, "/", conv2, wrap_start = "rt.wrap(math.floor(", wrap_end = "))") - extend_num_op("_rem", c1, c2, conv1, ",", conv2, wrap_start = "rt.wrap(math.fmod(", wrap_end = "))") + if c1 != Integer or c2 != Integer: + extend_num_op("_rem", c1, c2, conv1, ",", conv2, wrap_start = "rt.wrap(math.fmod(", wrap_end = "))") for (op, sym) in [("_num_eq", "=="), ("_lt", "<"), ("_gt", ">"), ("_lte", "<="), ("_gte", ">=")]: extend_num_op(op, c1, c2, conv1, sym, conv2, wrap_start = "true if ", wrap_end = " else false") From e53412cf1b7c957ee1d896be5c15f4a7d96fce9a Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Mon, 20 Apr 2015 10:39:26 -0400 Subject: [PATCH 718/909] Add note for Mac OS X building --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index 658d8c59..69a01bf9 100644 --- a/README.md +++ b/README.md @@ -30,6 +30,8 @@ Some planned and implemented features: make build_with_jit ./pixie-vm +Note: Mac OS X does not come with the build tools required by default. Install the XCode Command Line tools ([Apple Developer Site](http://developer.apple.com)) or install them independently. + ## Running the tests From a07f9e310552658cc2dcfd9df624aa3c2b5e6f74 Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Mon, 20 Apr 2015 17:35:38 -0400 Subject: [PATCH 719/909] Wrap _quot in a check whether its arguments are not both ints --- pixie/vm/numbers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 6f1d029e..ed54bd53 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -134,9 +134,9 @@ def define_num_ops(): if op == "_div" and c1 == Integer and c2 == Integer: continue extend_num_op(op, c1, c2, conv1, sym, conv2) - extend_num_op("_quot", c1, c2, conv1, "/", conv2, wrap_start = "rt.wrap(math.floor(", wrap_end = "))") if c1 != Integer or c2 != Integer: extend_num_op("_rem", c1, c2, conv1, ",", conv2, wrap_start = "rt.wrap(math.fmod(", wrap_end = "))") + extend_num_op("_quot", c1, c2, conv1, "/", conv2, wrap_start = "rt.wrap(math.floor(", wrap_end = "))") for (op, sym) in [("_num_eq", "=="), ("_lt", "<"), ("_gt", ">"), ("_lte", "<="), ("_gte", ">=")]: extend_num_op(op, c1, c2, conv1, sym, conv2, wrap_start = "true if ", wrap_end = " else false") From 74231847156eed08d9cfa0b54b6ee3dacfd32554 Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Mon, 20 Apr 2015 21:57:17 -0400 Subject: [PATCH 720/909] Added tests to confirm changes --- tests/pixie/tests/test-numbers.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/pixie/tests/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi index 1d88d9d0..092aff5d 100644 --- a/tests/pixie/tests/test-numbers.pxi +++ b/tests/pixie/tests/test-numbers.pxi @@ -49,3 +49,16 @@ (t/deftest test-float (doseq [[x f] [[1 1.0] [3 3.0] [3.333 3.333] [3/2 1.5] [1/7 (/ 1.0 7.0)]]] (t/assert= (float x) f))) + +(t/deftest rem-types + (t/assert= "" (str (type (rem 5 3)))) + (t/assert= "" (str (type (rem 5.0 3)))) + (t/assert= "" (str (type (rem 7/2 3)))) + (t/assert= "" (str (type (rem 7/2 3.0))))) + +(t/deftest quot-types + (t/assert= "" (str (type (quot 5 3)))) + (t/assert= "" (str (type (quot 5.0 3)))) + (t/assert= "" (str (type (quot 7/2 3/7)))) + (t/assert= "" (str (type (quot 7/2 3.0))))) + From 8871f30356d2668f2fe96a501a5195d8459c15ea Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Tue, 21 Apr 2015 12:05:55 -0400 Subject: [PATCH 721/909] Change test to remove stringification --- tests/pixie/tests/test-numbers.pxi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pixie/tests/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi index 092aff5d..efe36e2a 100644 --- a/tests/pixie/tests/test-numbers.pxi +++ b/tests/pixie/tests/test-numbers.pxi @@ -57,8 +57,8 @@ (t/assert= "" (str (type (rem 7/2 3.0))))) (t/deftest quot-types - (t/assert= "" (str (type (quot 5 3)))) - (t/assert= "" (str (type (quot 5.0 3)))) - (t/assert= "" (str (type (quot 7/2 3/7)))) - (t/assert= "" (str (type (quot 7/2 3.0))))) + (t/assert= Integer (type (quot 5 3))) + (t/assert= Float (type (quot 5.0 3))) + (t/assert= Integer (type (quot 7/2 3/7))) + (t/assert= Float (type (quot 7/2 3.0)))) From 103d36732fa54af0d6f6965d1fd75249daa6a5fd Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Tue, 21 Apr 2015 23:26:00 -0400 Subject: [PATCH 722/909] Fix to not use stringification for rem tests --- tests/pixie/tests/test-numbers.pxi | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/pixie/tests/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi index efe36e2a..3b5f0aeb 100644 --- a/tests/pixie/tests/test-numbers.pxi +++ b/tests/pixie/tests/test-numbers.pxi @@ -51,10 +51,10 @@ (t/assert= (float x) f))) (t/deftest rem-types - (t/assert= "" (str (type (rem 5 3)))) - (t/assert= "" (str (type (rem 5.0 3)))) - (t/assert= "" (str (type (rem 7/2 3)))) - (t/assert= "" (str (type (rem 7/2 3.0))))) + (t/assert= Integer (type (rem 5 3))) + (t/assert= Float (type (rem 5.0 3))) + (t/assert= Ratio (type (rem 7/2 3))) + (t/assert= Float (type (rem 7/2 3.0)))) (t/deftest quot-types (t/assert= Integer (type (quot 5 3))) From 0c367748fc8737a23140d82c476ca3102150602c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 22 Apr 2015 20:06:56 +0100 Subject: [PATCH 723/909] add if-not macro --- pixie/stdlib.pxi | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index aa648128..4114ed8f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1467,6 +1467,11 @@ The new value is thus `(apply f current-value-of-atom args)`." (let [~bind tmp#] ~@body))))) +(defmacro if-not + ([test then] `(if-not ~test ~then nil)) + ([test then else] + `(if (not ~test) ~then ~else))) + (defmacro if-let ([binding then] `(if-let ~binding ~then nil)) ([binding then else] From 85a9d7b6b85ba9507d7475d3edf12548db70a8ce Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Wed, 22 Apr 2015 21:45:21 +0200 Subject: [PATCH 724/909] exit the repl when the form is just `:exit` It does *not* exit when `:exit` is the result of evaluating an expression, e.g. `(do :exit)` and `(or nil false :exit)` both don't exit, but just `:exit` does. A similar feature was present in the old repl, but hadn't made it into the current one. --- pixie/repl.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/repl.pxi b/pixie/repl.pxi index e8432cba..639035db 100644 --- a/pixie/repl.pxi +++ b/pixie/repl.pxi @@ -19,7 +19,7 @@ ""))))] (loop [] (try (let [form (read rdr false)] - (if (= form eof) + (if (or (= form eof) (= form :exit)) (exit 0) (let [x (eval form)] (pixie.stdlib/-push-history x) From 6fcadacb5e58df752d34e269805672b980d289c7 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 22 Apr 2015 22:14:55 +0100 Subject: [PATCH 725/909] add a time macro --- pixie/time.pxi | 9 +++++++++ pixie/uv.pxi | 3 +++ 2 files changed, 12 insertions(+) create mode 100644 pixie/time.pxi diff --git a/pixie/time.pxi b/pixie/time.pxi new file mode 100644 index 00000000..f392646d --- /dev/null +++ b/pixie/time.pxi @@ -0,0 +1,9 @@ +(ns pixie.time + (:require [pixie.uv :as uv])) + +(defmacro time + [body] + `(let [start# (uv/uv_hrtime) + return# ~body] + (prn (str "Elapsed time: " (/ (- (uv/uv_hrtime) start#) 1000.0) "ms")) + return#)) diff --git a/pixie/uv.pxi b/pixie/uv.pxi index d49da8f7..199942f4 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -52,6 +52,9 @@ (f/defcfn uv_timer_set_repeat) (f/defcfn uv_timer_get_repeat) + ;; Time + (f/defcfn uv_hrtime) + ;; Filesystem From fbbd9c2598fb1fd1eafff7e382b852bef423a5a4 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 23 Apr 2015 12:16:16 +0100 Subject: [PATCH 726/909] fix --- pixie/time.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/time.pxi b/pixie/time.pxi index f392646d..38f8a46b 100644 --- a/pixie/time.pxi +++ b/pixie/time.pxi @@ -5,5 +5,5 @@ [body] `(let [start# (uv/uv_hrtime) return# ~body] - (prn (str "Elapsed time: " (/ (- (uv/uv_hrtime) start#) 1000.0) "ms")) + (prn (str "Elapsed time: " (/ (- (uv/uv_hrtime) start#) 1000000.0) "ms")) return#)) From eb8278ea9cc44cf7eda30eafc059d8fadf5e2a50 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Thu, 23 Apr 2015 13:48:52 -0400 Subject: [PATCH 727/909] loop destructure --- pixie/stdlib.pxi | 98 ++++++++++++++++++++++--------- pixie/vm/compiler.py | 6 +- tests/pixie/tests/test-stdlib.pxi | 8 +++ 3 files changed, 81 insertions(+), 31 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4114ed8f..9842dc1f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -37,6 +37,10 @@ (cons 'let* args))) (set-macro! let) +(def loop (fn* [& args] + (cons 'loop* args))) +(set-macro! loop) + (def identity (fn ^{:doc "The identity function. Returns its argument." :added "0.1"} @@ -132,6 +136,15 @@ result (-reduce coll f init)] (f result))))) +(defn every? + {:doc "Check if every element of the collection satisfies the predicate." + :added "0.1"} + [pred coll] + (cond + (nil? (seq coll)) true + (pred (first coll)) (recur pred (next coll)) + :else false)) + (def map (fn ^{:doc "map - creates a transducer that applies f to every input element" :signatures [[f] [f coll]] :added "0.1"} @@ -1322,6 +1335,24 @@ and implements IAssociative, ILookup and IObject." ([f] (lazy-seq (cons (f) (repeatedly f)))) ([n f] (take n (repeatedly f)))) +(defn interleave + "Returns a seq of all the items in the input collections interleaved" + ([] ()) + ([c1] (seq c1)) + ([c1 c2] + (lazy-seq + (let [s1 (seq c1) + s2 (seq c2)] + (when (and s1 s2) + (cons (first s1) (cons (first s2) + (interleave (next s1) (next s2)))))))) + ([& colls] + (lazy-seq + (let [ss (map seq colls)] + (when (every? identity ss) + (concat (map first ss) + (apply interleave (map next ss)))))))) + (defmacro doseq {:doc "Evaluates all elements of the seq, presumably for side effects. Returns nil." :added "0.1"} @@ -1710,7 +1741,7 @@ not enough elements were present." res)) (defmacro let - {:doc "Makes the bindings availlable in the body. + {:doc "Makes the bindings available in the body. The bindings must be a vector of binding-expr pairs. The binding can be a destructuring binding, as below. @@ -1744,6 +1775,44 @@ For more information, see http://clojure.org/special_forms#binding-forms"} `(let* ~destructured-bindings ~@body))) +(defn take-nth + "Returns a lazy seq of every nth item in coll. Returns a stateful + transducer when no collection is provided." + ([n] + (fn [rf] + (let [ia (atom -1)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [i (swap! ia inc)] + (if (zero? (rem i n)) + (rf result input) + result))))))) + ([n coll] + (lazy-seq + (when-let [s (seq coll)] + (cons (first s) (take-nth n (drop n s))))))) + +(defmacro loop + [bindings & body] + (let [vals (take-nth 2 (drop 1 bindings)) + bindings (take-nth 2 bindings) + binding-syms (map (fn [b] (if (symbol? b) b (gensym))) bindings) + binding-forms (transduce + (map (fn [bind] + (let [[b v s] bind] + (if (symbol? b) + [b v] + [s v b s])))) + concat + [] + (map vector bindings vals binding-syms))] + `(let ~(vec binding-forms) + (loop* ~(vec (interleave binding-syms binding-syms)) + (let ~(vec (interleave bindings binding-syms)) + ~@body))))) + (extend -nth ISeq (fn [s n] (when (empty? s) (throw @@ -1986,15 +2055,6 @@ user => (refer 'pixie.string :exclude '(substring))" m2))] (reduce merge2 (first maps) (next maps))))) -(defn every? - {:doc "Check if every element of the collection satisfies the predicate." - :added "0.1"} - [pred coll] - (cond - (nil? (seq coll)) true - (pred (first coll)) (recur pred (next coll)) - :else false)) - ; If you want a fn that uses destructuring in its parameter list, place ; it after this definition. If you don't, you will get compile failures ; in unrelated files. @@ -2347,24 +2407,6 @@ Calling this function on something that is not ISeqable returns a seq with that (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) -(defn interleave - "Returns a seq of all the items in the input collections interleaved" - ([] ()) - ([c1] (seq c1)) - ([c1 c2] - (lazy-seq - (let [s1 (seq c1) - s2 (seq c2)] - (when (and s1 s2) - (cons (first s1) (cons (first s2) - (interleave (next s1) (next s2)))))))) - ([& colls] - (lazy-seq - (let [ss (map seq colls)] - (when (every? identity ss) - (concat (map first ss) - (apply interleave (map next ss)))))))) - (defn min "Returns the smallest of all the arguments to this function. Assumes arguments are numeric" ([x] x) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index f314ac1c..76ca7fae 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -508,7 +508,7 @@ def compile_fn(form, ctx): if rt.meta(name) is not nil: compile_meta(rt.meta(name), ctx) -LOOP = symbol.symbol(u"loop") +LOOP = symbol.symbol(u"loop*") def compile_fn_body(name, args, body, ctx): new_ctx = Context(rt.name(name), rt.count(args), ctx) @@ -698,7 +698,7 @@ def compile_loop(form, ctx): for i in range(0, rt.count(bindings), 2): binding_count += 1 name = rt.nth(bindings, rt.wrap(i)) - affirm(isinstance(name, symbol.Symbol), u"Loop must bindings must be symbols") + affirm(isinstance(name, symbol.Symbol), u"Loop bindings must be symbols") bind = rt.nth(bindings, rt.wrap(i + 1)) compile_form(bind, ctx) @@ -787,7 +787,7 @@ def compile_local_macro(form, ctx): u"quote": compile_quote, u"recur": compile_recur, u"let*": compile_let, - u"loop": compile_loop, + u"loop*": compile_loop, u"comment": compile_comment, u"var": compile_var, u"catch": compile_catch, diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index b5dd3741..1e938a1b 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -405,6 +405,14 @@ (swap! cnt inc))) @cnt))) +(t/deftest test-loop-destructure + (t/assert= + [3 -3] + (loop [[a b :as vs] [0 0]] + (if (> a 2) + vs + (recur [(inc a) (dec b)]))))) + (t/deftest test-take-while (t/assert= (take-while pos? [1 2 3 -1]) [1 2 3]) (t/assert= (take-while pos? [-1 2]) ()) From e63ba1614351231a65f528995a8d007119806d28 Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Fri, 24 Apr 2015 11:51:54 -0400 Subject: [PATCH 728/909] From #176 Add 2-argument reduce with function identity --- pixie/stdlib.pxi | 7 +++++-- tests/pixie/tests/test-stdlib.pxi | 5 +++++ 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4114ed8f..3af57b15 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -160,8 +160,11 @@ nil)))))] (map (fn [args] (apply f args)) (step colls)))))) -(def reduce (fn [rf init col] - (-reduce col rf init))) +(def reduce (fn + ([rf col] + (reduce rf (rf) col)) + ([rf init col] + (-reduce col rf init)))) (def instance? (fn ^{:doc "Checks if x is an instance of t. diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index b5dd3741..551cbd8b 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -508,3 +508,8 @@ (t/assert-throws? RuntimeException "proto must be a Protocol" (satisfies? [IIndexed :also-not-a-proto] [1 2]))) + +(t/deftest test-reduce + (t/assert= 5050 (reduce + (range 101))) + (t/assert= 3628800 (reduce * (range 1 11))) + (t/assert= 5051 (reduce + 1 (range 101)))) From a6afbfb90b73bd6565c90f1c8cfe784494b5692a Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 26 Apr 2015 14:23:05 -0400 Subject: [PATCH 729/909] additional tests, reorder loop macro in stdlib --- pixie/stdlib.pxi | 134 +++++++++++++++--------------- tests/pixie/tests/test-stdlib.pxi | 10 ++- 2 files changed, 75 insertions(+), 69 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 4f87065c..6f260ade 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -38,7 +38,7 @@ (set-macro! let) (def loop (fn* [& args] - (cons 'loop* args))) + (cons 'loop* args))) (set-macro! loop) (def identity @@ -136,15 +136,6 @@ result (-reduce coll f init)] (f result))))) -(defn every? - {:doc "Check if every element of the collection satisfies the predicate." - :added "0.1"} - [pred coll] - (cond - (nil? (seq coll)) true - (pred (first coll)) (recur pred (next coll)) - :else false)) - (def map (fn ^{:doc "map - creates a transducer that applies f to every input element" :signatures [[f] [f coll]] :added "0.1"} @@ -1338,24 +1329,6 @@ and implements IAssociative, ILookup and IObject." ([f] (lazy-seq (cons (f) (repeatedly f)))) ([n f] (take n (repeatedly f)))) -(defn interleave - "Returns a seq of all the items in the input collections interleaved" - ([] ()) - ([c1] (seq c1)) - ([c1 c2] - (lazy-seq - (let [s1 (seq c1) - s2 (seq c2)] - (when (and s1 s2) - (cons (first s1) (cons (first s2) - (interleave (next s1) (next s2)))))))) - ([& colls] - (lazy-seq - (let [ss (map seq colls)] - (when (every? identity ss) - (concat (map first ss) - (apply interleave (map next ss)))))))) - (defmacro doseq {:doc "Evaluates all elements of the seq, presumably for side effects. Returns nil." :added "0.1"} @@ -1744,7 +1717,7 @@ not enough elements were present." res)) (defmacro let - {:doc "Makes the bindings available in the body. + {:doc "Makes the bindings availlable in the body. The bindings must be a vector of binding-expr pairs. The binding can be a destructuring binding, as below. @@ -1778,44 +1751,6 @@ For more information, see http://clojure.org/special_forms#binding-forms"} `(let* ~destructured-bindings ~@body))) -(defn take-nth - "Returns a lazy seq of every nth item in coll. Returns a stateful - transducer when no collection is provided." - ([n] - (fn [rf] - (let [ia (atom -1)] - (fn - ([] (rf)) - ([result] (rf result)) - ([result input] - (let [i (swap! ia inc)] - (if (zero? (rem i n)) - (rf result input) - result))))))) - ([n coll] - (lazy-seq - (when-let [s (seq coll)] - (cons (first s) (take-nth n (drop n s))))))) - -(defmacro loop - [bindings & body] - (let [vals (take-nth 2 (drop 1 bindings)) - bindings (take-nth 2 bindings) - binding-syms (map (fn [b] (if (symbol? b) b (gensym))) bindings) - binding-forms (transduce - (map (fn [bind] - (let [[b v s] bind] - (if (symbol? b) - [b v] - [s v b s])))) - concat - [] - (map vector bindings vals binding-syms))] - `(let ~(vec binding-forms) - (loop* ~(vec (interleave binding-syms binding-syms)) - (let ~(vec (interleave bindings binding-syms)) - ~@body))))) - (extend -nth ISeq (fn [s n] (when (empty? s) (throw @@ -2058,6 +1993,15 @@ user => (refer 'pixie.string :exclude '(substring))" m2))] (reduce merge2 (first maps) (next maps))))) +(defn every? + {:doc "Check if every element of the collection satisfies the predicate." + :added "0.1"} + [pred coll] + (cond + (nil? (seq coll)) true + (pred (first coll)) (recur pred (next coll)) + :else false)) + ; If you want a fn that uses destructuring in its parameter list, place ; it after this definition. If you don't, you will get compile failures ; in unrelated files. @@ -2410,6 +2354,24 @@ Calling this function on something that is not ISeqable returns a seq with that (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) +(defn interleave + "Returns a seq of all the items in the input collections interleaved" + ([] ()) + ([c1] (seq c1)) + ([c1 c2] + (lazy-seq + (let [s1 (seq c1) + s2 (seq c2)] + (when (and s1 s2) + (cons (first s1) (cons (first s2) + (interleave (next s1) (next s2)))))))) + ([& colls] + (lazy-seq + (let [ss (map seq colls)] + (when (every? identity ss) + (concat (map first ss) + (apply interleave (map next ss)))))))) + (defn min "Returns the smallest of all the arguments to this function. Assumes arguments are numeric" ([x] x) @@ -2421,3 +2383,41 @@ Calling this function on something that is not ISeqable returns a seq with that ([x] x) ([x y] (if (> x y) x y)) ([x y & zs] (apply max (max x y) zs))) + +(defn take-nth + "Returns a lazy seq of every nth item in coll. Returns a stateful + transducer when no collection is provided." + ([n] + (fn [rf] + (let [ia (atom -1)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [i (swap! ia inc)] + (if (zero? (rem i n)) + (rf result input) + result))))))) + ([n coll] + (lazy-seq + (when-let [s (seq coll)] + (cons (first s) (take-nth n (drop n s))))))) + +(defmacro loop + [bindings & body] + (let [vals (take-nth 2 (drop 1 bindings)) + bindings (take-nth 2 bindings) + binding-syms (map (fn [b] (if (symbol? b) b (gensym))) bindings) + binding-forms (transduce + (map (fn [bind] + (let [[b v s] bind] + (if (symbol? b) + [b v] + [s v b s])))) + concat + [] + (map vector bindings vals binding-syms))] + `(let ~(vec binding-forms) + (loop* ~(vec (interleave binding-syms binding-syms)) + (let ~(vec (interleave bindings binding-syms)) + ~@body))))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index c40dc7ca..c4816b44 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -405,13 +405,19 @@ (swap! cnt inc))) @cnt))) -(t/deftest test-loop-destructure +(t/deftest test-loop (t/assert= [3 -3] (loop [[a b :as vs] [0 0]] (if (> a 2) vs - (recur [(inc a) (dec b)]))))) + (recur [(inc a) (dec b)])))) + (t/assert= + 3 + (loop [a 1] + (if (> a 2) + a + (recur (inc a)))))) (t/deftest test-take-while (t/assert= (take-while pos? [1 2 3 -1]) [1 2 3]) From 34e129a41ccbd244e44934e81f3668af4e847837 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Sun, 26 Apr 2015 10:16:41 +0100 Subject: [PATCH 730/909] Add comment reader macro --- pixie/vm/reader.py | 10 ++++++++-- tests/pixie/tests/test-readeval.pxi | 8 ++++++++ 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 6173859b..d3872d12 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -627,9 +627,15 @@ def invoke(self, rdr, ch): if itm != rdr: acc = acc.conj(itm) +class CommentReader(ReaderHandler): + def invoke(self, rdr, ch): + read_inner(rdr, True, always_return_form=True) + return rdr + dispatch_handlers = { - u"{": SetReader(), - u"(": FnReader() + u"{": SetReader(), + u"(": FnReader(), + u"_": CommentReader() } class DispatchReader(ReaderHandler): diff --git a/tests/pixie/tests/test-readeval.pxi b/tests/pixie/tests/test-readeval.pxi index 4344bff6..636dca23 100644 --- a/tests/pixie/tests/test-readeval.pxi +++ b/tests/pixie/tests/test-readeval.pxi @@ -63,3 +63,11 @@ (t/assert= (read-string "{:foo :bar ; a comment\n }") '{:foo :bar}) (t/assert= (read-string "#{:foo ; a comment\n }") '#{:foo}) (t/assert= (read-string "{:foo ; a comment\n :bar }") '{:foo :bar})) + +(t/deftest test-comment-reader-macro + (t/assert= (read-string "(foo #_bar baz)") '(foo baz)) + (t/assert= (read-string "(foo #_ bar baz)") '(foo baz)) + (t/assert= (read-string "(foo #_ #_ bar baz)") '(foo)) + (t/assert= (read-string "(foo #_(bar goo) baz)") '(foo baz)) + (t/assert= (read-string "(foo bar #_baz)") '(foo bar)) + (t/assert= (read-string "(foo bar #_ ; comment \n baz)") '(foo bar))) From d954d72f7752634570be21acc162639dea1ad99c Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Mon, 4 May 2015 06:33:38 -0600 Subject: [PATCH 731/909] optimized performance and memory usage of custom types --- pixie/vm/custom_types.py | 104 ++++++++++++++++++++++++++++++++++----- 1 file changed, 91 insertions(+), 13 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 081e57fc..b1aa73c9 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -6,6 +6,9 @@ from pixie.vm.keyword import Keyword import pixie.vm.rt as rt +MAX_FIELDS = 32 + + class CustomType(Type): _immutable_fields_ = ["_slots", "_rev?"] def __init__(self, name, slots): @@ -38,10 +41,9 @@ def get_num_slots(self): class CustomTypeInstance(Object): _immutable_fields_ = ["_type"] - def __init__(self, type, fields): + def __init__(self, type): affirm(isinstance(type, CustomType), u"Can't create a instance of a non custom type") self._custom_type = type - self._fields = fields def type(self): return self._custom_type @@ -52,17 +54,17 @@ def set_field(self, name, val): runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type())), u"pixie.stdlib/InvalidFieldException") - old_val = self._fields[idx] + old_val = self.get_field_by_idx(idx) if isinstance(old_val, AbstractMutableCell): - old_val.set_mutable_cell_value(self._custom_type, self._fields, name, idx, val) + old_val.set_mutable_cell_value(self._custom_type, self, name, idx, val) else: self._custom_type.set_mutable(name) - self._fields[idx] = val + self.set_field_by_idx(idx, val) return self @jit.elidable_promote() def _get_field_immutable(self, idx, rev): - return self._fields[idx] + return self.get_field_by_idx(idx) def get_field_immutable(self, idx): tp = self._custom_type @@ -76,7 +78,7 @@ def get_field(self, name): u"pixie.stdlib/InvalidFieldException") if self._custom_type.is_mutable(name): - value = self._fields[idx] + value = self.get_field_by_idx(idx) else: value = self.get_field_immutable(idx) @@ -86,11 +88,86 @@ def get_field(self, name): return value +create_type_prefix = """ +def new_inst(tp, fields): + l = len(fields) + if l >= {max_c}: + runtime_error(u"Too many fields for type", u"pixie.stdlib/TooManyFields") +""" + +clause_template = """ + elif l == {c}: + return CustomType{c}(tp, fields) +""" + +def gen_ct_code(): + acc = create_type_prefix.format(max_c=MAX_FIELDS + 1) + for x in range(MAX_FIELDS + 1): + acc += clause_template.format(c=x) + + #print acc + return acc + +exec gen_ct_code() + + +type_template = """ +class CustomType{c}(CustomTypeInstance): + def __init__(self, tp, fields): + CustomTypeInstance.__init__(self, tp) + {self_fields_list} = fields + + def get_field_by_idx(self, idx): + if idx >= {max}: + return None +""" + +get_field_by_idx_template = """ + elif idx == {c}: + return self._custom_field{c} +""" + +set_field_prefix_template = """ + def set_field_by_idx(self, idx, val): + if idx >= {max}: + return +""" + +set_field_by_idx_template = """ + elif idx == {c}: + self._custom_field{c} = val +""" + +def gen_ct_class_code(): + acc = "" + for x in range(MAX_FIELDS + 1): + if x == 0: + self_fields_list = "_null_" + elif x == 1: + self_fields_list = "self._custom_field0," + else: + self_fields_list = ",".join(map(lambda x: "self._custom_field" + str(x), range(x))) + acc += type_template.format(c=x, self_fields_list=self_fields_list, max=x) + for y in range(x): + acc += get_field_by_idx_template.format(c=y) + + acc += set_field_prefix_template.format(max=x) + + for y in range(x): + acc += set_field_by_idx_template.format(c=y) + + + + #print acc + return acc + +exec gen_ct_class_code() + + @as_var("create-type") def create_type(type_name, fields): affirm(isinstance(type_name, Keyword), u"Type name must be a keyword") - field_count = rt.count(fields) acc = {} for i in range(rt.count(fields)): val = rt.nth(fields, rt.wrap(i)) @@ -101,6 +178,7 @@ def create_type(type_name, fields): return CustomType(rt.name(type_name), acc) @as_var("new") +@jit.unroll_safe def _new__args(args): affirm(len(args) >= 1, u"new takes at least one parameter") tp = args[0] @@ -116,7 +194,7 @@ def _new__args(args): val = FloatMutableCell(val.float_val()) arr[x] = val - return CustomTypeInstance(tp, arr) + return new_inst(tp, arr) @as_var("set-field!") def set_field(inst, field, val): @@ -152,9 +230,9 @@ def set_mutable_cell_value(self, ct, fields, nm, idx, value): if not isinstance(value, Integer): ct.set_mutable(nm) if isinstance(value, Float): - fields[idx] = FloatMutableCell(value.float_val()) + fields.set_field_by_idx(idx, FloatMutableCell(value.float_val())) else: - fields[idx] = value + fields.set_field_by_idx(idx, value) else: self._mutable_integer_val = value.int_val() @@ -169,9 +247,9 @@ def set_mutable_cell_value(self, ct, fields, nm, idx, value): if not isinstance(value, Float): ct.set_mutable(nm) if isinstance(value, Integer): - fields[idx] = IntegerMutableCell(value.int_val()) + fields.set_field_by_idx(idx, IntegerMutableCell(value.int_val())) else: - fields[idx] = value + fields.set_field_by_idx(idx, value) else: self._mutable_float_val = value.float_val() From 765b20c7685df980a0ab1fd231958deab2b90c44 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 4 May 2015 16:58:22 +0100 Subject: [PATCH 732/909] fix formatting of time --- pixie/time.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/time.pxi b/pixie/time.pxi index 38f8a46b..a045b14f 100644 --- a/pixie/time.pxi +++ b/pixie/time.pxi @@ -5,5 +5,5 @@ [body] `(let [start# (uv/uv_hrtime) return# ~body] - (prn (str "Elapsed time: " (/ (- (uv/uv_hrtime) start#) 1000000.0) "ms")) + (prn (str "Elapsed time: " (/ (- (uv/uv_hrtime) start#) 1000000.0) " ms")) return#)) From 6efa5cf6e86ed510e2b3154a273f9ecda2f44acb Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Tue, 12 May 2015 17:10:20 +0100 Subject: [PATCH 733/909] add remove function --- pixie/stdlib.pxi | 9 +++++++++ tests/pixie/tests/test-stdlib.pxi | 3 +++ 2 files changed, 12 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6f260ade..6f14c9db 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1876,6 +1876,15 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (cons f (filter pred r)) (filter pred r))))))) +(defn remove + {:doc "Removes any element from the collection which matches the predicate. The complement of filter." + :signatures [[pred] [pred coll]] + :added "0.1"} + ([pred] + (filter (complement pred))) + ([pred coll] + (filter (complement pred) coll))) + (defn distinct {:doc "Returns the distinct elements in the collection." :signatures [[] [coll]] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index c4816b44..25643160 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -302,6 +302,9 @@ (t/assert= (into {} (filter (fn [[_ v]] (odd? v)) {:a 1, :b 2, :c 3, :d 4})) {:a 1 :c 3})) +(t/deftest test-remove + (t/assert= (remove even? [1 2 3 4 5]) '(1 3 5))) + (t/deftest test-distinct (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) From 3c62749795970a4b2ca6c3b1c7a8816ec36518c0 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 15 May 2015 12:10:57 +0100 Subject: [PATCH 734/909] tcp streams satisfy IReduce * refactor out common io functionality to io/common --- pixie/io-blocking.pxi | 22 +++------------- pixie/io.pxi | 18 +++----------- pixie/io/common.pxi | 58 +++++++++++++++++++++++++++++++++++++++++++ pixie/io/tcp.pxi | 8 +++++- pixie/io/tty.pxi | 11 +------- 5 files changed, 73 insertions(+), 44 deletions(-) create mode 100644 pixie/io/common.pxi diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index 9b4f3943..2d359d5b 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -1,5 +1,6 @@ (ns pixie.io-blocking - (:require [pixie.streams :as st :refer :all])) + (:require [pixie.streams :as st :refer :all] + [pixie.io.common :as common])) (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) @@ -15,21 +16,6 @@ (def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) - -(def DEFAULT-BUFFER-SIZE 1024) - -(defn default-stream-reducer [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc))))) - (deftype FileStream [fp] IInputStream (read [this buffer len] @@ -50,7 +36,7 @@ (fclose fp)) IReduce (-reduce [this f init] - (default-stream-reducer this f init))) + (common/stream-reducer this f init))) (defn open-read {:doc "Open a file for reading, returning a IInputStream" @@ -144,7 +130,7 @@ (pclose fp)) IReduce (-reduce [this f init] - (default-stream-reducer this f init))) + (common/stream-reducer this f init))) (defn popen-read diff --git a/pixie/io.pxi b/pixie/io.pxi index 9929018e..b5092ddc 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -12,9 +12,6 @@ (uv/defuvfsfn fs_write [file bufs nbufs offset] :result) (uv/defuvfsfn fs_close [file] :result) - -(def DEFAULT-BUFFER-SIZE 1024) - (deftype FileStream [fp offset uvbuf] IInputStream (read [this buffer len] @@ -40,16 +37,7 @@ (fs_close fp)) IReduce (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) + (common/stream-reducer this f init))) (defn open-read @@ -134,13 +122,13 @@ (defn buffered-output-stream ([downstream] - (buffered-output-stream downstream DEFAULT-BUFFER-SIZE)) + (buffered-output-stream downstream common/DEFAULT-BUFFER-SIZE)) ([downstream size] (->BufferedOutputStream downstream 0 (buffer size)))) (defn buffered-input-stream ([upstream] - (buffered-input-stream upstream DEFAULT-BUFFER-SIZE)) + (buffered-input-stream upstream common/DEFAULT-BUFFER-SIZE)) ([upstream size] (let [b (buffer size)] (set-buffer-count! b size) diff --git a/pixie/io/common.pxi b/pixie/io/common.pxi new file mode 100644 index 00000000..9f4737c5 --- /dev/null +++ b/pixie/io/common.pxi @@ -0,0 +1,58 @@ +(ns pixie.io.common + "Common functionality for handling IO" + (:require [pixie.streams :refer :all])) + +(def DEFAULT-BUFFER-SIZE 1024) + +(defn stream-reducer [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) + +(defn stream-reader [this buf len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) + read-cb (atom nil)] + (st/call-cc (fn [k] + (reset! read-cb (ffi/ffi-prep-callback + uv/uv_read_cb + (fn [stream nread uv-buf] + (set-buffer-count! buffer nread) + (try + (dispose! alloc-cb) + (dispose! @read-cb) + ;(dispose! uv-buf) + (uv/uv_read_stop stream) + (st/run-and-process k (or + (st/exception-on-uv-error nread) + nread)) + (catch ex + (println ex)))))) + (uv/uv_read_start uv-client alloc-cb @read-cb))))) + +(defn stream-writer + [this buf] + (let [write-cb (atom nil) + uv_write (uv/uv_write_t)] + (ffi/set! uv-write-buf :base buffer) + (ffi/set! uv-write-buf :len (count buffer)) + (st/call-cc + (fn [k] + (reset! write-cb (ffi/ffi-prep-callback + uv/uv_write_cb + (fn [req status] + (try + (dispose! @write-cb) + ;(uv/uv_close uv_write st/close_cb) + (st/run-and-process k status) + (catch ex + (println ex)))))) + (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index 2ca440f4..5e22dda9 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -1,9 +1,12 @@ (ns pixie.io.tcp (:require [pixie.stacklets :as st] [pixie.streams :refer [IInputStream read IOutputStream write]] + [pixie.io.common :as common] [pixie.uv :as uv] [pixie.ffi :as ffi])) +(def DEFAULT-BUFFER-SIZE 1024) + (defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb] IDisposable (-dispose! [this] @@ -55,7 +58,10 @@ IDisposable (-dispose! [this] (dispose! uv-write-buf) - (uv/uv_close uv-client st/close_cb))) + (uv/uv_close uv-client st/close_cb)) + IReduce + (-reduce [this f init] + (common/default-stream-reducer this f init))) (defn launch-tcp-client-from-server [svr] (assert (instance? TCPServer svr) "Requires a TCPServer as the first argument") diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi index 24a909cd..060c5f0d 100644 --- a/pixie/io/tty.pxi +++ b/pixie/io/tty.pxi @@ -39,16 +39,7 @@ IReduce (-reduce [this f init] - (let [buf (buffer DEFAULT-BUFFER-SIZE) - rrf (preserving-reduced f)] - (loop [acc init] - (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] - (if (> read-count 0) - (let [result (reduce rrf acc buf)] - (if (not (reduced? result)) - (recur result) - @result)) - acc)))))) + (common/default-stream-reducer this f init))) (deftype TTYOutputStream [uv-client uv-write-buf] IOutputStream From 41203fe7c6ad152b5374055ea91f1f36348fb92b Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 15 May 2015 12:19:18 +0100 Subject: [PATCH 735/909] refactor tcp and tty --- pixie/io.pxi | 1 + pixie/io/common.pxi | 11 +++++++---- pixie/io/tcp.pxi | 44 +++----------------------------------------- pixie/io/tty.pxi | 45 ++++----------------------------------------- 4 files changed, 15 insertions(+), 86 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index b5092ddc..f90ec346 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -2,6 +2,7 @@ (:require [pixie.streams :as st :refer :all] [pixie.streams.utf8 :as utf8] [pixie.io-blocking :as io-blocking] + [pixie.io.common :as common] [pixie.uv :as uv] [pixie.stacklets :as st] [pixie.ffi :as ffi] diff --git a/pixie/io/common.pxi b/pixie/io/common.pxi index 9f4737c5..ee23bdef 100644 --- a/pixie/io/common.pxi +++ b/pixie/io/common.pxi @@ -1,6 +1,9 @@ (ns pixie.io.common "Common functionality for handling IO" - (:require [pixie.streams :refer :all])) + (:require [pixie.streams :refer :all] + [pixie.uv :as uv] + [pixie.stacklets :as st] + [pixie.ffi :as ffi])) (def DEFAULT-BUFFER-SIZE 1024) @@ -16,7 +19,7 @@ @result)) acc))))) -(defn stream-reader [this buf len] +(defn cb-stream-reader [uv-client buffer len] (assert (<= (buffer-capacity buffer) len) "Not enough capacity in the buffer") (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) @@ -38,8 +41,8 @@ (println ex)))))) (uv/uv_read_start uv-client alloc-cb @read-cb))))) -(defn stream-writer - [this buf] +(defn cb-stream-writer + [uv-client uv-write-buf buffer] (let [write-cb (atom nil) uv_write (uv/uv_write_t)] (ffi/set! uv-write-buf :base buffer) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index 5e22dda9..b3945782 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -5,8 +5,6 @@ [pixie.uv :as uv] [pixie.ffi :as ffi])) -(def DEFAULT-BUFFER-SIZE 1024) - (defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb] IDisposable (-dispose! [this] @@ -17,51 +15,17 @@ (deftype TCPStream [uv-client uv-write-buf] IInputStream (read [this buffer len] - (assert (<= (buffer-capacity buffer) len) - "Not enough capacity in the buffer") - (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) - read-cb (atom nil)] - (st/call-cc (fn [k] - (reset! read-cb (ffi/ffi-prep-callback - uv/uv_read_cb - (fn [stream nread uv-buf] - (set-buffer-count! buffer nread) - (try - (dispose! alloc-cb) - (dispose! @read-cb) - ;(dispose! uv-buf) - (uv/uv_read_stop stream) - (st/run-and-process k (or - (st/exception-on-uv-error nread) - nread)) - (catch ex - (println ex)))))) - (uv/uv_read_start uv-client alloc-cb @read-cb))))) + (common/cb-stream-reader uv-client buffer len)) IOutputStream (write [this buffer] - (let [write-cb (atom nil) - uv_write (uv/uv_write_t)] - (ffi/set! uv-write-buf :base buffer) - (ffi/set! uv-write-buf :len (count buffer)) - (st/call-cc - (fn [k] - (reset! write-cb (ffi/ffi-prep-callback - uv/uv_write_cb - (fn [req status] - (try - (dispose! @write-cb) - ;(uv/uv_close uv_write st/close_cb) - (st/run-and-process k status) - (catch ex - (println ex)))))) - (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) + (common/cb-stream-writer uv-client uv-write-buf buffer)) IDisposable (-dispose! [this] (dispose! uv-write-buf) (uv/uv_close uv-client st/close_cb)) IReduce (-reduce [this f init] - (common/default-stream-reducer this f init))) + (common/stream-reducer this f init))) (defn launch-tcp-client-from-server [svr] (assert (instance? TCPServer svr) "Requires a TCPServer as the first argument") @@ -74,8 +38,6 @@ (do (uv/uv_close client nil) svr)))) - - (defn tcp-server "Creates a TCP server on the given ip (as a string) and port (as an integer). Returns a TCPServer that can be shutdown with dispose!. on-connection is a function that will be passed a TCPStream for each connecting client." diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi index 060c5f0d..0acedfe8 100644 --- a/pixie/io/tty.pxi +++ b/pixie/io/tty.pxi @@ -3,63 +3,26 @@ [pixie.streams :refer [IInputStream read IOutputStream write]] [pixie.uv :as uv] [pixie.io :as io] + [pixie.io.common :as common] [pixie.system :as sys] [pixie.ffi :as ffi])) -(def DEFAULT-BUFFER-SIZE 1024) - (deftype TTYInputStream [uv-client uv-write-buf] IInputStream (read [this buf len] - (assert (<= (buffer-capacity buf) len) - "Not enough capacity in the buffer") - (let [alloc-cb (uv/-prep-uv-buffer-fn buf len) - read-cb (atom nil)] - (st/call-cc (fn [k] - (reset! read-cb (ffi/ffi-prep-callback - uv/uv_read_cb - (fn [stream nread uv-buf] - (set-buffer-count! buf nread) - (try - (dispose! alloc-cb) - (dispose! @read-cb) - ;(dispose! uv-buf) - (uv/uv_read_stop stream) - (st/run-and-process k (or - (st/exception-on-uv-error nread) - nread)) - (catch ex - (println ex)))))) - (uv/uv_read_start uv-client alloc-cb @read-cb))))) - + (common/cb-stream-reader uv-client buf len)) IDisposable (-dispose! [this] (dispose! uvbuf) (fs_close fp)) - IReduce (-reduce [this f init] - (common/default-stream-reducer this f init))) + (common/stream-reducer this f init))) (deftype TTYOutputStream [uv-client uv-write-buf] IOutputStream (write [this buffer] - (let [write-cb (atom nil) - uv_write (uv/uv_write_t)] - (ffi/set! uv-write-buf :base buffer) - (ffi/set! uv-write-buf :len (count buffer)) - (st/call-cc - (fn [k] - (reset! write-cb (ffi/ffi-prep-callback - uv/uv_write_cb - (fn [req status] - (try - (dispose! @write-cb) - (st/run-and-process k status) - (catch ex - (println ex)))))) - (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) - + (common/cb-stream-writer uv-client uv-write-buf buffer)) IDisposable (-dispose! [this] (dispose! uv-write-buf) From ce47508849b9cd52c6d60845c0921a877e61149a Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 15 May 2015 12:53:14 +0100 Subject: [PATCH 736/909] remove unused requires --- pixie/io-blocking.pxi | 3 --- pixie/io.pxi | 1 - pixie/io/tty.pxi | 4 +--- 3 files changed, 1 insertion(+), 7 deletions(-) diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index 2d359d5b..ecfbd5ee 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -91,7 +91,6 @@ (reduced cnt) (+ cnt written))))))) - (defn spit [filename val] (transduce (map int) (file-output-rf filename) @@ -132,7 +131,6 @@ (-reduce [this f init] (common/stream-reducer this f init))) - (defn popen-read {:doc "Open a file for reading, returning a IInputStream" :added "0.1"} @@ -140,7 +138,6 @@ (assert (string? command) "Command must be a string") (->ProcessInputStream (popen command "r"))) - (defn run-command [command] (let [c (->ProcessInputStream (popen command "r")) result (transduce diff --git a/pixie/io.pxi b/pixie/io.pxi index f90ec346..e589b34b 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -40,7 +40,6 @@ (-reduce [this f init] (common/stream-reducer this f init))) - (defn open-read {:doc "Open a file for reading, returning a IInputStream" :added "0.1"} diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi index 0acedfe8..cc75ad2b 100644 --- a/pixie/io/tty.pxi +++ b/pixie/io/tty.pxi @@ -2,10 +2,8 @@ (:require [pixie.stacklets :as st] [pixie.streams :refer [IInputStream read IOutputStream write]] [pixie.uv :as uv] - [pixie.io :as io] [pixie.io.common :as common] - [pixie.system :as sys] - [pixie.ffi :as ffi])) + [pixie.system :as sys])) (deftype TTYInputStream [uv-client uv-write-buf] IInputStream From eff528b20be407916713f1ddadb0f3a32a54d6b0 Mon Sep 17 00:00:00 2001 From: Julius Ring Date: Tue, 19 May 2015 20:24:35 +0200 Subject: [PATCH 737/909] Make comp work when the number of arguments is divisible by 4. --- pixie/stdlib.pxi | 1 + tests/pixie/tests/test-stdlib.pxi | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6f14c9db..f9754580 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -767,6 +767,7 @@ there's a value associated with the key. Use `some` for checking for values." :examples [["((comp inc first) [41 2 3])" nil 42]] :signatures [[f] [f & fs]] :added "0.1"} + ([] identity) ([f] f) ([f1 f2] (fn [& args] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 25643160..7b98f74a 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -530,3 +530,7 @@ (t/assert= 5050 (reduce + (range 101))) (t/assert= 3628800 (reduce * (range 1 11))) (t/assert= 5051 (reduce + 1 (range 101)))) + +(t/deftest test-comp + (t/assert= 5 ((comp inc inc inc inc) 1)) + (t/assert= :xyz ((comp) :xyz))) From d261289bf299c4fb57021428fecf35978a7e5ff4 Mon Sep 17 00:00:00 2001 From: Justin Jaffray Date: Tue, 19 May 2015 14:43:35 -0400 Subject: [PATCH 738/909] exchange_result_libffi -> exchange_result In current pypy (as of https://bitbucket.org/pypy/pypy/commits/749bf9a13d9ccf18ed9be89ac3f3ec4dd6b11a76), exchange_result_libffi has been removed. This means that anyone installing Pixie now and pulling down the latest pypy will be unable to build (as in #318). A lot of the logic for dealing with endianness has been pushed into rpython, meaning pixie/pypy don't need to deal with it. --- pixie/vm/libs/ffi.py | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index b8b92512..a733464d 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -119,7 +119,7 @@ def prep_exb(self, args): return exb, tokens def get_ret_val_from_buffer(self, exb): - offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_result_libffi)) + offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_result)) ret_val = self._ret_type.ffi_get_value(offset_p) return ret_val @@ -804,7 +804,6 @@ def prep_ffi_call__args(args): import sys -BIG_ENDIAN = sys.byteorder == 'big' USE_C_LIBFFI_MSVC = getattr(clibffi, 'USE_C_LIBFFI_MSVC', False) @@ -869,16 +868,6 @@ def fb_build_exchange(self, cif_descr): exchange_offset = rffi.sizeof(rffi.VOIDP) * nargs exchange_offset = self.align_arg(exchange_offset) cif_descr.exchange_result = exchange_offset - cif_descr.exchange_result_libffi = exchange_offset - - if BIG_ENDIAN and self.fresult.is_primitive_integer: - # For results of precisely these types, libffi has a - # strange rule that they will be returned as a whole - # 'ffi_arg' if they are smaller. The difference - # only matters on big-endian. - if self.fresult.size < SIZE_OF_FFI_ARG: - diff = SIZE_OF_FFI_ARG - self.fresult.size - cif_descr.exchange_result += diff # then enough room for the result, rounded up to sizeof(ffi_arg) exchange_offset += max(rffi.getintfield(self.rtype, 'c_size'), From 01c8776a2f9b9e3f87f78c6c70ab8bedd828b8d1 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 20 May 2015 12:54:08 +0100 Subject: [PATCH 739/909] keywords can now be fully qualified as in clojure --- pixie/vm/code.py | 20 ++++++++++++-------- pixie/vm/reader.py | 19 +++++++++++-------- tests/pixie/tests/test-keywords.pxi | 8 ++++++++ 3 files changed, 31 insertions(+), 16 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index b1ac4caa..bba4631a 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -544,6 +544,17 @@ def add_refer_symbol(self, sym, var): def include_stdlib(self): stdlib = _ns_registry.find_or_make(u"pixie.stdlib") self.add_refer(stdlib, refer_all=True) + + def resolve_ns(self, ns_alias): + refer = self._refers.get(ns_alias, None) + resolved_ns = None + if refer is not None: + resolved_ns = refer._namespace + if resolved_ns is None: + resolved_ns = _ns_registry.get(ns_alias, None) + if resolved_ns is None: + affirm(False, u"Unable to resolve namespace: " + ns_alias + u" inside namespace " + self._name) + return resolved_ns def resolve(self, s, use_refers=True): import pixie.vm.symbol as symbol @@ -552,14 +563,7 @@ def resolve(self, s, use_refers=True): name = rt.name(s) if ns is not None: - refer = self._refers.get(ns, None) - resolved_ns = None - if refer is not None: - resolved_ns = refer._namespace - if resolved_ns is None: - resolved_ns = _ns_registry.get(ns, None) - if resolved_ns is None: - affirm(False, u"Unable to resolve namespace: " + ns + u" inside namespace " + self._name) + resolved_ns = self.resolve_ns(ns) else: resolved_ns = self diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index d3872d12..64ad57c9 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -333,21 +333,24 @@ def invoke(self, rdr, ch): return cons(symbol(u"quote"), cons(itm)) class KeywordReader(ReaderHandler): + def fqd(self, itm): + ns_alias = rt.namespace(itm) + current_nms = rt.ns.deref() + + if ns_alias is None: + return keyword(rt.name(itm), rt.name(current_nms)) + else: + ns_fqd = current_nms.resolve_ns(ns_alias) + return keyword(rt.name(itm), rt.name(ns_fqd)) + def invoke(self, rdr, ch): - nms = u"" ch = rdr.read() if ch == u":": itm = read_inner(rdr, True) - nms = rt.name(rt.ns.deref()) + return self.fqd(itm) else: rdr.unread() itm = read_inner(rdr, True) - - affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") - if nms: - affirm(rt.namespace(itm) is None, u"Kewyword cannot have two namespaces") - return keyword(rt.name(itm), nms) - else: return keyword(rt.name(itm), rt.namespace(itm)) class LiteralStringReader(ReaderHandler): diff --git a/tests/pixie/tests/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi index 5156d27a..cdd0672c 100644 --- a/tests/pixie/tests/test-keywords.pxi +++ b/tests/pixie/tests/test-keywords.pxi @@ -14,6 +14,14 @@ (t/assert= (namespace :cat/dog) "cat") (t/assert= (namespace ::foo) "pixie.tests.test-keywords")) +(t/deftest fqd-keywords + (t/assert-throws? (read-string "::x/bar")) + (t/assert-throws? (read-string "::a.b/foo")) + (refer-ns 'my.other.ns 'my.fake.core 'fake) + (binding [*ns* (the-ns 'my.other.ns)] + (t/assert= :my.fake.core/foo (read-string "::fake/foo")) + (t/assert= :my.fake.core/foo (read-string "::my.fake.core/foo")) + (t/assert-throws? (read-string "::f/foo")))) (t/deftest keyword-equality (t/assert= :foo/bar :foo/bar) From d99623d395f23a0ab4276a36df85fcc43d49fe98 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 31 May 2015 10:29:35 +0100 Subject: [PATCH 740/909] fixes typos --- pixie/channels.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/channels.pxi b/pixie/channels.pxi index c44cc060..069bb2c6 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -46,7 +46,7 @@ (b/empty-buffer? puts)) nil (let [[val cfn] (b/remove! puts)] - (if (cancelled? cfn) + (if (canceled? cfn) (recur) (do (st/-run-later (partial cfn true)) (b/add! buffer val) From 54d503272e54256f7d4be7eb9b007c4ee3b7f92a Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 31 May 2015 13:45:52 +0100 Subject: [PATCH 741/909] adds ns-aliases lists all the aliases in a given a namespace as a map of alias -> ns --- pixie/stdlib.pxi | 5 +++++ pixie/vm/code.py | 2 +- pixie/vm/stdlib.py | 20 ++++++++++++++++++++ tests/pixie/tests/test-stdlib.pxi | 9 +++++++++ 4 files changed, 35 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6f14c9db..a973ec1e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2430,3 +2430,8 @@ Calling this function on something that is not ISeqable returns a seq with that (loop* ~(vec (interleave binding-syms binding-syms)) (let ~(vec (interleave bindings binding-syms)) ~@body))))) + +(extend -str Namespace + (fn [v] (str ""))) + +(extend -repr Namespace -str) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index b1ac4caa..b79fd8cd 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -540,7 +540,7 @@ def add_refer_symbol(self, sym, var): self._registry[name] = var return var - + def include_stdlib(self): stdlib = _ns_registry.find_or_make(u"pixie.stdlib") self.add_refer(stdlib, refer_all=True) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 05c309c3..5b4326a0 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -553,6 +553,26 @@ def ns_map(ns): return nil +@as_var("ns-aliases") +def ns_aliases(ns): + from pixie.vm.code import Namespace + from pixie.vm.symbol import Symbol + affirm(isinstance(ns, Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") + + if isinstance(ns, Symbol): + ns = rt.the_ns(ns) + if ns is nil: + return nil + + if isinstance(ns, Namespace): + m = rt.hashmap() + for alias in ns._refers: + refered_ns = ns._refers[alias]._namespace + m = rt.assoc(m, rt.symbol(rt.wrap(alias)), refered_ns) + return m + + return nil + @as_var("refer-ns") def refer(ns, refer, alias): from pixie.vm.symbol import Symbol diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 25643160..d291874b 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -392,6 +392,15 @@ (t/assert= (set (keys (ns-map 'foo))) #{'bar 'baz})) +(t/deftest test-ns-aliases + (in-ns :ns-to-require) + (in-ns :my-fake-ns) + (require ns-to-require :as some-alias) + (in-ns :pixie.tests.test-stdlib) + (t/assert= {'some-alias (the-ns 'ns-to-require) + 'pixie.stdlib (the-ns 'pixie.stdlib)} + (ns-aliases (the-ns 'my-fake-ns)))) + (t/deftest test-while (t/assert= (while (pos? 0) true ) nil) (t/assert= (while (pos? 0) false) nil) From 41aca09e3a737a64eed97333bbef083ae50a3f34 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 1 Jun 2015 09:41:47 +0100 Subject: [PATCH 742/909] remove unused buffer for input streams --- pixie/channels.pxi | 2 +- pixie/io.pxi | 8 ++++---- pixie/io/tty.pxi | 10 ++++------ pixie/stacklets.pxi | 2 +- pixie/uv.pxi | 2 +- tests/pixie/tests/test-io.pxi | 4 +++- 6 files changed, 14 insertions(+), 14 deletions(-) diff --git a/pixie/channels.pxi b/pixie/channels.pxi index c44cc060..84b60474 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -23,7 +23,7 @@ (cond (= idx 0) val (= idx 1) cfn - :else (throw "Index out of range"))) + :else (throw [::OutOfRangeException "Index out of range"]))) (-nth-not-found [this idx not-found] (cond (= idx 0) val diff --git a/pixie/io.pxi b/pixie/io.pxi index e589b34b..a37188f6 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -79,7 +79,7 @@ _ (pixie.ffi/set! uvbuf :len (- (count buffer) buffer-offset)) write-count (fs_write fp uvbuf 1 offset)] (when (neg? write-count) - (throw (uv/uv_err_name read-count))) + (throw [::FileOutputStreamException (uv/uv_err_name read-count)])) (set-field! this :offset (+ offset write-count)) (if (< (+ buffer-offset write-count) (count buffer)) (recur (+ buffer-offset write-count)) @@ -136,7 +136,7 @@ (defn throw-on-error [result] (when (neg? result) - (throw (uv/uv_err_name result))) + (throw [::UVException (uv/uv_err_name result)])) result) (defn open-write @@ -169,7 +169,7 @@ utf8/utf8-output-stream-rf) (str content)) - :else (throw "Expected a string or IOutputStream"))) + :else (throw [::Exception "Expected a string or IOutputStream"]))) (defn slurp "Reads in the contents of input. Input must be a filename or an IInputStream" @@ -177,7 +177,7 @@ (let [stream (cond (string? input) (open-read input) (satisfies? IInputStream input) input - :else (throw "Expected a string or an IInputStream")) + :else (throw [:pixie.io/Exception "Expected a string or an IInputStream"])) result (transduce (map char) string-builder diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi index cc75ad2b..44386f3e 100644 --- a/pixie/io/tty.pxi +++ b/pixie/io/tty.pxi @@ -5,14 +5,13 @@ [pixie.io.common :as common] [pixie.system :as sys])) -(deftype TTYInputStream [uv-client uv-write-buf] +(deftype TTYInputStream [uv-client] IInputStream (read [this buf len] (common/cb-stream-reader uv-client buf len)) IDisposable (-dispose! [this] - (dispose! uvbuf) - (fs_close fp)) + (uv/uv_close uv-client st/close_cb)) IReduce (-reduce [this f init] (common/stream-reducer this f init))) @@ -35,10 +34,9 @@ (defn tty-input-stream [fd] ;(assert (= uv/UV_TTY (uv/uv_guess_handle fd)) "fd is not a TTY") - (let [buf (uv/uv_buf_t) - tty (uv/uv_tty_t)] + (let [tty (uv/uv_tty_t)] (uv/uv_tty_init (uv/uv_default_loop) tty fd 0) - (->TTYInputStream tty buf))) + (->TTYInputStream tty))) (def stdin (tty-input-stream sys/stdin)) (def stdout (tty-output-stream sys/stdout)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 30e0ada3..1255fde0 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -36,7 +36,7 @@ (reset! stacklet-loop-h h) (-set-current-var-frames nil frames) (if (instance? ThrowException val) - (throw (:ex val)) + (throw [::Exception (:ex val)]) val))) (defn -run-later [f] diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 199942f4..06f7af7d 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -231,7 +231,7 @@ (defn throw-on-error [result] (if (neg? result) - (throw (str "UV Error: " (uv_err_name result))) + (throw [::UVException (str "UV Error: " (uv_err_name result))]) result)) (defmacro defuvfsfn diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index c93e3dd9..eb47b86c 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -38,4 +38,6 @@ (t/deftest test-slurp-spit (let [val (vec (range 1280))] (io/spit "test.tmp" val) - (t/assert= val (read-string (io/slurp "test.tmp"))))) + (t/assert= val (read-string (io/slurp "test.tmp")))) + (t/assert-throws? (io/slurp 1)) + (t/assert-throws? (io/slurp :foo))) From 7225c81bb5750600ae3de3207128bec0e2a5978c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 1 Jun 2015 09:34:25 +0100 Subject: [PATCH 743/909] slurping a utf-8 file works --- pixie/io.pxi | 11 ++++--- pixie/streams/utf8.pxi | 53 ++++++++++++++++++------------ tests/pixie/tests/test-io-utf8.txt | 1 + tests/pixie/tests/test-io.pxi | 4 ++- 4 files changed, 43 insertions(+), 26 deletions(-) create mode 100644 tests/pixie/tests/test-io-utf8.txt diff --git a/pixie/io.pxi b/pixie/io.pxi index a37188f6..56452320 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -113,9 +113,10 @@ (when (= idx (count buffer)) (set-field! this :idx 0) (read upstream buffer (buffer-capacity buffer))) - (let [val (nth buffer idx)] - (set-field! this :idx (inc idx)) - val)) + (when (pos? (count buffer)) + (let [val (nth buffer idx)] + (set-field! this :idx (inc idx)) + val))) IDisposable (-dispose! [this] (dispose! buffer))) @@ -181,7 +182,9 @@ result (transduce (map char) string-builder - stream)] + (-> stream + buffered-input-stream + utf8/utf8-input-stream))] (dispose! stream) result)) diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi index bc9b4044..e5605e18 100644 --- a/pixie/streams/utf8.pxi +++ b/pixie/streams/utf8.pxi @@ -31,29 +31,40 @@ (deftype UTF8InputStream [in bad-char] IUTF8InputStream (read-char [this] - (let [ch (int (read-byte in)) - [n bytes error?] (cond - (>= 0x7F ch) [ch 1] - (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2 false] - (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3 false] - (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4 false] - (= 0xF8 (bit-and ch 0xF8)) [(bit-and ch 3) 5 true] - (= 0xFC (bit-and ch 0xFE)) [(bit-and ch 1) 6 true] - :else [n 1 true])] - (loop [i (dec bytes) - n n] - (if (pos? i) - (recur (dec i) - (bit-or (bit-shift-left n 6) - (bit-and (read-byte in) 0x3F))) - (if error? - (if bad-char - bad-char - (throw (str "Invalid UTF8 character decoded: " n))) - (char n)))))) + (when-let [byte (read-byte in)] + (let [ch (int byte) + [n bytes error?] (cond + (>= 0x7F ch) [ch 1] + (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2 false] + (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3 false] + (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4 false] + (= 0xF8 (bit-and ch 0xF8)) [(bit-and ch 3) 5 true] + (= 0xFC (bit-and ch 0xFE)) [(bit-and ch 1) 6 true] + :else [n 1 true])] + (loop [i (dec bytes) + n n] + (if (pos? i) + (recur (dec i) + (bit-or (bit-shift-left n 6) + (bit-and (read-byte in) 0x3F))) + (if error? + (if bad-char + bad-char + (throw [::invalid-character (str "Invalid UTF8 character decoded: " n)])) + (char n))))))) IDisposable (-dispose! [this] - (dispose! in))) + (dispose! in)) + IReduce + (-reduce [this f init] + (let [rrf (preserving-reduced f)] + (loop [acc init] + (if-let [char (read-char this)] + (let [result (rrf acc (int char))] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) (defn utf8-input-stream "Creates a UTF8 decoder that reads characters from the given IByteInputStream. If a bad character is found diff --git a/tests/pixie/tests/test-io-utf8.txt b/tests/pixie/tests/test-io-utf8.txt new file mode 100644 index 00000000..37bf4044 --- /dev/null +++ b/tests/pixie/tests/test-io-utf8.txt @@ -0,0 +1 @@ +I love 🍺 . This is a thumbs up 👍 diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index eb47b86c..16f3778b 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -40,4 +40,6 @@ (io/spit "test.tmp" val) (t/assert= val (read-string (io/slurp "test.tmp")))) (t/assert-throws? (io/slurp 1)) - (t/assert-throws? (io/slurp :foo))) + (t/assert-throws? (io/slurp :foo)) + (t/assert= "I love 🍺 . This is a thumbs up 👍\n" + (io/slurp "tests/pixie/tests/test-io-utf8.txt"))) From 5970927cae69439d6c10d4d34b2aa30294536e3c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 3 Jun 2015 17:17:29 +0100 Subject: [PATCH 744/909] adds promotion to bigint on overflow --- pixie/vm/numbers.py | 33 ++++++++++++++++++++++++++---- tests/pixie/tests/test-numbers.pxi | 11 ++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index ed54bd53..eac2878f 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -1,7 +1,7 @@ import pixie.vm.object as object from pixie.vm.object import affirm from pixie.vm.primitives import true, false -from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import r_uint, ovfcheck from rpython.rlib.rbigint import rbigint import rpython.rlib.jit as jit from pixie.vm.code import DoublePolymorphicFn, extend, Protocol, as_var, wrap_fn @@ -116,12 +116,29 @@ def {pfn}_{ty1}_{ty2}(a, b): return {wrap_start}a.{conv1}() {op} b.{conv2}(){wrap_end} """ +ovf_op_template = """@extend({pfn}, {ty1}._type, {ty2}._type) +def {pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + try: + z = ovfcheck(a.{conv1}() {op} b.{conv2}()) + return {wrap_start}z{wrap_end} + except OverflowError: + z = rbigint.fromint(a.{conv1}()).{method_op}(rbigint.fromint(b.{conv2}())) + return {wrap_start}z{wrap_end} +""" + def extend_num_op(pfn, ty1, ty2, conv1, op, conv2, wrap_start = "rt.wrap(", wrap_end = ")"): tp = num_op_template.format(pfn=pfn, ty1=ty1.__name__, ty2=ty2.__name__, conv1=conv1, op=op, conv2=conv2, wrap_start=wrap_start, wrap_end=wrap_end) exec tp +def extend_ovf_op(pfn, ty1, ty2, conv1, op, method_op, conv2, wrap_start = "rt.wrap(", wrap_end = ")"): + tp = ovf_op_template.format(pfn=pfn, ty1=ty1.__name__, ty2=ty2.__name__, + conv1=conv1, op=op, method_op=method_op, conv2=conv2, + wrap_start=wrap_start, wrap_end=wrap_end) + exec tp + extend_num_op("_quot", Integer, Integer, "int_val", "/", "int_val") extend_num_op("_rem", Integer, Integer, "int_val", "%", "int_val") @@ -133,7 +150,14 @@ def define_num_ops(): for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/")]: if op == "_div" and c1 == Integer and c2 == Integer: continue - extend_num_op(op, c1, c2, conv1, sym, conv2) + elif op == "_add" and c1 == Integer and c2 == Integer: + extend_ovf_op(op, c1, c2, conv1, sym, "add", conv2) + elif op == "_sub" and c1 == Integer and c2 == Integer: + extend_ovf_op(op, c1, c2, conv1, sym, "sub", conv2) + elif op == "_mul" and c1 == Integer and c2 == Integer: + extend_ovf_op(op, c1, c2, conv1, sym, "mul", conv2) + else: + extend_num_op(op, c1, c2, conv1, sym, conv2) if c1 != Integer or c2 != Integer: extend_num_op("_rem", c1, c2, conv1, ",", conv2, wrap_start = "rt.wrap(math.fmod(", wrap_end = "))") extend_num_op("_quot", c1, c2, conv1, "/", conv2, wrap_start = "rt.wrap(math.floor(", wrap_end = "))") @@ -157,9 +181,10 @@ def define_bigint_ops(): continue for (pfn, op) in [("_add", "add"), ("_sub", "sub"), ("_mul", "mul"), ("_div", "div"), ("_num_eq", "eq"), ("_lt", "lt"), ("_gt", "gt"), ("_lte", "le"), ("_gte", "ge")]: + code = bigint_ops_tmpl.format(pfn=pfn, op=op, - ty1=c1.__name__, conv1=conv1, get1=get1, - ty2=c2.__name__, conv2=conv2, get2=get2) + ty1=c1.__name__, conv1=conv1, get1=get1, + ty2=c2.__name__, conv2=conv2, get2=get2) exec code define_bigint_ops() diff --git a/tests/pixie/tests/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi index 3b5f0aeb..0152adc4 100644 --- a/tests/pixie/tests/test-numbers.pxi +++ b/tests/pixie/tests/test-numbers.pxi @@ -62,3 +62,14 @@ (t/assert= Integer (type (quot 7/2 3/7))) (t/assert= Float (type (quot 7/2 3.0)))) +(t/deftest test-big-int-eq + (t/assert (-num-eq 1N 1)) + (t/assert (-num-eq 1 1N)) + ;(t/assert= 1N 1) this fails, should it? + (t/assert= 1 1N)) + +(t/deftest test-promotion + (t/assert= BigInteger (type (reduce * 1 (range 1 100)))) + (t/assert (-num-eq 1000000000000000000000N (* 10000000 + 10000000 + 10000000)))) From 45edc957c78dc6ad816ef1b34239adc4656452e7 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 3 Jun 2015 22:04:20 +0100 Subject: [PATCH 745/909] Add BigInteger to pxic_writer this should make compile_tests work --- pixie/vm/libs/pxic/writer.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index 8f1318ec..5dd39dd0 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -4,7 +4,7 @@ from pixie.vm.string import String from pixie.vm.keyword import Keyword from pixie.vm.symbol import Symbol -from pixie.vm.numbers import Integer, Float +from pixie.vm.numbers import Integer, BigInteger, Float from pixie.vm.code import Code, Var, NativeFn, Namespace from pixie.vm.primitives import nil, true, false from pixie.vm.reader import LinePromise @@ -245,6 +245,8 @@ def write_object(obj, wtr): write_string(rt.name(obj), wtr) elif isinstance(obj, Integer): write_int(obj.int_val(), wtr) + elif isinstance(obj, BigInteger): + write_int(obj.bigint_val(), wtr) elif isinstance(obj, Float): write_float(obj.float_val(), wtr) elif isinstance(obj, Code): From fde801ff0b1a914a0a80b537198499ded8b33531 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 5 Jun 2015 13:44:38 +0100 Subject: [PATCH 746/909] Add a BIGINT_STRING and BIGINT to reader & writer BIGINT has not yet been implemented, BigIntegers will be written as strings --- pixie/vm/libs/pxic/reader.py | 5 ++++- pixie/vm/libs/pxic/tags.py | 2 ++ pixie/vm/libs/pxic/writer.py | 8 +++++++- 3 files changed, 13 insertions(+), 2 deletions(-) diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index 571cbddc..d3b44613 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -4,7 +4,7 @@ from pixie.vm.string import String from pixie.vm.keyword import Keyword, keyword from pixie.vm.symbol import Symbol, symbol -from pixie.vm.numbers import Integer, Float +from pixie.vm.numbers import Integer, Float, BigInteger from pixie.vm.code import Code, Var, NativeFn, Namespace, intern_var import pixie.vm.code as code from pixie.vm.primitives import nil, true, false @@ -13,6 +13,7 @@ from pixie.vm.persistent_list import create_from_list from pixie.vm.reader import LinePromise from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rbigint import rbigint from pixie.vm.libs.pxic.util import read_handlers import pixie.vm.rt as rt @@ -172,6 +173,8 @@ def read_obj(rdr): return read_namespace(rdr) elif tag == INT_STRING: return Integer(int(read_raw_string(rdr))) + elif tag == BIGINT_STRING: + return BigInteger(rbigint.fromstr(str(read_raw_string(rdr)))) elif tag == NEW_CACHED_OBJ: diff --git a/pixie/vm/libs/pxic/tags.py b/pixie/vm/libs/pxic/tags.py index abe92ac2..eb7694f8 100644 --- a/pixie/vm/libs/pxic/tags.py +++ b/pixie/vm/libs/pxic/tags.py @@ -1,7 +1,9 @@ tag_name = ["INT", + "BIGINT", "FLOAT", "INT_STRING", + "BIGINT_STRING", "STRING", "CODE", "TRUE", diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index 5dd39dd0..c86f8013 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -10,6 +10,7 @@ from pixie.vm.reader import LinePromise from rpython.rlib.objectmodel import specialize from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rbigint import rbigint import pixie.vm.rt as rt MAX_INT32 = r_uint(1 << 31) @@ -107,6 +108,11 @@ def write_int(i, wtr): wtr.write(chr(INT_STRING)) write_string_raw(unicode(str(i)), wtr) +def write_bigint(i, wtr): + # TODO implement a non string BIGINT writer + wtr.write(chr(BIGINT_STRING)) + write_string_raw(unicode(i.str()), wtr) + def write_float(f, wtr): write_tag(FLOAT, wtr) write_string_raw(unicode(str(f)), wtr) @@ -246,7 +252,7 @@ def write_object(obj, wtr): elif isinstance(obj, Integer): write_int(obj.int_val(), wtr) elif isinstance(obj, BigInteger): - write_int(obj.bigint_val(), wtr) + write_bigint(obj.bigint_val(), wtr) elif isinstance(obj, Float): write_float(obj.float_val(), wtr) elif isinstance(obj, Code): From 4d8aef8c075bb81ddac66ba8ff929bdf312bd5b4 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 14 Jun 2015 17:31:57 -0400 Subject: [PATCH 747/909] add select-keys, rand-int, some? --- pixie/stdlib.pxi | 19 +++++++++++++++++++ tests/pixie/tests/test-stdlib.pxi | 14 ++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a973ec1e..2c3b9904 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -552,6 +552,11 @@ returns true" (defn rem [num div] (-rem num div)) +(defn rand-int + {:doc "random integer between 0 (inclusive) and n (exclusive)"} + [n] + (rem (rand) n)) + (defn = {:doc "Returns true if all the arguments are equivalent. Otherwise, returns false. Uses -eq to perform equality checks." @@ -946,6 +951,16 @@ If further arguments are passed, invokes the method named by symbol, passing the (fn [v] (transduce ordered-hash-reducing-fn v))) +(defn select-keys + {:doc "Produces a map with only the values in m contained in key-seq"} + [m key-seq] + (with-meta + (transduce + (comp (filter (fn [k] (contains? m k))) + (map (fn [k] [k (get m k)]))) + conj {} key-seq) + (meta m))) + (defn keys {:doc "If called with no arguments returns a transducer that will extract the key from each map entry. If passed a collection, will assume that it is a hashmap and return a vector of all keys from the collection." @@ -1411,6 +1426,10 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn nil? [x] (identical? x nil)) +(defn some? [x] + {:doc "true if x is not nil"} + (not (nil? x))) + (defn fnil [f else] (fn [x & args] (apply f (if (nil? x) else x) args))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index d291874b..e464b475 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -189,6 +189,16 @@ (t/assert= (set (vals v)) #{1 2 3}) (t/assert= (transduce (vals) conj! v) (vals v)))) +(t/deftest test-select-keys + (let [m ^{:k :v} {:a 1 :b 2}] + (t/assert= (select-keys m [:a :b]) m) + (t/assert= :v + (-> (select-keys m [:a]) + meta + :k)) + (t/assert= (select-keys m [:a :not-found]) {:a 1}) + (t/assert= (select-keys m nil) {}) + (t/assert= (select-keys {} [:a]) {}))) (t/deftest test-empty (t/assert= (empty '(1 2 3)) '()) @@ -273,6 +283,10 @@ (t/assert= (every? even? []) true) (t/assert= (every? odd? []) true)) +(t/deftest test-rand-int + (let [vs (repeatedly 10 #(rand-int 4))] + (t/assert (every? #(and (>= % 0) (< % 4)) vs)))) + (t/deftest test-some (t/assert= (some even? [2 4 6 8]) true) (t/assert= (some odd? [2 4 6 8]) false) From 303226e0d324064370ce2041665d13ec1ac5e12c Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 14 Jun 2015 16:00:58 -0400 Subject: [PATCH 748/909] clarify parser docs, typo fix --- pixie/parser.pxi | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pixie/parser.pxi b/pixie/parser.pxi index e1b12a26..79e18f6e 100644 --- a/pixie/parser.pxi +++ b/pixie/parser.pxi @@ -191,10 +191,10 @@ rules))) (defmacro parser - "(parser nm inherits & rules) - Defines a new parser named `nm` that inherits from zero or more other parsers defined ion `inherits`. Rules are pairs - of names and rules that will be assigned to those names. Names are inherited from parent parsers in the order they are - defined." + "(parser inherits & rules) + Creates a parser that inherits from zero or more other parsers defined in `inherits`. Rules are pairs + of names and rules that will be assigned to those names. Names are inherited from parent parsers in the + order they are defined." [inherits & rules] (let [parted (apply merge (conj (mapv (fn [sym] @@ -303,7 +303,7 @@ fail)) (defn one-of - "Deines a parser that succeeds if the value being parsed is found in v" + "Defines a parser that succeeds if the value being parsed is found in v" [v] (parse-if (partial contains? v))) From 17117c8dac55eeb3cd22e60f5ffe7995f646ea5a Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 21 Jun 2015 14:17:43 +0100 Subject: [PATCH 749/909] fixes typo --- pixie/channels.pxi | 2 +- pixie/io.pxi | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/channels.pxi b/pixie/channels.pxi index 9937e7c2..f930bf17 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -7,7 +7,7 @@ (-commit! [this])) (defprotocol IReadPort - (-take! [this cfn] "Take a value from this port passing it to a cancellable function")) + (-take! [this cfn] "Take a value from this port passing it to a cancelable function")) (defprotocol IWritePort (-put! [this itm cfn] "Write a value to this port passing true if the write succeeds and the diff --git a/pixie/io.pxi b/pixie/io.pxi index 56452320..d3550c28 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -86,7 +86,7 @@ write-count)))) IDisposable (-dispose! [this] - (fclose fp))) + (fs_close fp))) (deftype BufferedOutputStream [downstream idx buffer] IByteOutputStream From 957f17b65b08f4c0e2be7bf009345f4e350ea919 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 24 Jun 2015 17:26:49 +0100 Subject: [PATCH 750/909] Removed circular dependency uv -> ffi-infer -> io-blocking -> io.common -> uv This is fixed by making io.common require only pixie.stream and move all uv related io stuff into io.uv-common. --- pixie/io-blocking.pxi | 2 -- pixie/io/common.pxi | 46 +----------------------------------------- pixie/io/tcp.pxi | 5 +++-- pixie/io/tty.pxi | 5 +++-- pixie/io/uv-common.pxi | 45 +++++++++++++++++++++++++++++++++++++++++ 5 files changed, 52 insertions(+), 51 deletions(-) create mode 100644 pixie/io/uv-common.pxi diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index ecfbd5ee..4ac84eb3 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -2,7 +2,6 @@ (:require [pixie.streams :as st :refer :all] [pixie.io.common :as common])) - (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) (def fseek (ffi-fn libc "fseek" [CVoidP CInt CInt] CInt)) (def ftell (ffi-fn libc "ftell" [CVoidP] CInt)) @@ -15,7 +14,6 @@ (def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) (def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) - (deftype FileStream [fp] IInputStream (read [this buffer len] diff --git a/pixie/io/common.pxi b/pixie/io/common.pxi index ee23bdef..75a3e685 100644 --- a/pixie/io/common.pxi +++ b/pixie/io/common.pxi @@ -1,9 +1,6 @@ (ns pixie.io.common "Common functionality for handling IO" - (:require [pixie.streams :refer :all] - [pixie.uv :as uv] - [pixie.stacklets :as st] - [pixie.ffi :as ffi])) + (:require [pixie.streams :refer :all])) (def DEFAULT-BUFFER-SIZE 1024) @@ -18,44 +15,3 @@ (recur result) @result)) acc))))) - -(defn cb-stream-reader [uv-client buffer len] - (assert (<= (buffer-capacity buffer) len) - "Not enough capacity in the buffer") - (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) - read-cb (atom nil)] - (st/call-cc (fn [k] - (reset! read-cb (ffi/ffi-prep-callback - uv/uv_read_cb - (fn [stream nread uv-buf] - (set-buffer-count! buffer nread) - (try - (dispose! alloc-cb) - (dispose! @read-cb) - ;(dispose! uv-buf) - (uv/uv_read_stop stream) - (st/run-and-process k (or - (st/exception-on-uv-error nread) - nread)) - (catch ex - (println ex)))))) - (uv/uv_read_start uv-client alloc-cb @read-cb))))) - -(defn cb-stream-writer - [uv-client uv-write-buf buffer] - (let [write-cb (atom nil) - uv_write (uv/uv_write_t)] - (ffi/set! uv-write-buf :base buffer) - (ffi/set! uv-write-buf :len (count buffer)) - (st/call-cc - (fn [k] - (reset! write-cb (ffi/ffi-prep-callback - uv/uv_write_cb - (fn [req status] - (try - (dispose! @write-cb) - ;(uv/uv_close uv_write st/close_cb) - (st/run-and-process k status) - (catch ex - (println ex)))))) - (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index b3945782..4836e7a6 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -3,6 +3,7 @@ [pixie.streams :refer [IInputStream read IOutputStream write]] [pixie.io.common :as common] [pixie.uv :as uv] + [pixie.io.uv-common :as uv-common] [pixie.ffi :as ffi])) (defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb] @@ -15,10 +16,10 @@ (deftype TCPStream [uv-client uv-write-buf] IInputStream (read [this buffer len] - (common/cb-stream-reader uv-client buffer len)) + (uv-common/cb-stream-reader uv-client buffer len)) IOutputStream (write [this buffer] - (common/cb-stream-writer uv-client uv-write-buf buffer)) + (uv-common/cb-stream-writer uv-client uv-write-buf buffer)) IDisposable (-dispose! [this] (dispose! uv-write-buf) diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi index 44386f3e..f4a1c091 100644 --- a/pixie/io/tty.pxi +++ b/pixie/io/tty.pxi @@ -3,12 +3,13 @@ [pixie.streams :refer [IInputStream read IOutputStream write]] [pixie.uv :as uv] [pixie.io.common :as common] + [pixie.io.uv-common :as uv-common] [pixie.system :as sys])) (deftype TTYInputStream [uv-client] IInputStream (read [this buf len] - (common/cb-stream-reader uv-client buf len)) + (uv/cb-stream-reader uv-client buf len)) IDisposable (-dispose! [this] (uv/uv_close uv-client st/close_cb)) @@ -19,7 +20,7 @@ (deftype TTYOutputStream [uv-client uv-write-buf] IOutputStream (write [this buffer] - (common/cb-stream-writer uv-client uv-write-buf buffer)) + (uv-common/cb-stream-writer uv-client uv-write-buf buffer)) IDisposable (-dispose! [this] (dispose! uv-write-buf) diff --git a/pixie/io/uv-common.pxi b/pixie/io/uv-common.pxi new file mode 100644 index 00000000..c8f10ee2 --- /dev/null +++ b/pixie/io/uv-common.pxi @@ -0,0 +1,45 @@ +(ns pixie.io.uv-common + (:require [pixie.stacklets :as st] + [pixie.uv :as uv] + [pixie.ffi :as ffi])) + +(defn cb-stream-reader [uv-client buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) + read-cb (atom nil)] + (st/call-cc (fn [k] + (reset! read-cb (ffi/ffi-prep-callback + uv/uv_read_cb + (fn [stream nread uv-buf] + (set-buffer-count! buffer nread) + (try + (dispose! alloc-cb) + (dispose! @read-cb) + ;(dispose! uv-buf) + (uv/uv_read_stop stream) + (st/run-and-process k (or + (st/exception-on-uv-error nread) + nread)) + (catch ex + (println ex)))))) + (uv/uv_read_start uv-client alloc-cb @read-cb))))) + +(defn cb-stream-writer + [uv-client uv-write-buf buffer] + (let [write-cb (atom nil) + uv_write (uv/uv_write_t)] + (ffi/set! uv-write-buf :base buffer) + (ffi/set! uv-write-buf :len (count buffer)) + (st/call-cc + (fn [k] + (reset! write-cb (ffi/ffi-prep-callback + uv/uv_write_cb + (fn [req status] + (try + (dispose! @write-cb) + ;(uv/uv_close uv_write st/close_cb) + (st/run-and-process k status) + (catch ex + (println ex)))))) + (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) From a4a16f214910d2662cbac2242f80940f325c4d66 Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Sun, 28 Jun 2015 16:33:05 -0400 Subject: [PATCH 751/909] Update Executable size and use ;; comments --- README.md | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/README.md b/README.md index 69a01bf9..eda32826 100644 --- a/README.md +++ b/README.md @@ -62,15 +62,14 @@ First of all, the word "magic" is in quotes as it's partly a play on words, pixi However there are a few features of pixie that although may not be uncommon, are perhaps unexpected from a lisp. -* Pixie implements its own virtual machine. It does not run on the JVM, CLR or Python VM. It implements its own bytecode, has its own GC and JIT. And it's small. Currently the interpreter, JIT, GC, and stdlib clock in at about 5.5MB once compiled down to an executable. +* Pixie implements its own virtual machine. It does not run on the JVM, CLR or Python VM. It implements its own bytecode, has its own GC and JIT. And it's small. Currently the interpreter, JIT, GC, and stdlib clock in at about 10.3MB once compiled down to an executable. * The JIT makes some things fast. Very fast. Code like the following compiles down to a loop with 6 CPU instructions. While this may not be too impressive for any language that uses a tracing jit, it is fairly unique for a language as young as Pixie. ```clojure -(comment - This code adds up to 10000 from 0 via calling a function that takes a variable number of arguments. - That function then reduces over the argument list to add up all given arguments.) +;; This code adds up to 10000 from 0 via calling a function that takes a variable number of arguments. +;; That function then reduces over the argument list to add up all given arguments. (defn add-fn [& args] (reduce -add 0 args)) From b82479ddf92c33b0761eb82301ae72cf257efe67 Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Sun, 28 Jun 2015 16:35:03 -0400 Subject: [PATCH 752/909] Abort compilation if boost library not found (#319) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It’s a bit of a hack way to implement, but essentially just makes compilation dependent on the boost library being found in either /usr/local/include or /usr/include. --- Makefile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 5829c583..57204a3b 100644 --- a/Makefile +++ b/Makefile @@ -19,11 +19,13 @@ help: @echo "make fetch_externals - download and unpack external deps" build_with_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py + @if [ ! -d /usr/local/include/boost -a ! -d /usr/include/boost ] ; then echo "Boost C++ Library not found" && false; fi && \ + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py && \ make compile_basics build_no_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target.py + @if [ ! -d /usr/local/include/boost -a ! -d /usr/include/boost ] ; then echo "Boost C++ Library not found" && false; fi && \ + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target.py && \ make compile_basics compile_basics: From f5c8c62eb552920e5c3c93b97b0176a4f6ba140b Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 29 Jun 2015 09:45:58 +0100 Subject: [PATCH 753/909] removed old scripts --- checkout-externals | 1 - make-no-jit | 1 - make-with-jit | 1 - run-interpreted | 1 - 4 files changed, 4 deletions(-) delete mode 100755 checkout-externals delete mode 100755 make-no-jit delete mode 100755 make-with-jit delete mode 100755 run-interpreted diff --git a/checkout-externals b/checkout-externals deleted file mode 100755 index db5f24d8..00000000 --- a/checkout-externals +++ /dev/null @@ -1 +0,0 @@ -echo "This script will be removed soon, please use 'make build_with_jit' or one of it's variants instead." \ No newline at end of file diff --git a/make-no-jit b/make-no-jit deleted file mode 100755 index c10d5f4b..00000000 --- a/make-no-jit +++ /dev/null @@ -1 +0,0 @@ -echo "This script will be removed soon, please use 'make build_no_jit' instead." \ No newline at end of file diff --git a/make-with-jit b/make-with-jit deleted file mode 100755 index 2d955b51..00000000 --- a/make-with-jit +++ /dev/null @@ -1 +0,0 @@ -echo "This script will be removed soon, please use 'make build_with_jit' instead." diff --git a/run-interpreted b/run-interpreted deleted file mode 100755 index cef13f95..00000000 --- a/run-interpreted +++ /dev/null @@ -1 +0,0 @@ -echo "This script will be removed soon, please use 'make run_interactive' instead." From 41acb27a324a4ef34b12d0c62c91027884e4420d Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 29 Jun 2015 13:51:51 +0100 Subject: [PATCH 754/909] fix typo --- pixie/io.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index d3550c28..d1e36168 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -79,7 +79,7 @@ _ (pixie.ffi/set! uvbuf :len (- (count buffer) buffer-offset)) write-count (fs_write fp uvbuf 1 offset)] (when (neg? write-count) - (throw [::FileOutputStreamException (uv/uv_err_name read-count)])) + (throw [::FileOutputStreamException (uv/uv_err_name write-count)])) (set-field! this :offset (+ offset write-count)) (if (< (+ buffer-offset write-count) (count buffer)) (recur (+ buffer-offset write-count)) From bb9c0fbffdba92de76925be49c78fe349aca0c2b Mon Sep 17 00:00:00 2001 From: "Matthew A. West" Date: Wed, 8 Jul 2015 12:59:01 -0400 Subject: [PATCH 755/909] Encode BigIntegers in pxic without using strings --- pixie/vm/compiler.py | 2 +- pixie/vm/libs/pxic/reader.py | 11 +++++++++++ pixie/vm/libs/pxic/writer.py | 21 +++++++++++++++++---- tests/pixie/tests/test-numbers.pxi | 4 ++-- 4 files changed, 31 insertions(+), 7 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 76ca7fae..a6bbd64f 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -1,4 +1,4 @@ -from pixie.vm.object import affirm +from pixie.vm.object import affirm from pixie.vm.primitives import nil, true, Bool from pixie.vm.persistent_vector import EMPTY, PersistentVector from pixie.vm.persistent_hash_set import PersistentHashSet diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py index d3b44613..2cc5d799 100644 --- a/pixie/vm/libs/pxic/reader.py +++ b/pixie/vm/libs/pxic/reader.py @@ -55,6 +55,15 @@ def read_tag(rdr): def read_raw_integer(rdr): return r_uint(ord(rdr.read()[0]) | (ord(rdr.read()[0]) << 8) | (ord(rdr.read()[0]) << 16) | (ord(rdr.read()[0]) << 24)) +def read_raw_bigint(rdr): + nchars = read_raw_integer(rdr) + n = rbigint.fromint(0) + for i in range(nchars): + a = rbigint.fromint(ord(rdr.read()[0])) + a = a.lshift(8*i) + n = n.add(a) + return n + def read_raw_string(rdr): s = rdr.read_cached_string() return s @@ -138,6 +147,8 @@ def read_obj(rdr): if tag == INT: return Integer(intmask(read_raw_integer(rdr))) + elif tag == BIGINT: + return BigInteger(read_raw_bigint(rdr)) elif tag == CODE: return read_code(rdr) elif tag == NIL: diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index c86f8013..7aedcb17 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -100,6 +100,16 @@ def write_int_raw(i, wtr): def write_string_raw(si, wtr): wtr.write_raw_cached_string(si) +def write_bigint_raw(i, wtr): + bits = i.bit_length() + nchars = r_uint(bits / 8) + if (bits) % 8 != 0: + nchars += 1 + assert nchars <= MAX_INT32 + write_int_raw(nchars, wtr) # nchars used to represent the bigint + for j in range(nchars): + wtr.write(chr((i.rshift(j * 8).int_and_(0xFF).toint()))) + def write_int(i, wtr): if 0 <= i <= MAX_INT32: wtr.write(chr(INT)) @@ -109,9 +119,12 @@ def write_int(i, wtr): write_string_raw(unicode(str(i)), wtr) def write_bigint(i, wtr): - # TODO implement a non string BIGINT writer - wtr.write(chr(BIGINT_STRING)) - write_string_raw(unicode(i.str()), wtr) + if i.int_ge(0): + wtr.write(chr(BIGINT)) + write_bigint_raw(i, wtr) + else: + wtr.write(chr(BIGINT_STRING)) + write_string_raw(unicode(i.str()), wtr) def write_float(f, wtr): write_tag(FLOAT, wtr) @@ -251,7 +264,7 @@ def write_object(obj, wtr): write_string(rt.name(obj), wtr) elif isinstance(obj, Integer): write_int(obj.int_val(), wtr) - elif isinstance(obj, BigInteger): + elif isinstance(obj, BigInteger): #TODO test write_bigint(obj.bigint_val(), wtr) elif isinstance(obj, Float): write_float(obj.float_val(), wtr) diff --git a/tests/pixie/tests/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi index 0152adc4..28ab8b92 100644 --- a/tests/pixie/tests/test-numbers.pxi +++ b/tests/pixie/tests/test-numbers.pxi @@ -70,6 +70,6 @@ (t/deftest test-promotion (t/assert= BigInteger (type (reduce * 1 (range 1 100)))) - (t/assert (-num-eq 1000000000000000000000N (* 10000000 - 10000000 + (t/assert (-num-eq 1000000000000000000000N (* 10000000 + 10000000 10000000)))) From 2a1760fc60b6a2da488fa2a0a5fd9fbb4e2c8d43 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 21 Jun 2015 14:12:15 +0100 Subject: [PATCH 756/909] working zlib implementation pixies ffi can convert nil to an NULL PTR callback --- .travis.yml | 2 +- pixie/io.pxi | 7 +- pixie/io/tty.pxi | 2 +- pixie/stdlib.pxi | 4 +- pixie/streams/zlib.pxi | 218 ++++++++++++++++++++++++++++++++++ pixie/streams/zlib/ffi.pxi | 59 +++++++++ pixie/vm/libs/ffi.py | 19 +-- tests/pixie/tests/test-io.pxi | 39 +++++- 8 files changed, 337 insertions(+), 13 deletions(-) create mode 100644 pixie/streams/zlib.pxi create mode 100644 pixie/streams/zlib/ffi.pxi diff --git a/.travis.yml b/.travis.yml index 9d48c020..28ac48e9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,7 +17,7 @@ script: - make run_built_tests before_install: - - sudo apt-get install libffi-dev libedit-dev libboost-all-dev + - sudo apt-get install libffi-dev libedit-dev libboost-all-dev zlib1g-dev zlib-bin notifications: irc: "chat.freenode.net#pixie-lang" diff --git a/pixie/io.pxi b/pixie/io.pxi index d1e36168..3a14ca7c 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -100,12 +100,15 @@ IDisposable (-dispose! [this] (set-buffer-count! buffer idx) - (write downstream buffer)) + (write downstream buffer) + (flush this)) IFlushableStream (flush [this] (set-buffer-count! buffer idx) (set-field! this :idx 0) - (write downstream buffer))) + (write downstream buffer) + (when (satisfies? IFlushableStream downstream) + (flush downstream)))) (deftype BufferedInputStream [upstream idx buffer] IByteInputStream diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi index f4a1c091..be8c0185 100644 --- a/pixie/io/tty.pxi +++ b/pixie/io/tty.pxi @@ -9,7 +9,7 @@ (deftype TTYInputStream [uv-client] IInputStream (read [this buf len] - (uv/cb-stream-reader uv-client buf len)) + (uv-common/cb-stream-reader uv-client buf len)) IDisposable (-dispose! [this] (uv/uv_close uv-client st/close_cb)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8815e226..880efd42 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -645,7 +645,9 @@ returns true" :signatures [[coll]] :added "0.1"} [coll] - (not (seq coll))) + (if (satisfies? ICounted coll) + (zero? (count coll)) + (not (seq coll)))) (defn not-empty? {:doc "returns true if the collection has items, otherwise false" diff --git a/pixie/streams/zlib.pxi b/pixie/streams/zlib.pxi new file mode 100644 index 00000000..bae72ade --- /dev/null +++ b/pixie/streams/zlib.pxi @@ -0,0 +1,218 @@ +(ns pixie.streams.zlib + (:require [pixie.streams.zlib.ffi :as zlib.ffi] + [pixie.ffi :as ffi] + [pixie.streams :refer :all])) + +(defprotocol IZStream + (version [this]) + (set-input! [this] + "Should be called before deflate! to set a new chunk of input + to deflate") + + (full-output? [this] + "Returns true if the ouput buffer is full of deflated (compressed) data") + + (reset-output-buffer! [this] + "Make the output buffer ready to be refilled with data") + + (consumed-input? [this] + "Returns true if zlib has finished reading the input-buffer") + + (set-output-buffer-count! [this] + "Set the buffers count so down stream can safely read the buffer") + + ;; Deflation + (deflate-init! [this opts] + "Set up the compression with desired parameters") + + ;; Inflation + (inflate-init! [this opts] + "Set up decompression with desired parameters") + + ;; In/deflate depending on what the stream has been + ;; initialized as + (flate! [this down-stream mode] + "Compress/decompress") + + (flate-end! + "Cleanup")) + +(defn handle-errors! [status] + (cond + (= status zlib.ffi/Z_ERRNO) + (throw [::Error "Something went wrong"]) + + (= status zlib.ffi/Z_STREAM_ERROR) + (throw [::Error "The stream doesn't appear to be a valid zlib/gzip stream"]) + + (= status zlib.ffi/Z_DATA_ERROR) + (throw [::Error "There was something wrong with the data"]) + + + ;; TODO go through the different status and show + ;; appropriate messages + (neg? status) + (throw [::Error "There was something wrong with the data"]))) + + +;; This wraps the C data structure and stores some information about +;; how its been initialized: :deflate or :inflate. +(deftype ZStream [z-stream inited] + IZStream + (version [this] + (zlib.ffi/zlibVersion)) + + (full-output? [this] + (zero? (get z-stream :avail_out))) + + (consumed-input? [this] + (zero? (get z-stream :avail_in))) + + (reset-output-buffer! [this output-buffer] + (ffi/set! z-stream :next_out output-buffer) + (ffi/set! z-stream :avail_out (buffer-capacity output-buffer))) + + (set-output-buffer-count! [this output-buffer] + (let [fill-count (- (buffer-capacity output-buffer) + (get z-stream :avail_out))] + (set-buffer-count! output-buffer fill-count))) + + (set-input! [this input-buffer] + (ffi/set! z-stream :next_in input-buffer) + (ffi/set! z-stream :avail_in (count input-buffer))) + + (deflate-init! [this opts] + (assert (nil? inited) "ZStream can only be initialized once.") + (let [status (zlib.ffi/deflateInit2_ + z-stream + (get opts :level zlib.ffi/Z_BEST_COMPRESSION) ;level + zlib.ffi/Z_DEFLATED ;method + (+ 15 16) ;window (set for gz header) + 8 ;memlevel + zlib.ffi/Z_DEFAULT_STRATEGY ;strategy + (version this) ;version + (ffi/struct-size zlib.ffi/z_stream))] + (assert (= zlib.ffi/Z_OK status) "Failed to initiate zstream") + (set-field! this :inited :deflate))) + + (inflate-init! [this opts] + (assert (nil? inited) "ZStream can only be initialized once.") + (let [status (zlib.ffi/inflateInit2_ + z-stream + (+ 15 16) ;window (set for gz header) + (version this) ;version + (ffi/struct-size zlib.ffi/z_stream))] + (assert (= zlib.ffi/Z_OK status) "Failed to initiate zstream") + (set-field! this :inited :inflate))) + + (flate! [this output-buffer mode] + (case inited + :inflate + (let [status (zlib.ffi/inflate z-stream mode)] + (handle-errors! status) + (set-output-buffer-count! this output-buffer) + status) + + :deflate + (let [status (zlib.ffi/deflate z-stream mode)] + (handle-errors! status) + (set-output-buffer-count! this output-buffer) + status) + + (assert false "ZStream must be initialized before calling flate!"))) + + (flate-end! [this] + (case inited + :inflate + (zlib.ffi/inflateEnd z-stream) + + :deflate + (zlib.ffi/deflateEnd z-stream) + + (assert false "ZStream must be initialized before calling flate-end!")))) + +(defn z-stream [] + (let [z-stream (zlib.ffi/z_stream)] + ;; Set all the callbacks to NULL so zlib uses its default ones. + (ffi/set! z-stream :avail_in 0) + (ffi/set! z-stream :zalloc nil) + (ffi/set! z-stream :opaque nil) + (ffi/set! z-stream :zfree nil) + (->ZStream z-stream nil))) + +(deftype GZInputStream + [up-stream input-buffer z-stream] + IDisposable + (-dispose! [this] + (flate-end! z-stream)) + + IInputStream + (read [this buffer len] + (set-buffer-count! buffer 0) + (reset-output-buffer! z-stream buffer) + ;; We keep reading from upstream until we have filled our output buffer + (loop [] + (when (consumed-input? z-stream) + ;; If z-stream has finished reading the input-buffer we last gave it, + ;; give it a new one. + (read up-stream input-buffer (buffer-capacity input-buffer)) + (set-input! z-stream input-buffer)) + (flate! z-stream buffer zlib.ffi/Z_NO_FLUSH) + (if-not (empty? input-buffer) + (if (full-output? z-stream) + ;; The buffer is now filled up + (count buffer) + ;; We can still do some more de/compression + (recur)) + 0)))) + +(deftype GZOutputStream + [down-stream output-buffer z-stream] + IOutputStream + (write [this input-buffer] + (set-input! z-stream input-buffer) + (loop [] + (reset-output-buffer! z-stream output-buffer) + (flate! z-stream output-buffer zlib.ffi/Z_NO_FLUSH) + (when-not (empty? output-buffer) + (write down-stream output-buffer)) + ;; if there is still more 'flating to do, do it. + (when (full-output? z-stream) + (recur)))) + + IFlushableStream + (flush [this] + (loop [] + (reset-output-buffer! z-stream output-buffer) + (flate! z-stream output-buffer zlib.ffi/Z_FINISH) + (write down-stream output-buffer) + (when (full-output? z-stream) + (recur))) + (when (satisfies? IFlushableStream down-stream) + (flush down-stream))) + + IDisposable + (-dispose! [this] + (flate-end! z-stream) + (when (satisfies? IDisposable down-stream) + (-dispose! down-stream)))) + +(defn compressing-output-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [down-stream output-buffer opts] + (->GZOutputStream down-stream output-buffer (deflate-init! (z-stream) opts))) + +(defn decompressing-output-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [down-stream output-buffer opts] + (->GZOutputStream down-stream output-buffer (inflate-init! (z-stream) opts))) + +(defn decompressing-input-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [up-stream input-buffer opts] + (->GZInputStream up-stream input-buffer (inflate-init! (z-stream) opts))) + +(defn compressing-input-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [up-stream input-buffer opts] + (->GZInputStream up-stream input-buffer (deflate-init! (z-stream) opts))) diff --git a/pixie/streams/zlib/ffi.pxi b/pixie/streams/zlib/ffi.pxi new file mode 100644 index 00000000..8c8bcdbb --- /dev/null +++ b/pixie/streams/zlib/ffi.pxi @@ -0,0 +1,59 @@ +(ns pixie.streams.zlib.ffi + (:require [pixie.ffi-infer :as f] + [pixie.ffi :as ffi])) + +(f/with-config {:library "z" + :includes ["zlib.h"]} + (f/defcstruct z_stream [:next_in + :avail_in + :total_in + :next_out + :avail_out + :total_out + + :msg + :state + + :zalloc + :zfree + :opaque + + :data_type + :adler + + :reserved]) + + (f/defcfn zError) + (f/defcfn zlibVersion) + + ;; Inflating (decompressing) + (f/defcfn inflate) + (f/defcfn inflateEnd) + (f/defcfn inflateInit2_) + + ;; Defalting (compressing) + (f/defcfn deflate) + (f/defcfn deflateInit_) + (f/defcfn deflateInit2_) + (f/defcfn deflateEnd)) + +(def Z_OK 0) +(def Z_NO_FLUSH 0) +(def Z_PARTIAL_FLUSH 1) +(def Z_SYNC_FLUSH 2) +(def Z_FULL_FLUSH 3) +(def Z_FINISH 4) +(def Z_BLOCK 5) + +(def Z_ERRNO -1) +(def Z_STREAM_ERROR -2) +(def Z_DATA_ERROR -3) + +(def Z_NO_COMPRESSION 0) +(def Z_BEST_SPEED 1) +(def Z_BEST_COMPRESSION 9) +(def Z_DEFAULT_COMPRESSION -1) + +(def Z_DEFLATED 8) + +(def Z_DEFAULT_STRATEGY 0) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index a733464d..7d961ce4 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -680,11 +680,16 @@ def ffi_get_value(self, ptr): runtime_error(u"Cannot get a callback value via FFI") def ffi_set_value(self, ptr, val): - affirm(isinstance(val, CCallback), u"Can only encode CCallbacks as function pointers") - - - casted = rffi.cast(rffi.VOIDPP, ptr) - casted[0] = val.get_raw_closure() + if isinstance(val, CCallback): + casted = rffi.cast(rffi.VOIDPP, ptr) + casted[0] = val.get_raw_closure() + elif val is nil: + casted = rffi.cast(rffi.VOIDPP, ptr) + casted[0] = rffi.cast(rffi.VOIDP, 0) + else: + frm_name = rt.name(rt.str(val.type())) + to_name = rt.name(rt.str(self)) + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) return None @@ -751,8 +756,8 @@ def c_struct(name, size, spec): offset = rt.nth(row, rt.wrap(2)) affirm(isinstance(nm, Keyword), u"c-struct field names must be keywords") - if not isinstance(tp, CType): - runtime_error(u"c-struct field types must be c types, got: " + rt.name(rt.str(tp))) + #if not isinstance(tp, CType): + # runtime_error(u"c-struct field types must be c types, got: " + rt.name(rt.str(tp))) d[nm] = (tp, offset.int_val()) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 16f3778b..a3be00c7 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -1,7 +1,9 @@ (ns pixie.tests.test-io (require pixie.test :as t) (require pixie.streams :as st :refer :all) - (require pixie.io :as io)) + (require pixie.streams.utf8 :as utf8 :refer :all) + (require pixie.io :as io) + (require pixie.streams.zlib :as zlib)) (t/deftest test-file-reduction (let [f (io/open-read "tests/pixie/tests/test-io.txt")] @@ -43,3 +45,38 @@ (t/assert-throws? (io/slurp :foo)) (t/assert= "I love 🍺 . This is a thumbs up 👍\n" (io/slurp "tests/pixie/tests/test-io-utf8.txt"))) + +(defn compress-content [output-stream content] + (transduce (map identity) + (-> output-stream + (zlib/compressing-output-stream (buffer 512) {}) + (io/buffered-output-stream 1024) + utf8/utf8-output-stream-rf) + (str content))) + +(defn compress-and-decompress-content [output-stream content] + (transduce (map identity) + (-> output-stream + (zlib/decompressing-output-stream (buffer 512) {}) + (zlib/compressing-output-stream (buffer 512) {}) + (io/buffered-output-stream 1024) + utf8/utf8-output-stream-rf) + (str content))) + +(t/deftest test-write-compressed + (io/run-command "rm compressed-output.gz") + (compress-content (io/open-write "compressed-output.gz") (range 1000)) + ;; decompress the file with zcat + (io/run-command "zcat compressed-output.gz > compressed-output.txt") + (t/assert= (range 1000) (read-string (io/slurp "compressed-output.txt"))) + + ;; Wrapping an IInputStream in decompressing-stream should get the same result + (t/assert= (range 1000) (read-string (io/slurp + (zlib/decompressing-input-stream + (io/open-read "compressed-output.gz") + (buffer 512) {}))))) + +;; sticking a compressor into a decompressor should result in the original data +(t/deftest test-decompression + (compress-and-decompress-content (io/open-write "decompressed-output.txt") (range 1000)) + (t/assert= (range 1000) (read-string (io/slurp "decompressed-output.txt")))) From 73cc88dab7dbadd7cc28bd3c7ffb27faa1e3670f Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 3 Jun 2015 14:24:48 +0100 Subject: [PATCH 757/909] BufferedInputStreams implement ISeekableStream --- pixie/io.pxi | 20 ++++++++++++ tests/pixie/tests/test-io.pxi | 61 +++++++++++++++++++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/pixie/io.pxi b/pixie/io.pxi index 3a14ca7c..e8c5beb2 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -120,6 +120,26 @@ (let [val (nth buffer idx)] (set-field! this :idx (inc idx)) val))) + ISeekableStream + (position [this] + (+ (- (position upstream) + (count buffer)) + idx)) + (rewind [this] + (seek this 0)) + (seek [this pos] + ;; We can be clever about seeking. If we are seeking to somewhere with in + ;; our current buffer, we can avoid seeking in upstream. + (let [upper-bounds (position upstream) + lower-bounds (- upper-bounds (count buffer))] + (if (and (>= pos lower-bounds) + (<= pos upper-bounds)) + ;; We're in the buffer window :-) + (set-field! this :idx (- pos lower-bounds)) + ;; Put the index at the end of the buffer to force a read from upstream + (do + (set-field! this :idx (count buffer)) + (seek upstream pos))))) IDisposable (-dispose! [this] (dispose! buffer))) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index a3be00c7..b06807d6 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -37,6 +37,67 @@ (io/seek f (- (position f) 6)) (t/assert= (io/read-line f) "line."))) +(t/deftest test-buffered-input-streams + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert= (char (io/read-byte f)) \T) + (t/assert= (char (io/read-byte f)) \h) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (char (io/read-byte f)) \s))) + +(t/deftest test-buffered-input-streams-seek + ;; We use a buffer size of 4 because the test file isn't huge and i am terrible at + ;; counting characters... + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt") 4)] + ;; Read the first word 'This' + (t/assert= (position f) 0) + (t/assert= (char (io/read-byte f)) \T) + (t/assert= (position f) 1) + (t/assert= (char (io/read-byte f)) \h) + (t/assert= (position f) 2) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 3) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (position f) 4) + + ;; Back to start of file (this is a seek with in the buffer) + (rewind f) + + ;; Should read the first word again + (t/assert= (char (io/read-byte f)) \T) + (t/assert= (position f) 1) + (t/assert= (char (io/read-byte f)) \h) + (t/assert= (position f) 2) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 3) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (position f) 4) + + ;; Skip the space (we will have caused a seek upstream) + (seek f 5) + + ;; Read 'is' + (t/assert= (position f) 5) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 6) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (position f) 7) + + ;; Jump ahead to 'file' (another seek upstream) + (seek f 15) + (t/assert= (position f) 15) + (t/assert= (char (io/read-byte f)) \f) + (t/assert= (position f) 16) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 17) + (t/assert= (char (io/read-byte f)) \l) + (t/assert= (position f) 18) + (t/assert= (char (io/read-byte f)) \e) + + ;; Another seek with in a buffer + (seek f 16) + (t/assert= (position f) 16) + (t/assert= (char (io/read-byte f)) \i))) + (t/deftest test-slurp-spit (let [val (vec (range 1280))] (io/spit "test.tmp" val) From 3c86e86c6687bb6d5317f220295a63b12ae7afce Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 19 Jul 2015 14:10:07 +0100 Subject: [PATCH 758/909] use empty? instead --- pixie/io.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index e8c5beb2..ee1d34bf 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -116,7 +116,7 @@ (when (= idx (count buffer)) (set-field! this :idx 0) (read upstream buffer (buffer-capacity buffer))) - (when (pos? (count buffer)) + (when-not (empty? buffer) (let [val (nth buffer idx)] (set-field! this :idx (inc idx)) val))) From b36e750423d146988579947cec7de41b426ff80c Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 19 Jul 2015 15:09:04 +0100 Subject: [PATCH 759/909] fixed ILookup on vectors --- pixie/vm/persistent_vector.py | 2 +- tests/pixie/tests/test-stdlib.pxi | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index a0e1ae55..202e3c72 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -439,7 +439,7 @@ def _nth_not_found(self, idx, not_found): def _val_at(self, key, not_found): assert isinstance(self, PersistentVector) if isinstance(key, Integer): - return self.nth(key.int_val()) + return self.nth(key.int_val(), not_found) else: return not_found diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index f5650df7..441cf00b 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -109,6 +109,14 @@ (t/assert= (nth (range 4) 1 :default) 1) (t/assert= (nth "hithere" 1 :deafult) \i)) +(t/deftest test-get-on-vector + (t/assert= (get [1 2 3] 0) 1) + (t/assert= (get [1 2 3] 1) 2) + (t/assert= (get [1 2 3] 2) 3) + (t/assert= (get [1 2 3] 4) nil) + (t/assert= (get [1 2 3] 5) nil) + (t/assert= (get [1 2 3] 5 :not-found) :not-found)) + (t/deftest test-first (t/assert= (first []) nil) (t/assert= (first '()) nil) From 314007d4e3c6a31d5f07cf67d86b3a59da76eccb Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 20 Jul 2015 12:58:01 +0100 Subject: [PATCH 760/909] flush on dispose not write --- pixie/io.pxi | 1 - 1 file changed, 1 deletion(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 3a14ca7c..06e8847c 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -100,7 +100,6 @@ IDisposable (-dispose! [this] (set-buffer-count! buffer idx) - (write downstream buffer) (flush this)) IFlushableStream (flush [this] From e1d8e3fb36d11b8514b13af8a4516a6fff6c0acd Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 20 Jul 2015 13:30:31 +0100 Subject: [PATCH 761/909] do not ignore length arg passed to read previously the length of the buffer was being read rather than the number passed in by the user --- pixie/io.pxi | 4 ++-- tests/pixie/tests/test-io.pxi | 16 ++++++++++++++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 06e8847c..d1a6738c 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -16,10 +16,10 @@ (deftype FileStream [fp offset uvbuf] IInputStream (read [this buffer len] - (assert (<= (buffer-capacity buffer) len) + (assert (>= (buffer-capacity buffer) len) "Not enough capacity in the buffer") (let [_ (pixie.ffi/set! uvbuf :base buffer) - _ (pixie.ffi/set! uvbuf :len (buffer-capacity buffer)) + _ (pixie.ffi/set! uvbuf :len len) read-count (fs_read fp uvbuf 1 offset)] (assert (not (neg? read-count)) "Read Error") (set-field! this :offset (+ offset read-count)) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index a3be00c7..01fa0839 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -16,6 +16,22 @@ (let [f (io/run-command "ls tests/pixie/tests/test-io.txt")] (t/assert= f "tests/pixie/tests/test-io.txt\n"))) +(t/deftest test-read-into-buffer + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (let [buf (buffer 16)] + (io/read f buf 4) + (t/assert= (transduce (map char) string-builder buf) "This") + + (io/read f buf 4) + (t/assert= (transduce (map char) string-builder buf) " is ") + + + (io/read f buf 0) + (t/assert= (transduce (map char) string-builder buf) "") + + (t/assert-throws? (io/read f buf 17)) + (t/assert-throws? (io/read f buf -2))))) + (t/deftest test-read-line (let [f (io/open-read "tests/pixie/tests/test-io.txt")] (io/read-line f) From 1524626651f3728313c083d21e37a9e373f503ed Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 20 Jul 2015 17:21:44 +0100 Subject: [PATCH 762/909] clean up unused code --- pixie/vm/bootstrap.py | 1 - pixie/vm/compiler.py | 8 +++----- pixie/vm/custom_types.py | 4 +--- pixie/vm/object.py | 5 +---- pixie/vm/stdlib.py | 5 ----- 5 files changed, 5 insertions(+), 18 deletions(-) diff --git a/pixie/vm/bootstrap.py b/pixie/vm/bootstrap.py index 623d9442..10b9f49b 100644 --- a/pixie/vm/bootstrap.py +++ b/pixie/vm/bootstrap.py @@ -3,7 +3,6 @@ @wrap_fn def bootstrap(): import pixie.vm.rt as rt - from pixie.vm.string import String assert False rt.load_ns(rt.wrap(u"pixie/stdlib.pxi")) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index a6bbd64f..7dbb7477 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -258,7 +258,7 @@ def emit(self, ctx): -def resolve_var(ctx, name): +def resolve_var(name): return NS_VAR.deref().resolve(name) def resolve_local(ctx, name): @@ -270,7 +270,7 @@ def is_macro_call(form, ctx): name = rt.name(rt.first(form)) if resolve_local(ctx, name): return None - var = resolve_var(ctx, rt.first(form)) + var = resolve_var(rt.first(form)) if var and var.is_defined(): val = var.deref() @@ -377,7 +377,7 @@ def compile_form(form, ctx): ns = rt.namespace(form) loc = resolve_local(ctx, name) - var = resolve_var(ctx, form) + var = resolve_var(form) if var is None and loc: loc.emit(ctx) @@ -409,7 +409,6 @@ def compile_form(form, ctx): return if isinstance(form, PersistentVector): - vector_var = rt.vector() size = rt.count(form) #assert rt.count(form).int_val() == 0 ctx.push_const(code.intern_var(u"pixie.stdlib", u"vector")) @@ -513,7 +512,6 @@ def compile_fn(form, ctx): def compile_fn_body(name, args, body, ctx): new_ctx = Context(rt.name(name), rt.count(args), ctx) required_args = add_args(rt.name(name), args, new_ctx) - bc = 0 affirm(isinstance(name, symbol.Symbol), u"Function names must be symbols") diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index b1aa73c9..46c0f92a 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -1,6 +1,5 @@ from pixie.vm.object import Object, Type, affirm, runtime_error import rpython.rlib.jit as jit -from rpython.rlib.rarithmetic import r_uint from pixie.vm.code import as_var from pixie.vm.numbers import Integer, Float from pixie.vm.keyword import Keyword @@ -8,7 +7,6 @@ MAX_FIELDS = 32 - class CustomType(Type): _immutable_fields_ = ["_slots", "_rev?"] def __init__(self, name, slots): @@ -254,4 +252,4 @@ def set_mutable_cell_value(self, ct, fields, nm, idx, value): self._mutable_float_val = value.float_val() def get_mutable_cell_value(self): - return rt.wrap(self._mutable_float_val) \ No newline at end of file + return rt.wrap(self._mutable_float_val) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 4c39f162..4632eaf0 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -41,7 +41,6 @@ def register_type(self, nm, tp): self.var_for_type_and_name(nm, tp) def var_for_type_and_name(self, nm, tp): - from pixie.vm.symbol import symbol splits = nm.split(u".") size = len(splits) - 1 assert size >= 0 @@ -242,7 +241,6 @@ def __repr__(self): def trace_map(self): from pixie.vm.string import String - from pixie.vm.numbers import Integer from pixie.vm.keyword import keyword tp = self._tp @@ -261,7 +259,6 @@ def __repr__(self): def trace_map(self): from pixie.vm.string import String - from pixie.vm.numbers import Integer from pixie.vm.keyword import keyword tm = {keyword(u"type") : keyword(u"pixie")} @@ -292,4 +289,4 @@ def trace_map(self): def add_info(ex, data): assert isinstance(ex, WrappedException) ex._ex._trace.append(ExtraCodeInfo(data)) - return ex \ No newline at end of file + return ex diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 5b4326a0..9abbef3e 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -8,7 +8,6 @@ import pixie.vm.numbers as numbers import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import r_uint -from pixie.vm.interpreter import ShallowContinuation from rpython.rlib.objectmodel import we_are_translated defprotocol("pixie.stdlib", "ISeq", ["-first", "-next"]) @@ -99,7 +98,6 @@ def __meta(self, meta): def default_str(x): - from pixie.vm.string import String tp = x.type() assert isinstance(tp, Type) return rt.wrap(u"") @@ -284,7 +282,6 @@ def nth_not_found(a, b, c): @as_var("str") def str__args(args): - from pixie.vm.string import String acc = [] for x in args: acc.append(rt.name(rt._str(x))) @@ -401,7 +398,6 @@ def _load_file(filename, compile=False): import pixie.vm.reader as reader import pixie.vm.libs.pxic.writer as pxic_writer import os.path as path - import os affirm(isinstance(filename, String), u"filename must be a string") @@ -684,7 +680,6 @@ def _try_catch(main_fn, catch_fn, final): return main_fn.invoke([]) except Exception as ex: if not isinstance(ex, WrappedException): - from pixie.vm.string import String if isinstance(ex, Exception): if not we_are_translated(): print "Python Error Info: ", ex.__dict__, ex From 9eed1c78e23f417ae70f29332871f1d5d5b03a60 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 20 Jul 2015 17:31:13 +0100 Subject: [PATCH 763/909] rm PromptReader --- pixie/vm/reader.py | 26 -------------------------- target.py | 7 +++---- 2 files changed, 3 insertions(+), 30 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 64ad57c9..0df93adc 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -69,32 +69,6 @@ def read(self): def unread(self): self._idx -= 1 -class PromptReader(PlatformReader): - def __init__(self): - self._string_reader = None - - - def read(self): - if self._string_reader is None: - result = _readline(str(rt.name(rt.ns.deref())) + " => ") - if result == u"": - raise EOFError() - self._string_reader = StringReader(result) - - try: - return self._string_reader.read() - except EOFError: - self._string_reader = None - return self.read() - - def reset_line(self): - self._string_reader = None - - def unread(self): - assert self._string_reader is not None - self._string_reader.unread() - - class UserSpaceReader(PlatformReader): def __init__(self, reader_fn): self._string_reader = None diff --git a/target.py b/target.py index 66f397c5..a4126317 100644 --- a/target.py +++ b/target.py @@ -1,12 +1,11 @@ -from pixie.vm.compiler import compile, with_ns, NS_VAR -from pixie.vm.reader import StringReader, read_inner, eof, PromptReader, MetaDataReader -from pixie.vm.interpreter import interpret +from pixie.vm.compiler import with_ns, NS_VAR +from pixie.vm.reader import StringReader from rpython.jit.codewriter.policy import JitPolicy from rpython.rlib.jit import JitHookInterface, Counters from rpython.rlib.rfile import create_stdio from rpython.annotator.policy import AnnotatorPolicy from pixie.vm.code import wrap_fn, NativeFn, intern_var, Var -from pixie.vm.object import RuntimeException, WrappedException +from pixie.vm.object import WrappedException from rpython.translator.platform import platform from pixie.vm.primitives import nil from pixie.vm.atom import Atom From e452d808d1cfedc766e840e8c9b39e98fe7de884 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Tue, 21 Jul 2015 09:28:53 +0100 Subject: [PATCH 764/909] s/False/false --- pixie/ffi-infer.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index f2074f53..9f13ac74 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -108,7 +108,7 @@ return 0; [{:keys [size]} _] (cond (= size 8) 'pixie.stdlib/CDouble - :else (assert False "unknown type"))) + :else (assert false "unknown type"))) (defmethod edn-to-ctype :void [_ _] From 0fc2621bd37fb4cb26be4ede06bd6636afb17110 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Mon, 27 Jul 2015 18:16:33 -0500 Subject: [PATCH 765/909] Adding a string reverse function. --- pixie/string.pxi | 9 ++++++++- tests/pixie/tests/test-strings.pxi | 6 ++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/pixie/string.pxi b/pixie/string.pxi index fdbbcbb3..4d1bf9a1 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -1,5 +1,6 @@ (ns pixie.string - (:require [pixie.string.internal :as si])) + (:require [pixie.stdlib :as std] + [pixie.string.internal :as si])) ; reexport native string functions (def substring si/substring) @@ -45,6 +46,12 @@ (str (substring s 0 i) r (substring s (+ i (count x)))) s)) +(defn reverse + "Returns s with its characters reversed." + [s] + (when s + (apply str (std/reverse s)))) + (defn join {:doc "Join the elements of the collection using an optional separator" :examples [["(require pixie.string :as s)"] diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 27b3f13b..c769cd8b 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -101,6 +101,12 @@ (t/assert= (s/replace-first "&&&" "&" "&&") "&&&&") (t/assert= (s/replace-first "oops" "" "WAT") "WAToops")) +(t/deftest test-reverse + (t/assert= (s/reverse "not a palindrome") "emordnilap a ton") + (t/assert= (s/reverse "tacocat") "tacocat") + (t/assert= (s/reverse "") "") + (t/assert= (s/reverse nil) nil)) + (t/deftest test-join (t/assert= (s/join []) "") (t/assert= (s/join [1]) "1") From 022631929813f256e3af56cddea3f03e437b5ca6 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 6 Aug 2015 19:49:16 -0500 Subject: [PATCH 766/909] Adding a string escape function. --- pixie/string.pxi | 13 +++++++++++++ tests/pixie/tests/test-strings.pxi | 9 +++++++++ 2 files changed, 22 insertions(+) diff --git a/pixie/string.pxi b/pixie/string.pxi index fdbbcbb3..c65dfda5 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -73,6 +73,19 @@ false)))) true)) +(defn escape + "Return a new string, using cmap to escape each character ch + from s as follows: + + If (cmap ch) is nil, append ch to the new string. + If (cmap ch) is non-nil, append (str (cmap ch)) instead." + [s cmap] + (if (or (nil? s) + (nil? cmap)) + s + (apply str (map #(if-let [c (cmap %)] c %) + (vec s))))) + (defmacro interp ; TODO: This might merit special read syntax {:doc "String interpolation." diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 27b3f13b..e9201e1b 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -141,3 +141,12 @@ (t/assert= (s/blank? " ") true) (t/assert= (s/blank? " \t \n \r ") true) (t/assert= (s/blank? " foo ") false)) + +(t/deftest test-escape + (t/assert= (s/escape "foo" {\f \z}) "zoo") + (t/assert= (s/escape "foo" {\z \f}) "foo") + (t/assert= (s/escape "foobar" {\f \b \o \e \b \j}) "beejar") + (t/assert= (s/escape "foo" {}) "foo") + (t/assert= (s/escape "foo" nil) "foo") + (t/assert= (s/escape "" {\f \z}) "") + (t/assert= (s/escape nil {\f \z}) nil)) From ece248621a36e1215017d583f1a90109f7a182ed Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Tue, 11 Aug 2015 12:59:33 -0500 Subject: [PATCH 767/909] Adding a string-split function to pixie.string. --- pixie/string.pxi | 5 +++++ tests/pixie/tests/test-strings.pxi | 21 +++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/pixie/string.pxi b/pixie/string.pxi index 9b40d8fd..866085a8 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -7,6 +7,11 @@ (def index-of (comp #(if (not= -1 %) %) si/index-of)) (def split si/split) +(defn split-lines + "Splits on \\n or \\r\\n, the two typical line breaks." + [s] + (when s (apply concat (map #(split % "\n") (split s "\r\n"))))) + (def ends-with? si/ends-with) (def starts-with? si/starts-with) diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 3c244c56..7e3f6376 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -27,6 +27,27 @@ (t/assert= (s/split s ",") ["hey" "ho" "huh"]) (t/assert= (s/split s "h") ["" "ey," "o," "u" ""]))) +(t/deftest test-split-lines + ;; Splits unix-style lines. + (t/assert= (s/split-lines "bibbidi\nbobbidi\nboo") + ["bibbidi" "bobbidi" "boo"]) + ;; Splits windows-style lines. + (t/assert= (s/split-lines "bibbidi\r\nbobbidi\r\nboo") + ["bibbidi" "bobbidi" "boo"]) + ;; Splits mixed up stuff. + (t/assert= (s/split-lines "bibbidi\nbobbidi\r\nboo") + ["bibbidi" "bobbidi" "boo"]) + ;; Doesn't split lonely \r + (t/assert= (s/split-lines "bibbidi\nbobbidi\rboo") + ["bibbidi" "bobbidi\rboo"]) + ;; Works with a single line. + (t/assert= (s/split-lines "BibbidiBobbidiBoo") + ["BibbidiBobbidiBoo"]) + ;; Works with empty strings. + (t/assert= (s/split-lines "") "") + ;; Nil pass-through. + (t/assert= (s/split-lines nil) nil)) + (t/deftest test-index-of (let [s "heyhohuh"] (t/assert= (s/index-of s "hey") 0) From 429962a63eff75ac5ce017ee6235e97ec7128bfa Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Tue, 11 Aug 2015 13:52:28 -0500 Subject: [PATCH 768/909] Fixing empty string test for split-lines. --- tests/pixie/tests/test-strings.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 7e3f6376..129c5768 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -44,7 +44,7 @@ (t/assert= (s/split-lines "BibbidiBobbidiBoo") ["BibbidiBobbidiBoo"]) ;; Works with empty strings. - (t/assert= (s/split-lines "") "") + (t/assert= (s/split-lines "") [""]) ;; Nil pass-through. (t/assert= (s/split-lines nil) nil)) From 0246159063300bcf8465b1ae65e6f05fc9ef4933 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 20 Jul 2015 17:21:44 +0100 Subject: [PATCH 769/909] clean up unused code --- pixie/vm/cons.py | 7 ------- pixie/vm/keyword.py | 2 +- pixie/vm/stdlib.py | 19 +++++++------------ tests/pixie/tests/test-keywords.pxi | 7 ++++++- tests/pixie/tests/test-stdlib.pxi | 7 +++++++ 5 files changed, 21 insertions(+), 21 deletions(-) diff --git a/pixie/vm/cons.py b/pixie/vm/cons.py index e9f0eb92..bc27357e 100644 --- a/pixie/vm/cons.py +++ b/pixie/vm/cons.py @@ -57,12 +57,5 @@ def _with_meta(self, meta): def cons(head, tail): return Cons(head, tail) -def count(self): - cnt = 0 - while self is not nil: - self = self.next() - cnt += 1 - return cnt - def cons(head, tail=nil): return Cons(head, tail, nil) diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index f6ce6dbc..7da9af14 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -77,5 +77,5 @@ def _hash(self): def _keyword(s): if not isinstance(s, String): from pixie.vm.object import runtime_error - runtime_error(u"Symbol name must be a string") + runtime_error(u"Keyword name must be a string") return keyword(s._str) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 9abbef3e..54358636 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,14 +1,16 @@ # -*- coding: utf-8 -*- from pixie.vm.object import Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ runtime_error, add_info, ExtraCodeInfo -from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ - list_copy, returns, intern_var +from pixie.vm.code import Namespace, BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ + list_copy, returns, intern_var, _ns_registry import pixie.vm.code as code from pixie.vm.primitives import true, false, nil import pixie.vm.numbers as numbers import rpython.rlib.jit as jit from rpython.rlib.rarithmetic import r_uint from rpython.rlib.objectmodel import we_are_translated +import os.path as path +import sys defprotocol("pixie.stdlib", "ISeq", ["-first", "-next"]) defprotocol("pixie.stdlib", "ISeqable", ["-seq"]) @@ -351,7 +353,6 @@ def is_undefined(var): def load_ns(filename): import pixie.vm.string as string import pixie.vm.symbol as symbol - import os.path as path if isinstance(filename, symbol.Symbol): affirm(rt.namespace(filename) is None, u"load-file takes a un-namespaced symbol") @@ -397,7 +398,6 @@ def _load_file(filename, compile=False): from pixie.vm.util import unicode_from_utf8 import pixie.vm.reader as reader import pixie.vm.libs.pxic.writer as pxic_writer - import os.path as path affirm(isinstance(filename, String), u"filename must be a string") @@ -439,7 +439,6 @@ def load_pxic_file(filename): from pixie.vm.libs.pxic.reader import Reader, read_obj from pixie.vm.reader import eof import pixie.vm.compiler as compiler - import sys if not we_are_translated(): print "Loading precompiled file while interpreted, this may take time" @@ -463,7 +462,6 @@ def load_pxic_file(filename): def load_reader(rdr): import pixie.vm.reader as reader import pixie.vm.compiler as compiler - import sys if not we_are_translated(): print "Loading file while interpreted, this may take time" @@ -531,7 +529,6 @@ def in_ns(ns_name): @as_var("ns-map") def ns_map(ns): - from pixie.vm.code import Namespace from pixie.vm.symbol import Symbol affirm(isinstance(ns, Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") @@ -551,7 +548,6 @@ def ns_map(ns): @as_var("ns-aliases") def ns_aliases(ns): - from pixie.vm.code import Namespace from pixie.vm.symbol import Symbol affirm(isinstance(ns, Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") @@ -573,7 +569,6 @@ def ns_aliases(ns): def refer(ns, refer, alias): from pixie.vm.symbol import Symbol from pixie.vm.string import String - from pixie.vm.code import _ns_registry if isinstance(ns, Symbol) or isinstance(ns, String): ns = _ns_registry.find_or_make(rt.name(ns)) @@ -594,9 +589,9 @@ def refer(ns, refer, alias): def refer_symbol(ns, sym, var): from pixie.vm.symbol import Symbol - affirm(isinstance(ns, code.Namespace), u"First argument must be a namespace") + affirm(isinstance(ns, Namespace), u"First argument must be a namespace") affirm(isinstance(sym, Symbol) and rt.namespace(sym) is None, u"Second argument must be a non-namespaced symbol") - affirm(isinstance(var, code.Var), u"Third argument must be a var") + affirm(isinstance(var, Var), u"Third argument must be a var") ns.add_refer_symbol(sym, var) return nil @@ -712,7 +707,7 @@ def _throw(ex): @as_var("resolve-in") def _var(ns, nm): - affirm(isinstance(ns, code.Namespace), u"First argument to resolve-in must be a namespace") + affirm(isinstance(ns, Namespace), u"First argument to resolve-in must be a namespace") var = ns.resolve(nm) return var if var is not None else nil diff --git a/tests/pixie/tests/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi index cdd0672c..9b4357e7 100644 --- a/tests/pixie/tests/test-keywords.pxi +++ b/tests/pixie/tests/test-keywords.pxi @@ -30,4 +30,9 @@ (t/deftest string-to-keyword (t/assert= (keyword "foo") :foo) - (t/assert= (keyword "foo/bar") :foo/bar)) + (t/assert= (keyword "foo/bar") :foo/bar) + (t/assert-throws? (keyword 1)) + (t/assert-throws? (keyword :a)) + (t/assert-throws? (keyword 'a)) + (t/assert-throws? (keyword nil)) + (t/assert-throws? (keyword true))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 441cf00b..9591da37 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -544,6 +544,13 @@ "c must be a type" (instance? [Keyword :also-not-a-type] 123))) +(t/deftest test-types-are-types + (t/assert= Type (type Keyword)) + (t/assert= Type (type Integer)) + (t/assert= Type (type Number)) + (t/assert= Type (type Object)) + (t/assert= Type (type Type))) + (t/deftest test-satisfies? (t/assert= (satisfies? IIndexed [1 2]) true) (t/assert= (satisfies? IIndexed '(1 2)) false) From 3c9b164d0dc51e810b81845b50690989a5e52d65 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 12 Aug 2015 21:38:13 +0100 Subject: [PATCH 770/909] read-line can handle buffered-streams --- pixie/io.pxi | 42 +++++++++++++++++++++++++++++++++++------- 1 file changed, 35 insertions(+), 7 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 0993b53d..2e8f19b8 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -47,10 +47,21 @@ (assert (string? filename) "Filename must be a string") (->FileStream (fs_open filename uv/O_RDONLY 0) 0 (uv/uv_buf_t))) +(defn buffered-read-line + [input-stream] + (let [line-feed (into #{} (map int [\newline \return]))] + (loop [acc []] + (let [ch (read-byte input-stream)] + (cond + (nil? ch) nil + (zero? ch) nil -(defn read-line - "Read one line from input-stream for each invocation. - nil when all lines have been read" + (and (pos? ch) (not (line-feed ch))) + (recur (conj acc ch)) + + :else (transduce (map char) string-builder acc)))))) + +(defn unbuffered-read-line [input-stream] (let [line-feed (into #{} (map int [\newline \return])) buf (buffer 1)] @@ -62,7 +73,22 @@ (and (zero? len) (empty? acc)) nil - :else (apply str (map char acc))))))) + :else (transduce (map char) string-builder acc)))))) + +(defn read-line + "Read one line from input-stream for each invocation. + nil when all lines have been read. + Pass a BufferedInputStream for best performance." + [input-stream] + (cond + (instance? BufferedInputStream input-stream) + (buffered-read-line input-stream) + + (satisfies? IInputStream input-stream) + (unbuffered-read-line input-stream) + + :else + (throw [::Exception "Expected an IInputStream or IByteInputStream"]))) (defn line-seq "Returns the lines of text from input-stream as a lazy sequence of strings. @@ -153,9 +179,11 @@ ([upstream] (buffered-input-stream upstream common/DEFAULT-BUFFER-SIZE)) ([upstream size] - (let [b (buffer size)] - (set-buffer-count! b size) - (->BufferedInputStream upstream size b)))) + (if (satisfies? IInputStream upstream) + (let [b (buffer size)] + (set-buffer-count! b size) + (->BufferedInputStream upstream size b)) + (throw [::Exception "Expected upstream to satisfy IInputStream"])))) (defn throw-on-error [result] (when (neg? result) From 9b5757820baeb451c01b4a05d6bee9952539e9da Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 13 Aug 2015 20:28:40 +0100 Subject: [PATCH 771/909] Simplify string/blank? --- pixie/string.pxi | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/pixie/string.pxi b/pixie/string.pxi index 866085a8..dc0fc7b7 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -75,14 +75,8 @@ "True if s is nil, empty, or contains only whitespace." [s] (if s - (let [white (set whitespace) - length (count s)] - (loop [index 0] - (if (= length index) - true - (if (white (nth s index)) - (recur (inc index)) - false)))) + (let [white (set whitespace)] + (every? white s)) true)) (defn escape From 8d0b9dc91eb9ef940d89d6b49ed23254907393d1 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 13 Aug 2015 20:29:33 +0100 Subject: [PATCH 772/909] added benchmarks --- benchmarks/read-line.pxi | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 benchmarks/read-line.pxi diff --git a/benchmarks/read-line.pxi b/benchmarks/read-line.pxi new file mode 100644 index 00000000..f09e687a --- /dev/null +++ b/benchmarks/read-line.pxi @@ -0,0 +1,18 @@ +(ns benchmarks.readline + (:require [pixie.time :as time] + [pixie.io :as io])) + +(def file-name "/usr/share/dict/words") + +(println "testing unbuffered") +(time/time (-> file-name + (io/open-read) + (io/line-seq) + (count))) + +(println "testing buffered") +(time/time (-> file-name + (io/open-read) + (io/buffered-input-stream) + (io/line-seq) + (count))) From a27b4fb6f1869476fa81a3e338a994d47fdb2843 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 13 Aug 2015 20:38:31 +0100 Subject: [PATCH 773/909] added test around buffered-input-stream creation --- pixie/io.pxi | 2 +- tests/pixie/tests/test-io.pxi | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 2e8f19b8..c07ca1a4 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -88,7 +88,7 @@ (unbuffered-read-line input-stream) :else - (throw [::Exception "Expected an IInputStream or IByteInputStream"]))) + (throw [::Exception "Expected an IInputStream or BufferedInputStream"]))) (defn line-seq "Returns the lines of text from input-stream as a lazy sequence of strings. diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 387c1440..532d9890 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -60,6 +60,10 @@ (t/assert= (char (io/read-byte f)) \i) (t/assert= (char (io/read-byte f)) \s))) +(t/deftest test-buffered-input-streams-throws-on-non-input-streams + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert-throws? (io/buffered-input-stream f)))) + (t/deftest test-buffered-input-streams-seek ;; We use a buffer size of 4 because the test file isn't huge and i am terrible at ;; counting characters... From 920d05d266344585119d676de4b2d656062b1377 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 26 Jun 2015 11:41:28 +0100 Subject: [PATCH 774/909] Basic comparisons on some primitives --- pixie/stdlib.pxi | 97 ++++++++++++++++++++++++++++++ pixie/vm/object.py | 3 + pixie/vm/stdlib.py | 18 ++++-- tests/pixie/tests/test-compare.pxi | 59 ++++++++++++++++++ tests/pixie/tests/test-stdlib.pxi | 10 ++- 5 files changed, 182 insertions(+), 5 deletions(-) create mode 100644 tests/pixie/tests/test-compare.pxi diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 880efd42..fd323a65 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2457,3 +2457,100 @@ Calling this function on something that is not ISeqable returns a seq with that (fn [v] (str ""))) (extend -repr Namespace -str) + + +(defn bool? + [x] + (instance? Bool x)) + +(defprotocol IComparable + (-compare [x y] + "Compare to objects returing 0 if the same -1 with x is logically smaller than y and 1 if x is logically larger")) + +(defn compare-numbers + [x y] + (cond + (> x y) 1 + (< x y) -1 + :else 0)) + +(defn compare-counted + [x y] + (if (= x y) + 0 + (let [min-length (min (count x) (count y))] + (loop [n 0] + (if (not= min-length n) + (let [diff (-compare (nth x n) + (nth y n))] + (if-not (zero? diff) + diff + (recur (inc n)))) + ;; We have compared all characters of the smallest string + ;; against the largest string. + ;; If equal lengths 0 otherwise -1 or 1 + (compare-numbers (count x) (count y))))))) + +(defn compare-named + [x y] + (if (= x y) + 0 + (compare-counted (str x) (str y)))) + +(extend-protocol IComparable + Number + (-compare [x y] + (if (number? y) + (compare-numbers x y) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Character + (-compare [x y] + (if (char? y) + (compare-numbers (int x) (int y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + PersistentVector + (-compare [x y] + (if (vector? y) + (compare-counted x y) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + String + (-compare [x y] + (if (string? y) + (compare-counted (str x) (str y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Keyword + (-compare [x y] + (if (keyword? y) + (compare-counted (str x) (str y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Symbol + (-compare [x y] + (if (symbol? y) + (compare-counted (str x) (str y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Bool + (-compare [x y] + (if (bool? y) + (cond + (= x y) 0 + (and (true? x) (false? y)) 1 + :else -1)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)])) + + Nil + (-compare [x y] + (if (nil? y) + 0 + (throw [::ComparisonError (str "Cannot compare: " x " to " y)])))) + +(defn compare + [x y] + (if (satisfies? IComparable x) + (-compare x y) + (throw [::ComparisonError (str x " does not satisfy IComparable")]))) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 4632eaf0..b80259e8 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -86,6 +86,9 @@ def name(self): def type(self): return Type._type + def parent(self): + return self._parent + def add_subclass(self, tp): self._subclasses.append(tp) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 9abbef3e..3a698060 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from pixie.vm.object import Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ +from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ runtime_error, add_info, ExtraCodeInfo from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ list_copy, returns, intern_var @@ -319,13 +319,23 @@ def _instance(c, o): return true if istypeinstance(o, c) else false +def type_satisfies(proto, type): + affirm(isinstance(type, Type), u"type must be a Type") + if proto.satisfies(type): + return true + elif type == Object._type: + # top level type do not recurse + return false + elif type.parent(): + return type_satisfies(proto, type.parent()) + else: + return false + @returns(bool) @as_var("-satisfies?") def _satisfies(proto, o): affirm(isinstance(proto, Protocol), u"proto must be a Protocol") - - return true if proto.satisfies(o.type()) else false - + return type_satisfies(proto, o.type()) import pixie.vm.rt as rt diff --git a/tests/pixie/tests/test-compare.pxi b/tests/pixie/tests/test-compare.pxi new file mode 100644 index 00000000..3811e7c8 --- /dev/null +++ b/tests/pixie/tests/test-compare.pxi @@ -0,0 +1,59 @@ +(ns pixie.tests.test-compare + (require pixie.test :as t)) + +(t/deftest test-compare-numbers + (t/assert= (compare 1 1) 0) + (t/assert= (compare 1 2) -1) + (t/assert= (compare 1 -1) 1) + + (t/assert= (compare 1 1.0) 0) + (t/assert= (compare 1.0 1) 0) + (t/assert= (compare 1.0 2.0) -1) + (t/assert= (compare 1.0 -1.0) 1) + + (t/assert= (compare 1/2 1/2) 0) + (t/assert= (compare 1/3 1/2) -1) + (t/assert= (compare 1/2 1/3) 1)) + +(t/deftest test-compare-strings + (t/assert= (compare "a" "a") 0) + (t/assert= (compare "a" "b") -1) + (t/assert= (compare "b" "a") 1) + + (t/assert= (compare "aa" "a") 1) + (t/assert= (compare "a" "aa") -1) + + (t/assert= (compare "aa" "b") -1) + (t/assert= (compare "b" "aa") 1) + + (t/assert= (compare "aaaaaaaa" "azaaaaaa") -1) + (t/assert= (compare "azaaaaaa" "aaaaaaaa") 1)) + +(t/deftest test-compare-keywords + (t/assert= (compare :a :a) 0) + (t/assert= (compare :a :b) -1) + (t/assert= (compare :b :a) 1) + (t/assert= (compare :ns/a :ns/a) 0) + (t/assert= (compare :ns/a :ns/b) -1) + (t/assert= (compare :a :aa) -1) + (t/assert= (compare :aa :a) 1)) + +(t/deftest test-compare-symbols + (t/assert= (compare 'a 'a) 0) + (t/assert= (compare 'a 'b) -1) + (t/assert= (compare 'b 'a) 1) + (t/assert= (compare 'ns/a 'ns/a) 0) + (t/assert= (compare 'ns/a 'ns/b) -1) + (t/assert= (compare 'a 'aa) -1) + (t/assert= (compare 'aa 'a) 1)) + +(t/deftest test-compare-vectors + (t/assert= (compare [] []) 0) + (t/assert= (compare [1] []) 1) + (t/assert= (compare [1] [2]) -1) + (t/assert= (compare [1 2] [1 3]) -1) + (t/assert= (compare [:a] [:a]) 0) + (t/assert= (compare [:a] [:b]) -1) + (t/assert= (compare [:a] [:a :a]) -1) + (t/assert= (compare ["a"] ["a"]) 0) + (t/assert= (compare ["a"] ["a" "b"]) -1)) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 441cf00b..e41d9207 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -555,7 +555,15 @@ (satisfies? :not-a-proto 123)) (t/assert-throws? RuntimeException "proto must be a Protocol" - (satisfies? [IIndexed :also-not-a-proto] [1 2]))) + (satisfies? [IIndexed :also-not-a-proto] [1 2])) + (defprotocol IFoo (foo [this])) + (extend-protocol IFoo + Number + (foo [this] this)) + (t/assert= (satisfies? IFoo 1) true) + (t/assert= (satisfies? IFoo 1.0) true) + (t/assert= (satisfies? IFoo 1/2) true) + (t/assert= (satisfies? IFoo \a) false)) (t/deftest test-reduce (t/assert= 5050 (reduce + (range 101))) From 70ea4345674fdfb05b35e8a4cbd3c0a77959cb76 Mon Sep 17 00:00:00 2001 From: Joshua Greenberg Date: Mon, 17 Aug 2015 21:44:23 -0700 Subject: [PATCH 775/909] improve the speed of repeated get-field calls to same custom type --- pixie/vm/custom_types.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 46c0f92a..498bf9fe 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -8,7 +8,7 @@ MAX_FIELDS = 32 class CustomType(Type): - _immutable_fields_ = ["_slots", "_rev?"] + _immutable_fields_ = ["_slots[*]", "_rev?"] def __init__(self, name, slots): Type.__init__(self, name) @@ -16,7 +16,7 @@ def __init__(self, name, slots): self._mutable_slots = {} self._rev = 0 - @jit.elidable_promote() + @jit.elidable def get_slot_idx(self, nm): return self._slots.get(nm, -1) @@ -26,19 +26,19 @@ def set_mutable(self, nm): self._mutable_slots[nm] = nm - @jit.elidable_promote() + @jit.elidable def _is_mutable(self, nm, rev): return nm in self._mutable_slots def is_mutable(self, nm): return self._is_mutable(nm, self._rev) - @jit.elidable_promote() + @jit.elidable def get_num_slots(self): return len(self._slots) class CustomTypeInstance(Object): - _immutable_fields_ = ["_type"] + _immutable_fields_ = ["_custom_type"] def __init__(self, type): affirm(isinstance(type, CustomType), u"Can't create a instance of a non custom type") self._custom_type = type @@ -60,7 +60,7 @@ def set_field(self, name, val): self.set_field_by_idx(idx, val) return self - @jit.elidable_promote() + @jit.elidable def _get_field_immutable(self, idx, rev): return self.get_field_by_idx(idx) From e5a6d291af4e6669c87266787374a3b5066fe9cd Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 18 Aug 2015 21:57:26 -0600 Subject: [PATCH 776/909] rebase master --- pixie/stacklets.pxi | 59 ++++++++++++++++++++-------- pixie/uv.pxi | 2 + pixie/vm/code.py | 8 ++++ pixie/vm/libs/ffi.py | 91 ++++++++++++++++++++++++++++++-------------- pixie/vm/object.py | 37 ++++++++++++++++++ pixie/vm/reader.py | 6 +++ pixie/vm/stdlib.py | 18 +++++++++ 7 files changed, 177 insertions(+), 44 deletions(-) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi index 1255fde0..267c7530 100644 --- a/pixie/stacklets.pxi +++ b/pixie/stacklets.pxi @@ -65,21 +65,26 @@ pixie.ffi/dispose!)) ;;; Sleep -(defn sleep [ms] - (let [f (fn [k] - (let [cb (atom nil) - timer (uv/uv_timer_t)] - (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb - (fn [handle] - (try - (run-and-process k) - (uv/uv_timer_stop timer) - (-dispose! @cb) - (catch ex - (println ex)))))) - (uv/uv_timer_init (uv/uv_default_loop) timer) - (uv/uv_timer_start timer @cb ms 0)))] - (call-cc f))) +(defn sleep + ([ms] + (sleep ms false)) + ([ms background?] + (let [f (fn [k] + (let [cb (atom nil) + timer (uv/uv_timer_t)] + (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb + (fn [handle] + (try + (run-and-process k) + (uv/uv_timer_stop timer) + (-dispose! @cb) + (catch ex + (println ex)))))) + (uv/uv_timer_init (uv/uv_default_loop) timer) + (when background? + (uv/uv_unref timer)) + (uv/uv_timer_start timer @cb ms 0)))] + (call-cc f)))) ;; Spawn (defn -spawn [start-fn] @@ -101,6 +106,17 @@ (catch e (println e))))))) +(defmacro spawn-background [& body] + `(let [frames (-get-current-var-frames nil)] + (-spawn (fn [h# _] + (-set-current-var-frames nil frames) + (try + (reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (call-cc (fn [_] nil))) + (catch e + (println e))))))) + (defn spawn-from-non-stacklet [f] (let [s (new-stacklet (fn [h _] @@ -117,6 +133,14 @@ (run-and-process s))))) +(defn finalizer-loop [] + (spawn-background + (loop [] + (-run-finalizers) + (sleep 1000 true) + (recur)))) + + (defn -with-stacklets [fn] (swap! running-threads inc) (reset! main-loop-running? true) @@ -134,12 +158,15 @@ (fn [h# _] (try (reset! stacklet-loop-h h#) + (finalizer-loop) (let [result# (do ~@body)] (swap! running-threads dec) (call-cc (fn [_] nil))) (catch e (println e)))))) + + (defn run-with-stacklets [f] (with-stacklets (f))) @@ -166,3 +193,5 @@ (fn [] (let [result (apply f args)] (-run-later (fn [] (run-and-process k result))))))))) + + diff --git a/pixie/uv.pxi b/pixie/uv.pxi index 06f7af7d..0bcf4fba 100644 --- a/pixie/uv.pxi +++ b/pixie/uv.pxi @@ -20,6 +20,8 @@ (f/defcfn uv_run) (f/defcfn uv_loop_alive) + (f/defcfn uv_unref) + (f/defcfn uv_ref) (f/defcfn uv_stop) (f/defcfn uv_loop_size) (f/defcfn uv_backend_fd) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 83fa6cd1..573273c1 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -693,6 +693,14 @@ def extend(self, tp, fn): self._fn_cache = {} self._protocol.add_satisfies(tp) + ## We have to special case this so that the GC doesn't go nuts trying to do a ton during + ## collection. + self.maybe_mark_finalizer(tp) + + def maybe_mark_finalizer(self, tp): + ## Gets overridden in stdlib + pass + def _find_parent_fn(self, tp): ## Search the entire object tree to find the function to execute assert isinstance(tp, object.Type) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 7d961ce4..0fc34ddb 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -29,6 +29,9 @@ """ +class PointerType(object.Object): + pass + class CType(object.Type): def __init__(self, name): @@ -89,54 +92,54 @@ def get_fn_ptr(self, nm): class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") - _immutable_fields_ = ["_is_inited", "_lib", "_name", "_arg_types[*]", "_arity", "_ret_type", "_is_variadic", \ - "_transfer_size", "_arg0_offset", "_ret_offset", "_cd"] + _immutable_fields_ = ["_name", "_f_ptr", "_c_fn_type"] def type(self): return FFIFn._type - def __init__(self, lib, name, arg_types, ret_type, is_variadic): + def __init__(self, name, fn_ptr, c_fn_type): + assert isinstance(c_fn_type, CFunctionType) self._rev = 0 self._name = name - self._lib = lib - self._arg_types = arg_types - self._arity = len(arg_types) - self._ret_type = ret_type - self._is_variadic = is_variadic - self._f_ptr = self._lib.get_fn_ptr(self._name) - self._cd = CifDescrBuilder(self._arg_types, self._ret_type).rawallocate() + self._f_ptr = fn_ptr + self._c_fn_type = c_fn_type @jit.unroll_safe def prep_exb(self, args): - size = jit.promote(self._cd.exchange_size) + cd = self._c_fn_type.get_cd() + fn_tp = self._c_fn_type + + size = jit.promote(cd.exchange_size) exb = rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, size, flavor="raw")) tokens = [None] * len(args) - for i, tp in enumerate(self._arg_types): - offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_args[i])) + for i, tp in enumerate(fn_tp._arg_types): + offset_p = rffi.ptradd(exb, jit.promote(cd.exchange_args[i])) tokens[i] = tp.ffi_set_value(offset_p, args[i]) return exb, tokens def get_ret_val_from_buffer(self, exb): - offset_p = rffi.ptradd(exb, jit.promote(self._cd.exchange_result)) - ret_val = self._ret_type.ffi_get_value(offset_p) + cd = self._c_fn_type.get_cd() + offset_p = rffi.ptradd(exb, jit.promote(cd.exchange_result)) + ret_val = self._c_fn_type._ret_type.ffi_get_value(offset_p) return ret_val @jit.unroll_safe def _invoke(self, args): arity = len(args) - if self._is_variadic: - if arity < self._arity: + tp_arity = len(self._c_fn_type._arg_types) + if self._c_fn_type._is_variadic: + if arity < tp_arity: runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + - u", expected at least " + unicode(str(self._arity))) + u", expected at least " + unicode(str(tp_arity))) else: - if arity != self._arity: + if arity != tp_arity: runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + - u", expected " + unicode(str(self._arity))) + u", expected " + unicode(str(tp_arity))) exb, tokens = self.prep_exb(args) - cd = jit.promote(self._cd) + cd = jit.promote(self._c_fn_type.get_cd()) #fp = jit.promote(self._f_ptr) jit_ffi_call(cd, self._f_ptr, @@ -194,7 +197,9 @@ def _ffi_fn__args(args): else: affirm(False, u"unknown ffi-fn option: :" + k) - f = FFIFn(lib, rt.name(nm), new_args, ret_type, is_variadic) + tp = CFunctionType(new_args, ret_type, is_variadic) + nm = rt.name(nm) + f = FFIFn(nm, lib.get_fn_ptr(nm), tp) return f @as_var("ffi-voidp") @@ -209,7 +214,7 @@ def _ffi_voidp(lib, nm): -class Buffer(object.Object): +class Buffer(PointerType): """ Defines a byte buffer with non-gc'd (therefore non-movable) contents """ _type = object.Type(u"pixie.stdlib.Buffer") @@ -474,9 +479,10 @@ def ffi_type(self): return clibffi.ffi_type_pointer cvoidp = CVoidP() -class VoidP(object.Object): +class VoidP(PointerType): + _type = cvoidp def type(self): - return cvoidp + return VoidP._type def __init__(self, raw_data): self._raw_data = raw_data @@ -567,7 +573,7 @@ def invoke(self, args): registered_callbacks = {} -class CCallback(object.Object): +class CCallback(PointerType): _type = object.Type(u"pixie.ffi.CCallback") def __init__(self, cft, raw_closure, id, fn): @@ -625,6 +631,17 @@ def ffi_callback(args, ret_type): return CFunctionType(args_w, ret_type) + +class TranslatedIDGenerator(py_object): + def __init__(self): + self._val = 0 + + def get_next(self): + self._val += 1 + return self._val + +id_generator = TranslatedIDGenerator() + @as_var(u"pixie.ffi", u"ffi-prep-callback") def ffi_prep_callback(tp, f): """(ffi-prep-callback callback-tp fn) @@ -634,7 +651,10 @@ def ffi_prep_callback(tp, f): affirm(isinstance(tp, CFunctionType), u"First argument to ffi-prep-callback must be a CFunctionType") raw_closure = rffi.cast(rffi.VOIDP, clibffi.closureHeap.alloc()) - unique_id = rffi.cast(lltype.Signed, raw_closure) + if not we_are_translated(): + unique_id = id_generator.get_next() + else: + unique_id = rffi.cast(lltype.Signed, raw_closure) res = clibffi.c_ffi_prep_closure(rffi.cast(clibffi.FFI_CLOSUREP, raw_closure), tp.get_cd().cif, invoke_callback, @@ -670,14 +690,16 @@ class CFunctionType(object.Type): base_type = object.Type(u"pixie.ffi.CType") _immutable_fields_ = ["_arg_types", "_ret-type", "_cd"] - def __init__(self, arg_types, ret_type): + def __init__(self, arg_types, ret_type, is_variadic=False): object.Type.__init__(self, name_gen.next(), CStructType.base_type) self._arg_types = arg_types self._ret_type = ret_type + self._is_variadic = is_variadic self._cd = CifDescrBuilder(self._arg_types, self._ret_type).rawallocate() def ffi_get_value(self, ptr): - runtime_error(u"Cannot get a callback value via FFI") + casted = rffi.cast(rffi.VOIDPP, ptr) + return FFIFn(u"", casted[0], self) def ffi_set_value(self, ptr, val): if isinstance(val, CCallback): @@ -804,7 +826,18 @@ def prep_ffi_call__args(args): affirm(isinstance(fn, CFunctionType), u"First arg must be a FFI function") +def comp_ptrs(a, b): + print a, b + assert isinstance(a, PointerType) + if not isinstance(b, PointerType): + return false + if a.raw_data() == b.raw_data(): + return true + return false +extend(proto._eq, Buffer)(comp_ptrs) +extend(proto._eq, CStructType.base_type)(comp_ptrs) +extend(proto._eq, VoidP)(comp_ptrs) import sys diff --git a/pixie/vm/object.py b/pixie/vm/object.py index b80259e8..54d1c38f 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -1,6 +1,26 @@ from rpython.rlib.objectmodel import compute_identity_hash import rpython.rlib.jit as jit + +class FinalizerRegistry(object): + def __init__(self): + # TODO: PyPy uses a linked list, investigate if we need that too + self._registry = [] + + def register(self, o): + print "register finalizer ", o + self._registry.append(o) + + def run_finalizers(self): + import pixie.vm.rt as rt + + vals = self._registry + self._registry = [] + for x in vals: + rt._finalize_BANG_(x) + +finalizer_registry = FinalizerRegistry() + class Object(object): """ Base Object for all VM objects """ @@ -29,6 +49,13 @@ def hash(self): def promote(self): return self + def __del__(self): + if self.type().has_finalizer(): + finalizer_registry.register(self) + + + + class TypeRegistry(object): def __init__(self): self._types = {} @@ -66,6 +93,7 @@ def get_type_by_name(nm): return _type_registry.get_by_name(nm) class Type(Object): + _immutable_fields_ = ["_name", "_has_finalizer?"] def __init__(self, name, parent=None, object_inited=True): assert isinstance(name, unicode), u"Type names must be unicode" _type_registry.register_type(name, self) @@ -79,6 +107,7 @@ def __init__(self, name, parent=None, object_inited=True): self._parent = parent self._subclasses = [] + self._has_finalizer = False def name(self): return self._name @@ -95,6 +124,14 @@ def add_subclass(self, tp): def subclasses(self): return self._subclasses + @jit.elidable_promote() + def has_finalizer(self): + return self._has_finalizer + + def set_finalizer(self): + self._has_finalizer = True + + Object._type = Type(u"pixie.stdlib.Object", None, False) Type._type = Type(u"pixie.stdlib.Type") diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 0df93adc..a6ae64cd 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -43,6 +43,9 @@ class PlatformReader(object.Object): _type = object.Type(u"PlatformReader") + def type(self): + return PlatformReader._type + def read(self): assert False @@ -53,6 +56,9 @@ def reset_line(self): return self class StringReader(PlatformReader): + _type = object.Type(u"pixie.stdlib.StringReader") + def type(self): + return StringReader._type def __init__(self, s): affirm(isinstance(s, unicode), u"StringReader requires unicode") diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 52fa547c..a69370d1 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,4 +1,8 @@ # -*- coding: utf-8 -*- +from pixie.vm.object import Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ + runtime_error, add_info, ExtraCodeInfo, finalizer_registry +from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ + list_copy, returns, intern_var from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ runtime_error, add_info, ExtraCodeInfo from pixie.vm.code import Namespace, BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ @@ -59,6 +63,15 @@ defprotocol("pixie.stdlib", "ITransientStack", ["-push!", "-pop!"]) defprotocol("pixie.stdlib", "IDisposable", ["-dispose!"]) +defprotocol("pixie.stdlib", "IFinalize", ["-finalize!"]) + +def maybe_mark_finalizer(self, tp): + if self is _finalize_BANG_: + print "MARKING ", tp + + tp.set_finalizer() + +code.PolymorphicFn.maybe_mark_finalizer = maybe_mark_finalizer @as_var("pixie.stdlib.internal", "-defprotocol") def _defprotocol(name, methods): @@ -913,3 +926,8 @@ def _add_exception_info(ex, str, data): assert isinstance(ex, RuntimeException) ex._trace.append(ExtraCodeInfo(rt.name(str), data)) return ex + +@as_var("-run-finalizers") +def _run_finalizers(): + finalizer_registry.run_finalizers() + return nil From f0be04a78cb6a9a6417318f38e79041ff8c6f161 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 6 Aug 2015 22:07:53 -0600 Subject: [PATCH 777/909] added finalizers latest work on some interop stuff --- Makefile | 8 +++- lib_pixie.py | 11 +++++ pixie/lib_pixie.py | 15 +++++++ pixie/stdlib.pxi | 87 +++++++++++++++++++++---------------- pixie/vm/c_api.py | 31 +++++++++++++ pixie/vm/compiler.py | 45 +++++++++++-------- pixie/vm/custom_types.py | 7 ++- pixie/vm/libs/ffi.py | 14 ++++-- pixie/vm/object.py | 3 -- pixie/vm/persistent_list.py | 3 ++ pixie/vm/rt.py | 2 + pixie/vm/stdlib.py | 8 ++-- setup.py | 17 ++++++++ target.py | 17 +++++--- 14 files changed, 193 insertions(+), 75 deletions(-) create mode 100644 lib_pixie.py create mode 100644 pixie/lib_pixie.py create mode 100644 pixie/vm/c_api.py create mode 100644 setup.py diff --git a/Makefile b/Makefile index 57204a3b..f857a332 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ PYTHON ?= python PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy -COMMON_BUILD_OPTS?=--thread --no-shared --gcrootfinder=shadowstack --continuation +COMMON_BUILD_OPTS?=--thread --gcrootfinder=shadowstack --continuation JIT_OPTS?=--opt=jit TARGET_OPTS?=target.py @@ -28,6 +28,12 @@ build_no_jit: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target.py && \ make compile_basics +build_no_jit_shared: fetch_externals + @if [ ! -d /usr/local/include/boost -a ! -d /usr/include/boost ] ; then echo "Boost C++ Library not found" && false; fi && \ + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --shared target.py && \ + make compile_basics + + compile_basics: @echo -e "\n\n\n\nWARNING: Compiling core libs. If you want to modify one of these files delete the .pxic files first\n\n\n\n" ./pixie-vm -c pixie/uv.pxi -c pixie/io.pxi -c pixie/stacklets.pxi -c pixie/stdlib.pxi -c pixie/repl.pxi diff --git a/lib_pixie.py b/lib_pixie.py new file mode 100644 index 00000000..1a9f2967 --- /dev/null +++ b/lib_pixie.py @@ -0,0 +1,11 @@ +import ctypes, sys + +dll = ctypes.CDLL("libpixie-vm.dylib") + +dll.rpython_startup_code() +dll.pixie_init(sys.argv[0]) + +def repl(): + dll.pixie_execute_source("(ns user (:require [pixie.repl :as repl])) (pixie.repl/repl)") + +repl() \ No newline at end of file diff --git a/pixie/lib_pixie.py b/pixie/lib_pixie.py new file mode 100644 index 00000000..4bf67b59 --- /dev/null +++ b/pixie/lib_pixie.py @@ -0,0 +1,15 @@ +import ctypes +import os.path + +dll_name = os.path.join(os.path.dirname(__file__), "libpixie-vm.dylib") +print("Loading Pixie from " + dll_name) +dll = ctypes.CDLL(dll_name) + +dll.rpython_startup_code() +dll.pixie_init("/Users/tim/oss/pixie/pixie-vm".encode("ascii")) + + +def repl(): + dll.pixie_execute_source("(ns user (:require [pixie.repl :as repl])) (pixie.repl/repl)".encode('ascii')) + +#repl() \ No newline at end of file diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index fd323a65..fe4a85bb 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -841,7 +841,7 @@ there's a value associated with the key. Use `some` for checking for values." (fn [] ~@finally))))))) -(defn . +#_(defn . {:doc "Access the field named by the symbol. If further arguments are passed, invokes the method named by symbol, passing the object and arguments." @@ -1279,9 +1279,7 @@ and implements IAssociative, ILookup and IObject." "dissoc is not supported on defrecords"])) 'ILookup `(-val-at [self k not-found] - (if (contains? ~(set fields) k) - (. self k) - not-found)) + (get-field self k)) 'IObject `(-str [self] (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (. self %2)) {} ~fields) ">")) @@ -2072,7 +2070,48 @@ The params can be destructuring bindings, see `(doc let)` for details."} `(fn* ~@name ~(first (first decls)) ~@(next (first decls))) `(fn* ~@name ~@decls)))) +;; TODO: implement :>> like in Clojure? +(defmacro condp + "Takes a binary predicate, an expression and a number of two-form clauses. +Calls the predicate on the first value of each clause and the expression. +If the result is truthy returns the second value of the clause. + +If the number of arguments is odd and no clause matches, the last argument is returned. +If the number of arguments is even and no clause matches, throws an exception." + [pred-form expr & clauses] + (let [x (gensym 'expr), pred (gensym 'pred)] + `(let [~x ~expr, ~pred ~pred-form] + (cond ~@(mapcat + (fn [[a b :as clause]] + (if (> (count clause) 1) + `((~pred ~a ~x) ~b) + `(:else ~a))) + (partition 2 clauses)) + :else (throw [:pixie.stdlib/MissingClauseException + "No matching clause!"]))))) + +(defmacro case + "Takes an expression and a number of two-form clauses. +Checks for each clause if the first part is equal to the expression. +If yes, returns the value of the second part. + +The first part of each clause can also be a set. If that is the case, the clause matches when the result of the expression is in the set. + +If the number of arguments is odd and no clause matches, the last argument is returned. +If the number of arguments is even and no clause matches, throws an exception." + [expr & args] + `(condp #(if (set? %1) (%1 %2) (= %1 %2)) + ~expr ~@args)) + + + + (deftype MultiMethod [dispatch-fn default-val methods] + IMessageObject + (-get-attr [this kw] + (case kw + :methods methods + :else nil)) IFn (-invoke [self & args] (let [dispatch-val (apply dispatch-fn args) @@ -2082,6 +2121,7 @@ The params can be destructuring bindings, see `(doc let)` for details."} _ (assert method (str "no method defined for " dispatch-val))] (apply method args)))) + (defmacro defmulti {:doc "Define a multimethod, which dispatches to its methods based on dispatch-fn." :examples [["(defmulti greet first)"] @@ -2107,13 +2147,17 @@ The params can be destructuring bindings, see `(doc let)` for details."} :added "0.1"} [name dispatch-val params & body] `(do - (let [methods (.methods ~name)] + (let [methods (.-methods ~name)] (swap! methods assoc ~dispatch-val (fn ~params ~@body)) ~name))) +(defmulti Foo :r) +(defmethod Foo :r + [x] x) + (defmacro declare {:doc "Forward declare the given variable names, setting them to nil." :added "0.1"} @@ -2215,39 +2259,6 @@ Expands to calls to `extend-type`." "Returns a collection that contains all the elements of the argument in reverse order." (into () coll)) -;; TODO: implement :>> like in Clojure? -(defmacro condp - "Takes a binary predicate, an expression and a number of two-form clauses. -Calls the predicate on the first value of each clause and the expression. -If the result is truthy returns the second value of the clause. - -If the number of arguments is odd and no clause matches, the last argument is returned. -If the number of arguments is even and no clause matches, throws an exception." - [pred-form expr & clauses] - (let [x (gensym 'expr), pred (gensym 'pred)] - `(let [~x ~expr, ~pred ~pred-form] - (cond ~@(mapcat - (fn [[a b :as clause]] - (if (> (count clause) 1) - `((~pred ~a ~x) ~b) - `(:else ~a))) - (partition 2 clauses)) - :else (throw [:pixie.stdlib/MissingClauseException - "No matching clause!"]))))) - -(defmacro case - "Takes an expression and a number of two-form clauses. -Checks for each clause if the first part is equal to the expression. -If yes, returns the value of the second part. - -The first part of each clause can also be a set. If that is the case, the clause matches when the result of the expression is in the set. - -If the number of arguments is odd and no clause matches, the last argument is returned. -If the number of arguments is even and no clause matches, throws an exception." - [expr & args] - `(condp #(if (set? %1) (%1 %2) (= %1 %2)) - ~expr ~@args)) - (defmacro use [ns] `(do diff --git a/pixie/vm/c_api.py b/pixie/vm/c_api.py new file mode 100644 index 00000000..38b30e6e --- /dev/null +++ b/pixie/vm/c_api.py @@ -0,0 +1,31 @@ + + +from rpython.rlib.entrypoint import entrypoint, RPython_StartupCode +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.lltypesystem.lloperation import llop + + +@entrypoint('main', [rffi.CCHARP], c_name='pixie_init') +def pypy_execute_source(ll_progname): + from target import init_vm + after = rffi.aroundstate.after + if after: after() + progname = rffi.charp2str(ll_progname) + init_vm(progname) + res = 0 + before = rffi.aroundstate.before + if before: before() + return rffi.cast(rffi.INT, res) + +@entrypoint('main', [rffi.CCHARP], c_name='pixie_execute_source') +def pypy_execute_source(ll_source): + from target import EvalFn, run_with_stacklets + after = rffi.aroundstate.after + if after: after() + source = rffi.charp2str(ll_source) + f = EvalFn(source) + run_with_stacklets.invoke([f]) + res = 0 + before = rffi.aroundstate.before + if before: before() + return rffi.cast(rffi.INT, res) \ No newline at end of file diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 7dbb7477..5d0f214f 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -10,6 +10,8 @@ from pixie.vm.atom import Atom from rpython.rlib.rarithmetic import r_uint, intmask from pixie.vm.persistent_list import EmptyList +from pixie.vm.cons import cons +from pixie.vm.persistent_list import create_from_list import pixie.vm.rt as rt @@ -321,23 +323,6 @@ def compile_set_literal(form, ctx): compile_cons(set_call, ctx) -def macroexpand(form): - sym = rt.first(form) - if isinstance(sym, symbol.Symbol): - s = rt.name(sym) - if s.startswith(".") and s != u".": - if rt.count(form) < 2: - raise Exception("malformed dot expression, expecting (.member obj ...)") - - method = rt.keyword(rt.wrap(rt.name(sym)[1:])) - obj = rt.first(rt.next(form)) - dot = rt.symbol(rt.wrap(u".")) - call = rt.cons(dot, rt.cons(obj, rt.cons(method, rt.next(rt.next(form))))) - - return call - - return form - def compile_meta(meta, ctx): ctx.push_const(code.intern_var(u"pixie.stdlib", u'with-meta')) ctx.bytecode.append(code.DUP_NTH) @@ -351,13 +336,37 @@ def compile_meta(meta, ctx): ctx.bytecode.append(1) ctx.sub_sp(1) +def maybe_oop_invoke(form): + head = rt.first(form) + if isinstance(rt.first(form), symbol.Symbol) and rt.name(head).startswith(".-"): + postfix = rt.next(form) + affirm(rt.count(postfix) == 1, u" Attribute lookups must only have one argument") + subject = rt.first(postfix) + kw = keyword(rt.name(head)[2:]) + fn = symbol.symbol(u"pixie.stdlib/-get-attr") + return create_from_list([fn, subject, kw]) + + elif isinstance(rt.first(form), symbol.Symbol) and rt.name(head).startswith("."): + subject = rt.first(rt.next(form)) + postfix = rt.next(rt.next(form)) + form = cons(keyword(rt.name(head)[1:]), postfix) + form = cons(subject, form) + form = cons(symbol.symbol(u"pixie.stdlib/-call-method"), form) + return form + + else: + return form + + def compile_form(form, ctx): if form is nil: ctx.push_const(nil) return if rt._satisfies_QMARK_(rt.ISeq.deref(), form) and form is not nil: - form = macroexpand(form) + + form = maybe_oop_invoke(form) + return compile_cons(form, ctx) if isinstance(form, numbers.Integer): ctx.push_const(form) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 498bf9fe..68a10d8b 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -1,4 +1,4 @@ -from pixie.vm.object import Object, Type, affirm, runtime_error +from pixie.vm.object import Object, Type, affirm, runtime_error, finalizer_registry import rpython.rlib.jit as jit from pixie.vm.code import as_var from pixie.vm.numbers import Integer, Float @@ -86,6 +86,11 @@ def get_field(self, name): return value + def __del__(self): + if self.type().has_finalizer(): + finalizer_registry.register(self) + + create_type_prefix = """ def new_inst(tp, fields): l = len(fields) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 0fc34ddb..86fdc840 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -688,7 +688,7 @@ def next(self): class CFunctionType(object.Type): base_type = object.Type(u"pixie.ffi.CType") - _immutable_fields_ = ["_arg_types", "_ret-type", "_cd"] + _immutable_fields_ = ["_arg_types", "_ret_type", "_cd", "_is_variadic"] def __init__(self, arg_types, ret_type, is_variadic=False): object.Type.__init__(self, name_gen.next(), CStructType.base_type) @@ -724,7 +724,7 @@ def ffi_size(self): def ffi_type(self): return clibffi.ffi_type_pointer -class CStruct(object.Object): +class CStruct(PointerType): _immutable_fields_ = ["_type", "_buffer"] def __init__(self, tp, buffer): self._type = tp @@ -827,7 +827,6 @@ def prep_ffi_call__args(args): def comp_ptrs(a, b): - print a, b assert isinstance(a, PointerType) if not isinstance(b, PointerType): return false @@ -835,10 +834,19 @@ def comp_ptrs(a, b): return true return false +def hash_ptr(a): + assert isinstance(a, PointerType) + hashval = rffi.cast(lltype.Signed, a.raw_data()) + return rt.wrap(hashval) + extend(proto._eq, Buffer)(comp_ptrs) extend(proto._eq, CStructType.base_type)(comp_ptrs) extend(proto._eq, VoidP)(comp_ptrs) +extend(proto._hash, Buffer)(hash_ptr) +extend(proto._hash, CStructType.base_type)(hash_ptr) +extend(proto._hash, VoidP)(hash_ptr) + import sys diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 54d1c38f..ccddf276 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -49,9 +49,6 @@ def hash(self): def promote(self): return self - def __del__(self): - if self.type().has_finalizer(): - finalizer_registry.register(self) diff --git a/pixie/vm/persistent_list.py b/pixie/vm/persistent_list.py index 91f76da8..54d6b11a 100644 --- a/pixie/vm/persistent_list.py +++ b/pixie/vm/persistent_list.py @@ -82,6 +82,9 @@ def create_from_list(lst): i -= 1 return acc +def create(*args): + return create_from_list(args) + @extend(proto._meta, PersistentList) def _meta(self): assert isinstance(self, PersistentList) diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 37fa0676..c17389cb 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -73,6 +73,8 @@ def wrapper(*args): import pixie.vm.string_builder import pixie.vm.stacklet + import pixie.vm.c_api + @specialize.argtype(0) def wrap(x): if isinstance(x, bool): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index a69370d1..a9b9aea9 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,10 +1,6 @@ # -*- coding: utf-8 -*- -from pixie.vm.object import Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ - runtime_error, add_info, ExtraCodeInfo, finalizer_registry -from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ - list_copy, returns, intern_var from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ - runtime_error, add_info, ExtraCodeInfo + runtime_error, add_info, ExtraCodeInfo, finalizer_registry from pixie.vm.code import Namespace, BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ list_copy, returns, intern_var, _ns_registry import pixie.vm.code as code @@ -65,6 +61,8 @@ defprotocol("pixie.stdlib", "IDisposable", ["-dispose!"]) defprotocol("pixie.stdlib", "IFinalize", ["-finalize!"]) +defprotocol("pixie.stdlib", "IMessageObject", ["-call-method", "-get-attr"]) + def maybe_mark_finalizer(self, tp): if self is _finalize_BANG_: print "MARKING ", tp diff --git a/setup.py b/setup.py new file mode 100644 index 00000000..6046e6f4 --- /dev/null +++ b/setup.py @@ -0,0 +1,17 @@ +from setuptools import setup, find_packages +from codecs import open +from os import path + +here = path.abspath(path.dirname(__file__)) + +setup( + name="pixie", + version="0.2.0", + description="A fast lightweight lisp with 'magical' powers", + + license="LGPL", + + keywords="development language", + + package_data={"pixie": ["libpixie-vm.dylib", "pixie-vm"]} +) \ No newline at end of file diff --git a/target.py b/target.py index a4126317..c5c3fed8 100644 --- a/target.py +++ b/target.py @@ -147,19 +147,23 @@ def load_stdlib(): from pixie.vm.code import intern_var run_with_stacklets = intern_var(u"pixie.stacklets", u"run-with-stacklets") +def init_vm(progname): + import pixie.vm.stacklet + pixie.vm.stacklet.init() + + init_load_path(progname) + load_stdlib() + add_to_load_paths(".") + def entry_point(args): try: - import pixie.vm.stacklet - pixie.vm.stacklet.init() + + init_vm(args[0]) interactive = True exit = False script_args = [] - init_load_path(args[0]) - load_stdlib() - add_to_load_paths(".") - i = 1 while i < len(args): arg = args[i] @@ -227,6 +231,7 @@ def entry_point(args): def add_to_load_paths(path): rt.reset_BANG_(LOAD_PATHS.deref(), rt.conj(rt.deref(LOAD_PATHS.deref()), rt.wrap(path))) + def init_load_path(self_path): if not path.isfile(self_path): self_path = find_in_path(self_path) From 5865ccefc91df08dceb325a781e42fd7b4d708d3 Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Tue, 18 Aug 2015 22:01:40 -0600 Subject: [PATCH 778/909] remove old file --- setup.py | 17 ----------------- 1 file changed, 17 deletions(-) delete mode 100644 setup.py diff --git a/setup.py b/setup.py deleted file mode 100644 index 6046e6f4..00000000 --- a/setup.py +++ /dev/null @@ -1,17 +0,0 @@ -from setuptools import setup, find_packages -from codecs import open -from os import path - -here = path.abspath(path.dirname(__file__)) - -setup( - name="pixie", - version="0.2.0", - description="A fast lightweight lisp with 'magical' powers", - - license="LGPL", - - keywords="development language", - - package_data={"pixie": ["libpixie-vm.dylib", "pixie-vm"]} -) \ No newline at end of file From 3d25e8d25ce656b392d8df05279a2b44ef6cc96a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 19 Aug 2015 06:38:17 -0600 Subject: [PATCH 779/909] fixup tests and deftype to use the new OOP model --- pixie/stdlib.pxi | 51 +++++++++++++++++++--------- tests/pixie/tests/test-defrecord.pxi | 3 +- tests/pixie/tests/test-deftype.pxi | 31 +++++++---------- tests/pixie/tests/test-io.pxi | 2 +- 4 files changed, 51 insertions(+), 36 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index fe4a85bb..26bb1b6a 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1183,14 +1183,9 @@ Creates new maps if the keys are not present." (defmacro deftype {:doc "Define a custom type." :examples [["(deftype Person [name] - Object - (say-hi [self other-name] - (str \"Hi, I'm \" name \". You're \" other-name \", right?\")) - IObject (-str [self] (str \"\")))"] - ["(.say-hi (->Person \"James\") \"Paul\")" nil "Hi, I'm James. You're Paul, right?"] ["(str (->Person \"James\"))" nil ""]] :added "0.1"} [nm fields & body] @@ -1256,40 +1251,64 @@ Creates new maps if the keys are not present." ~ctor ~@proto-bodies))) +(defn -make-record-assoc-body [cname fields] + (let [k-sym (gensym "k") + v-sym (gensym "v") + this-sym (gensym "this") + result `(-assoc [~this-sym ~k-sym ~v-sym] + (case ~k-sym + ~@(mapcat + (fn [k] + [k `(~cname ~@(mapv (fn [x] + (if (= x k) + v-sym + `(get-field ~this-sym ~x))) + fields))]) + fields) + (throw [:pixie.stdlib/NotImplementedException + (str "Can't assoc to a unknown field: " ~k-sym)])))] + result)) + (defmacro defrecord {:doc "Define a record type. Similar to `deftype`, but supports construction from a map using `map->Type` and implements IAssociative, ILookup and IObject." :added "0.1"} - [nm fields & body] + [nm field-syms & body] (let [ctor-name (symbol (str "->" (name nm))) map-ctor-name (symbol (str "map" (name ctor-name))) - fields (transduce (map (comp keyword name)) conj fields) + fields (transduce (map (comp keyword name)) conj field-syms) type-from-map `(defn ~map-ctor-name [m] (apply ~ctor-name (map #(get m %) ~fields))) default-bodies ['IAssociative - `(-assoc [self k v] - (let [m (reduce #(assoc %1 %2 (. self %2)) {} ~fields)] - (~map-ctor-name (assoc m k v)))) + (-make-record-assoc-body ctor-name fields) + `(-contains-key [self k] (contains? ~(set fields) k)) `(-dissoc [self k] (throw [:pixie.stdlib/NotImplementedException "dissoc is not supported on defrecords"])) 'ILookup - `(-val-at [self k not-found] - (get-field self k)) + (let [self-nm (gensym "self") + k-nm (gensym "k")] + `(-val-at [~self-nm ~k-nm not-found#] + (case ~k-nm + ~@(mapcat + (fn [k] + [k `(get-field ~self-nm ~k-nm)]) + fields) + not-found#))) 'IObject - `(-str [self] - (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (. self %2)) {} ~fields) ">")) + `(-str [self#] + (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (%2 self#)) {} ~fields) ">")) `(-eq [self other] (and (instance? ~nm other) ~@(map (fn [field] - `(= (. self ~field) (. other ~field))) + `(= (~field self) (~field other))) fields))) `(-hash [self] - (throw [:pixie.stdlib/NotImplementedException "not implemented"]))] + (hash [~@field-syms]))] deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] `(do ~type-from-map ~deftype-decl))) diff --git a/tests/pixie/tests/test-defrecord.pxi b/tests/pixie/tests/test-defrecord.pxi index 382247d1..bc63724a 100644 --- a/tests/pixie/tests/test-defrecord.pxi +++ b/tests/pixie/tests/test-defrecord.pxi @@ -38,7 +38,8 @@ (t/assert (satisfies? IAssociative t)) (t/assert= t (assoc t4 :three 3)) (let [t' (assoc t :one 42) - t-oops (assoc t :oops 'never-found)] + t-oops (try (assoc t :oops 'never-found) + (catch ex t))] (t/assert (not (= t t'))) (t/assert= (get t' :one) 42) (t/assert= t t-oops) diff --git a/tests/pixie/tests/test-deftype.pxi b/tests/pixie/tests/test-deftype.pxi index 49f5e444..f6123aa0 100644 --- a/tests/pixie/tests/test-deftype.pxi +++ b/tests/pixie/tests/test-deftype.pxi @@ -10,8 +10,7 @@ (foreach [obj-and-val [[o1 1] [o2 2]]] (let [o (first obj-and-val) v (second obj-and-val)] - (t/assert= (. o :val) v) - (t/assert= (.val o) v))))) + (t/assert= (get-field o :val) v))))) (deftype MagicalVectorMap [] IMap IVector) @@ -34,14 +33,17 @@ (foreach [obj-and-val [[o1 1] [o2 2]]] (let [o (first obj-and-val) v (second obj-and-val)] - (t/assert= (. o :val) v) - (t/assert= (.val o) v) + (t/assert= (get-field o :val) v) (t/assert (satisfies? ICounted o)) (t/assert= (-count o) v) (t/assert= (count o) v))))) +(defprotocol TestObject + (add [self x & args]) + (one-plus [self x & xs])) + (deftype Three [:one :two :three] - Object + TestObject (add [self x & args] (apply + x args)) (one-plus [self x & xs] @@ -50,7 +52,7 @@ (-count [self] (+ one two three))) (deftype Three2 [one two three] - Object + TestObject (add [self x & args] (apply + x args)) (one-plus [self x & xs] @@ -66,20 +68,13 @@ one (second obj-and-vals) two (third obj-and-vals) three (fourth obj-and-vals)] - (t/assert= (. o :one) one) - (t/assert= (.one o) one) - (t/assert= (. o :two) two) - (t/assert= (.two o) two) - (t/assert= (. o :three) three) - (t/assert= (.three o) three) + (t/assert= (get-field o :one) one) + (t/assert= (get-field o :two) two) + (t/assert= (get-field o :three) three) (t/assert (satisfies? ICounted o)) (t/assert= (-count o) (+ one two three)) (t/assert= (count o) (+ one two three)) - (t/assert= (.add o 21 21) 42) - (t/assert= (.one-plus o 9) (+ one 9)) - - ; arity-1 (just the self arg) not supported for now - (t/assert= (.add o) (. o :add)) - (t/assert= (.one-plus o) (. o :one-plus)))))) + (t/assert= (add o 21 21) 42) + (t/assert= (one-plus o 9) (+ one 9)))))) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 532d9890..523d124e 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -148,7 +148,7 @@ (io/run-command "rm compressed-output.gz") (compress-content (io/open-write "compressed-output.gz") (range 1000)) ;; decompress the file with zcat - (io/run-command "zcat compressed-output.gz > compressed-output.txt") + (io/run-command "cat compressed-coutput | zcat > compressed-output.txt") (t/assert= (range 1000) (read-string (io/slurp "compressed-output.txt"))) ;; Wrapping an IInputStream in decompressing-stream should get the same result From b7036d1dc56386f78b9a36e2b723d0d137e636cd Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 19 Aug 2015 06:42:53 -0600 Subject: [PATCH 780/909] fixed compat issue with OSX and gzip tests --- tests/pixie/tests/test-io.pxi | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 523d124e..26cf8e70 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -148,7 +148,6 @@ (io/run-command "rm compressed-output.gz") (compress-content (io/open-write "compressed-output.gz") (range 1000)) ;; decompress the file with zcat - (io/run-command "cat compressed-coutput | zcat > compressed-output.txt") (t/assert= (range 1000) (read-string (io/slurp "compressed-output.txt"))) ;; Wrapping an IInputStream in decompressing-stream should get the same result From 884d98397df4d58d230cc79c0226be9a50681b7a Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Wed, 19 Aug 2015 16:03:32 -0600 Subject: [PATCH 781/909] use gunzip --- tests/pixie/tests/test-io.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 26cf8e70..663bf52b 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -148,6 +148,7 @@ (io/run-command "rm compressed-output.gz") (compress-content (io/open-write "compressed-output.gz") (range 1000)) ;; decompress the file with zcat + (io/run-command "gunzip compressed-output.gz -c > compressed-output.txt") (t/assert= (range 1000) (read-string (io/slurp "compressed-output.txt"))) ;; Wrapping an IInputStream in decompressing-stream should get the same result From b408e7e4868f3bfb839297cc45a5d3eeb997b2fa Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 20 Aug 2015 15:31:33 +0100 Subject: [PATCH 782/909] simplifies -repr and -str code --- pixie/stdlib.pxi | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 26bb1b6a..fd105624 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -343,17 +343,17 @@ (extend -str PersistentVector (fn [v] - (apply str "[" (conj (transduce (interpose " ") conj v) "]")))) + (str "[" (transduce (interpose " ") string-builder v) "]"))) (extend -repr PersistentVector (fn [v] - (apply str "[" (conj (transduce (comp (map -repr) (interpose " ")) conj v) "]")))) + (str "[" (transduce (comp (map -repr) (interpose " ")) string-builder v) "]"))) (extend -str Cons (fn [v] - (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) + (str "(" (transduce (interpose " ") string-builder v) ")"))) (extend -repr Cons (fn [v] - (apply str "(" (conj (transduce (comp (map -repr) (interpose " ")) conj v) ")")))) + (str "(" (transduce (comp (map -repr) (interpose " ")) string-builder v) ")"))) (extend -hash Cons (fn [v] @@ -361,10 +361,10 @@ (extend -str PersistentList (fn [v] - (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) + (str "(" (transduce (interpose " ") string-builder v) ")"))) (extend -repr PersistentList (fn [v] - (apply str "(" (conj (transduce (comp (map -repr) (interpose " ")) conj v) ")")))) + (str "(" (transduce (comp (map -repr) (interpose " ")) string-builder v) ")"))) (extend -hash PersistentList (fn [v] @@ -373,10 +373,10 @@ (extend -str LazySeq (fn [v] - (apply str "(" (conj (transduce (interpose " ") conj v) ")")))) + (str "(" (transduce (interpose " ") string-builder v) ")"))) (extend -repr LazySeq (fn [v] - (apply str "(" (conj (transduce (comp (map -repr) (interpose " ")) conj v) ")")))) + (str "(" (transduce (comp (map -repr) (interpose " ")) string-builder v) ")"))) (extend -hash PersistentVector (fn [v] @@ -945,10 +945,10 @@ If further arguments are passed, invokes the method named by symbol, passing the (extend -str MapEntry (fn [v] - (apply str "[" (conj (transduce (interpose " ") conj v) "]")))) + (str "[" (transduce (interpose " ") string-builder v) "]"))) (extend -repr MapEntry (fn [v] - (apply str "[" (conj (transduce (comp (map -repr) (interpose " ")) conj v) "]")))) + (str "[" (transduce (comp (map -repr) (interpose " ")) string-builder v) "]"))) (extend -hash MapEntry (fn [v] @@ -989,11 +989,11 @@ If further arguments are passed, invokes the method named by symbol, passing the (extend -str PersistentHashMap (fn [v] (let [entry->str (map (fn [e] (vector (key e) " " (val e))))] - (apply str "{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) + (str "{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) (extend -repr PersistentHashMap (fn [v] (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] - (apply str "{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) + (str "{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) (extend -hash PersistentHashMap (fn [v] @@ -1005,10 +1005,10 @@ If further arguments are passed, invokes the method named by symbol, passing the (extend -str PersistentHashSet (fn [s] - (apply str "#{" (conj (transduce (interpose " ") conj s) "}")))) + (str "#{" (transduce (interpose " ") string-builder s) "}"))) (extend -repr PersistentHashSet (fn [s] - (apply str "#{" (conj (transduce (comp (map -repr) (interpose " ")) conj s) "}")))) + (str "#{" (transduce (comp (map -repr) (interpose " ")) string-builder s) "}"))) (extend -empty Cons (fn [_] '())) (extend -empty LazySeq (fn [_] '())) @@ -2408,12 +2408,12 @@ Calling this function on something that is not ISeqable returns a seq with that (extend -str Environment (fn [v] (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] - (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) + (str "#Environment{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) (extend -repr Environment (fn [v] (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] - (apply str "#Environment{" (conj (transduce (comp entry->str (interpose [", "]) cat) conj v) "}"))))) + (str "#Environment{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) (defn interleave "Returns a seq of all the items in the input collections interleaved" From d4954723a85babf8aecde90276257095a3170162 Mon Sep 17 00:00:00 2001 From: Joshua Greenberg Date: Thu, 20 Aug 2015 20:06:57 -0700 Subject: [PATCH 783/909] add back jit.elidable_promote in the CustomType class --- pixie/vm/custom_types.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 68a10d8b..0cbbc4ef 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -16,7 +16,7 @@ def __init__(self, name, slots): self._mutable_slots = {} self._rev = 0 - @jit.elidable + @jit.elidable_promote() def get_slot_idx(self, nm): return self._slots.get(nm, -1) @@ -26,14 +26,14 @@ def set_mutable(self, nm): self._mutable_slots[nm] = nm - @jit.elidable + @jit.elidable_promote() def _is_mutable(self, nm, rev): return nm in self._mutable_slots def is_mutable(self, nm): return self._is_mutable(nm, self._rev) - @jit.elidable + @jit.elidable_promote() def get_num_slots(self): return len(self._slots) From af63494ddc48612f51fbdbe743fc91437607fbe7 Mon Sep 17 00:00:00 2001 From: Joshua Greenberg Date: Thu, 20 Aug 2015 21:26:36 -0700 Subject: [PATCH 784/909] improve "repeat": faster reducing, yet stays lazy even for finite sizes --- pixie/stdlib.pxi | 52 +++++++++++++++++++++++++++---- tests/pixie/tests/test-stdlib.pxi | 21 +++++++++++++ 2 files changed, 67 insertions(+), 6 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 26bb1b6a..398a6144 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1347,12 +1347,6 @@ and implements IAssociative, ILookup and IObject." (puts (apply pr-str args)) nil) -(defn repeat - ([x] - (cons x (lazy-seq* (fn [] (repeat x))))) - ([n x] - (take n (repeat x)))) - (defn repeatedly {:doc "Returns a lazy seq that contains the return values of repeated calls to f. @@ -1814,6 +1808,52 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (* -1 x) x)) +(deftype Repeat [n x] + IReduce + (-reduce [self f init] + (loop [i 0 + acc init] + (if (< i n) + (let [acc (f acc x)] + (if (reduced? acc) + @acc + (recur (inc i) acc))) + acc))) + ICounted + (-count [self] + n) + IIndexed + (-nth [self idx] + (if (and (>= idx 0) (< idx n)) + x + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"]))) + (-nth-not-found [self idx not-found] + (if (and (>= idx 0) (< idx n)) + x + not-found)) + ISeqable + (-seq [self] + (when (>= n 1) + (cons x (lazy-seq* (fn [] (->Repeat (dec n) x))))))) + +(extend -str Repeat + (fn [v] + (-str (seq v)))) + +(extend -repr Repeat -str) + +(defn repeat + {:doc "Returns a seqable of repetitions of a value." + :examples [["(repeat 3 :buffalo)" nil (:buffalo :buffalo :buffalo)] + ["(map vector '(1 2 3) (repeat :ahahah))" nil ([1 :ahahah] [2 :ahahah] [3 :ahahah])]] + :signatures [[x] [n x]] + :added "0.1"} + ([x] + (let [positive-infinity (/ 1.0 0)] + (repeat positive-infinity x))) + ([n x] + (->Repeat n x))) + (deftype Range [start stop step] IReduce (-reduce [self f init] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index fa6d3f16..9dc61e20 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -10,6 +10,7 @@ (t/assert= (map inc [1 2 3]) [2 3 4]) (t/assert= (map + [1 2 3] [4 5 6]) [5 7 9]) (t/assert= (map + [1 2 3] [4 5 6] [7 8 9]) [12 15 18]) + (t/assert= (map + [1 2 3] (repeat 7)) [8 9 10]) (let [value (map identity [1 2 3])] (t/assert= (seq value) [1 2 3]) (t/assert= (seq value) [1 2 3]))) @@ -37,6 +38,7 @@ (t/assert= (str (type 3)) "") (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]") (t/assert= (seq (map identity "iterable")) '(\i \t \e \r \a \b \l \e)) + (t/assert= (str (repeat 3 7)) "(7 7 7)") (t/assert= (str (range 3)) "(0 1 2)")) (t/deftest test-repr @@ -60,6 +62,7 @@ (t/assert= (-repr (type 3)) "pixie.stdlib.Integer") (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]") + (t/assert= (-repr (repeat 3 7)) "(7 7 7)") (t/assert= (-repr (range 3)) "(0 1 2)")) (t/deftest test-nth @@ -67,6 +70,7 @@ (t/assert= (nth [1 2 3] 1) 2) (t/assert= (nth '(1 2 3) 1) 2) (t/assert= (nth (make-array 3) 2) nil) + (t/assert= (nth (repeat 4 1) 3) 1) (t/assert= (nth (range 4) 1) 1) (t/assert= (nth "hithere" 1) \i) @@ -92,6 +96,9 @@ (try (nth '() 0) (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (repeat 99 nil) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) (try (nth (range 0 0) 0) (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) @@ -100,12 +107,14 @@ (t/assert= (nth [1 2 3] 99 :default) :default) (t/assert= (nth '(1 2 3) 99 :default) :default) (t/assert= (nth (make-array 3) 99 :default) :default) + (t/assert= (nth (repeat 4 1) 99 :default) :default) (t/assert= (nth (range 4) 99 :default) :default) (t/assert= (nth "hithere" 99 :default) :default) (t/assert= (nth [1 2 3] 1 :default) 2) (t/assert= (nth '(1 2 3) 1 :default) 2) (t/assert= (nth (make-array 3) 2 :default) nil) + (t/assert= (nth (repeat 4 1) 3 :default) 1) (t/assert= (nth (range 4) 1 :default) 1) (t/assert= (nth "hithere" 1 :deafult) \i)) @@ -138,6 +147,7 @@ r (range 1 6)] (t/assert= (last nil) nil) (t/assert= (last []) nil) + (t/assert= (last (repeat 3 nil)) nil) (t/assert= (last (range 0 0)) nil) (t/assert= (last v) 5) (t/assert= (last l) 5) @@ -161,6 +171,7 @@ (t/assert= (empty? (make-array 0)) true) (t/assert= (empty? {}) true) (t/assert= (empty? #{}) true) + (t/assert= (empty? (repeat '())) false) (t/assert= (empty? (range 1 5)) false) (t/assert= (empty? [1 2 3]) false) @@ -177,6 +188,7 @@ (t/assert= (not-empty? (make-array 0)) false) (t/assert= (not-empty? {}) false) (t/assert= (not-empty? #{}) false) + (t/assert= (not-empty? (repeat 0 'x)) false) (t/assert= (not-empty? (range 1 5)) true) (t/assert= (not-empty? [1 2 3]) true) @@ -399,6 +411,12 @@ (t/assert= (f ::catch-this) :found) (t/assert= (f :something-else) :not-found))) +(t/deftest test-repeat + (t/assert= (seq (repeat 3 1)) '(1 1 1)) + (t/assert= (seq (repeat 0 1)) nil) + (t/assert= (count (repeat 4096 'x)) 4096) + (t/assert= (count (repeat 0 'x)) 0)) + (t/deftest test-range (t/assert= (= (-seq (range 10)) '(0 1 2 3 4 5 6 7 8 9)) @@ -573,6 +591,9 @@ (t/assert= (satisfies? IFoo \a) false)) (t/deftest test-reduce + (t/assert= 300 (reduce + (repeat 100 3))) + (t/assert= 0 (reduce + (repeat 0 3))) + (t/assert= [9 9 9] (reduce (partial map +) (repeat 0) (repeat 3 (repeat 3 3)))) (t/assert= 5050 (reduce + (range 101))) (t/assert= 3628800 (reduce * (range 1 11))) (t/assert= 5051 (reduce + 1 (range 101)))) From ab082c93f2bc3385519c38f19992998300a86edb Mon Sep 17 00:00:00 2001 From: Joshua Greenberg Date: Fri, 28 Aug 2015 11:42:22 -0700 Subject: [PATCH 785/909] fixed (count (lazy-seq [])) incorrectly returning 1 --- pixie/vm/stdlib.py | 7 +++++-- tests/pixie/tests/test-stdlib.pxi | 22 ++++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index a9b9aea9..487080c6 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -258,9 +258,12 @@ def count(x): while True: _count_driver.jit_merge_point(tp=rt.type(x)) if ICounted.satisfies(rt.type(x)): - return rt._add(rt.wrap(acc), rt._count(x)) + return rt._add(rt.wrap(acc), rt._count(x)) + seq = rt.seq(x) + if seq is nil: + return rt.wrap(acc) acc += 1 - x = rt.next(rt.seq(x)) + x = rt._next(seq) @as_var("+") diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index fa6d3f16..9462bada 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -218,6 +218,28 @@ (t/assert= (empty {:a 1, :b 2, :c 3}) {}) (t/assert= (empty #{1 2 3}) #{})) +(t/deftest test-count + (t/assert= (count nil) 0) + (t/assert= (count '()) 0) + (t/assert= (count '(1 2 '(3 4))) 3) + (t/assert= (count '(nil nil nil)) 3) + (t/assert= (count (cons 1 [2 3])) 3) + (t/assert= (count (list)) 0) + (t/assert= (count (list 1 2 3)) 3) + (t/assert= (count []) 0) + (t/assert= (count [1 2 3]) 3) + (t/assert= (count {}) 0) + (t/assert= (count {:a 1, :b 2, :c 3}) 3) + (t/assert= (count (first (seq {:a 1, :b 2, :c 3}))) 2) + (t/assert= (count #{}) 0) + (t/assert= (count (conj #{1 2 3} 3)) 3) + (t/assert= (count (lazy-seq '())) 0) + (t/assert= (count (lazy-seq '(1 2 3))) 3) + (t/assert= (count (cons 1 (lazy-seq [2 3]))) 3) + (t/assert= (count (range 0 -9 -3)) 3) + (t/assert= (count "") 0) + (t/assert= (count "123") 3) + (t/assert= (count (make-array 3)) 3)) (t/deftest test-vec (let [v '(1 2 3 4 5)] From 3e589e510fc71d1ee11553529b9540287e5f407a Mon Sep 17 00:00:00 2001 From: The Gitter Badger Date: Tue, 1 Sep 2015 09:20:46 +0000 Subject: [PATCH 786/909] Added Gitter badge --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index eda32826..d66d94d1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,8 @@ [![Build Status](https://travis-ci.org/pixie-lang/pixie.svg?branch=master)](https://travis-ci.org/pixie-lang/pixie)[![License: LGPL] (http://img.shields.io/badge/license-LGPL-green.svg)](http://img.shields.io/badge/license-LGPL-green.svg) # Pixie +[![Join the chat at https://gitter.im/pixie-lang/pixie](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pixie-lang/pixie?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + ## Intro Pixie is a lightweight lisp suitable for both general use as well as shell scripting. The language is still in a "pre-alpha" phase and as such changes fairly quickly. From c5f3c56f357608fab3212b1279e35cc63e8eb3eb Mon Sep 17 00:00:00 2001 From: Timothy Baldridge Date: Thu, 3 Sep 2015 16:52:29 -0600 Subject: [PATCH 787/909] improvements to jit and batch mode --- pixie/stdlib.pxi | 62 ++++++++++++++++++++++++++++------------------ pixie/vm/code.py | 29 ++++++++++++++-------- pixie/vm/stdlib.py | 11 ++++++++ target.py | 8 ++++-- 4 files changed, 73 insertions(+), 37 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 398a6144..40e7a190 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -21,7 +21,6 @@ (def libm (ffi-library (str "libm." pixie.platform/so-ext))) (def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) -(def floor (ffi-fn libm "floor" [CDouble] CDouble)) (def lround (ffi-fn libm "lround" [CDouble] CInt)) @@ -872,29 +871,6 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn indexed? [v] (satisfies? IIndexed v)) (defn counted? [v] (satisfies? ICounted v)) -(defn float - {:doc "Converts a number to a float." - :since "0.1"} - [x] - (cond - (number? x) (+ x 0.0) - :else (throw - [:pixie.stdlib/ConversionException - (str "Can't convert a value of type " (type x) " to a Float")]))) - -(defn int - {:doc "Converts a number to an integer." - :since "0.1"} - [x] - (cond - (integer? x) x - (float? x) (lround (floor x)) - (ratio? x) (int (/ (float (numerator x)) (float (denominator x)))) - (char? x) (+ x 0) - :else (throw - [:pixie.stdlib/ConversionException - (throw (str "Can't convert a value of type " (type x) " to an Integer"))]))) - (defn last {:doc "Returns the last element of the collection, or nil if none." :signatures [[coll]] @@ -2286,6 +2262,44 @@ Expands to calls to `extend-type`." tps)] `(do ~@exts))) +(defprotocol IToFloat + (-float [this])) + +(defn float + {:doc "Converts a number to a float." + :since "0.1"} + [x] + (-float x)) + +(extend-type Number + IToFloat + (-float [x] (+ x 0.0))) + +(defprotocol IToInteger + (-int [x])) + +(extend-protocol IToInteger + Integer + (-int [x] x) + + Float + (-int [x] (floor x)) + + Ratio + (-int [x] + (int (/ (float (numerator x)) (float (denominator x))))) + + Character + (-int [x] + (+ x 0))) + +(defn int + {:doc "Converts a number to an integer." + :since "0.1"} + [x] + (-int x)) + + (defmacro for {:doc "A list comprehension for the bindings." :examples [["(for [x [1 2 3]] x)" nil [1 2 3]] diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 573273c1..920c3dd9 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -91,7 +91,7 @@ def slice_from_start(from_list, count, extra=r_uint(0)): class BaseCode(object.Object): - _immutable_fields_ = ["_meta"] + _immutable_fields_ = ["_meta", "_name"] def __init__(self): assert isinstance(self, BaseCode) self._name = u"unknown" @@ -215,7 +215,7 @@ def invoke_with(self, args, this_fn): class Code(BaseCode): """Interpreted code block. Contains consts and """ _type = object.Type(u"pixie.stdlib.Code") - _immutable_fields_ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta"] + _immutable_fields_ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta", "_debug_points"] def type(self): return Code._type @@ -411,22 +411,22 @@ def set_var_value(self, var, val): class Var(BaseCode): _type = object.Type(u"pixie.stdlib.Var") - _immutable_fields_ = ["_rev?"] + _immutable_fields_ = ["_ns_ref"] def type(self): return Var._type - def __init__(self, ns, name): + def __init__(self, ns_ref, ns, name): BaseCode.__init__(self) + self._ns_ref = ns_ref self._ns = ns self._name = name - self._rev = 0 self._root = undefined self._dynamic = False def set_root(self, o): affirm(o is not None, u"Invalid var set") - self._rev += 1 + self._ns_ref._rev += 1 self._root = o return self @@ -437,7 +437,7 @@ def set_value(self, val): def set_dynamic(self): self._dynamic = True - self._rev += 1 + self._ns_ref._rev += 1 def get_dynamic_value(self): @@ -450,12 +450,16 @@ def _is_dynamic(self, rev): return self._dynamic def is_dynamic(self): - return self._is_dynamic(self._rev) + return self._is_dynamic(self.ns_ref()._rev) @elidable_promote() def get_root(self, rev): return self._root + @elidable_promote() + def ns_ref(self): + return self._ns_ref + def deref(self): if self.is_dynamic(): if we_are_translated(): @@ -465,9 +469,9 @@ def deref(self): if globals().has_key("_dynamic_vars"): return self.get_dynamic_value() else: - return self.get_root(self._rev) + return self.get_root(self.ns_ref()._rev) else: - val = self.get_root(self._rev) + val = self.get_root(self.ns_ref()._rev) affirm(val is not undefined, u"Var " + self._name + u" is undefined") return val @@ -503,10 +507,13 @@ def __init__(self, ns, refer_syms=[], refer_all=False): class Namespace(object.Object): _type = object.Type(u"pixie.stdlib.Namespace") + _immutable_fields_ = ["_rev?"] + def type(self): return Namespace._type def __init__(self, name): + self._rev = 0 self._registry = {} self._name = name self._refers = {} @@ -516,7 +523,7 @@ def intern_or_make(self, name): affirm(isinstance(name, unicode), u"Var names must be unicode") v = self._registry.get(name, None) if v is None: - v = Var(self._name, name) + v = Var(self, self._name, name) self._registry[name] = v return v diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 487080c6..45287c85 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -427,6 +427,10 @@ def _load_file(filename, compile=False): affirm(isinstance(filename, String), u"filename must be a string") filename = str(rt.name(filename)) + if filename.endswith(".pxic"): + load_pxic_file(filename) + return nil + if path.isfile(filename + "c") and not compile: load_pxic_file(filename + "c") return nil @@ -932,3 +936,10 @@ def _add_exception_info(ex, str, data): def _run_finalizers(): finalizer_registry.run_finalizers() return nil + + +@as_var("floor") +def _floor(x): + affirm(isinstance(x, numbers.Float), u"floor expects a Float") + return numbers.Integer(int(x.float_val())) + diff --git a/target.py b/target.py index c5c3fed8..7eb8932c 100644 --- a/target.py +++ b/target.py @@ -39,7 +39,7 @@ def jitpolicy(driver): LOAD_PATHS = intern_var(u"pixie.stdlib", u"load-paths") LOAD_PATHS.set_root(nil) -load_path = Var(u"pixie.stdlib", u"internal-load-path") +load_path = intern_var(u"pixie.stdlib", u"internal-load-path") class ReplFn(NativeFn): def __init__(self, args): @@ -54,6 +54,8 @@ def inner_invoke(self, args): with with_ns(u"user"): repl.invoke([]) +load_file = intern_var(u"pixie.stdlib", u"load-file") + class BatchModeFn(NativeFn): def __init__(self, args): self._file = args[0] @@ -81,7 +83,9 @@ def inner_invoke(self, args): if not path.isfile(self._file): print "Error: Cannot open '" + self._file + "'" os._exit(1) - f = open(self._file) + load_file.invoke([rt.wrap(self._file)]) + return None + data = f.read() f.close() From ee883e52b0af421170272274a4cc141841883ec6 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 6 Sep 2015 13:19:00 -0400 Subject: [PATCH 788/909] reset! returns new value --- pixie/vm/atom.py | 2 +- tests/pixie/tests/test-stdlib.pxi | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/pixie/vm/atom.py b/pixie/vm/atom.py index d372e920..ae42ab2d 100644 --- a/pixie/vm/atom.py +++ b/pixie/vm/atom.py @@ -18,7 +18,7 @@ def __init__(self, boxed_value): def _reset(self, v): assert isinstance(self, Atom) self._boxed_value = v - return self + return v @extend(proto._deref, Atom) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index b8147548..61896a88 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -623,3 +623,9 @@ (t/deftest test-comp (t/assert= 5 ((comp inc inc inc inc) 1)) (t/assert= :xyz ((comp) :xyz))) + +(t/deftest test-swap-reset + (let [a (atom 0)] + (t/assert= 1 (swap! a inc)) + (t/assert= 2 (swap! a inc)) + (t/assert= 3 (reset! a 3)))) From 475a73a9990a1055a8e079427158af8e145b7dab Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 10 Sep 2015 21:40:59 +0100 Subject: [PATCH 789/909] take transducer --- pixie/stdlib.pxi | 20 ++++++++++++++++---- tests/pixie/tests/test-stdlib.pxi | 10 ++++++++++ 2 files changed, 26 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 398a6144..af9b40c0 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1569,10 +1569,22 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn take {:doc "Takes n elements from the collection, or fewer, if not enough." :added "0.1"} - [n coll] - (when (pos? n) - (when-let [s (seq coll)] - (cons (first s) (take (dec n) (next s)))))) + ([n] + (fn [rf] + (let [seen (atom 0)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [s (swap! seen inc)] + (if (<= s n) + (rf result input) + result))))))) + ([n coll] + (lazy-seq + (when (pos? n) + (when-let [s (seq coll)] + (cons (first s) (take (dec n) (next s)))))))) (defn drop {:doc "Drops n elements from the start of the collection." diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 61896a88..7cc23a84 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -493,6 +493,16 @@ a (recur (inc a)))))) +(t/deftest test-take + (t/assert= (take 0 [1 2 3 4]) ()) + (t/assert= (take 1 [1 2 3 4]) [1]) + (t/assert= (take 2 [1 2 3 4]) [1 2]) + (t/assert= (take 3 [1 2 3 4]) [1 2 3]) + (t/assert= (transduce (take 0) conj [1 2 3 4]) []) + (t/assert= (transduce (take 1) conj [1 2 3 4]) [1]) + (t/assert= (transduce (take 2) conj [1 2 3 4]) [1 2]) + (t/assert= (transduce (take 3) conj [1 2 3 4]) [1 2 3])) + (t/deftest test-take-while (t/assert= (take-while pos? [1 2 3 -1]) [1 2 3]) (t/assert= (take-while pos? [-1 2]) ()) From 83f0860dfacf99347c08bacc96a5e867aaf74a69 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 10 Sep 2015 22:31:18 +0100 Subject: [PATCH 790/909] add drop transducer --- pixie/stdlib.pxi | 32 ++++++++++++++++++++++--------- tests/pixie/tests/test-stdlib.pxi | 10 ++++++++++ 2 files changed, 33 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index af9b40c0..1ea331f7 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1571,15 +1571,16 @@ The new value is thus `(apply f current-value-of-atom args)`." :added "0.1"} ([n] (fn [rf] - (let [seen (atom 0)] + (let [rrf (preserving-reduced rf) + seen (atom 0)] (fn ([] (rf)) ([result] (rf result)) ([result input] (let [s (swap! seen inc)] - (if (<= s n) - (rf result input) - result))))))) + (if (<= s n) + (rrf result input) + (reduced result)))))))) ([n coll] (lazy-seq (when (pos? n) @@ -1589,11 +1590,24 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn drop {:doc "Drops n elements from the start of the collection." :added "0.1"} - [n coll] - (let [s (seq coll)] - (if (and (pos? n) s) - (recur (dec n) (next s)) - s))) + ([n] + (fn [rf] + (let [rrf (preserving-reduced rf) + seen (atom 0)] + (fn + ([] (rf)) + ([result] + (rf result)) + ([result input] + (let [s (swap! seen inc)] + (if (> s n) + (rrf result input) + result))))))) + ([n coll] + (let [s (seq coll)] + (if (and (pos? n) s) + (recur (dec n) (next s)) + s)))) (defn split-at {:doc "Returns a vector of the first n elements of the collection, and the remaining elements." diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 7cc23a84..d60f4e66 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -503,6 +503,16 @@ (t/assert= (transduce (take 2) conj [1 2 3 4]) [1 2]) (t/assert= (transduce (take 3) conj [1 2 3 4]) [1 2 3])) +(t/deftest test-drop + (t/assert= (drop 0 [1 2 3 4]) [1 2 3 4]) + (t/assert= (drop 1 [1 2 3 4]) [2 3 4]) + (t/assert= (drop 2 [1 2 3 4]) [3 4]) + (t/assert= (drop 3 [1 2 3 4]) [4]) + (t/assert= (transduce (drop 0) conj [1 2 3 4]) [1 2 3 4]) + (t/assert= (transduce (drop 1) conj [1 2 3 4]) [2 3 4]) + (t/assert= (transduce (drop 2) conj [1 2 3 4]) [3 4]) + (t/assert= (transduce (drop 3) conj [1 2 3 4]) [4])) + (t/deftest test-take-while (t/assert= (take-while pos? [1 2 3 -1]) [1 2 3]) (t/assert= (take-while pos? [-1 2]) ()) From 5a88bc4c53dc133c3af140cff76f48a6cf2bc8e8 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Wed, 16 Sep 2015 09:34:56 -0400 Subject: [PATCH 791/909] fn pre and post conditions --- pixie/stdlib.pxi | 19 ++++++++++++++++++- tests/pixie/tests/test-stdlib.pxi | 15 +++++++++++++++ 2 files changed, 33 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1ea331f7..8e199705 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2144,7 +2144,24 @@ The params can be destructuring bindings, see `(doc let)` for details."} (recur (inc i) bindings) (recur (inc i) (reduce conj bindings [(nth argv i) (nth names i)]))) bindings)) - body (next decl)] + body (next decl) + conds (when (and (next body) (map? (first body))) + (first body)) + pre (:pre conds) + post (:post conds) + body (if conds (next body) body) + body (if post + `((let [~'% ~(if (> (count body) 1) + `(do ~@body) + (first body))] + ~@(map (fn* [c] `(assert ~c)) post) + ~'%)) + body) + body (if pre + (seq (concat + (map (fn* [c] `(assert ~c)) pre) + body)) + body)] (if (every? symbol? argv) `(~argv ~@body) `(~names diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index d60f4e66..bf5faac8 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -649,3 +649,18 @@ (t/assert= 1 (swap! a inc)) (t/assert= 2 (swap! a inc)) (t/assert= 3 (reset! a 3)))) + +(t/deftest pre-post-conds + (let [f (fn ([a] {:pre [(even? a)] :post [(= % 6)]} (/ a 2)) + ([a b] {:pre [(= (+ 1 a) b)] :post [(odd? %)]} (+ a b)))] + (= 6 (f 12)) + (t/assert-throws? RuntimeException + "Assert failed" + (f 13)) + (t/assert-throws? RuntimeException + "Assert failed" + (f 14)) + (= 15 (f 7 8)) + (t/assert-throws? RuntimeException + "Assert failed" + (f 1 1)))) From 0a95ba6f385f7b70329275c1b96b7c3b44933e49 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Fri, 18 Sep 2015 10:54:43 -0400 Subject: [PATCH 792/909] pre & post assert msgs --- pixie/stdlib.pxi | 4 ++-- tests/pixie/tests/test-stdlib.pxi | 10 +++++----- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8e199705..1d202472 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2154,12 +2154,12 @@ The params can be destructuring bindings, see `(doc let)` for details."} `((let [~'% ~(if (> (count body) 1) `(do ~@body) (first body))] - ~@(map (fn* [c] `(assert ~c)) post) + ~@(map (fn* [c] `(assert ~c (str '~c))) post) ~'%)) body) body (if pre (seq (concat - (map (fn* [c] `(assert ~c)) pre) + (map (fn* [c] `(assert ~c (str '~c))) pre) body)) body)] (if (every? symbol? argv) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index bf5faac8..0b99ce67 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -653,14 +653,14 @@ (t/deftest pre-post-conds (let [f (fn ([a] {:pre [(even? a)] :post [(= % 6)]} (/ a 2)) ([a b] {:pre [(= (+ 1 a) b)] :post [(odd? %)]} (+ a b)))] - (= 6 (f 12)) + (t/assert= 6 (f 12)) (t/assert-throws? RuntimeException - "Assert failed" + "Assert failed: (even? a)" (f 13)) (t/assert-throws? RuntimeException - "Assert failed" + "Assert failed: (= % 6)" (f 14)) - (= 15 (f 7 8)) + (t/assert= 15 (f 7 8)) (t/assert-throws? RuntimeException - "Assert failed" + "Assert failed: (= (+ 1 a) b)" (f 1 1)))) From b5c2796615add7d20d963ef11cb0b3ae3806819f Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Tue, 15 Sep 2015 19:43:59 +0100 Subject: [PATCH 793/909] wip --- benchmarks/read-line.pxi | 22 ++++++----- pixie/io.pxi | 82 ++++++++++++++++++++++++---------------- 2 files changed, 63 insertions(+), 41 deletions(-) diff --git a/benchmarks/read-line.pxi b/benchmarks/read-line.pxi index f09e687a..441ccaeb 100644 --- a/benchmarks/read-line.pxi +++ b/benchmarks/read-line.pxi @@ -1,18 +1,22 @@ (ns benchmarks.readline (:require [pixie.time :as time] - [pixie.io :as io])) + [pixie.io :as io] + [pixie.streams.utf8 :as utf8])) (def file-name "/usr/share/dict/words") -(println "testing unbuffered") +(println "Lazy line-seq") (time/time (-> file-name (io/open-read) (io/line-seq) - (count))) + (count) + (println))) -(println "testing buffered") -(time/time (-> file-name - (io/open-read) - (io/buffered-input-stream) - (io/line-seq) - (count))) +(println "Reducing line-reader") +(time/time + (-> file-name + (io/open-read) + (io/line-reader) + (into []) + (count) + (println))) diff --git a/pixie/io.pxi b/pixie/io.pxi index c07ca1a4..a3160405 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -47,55 +47,64 @@ (assert (string? filename) "Filename must be a string") (->FileStream (fs_open filename uv/O_RDONLY 0) 0 (uv/uv_buf_t))) -(defn buffered-read-line - [input-stream] - (let [line-feed (into #{} (map int [\newline \return]))] - (loop [acc []] - (let [ch (read-byte input-stream)] - (cond - (nil? ch) nil - (zero? ch) nil +(defprotocol ILineReader + (-read-line [this])) - (and (pos? ch) (not (line-feed ch))) - (recur (conj acc ch)) +(def line-feed? #{\newline \return}) - :else (transduce (map char) string-builder acc)))))) +(deftype LineReader + [buffered-input-stream] + ILineReader + (-read-line [this] + (loop [string (string-builder)] + (if-let [i (read-byte buffered-input-stream)] + (let [ch (char i)] + (if-not (line-feed? ch) + (recur (conj! string ch)) + (str string)))))) + IReduce + (-reduce [this f init] + (let [rrf (preserving-reduced f)] + (loop [acc init] + (if-let [line (-read-line this)] + (let [result (rrf acc line)] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) -(defn unbuffered-read-line +(defn line-reader [input-stream] - (let [line-feed (into #{} (map int [\newline \return])) - buf (buffer 1)] - (loop [acc []] - (let [len (read input-stream buf 1)] - (cond - (and (pos? len) (not (line-feed (first buf)))) - (recur (conj acc (first buf))) + (cond + (satisfies? ILineReader input-stream) + input-stream - (and (zero? len) (empty? acc)) nil + (instance? BufferedInputStream input-stream) + (-> input-stream + (->LineReader)) - :else (transduce (map char) string-builder acc)))))) + (satisfies? IInputStream input-stream) + (-> input-stream + (buffered-input-stream) + (->LineReader)) + + :else + (throw [::Exception "Expected a LineReader, an IInputStream or BufferedInputStream"]))) (defn read-line "Read one line from input-stream for each invocation. nil when all lines have been read. Pass a BufferedInputStream for best performance." [input-stream] - (cond - (instance? BufferedInputStream input-stream) - (buffered-read-line input-stream) - - (satisfies? IInputStream input-stream) - (unbuffered-read-line input-stream) - - :else - (throw [::Exception "Expected an IInputStream or BufferedInputStream"]))) + (-read-line (line-reader input-stream))) (defn line-seq "Returns the lines of text from input-stream as a lazy sequence of strings. input-stream must implement IInputStream" [input-stream] - (when-let [line (read-line input-stream)] - (cons line (lazy-seq (line-seq input-stream))))) + (let [lr (line-reader input-stream)] + (when-let [line (-read-line lr)] + (cons line (lazy-seq (line-seq lr)))))) (deftype FileOutputStream [fp offset uvbuf] IOutputStream @@ -136,6 +145,15 @@ (flush downstream)))) (deftype BufferedInputStream [upstream idx buffer] + IReduce + (-reduce [this f init] + (loop [acc init] + (if-let [next-byte (read-byte this)] + (let [step (f acc next-byte)] + (if (reduced? step) + @step + (recur step))) + acc))) IByteInputStream (read-byte [this] (when (= idx (count buffer)) From 118e36a7556e9798347902813796879a414ff4a8 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Sun, 20 Sep 2015 16:04:36 -0400 Subject: [PATCH 794/909] doto macro --- pixie/stdlib.pxi | 11 +++++++++++ tests/pixie/tests/test-stdlib.pxi | 5 +++++ 2 files changed, 16 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1ea331f7..4eee557e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2337,6 +2337,17 @@ Expands to calls to `extend-type`." [~@body])))] `(or (seq ~(gen-loop [] bindings)) '()))) +(defmacro doto + {:doc "Evaluate o, uses the value as the first argument in each form. Returns o"} + [o & forms] + (let [s (gensym o)] + `(let [~s ~o] + ~@(for [f forms] + (if (seq? f) + `(~(first f) ~s ~@(rest f)) + `(~f ~s))) + ~s))) + (defn reverse ; TODO: We should probably have a protocol IReversible, so we can e.g. ; reverse vectors efficiently, etc.. diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index d60f4e66..943a9d1e 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -399,6 +399,11 @@ [2 :a] [2 :b] [2 :c] [3 :a] [3 :b] [3 :c]])) +(t/deftest test-doto + (let [a (atom 0)] + (t/assert= a (doto a (swap! + 3) (swap! str))) + (t/assert= @a "3"))) + (t/deftest test-into (t/assert= [1 3] (into [] (comp (map inc) (filter odd?)) (range 3))) (t/assert= {:a 1 :b 2} (into {} [[:a 1] [:b 2]]))) From 880331c2575417fa151b2ca3d08aaa47a2b1212e Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 23 Sep 2015 20:25:34 +0100 Subject: [PATCH 795/909] fix first on Seqables --- pixie/stdlib.pxi | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1ea331f7..c24e6db7 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -686,7 +686,9 @@ returns true" [coll] (if (satisfies? IIndexed coll) (nth coll 0 nil) - (-first coll))) + (if (satisfies? ISeq coll) + (-first coll) + (-first (seq coll))))) (defn second {:doc "Returns the second item in coll, if coll implements IIndexed nth will be used to retrieve From c927e8d0cb3125e1d02c5686f67ae7d317b71b69 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 23 Sep 2015 20:27:41 +0100 Subject: [PATCH 796/909] enable tests --- tests/pixie/tests/test-stdlib.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index d60f4e66..0e0ec67c 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -130,8 +130,8 @@ (t/assert= (first []) nil) (t/assert= (first '()) nil) (t/assert= (first (make-array 0)) nil) - (comment (t/assert= (first {}) nil)) - (comment (t/assert= (first #{}) nil)) + (t/assert= (first {}) nil) + (t/assert= (first #{}) nil) (t/assert= (first [1 2 3]) 1) (t/assert= (first '(1 2 3)) 1) From ae3bb32ab993ab23bbfa5b847aad73e8899771ec Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Wed, 23 Sep 2015 17:01:02 -0400 Subject: [PATCH 797/909] readline/add_history --- pixie/repl.pxi | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/pixie/repl.pxi b/pixie/repl.pxi index 639035db..6b21c862 100644 --- a/pixie/repl.pxi +++ b/pixie/repl.pxi @@ -5,7 +5,8 @@ (f/with-config {:library "edit" :includes ["editline/readline.h"]} - (f/defcfn readline)) + (f/defcfn readline) + (f/defcfn add_history)) (defn repl [] @@ -15,7 +16,9 @@ "") line (st/apply-blocking readline prompt)] (if line - (str line "\n") + (do + (add_history line) + (str line "\n")) ""))))] (loop [] (try (let [form (read rdr false)] From c0ac40c28b720a424e6c84bf496073e6fe9cc770 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Wed, 23 Sep 2015 22:10:37 +0100 Subject: [PATCH 798/909] alt solution --- pixie/stdlib.pxi | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index c24e6db7..f32b6b25 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -686,9 +686,7 @@ returns true" [coll] (if (satisfies? IIndexed coll) (nth coll 0 nil) - (if (satisfies? ISeq coll) - (-first coll) - (-first (seq coll))))) + (-first coll))) (defn second {:doc "Returns the second item in coll, if coll implements IIndexed nth will be used to retrieve @@ -2595,6 +2593,11 @@ Calling this function on something that is not ISeqable returns a seq with that 0 (compare-counted (str x) (str y)))) +(extend-protocol ISeq + ISeqable + (-first [coll] (-first (seq coll))) + (-next [coll] (-next (seq coll)))) + (extend-protocol IComparable Number (-compare [x y] From 4adcc1a1347733380df0cad931ca945af70026df Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 24 Sep 2015 09:58:33 +0100 Subject: [PATCH 799/909] add non empty tests and sort MapEntry equality --- pixie/stdlib.pxi | 5 +++++ tests/pixie/tests/test-stdlib.pxi | 3 +++ 2 files changed, 8 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index f32b6b25..41f16056 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -940,6 +940,11 @@ If further arguments are passed, invokes the method named by symbol, passing the (cond (= idx 0) (-key self) (= idx 1) (-val self) :else not-found))) +(extend -eq MapEntry (fn [self other] + (and (= (-key self) + (-key other)) + (= (-val self) + (-val other))))) (extend -reduce MapEntry indexed-reduce) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 0e0ec67c..6eed1535 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -133,6 +133,9 @@ (t/assert= (first {}) nil) (t/assert= (first #{}) nil) + (t/assert= (first {:a 1}) (map-entry :a 1)) + (t/assert= (first #{:a}) :a) + (t/assert= (first [1 2 3]) 1) (t/assert= (first '(1 2 3)) 1) (let [a (make-array 3)] From 43345421b5633d44d4aad44005ad3f57fad14b15 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 20 Sep 2015 13:58:39 +0100 Subject: [PATCH 800/909] Improves line-seq and read-line performance Introduces a LineReader which is reducable and speeds up UTF8 decoding --- benchmarks/read-line.pxi | 42 ++++++++++++++++++++++++++--------- pixie/io.pxi | 40 ++++++++++++++++----------------- pixie/stdlib.pxi | 13 ++++++----- pixie/streams/utf8.pxi | 39 +++++++++++++++----------------- tests/pixie/tests/test-io.pxi | 6 ++--- 5 files changed, 79 insertions(+), 61 deletions(-) diff --git a/benchmarks/read-line.pxi b/benchmarks/read-line.pxi index 441ccaeb..c252fd09 100644 --- a/benchmarks/read-line.pxi +++ b/benchmarks/read-line.pxi @@ -6,17 +6,37 @@ (def file-name "/usr/share/dict/words") (println "Lazy line-seq") -(time/time (-> file-name - (io/open-read) - (io/line-seq) - (count) - (println))) +(time/time + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (io/line-seq) + (count))) (println "Reducing line-reader") (time/time - (-> file-name - (io/open-read) - (io/line-reader) - (into []) - (count) - (println))) + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (io/line-reader) + (into []) + (count))) + +(println "Lazy UTF8 line-seq") +(time/time + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (utf8/utf8-input-stream) + (io/line-seq) + (count))) + +(println "Reducing UTF8 line-reader") +(time/time + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (utf8/utf8-input-stream) + (io/line-reader) + (into []) + (count))) diff --git a/pixie/io.pxi b/pixie/io.pxi index a3160405..4143386e 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -53,12 +53,14 @@ (def line-feed? #{\newline \return}) (deftype LineReader - [buffered-input-stream] + [input-stream] ILineReader (-read-line [this] - (loop [string (string-builder)] - (if-let [i (read-byte buffered-input-stream)] - (let [ch (char i)] + (let [read-fn (if (satisfies? utf8/IUTF8InputStream input-stream) + utf8/read-char + (fn [stream] (when-let [i (read-byte stream)] (char i))))] + (loop [string (string-builder)] + (if-let [ch (read-fn input-stream)] (if-not (line-feed? ch) (recur (conj! string ch)) (str string)))))) @@ -79,17 +81,14 @@ (satisfies? ILineReader input-stream) input-stream + (satisfies? utf8/IUTF8InputStream input-stream) + (-> input-stream ->LineReader) + (instance? BufferedInputStream input-stream) - (-> input-stream - (->LineReader)) + (-> input-stream ->LineReader) - (satisfies? IInputStream input-stream) - (-> input-stream - (buffered-input-stream) - (->LineReader)) - :else - (throw [::Exception "Expected a LineReader, an IInputStream or BufferedInputStream"]))) + (throw [::Exception "Expected a LineReader, UTF8InputStream, or BufferedInputStream"]))) (defn read-line "Read one line from input-stream for each invocation. @@ -147,13 +146,14 @@ (deftype BufferedInputStream [upstream idx buffer] IReduce (-reduce [this f init] - (loop [acc init] - (if-let [next-byte (read-byte this)] - (let [step (f acc next-byte)] - (if (reduced? step) - @step - (recur step))) - acc))) + (let [rrf (preserving-reduced f)] + (loop [acc init] + (if-let [next-byte (read-byte this)] + (let [step (rrf acc next-byte)] + (if (reduced? step) + @step + (recur step))) + acc)))) IByteInputStream (read-byte [this] (when (= idx (count buffer)) @@ -248,7 +248,7 @@ (satisfies? IInputStream input) input :else (throw [:pixie.io/Exception "Expected a string or an IInputStream"])) result (transduce - (map char) + (map identity) string-builder (-> stream buffered-input-stream diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 1ea331f7..a99b1059 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1633,13 +1633,14 @@ The new value is thus `(apply f current-value-of-atom args)`." :added "0.1"} ([pred] (fn [rf] - (fn - ([] (rf)) - ([result] (rf result)) - ([result input] + (let [rrf (preserving-reduced rf)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] (if (pred input) - (rf result input) - (reduced result)))))) + (rrf result input) + (reduced result))))))) ([pred coll] (lazy-seq (when-let [s (seq coll)] diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi index e5605e18..8e8ab95f 100644 --- a/pixie/streams/utf8.pxi +++ b/pixie/streams/utf8.pxi @@ -27,30 +27,27 @@ (-dispose! [this] (dispose! out))) - (deftype UTF8InputStream [in bad-char] IUTF8InputStream (read-char [this] (when-let [byte (read-byte in)] - (let [ch (int byte) - [n bytes error?] (cond - (>= 0x7F ch) [ch 1] - (= 0xC0 (bit-and ch 0xE0)) [(bit-and ch 31) 2 false] - (= 0xE0 (bit-and ch 0xF0)) [(bit-and ch 15) 3 false] - (= 0xF0 (bit-and ch 0xF8)) [(bit-and ch 7) 4 false] - (= 0xF8 (bit-and ch 0xF8)) [(bit-and ch 3) 5 true] - (= 0xFC (bit-and ch 0xFE)) [(bit-and ch 1) 6 true] - :else [n 1 true])] - (loop [i (dec bytes) - n n] - (if (pos? i) - (recur (dec i) - (bit-or (bit-shift-left n 6) - (bit-and (read-byte in) 0x3F))) - (if error? - (if bad-char - bad-char - (throw [::invalid-character (str "Invalid UTF8 character decoded: " n)])) + (let [[n bytes error?] + (cond + (>= 0x7F byte) [byte 1 false] + (= 0xC0 (bit-and byte 0xE0)) [(bit-and byte 31) 2 false] + (= 0xE0 (bit-and byte 0xF0)) [(bit-and byte 15) 3 false] + (= 0xF0 (bit-and byte 0xF8)) [(bit-and byte 7) 4 false] + (= 0xF8 (bit-and byte 0xF8)) [(bit-and byte 3) 5 true] + (= 0xFC (bit-and byte 0xFE)) [(bit-and byte 1) 6 true] + :else [n 1 true])] + (if error? + (or bad-char + (throw [::invalid-character (str "Invalid UTF8 character decoded: " n)])) + (loop [i (dec bytes) n n] + (if (pos? i) + (recur (dec i) + (bit-or (bit-shift-left n 6) + (bit-and (read-byte in) 0x3F))) (char n))))))) IDisposable (-dispose! [this] @@ -60,7 +57,7 @@ (let [rrf (preserving-reduced f)] (loop [acc init] (if-let [char (read-char this)] - (let [result (rrf acc (int char))] + (let [result (rrf acc char)] (if (not (reduced? result)) (recur result) @result)) diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 663bf52b..1d41c4a5 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -33,18 +33,18 @@ (t/assert-throws? (io/read f buf -2))))) (t/deftest test-read-line - (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] (io/read-line f) (t/assert= (io/read-line f) "Second line.") (t/assert= (io/read-line f) nil))) (t/deftest test-line-seq - (let [f (io/open-read "tests/pixie/tests/test-io.txt") + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt")) s (io/line-seq f)] (t/assert= (last s) "Second line."))) (t/deftest test-seek - (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] (io/read-line f) (t/assert= (io/read-line f) "Second line.") (io/rewind f) From 4f38016810399870e7ba9a6721b61ee44c30ec7b Mon Sep 17 00:00:00 2001 From: Brandon Adams Date: Fri, 25 Sep 2015 01:13:34 -0500 Subject: [PATCH 801/909] Let LazySeq conj --- pixie/stdlib.pxi | 12 ++++++++---- tests/pixie/tests/test-stdlib.pxi | 3 +++ 2 files changed, 11 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 216c4a12..6e40bf01 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1049,6 +1049,10 @@ If further arguments are passed, invokes the method named by symbol, passing the (fn [coll x] (cons x coll))) +(extend -conj LazySeq + (fn [coll x] + (cons x coll))) + (defn empty {:doc "Returns an empty collection of the same type, or nil." :added "0.1"} @@ -1601,7 +1605,7 @@ The new value is thus `(apply f current-value-of-atom args)`." seen (atom 0)] (fn ([] (rf)) - ([result] + ([result] (rf result)) ([result input] (let [s (swap! seen inc)] @@ -2611,7 +2615,7 @@ Calling this function on something that is not ISeqable returns a seq with that (let [min-length (min (count x) (count y))] (loop [n 0] (if (not= min-length n) - (let [diff (-compare (nth x n) + (let [diff (-compare (nth x n) (nth y n))] (if-not (zero? diff) diff @@ -2644,7 +2648,7 @@ Calling this function on something that is not ISeqable returns a seq with that (if (char? y) (compare-numbers (int x) (int y)) (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) - + PersistentVector (-compare [x y] (if (vector? y) @@ -2677,7 +2681,7 @@ Calling this function on something that is not ISeqable returns a seq with that (and (true? x) (false? y)) 1 :else -1)) (throw [::ComparisonError (str "Cannot compare: " x " to " y)])) - + Nil (-compare [x y] (if (nil? y) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index aa8547c8..a4cb9c38 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -672,3 +672,6 @@ (t/assert-throws? RuntimeException "Assert failed: (= (+ 1 a) b)" (f 1 1)))) + +(t/deftest test-lazyseq-conj + (t/assert= '(1 2 3) (conj (lazy-seq '(2 3)) 1))) From 9990b235a186454e36171f84c78a0ebfb18ee5d1 Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Tue, 29 Sep 2015 11:27:01 -0400 Subject: [PATCH 802/909] Mandelbrot demo from Strange Loop talk --- examples/mandelbrot.pxi | 101 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 101 insertions(+) create mode 100644 examples/mandelbrot.pxi diff --git a/examples/mandelbrot.pxi b/examples/mandelbrot.pxi new file mode 100644 index 00000000..4258280d --- /dev/null +++ b/examples/mandelbrot.pxi @@ -0,0 +1,101 @@ +;; Mandelbrot demo from Timothy Baldridge's 2015 Strange Loop talk +;; https://www.youtube.com/watch?v=1AjhFZVfB9c + + +(ns mandelbrot + (:require [pixie.ffi-infer :refer :all] + [pixie.ffi :as ffi] + [pixie.time :refer [time]])) + +(with-config {:library "SDL2" + :cxx-flags ["`sdl2-config --cflags`"] + :includes ["SDL.h"]} + + (defcfn SDL_Init) + + (defconst SDL_INIT_VIDEO) + (defconst SDL_WINDOWPOS_UNDEFINED) + (defcfn SDL_CreateWindow) + (defcfn SDL_CreateRenderer) + (defcfn SDL_CreateTexture) + (defconst SDL_PIXELFORMAT_RGBA8888) + (defconst SDL_TEXTUREACCESS_STREAMING) + (defcfn SDL_UpdateTexture) + (defcfn SDL_RenderClear) + (defcfn SDL_RenderCopy) + + (defconst SDL_WINDOW_SHOWN) + (defcfn SDL_RenderPresent) + (defcfn SDL_LockSurface)) + +(println "starting") +(def WIDTH 1024) +(def HEIGHT 512) + +(assert (>= (SDL_Init SDL_INIT_VIDEO) 0)) + +(def WINDOW (SDL_CreateWindow "Pixie MandelBrot" + SDL_WINDOWPOS_UNDEFINED + SDL_WINDOWPOS_UNDEFINED + WIDTH + HEIGHT + SDL_WINDOW_SHOWN)) + +(assert WINDOW "Could not create window") + +(def RENDERER (SDL_CreateRenderer WINDOW -1 0)) + +(def DRAW_SURFACE (SDL_CreateTexture RENDERER + SDL_PIXELFORMAT_RGBA8888 + SDL_TEXTUREACCESS_STREAMING + WIDTH + HEIGHT)) + +(defn bit-or [& args] + (reduce pixie.stdlib/bit-or 0 args)) + +(defn put-pixel [ptr x y r g b] + (let [loc (* 4 (+(* y WIDTH) x))] + (ffi/pack! ptr loc CUInt32 (bit-or (bit-shift-left r 24) + (bit-shift-left g 16) + (bit-shift-left b 8) + 255)))) + +(def BUFFER (buffer (* 4 WIDTH HEIGHT))) + +(defn mandel-point [x y width height max_iterations] + (let [x0 (float (- (* (/ x width) 3.5) 2.5)) + y0 (float (- (* (/ y height) 2) 1))] + (loop [x 0.0 + y 0.0 + iteration 0] + (let [xsq (* x x) + ysq (* y y)] + (if (and (< (+ xsq + ysq) + (* 2 2)) + (< iteration max_iterations)) + (let [xtemp (+ (- xsq + ysq) + x0) + y (+ (* 2 x y) y0)] + (recur xtemp y (inc iteration))) + (- 1 (/ iteration max_iterations))))))) + +(dotimes [x 3] + (time + (let [max (* WIDTH HEIGHT)] + (dotimes [y HEIGHT] + (dotimes [x WIDTH] + (let [result (mandel-point (float x) (float y) (float WIDTH) (float HEIGHT) 1000) + color (int (* 16777216 result))] + (put-pixel BUFFER x y + (bit-shift-right color 16) + (bit-and (bit-shift-right color 8) 0xff) + (bit-and color 0xff)))))))) + +(SDL_UpdateTexture DRAW_SURFACE nil BUFFER (* 4 WIDTH)) +(SDL_RenderCopy RENDERER DRAW_SURFACE nil nil) +(SDL_RenderPresent RENDERER) + +(pixie.stacklets/sleep 10000) From 8d89628e029f2809c2ea55c93bcb3065a4dcc285 Mon Sep 17 00:00:00 2001 From: Brandon Adams Date: Thu, 24 Sep 2015 08:17:04 -0500 Subject: [PATCH 803/909] Added map-entry? predicate --- pixie/stdlib.pxi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index f019e838..6720693c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -871,6 +871,8 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn indexed? [v] (satisfies? IIndexed v)) (defn counted? [v] (satisfies? ICounted v)) +(defn map-entry? [v] (satisfies? IMapEntry v)) + (defn last {:doc "Returns the last element of the collection, or nil if none." :signatures [[coll]] @@ -2320,7 +2322,7 @@ Expands to calls to `extend-type`." (defn float {:doc "Converts a number to a float." - :since "0.1"} + :since "0.1"} [x] (-float x)) @@ -2348,7 +2350,7 @@ Expands to calls to `extend-type`." (defn int {:doc "Converts a number to an integer." - :since "0.1"} + :since "0.1"} [x] (-int x)) From ad286c674817f58841753518fbbb0a8a383945bd Mon Sep 17 00:00:00 2001 From: Brandon Adams Date: Thu, 24 Sep 2015 08:19:20 -0500 Subject: [PATCH 804/909] Extend map-entry's -eq to vectors, add -seq for MapEntry This enables (= (map-entry :a 1) [:a 1]) and the reverse (= [:a 1] (map-entry :a 1)). Tests are included. MapEntry falls back to seq equality if not compared to another MapEntry or Vector --- pixie/stdlib.pxi | 16 ++++++++++++---- tests/pixie/tests/test-stdlib.pxi | 16 ++++++++++++++++ 2 files changed, 28 insertions(+), 4 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6720693c..cf38d04f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -919,10 +919,14 @@ If further arguments are passed, invokes the method named by symbol, passing the (= idx 1) (-val self) :else not-found))) (extend -eq MapEntry (fn [self other] - (and (= (-key self) - (-key other)) - (= (-val self) - (-val other))))) + (cond + (map-entry? other) (and (= (-key self) + (-key other)) + (= (-val self) + (-val other))) + (vector? other) (= [(-key self) (-val self)] + other) + :else (= (seq self) other)))) (extend -reduce MapEntry indexed-reduce) @@ -937,6 +941,10 @@ If further arguments are passed, invokes the method named by symbol, passing the (fn [v] (transduce ordered-hash-reducing-fn v))) +(extend -seq MapEntry + (fn [self] + (list (-key self) (-val self)))) + (defn select-keys {:doc "Produces a map with only the values in m contained in key-seq"} [m key-seq] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index a4cb9c38..8334d9e0 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -675,3 +675,19 @@ (t/deftest test-lazyseq-conj (t/assert= '(1 2 3) (conj (lazy-seq '(2 3)) 1))) + +(t/deftest map-entry-seq + (let [m (map-entry :a 1)] + (t/assert= (seq m) '(:a 1)) + ;; vectors are equal to seqs if their indexed values are equal + ;; thus, a vector of length 2 is equal to a map-entry's seq IFoo + ;; their values are equal + (t/assert= [:a 1] m) + (t/assert= '(:a 1) m))) + +(t/deftest map-entry-equals + (let [m (map-entry :a 1)] + (t/assert= m m) + (t/assert= m (map-entry :a 1)) + (t/assert= m [:a 1]) + (t/assert= m '(:a 1)))) From ab2eb8aa287c0b5603738de84222a661d9621c46 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Fri, 2 Oct 2015 14:40:27 -0500 Subject: [PATCH 805/909] Adding a pixie.string/trim-newline function based off the one in Clojure. --- pixie/string.pxi | 1 + pixie/vm/libs/string.py | 10 ++++++++++ tests/pixie/tests/test-strings.pxi | 10 ++++++++++ 3 files changed, 21 insertions(+) diff --git a/pixie/string.pxi b/pixie/string.pxi index dc0fc7b7..73716df4 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -18,6 +18,7 @@ (def trim si/trim) (def triml si/triml) (def trimr si/trimr) +(def trim-newline si/trim-newline) (def capitalize si/capitalize) (def lower-case si/lower-case) diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 967046c8..22403616 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -121,3 +121,13 @@ def trimr(a): if j <= 0: return rt.wrap(u"") return rt.wrap(a[0:j]) + +@as_var("pixie.string.internal", "trim-newline") +def trim_newline(a): + a = rt.name(a) + j = len(a) + while j > 0 and unicodedb.islinebreak(ord(a[j - 1])): + j -= 1 + if j <= 0: + return rt.wrap(u"") + return rt.wrap(a[0:j]) diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi index 129c5768..0013f6e2 100644 --- a/tests/pixie/tests/test-strings.pxi +++ b/tests/pixie/tests/test-strings.pxi @@ -110,6 +110,16 @@ (t/assert= (s/trimr " hey ") " hey") (t/assert= (s/trimr " h ey ") " h ey")) +(t/deftest test-trim-newline + (t/assert= (s/trim-newline "") "") + (t/assert= (s/trim-newline "\r\n") "") + (t/assert= (s/trim-newline "hey\r\n") "hey") + (t/assert= (s/trim-newline "hey\n\r") "hey") + (t/assert= (s/trim-newline "hey\r") "hey") + (t/assert= (s/trim-newline "hey\n") "hey") + (t/assert= (s/trim-newline "hey\r\nthere\r\n") "hey\r\nthere") + (t/assert= (s/trim-newline "\r\nhey\r\n") "\r\nhey")) + (t/deftest test-replace (t/assert= (s/replace "hey,you,there" "," ", ") "hey, you, there") (t/assert= (s/replace "hey,you,there" "," "") "heyyouthere") From 20a442bae69ce4bb16d5c965a9b4c7d99085cee4 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Fri, 2 Oct 2015 16:07:34 -0500 Subject: [PATCH 806/909] Adding a set union function. --- pixie/set.pxi | 16 ++++++++++++++++ tests/pixie/tests/test-sets.pxi | 20 ++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 pixie/set.pxi create mode 100644 tests/pixie/tests/test-sets.pxi diff --git a/pixie/set.pxi b/pixie/set.pxi new file mode 100644 index 00000000..4479d2f8 --- /dev/null +++ b/pixie/set.pxi @@ -0,0 +1,16 @@ +(ns pixie.set + (:require [pixie.stdlib :as std])) + +(defn- -union [s t] + (if (< (count s) (count t)) + (reduce conj t s) + (reduce conj s t))) + +(defn union + "Returns a set that is the union of the input sets." + ([] #{}) + ([s] s) + ([s t] + (-union s t)) + ([s t & sets] + (reduce -union (-union s t) sets))) diff --git a/tests/pixie/tests/test-sets.pxi b/tests/pixie/tests/test-sets.pxi new file mode 100644 index 00000000..1e182c1e --- /dev/null +++ b/tests/pixie/tests/test-sets.pxi @@ -0,0 +1,20 @@ +(ns pixie.test.test-sets + (require pixie.test :as t) + (require pixie.set :as s)) + +(t/deftest test-union + (let [magic #{:bibbidi :bobbidi :boo} + work #{:got :no :time :to :dilly-dally} + dreams #{:the :dreams :that :i :wish} + magical-work #{:bibbidi :bobbidi :boo + :got :no :time :to :dilly-dally} + dream-of-magical-work #{:bibbidi :bobbidi :boo + :got :no :time :to :dilly-dally + :the :dreams :that :i :wish}] + (t/assert= (s/union) #{}) + (t/assert= (s/union magic) magic) + (t/assert= (s/union magic magic) magic) + (t/assert= (s/union magic magic magic) magic) + (t/assert= (s/union magic work) magical-work) + (t/assert= (s/union work magic) magical-work) + (t/assert= (s/union magic work dreams) dream-of-magical-work))) From e8206e7c119242a09ae7be963008ef094ad5089b Mon Sep 17 00:00:00 2001 From: Marcin Gasperowicz Date: Sat, 3 Oct 2015 15:48:39 +0200 Subject: [PATCH 807/909] Fix issue #203 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fixes issue #203 „Syntax quoting for maps is broken” Adds `map?` Adds tests for syntax quoting in maps --- pixie/vm/reader.py | 12 ++++++++++++ pixie/vm/stdlib.py | 4 ++++ tests/pixie/tests/test-macros.pxi | 8 ++++++++ 3 files changed, 24 insertions(+) create mode 100644 tests/pixie/tests/test-macros.pxi diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index a6ae64cd..84dccfba 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -430,6 +430,7 @@ def invoke(self, rdr, ch): CONCAT = symbol(u"concat") SEQ = symbol(u"seq") LIST = symbol(u"list") +HASHMAP = symbol(u"hashmap") def is_unquote(form): return True if rt._satisfies_QMARK_(rt.ISeq.deref(), form) \ @@ -477,16 +478,27 @@ def syntax_quote(form): return runtime_error(u"Unquote splicing not used inside list") elif rt.vector_QMARK_(form) is true: ret = rt.list(APPLY, CONCAT, SyntaxQuoteReader.expand_list(form)) + elif rt.map_QMARK_(form) is true: + mes = SyntaxQuoteReader.flatten_map(form) + ret = rt.list(APPLY, HASHMAP, rt.list(APPLY, CONCAT, SyntaxQuoteReader.expand_list(mes))) elif form is not nil and rt.seq_QMARK_(form) is true: ret = rt.list(APPLY, LIST, rt.cons(CONCAT, SyntaxQuoteReader.expand_list(rt.seq(form)))) else: ret = rt.list(QUOTE, form) return ret + @staticmethod + def flatten_map(form): + return rt.reduce(flatten_map_rfn, EMPTY_VECTOR, form) + @staticmethod def expand_list(form): return rt.reduce(expand_list_rfn, EMPTY_VECTOR, form) +@wrap_fn +def flatten_map_rfn(ret, item): + return rt.conj(rt.conj(ret, rt.first(item)), rt.first(rt.next(item))) + @wrap_fn def expand_list_rfn(ret, item): if is_unquote(item): diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 45287c85..79a9061c 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -667,6 +667,10 @@ def identical(a, b): def vector_QMARK_(a): return true if rt._satisfies_QMARK_(rt.IVector.deref(), a) else false +@as_var("map?") +def map_QMARK_(a): + return true if rt._satisfies_QMARK_(rt.IMap.deref(), a) else false + @returns(bool) @as_var("eq") def eq(a, b): diff --git a/tests/pixie/tests/test-macros.pxi b/tests/pixie/tests/test-macros.pxi new file mode 100644 index 00000000..5678a1ee --- /dev/null +++ b/tests/pixie/tests/test-macros.pxi @@ -0,0 +1,8 @@ +(ns collections.test-macros + (require pixie.test :as t)) + +(t/deftest hashmap-unquote + (let [x 10 k :boop] + (t/assert= (-eq `{:x ~x} {:x 10}) true) + (t/assert= (-eq `{~k ~x} {:boop 10}) true) + (t/assert= (-eq `{:x {:y ~x}} {:x {:y 10}}) true))) \ No newline at end of file From 7049c2bf6e9de8d2a03b38a43fcb7d682d44dbf6 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Tue, 6 Oct 2015 14:50:49 -0500 Subject: [PATCH 808/909] Set union asserts the arguments must be sets. --- pixie/set.pxi | 22 +++++++++++++++------- tests/pixie/tests/test-sets.pxi | 3 ++- 2 files changed, 17 insertions(+), 8 deletions(-) diff --git a/pixie/set.pxi b/pixie/set.pxi index 4479d2f8..0fd73cdb 100644 --- a/pixie/set.pxi +++ b/pixie/set.pxi @@ -1,16 +1,24 @@ (ns pixie.set (:require [pixie.stdlib :as std])) +(defn- -must-be-set [s] + (if (set? s) + s + (throw [:pixie.stdlib/InvalidArgumentException + (str "Not a set: " s)]))) + (defn- -union [s t] + (-must-be-set s) + (-must-be-set t) (if (< (count s) (count t)) (reduce conj t s) (reduce conj s t))) (defn union - "Returns a set that is the union of the input sets." - ([] #{}) - ([s] s) - ([s t] - (-union s t)) - ([s t & sets] - (reduce -union (-union s t) sets))) + "Returns a set that is the union of the input sets." + ([] #{}) + ([s] (-must-be-set s)) + ([s t] + (-union s t)) + ([s t & sets] + (reduce -union (-union s t) sets))) diff --git a/tests/pixie/tests/test-sets.pxi b/tests/pixie/tests/test-sets.pxi index 1e182c1e..e65ae454 100644 --- a/tests/pixie/tests/test-sets.pxi +++ b/tests/pixie/tests/test-sets.pxi @@ -17,4 +17,5 @@ (t/assert= (s/union magic magic magic) magic) (t/assert= (s/union magic work) magical-work) (t/assert= (s/union work magic) magical-work) - (t/assert= (s/union magic work dreams) dream-of-magical-work))) + (t/assert= (s/union magic work dreams) dream-of-magical-work) + (t/assert-throws? (s/union [:i :only] [:love :sets])))) From ae1fa4a1a7d717902726bb03ae83b2d003b4acc0 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Tue, 6 Oct 2015 16:40:37 -0500 Subject: [PATCH 809/909] Rewriting trim-newline as pure Pixie. --- pixie/string.pxi | 12 +++++++++++- pixie/vm/libs/string.py | 10 ---------- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/pixie/string.pxi b/pixie/string.pxi index 73716df4..cc72b96f 100644 --- a/pixie/string.pxi +++ b/pixie/string.pxi @@ -18,7 +18,6 @@ (def trim si/trim) (def triml si/triml) (def trimr si/trimr) -(def trim-newline si/trim-newline) (def capitalize si/capitalize) (def lower-case si/lower-case) @@ -35,6 +34,17 @@ (def hexdigits "0123456789abcdefABCDEF") (def octdigits "012345678") +(defn trim-newline + "Replace all trailing newline characters (\\r and \\n) from the end of a string." + [s] + (loop [index (count s)] + (if (zero? index) + "" + (let [ch (nth s (dec index))] + (if (or (= ch \newline) (= ch \return)) + (recur (dec index)) + (substring s 0 index)))))) + (defn replace "Replace all occurrences of x in s with r." [s x r] diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py index 22403616..967046c8 100644 --- a/pixie/vm/libs/string.py +++ b/pixie/vm/libs/string.py @@ -121,13 +121,3 @@ def trimr(a): if j <= 0: return rt.wrap(u"") return rt.wrap(a[0:j]) - -@as_var("pixie.string.internal", "trim-newline") -def trim_newline(a): - a = rt.name(a) - j = len(a) - while j > 0 and unicodedb.islinebreak(ord(a[j - 1])): - j -= 1 - if j <= 0: - return rt.wrap(u"") - return rt.wrap(a[0:j]) From e37f888a3d26102321ed4ce4fc3c45d0d6acff1e Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Wed, 7 Oct 2015 14:08:43 -0500 Subject: [PATCH 810/909] Adding an implementation of set intersection. --- pixie/set.pxi | 18 ++++++++++++++++++ tests/pixie/tests/test-sets.pxi | 13 +++++++++++++ 2 files changed, 31 insertions(+) diff --git a/pixie/set.pxi b/pixie/set.pxi index 0fd73cdb..8d1792ba 100644 --- a/pixie/set.pxi +++ b/pixie/set.pxi @@ -22,3 +22,21 @@ (-union s t)) ([s t & sets] (reduce -union (-union s t) sets))) + +(defn- -intersection [s t] + (-must-be-set s) + (-must-be-set t) + (let [result (atom #{})] + (doseq [i s] + (when (contains? t i) + (swap! result conj i))) + @result)) + +(defn intersection + "Returns a set that is the intersection of the input sets." + ([] #{}) + ([s] (-must-be-set s)) + ([s t] + (-intersection s t)) + ([s t & sets] + (reduce -intersection (-intersection s t) sets))) diff --git a/tests/pixie/tests/test-sets.pxi b/tests/pixie/tests/test-sets.pxi index e65ae454..90d91c2a 100644 --- a/tests/pixie/tests/test-sets.pxi +++ b/tests/pixie/tests/test-sets.pxi @@ -19,3 +19,16 @@ (t/assert= (s/union work magic) magical-work) (t/assert= (s/union magic work dreams) dream-of-magical-work) (t/assert-throws? (s/union [:i :only] [:love :sets])))) + +(t/deftest test-intersection + (let [magic #{:bibbidi :bobbidi :boo} + work #{:got :no :time :to :dilly-dally}] + (t/assert= (s/intersection) #{}) + (t/assert= (s/intersection magic) magic) + (t/assert= (s/intersection magic magic) magic) + (t/assert= (s/intersection magic work) #{}) + (t/assert= (s/intersection magic #{:boo}) #{:boo}) + (t/assert= (s/intersection #{:boo} magic) #{:boo}) + (t/assert= (s/intersection magic #{:bobbidi :boo}) #{:bobbidi :boo}) + (t/assert= (s/intersection magic #{:bibbidi :boo} #{:bobbidi :boo}) #{:boo}) + (t/assert-throws? (s/intersection [:i :only] [:love :sets])))) From c917853505a569e1d95cf63a1c3ef1cdbab4b1bd Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Wed, 7 Oct 2015 16:08:50 -0500 Subject: [PATCH 811/909] Using into instead of an atom to build the intersection. --- pixie/set.pxi | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/pixie/set.pxi b/pixie/set.pxi index 8d1792ba..8760093e 100644 --- a/pixie/set.pxi +++ b/pixie/set.pxi @@ -26,11 +26,7 @@ (defn- -intersection [s t] (-must-be-set s) (-must-be-set t) - (let [result (atom #{})] - (doseq [i s] - (when (contains? t i) - (swap! result conj i))) - @result)) + (into #{} (filter #(contains? s %)) t)) (defn intersection "Returns a set that is the intersection of the input sets." From 7a5b3e3897218e3c78d7933a4e8e988e1453fde1 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Wed, 7 Oct 2015 18:07:42 -0500 Subject: [PATCH 812/909] Adding a set difference function. --- pixie/set.pxi | 14 ++++++++++ tests/pixie/tests/test-sets.pxi | 47 +++++++++++++++++++-------------- 2 files changed, 41 insertions(+), 20 deletions(-) diff --git a/pixie/set.pxi b/pixie/set.pxi index 8760093e..483dc26d 100644 --- a/pixie/set.pxi +++ b/pixie/set.pxi @@ -23,6 +23,20 @@ ([s t & sets] (reduce -union (-union s t) sets))) +(defn- -difference [s t] + (-must-be-set s) + (-must-be-set t) + (into #{} (filter #(not (contains? t %))) s)) + +(defn difference + "Returns a set that is the difference of the input sets." + ([] #{}) + ([s] (-must-be-set s)) + ([s t] + (-difference s t)) + ([s t & sets] + (reduce -difference (-difference s t) sets))) + (defn- -intersection [s t] (-must-be-set s) (-must-be-set t) diff --git a/tests/pixie/tests/test-sets.pxi b/tests/pixie/tests/test-sets.pxi index 90d91c2a..e9212496 100644 --- a/tests/pixie/tests/test-sets.pxi +++ b/tests/pixie/tests/test-sets.pxi @@ -2,27 +2,34 @@ (require pixie.test :as t) (require pixie.set :as s)) -(t/deftest test-union - (let [magic #{:bibbidi :bobbidi :boo} - work #{:got :no :time :to :dilly-dally} - dreams #{:the :dreams :that :i :wish} - magical-work #{:bibbidi :bobbidi :boo - :got :no :time :to :dilly-dally} - dream-of-magical-work #{:bibbidi :bobbidi :boo - :got :no :time :to :dilly-dally - :the :dreams :that :i :wish}] - (t/assert= (s/union) #{}) - (t/assert= (s/union magic) magic) - (t/assert= (s/union magic magic) magic) - (t/assert= (s/union magic magic magic) magic) - (t/assert= (s/union magic work) magical-work) - (t/assert= (s/union work magic) magical-work) - (t/assert= (s/union magic work dreams) dream-of-magical-work) - (t/assert-throws? (s/union [:i :only] [:love :sets])))) +(let [magic #{:bibbidi :bobbidi :boo} + work #{:got :no :time :to :dilly-dally}] -(t/deftest test-intersection - (let [magic #{:bibbidi :bobbidi :boo} - work #{:got :no :time :to :dilly-dally}] + (t/deftest test-union + (let [dreams #{:the :dreams :that :i :wish} + magical-work #{:bibbidi :bobbidi :boo + :got :no :time :to :dilly-dally} + dream-of-magical-work #{:bibbidi :bobbidi :boo + :got :no :time :to :dilly-dally + :the :dreams :that :i :wish}] + (t/assert= (s/union) #{}) + (t/assert= (s/union magic) magic) + (t/assert= (s/union magic magic) magic) + (t/assert= (s/union magic magic magic) magic) + (t/assert= (s/union magic work) magical-work) + (t/assert= (s/union work magic) magical-work) + (t/assert= (s/union magic work dreams) dream-of-magical-work) + (t/assert-throws? (s/union [:i :only] [:love :sets])))) + + (t/deftest test-difference + (t/assert= (s/difference) #{}) + (t/assert= (s/difference magic) magic) + (t/assert= (s/difference magic #{:boo}) #{:bibbidi :bobbidi}) + (t/assert= (s/difference magic work) magic) + (t/assert= (s/difference magic #{:bibbidi} #{:bobbidi}) #{:boo}) + (t/assert-throws? (s/difference [:i :only] [:love :sets]))) + + (t/deftest test-intersection (t/assert= (s/intersection) #{}) (t/assert= (s/intersection magic) magic) (t/assert= (s/intersection magic magic) magic) From a524fa3122aa49b1140dde8fe9f1cfa851a02195 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 8 Oct 2015 11:57:29 -0500 Subject: [PATCH 813/909] Adding several set relational predicates. --- pixie/set.pxi | 39 +++++++++++++++++++++++++++------ tests/pixie/tests/test-sets.pxi | 30 ++++++++++++++++++++++++- 2 files changed, 61 insertions(+), 8 deletions(-) diff --git a/pixie/set.pxi b/pixie/set.pxi index 483dc26d..2a113e5e 100644 --- a/pixie/set.pxi +++ b/pixie/set.pxi @@ -7,12 +7,15 @@ (throw [:pixie.stdlib/InvalidArgumentException (str "Not a set: " s)]))) +(defn- -must-be-sets [& sets] + (doseq [set sets] + (-must-be-set set))) + (defn- -union [s t] - (-must-be-set s) - (-must-be-set t) - (if (< (count s) (count t)) - (reduce conj t s) - (reduce conj s t))) + (-must-be-sets s t) + (if (< (count s) (count t)) + (reduce conj t s) + (reduce conj s t))) (defn union "Returns a set that is the union of the input sets." @@ -24,8 +27,7 @@ (reduce -union (-union s t) sets))) (defn- -difference [s t] - (-must-be-set s) - (-must-be-set t) + (-must-be-sets s t) (into #{} (filter #(not (contains? t %))) s)) (defn difference @@ -50,3 +52,26 @@ (-intersection s t)) ([s t & sets] (reduce -intersection (-intersection s t) sets))) + +(defn subset? + "Returns true if the first set is a subset of the second." + [s t] + (-must-be-sets s t) + (and (<= (count s) (count t)) + (every? #(contains? t %) (vec s)))) + +(defn strict-subset? + "Returns true if the first set is a subset of the second." + [s t] + (and (subset? s t) + (< (count s) (count t)))) + +(defn superset? + "Returns true if the first set is a superset of the second." + [s t] + (subset? t s)) + +(defn strict-superset? + "Returns true if the first set is a strict superset of the second." + [s t] + (strict-subset? t s)) diff --git a/tests/pixie/tests/test-sets.pxi b/tests/pixie/tests/test-sets.pxi index e9212496..bccc8041 100644 --- a/tests/pixie/tests/test-sets.pxi +++ b/tests/pixie/tests/test-sets.pxi @@ -38,4 +38,32 @@ (t/assert= (s/intersection #{:boo} magic) #{:boo}) (t/assert= (s/intersection magic #{:bobbidi :boo}) #{:bobbidi :boo}) (t/assert= (s/intersection magic #{:bibbidi :boo} #{:bobbidi :boo}) #{:boo}) - (t/assert-throws? (s/intersection [:i :only] [:love :sets])))) + (t/assert-throws? (s/intersection [:i :only] [:love :sets]))) + + (t/deftest test-subset? + (t/assert (not (s/subset? magic work))) + (t/assert (s/subset? magic magic)) + (t/assert (not (s/subset? magic #{:foo}))) + (t/assert (s/subset? #{:boo} magic)) + (t/assert-throws? (s/subset? [:i :only] [:love :sets]))) + + (t/deftest test-strict-subset? + (t/assert (not (s/strict-subset? magic work))) + (t/assert (not (s/strict-subset? magic magic))) + (t/assert (not (s/strict-subset? magic #{:foo}))) + (t/assert (s/strict-subset? #{:boo} magic)) + (t/assert-throws? (s/strict-subset? [:i :only] [:love :sets]))) + + (t/deftest test-superset? + (t/assert (not (s/superset? magic work))) + (t/assert (s/superset? magic magic)) + (t/assert (not (s/superset? #{:foo} magic))) + (t/assert (s/superset? magic #{:boo})) + (t/assert-throws? (s/superset? [:i :only] [:love :sets]))) + + (t/deftest test-strict-superset? + (t/assert (not (s/strict-superset? magic work))) + (t/assert (not (s/strict-superset? magic magic))) + (t/assert (not (s/strict-superset? #{:foo} magic))) + (t/assert (s/strict-superset? magic #{:boo})) + (t/assert-throws? (s/strict-superset? [:i :only] [:love :sets])))) From e19d9a895f3d3436fab6f084e5524ec57a3b299d Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 8 Oct 2015 12:05:07 -0500 Subject: [PATCH 814/909] It'll probably be faster to do the count check first. --- pixie/set.pxi | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pixie/set.pxi b/pixie/set.pxi index 2a113e5e..18060759 100644 --- a/pixie/set.pxi +++ b/pixie/set.pxi @@ -63,8 +63,9 @@ (defn strict-subset? "Returns true if the first set is a subset of the second." [s t] - (and (subset? s t) - (< (count s) (count t)))) + (-must-be-sets s t) + (and (< (count s) (count t)) + (subset? s t))) (defn superset? "Returns true if the first set is a superset of the second." From 97154127ac4075009b382f3532d638a36e65b035 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 8 Oct 2015 18:48:50 -0500 Subject: [PATCH 815/909] Faster set union. --- pixie/set.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/set.pxi b/pixie/set.pxi index 18060759..23c4b425 100644 --- a/pixie/set.pxi +++ b/pixie/set.pxi @@ -14,8 +14,8 @@ (defn- -union [s t] (-must-be-sets s t) (if (< (count s) (count t)) - (reduce conj t s) - (reduce conj s t))) + (into t s) + (into s t))) (defn union "Returns a set that is the union of the input sets." From e971bcf1735394e14e88abd912055ad780d662a5 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 10 Oct 2015 09:38:52 +0200 Subject: [PATCH 816/909] Use python2 as the default python executable This should make building on systems with python3 as default easier, and also make it clear that we need python2. Fixes #337. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index f857a332..eae4c76b 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=externals -PYTHON ?= python +PYTHON ?= python2 PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy From 4c5d3ee0f35a1d50d5ad245bcb9ab0eecb77ac9a Mon Sep 17 00:00:00 2001 From: puffybsd Date: Sun, 11 Oct 2015 00:27:18 -0400 Subject: [PATCH 817/909] Fix make clean_pxic 'rm: missing operand' error make clean_pxic fails if no files are found due to invalid rm command 'rm: missing operand'. Add --no-run-if-empty to xarg to fix. --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index eae4c76b..cbe6086c 100644 --- a/Makefile +++ b/Makefile @@ -87,7 +87,7 @@ compile_src: find * -name "*.pxi" | grep "^pixie/" | xargs -L1 ./pixie-vm $(EXTERNALS_FLAGS) -c clean_pxic: - find * -name "*.pxic" | xargs rm + find * -name "*.pxic" | xargs --no-run-if-empty rm clean: clean_pxic rm -rf ./lib From 83e2ca7781bcfa2e943035c3bdd6e9dc97cc89df Mon Sep 17 00:00:00 2001 From: Philipp Rustemeier Date: Thu, 15 Oct 2015 12:37:07 +0200 Subject: [PATCH 818/909] FFI: coerce numbers to double if required and possible If a function invoked over FFI requires a double and a (Big)Integer or a Ratio is supplied, automatically coerce to double. Should help with the rest of #377. --- pixie/vm/libs/ffi.py | 8 +++++++- pixie/vm/numbers.py | 2 ++ tests/pixie/tests/test-ffi.pxi | 7 +++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 86fdc840..eb24e44d 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -8,7 +8,7 @@ import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype, llmemory from pixie.vm.primitives import nil, true, false -from pixie.vm.numbers import Integer, Float +from pixie.vm.numbers import Integer, Float, BigInteger, Ratio from pixie.vm.string import String from pixie.vm.keyword import Keyword from pixie.vm.util import unicode_to_utf8 @@ -359,6 +359,12 @@ def ffi_get_value(self, ptr): return Float(casted[0]) def ffi_set_value(self, ptr, val): + if isinstance(val, Integer): + val = Float(float(val.int_val())) + elif isinstance(val, BigInteger): + val = Float(val.bigint_val().tofloat()) + elif isinstance(val, Ratio): + val = Float(val.numerator() / float(val.denominator())) casted = rffi.cast(rffi.DOUBLEP, ptr) casted[0] = rffi.cast(rffi.DOUBLE, val.float_val()) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index eac2878f..4505840a 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -302,6 +302,8 @@ def to_float(x): return x if isinstance(x, Ratio): return rt.wrap(x.numerator() / float(x.denominator())) + if isinstance(x, Integer): + return rt.wrap(float(x.int_val())) if isinstance(x, BigInteger): return rt.wrap(x.bigint_val().tofloat()) assert False diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index beaaa079..f2d997c7 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -58,3 +58,10 @@ (t/deftest test-size (t/assert= (pixie.ffi/struct-size (pixie.ffi/c-struct "struct" 1234 [])) 1234)) + + +(t/deftest test-double-coercion + (t/assert= (m/sin 1) (m/sin 1.0)) + (let [big (reduce * 1 (range 1 100))] + (t/assert= (m/sin big) (m/sin (float big)))) + (t/assert= (m/sin (/ 1 2)) (m/sin (float (/ 1 2))))) From a007eee58a488f3abe806c28ef9b93b7309bf194 Mon Sep 17 00:00:00 2001 From: Philipp Rustemeier Date: Thu, 15 Oct 2015 13:14:08 +0200 Subject: [PATCH 819/909] Use numbers.to_float to do the coercion --- pixie/vm/libs/ffi.py | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index eb24e44d..57b53184 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -8,7 +8,7 @@ import pixie.vm.rt as rt from rpython.rtyper.lltypesystem import rffi, lltype, llmemory from pixie.vm.primitives import nil, true, false -from pixie.vm.numbers import Integer, Float, BigInteger, Ratio +from pixie.vm.numbers import to_float, Integer, Float, BigInteger, Ratio from pixie.vm.string import String from pixie.vm.keyword import Keyword from pixie.vm.util import unicode_to_utf8 @@ -359,12 +359,8 @@ def ffi_get_value(self, ptr): return Float(casted[0]) def ffi_set_value(self, ptr, val): - if isinstance(val, Integer): - val = Float(float(val.int_val())) - elif isinstance(val, BigInteger): - val = Float(val.bigint_val().tofloat()) - elif isinstance(val, Ratio): - val = Float(val.numerator() / float(val.denominator())) + if isinstance(val, Integer) or isinstance(val, BigInteger) or isinstance(val, Ratio): + val = to_float(val) casted = rffi.cast(rffi.DOUBLEP, ptr) casted[0] = rffi.cast(rffi.DOUBLE, val.float_val()) From f0faaabb58027a3ca05ed527598d11c42f31b091 Mon Sep 17 00:00:00 2001 From: Philipp Rustemeier Date: Thu, 15 Oct 2015 13:49:01 +0200 Subject: [PATCH 820/909] Drop explicit type check (to_float covers this already). --- pixie/vm/libs/ffi.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 57b53184..193b56ce 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -359,8 +359,7 @@ def ffi_get_value(self, ptr): return Float(casted[0]) def ffi_set_value(self, ptr, val): - if isinstance(val, Integer) or isinstance(val, BigInteger) or isinstance(val, Ratio): - val = to_float(val) + val = to_float(val) casted = rffi.cast(rffi.DOUBLEP, ptr) casted[0] = rffi.cast(rffi.DOUBLE, val.float_val()) From 13b7c2d350466138820cc3d3cdff311a78958a7f Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 15 Oct 2015 14:36:15 +0100 Subject: [PATCH 821/909] this should make our builds faster enables the new infrastructure for travis builds --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 28ac48e9..490d9e48 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,4 @@ +sudo: false env: - JIT_OPTS='--opt=jit' TARGET_OPTS='target.py' - JIT_OPTS='' TARGET_OPTS='target.py' From fc04fba3567f8741e7e09ce4312e9bdc9df55d73 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Thu, 15 Oct 2015 15:00:48 +0100 Subject: [PATCH 822/909] use travis apt replacement --- .travis.yml | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 490d9e48..7176dc4b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -17,8 +17,14 @@ script: - make compile_tests - make run_built_tests -before_install: - - sudo apt-get install libffi-dev libedit-dev libboost-all-dev zlib1g-dev zlib-bin +addons: + apt: + packages: + - libffi-dev + - libedit-dev + - libboost-all-dev + - zlib1g-dev + - zlib-bin notifications: irc: "chat.freenode.net#pixie-lang" From 15aa7f8f1748f1014d9ffe53e6ef00b095212970 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 16 Oct 2015 16:16:20 +0100 Subject: [PATCH 823/909] fixes getting docs from aliased ns --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 434206a0..7d16a0c6 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1371,7 +1371,7 @@ and implements IAssociative, ILookup and IObject." :added "0.1"} [v] - (let [vr (resolve v) + (let [vr (resolve-in *ns* v) x (if vr @vr) doc (get (meta x) :doc) has-doc? (if doc true (get (meta x) :signatures))] From e4f4bf666b45da559aa4a88056cf2a9abfd3e184 Mon Sep 17 00:00:00 2001 From: alejandrodob Date: Mon, 19 Oct 2015 17:28:54 +0200 Subject: [PATCH 824/909] Add coll? function and tests --- pixie/stdlib.pxi | 1 + tests/pixie/tests/test-stdlib.pxi | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 434206a0..5a0b0729 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -867,6 +867,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn set? [v] (instance? PersistentHashSet v)) (defn map? [v] (satisfies? IMap v)) (defn fn? [v] (satisfies? IFn v)) +(defn coll? [v] (satisfies? IPersistentCollection v)) (defn indexed? [v] (satisfies? IIndexed v)) (defn counted? [v] (satisfies? ICounted v)) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 8334d9e0..3e77b9b8 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -304,6 +304,16 @@ (t/assert= (fn? "foo") false) (t/assert= (fn? (let [x 8] (fn [y] (+ x y)))) true)) +(t/deftest test-coll? + (t/assert= (coll? '()) true) + (t/assert= (coll? []) true) + (t/assert= (coll? {:foo "bar"}) true) + (t/assert= (coll? #{:foo :bar}) true) + (t/assert= (coll? #(%)) false) + (t/assert= (coll? :foo) false) + (t/assert= (coll? "foo") false) + (t/assert= (coll? 1) false)) + (t/deftest test-macro? (t/assert= (macro? and) true) (t/assert= (macro? or) true) From ca9150b8e02a849245bdaa8d9ff2f689f9afec2e Mon Sep 17 00:00:00 2001 From: Max Penet Date: Tue, 27 Oct 2015 22:54:57 +0100 Subject: [PATCH 825/909] add vary-meta --- pixie/stdlib.pxi | 6 ++++++ tests/pixie/tests/test-stdlib.pxi | 3 +++ 2 files changed, 9 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7d16a0c6..a9759630 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2717,3 +2717,9 @@ Calling this function on something that is not ISeqable returns a seq with that (if (satisfies? IComparable x) (-compare x y) (throw [::ComparisonError (str x " does not satisfy IComparable")]))) + +(defn vary-meta + [x f & args] + "Returns x with meta data updated with the application of f and args to it. +ex: (vary-meta x assoc :foo 42)" + (with-meta x (apply f (meta x) args))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 8334d9e0..167177e1 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -691,3 +691,6 @@ (t/assert= m (map-entry :a 1)) (t/assert= m [:a 1]) (t/assert= m '(:a 1)))) + +(t/deftest vary-meta + (t/assert= 42 (-> {} (vary-meta assoc :foo 42) meta :foo))) From 0e9d3e0f0aee9436daabf87691d7e32a2c5f0d0d Mon Sep 17 00:00:00 2001 From: Max Penet Date: Tue, 27 Oct 2015 23:39:56 +0100 Subject: [PATCH 826/909] add IRecord marker protocol and record? predicate --- pixie/stdlib.pxi | 19 ++++++++++++------- tests/pixie/tests/test-defrecord.pxi | 4 ++++ 2 files changed, 16 insertions(+), 7 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7d16a0c6..e65055e7 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -339,6 +339,11 @@ ([state] (finish-hash-state state)) ([state itm] (update-hash-unordered! state itm)))) +(def string-builder + (fn ^{:doc "Creates a reducing function that builds a string based on calling str on the transduced collection"} + ([] (-string-builder)) + ([sb] (str sb)) + ([sb item] (conj! sb item)))) (extend -str PersistentVector (fn [v] @@ -1303,7 +1308,8 @@ and implements IAssociative, ILookup and IObject." `(= (~field self) (~field other))) fields))) `(-hash [self] - (hash [~@field-syms]))] + (hash [~@field-syms])) + `IRecord] deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] `(do ~type-from-map ~deftype-decl))) @@ -2362,6 +2368,11 @@ Expands to calls to `extend-type`." [x] (-int x)) +(defprotocol IRecord) + +(defn record? + [x] + (satisfies? IRecord x)) (defmacro for {:doc "A list comprehension for the bindings." @@ -2418,12 +2429,6 @@ Expands to calls to `extend-type`." ([result] result) ([result _] (inc result))) -(defn string-builder - "Creates a reducing function that builds a string based on calling str on the transduced collection" - ([] (-string-builder)) - ([sb] (str sb)) - ([sb item] (conj! sb item))) - (defn dispose! "Finalizes use of the object by cleaning up resources used by the object" [x] diff --git a/tests/pixie/tests/test-defrecord.pxi b/tests/pixie/tests/test-defrecord.pxi index bc63724a..29844e6e 100644 --- a/tests/pixie/tests/test-defrecord.pxi +++ b/tests/pixie/tests/test-defrecord.pxi @@ -12,9 +12,13 @@ (t/deftest test-satisfies (foreach [t [t1 t2 t3 t4 t5]] (t/assert= t t) + (t/assert (satisfies? IRecord t)) (t/assert (satisfies? IAssociative t)) (t/assert (satisfies? ILookup t)))) +(t/deftest test-record-pred + (t/assert (record? t1))) + (t/deftest test-eq (t/assert= t1 t2) (t/assert= t2 t3) From 93d8426a763dcf1e6c81ba23231cd069679b33b0 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Wed, 28 Oct 2015 09:10:45 +0100 Subject: [PATCH 827/909] docstring --- pixie/stdlib.pxi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index e65055e7..0c7fa28e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2371,6 +2371,8 @@ Expands to calls to `extend-type`." (defprotocol IRecord) (defn record? + {:doc "Returns true if x implements IRecord" + :since "0.1"} [x] (satisfies? IRecord x)) From 854d838c6b51e4a5ad994ddc871872810a65739e Mon Sep 17 00:00:00 2001 From: Max Penet Date: Wed, 28 Oct 2015 09:14:15 +0100 Subject: [PATCH 828/909] improve metadata --- pixie/stdlib.pxi | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a9759630..614befd6 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2719,7 +2719,9 @@ Calling this function on something that is not ISeqable returns a seq with that (throw [::ComparisonError (str x " does not satisfy IComparable")]))) (defn vary-meta - [x f & args] - "Returns x with meta data updated with the application of f and args to it. + {:doc "Returns x with meta data updated with the application of f and args to it. ex: (vary-meta x assoc :foo 42)" + :signatures [[x f & args]] + :added "0.1"} + [x f & args] (with-meta x (apply f (meta x) args))) From 499430529b04f6375d9406dfa3be630cee459eec Mon Sep 17 00:00:00 2001 From: Max Penet Date: Wed, 28 Oct 2015 10:12:34 +0100 Subject: [PATCH 829/909] deftest compile down to a function with the same name, so it clashes with the stdlib var --- tests/pixie/tests/test-stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 167177e1..217784d1 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -692,5 +692,5 @@ (t/assert= m [:a 1]) (t/assert= m '(:a 1)))) -(t/deftest vary-meta +(t/deftest test-vary-meta (t/assert= 42 (-> {} (vary-meta assoc :foo 42) meta :foo))) From d613d130f321548341bc0cfa93863a3b654ea57f Mon Sep 17 00:00:00 2001 From: Max Penet Date: Wed, 28 Oct 2015 15:08:52 +0100 Subject: [PATCH 830/909] add metadata support to records --- pixie/stdlib.pxi | 16 ++++++++++++++-- tests/pixie/tests/test-defrecord.pxi | 4 ++++ 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 00cbf29e..ab0cdc3d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1281,6 +1281,7 @@ and implements IAssociative, ILookup and IObject." fields (transduce (map (comp keyword name)) conj field-syms) type-from-map `(defn ~map-ctor-name [m] (apply ~ctor-name (map #(get m %) ~fields))) + meta-gs (gensym "meta") default-bodies ['IAssociative (-make-record-assoc-body ctor-name fields) @@ -1299,6 +1300,13 @@ and implements IAssociative, ILookup and IObject." [k `(get-field ~self-nm ~k-nm)]) fields) not-found#))) + + 'IMeta + `(-with-meta [self# ~meta-gs] + (new ~nm + ~@(conj field-syms meta-gs))) + `(-meta [self#] __meta) + 'IObject `(-str [self#] (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (%2 self#)) {} ~fields) ">")) @@ -1310,9 +1318,13 @@ and implements IAssociative, ILookup and IObject." `(-hash [self] (hash [~@field-syms])) `IRecord] - deftype-decl `(deftype ~nm ~fields ~@default-bodies ~@body)] + deftype-decl `(deftype ~nm ~(conj fields '__meta) ~@default-bodies ~@body) + ctor `(defn ~ctor-name ~field-syms + (new ~nm + ~@(conj field-syms nil)))] `(do ~type-from-map - ~deftype-decl))) + ~deftype-decl + ~ctor))) (defn print {:doc "Prints the arguments, seperated by spaces." diff --git a/tests/pixie/tests/test-defrecord.pxi b/tests/pixie/tests/test-defrecord.pxi index 29844e6e..46a48d99 100644 --- a/tests/pixie/tests/test-defrecord.pxi +++ b/tests/pixie/tests/test-defrecord.pxi @@ -53,3 +53,7 @@ (t/assert (contains? t :one)) (t/assert (contains? t :two)) (t/assert (contains? t :three)))) + +(t/deftest test-record-metadata + (t/assert= nil (meta t1)) + (t/assert= :foo (-> t1 (with-meta :foo) meta))) From 62cacaca12be3005f722d427161d5a4933ce8798 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Thu, 29 Oct 2015 08:42:56 +0100 Subject: [PATCH 831/909] add *-> threading macros --- pixie/stdlib.pxi | 144 +++++++++++++++++++++++------- tests/pixie/tests/test-macros.pxi | 46 +++++++++- 2 files changed, 156 insertions(+), 34 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ab0cdc3d..3cd4f707 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -226,7 +226,6 @@ ([val coll] (transduce (interpose val) conj coll)))) - (def preserving-reduced (fn preserving-reduced [rf] (fn pr-inner [a b] @@ -480,38 +479,6 @@ (let [nm (with-meta nm (assoc (meta nm) :private true))] (cons `defn (cons nm rest)))) -(defmacro -> - {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next." - :examples [["(-> 3 inc inc)" nil 5] - ["(-> \"James\" (str \" is \" \"awesome \") (str \"(and stuff)\" \"!\"))" nil "James is awesome (and stuff)!"]] - :signatures [[x & forms]] - :added "0.1"} - [x & forms] - (loop [x x, forms forms] - (if forms - (let [form (first forms) - threaded (if (seq? form) - (with-meta `(~(first form) ~x ~@(next form)) (meta form)) - (list form x))] - (recur threaded (next forms))) - x))) - -(defmacro ->> - {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next." - :examples [["(->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \"))" nil "you know what? we like James"] - ["(->> 5 (range) (map inc) seq)" nil (1 2 3 4 5)]] - :signatures [[x & forms]] - :added "0.1"} - [x & forms] - (loop [x x, forms forms] - (if forms - (let [form (first forms) - threaded (if (seq? form) - (with-meta `(~(first form) ~@(next form) ~x) (meta form)) - (list form x))] - (recur threaded (next forms))) - x))) - (defn not {:doc "Inverts the input, if a truthy value is supplied, returns false, otherwise returns true" @@ -2640,6 +2607,117 @@ Calling this function on something that is not ISeqable returns a seq with that [x] (instance? Bool x)) +(defmacro -> + {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next." + :examples [["(-> 3 inc inc)" nil 5] + ["(-> \"James\" (str \" is \" \"awesome \") (str \"(and stuff)\" \"!\"))" nil "James is awesome (and stuff)!"]] + :signatures [[x & forms]] + :added "0.1"} + [x & forms] + (loop [x x, forms forms] + (if forms + (let [form (first forms) + threaded (if (seq? form) + (with-meta `(~(first form) ~x ~@(next form)) (meta form)) + (list form x))] + (recur threaded (next forms))) + x))) + +(defmacro ->> + {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next." + :examples [["(->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \"))" nil "you know what? we like James"] + ["(->> 5 (range) (map inc) seq)" nil (1 2 3 4 5)]] + :signatures [[x & forms]] + :added "0.1"} + [x & forms] + (loop [x x, forms forms] + (if forms + (let [form (first forms) + threaded (if (seq? form) + (with-meta `(~(first form) ~@(next form) ~x) (meta form)) + (list form x))] + (recur threaded (next forms))) + x))) + +(defmacro some-> + {:doc "When expr is not nil, threads it into the first form (via ->), + and when that result is not nil, through the next etc" + :signatures [[expr & forms]] + :added "0.1"} + [expr & forms] + (let [g (gensym) + steps (map (fn [step] `(if (nil? ~g) nil (-> ~g ~step))) + forms)] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro some->> + {:doc "When expr is not nil, threads it into the first form (via ->>), + and when that result is not nil, through the next etc" + :signatures [[x & forms]] + :added "0,1"} + [expr & forms] + (let [g (gensym) + steps (map (fn [step] `(if (nil? ~g) nil (->> ~g ~step))) + forms)] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro cond-> + {:added "0.1" + :signatures [[expr & clauses]] + :doc "Takes an expression and a set of test/form pairs. Threads expr (via ->) + through each form for which the corresponding test + expression is true. Note that, unlike cond branching, cond-> threading does + not short circuit after the first true test expression."} + [expr & clauses] + (assert (even? (count clauses))) + (let [g (gensym) + steps (map (fn [[test step]] `(if ~test (-> ~g ~step) ~g)) + (partition 2 clauses))] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro cond->> + {:doc "Takes an expression and a set of test/form pairs. Threads expr (via ->>) + through each form for which the corresponding test expression + is true. Note that, unlike cond branching, cond->> threading does not short circuit + after the first true test expression." + :signatures [[expr & clauses]] + :added "0.1"} + [expr & clauses] + (assert (even? (count clauses))) + (let [g (gensym) + steps (map (fn [[test step]] `(if ~test (->> ~g ~step) ~g)) + (partition 2 clauses))] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro as-> + {:doc "Binds name to expr, evaluates the first form in the lexical context + of that binding, then binds name to that result, repeating for each + successive form, returning the result of the last form." + :signatures [[expr name & forms]] + :added "0,1"} + [expr name & forms] + `(let [~name ~expr + ~@(interleave (repeat name) (butlast forms))] + ~(if (empty? forms) + name + (last forms)))) + (defprotocol IComparable (-compare [x y] "Compare to objects returing 0 if the same -1 with x is logically smaller than y and 1 if x is logically larger")) diff --git a/tests/pixie/tests/test-macros.pxi b/tests/pixie/tests/test-macros.pxi index 5678a1ee..238408d3 100644 --- a/tests/pixie/tests/test-macros.pxi +++ b/tests/pixie/tests/test-macros.pxi @@ -5,4 +5,48 @@ (let [x 10 k :boop] (t/assert= (-eq `{:x ~x} {:x 10}) true) (t/assert= (-eq `{~k ~x} {:boop 10}) true) - (t/assert= (-eq `{:x {:y ~x}} {:x {:y 10}}) true))) \ No newline at end of file + (t/assert= (-eq `{:x {:y ~x}} {:x {:y 10}}) true))) + +(def constantly-nil (constantly nil)) + +(t/deftest some->test + (t/assert (nil? (some-> nil))) + (t/assert (= 0 (some-> 0))) + (t/assert (= -1 (some-> 1 (- 2)))) + (t/assert (nil? (some-> 1 constantly-nil (- 2))))) + +(t/deftest some->>test + (t/assert (nil? (some->> nil))) + (t/assert (= 0 (some->> 0))) + (t/assert (= 1 (some->> 1 (- 2)))) + (t/assert (nil? (some->> 1 constantly-nil (- 2))))) + +(t/deftest cond->test + (t/assert (= 0 (cond-> 0))) + (t/assert (= -1 (cond-> 0 true inc true (- 2)))) + (t/assert (= 0 (cond-> 0 false inc))) + (t/assert (= -1 (cond-> 1 true (- 2) false inc)))) + +(t/deftest cond->>test + (t/assert (= 0 (cond->> 0))) + (t/assert (= 1 (cond->> 0 true inc true (- 2)))) + (t/assert (= 0 (cond->> 0 false inc))) + (t/assert (= 1 (cond->> 1 true (- 2) false inc)))) + +(t/deftest as->test + (t/assert (= 0 (as-> 0 x))) + (t/assert (= 1 (as-> 0 x (inc x)))) + (t/assert (= 2 (as-> [0 1] x + (map inc x) + (reverse x) + (first x))))) + +(t/deftest threading-loop-recur + (t/assert (nil? (loop [] + (as-> 0 x + (when-not (zero? x) + (recur)))))) + (t/assert (nil? (loop [x nil] (some-> x recur)))) + (t/assert (nil? (loop [x nil] (some->> x recur)))) + (t/assert (= 0 (loop [x 0] (cond-> x false recur)))) + (t/assert (= 0 (loop [x 0] (cond->> x false recur))))) From 7414496afbc2cf497bb3422851174b05cc648af3 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Fri, 30 Oct 2015 11:59:45 +0100 Subject: [PATCH 832/909] Implement IReduce on Records --- pixie/stdlib.pxi | 13 +++++++++++++ tests/pixie/tests/test-defrecord.pxi | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 3cd4f707..cd94876a 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1268,6 +1268,19 @@ and implements IAssociative, ILookup and IObject." fields) not-found#))) + 'IReduce + `(-reduce [self# f# init#] + (loop [fields# ~fields + acc# init#] + (if-let [field# (first fields#)] + (let [acc# (f# acc# (map-entry field# + (get-field self# + field#)))] + (if (reduced? acc#) + @acc# + (recur (next fields#) acc#))) + acc#))) + 'IMeta `(-with-meta [self# ~meta-gs] (new ~nm diff --git a/tests/pixie/tests/test-defrecord.pxi b/tests/pixie/tests/test-defrecord.pxi index 46a48d99..e45788ea 100644 --- a/tests/pixie/tests/test-defrecord.pxi +++ b/tests/pixie/tests/test-defrecord.pxi @@ -57,3 +57,9 @@ (t/deftest test-record-metadata (t/assert= nil (meta t1)) (t/assert= :foo (-> t1 (with-meta :foo) meta))) + + +(t/deftest ireduce [] + (t/assert= [[:one 1] [:two 2] [:three 3]] (reduce conj [] t1)) + (t/assert= [1 2 3] (vals t1)) + (t/assert= [:one :two :three] (keys t1))) From 81cdc2df08ab710630136391e6bb461c33c553c3 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Fri, 30 Oct 2015 12:35:24 +0100 Subject: [PATCH 833/909] allows kw/map/set invocations with default value --- pixie/stdlib.pxi | 15 ++++++++++----- tests/pixie/tests/collections/test-maps.pxi | 5 +++-- tests/pixie/tests/collections/test-sets.pxi | 4 +++- tests/pixie/tests/test-keywords.pxi | 3 ++- 4 files changed, 18 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ab0cdc3d..e47f7498 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1061,17 +1061,22 @@ If further arguments are passed, invokes the method named by symbol, passing the (extend -repr Symbol -str) -(extend -invoke Keyword (fn [k m] (-val-at m k nil))) -(extend -invoke PersistentHashMap (fn [m k] (-val-at m k nil))) -(extend -invoke PersistentHashSet (fn [m k] (-val-at m k nil))) - (defn get {:doc "Get an element from a collection implementing ILookup, return nil or the default value if not found." :added "0.1"} ([mp k] (get mp k nil)) ([mp k not-found] - (-val-at mp k not-found))) + (-val-at mp k not-found))) + +(extend -invoke Keyword (fn + ([k m not-found] + (-val-at m k not-found)) + ([k m] + (-val-at m k nil)))) +(extend -invoke PersistentHashMap get) +(extend -invoke PersistentHashSet get) + (defn get-in {:doc "Get a value from a nested collection at the \"path\" given by the keys." diff --git a/tests/pixie/tests/collections/test-maps.pxi b/tests/pixie/tests/collections/test-maps.pxi index a4072fe1..246e4af5 100644 --- a/tests/pixie/tests/collections/test-maps.pxi +++ b/tests/pixie/tests/collections/test-maps.pxi @@ -25,14 +25,15 @@ (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) - (t/deftest map-val-at-and-invoke (let [m {:a 1, :b 2, :c 3}] (foreach [e m] (t/assert= (get m (key e)) (val e)) (t/assert= (m (key e)) (val e))) (t/assert= (get m :d) nil) - (t/assert= (m :d) nil))) + (t/assert= (m :d) nil) + (t/assert= (m :d :foo) :foo) + (t/assert= (:d m :foo) :foo))) (t/deftest map-without (let [m {:a 1 :b 2}] diff --git a/tests/pixie/tests/collections/test-sets.pxi b/tests/pixie/tests/collections/test-sets.pxi index 376f95bf..7b91c342 100644 --- a/tests/pixie/tests/collections/test-sets.pxi +++ b/tests/pixie/tests/collections/test-sets.pxi @@ -58,7 +58,9 @@ (t/assert= (s 3) 3) (t/assert= (s -1) nil) - (t/assert= (s 4) nil))) + (t/assert= (s 4) nil) + (t/assert= (s :d :foo) :foo) + (t/assert= (:d s :foo) :foo))) (t/deftest test-has-meta (let [m {:has-meta true} diff --git a/tests/pixie/tests/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi index 9b4357e7..06e2deb7 100644 --- a/tests/pixie/tests/test-keywords.pxi +++ b/tests/pixie/tests/test-keywords.pxi @@ -7,7 +7,8 @@ (t/assert= (:b m) 2) (t/assert= (:c m) 3) - (t/assert= (:d m) nil))) + (t/assert= (:d m) nil) + (t/assert= (:d m :foo) :foo))) (t/deftest keyword-namespace (t/assert= (namespace :foo/bar) "foo") From d92c6f88ab9afa150091adb6db4fd810a761afc3 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Mon, 2 Nov 2015 09:51:07 +0100 Subject: [PATCH 834/909] adds keep-indexed, map-indexed and reductions functions --- pixie/stdlib.pxi | 73 +++++++++++++++++++++++++++++++ tests/pixie/tests/test-stdlib.pxi | 13 ++++++ 2 files changed, 86 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 3023ce0e..19c20a00 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1727,6 +1727,79 @@ not enough elements were present." (cons (take n s) (partitionf f (drop n s))))))) +(defn map-indexed + {:doc "Returns a lazy sequence consisting of the + result of applying f to 0 and the first item of coll, followed by + applying f to 1 and the second item in coll, etc, until coll is + exhausted. Thus function f should accept 2 arguments, index and + item. Returns a stateful transducer when no collection is provided." + :added "0.1" + :signatures [[f] [f coll]]} + ([f] + (fn [rf] + (let [i (atom -1) + rrf (preserving-reduced rf)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (rrf result (f (swap! i inc) input))))))) + ([f coll] + (let [mapi (fn mapi [i coll] + (lazy-seq + (when-let [s (seq coll)] + (cons (f i (first s)) + (mapi (inc i) (rest s))))))] + (mapi 0 coll)))) + +(defn keep-indexed + {:doc "Returns a lazy sequence of the non-nil + results of (f index item). Note, this means false return values will + be included. f must be free of side-effects. Returns a stateful + transducer when no collection is provided." + :signatures [[f] [f coll]] + :added "0.1"} + ([f] + (fn [rf] + (let [iv (atom -1) + rrf (preserving-reduced)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [i (swap! iv inc) + v (f i input)] + (if (nil? v) + result + (rrf result v)))))))) + ([f coll] + (let [keepi (fn keepi [i coll] + (lazy-seq + (when-let [s (seq coll)] + (let [x (f i (first s))] + (if (nil? x) + (keepi (inc i) (rest s)) + (cons x (keepi (inc i) (rest s))))))))] + (keepi 0 coll)))) + +(defn reductions + {:doc "Returns a lazy seq of the intermediate values of the + reduction (as per reduce) of coll by f, starting with init." + :added "0.1" + :signatures [[f coll] [f init coll]]} + ([f coll] + (lazy-seq + (if-let [s (seq coll)] + (reductions f (first s) (rest s)) + (list (f))))) + ([f init coll] + (if (reduced? init) + (list @init) + (cons init + (lazy-seq + (when-let [s (seq coll)] + (reductions f (f init (first s)) (rest s)))))))) + (defn destructure [binding expr] (cond (symbol? binding) [binding expr] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index f3dc0a0a..ca71c2a4 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -19,6 +19,19 @@ (t/assert= (mapcat identity []) []) (t/assert= (mapcat first [[[1 2]] [[3] [:not :present]] [[4 5 6]]]) [1 2 3 4 5 6])) +(t/deftest test-indexed + (t/assert= (map-indexed (fn [& xs] xs) []) []) + (t/assert= (map-indexed (fn [& xs] xs) [:a :b]) [[0 :a] [1 :b]]) + (t/assert= (transduce (map-indexed (fn [& xs] xs)) conj [:a :b]) [[0 :a] [1 :b]])) + +(t/deftest test-reductions + (t/assert= (reductions + nil) + [0]) + (t/assert= (reductions + [1 2 3 4 5]) + [1 3 6 10 15]) + (t/assert= (reductions + 10 [1 2 3 4 5]) + [10 11 13 16 20 25])) + (t/deftest test-str (t/assert= (str nil) "nil") (t/assert= (str true) "true") From 1d76fc3b743e6b4f362e01d25eadadf4c6750b18 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Mon, 2 Nov 2015 10:26:25 +0100 Subject: [PATCH 835/909] fix keep-indexed transducer and add tests --- pixie/stdlib.pxi | 2 +- tests/pixie/tests/test-stdlib.pxi | 10 +++++++++- 2 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 19c20a00..285ecad8 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1762,7 +1762,7 @@ not enough elements were present." ([f] (fn [rf] (let [iv (atom -1) - rrf (preserving-reduced)] + rrf (preserving-reduced rf)] (fn ([] (rf)) ([result] (rf result)) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index ca71c2a4..36b6d09f 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -22,7 +22,15 @@ (t/deftest test-indexed (t/assert= (map-indexed (fn [& xs] xs) []) []) (t/assert= (map-indexed (fn [& xs] xs) [:a :b]) [[0 :a] [1 :b]]) - (t/assert= (transduce (map-indexed (fn [& xs] xs)) conj [:a :b]) [[0 :a] [1 :b]])) + (t/assert= (transduce (map-indexed (fn [& xs] xs)) conj [:a :b]) [[0 :a] [1 :b]]) + + (t/assert= (keep-indexed (constantly true) []) []) + (t/assert= (keep-indexed (constantly nil) []) []) + (t/assert= (keep-indexed (fn [i x] [i x]) [:a :b]) [[0 :a] [1 :b]]) + + (t/assert= (transduce (keep-indexed (constantly true)) conj []) []) + (t/assert= (transduce (keep-indexed (constantly nil)) conj []) []) + (t/assert= (transduce (keep-indexed (fn [i x] [i x])) conj [:a :b]) [[0 :a] [1 :b]])) (t/deftest test-reductions (t/assert= (reductions + nil) From 373a44da19e67ca96a90ead02a84d5372b1f3b0e Mon Sep 17 00:00:00 2001 From: Max Penet Date: Mon, 2 Nov 2015 13:10:45 +0100 Subject: [PATCH 836/909] implement ICounted and ISeqable on records --- pixie/stdlib.pxi | 8 +++++++- tests/pixie/tests/test-defrecord.pxi | 8 ++++++++ 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 285ecad8..23ac2e67 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -11,7 +11,6 @@ (def printf (ffi-fn libc "printf" [CCharP] CInt :variadic? true)) (def getenv (ffi-fn libc "getenv" [CCharP] CCharP)) - (def libedit (ffi-library (str "libedit." pixie.platform/so-ext))) (def readline (ffi-fn libedit "readline" [CCharP] CCharP)) (def rand (ffi-fn libc "rand" [] CInt)) @@ -1286,6 +1285,13 @@ and implements IAssociative, ILookup and IObject." @acc# (recur (next fields#) acc#))) acc#))) + 'ICounted + `(-count [self] ~(count fields)) + + 'ISeqable + `(-seq [self#] + (map #(map-entry % (get-field self# %)) + ~fields)) 'IMeta `(-with-meta [self# ~meta-gs] diff --git a/tests/pixie/tests/test-defrecord.pxi b/tests/pixie/tests/test-defrecord.pxi index e45788ea..6c404bf5 100644 --- a/tests/pixie/tests/test-defrecord.pxi +++ b/tests/pixie/tests/test-defrecord.pxi @@ -63,3 +63,11 @@ (t/assert= [[:one 1] [:two 2] [:three 3]] (reduce conj [] t1)) (t/assert= [1 2 3] (vals t1)) (t/assert= [:one :two :three] (keys t1))) + +(t/deftest icounted [] + (t/assert= (count t1) 3)) + +(t/deftest seqable [] + (t/assert= (key (first (seq t1))) :one) + (t/assert= (val (first (seq t1))) 1) + (t/assert (instance? LazySeq (seq t1)))) From 3cbcec4dec200e9053dfafd9da39744ca9b1ec6e Mon Sep 17 00:00:00 2001 From: Max Penet Date: Mon, 2 Nov 2015 13:35:24 +0100 Subject: [PATCH 837/909] check for correct EOF value - fix #413 --- pixie/io-blocking.pxi | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index 4ac84eb3..c6547486 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -13,6 +13,7 @@ (def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) (def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) (def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) +(def EOF -1) (deftype FileStream [fp] IInputStream @@ -85,7 +86,7 @@ ([cnt chr] (assert (integer? chr)) (let [written (write-byte fp chr)] - (if (= written 0) + (if (= written EOF) (reduced cnt) (+ cnt written))))))) From 2d8b2a18229b38e4125101d66c8515e49fa2add5 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Mon, 2 Nov 2015 15:32:45 +0100 Subject: [PATCH 838/909] add meta data support to atoms --- pixie/vm/atom.py | 18 +++++++++++++++++- tests/pixie/tests/test-stdlib.pxi | 5 +++-- 2 files changed, 20 insertions(+), 3 deletions(-) diff --git a/pixie/vm/atom.py b/pixie/vm/atom.py index ae42ab2d..77ecd50d 100644 --- a/pixie/vm/atom.py +++ b/pixie/vm/atom.py @@ -10,8 +10,15 @@ class Atom(object.Object): def type(self): return Atom._type - def __init__(self, boxed_value): + def with_meta(self, meta): + return Atom(self._boxed_value, meta) + + def meta(self): + return self._meta + + def __init__(self, boxed_value, meta=nil): self._boxed_value = boxed_value + self._meta = meta @extend(proto._reset_BANG_, Atom) @@ -26,6 +33,15 @@ def _deref(self): assert isinstance(self, Atom) return self._boxed_value +@extend(proto._meta, Atom) +def _meta(self): + assert isinstance(self, Atom) + return self.meta() + +@extend(proto._with_meta, Atom) +def _with_meta(self, meta): + assert isinstance(self, Atom) + return self.with_meta(meta) @as_var("atom") def atom(val=nil): diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 36b6d09f..bf7f9323 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -683,11 +683,12 @@ (t/assert= 5 ((comp inc inc inc inc) 1)) (t/assert= :xyz ((comp) :xyz))) -(t/deftest test-swap-reset +(t/deftest test-atom (let [a (atom 0)] (t/assert= 1 (swap! a inc)) (t/assert= 2 (swap! a inc)) - (t/assert= 3 (reset! a 3)))) + (t/assert= 3 (reset! a 3)) + (t/assert= :bar (-> a (with-meta {:foo :bar}) meta :foo)))) (t/deftest pre-post-conds (let [f (fn ([a] {:pre [(even? a)] :post [(= % 6)]} (/ a 2)) From 215c90db9703b439ce57c2455daf790d058c074f Mon Sep 17 00:00:00 2001 From: Max Penet Date: Tue, 3 Nov 2015 11:07:09 +0100 Subject: [PATCH 839/909] add conj support to records (implements IPersistentCollection) --- pixie/stdlib.pxi | 16 ++++++++++++++++ tests/pixie/tests/test-defrecord.pxi | 7 ++++++- 2 files changed, 22 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 23ac2e67..acf7ca7e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1293,6 +1293,22 @@ and implements IAssociative, ILookup and IObject." (map #(map-entry % (get-field self# %)) ~fields)) + 'IPersistentCollection + `(-conj [self# x] + (cond + (instance? MapEntry x) + (assoc self# (key x) (val x)) + (instance? PersistentVector x) + (if (= (count x) 2) + (assoc self# (first x) (second x)) + (throw + [:pixie.stdlib/InvalidArgumentException + "Vector arg to record conj must be a pair"])))) + + `(-disj [self# x] + (throw [:pixie.stdlib/NotImplementedException + "disj is not supported on defrecords"])) + 'IMeta `(-with-meta [self# ~meta-gs] (new ~nm diff --git a/tests/pixie/tests/test-defrecord.pxi b/tests/pixie/tests/test-defrecord.pxi index 6c404bf5..c2ae7a8c 100644 --- a/tests/pixie/tests/test-defrecord.pxi +++ b/tests/pixie/tests/test-defrecord.pxi @@ -14,7 +14,8 @@ (t/assert= t t) (t/assert (satisfies? IRecord t)) (t/assert (satisfies? IAssociative t)) - (t/assert (satisfies? ILookup t)))) + (t/assert (satisfies? ILookup t)) + (t/assert (satisfies? IPersistentCollection t)))) (t/deftest test-record-pred (t/assert (record? t1))) @@ -54,6 +55,10 @@ (t/assert (contains? t :two)) (t/assert (contains? t :three)))) +(t/deftest test-ipersistentcoll + (t/assert= 11 (-> t1 (conj [:one 11]) :one)) + (t/assert= 11 (-> t1 (conj (map-entry :one 11)) :one))) + (t/deftest test-record-metadata (t/assert= nil (meta t1)) (t/assert= :foo (-> t1 (with-meta :foo) meta))) From a9c0f667bb1891c7596cf589917e9c14608043f7 Mon Sep 17 00:00:00 2001 From: John Gabriele Date: Wed, 4 Nov 2015 11:58:11 -0500 Subject: [PATCH 840/909] fix typo in metadata --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index acf7ca7e..a64a2046 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -701,7 +701,7 @@ returns true" (defn dissoc {:doc "Removes the value associated with the keys from the collection" :signatures [[m] [m & ks]] - :addded "0.1"} + :added "0.1"} ([m] m) ([m & ks] (reduce -dissoc m ks))) From 89cc9e849d679e92747ef7b809a3d828962d09ec Mon Sep 17 00:00:00 2001 From: John Gabriele Date: Wed, 4 Nov 2015 12:59:49 -0500 Subject: [PATCH 841/909] My attempt to improve docstring for `select-keys`. --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a64a2046..b55667bb 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -918,7 +918,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (list (-key self) (-val self)))) (defn select-keys - {:doc "Produces a map with only the values in m contained in key-seq"} + {:doc "Returns a map containing only the entries of `m` where the entry's key is in `key-seq`."} [m key-seq] (with-meta (transduce From 6bf2e36e7d89e4fa7c09124f86346d229901636f Mon Sep 17 00:00:00 2001 From: Mark Reuter Date: Mon, 2 Nov 2015 13:18:49 -0600 Subject: [PATCH 842/909] Added Unchecked Math Operations Added Tests for Unchecked Math Operations Adjustments to unchecked math Made changes to the implementation of unchecked math in regards to: implementing them under their own protocol, mirrored the api of clojure, reduced implementations of the protocol to just integer and float Fixed errors in the tests that caused the build to fail --- pixie/stdlib.pxi | 38 ++++++++++++++++++++++++++++++ pixie/vm/numbers.py | 16 ++++++++++++- tests/pixie/tests/test-numbers.pxi | 4 ++++ 3 files changed, 57 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a64a2046..b0d5ec96 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -510,6 +510,30 @@ returns true" ([x y & args] (reduce -mul (-mul x y) args))) +(defn unchecked-add + {:doc "Adds the arguments, returning 0 if no arguments" + :signatures [[& args]] + :added "0.1"} + ([] 0) + ([x] x) + ([x y] (-unchecked-add x y)) + ([x y & args] + (reduce -unchecked-add (-unchecked-add x y) args))) + +(defn unchecked-subtract + ([] 0) + ([x] (-unchecked-subtract 0 x)) + ([x y] (-unchecked-subtract x y)) + ([x y & args] + (reduce -unchecked-subtract (-unchecked-subtract x y) args))) + +(defn unchecked-multiply + ([] 1) + ([x] x) + ([x y] (-unchecked-multiply x y)) + ([x y & args] + (reduce -unchecked-multiply (-unchecked-multiply x y) args))) + (defn / ([x] (-div 1 x)) ([x y] (-div x y)) @@ -610,6 +634,20 @@ returns true" [x] (- x 1)) +(defn unchecked-inc + {:doc "Increments x by one" + :signatures [[x]] + :added "0.1"} + [x] + (unchecked-add x 1)) + +(defn unchecked-dec + {:doc "Decrements x by one" + :signatures [[x]] + :added "0.1"} + [x] + (unchecked-subtract x 1)) + (defn empty? {:doc "returns true if the collection has no items, otherwise false" :signatures [[coll]] diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index 4505840a..ab80edbe 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -107,8 +107,12 @@ def ratio_read(obj): _num_eq = as_var("-num-eq")(DoublePolymorphicFn(u"-num-eq", IMath)) _num_eq.set_default_fn(wrap_fn(lambda a, b: false)) -as_var("MAX-NUMBER")(Integer(100000)) # TODO: set this to a real max number +IUncheckedMath = as_var("IUncheckedMath")(Protocol(u"IUncheckedMath")) +_unchecked_add = as_var("-unchecked-add")(DoublePolymorphicFn(u"-unchecked-add", IUncheckedMath)) +_unchecked_subtract = as_var("-unchecked-subtract")(DoublePolymorphicFn(u"-unchecked-subtract", IUncheckedMath)) +_unchecked_multiply = as_var("-unchecked-multiply")(DoublePolymorphicFn(u"-unchecked-multiply", IUncheckedMath)) +as_var("MAX-NUMBER")(Integer(100000)) # TODO: set this to a real max number num_op_template = """@extend({pfn}, {ty1}._type, {ty2}._type) def {pfn}_{ty1}_{ty2}(a, b): @@ -167,6 +171,16 @@ def define_num_ops(): define_num_ops() +def define_unchecked_num_ops(): + # maybe define get_val() instead of using tuples? + num_classes = [(Integer, "int_val"), (Float, "float_val")] + for (c1, conv1) in num_classes: + for (c2, conv2) in num_classes: + for (op, sym) in [("_unchecked_add", "+"), ("_unchecked_subtract", "-"), ("_unchecked_multiply", "*")]: + extend_num_op(op, c1, c2, conv1, sym, conv2) + +define_unchecked_num_ops() + bigint_ops_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) def _{pfn}_{ty1}_{ty2}(a, b): assert isinstance(a, {ty1}) and isinstance(b, {ty2}) diff --git a/tests/pixie/tests/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi index 28ab8b92..270ba742 100644 --- a/tests/pixie/tests/test-numbers.pxi +++ b/tests/pixie/tests/test-numbers.pxi @@ -73,3 +73,7 @@ (t/assert (-num-eq 1000000000000000000000N (* 10000000 10000000 10000000)))) + +(t/deftest test-wrap-around + (t/assert= Integer (type (unchecked-add 9223372036854775807 1))) + (t/assert (-num-eq -9223372036854775808 (unchecked-add 9223372036854775807 1)))) From b7db173bf3eeefa8c6e43eab8390c66c0293e168 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Fri, 6 Nov 2015 16:40:56 +0100 Subject: [PATCH 843/909] add pixie.walk --- pixie/walk.pxi | 113 ++++++++++++++++++++++++++++++++ tests/pixie/tests/test-walk.pxi | 59 +++++++++++++++++ 2 files changed, 172 insertions(+) create mode 100644 pixie/walk.pxi create mode 100644 tests/pixie/tests/test-walk.pxi diff --git a/pixie/walk.pxi b/pixie/walk.pxi new file mode 100644 index 00000000..cef60c93 --- /dev/null +++ b/pixie/walk.pxi @@ -0,0 +1,113 @@ +(ns pixie.walk) + +(defprotocol IWalk + (-walk [x f])) + +(extend-protocol IWalk + PersistentList + (-walk [x f] + (apply list (map f x))) + + Cons + (-walk [x f] + (cons (f (first x)) (map f (next x)))) + + IMapEntry + (-walk [x f] + (map-entry (f (key x)) (f (val x)))) + + PersistentVector + (-walk [x f] + (into [] (map f) x)) + + PersistentHashSet + (-walk [x f] + (into #{} (map f) x)) + + PersistentHashMap + (-walk [x f] + (into {} (map f) x)) + + IRecord + (-walk [x f] + (into x (map f) x)) + + ISeqable + (-walk [x f] + (map f x)) + + IObject + (-walk [x f] x) + + Nil + (-walk [x f] nil)) + +(defn walk + {:doc "Traverses form, an arbitrary data structure. f is a + function. Applies f to each element of form, building up a data + structure of the same type. Recognizes all Pixie data + structures. Consumes seqs." + :signatures [[f x]] + :added "0.1"} + [f x] + (-walk x f)) + +(defn postwalk + {:doc "Performs a depth-first, post-order traversal of form. Calls f on + each sub-form, uses f's return value in place of the original. + Recognizes all Pixie data structures." + :signatures [[f x]] + :added "0.1"} + [f x] + (f (walk (partial postwalk f) x))) + +(defn prewalk + {:doc "Like postwalk, but does pre-order traversal." + :added "0.1"} + [f x] + (walk (partial prewalk f) (f x))) + +(defn prewalk-replace + {:doc "Recursively transforms form by replacing + keys in smap with their values. Like `replace` but works on + any data structure. Does replacement at the root of the tree + first." + :signatures [[f x]] + :added "0.1"} + [smap x] + (prewalk (fn [x] (if (contains? smap x) (smap x) x)) x)) + +(defn postwalk-replace + {:doc "Recursively transforms form by replacing keys in smap with + their values. Like `replace` but works on any data structure. + Does replacement at the leaves of the tree first." + :signatures [[smap x]] + :added "0.1"} + [smap x] + (postwalk (fn [x] (if (contains? smap x) (smap x) x)) x)) + +(defn keywordize-keys + {:doc "Recursively transforms all map keys from strings to keywords." + :signatures [[m]] + :added "0.1"} + [m] + (let [f (fn [[k v]] (if (string? k) [(keyword k) v] [k v]))] + ;; only apply to maps + (postwalk (fn [x] (if (map? x) (into {} (map f x)) x)) m))) + +(defn stringify-keys + {:doc "Recursively transforms all map keys from keywords to strings." + :signatures [[m]] + :added "0.1"} + [m] + (let [f (fn [[k v]] (if (keyword? k) [(name k) v] [k v]))] + ;; only apply to maps + (postwalk (fn [x] (if (map? x) (into {} (map f x)) x)) m))) + +(defn macroexpand-all + {:doc "Recursively performs all possible macroexpansions in + form. For development use only." + :added "0.1" + :signatures [[x]]} + [x] + (prewalk (fn [x] (if (seq? x) (macroexpand-1 x) x)) x)) diff --git a/tests/pixie/tests/test-walk.pxi b/tests/pixie/tests/test-walk.pxi new file mode 100644 index 00000000..d7a0aa1a --- /dev/null +++ b/tests/pixie/tests/test-walk.pxi @@ -0,0 +1,59 @@ +(ns pixie.tests.test-walk + (:require [pixie.walk :as w] + [pixie.test :as t])) + +(t/deftest t-prewalk-replace + (t/assert (= (w/prewalk-replace {:a :b} [:a {:a :a} (list 3 :c :a)]) + [:b {:b :b} (list 3 :c :b)]))) + +(t/deftest t-postwalk-replace + (t/assert (= (w/postwalk-replace {:a :b} [:a {:a :a} (list 3 :c :a)]) + [:b {:b :b} (list 3 :c :b)]))) + +(t/deftest t-prewalk-order + (t/assert (= (let [a (atom [])] + (w/prewalk (fn [form] (swap! a conj form) form) + [1 2 {:a 3} (list 4 [5])]) + @a) + [[1 2 {:a 3} (list 4 [5])] + 1 2 {:a 3} [:a 3] :a 3 (list 4 [5]) + 4 [5] 5]))) + +(t/deftest t-postwalk-order + (t/assert (= (let [a (atom [])] + (w/postwalk (fn [form] (swap! a conj form) form) + [1 2 {:a 3} (list 4 [5])]) + @a) + [1 2 + :a 3 [:a 3] {:a 3} + 4 5 [5] (list 4 [5]) + [1 2 {:a 3} (list 4 [5])]]))) + +(defrecord Foo [a b c]) + +(t/deftest walk + "Checks that walk returns the correct result and type of collection" + (let [colls ['(1 2 3) + [1 2 3] + #{1 2 3} + {:a 1, :b 2, :c 3} + (->Foo 1 2 3)]] + (doseq [c colls] + (let [walked (w/walk identity c)] + (t/assert (= c walked)) + (t/assert (= (type c) (type walked))) + (if (or (map? c) + (record? c)) + (do + (t/assert (= (reduce + (vals (w/walk + #(map-entry + (key %) + (inc (val %))) + c))) + (reduce + (map (comp inc val) c))))) + (t/assert (= (reduce + (w/walk inc c)) + (reduce + (map inc c))))))))) + +(t/deftest t-stringify-keys + (t/assert (= (w/stringify-keys {:a 1, nil {:b 2 :c 3}, :d 4}) + {"a" 1, nil {"b" 2 "c" 3}, "d" 4}))) From 7c09904cb9039f37ab836d9b663122ccaeba67a0 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Wed, 11 Nov 2015 14:08:47 +0100 Subject: [PATCH 844/909] add csp/timout channel --- pixie/channels.pxi | 20 ++++++++++++++++++++ pixie/csp.pxi | 1 + tests/pixie/tests/test-csp.pxi | 15 +++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/pixie/channels.pxi b/pixie/channels.pxi index f930bf17..1f8067df 100644 --- a/pixie/channels.pxi +++ b/pixie/channels.pxi @@ -1,5 +1,7 @@ (ns pixie.channels (:require [pixie.stacklets :as st] + [pixie.uv :as uv] + [pixie.ffi :as ffi] [pixie.buffers :as b])) (defprotocol ICancelable @@ -143,6 +145,24 @@ false 0))))) +(defn timeout + "Returns a channel that will close after given delay in ms" + [ms] + (let [ch (chan) + timer (uv/uv_timer_t) + cb (atom nil)] + (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb + (fn [handle] + (try + (-close! ch) + (uv/uv_timer_stop timer) + (-dispose! @cb) + (catch ex + (println ex)))))) + (uv/uv_timer_init (uv/uv_default_loop) timer) + (uv/uv_timer_start timer @cb ms 0) + ch)) + (deftype AltHandler [atm f] ICancelable (-canceled? [this] diff --git a/pixie/csp.pxi b/pixie/csp.pxi index 94ca90d9..32c96c8c 100644 --- a/pixie/csp.pxi +++ b/pixie/csp.pxi @@ -4,6 +4,7 @@ [pixie.channels :as chans])) (def chan chans/chan) +(def timeout chans/timeout) (defn close! "Closes the channel, future writes will be rejected, future reads will diff --git a/tests/pixie/tests/test-csp.pxi b/tests/pixie/tests/test-csp.pxi index 0b4cc396..94a241bd 100644 --- a/tests/pixie/tests/test-csp.pxi +++ b/tests/pixie/tests/test-csp.pxi @@ -31,3 +31,18 @@ (assert= (alts! [c] :default 42) [:default 42]) (>! c 1) (assert= (alts! [c] :default 42) [c 1]))) + +(deftest test-timeout-channel + (let [ts [(timeout 300) + (timeout 200) + (timeout 100)]] + (-> (go + (loop [ts (set ts) + res []] + (if (empty? ts) + res + (let [[p _] (alts! ts)] + (recur (set (remove #{p} ts)) + (conj res p)))))) + Date: Thu, 12 Nov 2015 19:37:41 +0000 Subject: [PATCH 845/909] Improves the performance of range dramatically Calling seq on range produced a lazy sequence which has significant overhead. Now we give Range -first and -next properties, returning start and a new Range respectively. `last` is now more efficient on IIndexed collections --- pixie/stdlib.pxi | 35 ++++++++++++++++++++++++++--------- 1 file changed, 26 insertions(+), 9 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7d16a0c6..afd7b09e 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -878,9 +878,18 @@ If further arguments are passed, invokes the method named by symbol, passing the :signatures [[coll]] :added "0.1"} [coll] - (if (next coll) - (recur (next coll)) - (first coll))) + (cond + (satisfies? IIndexed coll) + (when (pos? (count coll)) + (nth coll (dec (count coll)))) + + (satisfies? ISeq coll) + (if (next coll) + (recur (next coll)) + (first coll)) + + (satisfies? ISeqable coll) + (recur (seq coll)))) (defn butlast {:doc "Returns all elements but the last from the collection." @@ -1911,18 +1920,26 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (if (cmp val stop) val not-found))) + ISeq + (-first [this] + (when (not= start stop) + start)) + (-next [this] + (let [i (+ step start)] + (when (or (and (> step 0) (< i stop)) + (and (< step 0) (> i stop)) + (and (= step 0))) + (range i stop step)))) ISeqable - (-seq [self] - (when (or (and (> step 0) (< start stop)) - (and (< step 0) (> start stop))) - (cons start (lazy-seq* #(range (+ start step) stop step)))))) + (-seq [self] self)) (extend -str Range (fn [v] - (-str (seq v)))) + (str "(" (transduce (interpose " ") string-builder v) ")"))) + (extend -repr Range (fn [v] - (-repr (seq v)))) + (str "(" (transduce (interpose " ") string-builder v) ")"))) (defn range {:doc "Returns a range of numbers." From b605646ca75cb43ea81f74970bca8a3ba8a4b9e8 Mon Sep 17 00:00:00 2001 From: Max Penet Date: Fri, 13 Nov 2015 16:42:02 +0100 Subject: [PATCH 846/909] add memoize --- pixie/stdlib.pxi | 17 +++++++++++++++++ tests/pixie/tests/test-stdlib.pxi | 4 ++++ 2 files changed, 21 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index b0d5ec96..100fff2c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2974,3 +2974,20 @@ ex: (vary-meta x assoc :foo 42)" :added "0.1"} [x f & args] (with-meta x (apply f (meta x) args))) + +(defn memoize + {:doc "Returns a memoized version of function f. The first call will + realize the return value and subsequent calls get the same value + from its cache." + :signatures [[f]] + :added "0.1"} + [f] + (let [cache (atom {})] + (fn [& args] + (let [argsv (vec args) + val (get @cache argsv ::not-found)] + (if (= val ::not-found) + (let [ret (apply f args)] + (swap! cache assoc argsv ret) + ret) + val))))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index bf7f9323..cb10ebeb 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -726,3 +726,7 @@ (t/deftest test-vary-meta (t/assert= 42 (-> {} (vary-meta assoc :foo 42) meta :foo))) + +(t/deftest test-memoize + (let [f (memoize rand)] + (t/assert= (f) (f)))) From 721cdf19afae4eb62276a912f6764c63bc6302ac Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 14 Nov 2015 00:42:40 +0000 Subject: [PATCH 847/909] fixes (rand-int 0) case --- pixie/stdlib.pxi | 4 +++- tests/pixie/tests/test-stdlib.pxi | 4 +++- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 7d16a0c6..a1604eb2 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -554,7 +554,9 @@ returns true" (defn rand-int {:doc "random integer between 0 (inclusive) and n (exclusive)"} [n] - (rem (rand) n)) + (if (zero? n) + 0 + (rem (rand) n))) (defn = {:doc "Returns true if all the arguments are equivalent. Otherwise, returns false. Uses diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 8334d9e0..968c7f1e 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -330,7 +330,9 @@ (t/deftest test-rand-int (let [vs (repeatedly 10 #(rand-int 4))] - (t/assert (every? #(and (>= % 0) (< % 4)) vs)))) + (t/assert (every? #(and (>= % 0) (< % 4)) vs))) + (let [vs (repeatedly 10 #(rand-int 0))] + (t/assert (every? zero? vs)))) (t/deftest test-some (t/assert= (some even? [2 4 6 8]) true) From a27d12cfdb2a3fe53eba8fa03c991afd500e5c8d Mon Sep 17 00:00:00 2001 From: John Gabriele Date: Mon, 16 Nov 2015 10:38:16 -0500 Subject: [PATCH 848/909] signature was missing from `into` :signatures --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 71853016..eab2357f 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -197,7 +197,7 @@ (-satisfies? p x)))) (def into (fn ^{:doc "Add the elements of `from` to the collection `to`." - :signatures [[to from]] + :signatures [[to from] [to xform from]] :added "0.1"} ([to from] (if (satisfies? IToTransient to) From ad325efdabfe85b6d63871f2d4a3a68023f3b040 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 27 Nov 2015 10:17:28 +0000 Subject: [PATCH 849/909] line-reader accepts a non buffered stream We create a buffered stream with a buffer size of one. --- pixie/io.pxi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/io.pxi b/pixie/io.pxi index 4143386e..16ec14be 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -87,6 +87,10 @@ (instance? BufferedInputStream input-stream) (-> input-stream ->LineReader) + + (satisfies? IInputStream input-stream) + (-> input-stream (buffered-input-stream 1) ->LineReader) + :else (throw [::Exception "Expected a LineReader, UTF8InputStream, or BufferedInputStream"]))) From 3d0af3fd5e3eeb99801b5a60a65d315390caf2cc Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Mon, 30 Nov 2015 18:05:44 -0600 Subject: [PATCH 850/909] Adding a lot of math constants from math.h. --- pixie/math.pxi | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pixie/math.pxi b/pixie/math.pxi index 47d45cd0..c6500c81 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -23,4 +23,19 @@ (i/defcfn ceil) (i/defcfn fabs) (i/defcfn floor) - (i/defcfn fmod)) + (i/defcfn fmod) + + (i/defconst M_E) ; base of natural logarithm, e + (i/defconst M_LOG2E) ; log2(e) + (i/defconst M_LOG10E) ; log10(e) + (i/defconst M_LN2) ; ln(2) + (i/defconst M_LN10) ; ln(10) + (i/defconst M_PI) ; pi + (i/defconst M_PI_2) ; pi / 2 + (i/defconst M_PI_4) ; pi / 4 + (i/defconst M_1_PI) ; 1 / pi + (i/defconst M_2_PI) ; 2 / pi + (i/defconst M_2_SQRTPI) ; 2 / sqrt(pi) + (i/defconst M_SQRT2) ; sqrt(2) + (i/defconst M_SQRT1_2)) ; sqrt(1/2) + From 2290fdb3c87802cbd263ee00602c2b6cde3d8d7b Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 3 Dec 2015 11:33:28 -0600 Subject: [PATCH 851/909] Adding tan. --- pixie/math.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/math.pxi b/pixie/math.pxi index c6500c81..644cf47c 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -12,6 +12,7 @@ (i/defcfn cosh) (i/defcfn sin) (i/defcfn sinh) + (i/defcfn tan) (i/defcfn tanh) (i/defcfn exp) (i/defcfn ldexp) From 11c5fee569cd2ea6301bb8c5e39f88528dcf46ed Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 3 Dec 2015 11:36:13 -0600 Subject: [PATCH 852/909] Grouping the trig functions more normally. --- pixie/math.pxi | 15 +++++++++------ 1 file changed, 9 insertions(+), 6 deletions(-) diff --git a/pixie/math.pxi b/pixie/math.pxi index 644cf47c..7f7faf3b 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -4,16 +4,19 @@ (i/with-config {:library "m" :cxx-flags ["-lm"] :includes ["math.h"]} - (i/defcfn acos) + (i/defcfn sin) + (i/defcfn cos) + (i/defcfn tan) + (i/defcfn asin) + (i/defcfn acos) (i/defcfn atan) - (i/defcfn atan2) - (i/defcfn cos) - (i/defcfn cosh) - (i/defcfn sin) + (i/defcfn atan2) ; Arc tangent function of two variables. + (i/defcfn sinh) - (i/defcfn tan) + (i/defcfn cosh) (i/defcfn tanh) + (i/defcfn exp) (i/defcfn ldexp) (i/defcfn log) From 64cb0f3a8d88c210ed964077028bb7967ea762f5 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 3 Dec 2015 11:37:45 -0600 Subject: [PATCH 853/909] Adding asinh, acosh, and atanh. --- pixie/math.pxi | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pixie/math.pxi b/pixie/math.pxi index 7f7faf3b..313dd904 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -17,6 +17,10 @@ (i/defcfn cosh) (i/defcfn tanh) + (i/defcfn asinh) + (i/defcfn acosh) + (i/defcfn atanh) + (i/defcfn exp) (i/defcfn ldexp) (i/defcfn log) From f991686dae2dc8334b4215ac9df27970ca5ba647 Mon Sep 17 00:00:00 2001 From: Christopher Mark Gore Date: Thu, 3 Dec 2015 13:58:01 -0600 Subject: [PATCH 854/909] Adding log2, log1p, logb, and ilogb. --- pixie/math.pxi | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/pixie/math.pxi b/pixie/math.pxi index 313dd904..15a778c4 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -23,9 +23,15 @@ (i/defcfn exp) (i/defcfn ldexp) + (i/defcfn log) + (i/defcfn log2) (i/defcfn log10) - ;(i/defcfn modf) ;; Needs ffi support + (i/defcfn log1p) + (i/defcfn logb) + (i/defcfn ilogb) + + ;; (i/defcfn modf) ;; Needs ffi support (i/defcfn pow) (i/defcfn sqrt) (i/defcfn ceil) From b952476ac767f8aa9f6c45c9f58a916f43ecea67 Mon Sep 17 00:00:00 2001 From: Herwig Hochleitner Date: Fri, 4 Dec 2015 13:13:56 +0100 Subject: [PATCH 855/909] Improve tempdir usage in ffi-infer - Make use of mkdtemp to create unique temporary directories per call to run-infer - If $TMPDIR is set, use it instead of /tmp - Remove temp files after finishing --- pixie/ffi-infer.pxi | 43 +++++++++++++++++++++++++++---------------- 1 file changed, 27 insertions(+), 16 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 9f13ac74..51bfa34a 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -178,24 +178,35 @@ return 0; `(def ~(symbol name) ~(callback-type of-type false))) +(def mkdtemp (ffi-fn libc "mkdtemp" [CCharP] CCharP)) +(def unlink (ffi-fn libc "unlink" [CCharP] CInt)) +(def rmdir (ffi-fn libc "rmdir" [CCharP] CInt)) +(def tempdir-template (str (or (getenv "TMPDIR") "/tmp") + "/ffiXXXXXX")) (defn run-infer [config cmds] - (io/spit "/tmp/tmp.cpp" (str (start-string) - (apply str (map emit-infer-code - cmds)) - (end-string))) - (println @load-paths) - (let [cmd-str (str "c++ " - (apply str (interpose " " pixie.platform/c-flags)) - " /tmp/tmp.cpp " - (apply str (map (fn [x] ( str " -I " x " ")) - @load-paths)) - (apply str " " (interpose " " (:cxx-flags *config*))) - " -o /tmp/a.out && /tmp/a.out") - _ (println cmd-str) - result (read-string (io/run-command cmd-str)) - gen (vec (map generate-code cmds result))] - `(do ~@gen))) + (let [tempdir (mkdtemp tempdir-template) + infile (str tempdir "/ffi.cpp") + outfile (str tempdir "/ffi.out")] + (io/spit infile (str (start-string) + (apply str (map emit-infer-code + cmds)) + (end-string))) + (println @load-paths) + (let [cmd-str (str "c++ " + (apply str (interpose " " pixie.platform/c-flags)) + " " infile " " + (apply str (map (fn [x] ( str " -I " x " ")) + @load-paths)) + (apply str " " (interpose " " (:cxx-flags *config*))) + " -o " outfile " && " outfile) + _ (println cmd-str) + result (read-string (io/run-command cmd-str)) + gen (vec (map generate-code cmds result))] + (unlink infile) + (unlink outfile) + (rmdir tempdir) + `(do ~@gen)))) (defn full-lib-name [library-name] (if (= library-name "c") From d758bb8d5ac6a6763f4af12c3982c57687d3870e Mon Sep 17 00:00:00 2001 From: Pauli Jaakkola Date: Fri, 4 Dec 2015 21:42:13 +0200 Subject: [PATCH 856/909] Add CFloat ffi type. --- pixie/ffi-infer.pxi | 11 +++++++---- pixie/vm/libs/ffi.py | 20 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 4 deletions(-) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi index 9f13ac74..efa3f431 100644 --- a/pixie/ffi-infer.pxi +++ b/pixie/ffi-infer.pxi @@ -104,11 +104,14 @@ return 0; (= (:type of-type) :function) (callback-type of-type in-struct?) :else 'pixie.stdlib/CVoidP)) +(def float-types {32 'pixie.stdlib/CFloat + 64 'pixie.stdlib/CDouble}) + (defmethod edn-to-ctype :float - [{:keys [size]} _] - (cond - (= size 8) 'pixie.stdlib/CDouble - :else (assert false "unknown type"))) + [{:keys [size] :as tp} _] + (let [tp-found (get float-types (* 8 size))] + (assert tp-found (str "No type found for " tp)) + tp-found)) (defmethod edn-to-ctype :void [_ _] diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 193b56ce..9cc0d348 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -350,6 +350,26 @@ def ffi_type(self): return clibffi.cast_type_to_ffitype(rffi.INT) CInt() +class CFloat(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CFloat") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.FLOATP, ptr) + return Float(rffi.cast(rffi.DOUBLE, casted[0])) + + def ffi_set_value(self, ptr, val): + val = to_float(val) + casted = rffi.cast(rffi.FLOATP, ptr) + casted[0] = rffi.cast(rffi.FLOAT, val.float_val()) + + def ffi_size(self): + return rffi.sizeof(rffi.FLOAT) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.FLOAT) +CFloat() + class CDouble(CType): def __init__(self): CType.__init__(self, u"pixie.stdlib.CDouble") From cee88a2513999c10fa039d7741e3986d61f300f8 Mon Sep 17 00:00:00 2001 From: Chris Shea Date: Fri, 4 Dec 2015 16:23:38 -0500 Subject: [PATCH 857/909] Normalize docstrings in stdlib Use present tense, capitalization, and punctuation in docstrings. --- pixie/stdlib.pxi | 128 +++++++++++++++++++++++------------------------ 1 file changed, 64 insertions(+), 64 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index eab2357f..ec1ef49c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -47,7 +47,7 @@ x)) (def conj - (fn ^{:doc "Adds elements to the collection. Elements are added to the end except in the case of Cons lists" + (fn ^{:doc "Adds elements to the collection. Elements are added to the end except in the case of Cons lists." :signatures [[] [coll] [coll item] [coll item & args]] :added "0.1"} conj @@ -58,7 +58,7 @@ (reduce -conj (-conj coll item) args)))) (def conj! - (fn ^{:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists" + (fn ^{:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists." :signatures [[] [coll] [coll item] [coll item & args]] :added "0.1"} conj! @@ -96,7 +96,7 @@ ([coll] (-pop coll)))) (def push - (fn ^{:doc "Push an element on to a stack." + (fn ^{:doc "Pushes an element on to a stack." :signatures [[] [coll] [coll item] [coll item & args]] :added "0.1"} push @@ -110,7 +110,7 @@ ([coll] (-pop! coll)))) (def push! - (fn ^{:doc "Push an element on to a transient stack." + (fn ^{:doc "Pushes an element on to a transient stack." :signatures [[] [coll] [coll item] [coll item & args]] :added "0.1"} push! @@ -134,7 +134,7 @@ result (-reduce coll f init)] (f result))))) -(def map (fn ^{:doc "map - creates a transducer that applies f to every input element" +(def map (fn ^{:doc "Creates a transducer that applies f to every input element." :signatures [[f] [f coll]] :added "0.1"} map @@ -246,7 +246,7 @@ (reduce rrf result input)))))) (def mapcat - (fn ^{:doc "Maps f over the elements of coll and concatenates the result" + (fn ^{:doc "Maps f over the elements of coll and concatenates the result." :added "0.1"} mapcat ([f] @@ -338,7 +338,7 @@ ([state itm] (update-hash-unordered! state itm)))) (def string-builder - (fn ^{:doc "Creates a reducing function that builds a string based on calling str on the transduced collection"} + (fn ^{:doc "Creates a reducing function that builds a string based on calling str on the transduced collection."} ([] (-string-builder)) ([sb] (str sb)) ([sb item] (conj! sb item)))) @@ -471,7 +471,7 @@ (set-macro! defmacro) (defmacro defn- - {:doc "Define a new non-public function. Otherwise the same as defn" + {:doc "Define a new non-public function. Otherwise the same as defn." :signatures [[nm doc? meta? & body]] :added "0.1"} [nm & rest] @@ -480,14 +480,14 @@ (defn not {:doc "Inverts the input, if a truthy value is supplied, returns false, otherwise -returns true" +returns true." :signatures [[x]] :added "0.1"} [x] (if x false true)) (defn + - {:doc "Adds the arguments, returning 0 if no arguments" + {:doc "Adds the arguments, returning 0 if no arguments." :signatures [[& args]] :added "0.1"} ([] 0) @@ -511,7 +511,7 @@ returns true" (reduce -mul (-mul x y) args))) (defn unchecked-add - {:doc "Adds the arguments, returning 0 if no arguments" + {:doc "Adds the arguments, returning 0 if no arguments." :signatures [[& args]] :added "0.1"} ([] 0) @@ -547,7 +547,7 @@ returns true" (-rem num div)) (defn rand-int - {:doc "random integer between 0 (inclusive) and n (exclusive)"} + {:doc "Returns a random integer between 0 (inclusive) and n (exclusive)."} [n] (if (zero? n) 0 @@ -602,56 +602,56 @@ returns true" false))) (defn pos? - {:doc "Returns true if x is greater than zero" + {:doc "Returns true if x is greater than zero." :signatures [[x]] :added "0.1"} [x] (> x 0)) (defn neg? - {:doc "Returns true if x is less than zero" + {:doc "Returns true if x is less than zero." :signatures [[x]] :added "0.1"} [x] (< x 0)) (defn zero? - {:doc "Returns true if x is equal to zero" + {:doc "Returns true if x is equal to zero." :signatures [[x]] :added "0.1"} [x] (= x 0)) (defn inc - {:doc "Increments x by one" + {:doc "Increments x by one." :signatures [[x]] :added "0.1"} [x] (+ x 1)) (defn dec - {:doc "Decrements x by one" + {:doc "Decrements x by one." :signatures [[x]] :added "0.1"} [x] (- x 1)) (defn unchecked-inc - {:doc "Increments x by one" + {:doc "Increments x by one." :signatures [[x]] :added "0.1"} [x] (unchecked-add x 1)) (defn unchecked-dec - {:doc "Decrements x by one" + {:doc "Decrements x by one." :signatures [[x]] :added "0.1"} [x] (unchecked-subtract x 1)) (defn empty? - {:doc "returns true if the collection has no items, otherwise false" + {:doc "Returns true if the collection has no items, otherwise false." :signatures [[coll]] :added "0.1"} [coll] @@ -660,21 +660,21 @@ returns true" (not (seq coll)))) (defn not-empty? - {:doc "returns true if the collection has items, otherwise false" + {:doc "Returns true if the collection has items, otherwise false." :signatures [[coll]] :added "0.1"} [coll] (if (seq coll) true false)) (defn even? - {:doc "Returns true if n is even" + {:doc "Returns true if n is even." :signatures [[n]] :added "0.1"} [n] (zero? (rem n 2))) (defn odd? - {:doc "Returns true of n is odd" + {:doc "Returns true of n is odd." :signatures [[n]] :added "0.1"} [n] @@ -682,7 +682,7 @@ returns true" (defn nth {:doc "Returns the element at the idx. If the index is not found it will return an error. - However, if you specify a not-found parameter, it will substitute that instead" + However, if you specify a not-found parameter, it will substitute that instead." :signatures [[coll idx] [coll idx not-found]] :added "0.1"} ([coll idx] (-nth coll idx)) @@ -729,7 +729,7 @@ returns true" (first (next (next (next coll)))))) (defn assoc - {:doc "Associates the key with the value in the collection" + {:doc "Associates the key with the value in the collection." :signatures [[m] [m k v] [m k v & kvs]] :added "0.1"} ([m] m) @@ -739,7 +739,7 @@ returns true" (apply assoc (-assoc m k v) rest))) (defn dissoc - {:doc "Removes the value associated with the keys from the collection" + {:doc "Removes the value associated with the keys from the collection." :signatures [[m] [m & ks]] :added "0.1"} ([m] m) @@ -763,7 +763,7 @@ there's a value associated with the key. Use `some` for checking for values." (-contains-key coll key)) (defn hash-set [& args] - {:doc "Creates a hash-set from the arguments of the function" + {:doc "Creates a hash-set from the arguments of the function." :added "0.1"} (set args)) @@ -797,7 +797,7 @@ there's a value associated with the key. Use `some` for checking for values." (apply (transduce comp (apply list f1 f2 f3 fs)) args)))) (defmacro cond - {:doc "Checks if any of the tests is truthy, if so, stops and returns the value of the corresponding body" + {:doc "Checks if any of the tests is truthy, if so, stops and returns the value of the corresponding body." :signatures [[] [test then & clauses]] :added "0.1"} ([] nil) @@ -915,8 +915,8 @@ If further arguments are passed, invokes the method named by symbol, passing the (seq res)))) (defn complement - {:doc "Given a function, return a new function which takes the same arguments - but returns the opposite truth value"} + {:doc "Given a function, returns a new function which takes the same arguments + but returns the opposite truth value."} [f] (assert (fn? f) "Complement must be passed a function") (fn @@ -926,7 +926,7 @@ If further arguments are passed, invokes the method named by symbol, passing the ([x y & more] (not (apply f x y more))))) (defn constantly [x] - {:doc "Return a function that always returns x, no matter what it is called with." + {:doc "Returns a function that always returns x, no matter what it is called with." :examples [["(let [f (constantly :me)] [(f 1) (f \"foo\") (f :abc) (f nil)])" nil [:me :me :me :me]]]} (fn [& _] x)) @@ -977,7 +977,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (meta m))) (defn keys - {:doc "If called with no arguments returns a transducer that will extract the key from each map entry. If passed + {:doc "If called with no arguments, returns a transducer that will extract the key from each map entry. If passed a collection, will assume that it is a hashmap and return a vector of all keys from the collection." :signatures [[] [coll]] :added "0.1"} @@ -986,7 +986,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (transduce (map key) conj! m))) (defn vals - {:doc "If called with no arguments returns a transducer that will extract the key from each map entry. If passed + {:doc "If called with no arguments, returns a transducer that will extract the key from each map entry. If passed a collection, will assume that it is a hashmap and return a vector of all keys from the collection." :signatures [[] [coll]] :added "0.1"} @@ -1078,7 +1078,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (extend -repr Symbol -str) (defn get - {:doc "Get an element from a collection implementing ILookup, return nil or the default value if not found." + {:doc "Gets an element from a collection implementing ILookup. Returns nil or the default value if not found." :added "0.1"} ([mp k] (get mp k nil)) @@ -1095,7 +1095,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (defn get-in - {:doc "Get a value from a nested collection at the \"path\" given by the keys." + {:doc "Gets a value from a nested collection at the \"path\" given by the keys." :examples [["(get-in {:a [{:b 42}]} [:a 0 :b])" nil 42]] :signatures [[m ks] [m ks not-found]] :added "0.1"} @@ -1113,7 +1113,7 @@ If further arguments are passed, invokes the method named by symbol, passing the m)))) (defn assoc-in - {:doc "Associate a value in a nested collection given by the path. + {:doc "Associates a value in a nested collection given by the path. Creates new maps if the keys are not present." :examples [["(assoc-in {} [:a :b :c] 42)" nil {:a {:b {:c 42}}}]] @@ -1127,7 +1127,7 @@ Creates new maps if the keys are not present." (assoc m k v))))) (defn update-in - {:doc "Update a value in a nested collection." + {:doc "Updates a value in a nested collection." :examples [["(update-in {:a {:b {:c 41}}} [:a :b :c] inc)" nil {:a {:b {:c 42}}}]] :added "0.1"} [m ks f & args] @@ -1154,7 +1154,7 @@ Creates new maps if the keys are not present." (str "Assert failed: " ~msg)])))) (defmacro resolve - {:doc "Resolve the var associated with the symbol in the current namespace." + {:doc "Resolves the var associated with the symbol in the current namespace." :added "0.1"} [sym] `(resolve-in (this-ns-name) ~sym)) @@ -1202,7 +1202,7 @@ Creates new maps if the keys are not present." (instance? Protocol x)) (defmacro deftype - {:doc "Define a custom type." + {:doc "Defines a custom type." :examples [["(deftype Person [name] IObject (-str [self] @@ -1291,7 +1291,7 @@ Creates new maps if the keys are not present." result)) (defmacro defrecord - {:doc "Define a record type. + {:doc "Defines a record type. Similar to `deftype`, but supports construction from a map using `map->Type` and implements IAssociative, ILookup and IObject." @@ -1511,7 +1511,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (identical? x nil)) (defn some? [x] - {:doc "true if x is not nil"} + {:doc "Returns true if x is not nil."} (not (nil? x))) (defn fnil [f else] @@ -1528,7 +1528,7 @@ The new value is thus `(apply f current-value-of-atom args)`." ~(nth binding 1 nil))) (defmacro dotimes - {:doc "Execute the expressions in the body n times." + {:doc "Executes the expressions in the body n times." :examples [["(dotimes [i 3] (println i))" "1\n2\n3\n"]] :signatures [[[i n] & body]] :added "0.1"} @@ -1542,7 +1542,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (recur (inc ~b)))))))) (defmacro and - {:doc "Check if the given expressions return truthy values, returning the last, or false." + {:doc "Checks if the given expressions return truthy values, returning the last, or false." :examples [["(and true false)" nil false] ["(and 1 2 3)" nil 3] ["(and 1 false 3)" nil false]] @@ -1604,7 +1604,7 @@ The new value is thus `(apply f current-value-of-atom args)`." false)) (defn nnext - {:doc "Equivalent to (next (next coll))" + {:doc "Equivalent to (next (next coll))." :added "0.1"} [coll] (next (next coll))) @@ -1624,7 +1624,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defn ith {:doc "Returns the ith element of the collection, negative values count from the end. If an index is out of bounds, will throw an Index out of Range exception. - However, if you specify a not-found parameter, it will substitute that instead" + However, if you specify a not-found parameter, it will substitute that instead." :signatures [[coll i] [coll idx not-found]] :added "0.1"} ([coll i] @@ -1688,7 +1688,7 @@ The new value is thus `(apply f current-value-of-atom args)`." (defmacro while {:doc "Repeatedly executes body while test expression is true. Presumes - some side-effect will cause test to become false/nil. Returns nil" + some side-effect will cause test to become false/nil. Returns nil." :added "0.1"} [test & body] `(loop [] @@ -1761,7 +1761,7 @@ The new value is thus `(apply f current-value-of-atom args)`." ;; TODO: use a transient map in the future (defn frequencies - {:doc "Returns a map with distinct elements as keys and the number of occurences as values" + {:doc "Returns a map with distinct elements as keys and the number of occurences as values." :added "0.1"} [coll] (reduce (fn [res elem] @@ -2114,7 +2114,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} self)))) (defn filter - {:doc "Filter the collection for elements matching the predicate." + {:doc "Filters the collection for elements matching the predicate." :signatures [[pred] [pred coll]] :added "0.1"} ([pred] @@ -2260,7 +2260,7 @@ user => (refer 'pixie.string :exclude '(substring))" (reduce merge2 (first maps) (next maps))))) (defn every? - {:doc "Check if every element of the collection satisfies the predicate." + {:doc "Checks if every element of the collection satisfies the predicate." :added "0.1"} [pred coll] (cond @@ -2377,7 +2377,7 @@ If the number of arguments is even and no clause matches, throws an exception." (defmacro defmulti - {:doc "Define a multimethod, which dispatches to its methods based on dispatch-fn." + {:doc "Defines a multimethod, which dispatches to its methods based on dispatch-fn." :examples [["(defmulti greet first)"] ["(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\"))"] ["(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".\"))"] @@ -2396,7 +2396,7 @@ If the number of arguments is even and no clause matches, throws an exception." `(def ~name (->MultiMethod ~dispatch-fn ~(get options :default :default) (atom {}))))) (defmacro defmethod - {:doc "Define a method of a multimethod. See `(doc defmulti)` for details." + {:doc "Defines a method of a multimethod. See `(doc defmulti)` for details." :signatures [[name dispatch-val [param*] & body]] :added "0.1"} [name dispatch-val params & body] @@ -2413,14 +2413,14 @@ If the number of arguments is even and no clause matches, throws an exception." [x] x) (defmacro declare - {:doc "Forward declare the given variable names, setting them to nil." + {:doc "Forward declares the given variable names, setting them to nil." :added "0.1"} [& nms] (let [defs (map (fn [nm] `(def ~nm)) (seq nms))] `(do ~@defs))) (defmacro defprotocol - {:doc "Define a new protocol." + {:doc "Defines a new protocol." :examples [["(defprotocol SayHi (hi [x]))"] ["(extend hi String (fn [name] (str \"Hi, \" name \"!\")))"] ["(hi \"Jane\")" nil "Hi, Jane!"]] @@ -2433,7 +2433,7 @@ If the number of arguments is even and no clause matches, throws an exception." sigs))) (defmacro extend-type - {:doc "Extend the protocols to the given type. + {:doc "Extends the protocols to the given type. Expands to calls to `extend`." :examples [["(defprotocol SayHi (hi [x]))"] @@ -2521,7 +2521,7 @@ Expands to calls to `extend-type`." (defprotocol IRecord) (defn record? - {:doc "Returns true if x implements IRecord" + {:doc "Returns true if x implements IRecord." :since "0.1"} [x] (satisfies? IRecord x)) @@ -2552,7 +2552,7 @@ Expands to calls to `extend-type`." `(or (seq ~(gen-loop [] bindings)) '()))) (defmacro doto - {:doc "Evaluate o, uses the value as the first argument in each form. Returns o"} + {:doc "Evaluate o, uses the value as the first argument in each form. Returns o."} [o & forms] (let [s (gensym o)] `(let [~s ~o] @@ -2598,14 +2598,14 @@ Expands to calls to `extend-type`." result#))) (defn pst - {:doc "Prints the trace of a Runtime Exception if given, or the last Runtime Exception in *e" + {:doc "Prints the trace of a Runtime Exception if given, or the last Runtime Exception in *e." :signatures [[] [e]] :added "0.1"} ([] (pst *e)) ([e] (when e (print (str e))))) (defn trace - {:doc "Returns a seq of the trace of a Runtime Exception or the last Runtime Exception in *e" + {:doc "Returns a seq of the trace of a Runtime Exception or the last Runtime Exception in *e." :signatures [[] [e]] :added "0.1"} ([] (trace *e)) @@ -2701,7 +2701,7 @@ Calling this function on something that is not ISeqable returns a seq with that (str "#Environment{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) (defn interleave - "Returns a seq of all the items in the input collections interleaved" + "Returns a seq of all the items in the input collections interleaved." ([] ()) ([c1] (seq c1)) ([c1 c2] @@ -2719,13 +2719,13 @@ Calling this function on something that is not ISeqable returns a seq with that (apply interleave (map next ss)))))))) (defn min - "Returns the smallest of all the arguments to this function. Assumes arguments are numeric" + "Returns the smallest of all the arguments to this function. Assumes arguments are numeric." ([x] x) ([x y] (if (< x y) x y)) ([x y & zs] (apply min (min x y) zs))) (defn max - "Returns the largest of all the arguments to this function. Assumes arguments are numeric" + "Returns the largest of all the arguments to this function. Assumes arguments are numeric." ([x] x) ([x y] (if (> x y) x y)) ([x y & zs] (apply max (max x y) zs))) @@ -2812,7 +2812,7 @@ Calling this function on something that is not ISeqable returns a seq with that (defmacro some-> {:doc "When expr is not nil, threads it into the first form (via ->), - and when that result is not nil, through the next etc" + and when that result is not nil, through the next etc." :signatures [[expr & forms]] :added "0.1"} [expr & forms] @@ -2827,7 +2827,7 @@ Calling this function on something that is not ISeqable returns a seq with that (defmacro some->> {:doc "When expr is not nil, threads it into the first form (via ->>), - and when that result is not nil, through the next etc" + and when that result is not nil, through the next etc." :signatures [[x & forms]] :added "0,1"} [expr & forms] @@ -2891,7 +2891,7 @@ Calling this function on something that is not ISeqable returns a seq with that (defprotocol IComparable (-compare [x y] - "Compare to objects returing 0 if the same -1 with x is logically smaller than y and 1 if x is logically larger")) + "Compare to objects returing 0 if the same -1 with x is logically smaller than y and 1 if x is logically larger.")) (defn compare-numbers [x y] From dee802dbe97daa3a3450adbb9234c94cb01295e3 Mon Sep 17 00:00:00 2001 From: Lucas Stadler Date: Sat, 5 Dec 2015 08:11:36 +0100 Subject: [PATCH 858/909] Add `is_recursive=True` to the main JitDriver This seems to be required in recent versions of pypy/rpython. --- pixie/vm/interpreter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 83e873df..d279e2fb 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -11,7 +11,7 @@ def get_location(ip, sp, is_continuation, bc, base_code): return code.BYTECODES[bc[ip]] + " in " + str(base_code._name) jitdriver = JitDriver(greens=["ip", "sp", "is_continuation", "bc", "base_code"], reds=["frame"], virtualizables=["frame"], - get_printable_location=get_location) + get_printable_location=get_location, is_recursive=True) @elidable From 0daaefef0fd35ee011c49a3583b216923272775f Mon Sep 17 00:00:00 2001 From: Josh Glover Date: Mon, 7 Dec 2015 11:23:25 +0100 Subject: [PATCH 859/909] Fix the build for the latest PyPy nightly build --- pixie/vm/c_api.py | 16 ++++------------ pixie/vm/interpreter.py | 2 +- 2 files changed, 5 insertions(+), 13 deletions(-) diff --git a/pixie/vm/c_api.py b/pixie/vm/c_api.py index 38b30e6e..45888295 100644 --- a/pixie/vm/c_api.py +++ b/pixie/vm/c_api.py @@ -1,31 +1,23 @@ -from rpython.rlib.entrypoint import entrypoint, RPython_StartupCode +from rpython.rlib.entrypoint import entrypoint_highlevel, RPython_StartupCode from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.lltypesystem.lloperation import llop -@entrypoint('main', [rffi.CCHARP], c_name='pixie_init') +@entrypoint_highlevel('main', [rffi.CCHARP], c_name='pixie_init') def pypy_execute_source(ll_progname): from target import init_vm - after = rffi.aroundstate.after - if after: after() progname = rffi.charp2str(ll_progname) init_vm(progname) res = 0 - before = rffi.aroundstate.before - if before: before() return rffi.cast(rffi.INT, res) -@entrypoint('main', [rffi.CCHARP], c_name='pixie_execute_source') +@entrypoint_highlevel('main', [rffi.CCHARP], c_name='pixie_execute_source') def pypy_execute_source(ll_source): from target import EvalFn, run_with_stacklets - after = rffi.aroundstate.after - if after: after() source = rffi.charp2str(ll_source) f = EvalFn(source) run_with_stacklets.invoke([f]) res = 0 - before = rffi.aroundstate.before - if before: before() - return rffi.cast(rffi.INT, res) \ No newline at end of file + return rffi.cast(rffi.INT, res) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index 83e873df..d279e2fb 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -11,7 +11,7 @@ def get_location(ip, sp, is_continuation, bc, base_code): return code.BYTECODES[bc[ip]] + " in " + str(base_code._name) jitdriver = JitDriver(greens=["ip", "sp", "is_continuation", "bc", "base_code"], reds=["frame"], virtualizables=["frame"], - get_printable_location=get_location) + get_printable_location=get_location, is_recursive=True) @elidable From 4e3b95191f4bf646523a9bbca0cc2f17a271292d Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Tue, 8 Dec 2015 13:44:28 +0000 Subject: [PATCH 860/909] delete dead code --- pixie/vm/code.py | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 920c3dd9..14883796 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -484,18 +484,6 @@ def invoke_with(self, args, this_fn): def invoke(self, args): return self.deref().invoke(args) -class bindings(py_object): - def __init__(self, *args): - self._args = list(args) - - def __enter__(self): - _dynamic_vars.push_binding_frame() - for x in range(0, len(self._args), 2): - self._args[x].set_value(self._args[x + 1]) - - def __exit__(self, exc_type, exc_val, exc_tb): - _dynamic_vars.pop_binding_frame() - class Refer(py_object): def __init__(self, ns, refer_syms=[], refer_all=False): From 75e7db741f48537cf2e111c44a3d77506f511990 Mon Sep 17 00:00:00 2001 From: Josh Glover Date: Tue, 8 Dec 2015 16:21:12 +0100 Subject: [PATCH 861/909] Add pixie.data.json for reading and writing JSON --- pixie/data/json.pxi | 40 ++++++++++++++++++++++++++++ tests/pixie/tests/data/test-json.pxi | 19 +++++++++++++ 2 files changed, 59 insertions(+) create mode 100644 pixie/data/json.pxi create mode 100644 tests/pixie/tests/data/test-json.pxi diff --git a/pixie/data/json.pxi b/pixie/data/json.pxi new file mode 100644 index 00000000..9ec7a995 --- /dev/null +++ b/pixie/data/json.pxi @@ -0,0 +1,40 @@ +(ns pixie.data.json + (:require [pixie.parser.json] + [pixie.string :as string])) + +(def read-string pixie.parser.json/read-string) + +(defprotocol IToJSON + (write-string [this])) + +(defn- write-map [m] + (if (empty? m) + "{}" + (str "{ " + (->> m + (map (fn [[k v]] (string/interp "$(write-string k)$: $(write-string v)$"))) + (string/join ", ")) + " }"))) + +(defn- write-sequential [xs] + (if (empty? xs) + "[]" + (str "[ " + (->> (map write-string xs) + (string/join ", ")) + " ]"))) + +(defn- write-str [s] + (string/interp "\"$s$\"")) + +(extend-protocol IToJSON + Character (write-string [this] (write-str this)) + Cons (write-string [this] (write-sequential this)) + EmptyList (write-string [this] (write-sequential this)) + Number (write-string [this] (str this)) + Keyword (write-string [this] (write-str (name this))) + LazySeq (write-string [this] (write-sequential this)) + IMap (write-string [this] (write-map this)) + IVector (write-string [this] (write-sequential this)) + Ratio (write-string [this] (str (float this))) + String (write-string [this] (write-str this))) diff --git a/tests/pixie/tests/data/test-json.pxi b/tests/pixie/tests/data/test-json.pxi new file mode 100644 index 00000000..1dc81891 --- /dev/null +++ b/tests/pixie/tests/data/test-json.pxi @@ -0,0 +1,19 @@ +(ns pixie.tests.data.test-json + (:require [pixie.test :refer :all] + [pixie.data.json :as json])) + + + +;; This test just ensures that we can use read-string; more thorough testing is +;; done in pixie.tests.parser.test-json +(deftest test-read-string + (assert= (json/read-string "{\"foo\": 42, \"bar\": [\"baz\"]}") + {"foo" 42, "bar" ["baz"]})) + +(deftest test-write-string + (assert= (json/write-string {:strings {:a \a, :b "b", :c "", :d :d} + :numbers {:one 1, :two 2.0, :three 3/1} + :lists {:empty {:vector [], :list '(), :seq (take 0 (range))} + :non-empty {:vector [1 2 3], :list '(1 2 3), :seq (take 3 (range))}} + :maps {:empty {}, :non-empty {:this "is covered, right? ;)"}}}) + "{ \"strings\": { \"b\": \"b\", \"c\": \"\", \"a\": \"a\", \"d\": \"d\" }, \"maps\": { \"non-empty\": { \"this\": \"is covered, right? ;)\" }, \"empty\": {} }, \"lists\": { \"non-empty\": { \"seq\": [ 0, 1, 2 ], \"list\": [ 1, 2, 3 ], \"vector\": [ 1, 2, 3 ] }, \"empty\": { \"seq\": [], \"list\": [], \"vector\": [] } }, \"numbers\": { \"one\": 1, \"two\": 2.000000, \"three\": 3 } }")) From 98fbd71749f68244fe8766131f9380cea3047085 Mon Sep 17 00:00:00 2001 From: Josh Glover Date: Tue, 8 Dec 2015 16:51:29 +0100 Subject: [PATCH 862/909] data.test-json : add round-trip tests --- tests/pixie/tests/data/test-json.pxi | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/pixie/tests/data/test-json.pxi b/tests/pixie/tests/data/test-json.pxi index 1dc81891..02d43ec4 100644 --- a/tests/pixie/tests/data/test-json.pxi +++ b/tests/pixie/tests/data/test-json.pxi @@ -17,3 +17,12 @@ :non-empty {:vector [1 2 3], :list '(1 2 3), :seq (take 3 (range))}} :maps {:empty {}, :non-empty {:this "is covered, right? ;)"}}}) "{ \"strings\": { \"b\": \"b\", \"c\": \"\", \"a\": \"a\", \"d\": \"d\" }, \"maps\": { \"non-empty\": { \"this\": \"is covered, right? ;)\" }, \"empty\": {} }, \"lists\": { \"non-empty\": { \"seq\": [ 0, 1, 2 ], \"list\": [ 1, 2, 3 ], \"vector\": [ 1, 2, 3 ] }, \"empty\": { \"seq\": [], \"list\": [], \"vector\": [] } }, \"numbers\": { \"one\": 1, \"two\": 2.000000, \"three\": 3 } }")) + +(deftest test-round-trip + (let [data {"foo" 1, "bar" [2 3]}] ; won't work with keywords because the parser doesn't keywordise them (yet) + (assert= (-> data json/write-string json/read-string) + data)) + + (let [string "{ \"foo\": [ 1, 2, 3 ] }"] + (assert= (-> string json/read-string json/write-string) + string))) From 4bd54cc349332ceee0e9c037fd37e293ddf0ee40 Mon Sep 17 00:00:00 2001 From: Josh Glover Date: Tue, 8 Dec 2015 17:02:51 +0100 Subject: [PATCH 863/909] data.json : use transducers for maps and sequences --- pixie/data/json.pxi | 11 +++++++---- tests/pixie/tests/data/test-json.pxi | 2 +- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/pixie/data/json.pxi b/pixie/data/json.pxi index 9ec7a995..cb161871 100644 --- a/pixie/data/json.pxi +++ b/pixie/data/json.pxi @@ -12,16 +12,19 @@ "{}" (str "{ " (->> m - (map (fn [[k v]] (string/interp "$(write-string k)$: $(write-string v)$"))) - (string/join ", ")) + (transduce (comp (map (fn [[k v]] (string/interp "$(write-string k)$: $(write-string v)$"))) + (interpose ", ")) + string-builder)) " }"))) (defn- write-sequential [xs] (if (empty? xs) "[]" (str "[ " - (->> (map write-string xs) - (string/join ", ")) + (->> xs + (transduce (comp (map write-string) + (interpose ", ")) + string-builder)) " ]"))) (defn- write-str [s] diff --git a/tests/pixie/tests/data/test-json.pxi b/tests/pixie/tests/data/test-json.pxi index 02d43ec4..3eee66c7 100644 --- a/tests/pixie/tests/data/test-json.pxi +++ b/tests/pixie/tests/data/test-json.pxi @@ -16,7 +16,7 @@ :lists {:empty {:vector [], :list '(), :seq (take 0 (range))} :non-empty {:vector [1 2 3], :list '(1 2 3), :seq (take 3 (range))}} :maps {:empty {}, :non-empty {:this "is covered, right? ;)"}}}) - "{ \"strings\": { \"b\": \"b\", \"c\": \"\", \"a\": \"a\", \"d\": \"d\" }, \"maps\": { \"non-empty\": { \"this\": \"is covered, right? ;)\" }, \"empty\": {} }, \"lists\": { \"non-empty\": { \"seq\": [ 0, 1, 2 ], \"list\": [ 1, 2, 3 ], \"vector\": [ 1, 2, 3 ] }, \"empty\": { \"seq\": [], \"list\": [], \"vector\": [] } }, \"numbers\": { \"one\": 1, \"two\": 2.000000, \"three\": 3 } }")) + "{ \"numbers\": { \"three\": 3, \"two\": 2.000000, \"one\": 1 }, \"lists\": { \"empty\": { \"vector\": [], \"list\": [], \"seq\": [] }, \"non-empty\": { \"vector\": [ 1, 2, 3 ], \"list\": [ 1, 2, 3 ], \"seq\": [ 0, 1, 2 ] } }, \"maps\": { \"empty\": {}, \"non-empty\": { \"this\": \"is covered, right? ;)\" } }, \"strings\": { \"d\": \"d\", \"a\": \"a\", \"c\": \"\", \"b\": \"b\" } }")) (deftest test-round-trip (let [data {"foo" 1, "bar" [2 3]}] ; won't work with keywords because the parser doesn't keywordise them (yet) From 6b6673d1eddeb48f6c4bb52ef3e1b70351af53e8 Mon Sep 17 00:00:00 2001 From: Josh Glover Date: Wed, 9 Dec 2015 10:16:42 +0100 Subject: [PATCH 864/909] data.json : handle Nil and PersistentList --- pixie/data/json.pxi | 30 ++++++------- tests/pixie/tests/data/test-json.pxi | 64 ++++++++++++++++++++++++---- 2 files changed, 70 insertions(+), 24 deletions(-) diff --git a/pixie/data/json.pxi b/pixie/data/json.pxi index cb161871..7fde6354 100644 --- a/pixie/data/json.pxi +++ b/pixie/data/json.pxi @@ -8,24 +8,20 @@ (write-string [this])) (defn- write-map [m] - (if (empty? m) - "{}" - (str "{ " - (->> m - (transduce (comp (map (fn [[k v]] (string/interp "$(write-string k)$: $(write-string v)$"))) - (interpose ", ")) - string-builder)) - " }"))) + (str "{" + (->> m + (transduce (comp (map (fn [[k v]] (string/interp "$(write-string k)$: $(write-string v)$"))) + (interpose ", ")) + string-builder)) + "}")) (defn- write-sequential [xs] - (if (empty? xs) - "[]" - (str "[ " - (->> xs - (transduce (comp (map write-string) - (interpose ", ")) - string-builder)) - " ]"))) + (str "[" + (->> xs + (transduce (comp (map write-string) + (interpose ", ")) + string-builder)) + "]")) (defn- write-str [s] (string/interp "\"$s$\"")) @@ -34,10 +30,12 @@ Character (write-string [this] (write-str this)) Cons (write-string [this] (write-sequential this)) EmptyList (write-string [this] (write-sequential this)) + Nil (write-string [_] "null") Number (write-string [this] (str this)) Keyword (write-string [this] (write-str (name this))) LazySeq (write-string [this] (write-sequential this)) IMap (write-string [this] (write-map this)) IVector (write-string [this] (write-sequential this)) + PersistentList (write-string [this] (write-sequential this)) Ratio (write-string [this] (str (float this))) String (write-string [this] (write-str this))) diff --git a/tests/pixie/tests/data/test-json.pxi b/tests/pixie/tests/data/test-json.pxi index 3eee66c7..098ad019 100644 --- a/tests/pixie/tests/data/test-json.pxi +++ b/tests/pixie/tests/data/test-json.pxi @@ -10,19 +10,67 @@ (assert= (json/read-string "{\"foo\": 42, \"bar\": [\"baz\"]}") {"foo" 42, "bar" ["baz"]})) -(deftest test-write-string - (assert= (json/write-string {:strings {:a \a, :b "b", :c "", :d :d} - :numbers {:one 1, :two 2.0, :three 3/1} - :lists {:empty {:vector [], :list '(), :seq (take 0 (range))} - :non-empty {:vector [1 2 3], :list '(1 2 3), :seq (take 3 (range))}} - :maps {:empty {}, :non-empty {:this "is covered, right? ;)"}}}) - "{ \"numbers\": { \"three\": 3, \"two\": 2.000000, \"one\": 1 }, \"lists\": { \"empty\": { \"vector\": [], \"list\": [], \"seq\": [] }, \"non-empty\": { \"vector\": [ 1, 2, 3 ], \"list\": [ 1, 2, 3 ], \"seq\": [ 0, 1, 2 ] } }, \"maps\": { \"empty\": {}, \"non-empty\": { \"this\": \"is covered, right? ;)\" } }, \"strings\": { \"d\": \"d\", \"a\": \"a\", \"c\": \"\", \"b\": \"b\" } }")) +(deftest test-strings + (assert-table [x y] (assert= (json/write-string x) y) + \a "\"a\"" + "foo" "\"foo\"" + :bar "\"bar\"" + "" "\"\"")) + +(deftest test-numbers + (assert-table [x y] (assert= (json/write-string x) y) + 1 "1" + 1.0 "1.000000" + 0.1 "0.100000" + 1.1 "1.100000" + 1234.5678 "1234.567800" + -1 "-1" + -0.1 "-0.100000" + -1.1 "-1.100000" + -1234.5678 "-1234.567800" + 1e1 "10.000000" + 3/1 "3")) + +(deftest test-vectors + (assert-table [x y] (assert= (json/write-string x) y) + [] "[]" + [nil] "[null]" + [1 2] "[1, 2]" + [1 1.0 nil] "[1, 1.000000, null]" + ["foo" 42] "[\"foo\", 42]")) + +(deftest test-seqs + (assert-table [x y] (assert= (json/write-string x) y) + (take 0 (range)) "[]" + (take 1 (repeat nil)) "[null]" + (map identity [1 2]) "[1, 2]" + (reduce + [1 2 3]) "6")) + +(deftest test-lists + (assert-table [x y] (assert= (json/write-string x) y) + '() "[]" + '(nil) "[null]" + '(1 2) "[1, 2]" + '(1 1.0 nil) "[1, 1.000000, null]" + '("foo" 42) "[\"foo\", 42]" + (list) "[]" + (list nil) "[null]" + (list 1 2) "[1, 2]" + (list 1 1.0 nil) "[1, 1.000000, null]" + (list "foo" 42) "[\"foo\", 42]")) + +(deftest test-maps + (assert-table [x y] (assert= (json/write-string x) y) + {} "{}" + {:foo 42} "{\"foo\": 42}" + {"foo" 42} "{\"foo\": 42}" + {"foo" 42, "bar" nil} "{\"foo\": 42, \"bar\": null}")) (deftest test-round-trip (let [data {"foo" 1, "bar" [2 3]}] ; won't work with keywords because the parser doesn't keywordise them (yet) (assert= (-> data json/write-string json/read-string) data)) - (let [string "{ \"foo\": [ 1, 2, 3 ] }"] + (let [string "{\"foo\": [1, 2, 3]}"] (assert= (-> string json/read-string json/write-string) string))) From e6741066162984363d1bf8fe1bafda030fdf31a9 Mon Sep 17 00:00:00 2001 From: Josh Glover Date: Wed, 9 Dec 2015 10:20:56 +0100 Subject: [PATCH 865/909] Update gitignore to ignore build artifacts --- .gitignore | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.gitignore b/.gitignore index d2f96fde..62c09a44 100644 --- a/.gitignore +++ b/.gitignore @@ -6,3 +6,7 @@ lib include *.pxic test.tmp +/libuv* +/uv-* +/uv.h +/*compressed-output.* From 5fa954e02d542c6f092db582953cca5afcd1b248 Mon Sep 17 00:00:00 2001 From: Chris Shea Date: Fri, 11 Dec 2015 15:54:26 -0500 Subject: [PATCH 866/909] Normalize docstrings in io-related namespaces --- pixie/io-blocking.pxi | 10 +++++----- pixie/io.pxi | 14 +++++++------- pixie/io/tcp.pxi | 2 +- 3 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi index c6547486..4af15739 100644 --- a/pixie/io-blocking.pxi +++ b/pixie/io-blocking.pxi @@ -38,15 +38,15 @@ (common/stream-reducer this f init))) (defn open-read - {:doc "Open a file for reading, returning a IInputStream" + {:doc "Opens a file for reading. Returns an IInputStream." :added "0.1"} [filename] (assert (string? filename) "Filename must be a string") (->FileStream (fopen filename "r"))) (defn read-line - "Read one line from input-stream for each invocation. - nil when all lines have been read" + "Reads one line from input-stream for each invocation. + Returns nil when all lines have been read." [input-stream] (let [line-feed (into #{} (map int [\newline \return])) buf (buffer 1)] @@ -62,7 +62,7 @@ (defn line-seq "Returns the lines of text from input-stream as a lazy sequence of strings. - input-stream must implement IInputStream" + input-stream must implement IInputStream." [input-stream] (when-let [line (read-line input-stream)] (cons line (lazy-seq (line-seq input-stream))))) @@ -131,7 +131,7 @@ (common/stream-reducer this f init))) (defn popen-read - {:doc "Open a file for reading, returning a IInputStream" + {:doc "Opens a file for reading. Returns an IInputStream." :added "0.1"} [command] (assert (string? command) "Command must be a string") diff --git a/pixie/io.pxi b/pixie/io.pxi index 16ec14be..3dd8bdf2 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -41,7 +41,7 @@ (common/stream-reducer this f init))) (defn open-read - {:doc "Open a file for reading, returning a IInputStream" + {:doc "Opens a file for reading. Returns an IInputStream." :added "0.1"} [filename] (assert (string? filename) "Filename must be a string") @@ -95,15 +95,15 @@ (throw [::Exception "Expected a LineReader, UTF8InputStream, or BufferedInputStream"]))) (defn read-line - "Read one line from input-stream for each invocation. - nil when all lines have been read. + "Reads one line from input-stream for each invocation. + Returns nil when all lines have been read. Pass a BufferedInputStream for best performance." [input-stream] (-read-line (line-reader input-stream))) (defn line-seq "Returns the lines of text from input-stream as a lazy sequence of strings. - input-stream must implement IInputStream" + input-stream must implement IInputStream." [input-stream] (let [lr (line-reader input-stream)] (when-let [line (-read-line lr)] @@ -213,7 +213,7 @@ result) (defn open-write - {:doc "Open a file for writing, returning a IOutputStream" + {:doc "Opens a file for writing. Returns an IOutputStream." :added "0.1"} [filename] (assert (string? filename) "Filename must be a string") @@ -244,8 +244,8 @@ :else (throw [::Exception "Expected a string or IOutputStream"]))) -(defn slurp - "Reads in the contents of input. Input must be a filename or an IInputStream" +(defn slurp + "Reads in the contents of input. Input must be a filename or an IInputStream." [input] (let [stream (cond (string? input) (open-read input) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi index 4836e7a6..0233d00a 100644 --- a/pixie/io/tcp.pxi +++ b/pixie/io/tcp.pxi @@ -64,7 +64,7 @@ (defn tcp-client - "Creates a TCP connection to the given ip (as a string) and port (an integer). Will return a TCPStream" + "Creates a TCP connection to the given ip (as a string) and port (an integer). Returns a TCPStream." [ip port] (let [client-addr (uv/sockaddr_in) uv-connect (uv/uv_connect_t) From 91309e886fce31598ec6912fcf2228e6c56d8124 Mon Sep 17 00:00:00 2001 From: Chris Shea Date: Fri, 18 Dec 2015 16:31:31 -0500 Subject: [PATCH 867/909] Add a few more docstrings --- pixie/stdlib.pxi | 30 +++++++++++++++++++++++++----- 1 file changed, 25 insertions(+), 5 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ec1ef49c..be13ccfe 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2245,6 +2245,11 @@ user => (refer 'pixie.string :exclude '(substring))" (defn merge-with + {:doc "Returns a map consisting of each map merged onto the first. If a + map contains a key that already exists in the result, the + value will be f applied to the value in the result map and + the value from the map being merged in." + :examples [["(merge-with + {:a 1 :b 2} {:a 3 :c 5} {:c 3 :d 4})" nil {:a 4, :b 2, :c 8 :d 4}]]} [f & maps] (cond (empty? maps) nil @@ -2570,24 +2575,29 @@ Expands to calls to `extend-type`." (into () coll)) (defmacro use + "Loads a namespace and refers all symbols from it." [ns] `(do (load-ns ~ns) (refer ~ns :refer :all))) (defn count-rf - "A Reducing function that counts the items reduced over" + "A Reducing function that counts the items reduced over." ([] 0) ([result] result) ([result _] (inc result))) (defn dispose! - "Finalizes use of the object by cleaning up resources used by the object" + "Finalizes use of the object by cleaning up resources used by the object." [x] (-dispose! x) nil) -(defmacro using [bindings & body] +(defmacro using + "Evaluates body with the bindings available as with let, + calling -dispose! on each name afterwards. Returns the value of the + last expression in body." + [bindings & body] (let [pairs (partition 2 bindings) names (map first pairs)] `(let [~@bindings @@ -2653,6 +2663,8 @@ Calling this function on something that is not ISeqable returns a seq with that {} m)) (defn mapv + {:doc "Returns a vector consisting of f applied to each element in col." + :examples [["(mapv inc '(1 2 3))" nil [2 3 4]]]} ([f col] (transduce (map f) conj col))) @@ -2687,7 +2699,10 @@ Calling this function on something that is not ISeqable returns a seq with that (def hash-map hashmap) -(defn zipmap [a b] +(defn zipmap + "Returns a map with the elements of a mapped to the corresponding + elements of b." + [a b] (into {} (map vector a b))) (extend -str Environment @@ -2775,6 +2790,7 @@ Calling this function on something that is not ISeqable returns a seq with that (defn bool? + "Returns true if x is a Bool." [x] (instance? Bool x)) @@ -2891,7 +2907,8 @@ Calling this function on something that is not ISeqable returns a seq with that (defprotocol IComparable (-compare [x y] - "Compare to objects returing 0 if the same -1 with x is logically smaller than y and 1 if x is logically larger.")) + "Compares two objects. Returns 0 when x is equal to y, -1 when x + is logically smaller than y, and 1 when x is logically larger.")) (defn compare-numbers [x y] @@ -2981,6 +2998,9 @@ Calling this function on something that is not ISeqable returns a seq with that (throw [::ComparisonError (str "Cannot compare: " x " to " y)])))) (defn compare + "Compares two objects. Returns 0 when x is equal to y, -1 when x is + logically smaller than y, and 1 when x is logically larger. x must + implement IComparable." [x y] (if (satisfies? IComparable x) (-compare x y) From 9788a3fad9d60fbaf014f8f3820278e5861379af Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 19 Dec 2015 19:27:33 +0000 Subject: [PATCH 868/909] freeze rpython --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index cbe6086c..0ef5b975 100644 --- a/Makefile +++ b/Makefile @@ -58,7 +58,7 @@ lib: $(EXTERNALS)/pypy: mkdir $(EXTERNALS); \ cd $(EXTERNALS); \ - curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2; \ + curl https://bitbucket.org/pypy/pypy/get/81254.tar.bz2 > pypy.tar.bz2; \ mkdir pypy; \ cd pypy; \ tar -jxf ../pypy.tar.bz2 --strip-components=1 From d3c18882baed7a9c001ed72ea7b468d04b059d47 Mon Sep 17 00:00:00 2001 From: Josh Glover Date: Mon, 21 Dec 2015 10:42:22 +0100 Subject: [PATCH 869/909] Makefile : do not fetch externals when present --- .gitignore | 2 +- Makefile | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/.gitignore b/.gitignore index 62c09a44..5928d907 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,5 @@ *.pyc -externals +externals* pixie-vm .idea lib diff --git a/Makefile b/Makefile index 0ef5b975..7a6848af 100644 --- a/Makefile +++ b/Makefile @@ -47,12 +47,13 @@ build_preload_no_jit: fetch_externals build: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) -fetch_externals: $(EXTERNALS)/pypy ./lib +fetch_externals: $(EXTERNALS)/pypy externals.fetched -lib: +externals.fetched: echo https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 curl -L https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 > /tmp/externals.tar.bz2 tar -jxf /tmp/externals.tar.bz2 --strip-components=2 + touch externals.fetched $(EXTERNALS)/pypy: @@ -92,6 +93,6 @@ clean_pxic: clean: clean_pxic rm -rf ./lib rm -rf ./include - rm -rf ./externals + rm -rf ./externals* rm -f ./pixie-vm rm -f ./*.pyc From 7a0c6906066f9ac8d0a55c7088a8da5d8847a4eb Mon Sep 17 00:00:00 2001 From: Stuart Hinson Date: Wed, 23 Dec 2015 16:32:40 -0500 Subject: [PATCH 870/909] implement cycle --- pixie/stdlib.pxi | 13 +++++++++++++ tests/pixie/tests/test-stdlib.pxi | 6 ++++++ 2 files changed, 19 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index be13ccfe..16c68916 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1746,6 +1746,19 @@ The new value is thus `(apply f current-value-of-atom args)`." s)))] (lazy-seq (step pred coll))))) +(defn cycle + [coll] + (if (empty? coll) + () + (let [cycle' + (fn cycle' [current] + (lazy-seq + (cons + (first current) + (let [rst (rest current)] + (cycle' (if (empty? rst) coll rst))))))] + (cycle' coll)))) + ;; TODO: use a transient map in the future (defn group-by {:doc "Groups the collection into a map keyed by the result of applying f on each element. The value at each key is a vector of elements in order of appearance." diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index ae219c75..8e96707f 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -568,6 +568,12 @@ (t/assert= (transduce (drop-while even?) conj [0 2] [1 4 6]) [0 2 1 4 6]) (t/assert= (transduce (drop-while even?) conj [0 2] [2 4 6 7 8]) [0 2 7 8])) +(t/deftest test-cycle + (t/assert= (cycle ()) ()) + (t/assert= (cycle nil) ()) + (t/assert= (take 5 (cycle '(1 2))) '(1 2 1 2 1)) + (t/assert= (take 3 (cycle [nil])) '(nil nil nil))) + (t/deftest test-trace (try (/ 0 0) From 98eedd2a24fede820c79beeaaff8be47603ac1b6 Mon Sep 17 00:00:00 2001 From: hswick Date: Wed, 30 Dec 2015 15:19:23 -0600 Subject: [PATCH 871/909] Added generate-docs.pxi --- generate-docs.pxi | 54 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) create mode 100644 generate-docs.pxi diff --git a/generate-docs.pxi b/generate-docs.pxi new file mode 100644 index 00000000..5f8abb5b --- /dev/null +++ b/generate-docs.pxi @@ -0,0 +1,54 @@ +(ns pixie.generate-docs + (:require [pixie.io :as io] + [pixie.string :as string])) + +(let [[namespace] program-arguments] + + (println "==============") + (println (name namespace)) + (println "==============") + + (load-ns (symbol namespace)) + (println) + + ;;Should be sorting the map + ;;Like so: (sort-by first map) + ;;However, I'm holding off until sort is properly supported + (doseq [[k v] (ns-map (the-ns namespace))] + (println (name k)) + (println "====================================") + (println) + + (if-let [m (meta @v)] + (do + ;(println m) + (if-let [doc (:doc m)];; + (println doc) + (println "No doc available :(")) + (println) + + (when-let (examples (:examples m)) + (println "**Examples:**") + (doseq [[code _ result] examples] + (println) + (println code) + (println) + (when (not (nil? result)) + (println "=> " result))) + (println)) + + (when-let (signatures (:signatures m)) + (println "**Signatures:**") + (println) + (doseq [sig signatures] + (println (str "- " sig))) + (println)) + + (when (and (:line-number m) (:file m)) + (let [file (str "pixie/" (last (string/split (:file m) "/")))] + (println (str "http://github.com/pixie-lang/pixie/blob/master/" + file "#L" (:line-number m)))) + (println))) + + (println "No meta data available :(")) + (println))) \ No newline at end of file From a7a81cd6d3946f188fb203f62383f64f1a431eed Mon Sep 17 00:00:00 2001 From: Pauli Jaakkola Date: Mon, 11 Jan 2016 18:05:05 +0200 Subject: [PATCH 872/909] Add CFloat test. --- tests/pixie/tests/test-ffi.pxi | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index f2d997c7..813098e9 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -1,6 +1,7 @@ (ns pixie.tests.test-ffi (require pixie.test :as t) - (require pixie.math :as m)) + (require pixie.math :as m) + (require pixie.ffi-infer :as i)) @@ -36,6 +37,17 @@ (t/deftest test-ffi-infer (t/assert= 0.5 (m/asin (m/sin 0.5)))) +(t/deftest test-cdouble + (i/with-config {:library "m" + :cxx-flags ["-lm"] + :includes ["math.h"]} + (i/defcfn sinf) + (i/defcfn asinf) + (i/defcfn cosf) + (i/defcfn powf)) + (t/assert= 0.5 (asinf (sinf 0.5))) + (t/assert= 1.0 (+ (powf (sinf 0.5) 2.0) (powf (cosf 0.5) 2.0)))) + (t/deftest test-ffi-callbacks (let [MAX 255 From baabdabcb1056624276caf07885ccfe25f7c0acf Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 21 Dec 2015 12:33:42 +0000 Subject: [PATCH 873/909] Remove remands of preloader --- .travis.yml | 5 ----- Makefile | 6 ------ target_preload.py | 32 -------------------------------- 3 files changed, 43 deletions(-) delete mode 100644 target_preload.py diff --git a/.travis.yml b/.travis.yml index 7176dc4b..77846892 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,14 +2,9 @@ sudo: false env: - JIT_OPTS='--opt=jit' TARGET_OPTS='target.py' - JIT_OPTS='' TARGET_OPTS='target.py' - #- JIT_OPTS='--opt=jit' TARGET_OPTS='target_preload.py' - #- JIT_OPTS='' TARGET_OPTS='target_preload.py' matrix: fast_finish: true - allow_failures: - - env: JIT_OPTS='--opt=jit' TARGET_OPTS='target_preload.py' - - env: JIT_OPTS='' TARGET_OPTS='target_preload.py' script: - make PYTHON=python build diff --git a/Makefile b/Makefile index 7a6848af..15f691e5 100644 --- a/Makefile +++ b/Makefile @@ -38,12 +38,6 @@ compile_basics: @echo -e "\n\n\n\nWARNING: Compiling core libs. If you want to modify one of these files delete the .pxic files first\n\n\n\n" ./pixie-vm -c pixie/uv.pxi -c pixie/io.pxi -c pixie/stacklets.pxi -c pixie/stdlib.pxi -c pixie/repl.pxi -build_preload_with_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target_preload.py 2>&1 >/dev/null | grep -v 'WARNING' - -build_preload_no_jit: fetch_externals - $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target_preload.py - build: fetch_externals $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) diff --git a/target_preload.py b/target_preload.py deleted file mode 100644 index 3bb0fbe8..00000000 --- a/target_preload.py +++ /dev/null @@ -1,32 +0,0 @@ -from target import entry_point, load_stdlib, init_load_path, LOAD_PATHS, load_path, BatchModeFn -from pixie.vm.atom import Atom -from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR -from pixie.vm.symbol import symbol -from pixie.vm.code import intern_var - -import pixie.vm.rt as rt -rt.init() - -load_path.set_root(rt.wrap(u"./")) -LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(u"./")))) -load_stdlib() - -BatchModeFn(["pixie/preload.pxi"]).invoke([]) - -def target(*args): - import pixie.vm.rt as rt - driver = args[0] - driver.exe_name = "pixie-vm" - rt.__config__ = args[0].config - - - - - - return entry_point, None - -import rpython.config.translationoption -print rpython.config.translationoption.get_combined_translation_config() - -if __name__ == "__main__": - entry_point(sys.argv) \ No newline at end of file From 9811e61c06f4750e217341c84bc55e2f28f1a30f Mon Sep 17 00:00:00 2001 From: Max Penet Date: Wed, 13 Jan 2016 09:36:44 +0100 Subject: [PATCH 874/909] improve pixie.math scope (adapted from work of @edw found here: https://gist.github.com/edw/87a4e234a33b5a1e0377) --- pixie/math.pxi | 107 ++++++++++++++++++++++++++++++++++--------------- 1 file changed, 74 insertions(+), 33 deletions(-) diff --git a/pixie/math.pxi b/pixie/math.pxi index 15a778c4..0cb53ae1 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -4,52 +4,93 @@ (i/with-config {:library "m" :cxx-flags ["-lm"] :includes ["math.h"]} - (i/defcfn sin) - (i/defcfn cos) - (i/defcfn tan) + (i/defconst M_E) + (i/defconst M_LOG2E) + (i/defconst M_LOG10E) + (i/defconst M_LN2) + (i/defconst M_LN10) + (i/defconst M_PI) + (i/defconst M_PI_2) + (i/defconst M_PI_4) + (i/defconst M_1_PI) + (i/defconst M_2_PI) + (i/defconst M_2_SQRTPI) + (i/defconst M_SQRT2) + (i/defconst M_SQRT1_2) - (i/defcfn asin) - (i/defcfn acos) - (i/defcfn atan) - (i/defcfn atan2) ; Arc tangent function of two variables. + (i/defcfn nan) + (i/defcfn ceil) + (i/defcfn floor) + (i/defcfn nearbyint) + (i/defcfn rint) + (i/defcfn lround) + (i/defcfn llrint) + (i/defcfn llround) + (i/defcfn trunc) - (i/defcfn sinh) - (i/defcfn cosh) - (i/defcfn tanh) + (i/defcfn fmod) + (i/defcfn remainder) + (i/defcfn remquo) - (i/defcfn asinh) - (i/defcfn acosh) - (i/defcfn atanh) + (i/defcfn fdim) + (i/defcfn fmax) + (i/defcfn fmin) + + (i/defcfn fma) + + (i/defcfn fabs) + (i/defcfn sqrt) + (i/defcfn cbrt) + (i/defcfn hypot) (i/defcfn exp) - (i/defcfn ldexp) + (i/defcfn exp2) + (i/defcfn exp10) + (i/defcfn expm1) (i/defcfn log) (i/defcfn log2) (i/defcfn log10) (i/defcfn log1p) + (i/defcfn logb) (i/defcfn ilogb) - ;; (i/defcfn modf) ;; Needs ffi support + (i/defcfn modf) + (i/defcfn frexp) + + (i/defcfn ldexp) + (i/defcfn scalbn) + (i/defcfn scalbln) + (i/defcfn pow) - (i/defcfn sqrt) - (i/defcfn ceil) - (i/defcfn fabs) - (i/defcfn floor) - (i/defcfn fmod) - (i/defconst M_E) ; base of natural logarithm, e - (i/defconst M_LOG2E) ; log2(e) - (i/defconst M_LOG10E) ; log10(e) - (i/defconst M_LN2) ; ln(2) - (i/defconst M_LN10) ; ln(10) - (i/defconst M_PI) ; pi - (i/defconst M_PI_2) ; pi / 2 - (i/defconst M_PI_4) ; pi / 4 - (i/defconst M_1_PI) ; 1 / pi - (i/defconst M_2_PI) ; 2 / pi - (i/defconst M_2_SQRTPI) ; 2 / sqrt(pi) - (i/defconst M_SQRT2) ; sqrt(2) - (i/defconst M_SQRT1_2)) ; sqrt(1/2) + (i/defcfn cos) + (i/defcfn sin) + (i/defcfn tan) + + (i/defcfn cosh) + (i/defcfn sinh) + (i/defcfn tanh) + + (i/defcfn acos) + (i/defcfn asin) + (i/defcfn atan) + (i/defcfn atan2) + + (i/defcfn acosh) + (i/defcfn asinh) + (i/defcfn atanh) + + (i/defcfn tgamma) + (i/defcfn lgamma) + + (i/defcfn j0) + (i/defcfn j1) + (i/defcfn jn) + (i/defcfn y0) + (i/defcfn y1) + (i/defcfn yn) + (i/defcfn erf) + (i/defcfn erfc)) From 780ee38e14b5bd069291a7a0f69f98049427749c Mon Sep 17 00:00:00 2001 From: David Schaefer Date: Sat, 23 Jan 2016 15:57:58 +0100 Subject: [PATCH 875/909] Raise runtime_error in to_float --- pixie/vm/numbers.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index ab80edbe..b9363665 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -320,7 +320,7 @@ def to_float(x): return rt.wrap(float(x.int_val())) if isinstance(x, BigInteger): return rt.wrap(x.bigint_val().tofloat()) - assert False + object.runtime_error(u"Cannot convert %s to float" %x.type().name()) def to_float_conv(c): if c == Float: From 034349daaf083970aedcc9aba44796d2291f231e Mon Sep 17 00:00:00 2001 From: David Schaefer Date: Mon, 25 Jan 2016 15:56:31 +0100 Subject: [PATCH 876/909] added to_float exception test --- tests/pixie/tests/test-ffi.pxi | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index 813098e9..ba5f9bd1 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -48,6 +48,14 @@ (t/assert= 0.5 (asinf (sinf 0.5))) (t/assert= 1.0 (+ (powf (sinf 0.5) 2.0) (powf (cosf 0.5) 2.0)))) +(t/deftest test-invalid-float-argument + (try + (m/sin "nil") + (catch ex (t/assert= (type ex) RuntimeException))) + (try + (m/exp nil) + (catch ex (t/assert= (type ex) RuntimeException))) + ) (t/deftest test-ffi-callbacks (let [MAX 255 From d397fd8131c58b38fd3b92ace9f77dcdff00e15e Mon Sep 17 00:00:00 2001 From: David Schaefer Date: Mon, 25 Jan 2016 16:38:41 +0100 Subject: [PATCH 877/909] using assert-throws? in test-invalid-float-argument --- tests/pixie/tests/test-ffi.pxi | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi index ba5f9bd1..4cc486dd 100644 --- a/tests/pixie/tests/test-ffi.pxi +++ b/tests/pixie/tests/test-ffi.pxi @@ -49,12 +49,8 @@ (t/assert= 1.0 (+ (powf (sinf 0.5) 2.0) (powf (cosf 0.5) 2.0)))) (t/deftest test-invalid-float-argument - (try - (m/sin "nil") - (catch ex (t/assert= (type ex) RuntimeException))) - (try - (m/exp nil) - (catch ex (t/assert= (type ex) RuntimeException))) + (t/assert-throws? (m/sin "nil")) + (t/assert-throws? (m/sin nil)) ) (t/deftest test-ffi-callbacks From d2b1b3acc3491273fa1e9b17af62078dcb3d491a Mon Sep 17 00:00:00 2001 From: Sebastiano Barrera Date: Wed, 3 Feb 2016 12:37:26 +0100 Subject: [PATCH 878/909] Add `eduction` and `completing` The `eduction` function was added, as requested in issue #483. The implementation and the tests were imported and adapted from Clojure's standard library (from [1] and [2], respectively). Since Clojure's implementation uses it, the function `completing` was imported as well (from the same source). As of now, Pixie's stdlib lacks a `sort`/`sort-by` function, one of the tests is disabled (commented out), and another is modified in order to avoid the call to `sort-by`. Sources: Both from Clojure's main repo, fetched at commit `3b98a00`. [1] https://github.com/clojure/clojure/blob/3b98a00e86f961550fb2eaee64e70754e04d1089/src/clj/clojure/core.clj#L7339 [2] https://github.com/clojure/clojure/blob/3b98a00e86f961550fb2eaee64e70754e04d1089/test/clojure/test_clojure/transducers.clj --- pixie/stdlib.pxi | 29 +++++++++++++++++++++++++++++ tests/pixie/tests/test-stdlib.pxi | 28 ++++++++++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 16c68916..46f656a1 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1884,6 +1884,35 @@ not enough elements were present." (when-let [s (seq coll)] (reductions f (f init (first s)) (rest s)))))))) +(defn completing + "Takes a reducing function f of 2 args and returns a fn suitable for + transduce by adding an arity-1 signature that calls cf (default - + identity) on the result argument." + ([f] (completing f identity)) + ([f cf] + (fn + ([] (f)) + ([x] (cf x)) + ([x y] (f x y))))) + +(deftype Eduction [xform coll] + IReduce + (-reduce [self f init] + ;; NB (completing f) isolates completion of inner rf from outer rf + (transduce xform (completing f) init coll)) + + ISeqable + (-seq [self] + (reduce conj self))) + +(defn eduction + "Returns a reducible/iterable application of the transducers + to the items in coll. Transducers are applied in order as if + combined with comp. Note that these applications will be + performed every time reduce/iterator is called." + [& xforms] + (->Eduction (apply comp (butlast xforms)) (last xforms))) + (defn destructure [binding expr] (cond (symbol? binding) [binding expr] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 8e96707f..48e3b4e1 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -574,6 +574,34 @@ (t/assert= (take 5 (cycle '(1 2))) '(1 2 1 2 1)) (t/assert= (take 3 (cycle [nil])) '(nil nil nil))) +(t/deftest test-eduction + ;; one xform + (t/assert= [1 2 3 4 5] + (eduction (map inc) (range 5))) + ;; multiple xforms + (t/assert= ["2" "4"] + (eduction (map inc) (filter even?) (map str) (range 5))) + ;; materialize at the end + ;; TODO: For when sort is implemented + ;; (t/assert= [1 1 1 1 2 2 2 3 3 4] + ;; (->> (range 5) + ;; (eduction (mapcat range) (map inc)) + ;; sort)) + (t/assert= [1 1 2 1 2 3 1 2 3 4] + (vec (->> (range 5) + (eduction (mapcat range) (map inc))))) + (t/assert= {1 4, 2 3, 3 2, 4 1} + (->> (range 5) + (eduction (mapcat range) (map inc)) + frequencies)) + (t/assert= ["tac" "god" "hsif" "drib" "kravdraa"] + (->> ["cat" "dog" "fish" "bird" "aardvark"] + (eduction (map pixie.string/reverse)) + (seq))) + ;; expanding transducer with nils + (t/assert= '(1 2 3 nil 4 5 6 nil) + (eduction cat [[1 2 3 nil] [4 5 6 nil]]))) + (t/deftest test-trace (try (/ 0 0) From 2789b879e5b3b0770bccabd28e429fad851d8769 Mon Sep 17 00:00:00 2001 From: Sebastiano Barrera Date: Wed, 3 Feb 2016 16:25:35 +0100 Subject: [PATCH 879/909] eduction: remove superfluous test --- tests/pixie/tests/test-stdlib.pxi | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 48e3b4e1..6dfd3efc 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -582,11 +582,6 @@ (t/assert= ["2" "4"] (eduction (map inc) (filter even?) (map str) (range 5))) ;; materialize at the end - ;; TODO: For when sort is implemented - ;; (t/assert= [1 1 1 1 2 2 2 3 3 4] - ;; (->> (range 5) - ;; (eduction (mapcat range) (map inc)) - ;; sort)) (t/assert= [1 1 2 1 2 3 1 2 3 4] (vec (->> (range 5) (eduction (mapcat range) (map inc))))) From df3b2a0a1b318e01aadb24908a60fc0273427fea Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Tue, 9 Feb 2016 21:31:09 +0000 Subject: [PATCH 880/909] make typing less verbose Object's type method is now used by default so all subtypes can use the inherited implementation --- pixie/vm/array.py | 8 -------- pixie/vm/atom.py | 3 --- pixie/vm/code.py | 33 --------------------------------- pixie/vm/cons.py | 3 --- pixie/vm/custom_types.py | 2 -- pixie/vm/interpreter.py | 4 ---- pixie/vm/lazy_seq.py | 3 --- pixie/vm/libs/env.py | 3 --- pixie/vm/libs/ffi.py | 14 -------------- pixie/vm/libs/path.py | 3 --- pixie/vm/libs/pxic/writer.py | 2 -- pixie/vm/map_entry.py | 3 --- pixie/vm/numbers.py | 15 --------------- pixie/vm/object.py | 15 ++++++--------- pixie/vm/persistent_hash_map.py | 6 ------ pixie/vm/persistent_hash_set.py | 3 --- pixie/vm/persistent_list.py | 5 ----- pixie/vm/persistent_vector.py | 9 --------- pixie/vm/primitives.py | 10 ---------- pixie/vm/reader.py | 10 ---------- pixie/vm/reduced.py | 2 -- pixie/vm/stacklet.py | 5 +---- pixie/vm/string.py | 6 ------ pixie/vm/string_builder.py | 5 +---- pixie/vm/threads.py | 3 --- pixie/vm/util.py | 3 --- 26 files changed, 8 insertions(+), 170 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 3bde8642..9d15f550 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -14,8 +14,6 @@ class Array(object.Object): _type = object.Type(u"pixie.stdlib.Array") _immutable_fields_ = ["_list"] - def type(self): - return Array._type def __init__(self, lst): self._list = lst @@ -98,9 +96,6 @@ def reduce(self, f, init): init = f.invoke([init, rt.nth(self._w_array, rt.wrap(x))]) return init - def type(self): - return self._type - @extend(proto._first, ArraySeq) def _first(self): assert isinstance(self, ArraySeq) @@ -174,9 +169,6 @@ def __init__(self, size): for x in range(size): self._buffer[x] = chr(0) - def type(self): - return ByteArray._type - def __del__(self): lltype.free(self._buffer, flavor="raw") diff --git a/pixie/vm/atom.py b/pixie/vm/atom.py index 77ecd50d..28254673 100644 --- a/pixie/vm/atom.py +++ b/pixie/vm/atom.py @@ -7,9 +7,6 @@ class Atom(object.Object): _type = object.Type(u"pixie.stdlib.Atom") - def type(self): - return Atom._type - def with_meta(self, meta): return Atom(self._boxed_value, meta) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index 14883796..5a54ea25 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -148,9 +148,6 @@ class MultiArityFn(BaseCode): _immutable_fields_ = ["_arities[*]", "_required_arity", "_rest_fn"] - def type(self): - return MultiArityFn._type - def __init__(self, name, arities, required_arity=0, rest_fn=None, meta=nil): BaseCode.__init__(self) self._name = name @@ -199,9 +196,6 @@ class NativeFn(BaseCode): def __init__(self, doc=None): BaseCode.__init__(self) - def type(self): - return NativeFn._type - def invoke(self, args): return self.inner_invoke(args) @@ -217,9 +211,6 @@ class Code(BaseCode): _type = object.Type(u"pixie.stdlib.Code") _immutable_fields_ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta", "_debug_points"] - def type(self): - return Code._type - def __init__(self, name, arity, bytecode, consts, stack_size, debug_points, meta=nil): BaseCode.__init__(self) self._arity = arity @@ -277,9 +268,6 @@ class VariadicCode(BaseCode): _immutable_fields_ = ["_required_arity", "_code", "_meta"] _type = object.Type(u"pixie.stdlib.VariadicCode") - def type(self): - return VariadicCode._type - def __init__(self, code, required_arity, meta=nil): BaseCode.__init__(self) self._required_arity = r_uint(required_arity) @@ -319,9 +307,6 @@ class Closure(BaseCode): _type = object.Type(u"pixie.stdlib.Closure") _immutable_fields_ = ["_closed_overs[*]", "_code", "_meta"] - def type(self): - return Closure._type - def __init__(self, code, closed_overs, meta=nil): BaseCode.__init__(self) affirm(isinstance(code, Code), u"Code argument to Closure must be an instance of Code") @@ -373,9 +358,6 @@ def get_debug_points(self): class Undefined(object.Object): _type = object.Type(u"pixie.stdlib.Undefined") - def type(self): - return Undefined._type - undefined = Undefined() @@ -413,9 +395,6 @@ class Var(BaseCode): _type = object.Type(u"pixie.stdlib.Var") _immutable_fields_ = ["_ns_ref"] - def type(self): - return Var._type - def __init__(self, ns_ref, ns, name): BaseCode.__init__(self) self._ns_ref = ns_ref @@ -497,9 +476,6 @@ class Namespace(object.Object): _immutable_fields_ = ["_rev?"] - def type(self): - return Namespace._type - def __init__(self, name): self._rev = 0 self._registry = {} @@ -636,9 +612,6 @@ class Protocol(object.Object): _immutable_fields_ = ["_rev?"] - def type(self): - return Protocol._type - def __init__(self, name): self._name = name self._polyfns = {} @@ -663,9 +636,6 @@ def satisfies(self, tp): class PolymorphicFn(BaseCode): _type = object.Type(u"pixie.stdlib.PolymorphicFn") - def type(self): - return PolymorphicFn._type - _immutable_fields_ = ["_rev?"] def __init__(self, name, protocol): @@ -745,9 +715,6 @@ class DoublePolymorphicFn(BaseCode): """A function that is polymorphic on the first two arguments""" _type = object.Type(u"pixie.stdlib.DoublePolymorphicFn") - def type(self): - return DefaultProtocolFn._type - _immutable_fields_ = ["_rev?"] def __init__(self, name, protocol): diff --git a/pixie/vm/cons.py b/pixie/vm/cons.py index bc27357e..27fcc109 100644 --- a/pixie/vm/cons.py +++ b/pixie/vm/cons.py @@ -7,9 +7,6 @@ class Cons(object.Object): _type = object.Type(u"pixie.stdlib.Cons") - def type(self): - return Cons._type - def __init__(self, head, tail, meta=nil): self._first = head self._next = tail diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 0cbbc4ef..d5fb3047 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -216,8 +216,6 @@ def get_field(inst, field): class AbstractMutableCell(Object): _type = Type(u"pixie.stdlib.AbstractMutableCell") - def type(self): - return self._type def set_mutable_cell_value(self, ct, fields, nm, idx, value): pass diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index d279e2fb..cb11508d 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -159,10 +159,6 @@ def __init__(self, frame, val): self._frame = frame self._val = val - def type(self): - return ShallowContinuation._type - - def is_finished(self): return self._frame.finished diff --git a/pixie/vm/lazy_seq.py b/pixie/vm/lazy_seq.py index b95c3c94..65cb744c 100644 --- a/pixie/vm/lazy_seq.py +++ b/pixie/vm/lazy_seq.py @@ -9,9 +9,6 @@ class LazySeq(object.Object): _type = object.Type(u"pixie.stdlib.LazySeq") - def type(self): - return LazySeq._type - def __init__(self, fn, meta=nil): self._fn = fn self._meta = meta diff --git a/pixie/vm/libs/env.py b/pixie/vm/libs/env.py index 25bfb7ca..39b806df 100644 --- a/pixie/vm/libs/env.py +++ b/pixie/vm/libs/env.py @@ -10,9 +10,6 @@ class Environment(Object): _type = Type(u"pixie.stdlib.Environment") - def type(self): - return Environment._type - def val_at(self, key, not_found): if not isinstance(key, String): runtime_error(u"Environment variables are strings ") diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py index 9cc0d348..cdd670b4 100644 --- a/pixie/vm/libs/ffi.py +++ b/pixie/vm/libs/ffi.py @@ -41,9 +41,6 @@ def __init__(self, name): class ExternalLib(object.Object): _type = object.Type(u"pixie.stdlib.ExternalLib") - def type(self): - return ExternalLib._type - def __init__(self, nm): assert isinstance(nm, unicode) self._name = nm @@ -94,9 +91,6 @@ class FFIFn(object.Object): _type = object.Type(u"pixie.stdlib.FFIFn") _immutable_fields_ = ["_name", "_f_ptr", "_c_fn_type"] - def type(self): - return FFIFn._type - def __init__(self, name, fn_ptr, c_fn_type): assert isinstance(c_fn_type, CFunctionType) self._rev = 0 @@ -219,9 +213,6 @@ class Buffer(PointerType): """ _type = object.Type(u"pixie.stdlib.Buffer") - def type(self): - return Buffer._type - def __init__(self, size): self._size = size self._used_size = 0 @@ -502,8 +493,6 @@ def ffi_type(self): class VoidP(PointerType): _type = cvoidp - def type(self): - return VoidP._type def __init__(self, raw_data): self._raw_data = raw_data @@ -604,9 +593,6 @@ def __init__(self, cft, raw_closure, id, fn): self._is_invoked = False self._unique_id = id - def type(self): - return CCallback._type - def get_raw_closure(self): return self._raw_closure diff --git a/pixie/vm/libs/path.py b/pixie/vm/libs/path.py index 8afd0ef2..e512efe8 100644 --- a/pixie/vm/libs/path.py +++ b/pixie/vm/libs/path.py @@ -8,9 +8,6 @@ class Path(Object): _type = Type(u"pixie.path.Path") - def type(self): - return Path._type - def __init__(self, top): self._path = rt.name(top) diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py index 7aedcb17..0d2e511a 100644 --- a/pixie/vm/libs/pxic/writer.py +++ b/pixie/vm/libs/pxic/writer.py @@ -73,8 +73,6 @@ def finish(self): class WriterBox(Object): _type = Type(u"pixie.stdlib.WriterBox") - def type(self): - return WriterBox._type def __init__(self, wtr): self._pxic_writer = wtr diff --git a/pixie/vm/map_entry.py b/pixie/vm/map_entry.py index 543fb7d3..3b14a6c4 100644 --- a/pixie/vm/map_entry.py +++ b/pixie/vm/map_entry.py @@ -6,9 +6,6 @@ class MapEntry(object.Object): _type = object.Type(u"pixie.stdlib.MapEntry") - def type(self): - return MapEntry._type - def __init__(self, key, val): self._key = key self._val = val diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index b9363665..8e637365 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -13,9 +13,6 @@ class Number(object.Object): _type = object.Type(u"pixie.stdlib.Number") - def type(self): - return Number._type - class Integer(Number): _type = object.Type(u"pixie.stdlib.Integer", Number._type) _immutable_fields_ = ["_int_val"] @@ -32,9 +29,6 @@ def r_uint_val(self): def promote(self): return Integer(jit.promote(self._int_val)) - def type(self): - return Integer._type - zero_int = Integer(0) one_int = Integer(1) @@ -48,9 +42,6 @@ def __init__(self, bi_val): def bigint_val(self): return self._bigint_val - def type(self): - return BigInteger._type - class Float(Number): _type = object.Type(u"pixie.stdlib.Float", Number._type) _immutable_fields_ = ["_float_val"] @@ -61,9 +52,6 @@ def __init__(self, f_val): def float_val(self): return self._float_val - def type(self): - return Float._type - class Ratio(Number): _type = object.Type(u"pixie.stdlib.Ratio", Number._type) _immutable_fields_ = ["_numerator", "_denominator"] @@ -79,9 +67,6 @@ def numerator(self): def denominator(self): return self._denominator - def type(self): - return Ratio._type - @wrap_fn def ratio_write(obj): assert isinstance(obj, Ratio) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index ccddf276..1b47e22c 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -26,8 +26,12 @@ class Object(object): """ _attrs_ = () + # Initialized after Type is defined. + _type = None + def type(self): - affirm(False, u".type isn't overloaded") + assert(isinstance(self.__class__._type, Type)) + return self.__class__._type @jit.unroll_safe def invoke(self, args): @@ -109,9 +113,6 @@ def __init__(self, name, parent=None, object_inited=True): def name(self): return self._name - def type(self): - return Type._type - def parent(self): return self._parent @@ -157,9 +158,6 @@ def __init__(self, msg, data): self._data = data self._trace = [] - def type(self): - return RuntimeException._type - def __repr__(self): import pixie.vm.rt as rt s = [] @@ -211,8 +209,7 @@ def safe_invoke(f, args): class ErrorInfo(Object): _type = Type(u"pixie.stdlib.ErrorInfo") - def type(self): - return ErrorInfo._type + def __init__(self): pass diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index c36dbbdc..7cab577c 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -19,9 +19,6 @@ def __init__(self): class PersistentHashMap(object.Object): _type = object.Type(u"pixie.stdlib.PersistentHashMap") - def type(self): - return PersistentHashMap._type - def __init__(self, cnt, root, meta=nil): self._cnt = cnt self._root = root @@ -62,9 +59,6 @@ def without(self, key): class INode(object.Object): _type = object.Type(u"pixie.stdlib.INode") - def type(self): - return INode._type - def assoc_inode(self, shift, hash_val, key, val, added_leaf): pass diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py index 9327ea42..f2bd5d4a 100644 --- a/pixie/vm/persistent_hash_set.py +++ b/pixie/vm/persistent_hash_set.py @@ -12,9 +12,6 @@ class PersistentHashSet(object.Object): _type = object.Type(u"pixie.stdlib.PersistentHashSet") - def type(self): - return PersistentHashSet._type - def __init__(self, meta, m): self._meta = meta self._map = m diff --git a/pixie/vm/persistent_list.py b/pixie/vm/persistent_list.py index 54d6b11a..10514b43 100644 --- a/pixie/vm/persistent_list.py +++ b/pixie/vm/persistent_list.py @@ -8,9 +8,6 @@ class PersistentList(object.Object): _type = object.Type(u"pixie.stdlib.PersistentList") - def type(self): - return PersistentList._type - def __init__(self, head, tail, cnt, meta=nil): self._first = head self._next = tail @@ -100,8 +97,6 @@ def _with_meta(self, meta): class EmptyList(object.Object): _type = object.Type(u"pixie.stdlib.EmptyList") - def type(self): - return EmptyList._type def __init__(self, meta=nil): self._meta = meta diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 202e3c72..2740caf5 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -13,9 +13,6 @@ class Node(object.Object): _type = object.Type(u"pixie.stdlib.PersistentVectorNode") - def type(self): - return Node._type - def __init__(self, edit, array=None): self._edit = edit self._array = [None] * 32 if array is None else array @@ -27,9 +24,6 @@ def __init__(self, edit, array=None): class PersistentVector(object.Object): _type = object.Type(u"pixie.stdlib.PersistentVector") - def type(self): - return PersistentVector._type - def __init__(self, meta, cnt, shift, root, tail): self._meta = meta self._cnt = cnt @@ -211,9 +205,6 @@ def new_path(edit, level, node): class TransientVector(object.Object): _type = object.Type(u"pixie.stdlib.TransientVector") - def type(self): - return TransientVector._type - def __init__(self, cnt, shift, root, tail): self._cnt = cnt self._shift = shift diff --git a/pixie/vm/primitives.py b/pixie/vm/primitives.py index 6bab30aa..5924b6a5 100644 --- a/pixie/vm/primitives.py +++ b/pixie/vm/primitives.py @@ -1,25 +1,15 @@ import pixie.vm.object as object - class Nil(object.Object): _type = object.Type(u"pixie.stdlib.Nil") def __repr__(self): return u"nil" - def type(self): - return Nil._type - - nil = Nil() - class Bool(object.Object): _type = object.Type(u"pixie.stdlib.Bool") - def type(self): - return Bool._type - - true = Bool() false = Bool() diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 84dccfba..09e4fdf9 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -43,9 +43,6 @@ class PlatformReader(object.Object): _type = object.Type(u"PlatformReader") - def type(self): - return PlatformReader._type - def read(self): assert False @@ -57,8 +54,6 @@ def reset_line(self): class StringReader(PlatformReader): _type = object.Type(u"pixie.stdlib.StringReader") - def type(self): - return StringReader._type def __init__(self, s): affirm(isinstance(s, unicode), u"StringReader requires unicode") @@ -111,8 +106,6 @@ def reader_fn(fn): class LinePromise(object.Object): _type = object.Type(u"pixie.stdlib.LinePromise") - def type(self): - return LinePromise._type def __init__(self): self._chrs = [] @@ -770,9 +763,6 @@ def read_symbol(rdr, ch): class EOF(object.Object): _type = object.Type(u"EOF") - def type(self): - return EOF._type - eof = EOF() diff --git a/pixie/vm/reduced.py b/pixie/vm/reduced.py index d1d703ea..c94739a6 100644 --- a/pixie/vm/reduced.py +++ b/pixie/vm/reduced.py @@ -6,8 +6,6 @@ class Reduced(object.Object): _type = object.Type(u"pixie.stdlib.Reduced") - def type(self): - return Reduced._type def __init__(self, boxed_value): self._boxed_value = boxed_value diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index da42cfc0..d4e8e5c4 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -25,9 +25,6 @@ def __init__(self, h): self._stacklet_handle = h self._used = False - def type(self): - return self._type - def invoke(self, args): affirm(not self._used, u"Can only call a given stacklet handle once.") affirm(len(args) == 1, u"Only one arg should be handed to a stacklet handle") @@ -53,4 +50,4 @@ def new_handler(h, _): def new_stacklet(fn): global_state._val = fn h = global_state._th.new(new_handler) - return StackletHandle(h) \ No newline at end of file + return StackletHandle(h) diff --git a/pixie/vm/string.py b/pixie/vm/string.py index c9be8844..17327fa9 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -11,9 +11,6 @@ class String(Object): _type = Type(u"pixie.stdlib.String") - def type(self): - return String._type - def __init__(self, s): #assert isinstance(s, unicode) self._str = s @@ -76,9 +73,6 @@ class Character(Object): _type = Type(u"pixie.stdlib.Character") _immutable_fields_ = ["_char_val"] - def type(self): - return Character._type - def __init__(self, i): assert isinstance(i, int) self._char_val = i diff --git a/pixie/vm/string_builder.py b/pixie/vm/string_builder.py index d2a94296..588b00a5 100644 --- a/pixie/vm/string_builder.py +++ b/pixie/vm/string_builder.py @@ -6,9 +6,6 @@ class StringBuilder(Object): _type = Type(u"pixie.stdlib.StringBuilder") - def type(self): - return StringBuilder._type - def __init__(self): self._strs = [] @@ -34,4 +31,4 @@ def _str(self): @as_var("-string-builder") def _string_builder(): - return StringBuilder() \ No newline at end of file + return StringBuilder() diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index 5140608b..b523aa2f 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -65,9 +65,6 @@ class Lock(Object): def __init__(self, ll_lock): self._ll_lock = ll_lock - def type(self): - return Lock._type - @as_var("-create-lock") def _create_lock(): diff --git a/pixie/vm/util.py b/pixie/vm/util.py index 931ba731..cd97ed7b 100644 --- a/pixie/vm/util.py +++ b/pixie/vm/util.py @@ -78,9 +78,6 @@ def mix_coll_hash(hash, count): class HashingState(Object): _type = Type(u"pixie.stdlib.HashingState") - def type(self): - return HashingState._type - def __init__(self): self._n = r_uint(0) self._hash = r_uint(1) From 6e9ec11b8608e80fc4bf817490c27666f6388909 Mon Sep 17 00:00:00 2001 From: keymone Date: Thu, 11 Feb 2016 18:17:44 +0100 Subject: [PATCH 881/909] Make pixie.math/exp10 platform-specific OS X math library doesn't define exp10, define it within math.pxi instead. --- pixie/math.pxi | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pixie/math.pxi b/pixie/math.pxi index 0cb53ae1..253bc132 100644 --- a/pixie/math.pxi +++ b/pixie/math.pxi @@ -1,5 +1,6 @@ (ns pixie.math - (:require [pixie.ffi-infer :as i])) + (:require [pixie.ffi-infer :as i] + [pixie.string :as s])) (i/with-config {:library "m" :cxx-flags ["-lm"] @@ -45,7 +46,8 @@ (i/defcfn exp) (i/defcfn exp2) - (i/defcfn exp10) + (if-not (s/starts-with? pixie.platform/name "darwin") + (i/defcfn exp10)) (i/defcfn expm1) (i/defcfn log) @@ -94,3 +96,6 @@ (i/defcfn erf) (i/defcfn erfc)) + +(if (s/starts-with? pixie.platform/name "darwin") + (defn exp10 [x] (pow 10.0 x))) From 7b0680be167c2f6aebf6586ed3ce255b17bf7e43 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 11 Feb 2016 16:12:40 +0000 Subject: [PATCH 882/909] Added iterate function and tests --- pixie/stdlib.pxi | 8 ++++++++ tests/pixie/tests/test-stdlib.pxi | 4 ++++ 2 files changed, 12 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 46f656a1..93abc01c 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -3072,3 +3072,11 @@ ex: (vary-meta x assoc :foo 42)" (swap! cache assoc argsv ret) ret) val))))) + +(defn iterate + {:doc "Returns a lazy sequence of x, (f x), (f (f x)) etc. f must be free of + side-effects" + :signatures [[f x]] + :added "0.1"} + [f x] + (lazy-seq (cons x (iterate f (f x))))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 6dfd3efc..0868cac8 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -761,3 +761,7 @@ (t/deftest test-memoize (let [f (memoize rand)] (t/assert= (f) (f)))) + +(t/deftest test-iterate + (t/assert= (take 5 (iterate inc 5)) '(5 6 7 8 9)) + (t/assert= (str (type (iterate inc 1))) "")) From 3b9d76a40bfc6614fafbe143de6b6743bc2556d2 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Fri, 12 Feb 2016 16:10:32 +0000 Subject: [PATCH 883/909] Added IReduce to iterate --- pixie/stdlib.pxi | 10 ++++++++++ tests/pixie/tests/test-stdlib.pxi | 3 ++- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 93abc01c..de683993 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -3073,6 +3073,16 @@ ex: (vary-meta x assoc :foo 42)" ret) val))))) +(deftype Iterate [f x] + IReduce + (-reduce [self f init] + (loop [acc (f (if (nil? init) + (first self) + init))] + (if (reduced? acc) + @acc + (recur (f acc)))))) + (defn iterate {:doc "Returns a lazy sequence of x, (f x), (f (f x)) etc. f must be free of side-effects" diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 0868cac8..70adcd6e 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -764,4 +764,5 @@ (t/deftest test-iterate (t/assert= (take 5 (iterate inc 5)) '(5 6 7 8 9)) - (t/assert= (str (type (iterate inc 1))) "")) + (t/assert= (str (type (iterate inc 1))) "") + (t/assert= (reduce (fn [a v] (if (< a 10) (+ a v) (reduced a))) (iterate (partial + 2) 1)) 16)) From 027220f17607852550e54e13abedd2e7ece55a31 Mon Sep 17 00:00:00 2001 From: keymone Date: Sat, 13 Feb 2016 11:11:15 +0100 Subject: [PATCH 884/909] find python using which -a --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 15f691e5..e27182ee 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ all: help EXTERNALS=externals -PYTHON ?= python2 +PYTHON ?= `env which -a python2 python2.7 | head -n1` PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy From ffbb0822a80e6f4344c5d179bba233b81c62410e Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Mon, 15 Feb 2016 16:27:04 +0000 Subject: [PATCH 885/909] Refactored iterate into type and added IReduce --- pixie/stdlib.pxi | 14 ++++++++------ tests/pixie/tests/test-stdlib.pxi | 5 +++-- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index de683993..311df7fb 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -3075,13 +3075,15 @@ ex: (vary-meta x assoc :foo 42)" (deftype Iterate [f x] IReduce - (-reduce [self f init] - (loop [acc (f (if (nil? init) - (first self) - init))] + (-reduce [self rf init] + (loop [col (rest self) + acc (rf init (first self))] (if (reduced? acc) @acc - (recur (f acc)))))) + (recur (rest col) (rf acc (first col)))))) + ISeq + (-seq [self] + (cons x (lazy-seq* (fn [] (->Iterate f (f x))))))) (defn iterate {:doc "Returns a lazy sequence of x, (f x), (f (f x)) etc. f must be free of @@ -3089,4 +3091,4 @@ ex: (vary-meta x assoc :foo 42)" :signatures [[f x]] :added "0.1"} [f x] - (lazy-seq (cons x (iterate f (f x))))) + (->Iterate f x)) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 70adcd6e..12ad9912 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -764,5 +764,6 @@ (t/deftest test-iterate (t/assert= (take 5 (iterate inc 5)) '(5 6 7 8 9)) - (t/assert= (str (type (iterate inc 1))) "") - (t/assert= (reduce (fn [a v] (if (< a 10) (+ a v) (reduced a))) (iterate (partial + 2) 1)) 16)) + (t/assert= (reduce (fn [a v] (reduced "foo")) 0 (iterate inc 1)) "foo") + (t/assert= (reduce (fn [a v] (if (< a 10) (+ a v) (reduced a))) 0 (iterate (partial + 2) 1)) 16)) + From 501b241d6e31753decb3d9435830180466cb3b5f Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 18 Feb 2016 16:56:06 +0100 Subject: [PATCH 886/909] Added mkdtmp to stdlib --- pixie/stdlib.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 311df7fb..8953592d 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -17,6 +17,7 @@ (def srand (ffi-fn libc "srand" [CInt] CInt)) (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) +(def mkdtemp (ffi-fn libc "mkdtemp" [CCharP] CCharP)) (def libm (ffi-library (str "libm." pixie.platform/so-ext))) (def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) From df8afb0bc1486b116373a7361832efe1f510d2ac Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 18 Feb 2016 17:08:30 +0100 Subject: [PATCH 887/909] Added rmdir to stdlib --- pixie/stdlib.pxi | 1 + 1 file changed, 1 insertion(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 8953592d..02916e72 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -18,6 +18,7 @@ (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) (def mkdtemp (ffi-fn libc "mkdtemp" [CCharP] CCharP)) +(def rmdir (ffi-fn libc "rmdir" [CCharP] CCharP)) (def libm (ffi-library (str "libm." pixie.platform/so-ext))) (def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) From 8bed3a84edac36705e6e765119d4a417610a75e6 Mon Sep 17 00:00:00 2001 From: Matt Carroll Date: Thu, 18 Feb 2016 17:26:15 +0100 Subject: [PATCH 888/909] Added mkdir and rm --- pixie/stdlib.pxi | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 02916e72..8ccfe2df 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -18,7 +18,9 @@ (def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) (def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) (def mkdtemp (ffi-fn libc "mkdtemp" [CCharP] CCharP)) +(def mkdir (ffi-fn libc "mkdir" [CCharP] CCharP)) (def rmdir (ffi-fn libc "rmdir" [CCharP] CCharP)) +(def rm (ffi-fn libc "remove" [CCharP] CCharP)) (def libm (ffi-library (str "libm." pixie.platform/so-ext))) (def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) From 3f81e6576eb58827a519ea985f42971e0ae1163f Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 19 Feb 2016 00:06:19 +0000 Subject: [PATCH 889/909] fixes the speed of iterates reduce the reduce implementation was creating some lazy lists --- pixie/stdlib.pxi | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 311df7fb..fab91331 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -3076,11 +3076,11 @@ ex: (vary-meta x assoc :foo 42)" (deftype Iterate [f x] IReduce (-reduce [self rf init] - (loop [col (rest self) - acc (rf init (first self))] + (loop [next (f x) + acc (rf init x)] (if (reduced? acc) @acc - (recur (rest col) (rf acc (first col)))))) + (recur (f next) (rf acc next))))) ISeq (-seq [self] (cons x (lazy-seq* (fn [] (->Iterate f (f x))))))) From 94d607944ecdc673964c2d6969c8e6f73ae8abbc Mon Sep 17 00:00:00 2001 From: keymone Date: Fri, 19 Feb 2016 16:19:14 +0100 Subject: [PATCH 890/909] deref macro? arg if it's a var --- pixie/vm/stdlib.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index 79a9061c..d8611d72 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -687,7 +687,8 @@ def set_macro(f): @returns(bool) @as_var("macro?") -def macro_QMARK_(f): +def macro_QMARK_(v): + f = v.deref() if isinstance(v, Var) else v return true if isinstance(f, BaseCode) and f.is_macro() else false @returns(unicode) From 02971267f7fa291c87d89962448f951c947d0854 Mon Sep 17 00:00:00 2001 From: keymone Date: Fri, 19 Feb 2016 17:37:58 +0100 Subject: [PATCH 891/909] look for symbol in current ns --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 3d9c2443..77b3cede 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2726,7 +2726,7 @@ Calling this function on something that is not ISeqable returns a seq with that (= () form)) form (let [[sym & args] form - fvar (resolve sym)] + fvar (resolve-in *ns* sym)] (if (and fvar (macro? @fvar)) (apply @fvar args) form)))) From bb7b335d89b141a4bcf4e7af21d17ce4e65ccce1 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sat, 5 Mar 2016 15:20:41 +0000 Subject: [PATCH 892/909] catch exception when compiling with -c --- target.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/target.py b/target.py index 7eb8932c..71be1c7d 100644 --- a/target.py +++ b/target.py @@ -117,8 +117,11 @@ def __init__(self, filename): def inner_invoke(self, args): import pixie.vm.rt as rt - - rt.compile_file(rt.wrap(self._filename)) + try: + rt.compile_file(rt.wrap(self._filename)) + except WrappedException as ex: + print "Error: ", ex._ex.__repr__() + os._exit(1) class IsPreloadFlag(object): From bbdde9cd2e9768b95ec6071619a15c1b54d3d080 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 6 Mar 2016 01:02:39 +0000 Subject: [PATCH 893/909] keywords can start with numbers or true, false, nil --- pixie/vm/reader.py | 21 ++++++++++++++++--- tests/pixie/tests/test-keywords.pxi | 32 ++++++++++++++++++++++++++++- 2 files changed, 49 insertions(+), 4 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 09e4fdf9..9cc2e36a 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -10,7 +10,6 @@ from pixie.vm.keyword import keyword, Keyword import pixie.vm.rt as rt from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR -from pixie.vm.libs.libedit import _readline from pixie.vm.string import Character from pixie.vm.code import wrap_fn from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP @@ -306,6 +305,21 @@ def invoke(self, rdr, ch): return cons(symbol(u"quote"), cons(itm)) class KeywordReader(ReaderHandler): + def read(self, rdr): + acc = [] + + try: + while True: + ch = rdr.read() + if is_whitespace(ch) or is_terminating_macro(ch): + rdr.unread() + break + acc.append(ch) + except EOFError: + pass + sym_str = u"".join(acc) + return symbol(sym_str) + def fqd(self, itm): ns_alias = rt.namespace(itm) current_nms = rt.ns.deref() @@ -319,11 +333,12 @@ def fqd(self, itm): def invoke(self, rdr, ch): ch = rdr.read() if ch == u":": - itm = read_inner(rdr, True) + itm = self.read(rdr) return self.fqd(itm) else: rdr.unread() - itm = read_inner(rdr, True) + itm = self.read(rdr) + return keyword(rt.name(itm), rt.namespace(itm)) class LiteralStringReader(ReaderHandler): diff --git a/tests/pixie/tests/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi index 06e2deb7..1094cf45 100644 --- a/tests/pixie/tests/test-keywords.pxi +++ b/tests/pixie/tests/test-keywords.pxi @@ -6,7 +6,6 @@ (t/assert= (:a m) 1) (t/assert= (:b m) 2) (t/assert= (:c m) 3) - (t/assert= (:d m) nil) (t/assert= (:d m :foo) :foo))) @@ -29,9 +28,40 @@ (t/assert= (not= :foo/bar :cat/bar) true) (t/assert= (not= :foo/cat :foo/dog) true)) +(t/deftest keyword-reader + (t/assert= (read-string ":1") :1) + (t/assert= (read-string ":1") :1) + (t/assert= (read-string ":1.0") :1.0) + (t/assert= (read-string ":foo") :foo) + (t/assert= (read-string ":1foo") :1foo) + (t/assert= (read-string ":foo/bar") :foo/bar) + (t/assert= (read-string ":1foo/1bar") :1foo/1bar) + (t/assert= (read-string ":nil") :nil) + (t/assert= (read-string ":true") :true) + (t/assert= (read-string ":false") :false) + + ;; We are reading at runtime so the namespace isn't + ;; going to be pixie.test.test-keywords. Its probably + ;; 'user but lets explicitly set it. + ;; The refer-ns is to initialize a new space + (refer-ns 'my.other.ns 'my.fake.core 'fake) + (binding [*ns* (the-ns 'my.other.ns)] + (t/assert= (read-string "::1") :my.other.ns/1) + (t/assert= (read-string "::1.0") :my.other.ns/1.0) + (t/assert= (read-string "::foo") :my.other.ns/foo) + (t/assert= (read-string "::true") :my.other.ns/true))) + (t/deftest string-to-keyword + (t/assert= (keyword "1") :1) + (t/assert= (keyword "1") :1) + (t/assert= (keyword "1.0") :1.0) (t/assert= (keyword "foo") :foo) + (t/assert= (keyword "1foo") :1foo) (t/assert= (keyword "foo/bar") :foo/bar) + (t/assert= (keyword "1foo/1bar") :1foo/1bar) + (t/assert= (keyword "nil") :nil) + (t/assert= (keyword "true") :true) + (t/assert= (keyword "false") :false) (t/assert-throws? (keyword 1)) (t/assert-throws? (keyword :a)) (t/assert-throws? (keyword 'a)) From a6ecc893395ef8d2f501962afbff7a03cb174901 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 6 Mar 2016 10:52:40 +0000 Subject: [PATCH 894/909] simplify --- pixie/vm/reader.py | 37 ++++++++++++------------------------- 1 file changed, 12 insertions(+), 25 deletions(-) diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 9cc2e36a..4a6ab292 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -305,21 +305,6 @@ def invoke(self, rdr, ch): return cons(symbol(u"quote"), cons(itm)) class KeywordReader(ReaderHandler): - def read(self, rdr): - acc = [] - - try: - while True: - ch = rdr.read() - if is_whitespace(ch) or is_terminating_macro(ch): - rdr.unread() - break - acc.append(ch) - except EOFError: - pass - sym_str = u"".join(acc) - return symbol(sym_str) - def fqd(self, itm): ns_alias = rt.namespace(itm) current_nms = rt.ns.deref() @@ -333,11 +318,11 @@ def fqd(self, itm): def invoke(self, rdr, ch): ch = rdr.read() if ch == u":": - itm = self.read(rdr) + ch = rdr.read() + itm = read_symbol(rdr, ch, False) return self.fqd(itm) else: - rdr.unread() - itm = self.read(rdr) + itm = read_symbol(rdr, ch, False) return keyword(rt.name(itm), rt.namespace(itm)) @@ -754,7 +739,7 @@ def read_number(rdr, ch): return parsed return Symbol(joined) -def read_symbol(rdr, ch): +def read_symbol(rdr, ch, convert_primitives=True): acc = [ch] try: while True: @@ -767,12 +752,14 @@ def read_symbol(rdr, ch): pass sym_str = u"".join(acc) - if sym_str == u"true": - return true - if sym_str == u"false": - return false - if sym_str == u"nil": - return nil + + if convert_primitives: + if sym_str == u"true": + return true + if sym_str == u"false": + return false + if sym_str == u"nil": + return nil return symbol(sym_str) class EOF(object.Object): From 90ea228d09cd03eca90f25acf0cb71e4a3b18a32 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Sun, 6 Mar 2016 16:07:59 +0000 Subject: [PATCH 895/909] Update to most recent pypy --- Makefile | 2 +- pixie/vm/c_api.py | 5 +---- pixie/vm/object.py | 1 - pixie/vm/threads.py | 50 +++------------------------------------------ 4 files changed, 5 insertions(+), 53 deletions(-) diff --git a/Makefile b/Makefile index e27182ee..8a5f9ccf 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ externals.fetched: $(EXTERNALS)/pypy: mkdir $(EXTERNALS); \ cd $(EXTERNALS); \ - curl https://bitbucket.org/pypy/pypy/get/81254.tar.bz2 > pypy.tar.bz2; \ + curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2; \ mkdir pypy; \ cd pypy; \ tar -jxf ../pypy.tar.bz2 --strip-components=1 diff --git a/pixie/vm/c_api.py b/pixie/vm/c_api.py index 45888295..a050a070 100644 --- a/pixie/vm/c_api.py +++ b/pixie/vm/c_api.py @@ -1,10 +1,7 @@ - - -from rpython.rlib.entrypoint import entrypoint_highlevel, RPython_StartupCode +from rpython.rlib.entrypoint import entrypoint_highlevel from rpython.rtyper.lltypesystem import rffi, lltype from rpython.rtyper.lltypesystem.lloperation import llop - @entrypoint_highlevel('main', [rffi.CCHARP], c_name='pixie_init') def pypy_execute_source(ll_progname): from target import init_vm diff --git a/pixie/vm/object.py b/pixie/vm/object.py index 1b47e22c..2ef03de3 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -8,7 +8,6 @@ def __init__(self): self._registry = [] def register(self, o): - print "register finalizer ", o self._registry.append(o) def run_finalizers(self): diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py index b523aa2f..e42d7d18 100644 --- a/pixie/vm/threads.py +++ b/pixie/vm/threads.py @@ -6,8 +6,6 @@ from pixie.vm.code import as_var import pixie.vm.rt as rt -from rpython.rlib.objectmodel import invoke_around_extcall - class Bootstrapper(object): def __init__(self): self._is_inited = False @@ -15,10 +13,9 @@ def __init__(self): def init(self): if not self._is_inited: - self._is_inited = True self._lock = rthread.allocate_lock() - rgil.gil_allocate() - invoke_around_extcall(before_external_call, after_external_call) + self._is_inited = True + rgil.allocate() def aquire(self, fn): self.init() @@ -55,7 +52,7 @@ def new_thread(fn): @as_var("-yield-thread") def yield_thread(): - do_yield_thread() + rgil.yield_thread() return nil # Locks @@ -86,46 +83,5 @@ def _release_lock(self): return rt.wrap(self._ll_lock.release()) - -## From PYPY - - -after_thread_switch = lambda: None # hook for signal.py - -# Fragile code below. We have to preserve the C-level errno manually... - -def before_external_call(): - # this function must not raise, in such a way that the exception - # transformer knows that it cannot raise! - rgil.gil_release() -before_external_call._gctransformer_hint_cannot_collect_ = True -before_external_call._dont_reach_me_in_del_ = True - -def after_external_call(): - rgil.gil_acquire() - rthread.gc_thread_run() - after_thread_switch() -after_external_call._gctransformer_hint_cannot_collect_ = True -after_external_call._dont_reach_me_in_del_ = True - -# The _gctransformer_hint_cannot_collect_ hack is needed for -# translations in which the *_external_call() functions are not inlined. -# They tell the gctransformer not to save and restore the local GC -# pointers in the shadow stack. This is necessary because the GIL is -# not held after the call to before_external_call() or before the call -# to after_external_call(). - -def do_yield_thread(): - # explicitly release the gil, in a way that tries to give more - # priority to other threads (as opposed to continuing to run in - # the same thread). - if rgil.gil_yield_thread(): - rthread.gc_thread_run() - after_thread_switch() -do_yield_thread._gctransformer_hint_close_stack_ = True -do_yield_thread._dont_reach_me_in_del_ = True -do_yield_thread._dont_inline_ = True - -# do_yield_thread() needs a different hint: _gctransformer_hint_close_stack_. # The *_external_call() functions are themselves called only from the rffi # module from a helper function that also has this hint. From 46d8277d6eefe0696f1cf12dc08da1850a17186e Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Mon, 7 Mar 2016 21:56:30 +0000 Subject: [PATCH 896/909] fix compiler --- pixie/vm/compiler.py | 8 +++---- tests/pixie/tests/test-compiler.pxi | 36 +++++++++++++++++++++++++++++ 2 files changed, 40 insertions(+), 4 deletions(-) diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index 5d0f214f..60cc353a 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -539,8 +539,8 @@ def compile_fn_body(name, args, body, ctx): compile_form(body, new_ctx) else: while body is not nil: - if rt.next(body) is nil: - new_ctx.enable_tail_call() + #if rt.next(body) is nil: + # new_ctx.enable_tail_call() compile_form(rt.first(body), new_ctx) body = rt.next(body) if body is not nil: @@ -635,7 +635,7 @@ def compile_quote(form, ctx): def compile_recur(form, ctx): form = form.next() - #affirm(ctx.can_tail_call, u"Can't recur in non-tail position") + affirm(ctx.can_tail_call, u"Can't recur in non-tail position") ctc = ctx.can_tail_call ctx.disable_tail_call() args = 0 @@ -697,7 +697,7 @@ def compile_loop(form, ctx): bindings = rt.first(form) affirm(isinstance(bindings, PersistentVector), u"Loop bindings must be a vector") body = rt.next(form) - + ctx.enable_tail_call() ctc = ctx.can_tail_call ctx.disable_tail_call() diff --git a/tests/pixie/tests/test-compiler.pxi b/tests/pixie/tests/test-compiler.pxi index 944343d4..751ee95e 100644 --- a/tests/pixie/tests/test-compiler.pxi +++ b/tests/pixie/tests/test-compiler.pxi @@ -35,3 +35,39 @@ (t/deftest test-deftype-mutables (mutate! (->Foo 0))) + +(t/deftest test-recur-must-tail + (t/assert-throws? + (eval + '(loop [n 0] + (inc (recur n))))) + + (t/assert-throws? + (eval + '(fn [n] + (if (zero? n) + 1 + (* n (recur (dec n))))))) + +(t/assert-throws? + (eval + '(fn [n] + (if (zero? n) + (* n (recur (dec n))) + 1)))) + + (t/assert= + (eval + '(loop [n 0] + (if (= 10 n) + n + (recur (inc n))))) + 10) + + (t/assert= + (eval + '(loop [n 0] + (if (not= 10 n) + (recur (inc n)) + n))) + 10)) From b3d41163d48fc51dc4c94acc6975e6fd890aa0b5 Mon Sep 17 00:00:00 2001 From: angusiguess Date: Tue, 8 Mar 2016 23:09:02 -0400 Subject: [PATCH 897/909] Add sequence to stdlib --- pixie/stdlib.pxi | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 77b3cede..a529de33 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -903,7 +903,7 @@ If further arguments are passed, invokes the method named by symbol, passing the (if (next coll) (recur (next coll)) (first coll)) - + (satisfies? ISeqable coll) (recur (seq coll)))) @@ -2106,7 +2106,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} val not-found))) ISeq - (-first [this] + (-first [this] (when (not= start stop) start)) (-next [this] @@ -2188,6 +2188,14 @@ For more information, see http://clojure.org/special_forms#binding-forms"} ([pred coll] (filter (complement pred) coll))) +(defn sequence + "Returns a lazy sequence of `data`, optionally transforming it using `xform`. + Given an `eduction`, produces a lazy sequence of it." + ([eduction] + (lazy-seq (cons (first eduction) (sequence (rest eduction))))) + ([xform data] + (sequence (eduction xform data)))) + (defn distinct {:doc "Returns the distinct elements in the collection." :signatures [[] [coll]] @@ -3080,7 +3088,7 @@ ex: (vary-meta x assoc :foo 42)" (deftype Iterate [f x] IReduce (-reduce [self rf init] - (loop [next (f x) + (loop [next (f x) acc (rf init x)] (if (reduced? acc) @acc From d340de4ee39936246a6f026f14e3a4b3ba047bfc Mon Sep 17 00:00:00 2001 From: angusiguess Date: Wed, 9 Mar 2016 10:22:12 -0400 Subject: [PATCH 898/909] Handle the empty case, fix eduction to conform to seqable --- pixie/stdlib.pxi | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index a529de33..ff070380 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1907,7 +1907,7 @@ not enough elements were present." ISeqable (-seq [self] - (reduce conj self))) + (seq (reduce conj self)))) (defn eduction "Returns a reducible/iterable application of the transducers @@ -2192,7 +2192,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} "Returns a lazy sequence of `data`, optionally transforming it using `xform`. Given an `eduction`, produces a lazy sequence of it." ([eduction] - (lazy-seq (cons (first eduction) (sequence (rest eduction))))) + (when (seq eduction) (lazy-seq (cons (first eduction) (sequence (rest eduction)))))) ([xform data] (sequence (eduction xform data)))) From 1626fff48d763d86f9c0d5c7b5a1ccff14f7017f Mon Sep 17 00:00:00 2001 From: angusiguess Date: Wed, 9 Mar 2016 10:22:20 -0400 Subject: [PATCH 899/909] Add tests --- tests/pixie/test-sequence.pxi | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) create mode 100644 tests/pixie/test-sequence.pxi diff --git a/tests/pixie/test-sequence.pxi b/tests/pixie/test-sequence.pxi new file mode 100644 index 00000000..e8a8e748 --- /dev/null +++ b/tests/pixie/test-sequence.pxi @@ -0,0 +1,23 @@ +(ns pixie.tests.test-sequence + (:require [pixie.test :as t])) + +(t/deftest empty-sequences + (t/assert= '() (take 1 (sequence (map inc) '()))) + (t/assert= '() (take 1 (sequence (map inc) []))) + (t/assert= '() (take 1 (sequence (map inc) #{}))) + (t/assert= '() (take 1 (sequence (map inc) {})))) + +(t/deftest non-empty-sequences + (t/assert= '(1 3) (take 2 (sequence (comp + (filter even?) + (map inc)) (range 3)))) + (t/assert= '(1) (take 1 (sequence (distinct) (repeat 4 1))))) + +(t/deftest early-terminating-sequences + (t/assert= '() (take 5 (sequence (filter (fn [x] false)) (repeat 8 8)))) + (t/assert= '(1 2) (take 3 (sequence (map identity) [1 2]))) + (t/assert= #{[:a 1] [:b 2]} (into #{} (take 3 (sequence (filter (fn [[k v]] + (keyword? k)) {:a 1 + :b 2 + "c" 3 + "d" 4})))))) From 283e6477709fa5e5d743f29abbe90da99fe5cc36 Mon Sep 17 00:00:00 2001 From: notnew <> Date: Sat, 12 Mar 2016 20:49:26 -0600 Subject: [PATCH 900/909] fix transducer composition problems, add tests remove `preserving-reduced` from several transducers. This was modifying the reducing function passed to the transducer, causing reduced values to be wrapped in `reduced` twice, so the reducing process wasn't completely unwrapping reduced values when finishing. Reducing functions only need to be wrapped with `preserving-reduced` if they call `reduce` themselves. (Like `cat` and `pixie.io.common/stream-reducer`) Also fixes a problem with the `take` transducer consuming one more element than needed. Add tests for these problems. --- pixie/io.pxi | 30 +++++++++--------- pixie/stdlib.pxi | 42 ++++++++++++------------- tests/pixie/tests/test-io.pxi | 33 ++++++++++++++++++-- tests/pixie/tests/test-stdlib.pxi | 52 ++++++++++++++++++++++++++----- 4 files changed, 110 insertions(+), 47 deletions(-) diff --git a/pixie/io.pxi b/pixie/io.pxi index 3dd8bdf2..a6ed1286 100644 --- a/pixie/io.pxi +++ b/pixie/io.pxi @@ -66,14 +66,13 @@ (str string)))))) IReduce (-reduce [this f init] - (let [rrf (preserving-reduced f)] - (loop [acc init] - (if-let [line (-read-line this)] - (let [result (rrf acc line)] - (if (not (reduced? result)) - (recur result) - @result)) - acc))))) + (loop [acc init] + (if-let [line (-read-line this)] + (let [result (f acc line)] + (if (reduced? result) + @result + (recur result))) + acc)))) (defn line-reader [input-stream] @@ -150,14 +149,13 @@ (deftype BufferedInputStream [upstream idx buffer] IReduce (-reduce [this f init] - (let [rrf (preserving-reduced f)] - (loop [acc init] - (if-let [next-byte (read-byte this)] - (let [step (rrf acc next-byte)] - (if (reduced? step) - @step - (recur step))) - acc)))) + (loop [acc init] + (if-let [next-byte (read-byte this)] + (let [result (f acc next-byte)] + (if (reduced? result) + @result + (recur result))) + acc))) IByteInputStream (read-byte [this] (when (= idx (count buffer)) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 77b3cede..92b09062 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1640,21 +1640,25 @@ The new value is thus `(apply f current-value-of-atom args)`." (let [idx (if (neg? i) (+ i (count coll)) i)] (nth coll idx not-found))))) +(defn ensure-reduced [x] + (if (reduced? x) + x + (reduced x))) + (defn take {:doc "Takes n elements from the collection, or fewer, if not enough." :added "0.1"} ([n] (fn [rf] - (let [rrf (preserving-reduced rf) - seen (atom 0)] + (let [seen (atom 0)] (fn ([] (rf)) ([result] (rf result)) ([result input] (let [s (swap! seen inc)] - (if (<= s n) - (rrf result input) - (reduced result)))))))) + (cond (< s n) (rf result input) + (= s n) (ensure-reduced (rf result input)) + :else (reduced result)))))))) ([n coll] (lazy-seq (when (pos? n) @@ -1666,8 +1670,7 @@ The new value is thus `(apply f current-value-of-atom args)`." :added "0.1"} ([n] (fn [rf] - (let [rrf (preserving-reduced rf) - seen (atom 0)] + (let [seen (atom 0)] (fn ([] (rf)) ([result] @@ -1675,7 +1678,7 @@ The new value is thus `(apply f current-value-of-atom args)`." ([result input] (let [s (swap! seen inc)] (if (> s n) - (rrf result input) + (rf result input) result))))))) ([n coll] (let [s (seq coll)] @@ -1707,14 +1710,13 @@ The new value is thus `(apply f current-value-of-atom args)`." :added "0.1"} ([pred] (fn [rf] - (let [rrf (preserving-reduced rf)] - (fn - ([] (rf)) - ([result] (rf result)) - ([result input] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] (if (pred input) - (rrf result input) - (reduced result))))))) + (rf result input) + (reduced result)))))) ([pred coll] (lazy-seq (when-let [s (seq coll)] @@ -1825,13 +1827,12 @@ not enough elements were present." :signatures [[f] [f coll]]} ([f] (fn [rf] - (let [i (atom -1) - rrf (preserving-reduced rf)] + (let [i (atom -1)] (fn ([] (rf)) ([result] (rf result)) ([result input] - (rrf result (f (swap! i inc) input))))))) + (rf result (f (swap! i inc) input))))))) ([f coll] (let [mapi (fn mapi [i coll] (lazy-seq @@ -1849,8 +1850,7 @@ not enough elements were present." :added "0.1"} ([f] (fn [rf] - (let [iv (atom -1) - rrf (preserving-reduced rf)] + (let [iv (atom -1)] (fn ([] (rf)) ([result] (rf result)) @@ -1859,7 +1859,7 @@ not enough elements were present." v (f i input)] (if (nil? v) result - (rrf result v)))))))) + (rf result v)))))))) ([f coll] (let [keepi (fn keepi [i coll] (lazy-seq diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi index 1d41c4a5..07a3606b 100644 --- a/tests/pixie/tests/test-io.pxi +++ b/tests/pixie/tests/test-io.pxi @@ -3,6 +3,7 @@ (require pixie.streams :as st :refer :all) (require pixie.streams.utf8 :as utf8 :refer :all) (require pixie.io :as io) + (require pixie.io-blocking :as blocking) (require pixie.streams.zlib :as zlib)) (t/deftest test-file-reduction @@ -10,11 +11,25 @@ (t/assert= (transduce (map identity) count-rf f) - 91))) + 91)) + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (comp (map char) (take 4)) string-builder f) + "This")) + (let [f (blocking/open-read "tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (map identity) + count-rf + f) + 91)) + (let [f (blocking/open-read "tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (comp (map char) (take 4)) string-builder f) + "This"))) (t/deftest test-process-reduction (let [f (io/run-command "ls tests/pixie/tests/test-io.txt")] - (t/assert= f "tests/pixie/tests/test-io.txt\n"))) + (t/assert= f "tests/pixie/tests/test-io.txt\n")) + (let [pipe (blocking/popen-read "ls tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (comp (map char) (take 6)) string-builder pipe) + "tests/"))) (t/deftest test-read-into-buffer (let [f (io/open-read "tests/pixie/tests/test-io.txt")] @@ -38,6 +53,14 @@ (t/assert= (io/read-line f) "Second line.") (t/assert= (io/read-line f) nil))) +(t/deftest test-line-reader + (let [lines (io/line-reader (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert= (transduce (map count) conj [] lines) + [77 12])) + (let [lines (io/line-reader (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert= (transduce (comp (map count) (take 1)) conj [] lines) + [77]))) + (t/deftest test-line-seq (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt")) s (io/line-seq f)] @@ -58,7 +81,11 @@ (t/assert= (char (io/read-byte f)) \T) (t/assert= (char (io/read-byte f)) \h) (t/assert= (char (io/read-byte f)) \i) - (t/assert= (char (io/read-byte f)) \s))) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (transduce (comp (map char) (take 5)) string-builder f) + " is a") + (t/assert= (transduce (comp (map char) (take 5)) string-builder f) + " test"))) (t/deftest test-buffered-input-streams-throws-on-non-input-streams (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 12ad9912..f4d2f763 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -28,9 +28,28 @@ (t/assert= (keep-indexed (constantly nil) []) []) (t/assert= (keep-indexed (fn [i x] [i x]) [:a :b]) [[0 :a] [1 :b]]) - (t/assert= (transduce (keep-indexed (constantly true)) conj []) []) - (t/assert= (transduce (keep-indexed (constantly nil)) conj []) []) - (t/assert= (transduce (keep-indexed (fn [i x] [i x])) conj [:a :b]) [[0 :a] [1 :b]])) + (t/assert= (transduce (keep-indexed (constantly true)) conj [:a :b]) [true true]) + (t/assert= (transduce (keep-indexed (constantly nil)) conj [:a :b]) []) + (t/assert= (transduce (keep-indexed (fn [i x] [i x])) conj [:a :b]) [[0 :a] [1 :b]]) + + (let [even-index? (fn [i x] + (when (even? i) + x))] + (t/assert= (transduce (keep-indexed even-index?) conj [:a :b :c :d]) + [:a :c]) + (t/assert= (transduce (map-indexed even-index?) conj [:a :b :c :d]) + [:a nil :c nil]) + + (t/assert= (transduce (comp (keep-indexed even-index?) + (take 3)) + conj + [:a :b :c :d :e :f]) + [:a :c :e]) + (t/assert= (transduce (comp (map-indexed even-index?) + (take 3)) + conj + [:a :b :c :d]) + [:a nil :c]))) (t/deftest test-reductions (t/assert= (reductions + nil) @@ -542,7 +561,21 @@ (t/assert= (transduce (take 0) conj [1 2 3 4]) []) (t/assert= (transduce (take 1) conj [1 2 3 4]) [1]) (t/assert= (transduce (take 2) conj [1 2 3 4]) [1 2]) - (t/assert= (transduce (take 3) conj [1 2 3 4]) [1 2 3])) + (t/assert= (transduce (take 3) conj [1 2 3 4]) [1 2 3]) + (t/assert= (transduce (take 10) conj [1 2 3 4]) [1 2 3 4]) + (t/assert= (transduce (comp (take 2) (take 1)) conj [1 2 3 4]) [1]) + (t/assert= (transduce (comp (take 1) (take 2)) conj [1 2 3 4]) [1]) + + (let [call-count (atom 0) + inc-call-count! (fn [x] + (swap! call-count inc) + x)] + (t/assert= (transduce (comp (map inc-call-count!) (take 2)) conj (range 10)) + [0 1]) + (t/assert= @call-count 2) + (t/assert= (transduce (comp (take 2) (map inc-call-count!)) conj (range 10)) + [0 1]) + (t/assert= @call-count 4))) (t/deftest test-drop (t/assert= (drop 0 [1 2 3 4]) [1 2 3 4]) @@ -552,21 +585,26 @@ (t/assert= (transduce (drop 0) conj [1 2 3 4]) [1 2 3 4]) (t/assert= (transduce (drop 1) conj [1 2 3 4]) [2 3 4]) (t/assert= (transduce (drop 2) conj [1 2 3 4]) [3 4]) - (t/assert= (transduce (drop 3) conj [1 2 3 4]) [4])) + (t/assert= (transduce (drop 3) conj [1 2 3 4]) [4]) + (t/assert= (transduce (drop 10) conj [1 2 3 4]) []) + (t/assert= (transduce (comp (drop 1) (take 2)) conj [1 2 3 4]) [2 3]) + (t/assert= (transduce (comp (take 2) (drop 1)) conj [1 2 3 4]) [2])) (t/deftest test-take-while (t/assert= (take-while pos? [1 2 3 -1]) [1 2 3]) (t/assert= (take-while pos? [-1 2]) ()) (t/assert= (transduce (take-while even?) conj [2 4 6 7 8]) [2 4 6]) (t/assert= (transduce (take-while even?) conj [0 2] [1 4 6]) [0 2]) - (t/assert= (transduce (take-while even?) conj [1 3] [2 4 6 7 8]) [1 3 2 4 6])) + (t/assert= (transduce (take-while even?) conj [1 3] [2 4 6 7 8]) [1 3 2 4 6]) + (t/assert= (transduce (comp (take-while even?) (take 2)) conj [1 3] [2 4 6 7 8]) [1 3 2 4])) (t/deftest test-drop-while (t/assert= (drop-while pos? [1 2 3 -1]) [-1]) (t/assert= (drop-while pos? [-1 2]) [-1 2]) (t/assert= (transduce (drop-while even?) conj [2 4 6 7 8]) [7 8]) (t/assert= (transduce (drop-while even?) conj [0 2] [1 4 6]) [0 2 1 4 6]) - (t/assert= (transduce (drop-while even?) conj [0 2] [2 4 6 7 8]) [0 2 7 8])) + (t/assert= (transduce (drop-while even?) conj [0 2] [2 4 6 7 8]) [0 2 7 8]) + (t/assert= (transduce (comp (drop-while even?) (take 2)) conj [0 2] [2 4 6 7 8]) [0 2 7 8])) (t/deftest test-cycle (t/assert= (cycle ()) ()) From c94144b43c3a01f1bf4c4921ddc8e8c6912dc97c Mon Sep 17 00:00:00 2001 From: angusiguess Date: Sun, 13 Mar 2016 20:33:58 -0300 Subject: [PATCH 901/909] Make sequence lazy, make educe seq use sequence --- pixie/stdlib.pxi | 20 +++++++++++++------- tests/pixie/test-sequence.pxi | 23 ----------------------- tests/pixie/tests/test-stdlib.pxi | 22 +++++++++++++++++++++- 3 files changed, 34 insertions(+), 31 deletions(-) delete mode 100644 tests/pixie/test-sequence.pxi diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index ff070380..23a9f7e4 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -1907,7 +1907,7 @@ not enough elements were present." ISeqable (-seq [self] - (seq (reduce conj self)))) + (sequence xform coll))) (defn eduction "Returns a reducible/iterable application of the transducers @@ -2189,12 +2189,18 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (filter (complement pred) coll))) (defn sequence - "Returns a lazy sequence of `data`, optionally transforming it using `xform`. - Given an `eduction`, produces a lazy sequence of it." - ([eduction] - (when (seq eduction) (lazy-seq (cons (first eduction) (sequence (rest eduction)))))) - ([xform data] - (sequence (eduction xform data)))) + "Returns a lazy sequence of `data`, optionally transforming it using `xform`" + ([coll] + (if (seq? coll) coll + (or (seq coll) ()))) + ([xform coll] + (let [step (defn step [xform acc xs] + (if-let [s (seq xs)] + (let [next-acc ((xform conj) acc (first s))] + (if (= acc next-acc) (step xform next-acc (next s)) + (concat (drop (count acc) next-acc) (step xform next-acc (next s))))) + nil))] + (lazy-seq (step xform [] coll))))) (defn distinct {:doc "Returns the distinct elements in the collection." diff --git a/tests/pixie/test-sequence.pxi b/tests/pixie/test-sequence.pxi deleted file mode 100644 index e8a8e748..00000000 --- a/tests/pixie/test-sequence.pxi +++ /dev/null @@ -1,23 +0,0 @@ -(ns pixie.tests.test-sequence - (:require [pixie.test :as t])) - -(t/deftest empty-sequences - (t/assert= '() (take 1 (sequence (map inc) '()))) - (t/assert= '() (take 1 (sequence (map inc) []))) - (t/assert= '() (take 1 (sequence (map inc) #{}))) - (t/assert= '() (take 1 (sequence (map inc) {})))) - -(t/deftest non-empty-sequences - (t/assert= '(1 3) (take 2 (sequence (comp - (filter even?) - (map inc)) (range 3)))) - (t/assert= '(1) (take 1 (sequence (distinct) (repeat 4 1))))) - -(t/deftest early-terminating-sequences - (t/assert= '() (take 5 (sequence (filter (fn [x] false)) (repeat 8 8)))) - (t/assert= '(1 2) (take 3 (sequence (map identity) [1 2]))) - (t/assert= #{[:a 1] [:b 2]} (into #{} (take 3 (sequence (filter (fn [[k v]] - (keyword? k)) {:a 1 - :b 2 - "c" 3 - "d" 4})))))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 12ad9912..8805cd14 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -595,7 +595,7 @@ (seq))) ;; expanding transducer with nils (t/assert= '(1 2 3 nil 4 5 6 nil) - (eduction cat [[1 2 3 nil] [4 5 6 nil]]))) + (seq (eduction cat [[1 2 3 nil] [4 5 6 nil]])))) (t/deftest test-trace (try @@ -767,3 +767,23 @@ (t/assert= (reduce (fn [a v] (reduced "foo")) 0 (iterate inc 1)) "foo") (t/assert= (reduce (fn [a v] (if (< a 10) (+ a v) (reduced a))) 0 (iterate (partial + 2) 1)) 16)) +(t/deftest test-sequence-empty-sequences + (t/assert= '() (take 1 (sequence (map inc) '()))) + (t/assert= '() (take 1 (sequence (map inc) []))) + (t/assert= '() (take 1 (sequence (map inc) #{}))) + (t/assert= '() (take 1 (sequence (map inc) {})))) + +(t/deftest test-sequence-non-empty-sequences + (t/assert= '(1 3) (take 2 (sequence (comp + (filter even?) + (map inc)) (range 3)))) + (t/assert= '(1) (take 1 (sequence (distinct) (repeat 4 1))))) + +(t/deftest test-sequence-early-terminating-sequences + (t/assert= '() (take 5 (sequence (filter (fn [x] false)) (repeat 8 8)))) + (t/assert= '(1 2) (take 3 (sequence (map identity) [1 2]))) + (t/assert= #{[:a 1] [:b 2]} (into #{} (take 3 (sequence (filter (fn [[k v]] + (keyword? k)) {:a 1 + :b 2 + "c" 3 + "d" 4})))))) From 06cfb08e2ad01671c1831c0dcc7220faadbdf273 Mon Sep 17 00:00:00 2001 From: Thomas Mulvaney Date: Fri, 25 Mar 2016 16:49:04 +0000 Subject: [PATCH 902/909] Throw RuntimeExceptions for array functions --- pixie/vm/array.py | 39 ++++++++++++++++++------------- tests/pixie/tests/test-arrays.pxi | 35 +++++++++++++++++++++++---- 2 files changed, 54 insertions(+), 20 deletions(-) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 9d15f550..4ca97813 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -26,6 +26,8 @@ def reduce_small(self, f, init): init = f.invoke([init, self._list[x]]) return init + def list(self): + return self._list def reduce_large(self, f, init): for x in range(len(self._list)): @@ -37,14 +39,14 @@ def reduce_large(self, f, init): @extend(proto._count, Array) def _count(self): assert isinstance(self, Array) - return rt.wrap(len(self._list)) + return rt.wrap(len(self.list())) @extend(proto._nth, Array) def _nth(self, idx): assert isinstance(self, Array) ival = idx.int_val() - if ival < len(self._list): - return self._list[ival] + if ival < len(self.list()): + return self.list()[ival] else: affirm(False, u"Index out of Range") @@ -52,15 +54,15 @@ def _nth(self, idx): def _nth_not_found(self, idx, not_found): assert isinstance(self, Array) ival = idx.int_val() - if ival < len(self._list): - return self._list[ival] + if ival < len(self.list()): + return self.list()[ival] else: return not_found @extend(proto._reduce, Array) def reduce(self, f, init): assert isinstance(self, Array) - if len(self._list) > UNROLL_IF_SMALLER_THAN: + if len(self.list()) > UNROLL_IF_SMALLER_THAN: return self.reduce_large(f, init) return self.reduce_small(f, init) @@ -122,34 +124,39 @@ def array(lst): @as_var("aget") def aget(self, idx): - assert isinstance(self, Array) - return self._list[idx.int_val()] + affirm(isinstance(self, Array), u"aget expects an Array as the first argument") + affirm(isinstance(idx, Integer), u"aget expects an Integer as the second argument") + return self.list()[idx.int_val()] @as_var("aset") def aset(self, idx, val): - assert isinstance(self, Array) - self._list[idx.int_val()] = val + affirm(isinstance(self, Array), u"aset expects an Array as the first argument") + affirm(isinstance(idx, Integer), u"aset expects an Integer as the second argument") + self.list()[idx.int_val()] = val return val @as_var("aslice") def aslice(self, offset): - assert isinstance(self, Array) and isinstance(offset, Integer) + affirm(isinstance(self, Array), u"aset expects an Array as the first argument") + affirm(isinstance(offset, Integer), u"aset expects an Integer as the second argument") offset = offset.int_val() if offset >= 0: - return Array(self._list[offset:]) + return Array(self.list()[offset:]) else: rt.throw(rt.wrap(u"offset must be an Integer >= 0")) @as_var("aconcat") def aconcat(self, other): - assert isinstance(self, Array) and isinstance(other, Array) - return Array(self._list + other._list) + affirm(isinstance(self, Array) and isinstance(other, Array), + u"aconcat expects 2 Arrays") + return Array(self.list() + other.list()) @as_var("alength") def alength(self): - assert isinstance(self, Array) - return rt.wrap(len(self._list)) + affirm(isinstance(self, Array), u"alength expects an Array") + + return rt.wrap(len(self.list())) @as_var("make-array") def make_array(l): diff --git a/tests/pixie/tests/test-arrays.pxi b/tests/pixie/tests/test-arrays.pxi index fbd55937..ecd651cb 100644 --- a/tests/pixie/tests/test-arrays.pxi +++ b/tests/pixie/tests/test-arrays.pxi @@ -8,6 +8,12 @@ (foreach [x a] (t/assert= x nil)))) +(t/deftest test-alength + (let [a (make-array 10)] + (t/assert= (alength a) 10) + (t/assert-throws? RuntimeException + (alength [])))) + (t/deftest test-aget-and-aset (let [a (make-array 10)] (dotimes [i 10] @@ -17,7 +23,13 @@ (aset a i i)) (dotimes [i 10] - (t/assert= (aget a i) i)))) + (t/assert= (aget a i) i)) + + (t/assert-throws? RuntimeException + (aget a 1.0)) + + (t/assert-throws? RuntimeException + (aset a 1.0 :foo)))) (t/deftest test-aconcat (let [a1 (make-array 10) @@ -30,7 +42,13 @@ (let [a3 (aconcat a1 a2)] (dotimes [i 20] - (t/assert= (aget a3 i) i))))) + (t/assert= (aget a3 i) i))) + + (t/assert-throws? RuntimeException + (t/aconcat a1 [])) + + (t/assert-throws? RuntimeException + (t/aconcat a1 '())))) (t/deftest test-aslice (let [a (make-array 10)] @@ -42,10 +60,19 @@ (foreach [i (range 0 7)] (t/assert= (aget a1 i) (+ i 3))) (foreach [i (range 0 3)] - (t/assert= (aget a2 i) (+ i 7)))))) + (t/assert= (aget a2 i) (+ i 7)))) + + (t/assert-throws? RuntimeException + (aslice [1 2 3 4] 0 2)) + + (t/assert-throws? RuntimeException + (aslice '() 0 2)) + + (t/assert-throws? RuntimeException + (aslice a 1.0 2)))) (t/deftest test-byte-array-creation (let [ba (byte-array 10)] (t/assert= (vec ba) [0 0 0 0 0 0 0 0 0 0]) - (t/assert= (count ba) 10))) \ No newline at end of file + (t/assert= (count ba) 10))) From 808d8298e31eaf5c760636021714ca8e5f4295e0 Mon Sep 17 00:00:00 2001 From: Tom Mulvaney Date: Sun, 14 Aug 2016 20:25:33 +0100 Subject: [PATCH 903/909] lock pypy version down --- Makefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Makefile b/Makefile index 8a5f9ccf..5979fde8 100644 --- a/Makefile +++ b/Makefile @@ -53,7 +53,7 @@ externals.fetched: $(EXTERNALS)/pypy: mkdir $(EXTERNALS); \ cd $(EXTERNALS); \ - curl https://bitbucket.org/pypy/pypy/get/default.tar.bz2 > pypy.tar.bz2; \ + curl https://bitbucket.org/pypy/pypy/get/91db1a9.tar.bz2 > pypy.tar.bz2; \ mkdir pypy; \ cd pypy; \ tar -jxf ../pypy.tar.bz2 --strip-components=1 From 946b408e2013c2a2b3f1c55796aba59e079ddef0 Mon Sep 17 00:00:00 2001 From: Bryce Covert Date: Thu, 29 Sep 2016 23:25:26 -0700 Subject: [PATCH 904/909] for bindings can depend on previous ones. --- pixie/stdlib.pxi | 14 +++++++------- tests/pixie/tests/test-stdlib.pxi | 6 +++++- 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index b1a39041..b30c6900 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2606,14 +2606,14 @@ Expands to calls to `extend-type`." `(loop [res# [] ~c (seq ~coll)] (if ~c - (recur (into res# - ~(gen-loop (into coll-bindings - [binding `(first ~c)]) - (nnext bindings))) - (next ~c)) + (let [~binding (first ~c)] + (recur (into res# + ~(gen-loop (into coll-bindings + [binding `(first ~c)]) + (nnext bindings))) + (next ~c))) res#))) - `(let ~coll-bindings - [~@body])))] + `[~@body]))] `(or (seq ~(gen-loop [] bindings)) '()))) (defmacro doto diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 6e1984e8..17e6bc5b 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -452,7 +452,11 @@ (t/assert= (for [x [1 2 3] y [:a :b :c]] [x y]) [[1 :a] [1 :b] [1 :c] [2 :a] [2 :b] [2 :c] - [3 :a] [3 :b] [3 :c]])) + [3 :a] [3 :b] [3 :c]]) + (t/assert= (for [x [[1 2 3]] + y x] + y) + [1 2 3])) (t/deftest test-doto (let [a (atom 0)] From e3861de3ba7b1f5948a64b5eb08c7d110eb5882c Mon Sep 17 00:00:00 2001 From: Angus Fletcher Date: Sat, 24 Dec 2016 13:27:26 -0400 Subject: [PATCH 905/909] Replace defn by fn --- pixie/stdlib.pxi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index b1a39041..4da12535 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -2194,7 +2194,7 @@ For more information, see http://clojure.org/special_forms#binding-forms"} (if (seq? coll) coll (or (seq coll) ()))) ([xform coll] - (let [step (defn step [xform acc xs] + (let [step (fn step [xform acc xs] (if-let [s (seq xs)] (let [next-acc ((xform conj) acc (first s))] (if (= acc next-acc) (step xform next-acc (next s)) From 060bb4a8058460b5af11ee325815099c763c87eb Mon Sep 17 00:00:00 2001 From: Shoozza Date: Sun, 2 Apr 2017 20:50:07 +0200 Subject: [PATCH 906/909] Fix LGPL svg link --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index d66d94d1..a290aa99 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -[![Build Status](https://travis-ci.org/pixie-lang/pixie.svg?branch=master)](https://travis-ci.org/pixie-lang/pixie)[![License: LGPL] (http://img.shields.io/badge/license-LGPL-green.svg)](http://img.shields.io/badge/license-LGPL-green.svg) +[![Build Status](https://travis-ci.org/pixie-lang/pixie.svg?branch=master)](https://travis-ci.org/pixie-lang/pixie) +[![License: LGPL](https://img.shields.io/badge/license-LGPL-green.svg)](https://img.shields.io/badge/license-LGPL-green.svg) # Pixie [![Join the chat at https://gitter.im/pixie-lang/pixie](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pixie-lang/pixie?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From f82bf5f577a950051bd0b106865e3bd5608d36bd Mon Sep 17 00:00:00 2001 From: egregius313 Date: Wed, 7 Jun 2017 20:58:02 -0400 Subject: [PATCH 907/909] Change calls to throw to vector instead of string Calls to throw previously used strings, which is not a valid way to call throw (Thus raising an exception raised a completely different exception). Changed all calls to use vectors instead. --- pixie/fs.pxi | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pixie/fs.pxi b/pixie/fs.pxi index f3e73960..679bd5f6 100644 --- a/pixie/fs.pxi +++ b/pixie/fs.pxi @@ -109,7 +109,7 @@ (rel [this other] (if (satisfies? IFSPath other) (rel-path this other) - (throw "Second argument must satisfy IFSPath"))) + (throw [::AssertionException "Second argument must satisfy IFSPath"]))) (abs [this] (path/-abs pathz)) @@ -157,7 +157,7 @@ (rel [this other] (if (satisfies? IFSPath other) (rel-path this other) - (throw "Second argument must satisfy IFSPath"))) + (throw [::AssertionException "Second argument must satisfy IFSPath"]))) (abs [this] (path/-abs pathz)) @@ -217,7 +217,7 @@ (cond (path/-file? x) (->File x) (not (path/-exists? x)) (->File x) - :else (throw (str "A non-file object exists at path: " x))))) + :else (throw [::NotAFileException (str "A non-file object exists at path: " x)])))) (defn dir "Returns a dir if the path is a dir or does not exist. If a different filesystem object exists at the path an error will be thrown." @@ -226,7 +226,7 @@ (cond (path/-dir? x) (->Dir x) (not (path/-exists? x)) (->Dir x) - :else (throw (str "A non-dir object exists at path: " x))))) + :else (throw [::NotADirectoryException (str "A non-dir object exists at path: " x)])))) (defn fspath "Returns either a File or Dir if they exist at the path" @@ -235,6 +235,6 @@ (cond (path/-file? x) (->File x) (path/-dir? x) (->Dir x) - :else (throw (str "No file or directory at path: " x))))) + :else (throw [::FileNotFoundException (str "No file or directory at path: " x)])))) From 42d6fc0cc78075028c5fcdf612b0f9815b554603 Mon Sep 17 00:00:00 2001 From: Edward Minnix III Date: Thu, 5 Oct 2017 11:13:07 -0400 Subject: [PATCH 908/909] Add test for the letfn macro --- tests/pixie/tests/test-stdlib.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi index 17e6bc5b..9b2174e0 100644 --- a/tests/pixie/tests/test-stdlib.pxi +++ b/tests/pixie/tests/test-stdlib.pxi @@ -829,3 +829,16 @@ :b 2 "c" 3 "d" 4})))))) + +(t/deftest test-letfn + (letfn [(hello [] "Hello") + (adder [x y] (+ x y))] + (t/assert= "Hello" (hello)) + (dotimes [i 10] + (dotimes [j 20] + (t/assert= (+ i j) (adder i j))))) + (letfn [(f [x n] (vec (repeat n x)))] + (t/assert= (f :x 3) [:x :x :x]) + (t/assert= (f 0 20) [0 0 0 0 0 + 0 0 0 0 0 + 0 0 0 0 0]))) From a25209bc464a7a3254636c445814ba4f92c870d5 Mon Sep 17 00:00:00 2001 From: Edward Minnix III Date: Thu, 5 Oct 2017 11:17:38 -0400 Subject: [PATCH 909/909] Added letfn --- pixie/stdlib.pxi | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi index 6c908a82..2501e2fe 100644 --- a/pixie/stdlib.pxi +++ b/pixie/stdlib.pxi @@ -482,6 +482,19 @@ (let [nm (with-meta nm (assoc (meta nm) :private true))] (cons `defn (cons nm rest)))) +(defmacro letfn + "fnspec ==> (fname [params*] exprs) or (fname ([params*] exprs)+) + + Takes a vector of function specs and a body, and generates a set of + bindings of functions to their names. All of the names are available + in all of the definitions of the functions, as well as the body." + {:added "0.2" + :forms '[(letfn [fnspecs*] exprs*)]} + [fnspecs & body] + `(letfn* ~(vec (interleave (map first fnspecs) + (map #(cons `fn %) fnspecs))) + ~@body)) + (defn not {:doc "Inverts the input, if a truthy value is supplied, returns false, otherwise returns true."