diff --git a/.gitignore b/.gitignore index dded1ddc..5928d907 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,12 @@ *.pyc -externals +externals* +pixie-vm .idea +lib +include +*.pxic +test.tmp +/libuv* +/uv-* +/uv.h +/*compressed-output.* diff --git a/.travis.yml b/.travis.yml index 1b9e9cb0..77846892 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,25 @@ +sudo: false +env: + - JIT_OPTS='--opt=jit' TARGET_OPTS='target.py' + - JIT_OPTS='' TARGET_OPTS='target.py' + +matrix: + fast_finish: true + script: - - ./checkout-externals - - ./make-with-jit + - make PYTHON=python build + - make compile_src + - make compile_tests + - make run_built_tests + +addons: + apt: + packages: + - libffi-dev + - libedit-dev + - libboost-all-dev + - zlib1g-dev + - zlib-bin + +notifications: + irc: "chat.freenode.net#pixie-lang" diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 00000000..76ca6849 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,15 @@ +FROM debian:sid + +# install dependencies +RUN apt-get update \ + && apt-get install -y gcc g++ libboost-dev pkg-config make curl bzip2 python2.7 \ + && apt-get install -y libffi-dev libuv-dev libedit-dev + +ADD . /usr/src/pixie + +# build the thing +RUN cd /usr/src/pixie \ + && make PYTHON=python2.7 build_with_jit \ + && ln -s /usr/src/pixie/pixie-vm /usr/bin/pxi + +ENTRYPOINT ["/usr/bin/pxi"] diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..5979fde8 --- /dev/null +++ b/Makefile @@ -0,0 +1,92 @@ +all: help + +EXTERNALS=externals + +PYTHON ?= `env which -a python2 python2.7 | head -n1` +PYTHONPATH=$$PYTHONPATH:$(EXTERNALS)/pypy + + +COMMON_BUILD_OPTS?=--thread --gcrootfinder=shadowstack --continuation +JIT_OPTS?=--opt=jit +TARGET_OPTS?=target.py + +help: + @echo "make help - display this message" + @echo "make run - run the compiled interpreter" + @echo "make run_interactive - run without compiling (slow)" + @echo "make build_with_jit - build with jit enabled" + @echo "make build_no_jit - build without jit" + @echo "make fetch_externals - download and unpack external deps" + +build_with_jit: fetch_externals + @if [ ! -d /usr/local/include/boost -a ! -d /usr/include/boost ] ; then echo "Boost C++ Library not found" && false; fi && \ + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --opt=jit target.py && \ + make compile_basics + +build_no_jit: fetch_externals + @if [ ! -d /usr/local/include/boost -a ! -d /usr/include/boost ] ; then echo "Boost C++ Library not found" && false; fi && \ + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) target.py && \ + make compile_basics + +build_no_jit_shared: fetch_externals + @if [ ! -d /usr/local/include/boost -a ! -d /usr/include/boost ] ; then echo "Boost C++ Library not found" && false; fi && \ + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) --shared target.py && \ + make compile_basics + + +compile_basics: + @echo -e "\n\n\n\nWARNING: Compiling core libs. If you want to modify one of these files delete the .pxic files first\n\n\n\n" + ./pixie-vm -c pixie/uv.pxi -c pixie/io.pxi -c pixie/stacklets.pxi -c pixie/stdlib.pxi -c pixie/repl.pxi + +build: fetch_externals + $(PYTHON) $(EXTERNALS)/pypy/rpython/bin/rpython $(COMMON_BUILD_OPTS) $(JIT_OPTS) $(TARGET_OPTS) + +fetch_externals: $(EXTERNALS)/pypy externals.fetched + +externals.fetched: + echo https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 + curl -L https://github.com/pixie-lang/external-deps/releases/download/1.0/`uname -s`-`uname -m`.tar.bz2 > /tmp/externals.tar.bz2 + tar -jxf /tmp/externals.tar.bz2 --strip-components=2 + touch externals.fetched + + +$(EXTERNALS)/pypy: + mkdir $(EXTERNALS); \ + cd $(EXTERNALS); \ + curl https://bitbucket.org/pypy/pypy/get/91db1a9.tar.bz2 > pypy.tar.bz2; \ + mkdir pypy; \ + cd pypy; \ + tar -jxf ../pypy.tar.bz2 --strip-components=1 + +run: + ./pixie-vm + + +run_interactive: + @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py + +run_interactive_stacklets: + @PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py pixie/stacklets.pxi + + +run_built_tests: pixie-vm + ./pixie-vm run-tests.pxi + +run_interpreted_tests: target.py + PYTHONPATH=$(PYTHONPATH) $(PYTHON) target.py run-tests.pxi + +compile_tests: + find "tests" -name "*.pxi" | xargs -L1 ./pixie-vm -l "tests" -c + +compile_src: + find * -name "*.pxi" | grep "^pixie/" | xargs -L1 ./pixie-vm $(EXTERNALS_FLAGS) -c + +clean_pxic: + find * -name "*.pxic" | xargs --no-run-if-empty rm + +clean: clean_pxic + rm -rf ./lib + rm -rf ./include + rm -rf ./externals* + rm -f ./pixie-vm + rm -f ./*.pyc diff --git a/README.md b/README.md index 5c557319..a290aa99 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,9 @@ [![Build Status](https://travis-ci.org/pixie-lang/pixie.svg?branch=master)](https://travis-ci.org/pixie-lang/pixie) +[![License: LGPL](https://img.shields.io/badge/license-LGPL-green.svg)](https://img.shields.io/badge/license-LGPL-green.svg) # Pixie +[![Join the chat at https://gitter.im/pixie-lang/pixie](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/pixie-lang/pixie?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) + ## Intro Pixie is a lightweight lisp suitable for both general use as well as shell scripting. The language is still in a "pre-alpha" phase and as such changes fairly quickly. @@ -10,42 +13,67 @@ The standard library is heavily inspired by Clojure as well as several other fun Some planned and implemented features: +* Immutable datastructures * Protocols first implementation -* Transducers at-the-bottom (most primitves are based off of reduce) -* Coroutines for transducer inversion of control (transducer to lazy-seq conversion) +* Transducers at-the-bottom (most primitives are based off of reduce) * A "good enough" JIT (implemented, tuning still a WIP, but not bad performance today) -* Easy FFI (TODO) +* Easy FFI * Pattern matching (TODO) +## Dependencies + +* python or pypy to build +* [libffi-dev](https://sourceware.org/libffi/) +* [libedit-dev](http://thrysoee.dk/editline/) +* [libuv-dev](https://github.com/libuv/libuv) Version 1.0 or higher +* [libboost-all-dev](http://www.boost.org/) (`brew install boost` for Mac) + ## Building - ./checkout-externals - ./make-with-jit - ./target-c - + make build_with_jit + ./pixie-vm + +Note: Mac OS X does not come with the build tools required by default. Install the XCode Command Line tools ([Apple Developer Site](http://developer.apple.com)) or install them independently. + + +## Running the tests + + ./pixie-vm run-tests.pxi + + +## Examples + +There are examples in the /examples directory. +Try out "Hello World" with: + + ./examples/hello-world.pxi + + +## Build Tool +Pixie now comes with a build tool called [dust](https://github.com/pixie-lang/dust). Try it and start making magic of your own. ## FAQ ### So this is written in Python? -It's actually written in the RPython, the same language PyPy is written in. The script `./make-with-jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `target-c` this executable is a flow blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... +It's actually written in RPython, the same language PyPy is written in. `make build_with_jit` will compile Pixie using the PyPy toolchain. After some time, it will produce an executable called `pixie-vm`. This executable is a full blown native interpreter with a JIT, GC, etc. So yes, the guts are written in RPython, just like the guts of most lisp interpreters are written in C. At runtime the only thing that is interpreted is the Pixie bytecode, that is until the JIT kicks in... + ### What's this bit about "magical powers"? -First of all, the word "magic" is in quotes as it's partly a play on words, pixies are small, light and often considered to have magical powers. +First of all, the word "magic" is in quotes as it's partly a play on words, pixies are small, light and often considered to have magical powers. -However there are a few features of pixie that although may not be uncommon, are perhaps unexpected from a lisp. +However there are a few features of pixie that although may not be uncommon, are perhaps unexpected from a lisp. -* Pixie implements its own virtual machine. It does not run on the JVM, CLR or Python VM. It implements its own bytecode, has its own GC and JIT. And it's small. Currently the interpreter, JIT, GC, and stdlib clock in at about 5.5MB once compiled down to an executable. +* Pixie implements its own virtual machine. It does not run on the JVM, CLR or Python VM. It implements its own bytecode, has its own GC and JIT. And it's small. Currently the interpreter, JIT, GC, and stdlib clock in at about 10.3MB once compiled down to an executable. -* The JIT makes some things fast. Very fast. Code like the following compiles down to a loop with 6 CPU instructions. While this may not be too impressive for any language that uses a tracing jit, it is faily unique for a language as young as Pixie. +* The JIT makes some things fast. Very fast. Code like the following compiles down to a loop with 6 CPU instructions. While this may not be too impressive for any language that uses a tracing jit, it is fairly unique for a language as young as Pixie. ```clojure -(comment - This code adds up to 10000 from 0 via calling a function that takes a variable number of arguments. - That function then reduces over the argument list to add up all given arguments.) - +;; This code adds up to 10000 from 0 via calling a function that takes a variable number of arguments. +;; That function then reduces over the argument list to add up all given arguments. + (defn add-fn [& args] (reduce -add 0 args)) @@ -53,45 +81,12 @@ However there are a few features of pixie that although may not be uncommon, are (if (eq x 10000) x (recur (add-fn x 1)))) - + ``` - - -* Inversion of control via stacklets. Most of Pixie makes heavy use of transducers. However there are times when transducers need to be converted into a cons style lazy sequence. For this we make use of stacklets to invert control of the loops. Because of this, any data collection need only define a method for the -reduce protocol and most collections functions will "just work" -```clojure -(def stacklet->lazy-seq - (fn [f] - (let [val (f nil)] - (if (identical? val :end) - nil - (cons val (lazy-seq* (fn [] (stacklet->lazy-seq f)))))))) - -(def sequence - (fn - ([data] - (let [f (create-stacklet - (fn [h] - (reduce (fn ([h item] (h item) h)) h data) - (h :end)))] - (stacklet->lazy-seq f))) - ([xform data] - (let [f (create-stacklet - (fn [h] - (transduce xform - (fn ([] h) - ([h item] (h item) h) - ([h] (h :end))) - data)))] - (stacklet->lazy-seq f))))) - -(comment - (sequence [1 2 3 4]) now produces '(1 2 3 4)) -``` - -* Math system is fully polymorphic. Math primitives (+,-, etc.) are built off of polymorphic functions that dispatch on the types of the first two arguments. This allows the math system to be extended to complex numbers, matricies, etc. The performance penalty of such a polymorphic call is completely removed by the RPython generated JIT. +* Math system is fully polymorphic. Math primitives (+,-, etc.) are built off of polymorphic functions that dispatch on the types of the first two arguments. This allows the math system to be extended to complex numbers, matrices, etc. The performance penalty of such a polymorphic call is completely removed by the RPython generated JIT. (Planned "magical" Features) @@ -99,17 +94,22 @@ However there are a few features of pixie that although may not be uncommon, are * STM for parallelism. Once STM gets merged into the mainline branch of PyPy, we'll adopt it pretty quickly. -* CSP for concurrency. We already have stacklets, it's not that hard to use them for CSP style concurrency as well. +* CSP for concurrency. We already have stacklets, it's not that hard to use them for CSP style concurrency as well. +## Where do the devs hangout? +Mostly on FreeNode at `#pixie-lang` stop by and say "hello". ## Contributing -Currently there isn't a whole lot that newcomers can help out with, since the entire codebase is in quite a bit of flux, and the main primitives are still under development. However, we do want to have a very open contribution process. If you have a feature you'd like to implement, submit a PR or file an issue and we'll see what we can do. +We have a very open contribution process. If you have a feature you'd like to implement, submit a PR or file an issue and we'll see what we can do. Most PRs are either rejected (if there is a technical flaw) or accepted within a day, so send an improvement our way and see what happens. ## Implementation Notes Although parts of the language may be very close to Clojure (they are both lisps after all), language parity is not a design goal. We will take the features from Clojure or other languages that are suitable to our needs, and feel free to reject those that aren't. Therefore this should not be considered a "Clojure Dialect", but instead a "Clojure inspired lisp". +## Disclaimer +This project is the personal work of Timothy Baldridge and contributors. It is not supported by any entity, including Timothy's employer, or any employers of any other contributors. + ## Copying -Free use of this software is granted under the terms of the GNU Lesser General Public License (LGPL). For details see the files `COPYING` and `COPYING.LESSER` included with the 0MQ distribution. +Free use of this software is granted under the terms of the GNU Lesser General Public License (LGPL). For details see the files `COPYING` and `COPYING.LESSER` included with the source distribution. All copyrights are owned by their respective authors. diff --git a/benchmarks/apply_varargs.lisp b/benchmarks/apply_varargs.pxi similarity index 100% rename from benchmarks/apply_varargs.lisp rename to benchmarks/apply_varargs.pxi diff --git a/benchmarks/deftype_fields.pxi b/benchmarks/deftype_fields.pxi new file mode 100644 index 00000000..fbee851b --- /dev/null +++ b/benchmarks/deftype_fields.pxi @@ -0,0 +1,16 @@ +;; Before Immutable Opt: 3.9 sec +;; After 3.2 sec + +(defprotocol IAdder + (add-them [this])) + +(deftype Adder [a b] + IAdder + (add-them [this] + (set-field! this :b (+ a b)) + b)) + + +(def adder (->Adder 1 0)) +(dotimes [x (* 1024 1024 1024)] + (assert (= (inc x) (add-them adder)))) diff --git a/benchmarks/dynamic_vars.lisp b/benchmarks/dynamic_vars.pxi similarity index 100% rename from benchmarks/dynamic_vars.lisp rename to benchmarks/dynamic_vars.pxi diff --git a/benchmarks/ffi_test.pxi b/benchmarks/ffi_test.pxi new file mode 100644 index 00000000..a96d4d71 --- /dev/null +++ b/benchmarks/ffi_test.pxi @@ -0,0 +1,5 @@ +(loop [x 0] + (if (= x 10000) + x + (do (printf ".") + (recur (inc x))))) diff --git a/benchmarks/get_from_hashmap.pxi b/benchmarks/get_from_hashmap.pxi new file mode 100644 index 00000000..7eb7583e --- /dev/null +++ b/benchmarks/get_from_hashmap.pxi @@ -0,0 +1,5 @@ +(let [map {:number 1}] + (loop [x 0] + (if (= x 10000) + x + (recur (+ x (get map :number)))))) diff --git a/benchmarks/read-line.pxi b/benchmarks/read-line.pxi new file mode 100644 index 00000000..c252fd09 --- /dev/null +++ b/benchmarks/read-line.pxi @@ -0,0 +1,42 @@ +(ns benchmarks.readline + (:require [pixie.time :as time] + [pixie.io :as io] + [pixie.streams.utf8 :as utf8])) + +(def file-name "/usr/share/dict/words") + +(println "Lazy line-seq") +(time/time + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (io/line-seq) + (count))) + +(println "Reducing line-reader") +(time/time + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (io/line-reader) + (into []) + (count))) + +(println "Lazy UTF8 line-seq") +(time/time + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (utf8/utf8-input-stream) + (io/line-seq) + (count))) + +(println "Reducing UTF8 line-reader") +(time/time + (->> file-name + (io/open-read) + (io/buffered-input-stream) + (utf8/utf8-input-stream) + (io/line-reader) + (into []) + (count))) diff --git a/benchmarks/reduce_varargs.lisp b/benchmarks/reduce_varargs.pxi similarity index 100% rename from benchmarks/reduce_varargs.lisp rename to benchmarks/reduce_varargs.pxi diff --git a/benchmarks/transduce_range_iterator.pxi b/benchmarks/transduce_range_iterator.pxi new file mode 100644 index 00000000..a30f9ab2 --- /dev/null +++ b/benchmarks/transduce_range_iterator.pxi @@ -0,0 +1 @@ +(reduce (fn [_ i] nil) nil (-iterator (range 10000000))) diff --git a/benchmarks/transduce_range_reduced.pxi b/benchmarks/transduce_range_reduced.pxi new file mode 100644 index 00000000..4939c02a --- /dev/null +++ b/benchmarks/transduce_range_reduced.pxi @@ -0,0 +1 @@ +(reduce (fn [_ i] nil) nil (range 10000000)) diff --git a/benchmarks/vector_assoc.lisp b/benchmarks/vector_assoc.pxi similarity index 100% rename from benchmarks/vector_assoc.lisp rename to benchmarks/vector_assoc.pxi diff --git a/benchmarks/vector_build_and_hash.pxi b/benchmarks/vector_build_and_hash.pxi new file mode 100644 index 00000000..f09c451e --- /dev/null +++ b/benchmarks/vector_build_and_hash.pxi @@ -0,0 +1,6 @@ +(loop [acc []] + (if (= (count acc) 10000) + (hash acc) + (recur (conj acc (count acc))))) + +:exit-repl diff --git a/checkout-externals b/checkout-externals deleted file mode 100755 index 0cb310f8..00000000 --- a/checkout-externals +++ /dev/null @@ -1,7 +0,0 @@ -mkdir ../externals -cd ../externals -curl https://bitbucket.org/halgari/pypy/get/e5d9ee0be134.tar.bz2 > pypy.tar.bz2 -mkdir pypy -cd pypy -tar -jxf ../pypy.tar.bz2 --strip-components=1 -cd - diff --git a/examples/gen-docs.pxi b/examples/gen-docs.pxi new file mode 100755 index 00000000..9cf965fa --- /dev/null +++ b/examples/gen-docs.pxi @@ -0,0 +1,64 @@ +#!/usr/bin/env pixie-vm + +; generate html docs for a given namespace +(ns gen-ns-docs + (use 'hiccup.core)) + +(defn munge [nm] + (-> (str nm) + (replace "-" "_") + (replace "?" ""))) + +(defn generate-docs [ns] + (html [:html + [:head + [:title ns] + [:meta {:charset "utf-8"}] + [:style {:type "text/css"} +" +.version { + color: #aaa; +} + +#overview ul { + -webkit-column-width: 15em; + -moz-column-width: 15em; + column-width: 15em; +} +"]] + [:body + [:h1 ns] + (let [syms (ns-map ns) + infos (transduce (comp (map (fn [sym] + (when-let [info (meta @(resolve-in (the-ns ns) sym))] + (assoc info :name (hiccup.util/escape-html sym))))) + (filter (complement nil?))) + conj + (keys (ns-map ns)))] + (list [:section#overview + [:h2 "Overview"] + [:ul + (for [{name :name} infos] + [:li [:a {:href (str "#" ns "/" name)} (str ns "/" name)]])]] + (seq (map (fn [{:keys [name doc signatures added examples]}] + [:article + [:h2 {:id (str ns "/" name)} name (when added [:span.version (str " (since " added ")")])] + (when signatures + [:pre (pr-str (seq signatures))]) + (when doc + [:pre doc]) + (when examples + [:section.examples + [:ul + (for [[expr output result] examples] + [:pre (str "user => " expr (as-str output) "\n" (as-str result))])]])]) + infos))))]])) + +(defn main [file ns] + (load-file file) + (println (str "\n" (generate-docs (read-string ns))))) + +(if (< (count program-arguments) 2) + (println "Usage: gen-ns-docs ") + (let [[file ns] program-arguments] + (main file ns))) diff --git a/examples/hello-world.pxi b/examples/hello-world.pxi new file mode 100755 index 00000000..5c434843 --- /dev/null +++ b/examples/hello-world.pxi @@ -0,0 +1,6 @@ +#!./pixie-vm + +(defn greet [name] + (println (str "Hello, " (or name "World") "!"))) + +(greet (first program-arguments)) diff --git a/examples/mandelbrot.pxi b/examples/mandelbrot.pxi new file mode 100644 index 00000000..4258280d --- /dev/null +++ b/examples/mandelbrot.pxi @@ -0,0 +1,101 @@ +;; Mandelbrot demo from Timothy Baldridge's 2015 Strange Loop talk +;; https://www.youtube.com/watch?v=1AjhFZVfB9c + + +(ns mandelbrot + (:require [pixie.ffi-infer :refer :all] + [pixie.ffi :as ffi] + [pixie.time :refer [time]])) + +(with-config {:library "SDL2" + :cxx-flags ["`sdl2-config --cflags`"] + :includes ["SDL.h"]} + + (defcfn SDL_Init) + + (defconst SDL_INIT_VIDEO) + (defconst SDL_WINDOWPOS_UNDEFINED) + (defcfn SDL_CreateWindow) + (defcfn SDL_CreateRenderer) + (defcfn SDL_CreateTexture) + (defconst SDL_PIXELFORMAT_RGBA8888) + (defconst SDL_TEXTUREACCESS_STREAMING) + (defcfn SDL_UpdateTexture) + (defcfn SDL_RenderClear) + (defcfn SDL_RenderCopy) + + (defconst SDL_WINDOW_SHOWN) + (defcfn SDL_RenderPresent) + (defcfn SDL_LockSurface)) + +(println "starting") +(def WIDTH 1024) +(def HEIGHT 512) + +(assert (>= (SDL_Init SDL_INIT_VIDEO) 0)) + +(def WINDOW (SDL_CreateWindow "Pixie MandelBrot" + SDL_WINDOWPOS_UNDEFINED + SDL_WINDOWPOS_UNDEFINED + WIDTH + HEIGHT + SDL_WINDOW_SHOWN)) + +(assert WINDOW "Could not create window") + +(def RENDERER (SDL_CreateRenderer WINDOW -1 0)) + +(def DRAW_SURFACE (SDL_CreateTexture RENDERER + SDL_PIXELFORMAT_RGBA8888 + SDL_TEXTUREACCESS_STREAMING + WIDTH + HEIGHT)) + +(defn bit-or [& args] + (reduce pixie.stdlib/bit-or 0 args)) + +(defn put-pixel [ptr x y r g b] + (let [loc (* 4 (+(* y WIDTH) x))] + (ffi/pack! ptr loc CUInt32 (bit-or (bit-shift-left r 24) + (bit-shift-left g 16) + (bit-shift-left b 8) + 255)))) + +(def BUFFER (buffer (* 4 WIDTH HEIGHT))) + +(defn mandel-point [x y width height max_iterations] + (let [x0 (float (- (* (/ x width) 3.5) 2.5)) + y0 (float (- (* (/ y height) 2) 1))] + (loop [x 0.0 + y 0.0 + iteration 0] + (let [xsq (* x x) + ysq (* y y)] + (if (and (< (+ xsq + ysq) + (* 2 2)) + (< iteration max_iterations)) + (let [xtemp (+ (- xsq + ysq) + x0) + y (+ (* 2 x y) y0)] + (recur xtemp y (inc iteration))) + (- 1 (/ iteration max_iterations))))))) + +(dotimes [x 3] + (time + (let [max (* WIDTH HEIGHT)] + (dotimes [y HEIGHT] + (dotimes [x WIDTH] + (let [result (mandel-point (float x) (float y) (float WIDTH) (float HEIGHT) 1000) + color (int (* 16777216 result))] + (put-pixel BUFFER x y + (bit-shift-right color 16) + (bit-and (bit-shift-right color 8) 0xff) + (bit-and color 0xff)))))))) + +(SDL_UpdateTexture DRAW_SURFACE nil BUFFER (* 4 WIDTH)) +(SDL_RenderCopy RENDERER DRAW_SURFACE nil nil) +(SDL_RenderPresent RENDERER) + +(pixie.stacklets/sleep 10000) diff --git a/examples/mu-kanren.pxi b/examples/mu-kanren.pxi new file mode 100644 index 00000000..3d48e9ab --- /dev/null +++ b/examples/mu-kanren.pxi @@ -0,0 +1,63 @@ +(ns examples.mu-kanren) + +(defn lvar [] + (gensym)) + +(defn lvar? [x] + (symbol? x)) + +(defn walk [s u] + (let [pr (get s u)] + (if (lvar? pr) + (recur s pr) + pr) + u)) + +(defn unify [s u v] + (let [u (walk s u) + v (walk s v)] + (cond + (and (lvar? u) + (lvar? v) + (= u v)) s + (lvar? u) (assoc s u v) + (lvar? v) (assoc s v u) + :else (and (= u v) s)))) + +(defn == [a b] + (keep (fn [s] (unify s a b)))) + + +(defn -disj [& xforms] + (fn [xf] + (let [xforms (transduce (map (fn [xform] + (xform xf))) + conj + xforms)] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (reduce + (fn [acc xform] + (xform acc i)) + acc + xforms)))))) + +(defn conde [& goals] + (apply -disj (map (fn [goals] + (apply comp goals)) + goals))) + + +"Use transduce to run eagerly" +(transduce (conde + [(== 'a 42)] + [ (== 'b 1)]) + conj + [{}]) + +"Use sequence to make it lazy (via stacklets)" +(sequence (conde + [(== 'a 42)] + [(== 'b 1)]) + [{}]) diff --git a/examples/tty-io-test.pxi b/examples/tty-io-test.pxi new file mode 100644 index 00000000..0fadfa3a --- /dev/null +++ b/examples/tty-io-test.pxi @@ -0,0 +1,29 @@ +(ns io-test + (:require [pixie.io :as io] + [pixie.system :as sys] + [pixie.io.tty :as tty])) + +(def history (atom [])) + +(defn depth [d] + (apply str (take d (repeat " ")))) + +(defn nested-trace + "Prints a stack trace with each level indented slightly" + [e] + (loop [d 0 traces (trace e)] + (io/spit tty/stdout (str (depth d) (pr-str (first traces)) "\n")) + (if (seq traces) + (recur (inc d) (rest traces))))) + +(io/spit tty/stdout "TTY Demo REPL\n") +(loop [] + (let [command-number (count @history)] + (io/spit tty/stdout (str "[ " command-number " ] < " )) + (let [input (io/read-line tty/stdin) + res (try (eval (read-string input)) + (catch e + (nested-trace e)))] + (io/spit tty/stdout (str "[ " command-number " ] > " res "\n")) + (swap! history conj input) + (recur)))) diff --git a/find_externals_name.py b/find_externals_name.py new file mode 100644 index 00000000..7f9b88d4 --- /dev/null +++ b/find_externals_name.py @@ -0,0 +1,3 @@ +from rpython.translator.platform import platform + +print "https://github.com/pixie-lang/external-deps/releases/download/1.0/externals-"+platform.name+".tar.bz2" \ No newline at end of file diff --git a/generate-docs.pxi b/generate-docs.pxi new file mode 100644 index 00000000..5f8abb5b --- /dev/null +++ b/generate-docs.pxi @@ -0,0 +1,54 @@ +(ns pixie.generate-docs + (:require [pixie.io :as io] + [pixie.string :as string])) + +(let [[namespace] program-arguments] + + (println "==============") + (println (name namespace)) + (println "==============") + + (load-ns (symbol namespace)) + (println) + + ;;Should be sorting the map + ;;Like so: (sort-by first map) + ;;However, I'm holding off until sort is properly supported + (doseq [[k v] (ns-map (the-ns namespace))] + (println (name k)) + (println "====================================") + (println) + + (if-let [m (meta @v)] + (do + ;(println m) + (if-let [doc (:doc m)];; + (println doc) + (println "No doc available :(")) + (println) + + (when-let (examples (:examples m)) + (println "**Examples:**") + (doseq [[code _ result] examples] + (println) + (println code) + (println) + (when (not (nil? result)) + (println "=> " result))) + (println)) + + (when-let (signatures (:signatures m)) + (println "**Signatures:**") + (println) + (doseq [sig signatures] + (println (str "- " sig))) + (println)) + + (when (and (:line-number m) (:file m)) + (let [file (str "pixie/" (last (string/split (:file m) "/")))] + (println (str "http://github.com/pixie-lang/pixie/blob/master/" + file "#L" (:line-number m)))) + (println))) + + (println "No meta data available :(")) + (println))) \ No newline at end of file diff --git a/lib_pixie.py b/lib_pixie.py new file mode 100644 index 00000000..1a9f2967 --- /dev/null +++ b/lib_pixie.py @@ -0,0 +1,11 @@ +import ctypes, sys + +dll = ctypes.CDLL("libpixie-vm.dylib") + +dll.rpython_startup_code() +dll.pixie_init(sys.argv[0]) + +def repl(): + dll.pixie_execute_source("(ns user (:require [pixie.repl :as repl])) (pixie.repl/repl)") + +repl() \ No newline at end of file diff --git a/make-with-jit b/make-with-jit deleted file mode 100755 index 8abafb1c..00000000 --- a/make-with-jit +++ /dev/null @@ -1 +0,0 @@ -python ../externals/pypy/rpython/bin/rpython --opt=jit --continuation --no-shared target.py diff --git a/pixie/PixieChecker.hpp b/pixie/PixieChecker.hpp new file mode 100644 index 00000000..7266cbea --- /dev/null +++ b/pixie/PixieChecker.hpp @@ -0,0 +1,443 @@ +// +// Pixie C++ typer +// +// This code (when compiled) will emit EDN data for a given C++ type. This includes C functions +// So calling this: +// PixieChecker::DumpType() +// will emit this in the compiled program: +// {:type :function :arity 1 :returns {:type :float :size 8} :arguments [{:type :pointer :of-type {:type :int :size 1 :signed? true}} ]} + +#include +#include "boost/type_traits.hpp" + +namespace PixieChecker { + +template +std::string GetType(); + +template +struct GenericChecker; + +template < typename T > std::string to_string( const T& n ) +{ + std::ostringstream stm ; + stm << n ; + return stm.str() ; +} + + +// Function Checker +template +struct FunctionTyper +{ + static std::string getType() + { + return "{:type :function :arity :unknown}"; + } +}; + +template +struct FunctionTyper<0, T> +{ + static std::string getType() + { + return "{:type :function :arity 0 :returns " + + GetType::result_type>() + + " :arguments []}"; + + } +}; + + + template + struct FunctionTyper<1, T> + { + static std::string getType() + { + return "{:type :function :arity 1 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<2, T> + { + static std::string getType() + { + return "{:type :function :arity 2 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + + " ]}"; + } + }; + + + template + struct FunctionTyper<3, T> + { + static std::string getType() + { + return "{:type :function :arity 3 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<4, T> + { + static std::string getType() + { + return "{:type :function :arity 4 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<5, T> + { + static std::string getType() + { + return "{:type :function :arity 5 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<6, T> + { + static std::string getType() + { + return "{:type :function :arity 6 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<7, T> + { + static std::string getType() + { + return "{:type :function :arity 7 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<8, T> + { + static std::string getType() + { + return "{:type :function :arity 8 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + GetType::arg8_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<9, T> + { + static std::string getType() + { + return "{:type :function :arity 9 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + GetType::arg8_type>() + " " + + GetType::arg9_type>() + " " + + " ]}"; + } + }; + + template + struct FunctionTyper<10, T> + { + static std::string getType() + { + return "{:type :function :arity 10 :returns " + + GetType::result_type>() + + " :arguments [" + + GetType::arg1_type>() + " " + + GetType::arg2_type>() + " " + + GetType::arg3_type>() + " " + + GetType::arg4_type>() + " " + + GetType::arg5_type>() + " " + + GetType::arg6_type>() + " " + + GetType::arg7_type>() + " " + + GetType::arg8_type>() + " " + + GetType::arg9_type>() + " " + + GetType::arg10_type>() + " " + + " ]}"; + } + }; + + +// End Function Typer + +// Is Enum + template + struct GenericCheckerIsEnum + { + static std::string getType() + { + return "Shouldn't happen in enum checker"; + } + }; + + template + struct GenericCheckerIsEnum + { + static std::string getType() + { + return GetType(); + } + }; + + template + struct GenericCheckerIsEnum + { + static std::string getType() + { + return "{:type :unknown}"; + } + }; +// End Is Enum + + + +// Is Void + template + struct GenericCheckerIsVoid + { + static std::string getType() + { + return "Shouldn't happen in void checker"; + } + }; + + template + struct GenericCheckerIsVoid + { + static std::string getType() + { + return "{:type :void}"; + } + }; + + template + struct GenericCheckerIsVoid + { + static std::string getType() + { + return GenericCheckerIsEnum::type, T>::getType(); + } + }; +// End Is Void + +// Is Pointer + +template +struct GenericCheckerIsPointer +{ + static std::string getType() + { + return "Shouldn't happen in pointer checker"; + } +}; + +template +struct GenericCheckerIsPointer +{ + static std::string getType() + { + return "{:type :pointer :of-type " + GetType::type>() + "}"; + } +}; + +template +struct GenericCheckerIsPointer +{ + static std::string getType() + { + return GenericCheckerIsVoid::type, T>::getType(); + } +}; + + +// End Is Pointer + + + +// Is Function + +template +struct GenericCheckerIsFunction +{ + static std::string getType() + { + return "Shouldn't happen in function checker"; + } +}; + +template +struct GenericCheckerIsFunction +{ + static std::string getType() + { + return FunctionTyper::arity, T>::getType(); + } +}; + +template +struct GenericCheckerIsFunction +{ + static std::string getType() + { + return GenericCheckerIsPointer::type, T>::getType(); + } +}; + +// End Is Function + + +// Is Float + +template +struct GenericCheckerIsFloat +{ + static std::string getType() + { + throw 42; + } +}; + +template +struct GenericCheckerIsFloat +{ + static std::string getType() + { + return "{:type :float :size " + to_string(sizeof(T)) + "}"; + } +}; + +template +struct GenericCheckerIsFloat +{ + static std::string getType() + { + return GenericCheckerIsFunction::type, T>::getType(); + } +}; + +// End is Float + +// IsInt + +template +struct GenericCheckerIsInt +{ + static std::string getType() + { + return "Shouldn't happen"; + } +}; + +template +struct GenericCheckerIsInt +{ + static std::string getType() + { + return "{:type :int :size " + to_string(sizeof(T)) + + " :signed? " + (boost::is_signed::value ? "true" : "false") + + "}"; + } +}; + +template +struct GenericCheckerIsInt +{ + static std::string getType() + { + return GenericCheckerIsFloat::type, T>::getType(); + } +}; + +// End is Int + +// Generic Typer +template +std::string GetType() +{ + return GenericCheckerIsInt::type, T>::getType(); +} + +// End Generic Typer + +template +void DumpValue(T t) +{ + std::cout << GetType() << std::endl; +} + +template +void DumpType() +{ + std::cout << GetType() << std::endl; +} + +} diff --git a/pixie/async.pxi b/pixie/async.pxi new file mode 100644 index 00000000..ca7862f8 --- /dev/null +++ b/pixie/async.pxi @@ -0,0 +1,31 @@ +(ns pixie.async + (:require [pixie.stacklets :as st])) + + +(deftype Promise [val pending-callbacks delivered?] + IDeref + (-deref [self] + (if delivered? + val + (do + (st/call-cc (fn [k] + (swap! pending-callbacks conj + (fn [v] + (st/-run-later (partial st/run-and-process k v))))))))) + IFn + (-invoke [self v] + (assert (not delivered?) "Can only deliver a promise once") + (set-field! self :val v) + (set-field! self :delivered? true) + (doseq [f @pending-callbacks] + (f v)) + (reset! pending-callbacks nil) + nil)) + +(defn promise [] + (->Promise nil (atom []) false)) + +(defmacro future [& body] + `(let [p# (promise)] + (st/spawn (p# (do ~@body))) + p#)) diff --git a/pixie/buffers.pxi b/pixie/buffers.pxi new file mode 100644 index 00000000..5e6009db --- /dev/null +++ b/pixie/buffers.pxi @@ -0,0 +1,146 @@ +(ns pixie.buffers) + +(defn acopy [src src-start dest dest-start len] + (loop [cnt 0] + (when (< cnt len) + (aset dest + (+ dest-start cnt) + (aget src (+ src-start cnt))) + (recur (inc cnt))))) + + +(defprotocol IMutableBuffer + (remove! [this]) + (add! [this]) + (full? [this])) + +(defprotocol IResizableMutableBuffer + (add-unbounded! [this val]) + (resize! [this new-size])) + +(deftype RingBuffer [head tail length arr] + IMutableBuffer + (remove! [this] + (when-not (zero? length) + (let [x (aget arr tail)] + (aset arr tail nil) + (set-field! this :tail (int (rem (inc tail) (alength arr)))) + (set-field! this :length (dec length)) + x))) + (add! [this x] + (assert (< length (alength arr))) + (aset arr head x) + (set-field! this :head (int (rem (inc head) (alength arr)))) + (set-field! this :length (inc length)) + nil) + + (full? [this] + (= length (alength arr))) + + + IResizableMutableBuffer + (resize! [this new-size] + (let [new-arr (make-array new-size)] + (cond + (< tail head) + (do (acopy arr tail new-arr 0 length) + (set-field! this :tail 0) + (set-field! this :head length) + (set-field! this :arr new-arr)) + + (> tail head) + (do (acopy arr tail new-arr 0 (- (alength arr) tail)) + (acopy arr 0 new-arr (- (alength arr) tail) head) + (set-field! this :tail 0) + (set-field! this :head length) + (set-field! this :arr new-arr)) + + + (full? this) + (do (acopy arr tail new-arr 0 length) + (set-field! this :tail 0) + (set-field! this :head length) + (set-field! this :arr new-arr)) + + + :else + (do (set-field! this :tail 0) + (set-field! this :head 0) + (set-field! this :arr new-arr))))) + + (add-unbounded! [this val] + (when (full? this) + (resize! this (* 2 length))) + (add! this val)) + + ICounted + (-count [this] + length)) + + +(defn ring-buffer [size] + (assert (> size 0) "Can't create a ring buffer of size <= 0") + (->RingBuffer 0 0 0 (make-array size))) + + +(defn fixed-buffer [size] + (ring-buffer size)) + + +(deftype DroppingBuffer [buf] + IMutableBuffer + (full? [this] + false) + (remove! [this] + (remove! buf)) + (add! [this val] + (when-not (full? buf) + (add! buf val))) + + ICounted + (-count [this] + (count buf))) + +(defn dropping-buffer [size] + (->DroppingBuffer (ring-buffer size))) + + +(deftype SlidingBuffer [buf] + IMutableBuffer + (full? [this] + false) + (remove! [this] + (remove! buf)) + (add! [this val] + (when (full? buf) + (remove! buf)) + (add! buf val)) + + ICounted + (-count [this] + (count buf))) + +(defn sliding-buffer [size] + (->SlidingBuffer (ring-buffer size))) + +(defn empty-buffer? [buf] + (= (count buf) 0)) + + +(deftype NullBuffer [] + IMutableBuffer + (full? [this] + true) + ICounted + (-count [this] 0)) + +(def null-buffer (->NullBuffer)) + +(extend -reduce IMutableBuffer + (fn [buf f acc] + (loop [acc acc] + (if (reduced? acc) + @acc + (if (pos? (count buf)) + (recur (f acc (remove! buf))) + acc))))) diff --git a/pixie/channels.pxi b/pixie/channels.pxi new file mode 100644 index 00000000..1f8067df --- /dev/null +++ b/pixie/channels.pxi @@ -0,0 +1,203 @@ +(ns pixie.channels + (:require [pixie.stacklets :as st] + [pixie.uv :as uv] + [pixie.ffi :as ffi] + [pixie.buffers :as b])) + +(defprotocol ICancelable + (-canceled? [this] "Determines if a request (such as a callback) that can be canceled") + (-commit! [this])) + +(defprotocol IReadPort + (-take! [this cfn] "Take a value from this port passing it to a cancelable function")) + +(defprotocol IWritePort + (-put! [this itm cfn] "Write a value to this port passing true if the write succeeds and the + callback isn't canceled")) + +(defprotocol ICloseable + (-close! [this] "Closes the channel, future writes will be rejected, future reads will + drain the channel before returning nil.")) + +(deftype OpCell [val cfn] + IIndexed + (-nth [this idx] + (cond + (= idx 0) val + (= idx 1) cfn + :else (throw [::OutOfRangeException "Index out of range"]))) + (-nth-not-found [this idx not-found] + (cond + (= idx 0) val + (= idx 1) cfn + :else not-found)) + ICounted + (-count [this] + 2) + ICancelable + (-canceled? [this] + (canceled? cfn))) + +(defn canceled? [this] + (-canceled? this)) + + +(defn -move-puts-to-buffer [puts buffer] + (loop [] + (if (or (b/full? buffer) + (b/empty-buffer? puts)) + nil + (let [[val cfn] (b/remove! puts)] + (if (canceled? cfn) + (recur) + (do (st/-run-later (partial cfn true)) + (b/add! buffer val) + (recur))))))) + +(defn -get-non-canceled! [buffer] + (loop [] + (if (b/empty-buffer? buffer) + nil + (let [v (b/remove! buffer)] + (if (canceled? v) + (recur) + v))))) + + +(deftype MultiReaderWriterChannel [puts takes buffer closed? ops-since-last-clean] + IReadPort + (-take! [this cfn] + (if (canceled? cfn) + false + (if (and closed? + (b/empty-buffer? buffer) + (b/empty-buffer? puts)) + (do (-commit! cfn) + (st/-run-later (partial cfn nil)) + false) + (if (not (b/empty-buffer? buffer)) + (do (-commit! cfn) + (st/-run-later (partial cfn (b/remove! buffer))) + (-move-puts-to-buffer puts buffer)) + + (if-let [[v pcfn] (-get-non-canceled! puts)] + (do (-commit! pcfn) + (-commit! cfn) + (st/-run-later (partial pcfn true)) + (st/-run-later (partial cfn v)) + true) + (do (set-field! this :ops-since-last-clean (inc ops-since-last-clean)) + (b/add-unbounded! takes cfn) + true)))))) + IWritePort + (-put! [this val cfn] + (if (or (canceled? cfn)) + false + (if closed? + (do (-commit! cfn) + (st/-run-later (partial cfn false)) + false) + (if-let [tfn (-get-non-canceled! takes)] + (do (-commit! cfn) + (-commit! tfn) + (st/-run-later (partial tfn val)) + (st/-run-later (partial cfn true)) + true) + (if (not (b/full? buffer)) + (do (b/add! buffer val) + (-commit! cfn) + (st/-run-later (partial cfn true)) + true) + (do (b/add-unbounded! puts (->OpCell val cfn)) + (set-field! this :ops-since-last-clean (inc ops-since-last-clean)) + true)))))) + ICloseable + (-close! [this] + (set-field! this :closed? true) + (when (not (b/empty-buffer? takes)) + (loop [] + (when-let [tfn (-get-non-canceled! takes)] + (-commit! tfn) + (st/-run-later (partial tfn nil)) + (recur)))))) + +(defn chan + "Creates a CSP channel with the given buffer. If an integer is provided as the argument + creates a channel with a fixed buffer of that size. " + ([] + (chan 0)) + ([size-or-buffer] + (if (= 0 size-or-buffer) + (->MultiReaderWriterChannel (b/ring-buffer 8) + (b/ring-buffer 8) + b/null-buffer + false + 0) + (if (integer? size-or-buffer) + (->MultiReaderWriterChannel (b/ring-buffer 8) + (b/ring-buffer 8) + (b/fixed-buffer size-or-buffer) + false + 0) + (->MultiReaderWriterChannel (b/ring-buffer 8) + (b/ring-buffer 8) + size-or-buffer + false + 0))))) + +(defn timeout + "Returns a channel that will close after given delay in ms" + [ms] + (let [ch (chan) + timer (uv/uv_timer_t) + cb (atom nil)] + (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb + (fn [handle] + (try + (-close! ch) + (uv/uv_timer_stop timer) + (-dispose! @cb) + (catch ex + (println ex)))))) + (uv/uv_timer_init (uv/uv_default_loop) timer) + (uv/uv_timer_start timer @cb ms 0) + ch)) + +(deftype AltHandler [atm f] + ICancelable + (-canceled? [this] + @atm) + (-commit! [this] + (reset! atm true)) + IFn + (-invoke [this & args] + (apply f args))) + +(defn alt-handlers [fns] + (mapv (partial ->AltHandler (atom false)) fns)) + +(extend -canceled? IFn + (fn [this] false)) + +(extend -commit! IFn + (fn [this] nil)) + +(defn alts! [ops k options] + (let [handler-atom (atom false)] + (reduce + (fn [_ op] + (if (vector? op) + (let [[c val] op + f (fn [v] + (st/-run-later (partial k [c v])))] + (-put! c val (->AltHandler handler-atom f))) + (let [c op + f (fn [v] + (st/-run-later (partial k [c v])))] + (-take! c (->AltHandler handler-atom f))))) + nil + ops) + (when (and (contains? options :default) + (not @handler-atom)) + (reset! handler-atom true) + (st/-run-later (partial k [:default (:default options)]))))) diff --git a/pixie/csp.pxi b/pixie/csp.pxi new file mode 100644 index 00000000..32c96c8c --- /dev/null +++ b/pixie/csp.pxi @@ -0,0 +1,63 @@ +(ns pixie.csp + (:require [pixie.stacklets :as st] + [pixie.buffers :as b] + [pixie.channels :as chans])) + +(def chan chans/chan) +(def timeout chans/timeout) + +(defn close! + "Closes the channel, future writes will be rejected, future reads will + drain the channel before returning nil." + [c] + (chans/-close! c)) + +(def -null-callback (fn [_] nil)) + +(defn put! + "Puts the value into the channel, calling the optional callback when the operation has + completed." + ([c v] + (chans/-put! c v -null-callback)) + ([c v f] + (chans/-put! c v f))) + +(defn take! + "Takes a value from a channel, calling the provided callback when completed" + ([c f] + (chans/-take! c f))) + +(defn >! [c v] + (st/call-cc (fn [k] + (chans/-put! c v (partial st/run-and-process k))))) + +(defn > m + (transduce (comp (map (fn [[k v]] (string/interp "$(write-string k)$: $(write-string v)$"))) + (interpose ", ")) + string-builder)) + "}")) + +(defn- write-sequential [xs] + (str "[" + (->> xs + (transduce (comp (map write-string) + (interpose ", ")) + string-builder)) + "]")) + +(defn- write-str [s] + (string/interp "\"$s$\"")) + +(extend-protocol IToJSON + Character (write-string [this] (write-str this)) + Cons (write-string [this] (write-sequential this)) + EmptyList (write-string [this] (write-sequential this)) + Nil (write-string [_] "null") + Number (write-string [this] (str this)) + Keyword (write-string [this] (write-str (name this))) + LazySeq (write-string [this] (write-sequential this)) + IMap (write-string [this] (write-map this)) + IVector (write-string [this] (write-sequential this)) + PersistentList (write-string [this] (write-sequential this)) + Ratio (write-string [this] (str (float this))) + String (write-string [this] (write-str this))) diff --git a/pixie/ffi-infer.pxi b/pixie/ffi-infer.pxi new file mode 100644 index 00000000..6b6920d6 --- /dev/null +++ b/pixie/ffi-infer.pxi @@ -0,0 +1,348 @@ +(ns pixie.ffi-infer + (:require [pixie.io-blocking :as io])) + + +(defn -add-rel-path [rel] + (swap! load-paths conj (str (first @load-paths) "/" rel))) + +(-add-rel-path "lib") +(-add-rel-path "include") +(-add-rel-path "../lib") +(-add-rel-path "../include") + + +(def *config* nil) +(set-dynamic! (var *config*)) +(def *bodies* nil) +(set-dynamic! (var *bodies*)) +(def *library* nil) +(set-dynamic! (var *library*)) + + +(defmulti emit-infer-code :op) + +(defmethod emit-infer-code :const + [{:keys [name]}] + (str "std::cout << \"{:value \" << " name " << \" :type \" " + ";\n PixieChecker::DumpValue(" name "); \n std::cout << \"}\" << std::endl; ")) + +(defmethod emit-infer-code :group + [{:keys [ops]}] + (str "std::cout << \"[\" << std::endl; " + (apply str (map emit-infer-code ops)) + "std::cout << \"]\" << std::endl; ")) + +(defmethod emit-infer-code :function + [{:keys [name]}] + (str "PixieChecker::DumpType<__typeof__(" name ")>(); \n")) + +(defmethod emit-infer-code :raw-struct + [{:keys [name members]}] + (str "std::cout << \"{:size \" << sizeof(struct " name ")" + " << \" :infered-members [\" << " + (apply str + (map (fn [member] + (str "\"{:type \"; PixieChecker::DumpValue((new (struct " name "))->" member "); " + " std::cout << \":offset \" << offsetof(struct " name ", " member ") << \" }\" << \n ")) + members)) + "\"]}\" << std::endl;")) + +(defmethod emit-infer-code :struct + [{:keys [name members]}] + (str "std::cout << \"{:size \" << sizeof(" name ")" + " << \" :infered-members [\" << " + (apply str + (map (fn [member] + (str "\"{:type \"; PixieChecker::DumpValue((new (" name "))->" member "); " + " std::cout << \":offset \" << offsetof(" name ", " member ") << \" }\" << \n ")) + members)) + "\"]}\" << std::endl;")) + +(defmethod emit-infer-code :callback + [{:keys [name]}] + (str "PixieChecker::DumpType<__typeof__(" name ")>();")) + +(defmethod emit-infer-code :global + [_] + (str "std::cout <<\"[]\" << std::endl;")) + + +(defn start-string [] + (str (apply str (map (fn [i] + (str "#include \"" i "\"\n")) + (:includes *config*))) + "#include \"pixie/PixieChecker.hpp\" + #include \"stdlib.h\" + + + int main(int argc, char* argv[]) { + std::cout << \"[\"; +")) + +(defn end-string [] + " std::cout << \"]\" << std::endl; +return 0; + }") + + +;; To Ctype conversion + +(defmulti edn-to-ctype (fn [n _] + (:type n))) + +(defn callback-type [{:keys [arguments returns]} in-struct?] + `(pixie.ffi/ffi-callback ~(vec (map (fn [x] (edn-to-ctype x in-struct?)) + arguments)) + ~(edn-to-ctype returns in-struct?))) + +(defmethod edn-to-ctype :pointer + [{:keys [of-type] :as ptr} in-struct?] + (cond + (and (= (:size of-type) 1) + (= (:type of-type) :int) + (not in-struct?)) 'pixie.stdlib/CCharP + (= (:type of-type) :function) (callback-type of-type in-struct?) + :else 'pixie.stdlib/CVoidP)) + +(def float-types {32 'pixie.stdlib/CFloat + 64 'pixie.stdlib/CDouble}) + +(defmethod edn-to-ctype :float + [{:keys [size] :as tp} _] + (let [tp-found (get float-types (* 8 size))] + (assert tp-found (str "No type found for " tp)) + tp-found)) + +(defmethod edn-to-ctype :void + [_ _] + `pixie.stdlib/CVoid) + +(def int-types {[8 true] 'pixie.stdlib/CInt8 + [8 false] 'pixie.stdlib/CUInt8 + [16 true] 'pixie.stdlib/CInt16 + [16 false] 'pixie.stdlib/CUInt16 + [32 true] 'pixie.stdlib/CInt32 + [32 false] 'pixie.stdlib/CUInt32 + [64 true] 'pixie.stdlib/CInt64 + [64 false] 'pixie.stdlib/CUInt64}) + +(defmethod edn-to-ctype :int + [{:keys [size signed?] :as tp} _] + (let [tp-found (get int-types [(* 8 size) signed?])] + (assert tp-found (str "No type found for " tp)) + tp-found)) + +;; Code Generation +(defmulti generate-code (fn [input output] + (:op input))) + +(defmethod generate-code :const + [{:keys [name]} {:keys [value type]}] + `(def ~(symbol name) ~value)) + +(defmethod generate-code :global + [{:keys [name]} _] + `(def ~(symbol name) (ffi-voidp *library* ~(str name)))) + + + +(defmethod generate-code :function + [{:keys [name]} {:keys [type arguments returns]}] + (assert (= type :function) (str name " is not infered to be a function")) + `(def ~(symbol name) + (ffi-fn *library* ~name + ~(vec (map (fn [x] (edn-to-ctype x false)) + arguments)) + ~(edn-to-ctype returns false)))) + +(defmethod generate-code :raw-struct + [{:keys [name members]} {:keys [size infered-members]}] + `(def ~(symbol name) + (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] + `[~(keyword name) ~(edn-to-ctype type true) ~offset]) + members infered-members)]))) + +(defmethod generate-code :struct + [{:keys [name members]} {:keys [size infered-members]}] + `(def ~(symbol name) + (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] + `[~(keyword name) ~(edn-to-ctype type true) ~offset]) + members infered-members)]))) + +(defmethod generate-code :type + [{:keys [name members]} {:keys [size infered-members]}] + `(def ~(symbol name) + (pixie.ffi/c-struct ~name ~size [~@(map (fn [name {:keys [type offset]}] + `[~(keyword name) ~(edn-to-ctype type true) ~offset]) + members infered-members)]))) + +(defmethod generate-code :callback + [{:keys [name]} {:keys [of-type]}] + `(def ~(symbol name) + ~(callback-type of-type false))) + +(def mkdtemp (ffi-fn libc "mkdtemp" [CCharP] CCharP)) +(def unlink (ffi-fn libc "unlink" [CCharP] CInt)) +(def rmdir (ffi-fn libc "rmdir" [CCharP] CInt)) +(def tempdir-template (str (or (getenv "TMPDIR") "/tmp") + "/ffiXXXXXX")) + +(defn run-infer [config cmds] + (let [tempdir (mkdtemp tempdir-template) + infile (str tempdir "/ffi.cpp") + outfile (str tempdir "/ffi.out")] + (io/spit infile (str (start-string) + (apply str (map emit-infer-code + cmds)) + (end-string))) + (println @load-paths) + (let [cmd-str (str "c++ " + (apply str (interpose " " pixie.platform/c-flags)) + " " infile " " + (apply str (map (fn [x] ( str " -I " x " ")) + @load-paths)) + (apply str " " (interpose " " (:cxx-flags *config*))) + " -o " outfile " && " outfile) + _ (println cmd-str) + result (read-string (io/run-command cmd-str)) + gen (vec (map generate-code cmds result))] + (unlink infile) + (unlink outfile) + (rmdir tempdir) + `(do ~@gen)))) + +(defn full-lib-name [library-name] + (if (= library-name "c") + pixie.platform/lib-c-name + (str "lib" library-name "." pixie.platform/so-ext))) + +(defmacro with-config [config & body] + (binding [*config* config + *bodies* (atom []) + *library* (ffi-library (full-lib-name (:library config)))] + (doseq [b body] + (eval b)) + `(binding [*library* (ffi-library ~(full-lib-name (:library config)))] + ~(run-infer *config* @*bodies*)))) + +(defmacro defcfn [nm] + (let [name-str (name nm)] + `(swap! *bodies* conj (assoc {:op :function} :name ~name-str)))) + +(defmacro defconst [nm] + (let [name-str (name nm)] + `(swap! *bodies* conj (assoc {:op :const} :name ~name-str)))) + +(defmacro defcstruct [nm members] + `(swap! *bodies* conj (assoc {:op :struct} + :name ~(name nm) + :members ~(vec (map name members))) )) + +(defmacro defc-raw-struct [nm members] + `(swap! *bodies* conj (assoc {:op :raw-struct} + :name ~(name nm) + :members ~(vec (map name members))) )) + +(defmacro defctype [nm members] + `(swap! *bodies* conj (assoc {:op :type} + :name ~(name nm) + :members ~(vec (map name members))) )) + +(defmacro defccallback [nm] + `(swap! *bodies* conj (assoc {:op :callback} + :name ~(name nm)))) + +(defmacro defglobal [nm] + `(swap! *bodies* conj (assoc {:op :global} + :name ~(name nm)))) + + +(defn compile-library [{:keys [prefix includes]} & source] + (let [c-name (str "/tmp/" prefix ".c") + source-header (apply str (map (fn [i] + (str "#include \"" i "\"\n")) + includes)) + source (apply str source-header (interpose "\n\n" source)) + lib-name (str "/tmp/" prefix "-" (hash source) "." pixie.platform/so-ext) + cmd (str "cc -dynamic-lang " + c-name + (apply str (map (fn [x] ( str " -I " x " ")) + @load-paths)) + + " -o " + lib-name)] + (io/spit c-name source) + (println cmd) + (io/run-command cmd))) + + + + +(comment + +(with-config {:library "SDL" + :cxx-flags ["`sdl2-config --cflags --libs`"] + :includes ["SDL.h"] + } + (defconst SDL_INIT_EVERYTHING) + (defcfn SDL_Init) + + (defconst SDL_WINDOW_SHOWN)) + +(f/with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["sys/stat.h"] + } + (f/defcstruct stat [:st_dev + :st_ino + :st_mode + :st_nlink + :st_uid + :st_gid + :st_size]) + (f/defcfn lstat64)) + + + + + +(let [s (stat)] + (pixie.ffi/set! s :st_size 42) + (println (str "\n" (:st_size s))) + (println (str "\n" (lstat64 "/tmp/tmp.cpp" s))) + (println "filesize " (:st_size s) " " (:st_uid s) " " (:st_gid s))) + +(with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["ctime.h"]}) + + +(f/with-config {:library "c" + :cxx-flags ["-lc"] + :includes ["time.h"] + } + (def time_t (pixie.ffi/c-struct :time_t 8 [[:val CInt 0 ]])) + (f/defcfn time) + (f/defcstruct tm [:tm_sec + :tm_min + :tm_hour + :tm_mday + :tm_mon + :tm_year + :tm_wday + :tm_yday + :tm_isdst]) + (f/defcfn localtime)) + + + +(type (pixie.ffi/cast (localtime (time_t)) tm)) + + +(let [t (time_t) + _ (time t) + tmi (pixie.ffi/cast (localtime t) tm)] + + (println (- (:tm_hour tmi) 12) " " (:tm_min tmi))) + + ) diff --git a/pixie/fs.pxi b/pixie/fs.pxi new file mode 100644 index 00000000..679bd5f6 --- /dev/null +++ b/pixie/fs.pxi @@ -0,0 +1,240 @@ +(ns pixie.fs + (:require [pixie.path :as path] + [pixie.string :as string] + [pixie.uv :as uv])) + +(defprotocol IFSPath + (path [this] + "Returns the path used to reference the File/Dir/Filesystem Object") + + (rel [this other] + "Returns the path relative to the other path") + + (abs [this] + "Returns the absolute path") + + (exists? [this] + "Returns true if the file exists") + + (basename [this] + "Returns the basename of the Filesystem Object") + + (permissions [this] + "Returns a string of the octal permissions") + + (size [this] + "Returns the size of the file/dir on disk") + + ;; TODO + + (mounted? [this] + "Returns true if the directory is a mounted")) + +(defprotocol IFile + (extension [this] + "Returns the extension") + + (extension? [this ext] + "Returns true if file has extension") + + ;; TODO + (touch [this] + "Create the file if it doesn't exist.")) + +(defprotocol IDir + (list [this] + "List files and dirs underneath") + + (empty? [this] + "Returns true if directory is not empty") + + (walk [this] + "Recursively returns all files and directories below") + + (walk-files [this] + "Recursively returns all files underneath") + + (walk-dirs [this] + "Recursivley returns all directories underneath")) + +(defn rel-path [a b] + (let [paths-a (string/split (abs a) "/") + paths-b (string/split (abs b) "/") + ;; Get the common root of the two paths and the bits that diverge + [common diff-a diff-b] (loop [ra paths-a rb paths-b common []] + (let [ca (first ra) + cb (first rb)] + (if (and ca cb (= ca cb)) + (recur (rest ra) (rest rb) (conj common ca)) + [common ra rb])))] + (let [level (- (count diff-a) (count diff-b))] + (cond + ;; Same level + (and (zero? (count diff-a)) (zero? (count diff-b))) + "." + + ;; If B diverges by one level and a is a Dir we use ".." + (and (= 1 (count diff-b)) (instance? Dir a)) + ".." + + ;; In all other cases we want to go back to the root, + :else + (apply str (interpose "/" (concat (repeat (count diff-b) "..") diff-a))))))) + +(defn- assert-existence [f] + (assert (exists? f) (str "No file or directory at \"" (abs f) "\""))) + +(uv/defuvfsfn fs-size pixie.uv/uv_fs_stat [file] :statbuf.st_size) +(uv/defuvfsfn fs-mode pixie.uv/uv_fs_stat [file] :statbuf.st_mode) + +(defn- size-of-path [path] + (assert-existence path) + (fs-size (abs path))) + +(defn- permission-string [n] + (apply str + (for [shift [6 3 0]] + (let [mask (bit-shift-left 7 shift) + masked (bit-and mask n)] + (bit-shift-right masked shift))))) + +(defn- permissions-of-path [path] + (assert-existence path) + (permission-string (fs-mode (abs path)))) + +;; File and Dir are just wrappers around paths. +(deftype File [pathz] + IFSPath + (path [this] pathz) + (rel [this other] + (if (satisfies? IFSPath other) + (rel-path this other) + (throw [::AssertionException "Second argument must satisfy IFSPath"]))) + + (abs [this] + (path/-abs pathz)) + + (exists? [this] + (path/-exists? pathz)) + + (basename [this] + (last (string/split (abs this) "/"))) + + (size [this] + (size-of-path this)) + + (permissions [this] + (permissions-of-path this)) + + IFile + ;; TODO: Sort out regex or make strings partitionable. So we can split at + ;; #".". + (extension [this] + (last (string/split (abs this) "."))) + + (extension? [this ext] + (string/ends-with? (abs this) ext)) + + IObject + (-hash [this] + (hash (abs this))) + + (-eq [this other] + (if (instance? File other) + (= (abs this) (abs other)) + false)) + + (-str [this] + (str (abs this))) + + (-repr [this] + (str ""))) + +(deftype Dir [pathz] + IFSPath + (path [this] pathz) + + (rel [this other] + (if (satisfies? IFSPath other) + (rel-path this other) + (throw [::AssertionException "Second argument must satisfy IFSPath"]))) + + (abs [this] + (path/-abs pathz)) + + (exists? [this] + (path/-exists? pathz)) + + (basename [this] + (last (string/split (abs this) "/"))) + + (size [this] + (size-of-path this)) + + (permissions [this] + (permissions-of-path pathz)) + + IDir + (list [this] + (vec (map fspath (path/-list-dir pathz)))) + + (walk [this] + (transduce (map #(if (path/-file? %) + (->File %) + (->Dir %))) + conj + pathz)) + + (walk-files [this] + (filter #(instance? File %) + (walk this))) + + (walk-dirs [this] + (filter #(instance? Dir %) + (walk this))) + + IObject + (-hash [this] + (hash (abs this))) + + (-eq [this other] + (if (instance? Dir other) + (= (abs this) (abs other)) + false)) + + (-str [this] + (str (abs this))) + + (-repr [this] + (str ""))) + +;; (deftype Fifo [pathz]) + +(defn file + "Returns a file if the path is a file or does not exist. If a different filesystem object exists at the path an error will be thrown." + [x] + (let [x (path/-path x)] + (cond + (path/-file? x) (->File x) + (not (path/-exists? x)) (->File x) + :else (throw [::NotAFileException (str "A non-file object exists at path: " x)])))) + +(defn dir + "Returns a dir if the path is a dir or does not exist. If a different filesystem object exists at the path an error will be thrown." + [x] + (let [x (path/-path x)] + (cond + (path/-dir? x) (->Dir x) + (not (path/-exists? x)) (->Dir x) + :else (throw [::NotADirectoryException (str "A non-dir object exists at path: " x)])))) + +(defn fspath + "Returns either a File or Dir if they exist at the path" + [x] + (let [x (path/-path x)] + (cond + (path/-file? x) (->File x) + (path/-dir? x) (->Dir x) + :else (throw [::FileNotFoundException (str "No file or directory at path: " x)])))) + + diff --git a/pixie/io-blocking.pxi b/pixie/io-blocking.pxi new file mode 100644 index 00000000..4af15739 --- /dev/null +++ b/pixie/io-blocking.pxi @@ -0,0 +1,147 @@ +(ns pixie.io-blocking + (:require [pixie.streams :as st :refer :all] + [pixie.io.common :as common])) + +(def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) +(def fseek (ffi-fn libc "fseek" [CVoidP CInt CInt] CInt)) +(def ftell (ffi-fn libc "ftell" [CVoidP] CInt)) +(def -rewind (ffi-fn libc "rewind" [CVoidP] CVoidP)) +(def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) +(def fgetc (ffi-fn libc "fgetc" [CVoidP] CInt)) +(def fputc (ffi-fn libc "fputc" [CInt CVoidP] CInt)) +(def fwrite (ffi-fn libc "fwrite" [CVoidP CInt CInt CVoidP] CInt)) +(def fclose (ffi-fn libc "fclose" [CVoidP] CInt)) +(def popen (ffi-fn libc "popen" [CCharP CCharP] CVoidP)) +(def pclose (ffi-fn libc "pclose" [CVoidP] CInt)) +(def EOF -1) + +(deftype FileStream [fp] + IInputStream + (read [this buffer len] + (assert (>= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [read-count (fread buffer 1 len fp)] + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (fgetc buffer)) + ISeekableStream + (seek [this pos] + (fseek fp pos 0)) + (rewind [this] + (-rewind fp)) + IDisposable + (-dispose! [this] + (fclose fp)) + IReduce + (-reduce [this f init] + (common/stream-reducer this f init))) + +(defn open-read + {:doc "Opens a file for reading. Returns an IInputStream." + :added "0.1"} + [filename] + (assert (string? filename) "Filename must be a string") + (->FileStream (fopen filename "r"))) + +(defn read-line + "Reads one line from input-stream for each invocation. + Returns nil when all lines have been read." + [input-stream] + (let [line-feed (into #{} (map int [\newline \return])) + buf (buffer 1)] + (loop [acc []] + (let [len (read input-stream buf 1)] + (cond + (and (pos? len) (not (line-feed (first buf)))) + (recur (conj acc (first buf))) + + (and (zero? len) (empty? acc)) nil + + :else (apply str (map char acc))))))) + +(defn line-seq + "Returns the lines of text from input-stream as a lazy sequence of strings. + input-stream must implement IInputStream." + [input-stream] + (when-let [line (read-line input-stream)] + (cons line (lazy-seq (line-seq input-stream))))) + +(deftype FileOutputStream [fp] + IByteOutputStream + (write-byte [this val] + (assert (integer? val) "Value must be a int") + (fputc val fp)) + IOutputStream + (write [this buffer] + (fwrite buffer 1 (count buffer) fp)) + IDisposable + (-dispose! [this] + (fclose fp))) + +(defn file-output-rf [filename] + (let [fp (->FileOutputStream (fopen filename "w"))] + (fn ([] 0) + ([cnt] (dispose! fp) nil) + ([cnt chr] + (assert (integer? chr)) + (let [written (write-byte fp chr)] + (if (= written EOF) + (reduced cnt) + (+ cnt written))))))) + +(defn spit [filename val] + (transduce (map int) + (file-output-rf filename) + (str val))) + +(defn slurp [filename] + (let [c (->FileStream (fopen filename "r")) + result (transduce + (map char) + string-builder + c)] + (dispose! c) + result)) + +(defn slurp-stream [stream] + (let [c stream + result (transduce + (map char) + string-builder + c)] + (dispose! c) + result)) + +(deftype ProcessInputStream [fp] + IInputStream + (read [this buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [read-count (fread buffer 1 len fp)] + (set-buffer-count! buffer read-count) + read-count)) + (read-byte [this] + (fgetc fp)) + IDisposable + (-dispose! [this] + (pclose fp)) + IReduce + (-reduce [this f init] + (common/stream-reducer this f init))) + +(defn popen-read + {:doc "Opens a file for reading. Returns an IInputStream." + :added "0.1"} + [command] + (assert (string? command) "Command must be a string") + (->ProcessInputStream (popen command "r"))) + +(defn run-command [command] + (let [c (->ProcessInputStream (popen command "r")) + result (transduce + (map char) + string-builder + c)] + (dispose! c) + result)) diff --git a/pixie/io.pxi b/pixie/io.pxi new file mode 100644 index 00000000..a6ed1286 --- /dev/null +++ b/pixie/io.pxi @@ -0,0 +1,275 @@ +(ns pixie.io + (:require [pixie.streams :as st :refer :all] + [pixie.streams.utf8 :as utf8] + [pixie.io-blocking :as io-blocking] + [pixie.io.common :as common] + [pixie.uv :as uv] + [pixie.stacklets :as st] + [pixie.ffi :as ffi] + [pixie.ffi-infer :as ffi-infer])) + +(uv/defuvfsfn fs_open [path flags mode] :result) +(uv/defuvfsfn fs_read [file bufs nbufs offset] :result) +(uv/defuvfsfn fs_write [file bufs nbufs offset] :result) +(uv/defuvfsfn fs_close [file] :result) + +(deftype FileStream [fp offset uvbuf] + IInputStream + (read [this buffer len] + (assert (>= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [_ (pixie.ffi/set! uvbuf :base buffer) + _ (pixie.ffi/set! uvbuf :len len) + read-count (fs_read fp uvbuf 1 offset)] + (assert (not (neg? read-count)) "Read Error") + (set-field! this :offset (+ offset read-count)) + (set-buffer-count! buffer read-count) + read-count)) + ISeekableStream + (position [this] + offset) + (rewind [this] + (set-field! this :offset 0)) + (seek [this pos] + (set-field! this :offset pos)) + IDisposable + (-dispose! [this] + (dispose! uvbuf) + (fs_close fp)) + IReduce + (-reduce [this f init] + (common/stream-reducer this f init))) + +(defn open-read + {:doc "Opens a file for reading. Returns an IInputStream." + :added "0.1"} + [filename] + (assert (string? filename) "Filename must be a string") + (->FileStream (fs_open filename uv/O_RDONLY 0) 0 (uv/uv_buf_t))) + +(defprotocol ILineReader + (-read-line [this])) + +(def line-feed? #{\newline \return}) + +(deftype LineReader + [input-stream] + ILineReader + (-read-line [this] + (let [read-fn (if (satisfies? utf8/IUTF8InputStream input-stream) + utf8/read-char + (fn [stream] (when-let [i (read-byte stream)] (char i))))] + (loop [string (string-builder)] + (if-let [ch (read-fn input-stream)] + (if-not (line-feed? ch) + (recur (conj! string ch)) + (str string)))))) + IReduce + (-reduce [this f init] + (loop [acc init] + (if-let [line (-read-line this)] + (let [result (f acc line)] + (if (reduced? result) + @result + (recur result))) + acc)))) + +(defn line-reader + [input-stream] + (cond + (satisfies? ILineReader input-stream) + input-stream + + (satisfies? utf8/IUTF8InputStream input-stream) + (-> input-stream ->LineReader) + + (instance? BufferedInputStream input-stream) + (-> input-stream ->LineReader) + + + (satisfies? IInputStream input-stream) + (-> input-stream (buffered-input-stream 1) ->LineReader) + + :else + (throw [::Exception "Expected a LineReader, UTF8InputStream, or BufferedInputStream"]))) + +(defn read-line + "Reads one line from input-stream for each invocation. + Returns nil when all lines have been read. + Pass a BufferedInputStream for best performance." + [input-stream] + (-read-line (line-reader input-stream))) + +(defn line-seq + "Returns the lines of text from input-stream as a lazy sequence of strings. + input-stream must implement IInputStream." + [input-stream] + (let [lr (line-reader input-stream)] + (when-let [line (-read-line lr)] + (cons line (lazy-seq (line-seq lr)))))) + +(deftype FileOutputStream [fp offset uvbuf] + IOutputStream + (write [this buffer] + (loop [buffer-offset 0] + (let [_ (pixie.ffi/set! uvbuf :base (ffi/ptr-add buffer buffer-offset)) + _ (pixie.ffi/set! uvbuf :len (- (count buffer) buffer-offset)) + write-count (fs_write fp uvbuf 1 offset)] + (when (neg? write-count) + (throw [::FileOutputStreamException (uv/uv_err_name write-count)])) + (set-field! this :offset (+ offset write-count)) + (if (< (+ buffer-offset write-count) (count buffer)) + (recur (+ buffer-offset write-count)) + write-count)))) + IDisposable + (-dispose! [this] + (fs_close fp))) + +(deftype BufferedOutputStream [downstream idx buffer] + IByteOutputStream + (write-byte [this val] + (pixie.ffi/pack! buffer idx CUInt8 val) + (set-field! this :idx (inc idx)) + (when (= idx (buffer-capacity buffer)) + (set-buffer-count! buffer (buffer-capacity buffer)) + (write downstream buffer) + (set-field! this :idx 0))) + IDisposable + (-dispose! [this] + (set-buffer-count! buffer idx) + (flush this)) + IFlushableStream + (flush [this] + (set-buffer-count! buffer idx) + (set-field! this :idx 0) + (write downstream buffer) + (when (satisfies? IFlushableStream downstream) + (flush downstream)))) + +(deftype BufferedInputStream [upstream idx buffer] + IReduce + (-reduce [this f init] + (loop [acc init] + (if-let [next-byte (read-byte this)] + (let [result (f acc next-byte)] + (if (reduced? result) + @result + (recur result))) + acc))) + IByteInputStream + (read-byte [this] + (when (= idx (count buffer)) + (set-field! this :idx 0) + (read upstream buffer (buffer-capacity buffer))) + (when-not (empty? buffer) + (let [val (nth buffer idx)] + (set-field! this :idx (inc idx)) + val))) + ISeekableStream + (position [this] + (+ (- (position upstream) + (count buffer)) + idx)) + (rewind [this] + (seek this 0)) + (seek [this pos] + ;; We can be clever about seeking. If we are seeking to somewhere with in + ;; our current buffer, we can avoid seeking in upstream. + (let [upper-bounds (position upstream) + lower-bounds (- upper-bounds (count buffer))] + (if (and (>= pos lower-bounds) + (<= pos upper-bounds)) + ;; We're in the buffer window :-) + (set-field! this :idx (- pos lower-bounds)) + ;; Put the index at the end of the buffer to force a read from upstream + (do + (set-field! this :idx (count buffer)) + (seek upstream pos))))) + IDisposable + (-dispose! [this] + (dispose! buffer))) + +(defn buffered-output-stream + ([downstream] + (buffered-output-stream downstream common/DEFAULT-BUFFER-SIZE)) + ([downstream size] + (->BufferedOutputStream downstream 0 (buffer size)))) + +(defn buffered-input-stream + ([upstream] + (buffered-input-stream upstream common/DEFAULT-BUFFER-SIZE)) + ([upstream size] + (if (satisfies? IInputStream upstream) + (let [b (buffer size)] + (set-buffer-count! b size) + (->BufferedInputStream upstream size b)) + (throw [::Exception "Expected upstream to satisfy IInputStream"])))) + +(defn throw-on-error [result] + (when (neg? result) + (throw [::UVException (uv/uv_err_name result)])) + result) + +(defn open-write + {:doc "Opens a file for writing. Returns an IOutputStream." + :added "0.1"} + [filename] + (assert (string? filename) "Filename must be a string") + (->FileOutputStream (throw-on-error (fs_open filename + (bit-or uv/O_WRONLY uv/O_CREAT) + uv/S_IRWXU)) + 0 + (uv/uv_buf_t))) + +(defn spit + "Writes the content to output. Output must be a file or an IOutputStream." + [output content] + (cond + (string? output) + (transduce (map identity) + (-> output + open-write + buffered-output-stream + utf8/utf8-output-stream-rf) + (str content)) + + (satisfies? IOutputStream output) + (transduce (map identity) + (-> output + buffered-output-stream + utf8/utf8-output-stream-rf) + (str content)) + + :else (throw [::Exception "Expected a string or IOutputStream"]))) + +(defn slurp + "Reads in the contents of input. Input must be a filename or an IInputStream." + [input] + (let [stream (cond + (string? input) (open-read input) + (satisfies? IInputStream input) input + :else (throw [:pixie.io/Exception "Expected a string or an IInputStream"])) + result (transduce + (map identity) + string-builder + (-> stream + buffered-input-stream + utf8/utf8-input-stream))] + (dispose! stream) + result)) + +(defn run-command [command] + (st/apply-blocking io-blocking/run-command command)) + + +(comment + (st/apply-blocking println "FROM OTHER THREAD <---!!!!!") + + + (tcp-server "0.0.0.0" 4242 nil)) +(comment + (defmacro make-readline-async [] + `(let [libname ~(ffi-infer/compile-library {:prefix "pixie.io.readline" + :includes ["uv.h" "editline/readline.h"]})])) + + (ffi-infer/compile-library)) diff --git a/pixie/io/common.pxi b/pixie/io/common.pxi new file mode 100644 index 00000000..75a3e685 --- /dev/null +++ b/pixie/io/common.pxi @@ -0,0 +1,17 @@ +(ns pixie.io.common + "Common functionality for handling IO" + (:require [pixie.streams :refer :all])) + +(def DEFAULT-BUFFER-SIZE 1024) + +(defn stream-reducer [this f init] + (let [buf (buffer DEFAULT-BUFFER-SIZE) + rrf (preserving-reduced f)] + (loop [acc init] + (let [read-count (read this buf DEFAULT-BUFFER-SIZE)] + (if (> read-count 0) + (let [result (reduce rrf acc buf)] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) diff --git a/pixie/io/tcp.pxi b/pixie/io/tcp.pxi new file mode 100644 index 00000000..0233d00a --- /dev/null +++ b/pixie/io/tcp.pxi @@ -0,0 +1,87 @@ +(ns pixie.io.tcp + (:require [pixie.stacklets :as st] + [pixie.streams :refer [IInputStream read IOutputStream write]] + [pixie.io.common :as common] + [pixie.uv :as uv] + [pixie.io.uv-common :as uv-common] + [pixie.ffi :as ffi])) + +(defrecord TCPServer [ip port on-connect uv-server bind-addr on-connection-cb] + IDisposable + (-dispose! [this] + (uv/uv_close uv-server st/close_cb) + (dispose! @on-connection-cb) + (dispose! bind-addr))) + +(deftype TCPStream [uv-client uv-write-buf] + IInputStream + (read [this buffer len] + (uv-common/cb-stream-reader uv-client buffer len)) + IOutputStream + (write [this buffer] + (uv-common/cb-stream-writer uv-client uv-write-buf buffer)) + IDisposable + (-dispose! [this] + (dispose! uv-write-buf) + (uv/uv_close uv-client st/close_cb)) + IReduce + (-reduce [this f init] + (common/stream-reducer this f init))) + +(defn launch-tcp-client-from-server [svr] + (assert (instance? TCPServer svr) "Requires a TCPServer as the first argument") + (let [client (uv/uv_tcp_t)] + (uv/uv_tcp_init (uv/uv_default_loop) client) + (if (= 0 (uv/uv_accept (:uv-server svr) client)) + (do (st/spawn-from-non-stacklet #((:on-connect svr) + (->TCPStream client (uv/uv_buf_t)))) + svr) + (do (uv/uv_close client nil) + svr)))) + +(defn tcp-server + "Creates a TCP server on the given ip (as a string) and port (as an integer). Returns a TCPServer that can be + shutdown with dispose!. on-connection is a function that will be passed a TCPStream for each connecting client." + [ip port on-connection] + (assert (string? ip) "Ip should be a string") + (assert (integer? port) "Port should be a int") + + (let [server (uv/uv_tcp_t) + bind-addr (uv/sockaddr_in) + _ (uv/throw-on-error (uv/uv_ip4_addr ip port bind-addr)) + on-new-connection (atom nil) + tcp-server (->TCPServer ip port on-connection server bind-addr on-new-connection)] + (reset! on-new-connection + (ffi/ffi-prep-callback + uv/uv_connection_cb + (fn [server status] + (launch-tcp-client-from-server tcp-server)))) + (uv/uv_tcp_init (uv/uv_default_loop) server) + (uv/uv_tcp_bind server bind-addr 0) + (uv/throw-on-error (uv/uv_listen server 128 @on-new-connection)) + (st/yield-control) + tcp-server)) + + +(defn tcp-client + "Creates a TCP connection to the given ip (as a string) and port (an integer). Returns a TCPStream." + [ip port] + (let [client-addr (uv/sockaddr_in) + uv-connect (uv/uv_connect_t) + client (uv/uv_tcp_t) + cb (atom nil)] + (uv/throw-on-error (uv/uv_ip4_addr ip port client-addr)) + (uv/uv_tcp_init (uv/uv_default_loop) client) + (st/call-cc (fn [k] + (reset! cb (ffi/ffi-prep-callback + uv/uv_connect_cb + (fn [_ status] + (try + (dispose! @cb) + (dispose! uv-connect) + (dispose! client-addr) + (st/run-and-process k (or (st/exception-on-uv-error status) + (->TCPStream client (uv/uv_buf_t)))) + (catch ex + (println ex)))))) + (uv/uv_tcp_connect uv-connect client client-addr @cb))))) diff --git a/pixie/io/tty.pxi b/pixie/io/tty.pxi new file mode 100644 index 00000000..be8c0185 --- /dev/null +++ b/pixie/io/tty.pxi @@ -0,0 +1,44 @@ +(ns pixie.io.tty + (:require [pixie.stacklets :as st] + [pixie.streams :refer [IInputStream read IOutputStream write]] + [pixie.uv :as uv] + [pixie.io.common :as common] + [pixie.io.uv-common :as uv-common] + [pixie.system :as sys])) + +(deftype TTYInputStream [uv-client] + IInputStream + (read [this buf len] + (uv-common/cb-stream-reader uv-client buf len)) + IDisposable + (-dispose! [this] + (uv/uv_close uv-client st/close_cb)) + IReduce + (-reduce [this f init] + (common/stream-reducer this f init))) + +(deftype TTYOutputStream [uv-client uv-write-buf] + IOutputStream + (write [this buffer] + (uv-common/cb-stream-writer uv-client uv-write-buf buffer)) + IDisposable + (-dispose! [this] + (dispose! uv-write-buf) + (uv/uv_close uv-client st/close_cb))) + +(defn tty-output-stream [fd] + ;(assert (= uv/UV_TTY (uv/uv_guess_handle fd)) "fd is not a TTY") + (let [buf (uv/uv_buf_t) + tty (uv/uv_tty_t)] + (uv/uv_tty_init (uv/uv_default_loop) tty fd 0) + (->TTYOutputStream tty buf))) + +(defn tty-input-stream [fd] + ;(assert (= uv/UV_TTY (uv/uv_guess_handle fd)) "fd is not a TTY") + (let [tty (uv/uv_tty_t)] + (uv/uv_tty_init (uv/uv_default_loop) tty fd 0) + (->TTYInputStream tty))) + +(def stdin (tty-input-stream sys/stdin)) +(def stdout (tty-output-stream sys/stdout)) +(def stderr (tty-output-stream sys/stderr)) diff --git a/pixie/io/uv-common.pxi b/pixie/io/uv-common.pxi new file mode 100644 index 00000000..c8f10ee2 --- /dev/null +++ b/pixie/io/uv-common.pxi @@ -0,0 +1,45 @@ +(ns pixie.io.uv-common + (:require [pixie.stacklets :as st] + [pixie.uv :as uv] + [pixie.ffi :as ffi])) + +(defn cb-stream-reader [uv-client buffer len] + (assert (<= (buffer-capacity buffer) len) + "Not enough capacity in the buffer") + (let [alloc-cb (uv/-prep-uv-buffer-fn buffer len) + read-cb (atom nil)] + (st/call-cc (fn [k] + (reset! read-cb (ffi/ffi-prep-callback + uv/uv_read_cb + (fn [stream nread uv-buf] + (set-buffer-count! buffer nread) + (try + (dispose! alloc-cb) + (dispose! @read-cb) + ;(dispose! uv-buf) + (uv/uv_read_stop stream) + (st/run-and-process k (or + (st/exception-on-uv-error nread) + nread)) + (catch ex + (println ex)))))) + (uv/uv_read_start uv-client alloc-cb @read-cb))))) + +(defn cb-stream-writer + [uv-client uv-write-buf buffer] + (let [write-cb (atom nil) + uv_write (uv/uv_write_t)] + (ffi/set! uv-write-buf :base buffer) + (ffi/set! uv-write-buf :len (count buffer)) + (st/call-cc + (fn [k] + (reset! write-cb (ffi/ffi-prep-callback + uv/uv_write_cb + (fn [req status] + (try + (dispose! @write-cb) + ;(uv/uv_close uv_write st/close_cb) + (st/run-and-process k status) + (catch ex + (println ex)))))) + (uv/uv_write uv_write uv-client uv-write-buf 1 @write-cb))))) diff --git a/pixie/lib_pixie.py b/pixie/lib_pixie.py new file mode 100644 index 00000000..4bf67b59 --- /dev/null +++ b/pixie/lib_pixie.py @@ -0,0 +1,15 @@ +import ctypes +import os.path + +dll_name = os.path.join(os.path.dirname(__file__), "libpixie-vm.dylib") +print("Loading Pixie from " + dll_name) +dll = ctypes.CDLL(dll_name) + +dll.rpython_startup_code() +dll.pixie_init("/Users/tim/oss/pixie/pixie-vm".encode("ascii")) + + +def repl(): + dll.pixie_execute_source("(ns user (:require [pixie.repl :as repl])) (pixie.repl/repl)".encode('ascii')) + +#repl() \ No newline at end of file diff --git a/pixie/math.pxi b/pixie/math.pxi new file mode 100644 index 00000000..253bc132 --- /dev/null +++ b/pixie/math.pxi @@ -0,0 +1,101 @@ +(ns pixie.math + (:require [pixie.ffi-infer :as i] + [pixie.string :as s])) + +(i/with-config {:library "m" + :cxx-flags ["-lm"] + :includes ["math.h"]} + (i/defconst M_E) + (i/defconst M_LOG2E) + (i/defconst M_LOG10E) + (i/defconst M_LN2) + (i/defconst M_LN10) + (i/defconst M_PI) + (i/defconst M_PI_2) + (i/defconst M_PI_4) + (i/defconst M_1_PI) + (i/defconst M_2_PI) + (i/defconst M_2_SQRTPI) + (i/defconst M_SQRT2) + (i/defconst M_SQRT1_2) + + (i/defcfn nan) + (i/defcfn ceil) + (i/defcfn floor) + (i/defcfn nearbyint) + (i/defcfn rint) + (i/defcfn lround) + (i/defcfn llrint) + (i/defcfn llround) + (i/defcfn trunc) + + (i/defcfn fmod) + (i/defcfn remainder) + (i/defcfn remquo) + + (i/defcfn fdim) + (i/defcfn fmax) + (i/defcfn fmin) + + (i/defcfn fma) + + (i/defcfn fabs) + (i/defcfn sqrt) + (i/defcfn cbrt) + (i/defcfn hypot) + + (i/defcfn exp) + (i/defcfn exp2) + (if-not (s/starts-with? pixie.platform/name "darwin") + (i/defcfn exp10)) + (i/defcfn expm1) + + (i/defcfn log) + (i/defcfn log2) + (i/defcfn log10) + (i/defcfn log1p) + + (i/defcfn logb) + (i/defcfn ilogb) + + (i/defcfn modf) + (i/defcfn frexp) + + (i/defcfn ldexp) + (i/defcfn scalbn) + (i/defcfn scalbln) + + (i/defcfn pow) + + (i/defcfn cos) + (i/defcfn sin) + (i/defcfn tan) + + (i/defcfn cosh) + (i/defcfn sinh) + (i/defcfn tanh) + + (i/defcfn acos) + (i/defcfn asin) + (i/defcfn atan) + (i/defcfn atan2) + + (i/defcfn acosh) + (i/defcfn asinh) + (i/defcfn atanh) + + (i/defcfn tgamma) + (i/defcfn lgamma) + + (i/defcfn j0) + (i/defcfn j1) + (i/defcfn jn) + (i/defcfn y0) + (i/defcfn y1) + (i/defcfn yn) + + (i/defcfn erf) + (i/defcfn erfc)) + +(if (s/starts-with? pixie.platform/name "darwin") + (defn exp10 [x] (pow 10.0 x))) diff --git a/pixie/parser.pxi b/pixie/parser.pxi new file mode 100644 index 00000000..79e18f6e --- /dev/null +++ b/pixie/parser.pxi @@ -0,0 +1,312 @@ +(ns pixie.parser + (:require [pixie.stdlib :as s])) + + +;; This file contans a small framework for writing generic parsers in Pixie. Although the generated +;; code is probably not the fastest, it is fairly simple, and that simplicity should open the road for +;; future optimizations. The parsers allowed support multiple inheritance and multiple input data types. +;; Backtracking is supported by snapshots taken at key parts of the parsing process. For a string parser +;; these snapshots are simply a integer index into the string being parsed. + +;; Cursors + +(defprotocol ICursor + (next! [this] "Advance to the next element") + (current [this] "Return the current element") + (snapshot [this] "Return a snapshot of the cursor's mutable state") + (rewind! [this snapshot] "Rewind the cursor to a previous snapshot") + (at-end? [this] "Is there more to parse?")) + +(deftype StringCursor [idx s] + ICursor + (next! [this] + (set-field! this :idx (inc idx))) + (current [this] + (when (< idx (count s)) + (nth s idx))) + (snapshot [this] + idx) + (rewind! [this val] + (set-field! this :idx val)) + (at-end? [this] + (= idx (count s)))) + +;; Create a cursor from the given string +(defn string-cursor [s] + (->StringCursor 0 s)) + +;; Mechanics + +(deftype ParseFailure []) + +;; If a parser returns this value, parsing has failed +(def fail (->ParseFailure)) + +(defn failure? + "Returns true if return value from a parser is a parse failure" + [v] + (identical? v fail)) + +(defn parse-if + "Parse and return the current value of the cursor if this predicate succeeds against the cursor. Advances + the cursor to the next element." + [pred] + (fn [cursor] + (if (pred (current cursor)) + (let [value (current cursor)] + (next! cursor) + value) + fail))) + + +(defprotocol IParserGenerator + (to-parser [this] "Convert the current object to a parser")) + +(extend-protocol IParserGenerator + IFn + (to-parser [this] + this) + Character + (to-parser [this] + (parse-if #(= % this))) + String + (to-parser [this] + (println "bad-parser " this) + (assert false))) + + + + +(defn or + "Defines a parser that succeeds if one of the provided parsers succeeds. Parsers are tried in-order." + ([a] a) + ([a b] + (let [a (to-parser a) + b (to-parser b) + m (atom #{})] + (fn [cursor] + (let [key [cursor (snapshot cursor)]] + (if-let [v (contains? @m key)] + (b cursor) + (let [_ (swap! m conj key) + state (snapshot cursor) + val (a cursor)] + (swap! m disj key) + (if (identical? val fail) + (do (rewind! cursor state) + (b cursor)) + val))))))) + ([a b & more] + (apply or (or a b) more))) + +(defn add-clauses [cursor-sym body [[sym goal] & more]] + (if sym + `(let [~sym (~sym ~cursor-sym)] + (if (identical? ~sym fail) + fail + ~(add-clauses cursor-sym body more))) + body)) + +(defn -parse-args + [args] + (loop [args args + rules [] + return nil] + (let [[arg & rest] args] + (assert (not (= '-> arg)) "invalid position for ->") + (if arg + (if (= '<- arg) + (let [return (first rest)] + (recur (next rest) + rules + return)) + (if (= (first rest) '->) + (let [binding (-> rest next first) + rest (-> rest next next)] + (recur rest + (conj rules [binding arg]) + return)) + (recur rest + (conj rules [(gensym "_") arg]) + return))) + [rules return])))) + +(defmacro and + "Defines a parser that succeeds only if all parsers succeed. Tried in order. Each parser clause can be followed + by a -> to give the parser's output a name. There may also be a single <- followed by any Pixie code that can be used + to post-process the parsed output." + [& args] + (let [[parsed body] (-parse-args args) + cursor-sym (gensym "cursor")] + `(let [~@(mapcat + (fn [[sym parser]] + [sym `(to-parser ~parser)]) + parsed)] + (fn [~cursor-sym] + (let [prev-pos# (snapshot ~cursor-sym) + result# ~(add-clauses cursor-sym body parsed)] + (if (identical? result# fail) + (do (rewind! ~cursor-sym prev-pos#) + fail) + result#)))))) + + +(defprotocol IDeliverable + (-deliver [this val])) + +(deftype PromiseFn [f name] + IDeliverable + (-deliver [this val] + (set-field! this :f val)) + IFn + (-invoke [this val] + (assert f (str "PromiseFN " name " has not been delivered")) + (f val))) + +(defn promise-fn + "Defines a promise that is callable." + [name] + (->PromiseFn nil name)) + +(defn -parse-parser-args [args] + (loop [rules {} + args args] + (if args + (let [name (first args) + body (second args) + args (-> args next next)] + (assert (symbol? name) "Must name all rules") + (if (= '<- (first args)) + (let [return (first (next args)) + full-rule `(let [p# (to-parser ~body)] + (fn [cursor#] + (let [~'value (p# cursor#)] + (if (failure? ~'value) + ~'value + ~return))))] + (recur (assoc rules name full-rule) + (next (next args)))) + (recur (assoc rules name body) + args))) + rules))) + +(defmacro parser + "(parser inherits & rules) + Creates a parser that inherits from zero or more other parsers defined in `inherits`. Rules are pairs + of names and rules that will be assigned to those names. Names are inherited from parent parsers in the + order they are defined." + [inherits & rules] + (let [parted (apply merge + (conj (mapv (fn [sym] + (::forms (deref (resolve-in *ns* sym)))) + inherits) + (-parse-parser-args rules))) + rules (apply concat parted) + syms (keys parted)] + `(let [~@(mapcat (fn [s] + `[~s (promise-fn (quote ~s))]) + syms)] + ~@(map (fn [[s goal]] + `(-deliver ~s ~goal)) + parted) + ~(assoc (zipmap (map (comp keyword name) syms) + syms) + ::forms (list 'quote (apply hashmap rules)))))) + +(defmacro defparser + "(defparser nm inherits rules) + Same as parser but assigns the resulting parser to a var with the name nm" + [nm inherits & rules] + `(def ~nm (parser ~inherits ~@rules))) + +;; Common parsers + +(defn char-range + "Defines a parser that parses a numerical range of characters" + [from to] + (parse-if (fn [v] + (when (char? v) + (<= (int from) (int v) (int to)))))) + +(defn one+ + "Defines a parser that succeeds if the given parser succeeds once or more. Will return a vector, but any + reducing function can be provided via rf as well." + ([g] + (one+ g conj)) + ([g rf] + (let [g (to-parser g)] + (fn [cursor] + (loop [acc (rf) + cnt 0] + (let [prev-pos (snapshot cursor) + v (g cursor)] + (if (identical? v fail) + (if (= 0 cnt) + (do (rewind! cursor prev-pos) + fail) + (rf acc)) + (recur (rf acc v) + (inc cnt))))))))) + +(def one+chars #(one+ % string-builder)) + +(defn zero+ + "Defines a parser that succeeds if a given parser succeeds zero or more times. Will return a vector, but + any reducing function can be provided via rf as well." + ([g] + (zero+ g conj)) + ([g rf] + (let [g (to-parser g)] + (fn [cursor] + (loop [acc (rf)] + (let [v (g cursor)] + (if (identical? v fail) + (rf acc) + (recur (rf acc v))))))))) + +(def zero+chars #(zero+ % string-builder)) + +(defn eat + "Eagerly parses as many values as possible until g fails. Discards the result, returns nil." + [g] + (fn [cursor] + (loop [] + (let [prev-pos (snapshot cursor) + v (g cursor)] + (if (identical? v fail) + (do (rewind! cursor prev-pos) + nil) + (recur)))))) + +(defn maybe + "Always succeeds, returns nil when the input did not match the parser." + ([g] + (maybe g nil)) + ([g default] + (let [g (to-parser g)] + (fn [cursor] + (let [v (g cursor)] + (if (failure? v) + default + v)))))) + +(defmacro sequence + [coll arrow body] + (assert (= '<- arrow) "Middle argument to sequence must be a return arrow") + `(and ~@coll ~'<- ~body)) + +(defn end + "A parser that only succeeds if there is no more input left to process." + [cursor] + (if (at-end? cursor) + nil + fail)) + +(defn one-of + "Defines a parser that succeeds if the value being parsed is found in v" + [v] + (parse-if (partial contains? v))) + +(def digits (parse-if (set "1234567890"))) + +(def whitespace (parse-if #{\newline \return \space \tab})) diff --git a/pixie/parser/json.pxi b/pixie/parser/json.pxi new file mode 100644 index 00000000..321d71c1 --- /dev/null +++ b/pixie/parser/json.pxi @@ -0,0 +1,111 @@ +(ns pixie.parser.json + (:require [pixie.parser :refer :all] + [pixie.stdlib :as std])) + + + +;; Basic numeric parser. Supports integers (1, 2, 43), decimals (0.1, 1.1, 1000.11) and exponents (1e42, 1E-2) +(defparser NumberParser [] + NUMBER (and (maybe \-) + -> sign + + (or (and + (parse-if (set "123456789")) -> first + (zero+chars digits) -> rest + <- (str first rest)) + (and \0 <- "0")) + -> integer-digits + + (maybe (and \. + (one+chars digits) -> digits + <- digits)) + -> fraction-digits + + + (maybe (and (parse-if (set "eE")) + (maybe (parse-if (set "-+"))) -> exp-sign + (one+chars digits) -> exp-digits + <- [(std/or exp-sign "") exp-digits])) + -> exp-data + + <- (std/read-string (str (std/or sign "") + integer-digits + (if fraction-digits (str "." fraction-digits) "") + (if exp-data (apply str "E" exp-data) ""))))) + +(def valid-escape-chars + {\\ \\ + \" \" + \/ \/ + \b \backspace + \f \formfeed + \n \newline + \r \return + \t \tab}) + + +;; Defines a JSON escaped string parser. Supports all the normal \n \f \r stuff as well +;; as \uXXXX unicode characters +(defparser EscapedStringParser [] + CHAR (or (and \\ + (one-of valid-escape-chars) -> char + <- (valid-escape-chars char)) + + (and \\ + \u + digits -> d1 + digits -> d2 + digits -> d3 + digits -> d4 + <- (char (std/read-string (str "0x" d1 d2 d3 d4)))) + + (parse-if #(not= % \"))) + + STRING (and \" + (zero+chars CHAR) -> s + \" + <- s)) + +;; Basic JSON parser +(defparser JSONParser [NumberParser EscapedStringParser] + + NULL (sequence "null" <- nil) + TRUE (sequence "true" <- true) + FALSE (sequence "false" <- false) + ARRAY (and \[ + (eat whitespace) + (zero+ (and ENTRY -> e + (maybe \,) + <- e)) -> items + (eat whitespace) + (eat whitespace) + \] + <- items) + MAP-ENTRY (and (eat whitespace) + STRING -> key + (eat whitespace) + \: + ENTRY -> value + (maybe \,) + <- [key value]) + MAP (and \{ + (zero+ MAP-ENTRY) -> items + (eat whitespace) + \} + <- (apply hashmap (apply concat items))) + ENTRY (and + (eat whitespace) + (or NUMBER MAP STRING NULL TRUE FALSE ARRAY) -> val + (eat whitespace) + <- val) + ENTRY-AT-END (and ENTRY -> e + (eat whitespace) + end + <- e)) + +(defn read-string [s] + (let [c (string-cursor s) + result ((:ENTRY-AT-END JSONParser) c)] + (if (failure? result) + (println (current c) (snapshot c)) + result))) diff --git a/pixie/repl.pxi b/pixie/repl.pxi new file mode 100644 index 00000000..6b21c862 --- /dev/null +++ b/pixie/repl.pxi @@ -0,0 +1,33 @@ +(ns pixie.repl + (:require [pixie.stacklets :as st] + [pixie.io :as io] + [pixie.ffi-infer :as f])) + +(f/with-config {:library "edit" + :includes ["editline/readline.h"]} + (f/defcfn readline) + (f/defcfn add_history)) + + +(defn repl [] + (let [rdr (reader-fn (fn [] + (let [prompt (if (= 0 pixie.stdlib/*reading-form*) + (str (name pixie.stdlib/*ns*) " => ") + "") + line (st/apply-blocking readline prompt)] + (if line + (do + (add_history line) + (str line "\n")) + ""))))] + (loop [] + (try (let [form (read rdr false)] + (if (or (= form eof) (= form :exit)) + (exit 0) + (let [x (eval form)] + (pixie.stdlib/-push-history x) + (prn x)))) + (catch ex + (pixie.stdlib/-set-*e ex) + (println "ERROR: \n" ex))) + (recur)))) diff --git a/pixie/set.pxi b/pixie/set.pxi new file mode 100644 index 00000000..23c4b425 --- /dev/null +++ b/pixie/set.pxi @@ -0,0 +1,78 @@ +(ns pixie.set + (:require [pixie.stdlib :as std])) + +(defn- -must-be-set [s] + (if (set? s) + s + (throw [:pixie.stdlib/InvalidArgumentException + (str "Not a set: " s)]))) + +(defn- -must-be-sets [& sets] + (doseq [set sets] + (-must-be-set set))) + +(defn- -union [s t] + (-must-be-sets s t) + (if (< (count s) (count t)) + (into t s) + (into s t))) + +(defn union + "Returns a set that is the union of the input sets." + ([] #{}) + ([s] (-must-be-set s)) + ([s t] + (-union s t)) + ([s t & sets] + (reduce -union (-union s t) sets))) + +(defn- -difference [s t] + (-must-be-sets s t) + (into #{} (filter #(not (contains? t %))) s)) + +(defn difference + "Returns a set that is the difference of the input sets." + ([] #{}) + ([s] (-must-be-set s)) + ([s t] + (-difference s t)) + ([s t & sets] + (reduce -difference (-difference s t) sets))) + +(defn- -intersection [s t] + (-must-be-set s) + (-must-be-set t) + (into #{} (filter #(contains? s %)) t)) + +(defn intersection + "Returns a set that is the intersection of the input sets." + ([] #{}) + ([s] (-must-be-set s)) + ([s t] + (-intersection s t)) + ([s t & sets] + (reduce -intersection (-intersection s t) sets))) + +(defn subset? + "Returns true if the first set is a subset of the second." + [s t] + (-must-be-sets s t) + (and (<= (count s) (count t)) + (every? #(contains? t %) (vec s)))) + +(defn strict-subset? + "Returns true if the first set is a subset of the second." + [s t] + (-must-be-sets s t) + (and (< (count s) (count t)) + (subset? s t))) + +(defn superset? + "Returns true if the first set is a superset of the second." + [s t] + (subset? t s)) + +(defn strict-superset? + "Returns true if the first set is a strict superset of the second." + [s t] + (strict-subset? t s)) diff --git a/pixie/stacklets.pxi b/pixie/stacklets.pxi new file mode 100644 index 00000000..267c7530 --- /dev/null +++ b/pixie/stacklets.pxi @@ -0,0 +1,197 @@ +(ns pixie.stacklets + (:require [pixie.uv :as uv] + [pixie.ffi :as ffi])) + +;; If we don't do this, compiling this file doesn't work since the def will clear out +;; the existing value. + +(when (undefined? (var stacklet-loop-h)) + (def stacklet-loop-h (atom nil)) + (def running-threads (atom 0)) + (def main-loop-running? (atom false)) + (def main-loop-lock (-create-lock)) + (-acquire-lock main-loop-lock true)) + + + +;; Yield + +(defrecord ThrowException [ex]) + +(defn run-and-process + ([k] + (run-and-process k nil)) + ([k val] + (let [[h f] (k val)] + (f h)))) + +(defn exception-on-uv-error [result] + (when (neg? result) + (->ThrowException (str "UV Error: " (uv/uv_err_name result))))) + + +(defn call-cc [f] + (let [frames (-get-current-var-frames nil) + [h val] (@stacklet-loop-h f)] + (reset! stacklet-loop-h h) + (-set-current-var-frames nil frames) + (if (instance? ThrowException val) + (throw [::Exception (:ex val)]) + val))) + +(defn -run-later [f] + (let [a (uv/uv_async_t) + cb (atom nil)] + (reset! cb (ffi/ffi-prep-callback uv/uv_async_cb + (fn [handle] + (try + (uv/uv_close a close_cb) + (-dispose! @cb) + (f) + (catch ex (println ex)))))) + (uv/uv_async_init (uv/uv_default_loop) a @cb) + (uv/uv_async_send a) + (when (not @main-loop-running?) + (reset! main-loop-running? true) + (-release-lock main-loop-lock)) + nil)) + + +(defn yield-control [] + (call-cc (fn [k] + (-run-later (partial run-and-process k))))) + +(def close_cb (ffi/ffi-prep-callback uv/uv_close_cb + pixie.ffi/dispose!)) + +;;; Sleep +(defn sleep + ([ms] + (sleep ms false)) + ([ms background?] + (let [f (fn [k] + (let [cb (atom nil) + timer (uv/uv_timer_t)] + (reset! cb (ffi/ffi-prep-callback uv/uv_timer_cb + (fn [handle] + (try + (run-and-process k) + (uv/uv_timer_stop timer) + (-dispose! @cb) + (catch ex + (println ex)))))) + (uv/uv_timer_init (uv/uv_default_loop) timer) + (when background? + (uv/uv_unref timer)) + (uv/uv_timer_start timer @cb ms 0)))] + (call-cc f)))) + +;; Spawn +(defn -spawn [start-fn] + (call-cc (fn [k] + (-run-later (fn [] + (run-and-process (new-stacklet start-fn)))) + (-run-later (partial run-and-process k))))) + +(defmacro spawn [& body] + `(let [frames (-get-current-var-frames nil)] + (-spawn (fn [h# _] + (-set-current-var-frames nil frames) + (try + (swap! running-threads inc) + (reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (swap! running-threads dec) + (call-cc (fn [_] nil))) + (catch e + (println e))))))) + +(defmacro spawn-background [& body] + `(let [frames (-get-current-var-frames nil)] + (-spawn (fn [h# _] + (-set-current-var-frames nil frames) + (try + (reset! stacklet-loop-h h#) + (let [result# (do ~@body)] + (call-cc (fn [_] nil))) + (catch e + (println e))))))) + + +(defn spawn-from-non-stacklet [f] + (let [s (new-stacklet (fn [h _] + (try + (reset! stacklet-loop-h h) + (swap! running-threads inc) + (f) + (swap! running-threads dec) + (call-cc (fn [_] nil)) + (catch e + (println e)))))] + (-run-later + (fn [] + (run-and-process s))))) + + +(defn finalizer-loop [] + (spawn-background + (loop [] + (-run-finalizers) + (sleep 1000 true) + (recur)))) + + +(defn -with-stacklets [fn] + (swap! running-threads inc) + (reset! main-loop-running? true) + (let [[h f] ((new-stacklet fn) nil)] + (f h) + (loop [] + (uv/uv_run (uv/uv_default_loop) uv/UV_RUN_DEFAULT) + (when (> @running-threads 0) + (reset! main-loop-running? false) + (-acquire-lock main-loop-lock true) + (recur))))) + +(defmacro with-stacklets [& body] + `(-with-stacklets + (fn [h# _] + (try + (reset! stacklet-loop-h h#) + (finalizer-loop) + (let [result# (do ~@body)] + (swap! running-threads dec) + (call-cc (fn [_] nil))) + (catch e + (println e)))))) + + + +(defn run-with-stacklets [f] + (with-stacklets + (f))) + +(defprotocol IThreadPool + (-execute [this work-fn])) + +;; Super basic Thread Pool, yes, this should be improved + +(deftype ThreadPool [] + IThreadPool + (-execute [this work-fn] + (-thread (fn [] (work-fn))))) + +(def basic-thread-pool (->ThreadPool)) + +(defn -run-in-other-thread [work-fn] + (-execute basic-thread-pool work-fn)) + + +(defn apply-blocking [f & args] + (call-cc (fn [k] + (-run-in-other-thread + (fn [] + (let [result (apply f args)] + (-run-later (fn [] (run-and-process k result))))))))) + + diff --git a/pixie/stdlib.lisp b/pixie/stdlib.lisp deleted file mode 100644 index 3bb90944..00000000 --- a/pixie/stdlib.lisp +++ /dev/null @@ -1,407 +0,0 @@ -(__ns__ pixie.stdlib) - -(def reset! -reset!) - -(def map (fn map [f] - (fn [xf] - (fn - ([] (xf)) - ([result] (xf result)) - ([result item] (xf result (f item))))))) - -(def conj (fn conj - ([] []) - ([result] result) - ([result item] (-conj result item)))) - -(def conj! (fn conj! - ([] (-transient [])) - ([result] (-persistent! result)) - ([result item] (-conj! result item)))) - - -(def transduce (fn transduce - ([f coll] - (let [result (-reduce coll f (f))] - (f result))) - - ([xform rf coll] - (let [f (xform rf) - result (-reduce coll f (f))] - (f result))) - ([xform rf init coll] - (let [f (xform rf) - result (-reduce coll f init)] - (f result))))) - -(def reduce (fn [rf init col] - (-reduce col rf init))) - - -(def interpose - (fn interpose [val] - (fn [xf] - (let [first? (atom true)] - (fn - ([] (xf)) - ([result] (xf result)) - ([result item] (if @first? - (do (reset! first? false) - (xf result item)) - (xf (xf result val) item)))))))) - - -(def preserving-reduced - (fn [rf] - (fn [a b] - (let [ret (rf a b)] - (if (reduced? ret) - (reduced ret) - ret))))) - -(def cat - (fn cat [rf] - (let [rrf (preserving-reduced rf)] - (fn cat-inner - ([] (rf)) - ([result] (rf result)) - ([result input] - (reduce rrf result input)))))) - - -(def seq-reduce (fn seq-reduce - [coll f init] - (loop [init init - coll (seq coll)] - (if (reduced? init) - @init - (if (seq coll) - (recur (f init (first coll)) - (seq (next coll))) - init))))) - -(def indexed-reduce (fn indexed-reduce - [coll f init] - (let [max (count coll)] - (loop [init init - i 0] - (if (reduced? init) - @init - (if (-eq i max) - init - (recur (f init (nth coll i)) (+ i 1)))))))) - - -(extend -reduce Cons seq-reduce) -(extend -reduce PersistentList seq-reduce) -(extend -reduce LazySeq seq-reduce) - -(comment (extend -reduce Array indexed-reduce)) - -(extend -str Bool - (fn [x] - (if (identical? x true) - "true" - "false"))) - -(extend -str Nil (fn [x] "nil")) -(extend -reduce Nil (fn [self f init] init)) -(extend -hash Nil (fn [self] 100000)) - -(extend -hash Integer hash-int) - -(extend -eq Integer -num-eq) - -(def ordered-hash-reducing-fn - (fn ordered-hash-reducing-fn - ([] (new-hash-state)) - ([state] (finish-hash-state state)) - ([state itm] (update-hash-ordered! state itm)))) - -(def unordered-hash-reducing-fn - (fn unordered-hash-reducing-fn - ([] (new-hash-state)) - ([state] (finish-hash-state state)) - ([state itm] (update-hash-unordered! state itm)))) - - -(extend -str PersistentVector - (fn [v] - (apply str "[" (conj (transduce (interpose ", ") conj v) "]")))) - - - - -(extend -str Cons - (fn [v] - (apply str "(" (conj (transduce (interpose ", ") conj v) ")")))) - -(extend -hash Cons - (fn [v] - (transduce ordered-hash-reducing-fn v))) - -(extend -str PersistentList - (fn [v] - (apply str "(" (conj (transduce (interpose ", ") conj v) ")")))) - -(extend -str LazySeq - (fn [v] - (apply str "(" (conj (transduce (interpose ", ") conj v) ")")))) - -(extend -hash PersistentVector - (fn [v] - (transduce ordered-hash-reducing-fn v))) - - -(def stacklet->lazy-seq - (fn [f] - (let [val (f nil)] - (if (identical? val :end) - nil - (cons val (lazy-seq* (fn [] (stacklet->lazy-seq f)))))))) - -(def sequence - (fn - ([data] - (let [f (create-stacklet - (fn [h] - (reduce (fn ([h item] (h item) h)) h data) - (h :end)))] - (stacklet->lazy-seq f))) - ([xform data] - (let [f (create-stacklet - (fn [h] - (transduce xform - (fn ([] h) - ([h item] (h item) h) - ([h] (h :end))) - data)))] - (stacklet->lazy-seq f))))) - -(extend -seq PersistentVector sequence) -(extend -seq Array sequence) - - - -(def concat (fn [& args] (transduce cat conj args))) - -(def defn (fn [nm & rest] `(def ~nm (fn ~nm ~@rest)))) -(set-macro! defn) - -(defn defmacro [nm & rest] - `(do (defn ~nm ~@rest) - (set-macro! ~nm) - ~nm)) - -(set-macro! defmacro) - -(defn + - [& args] - (reduce -add 0 args)) - - -(defn - - ([] 0) - ([x] (-sub 0 x)) - ([x & args] - (reduce -sub x args))) - -(defn = - ([x] true) - ([x y] (eq x y)) - ([x y & rest] (if (eq x y) - (apply = y rest) - false))) - -(def inc (fn [x] (+ x 1))) - -(def dec (fn [x] (- x 1))) - - -(defn assoc - ([m] m) - ([m k v] - (-assoc m k v)) - ([m k v & rest] - (apply assoc (-assoc m k v) rest))) - -(def slot-tp (create-type :slot [:val])) - -(defn ->Slot [x] - (let [inst (new slot-tp)] - (set-field! inst :val x))) - -(defn get-val [inst] - (get-field inst :val)) - -(defn comp - ([f] f) - ([f1 f2] - (fn [& args] - (f1 (apply f2 args)))) - ([f1 f2 f3] - (fn [& args] - (f1 (f2 (apply f3 args)))))) - - -(defn not [x] - (if x false true)) - -(defmacro cond - ([] nil) - ([test then & clauses] - `(if ~test - ~then - (cond ~@clauses)))) - - -(defmacro try [& body] - (loop [catch nil - catch-sym nil - body-items [] - finally nil - body (seq body)] - (let [form (first body)] - (if form - (if (not (seq? form)) - (recur catch catch-sym (conj body-items form) finally (next body)) - (let [head (first form)] - (cond - (= head 'catch) (if catch - (throw "Can only have one catch clause per try") - (recur (next (next form)) (first (next form)) body-items finally (next body))) - (= head 'finally) (if finally - (throw "Can only have one finally clause per try") - (recur catch catch-sym body-items (next form) (next body))) - :else (recur catch catch-sym (conj body-items form) finally (next body))))) - `(-try-catch - (fn [] ~@body-items) - ~(if catch - `(fn [~catch-sym] ~@catch) - `(fn [] nil)) - - (fn [] ~@finally)))))) - -(defn . - ([obj sym] - (get-field obj sym)) - ([obj sym & args] - (apply (get-field obj sym) args))) - - -(extend -count MapEntry (fn [self] 2)) -(extend -nth MapEntry (fn [self idx not-found] - (cond (= idx 0) (-key self) - (= idx 1) (-val self) - :else not-found))) - -(defn key [x] - (-key x)) - -(defn val [x] - (-val x)) - -(extend -reduce MapEntry indexed-reduce) - -(extend -str MapEntry - (fn [v] - (apply str "[" (conj (transduce (interpose ", ") conj v) "]")))) - -(extend -hash MapEntry - (fn [v] - (transduce ordered-hash-reducing-fn v))) - -(extend -str PersistentHashMap - (fn [v] - (apply str "{" (conj (transduce (comp cat (interpose " ")) conj v) "}")))) - -(extend -hash PersistentHashMap - (fn [v] - (transduce cat unordered-hash-reducing-fn v))) - -(extend -hash Symbol - (fn [v] - (hash [(namespace v) (name v)]))) - -(extend -hash Keyword - (fn [v] - (hash [(namespace v) (name v)]))) - -(extend -str Keyword - (fn [k] - (if (namespace k) - (str ":" (namespace k) "/" (name k)) - (str ":" (name k))))) - -(defn get - ([mp k] - (get mp k nil)) - ([mp k not-found] - (-val-at mp k not-found))) - -(defmacro assert - ([test] - `(if ~test - nil - (throw "Assert failed"))) - ([test msg] - `(if ~test - nil - (throw (str "Assert failed " ~msg))))) - -(defmacro resolve [sym] - `(resolve-in (this-ns-name) ~sym)) - -(defmacro with-bindings [binds & body] - `(do (push-binding-frame!) - (reduce (fn [_ map-entry] - (set! (resolve (key map-entry)) (val map-entry))) - nil - (apply hashmap ~binds)) - (let [ret (do ~@body)] - (pop-binding-frame!) - ret))) - -(def foo 42) -(set-dynamic! (resolve 'pixie.stdlib/foo)) - -(defmacro require [ns kw as-nm] - (assert (= kw :as) "Require expects :as as the second argument") - `(do (load-file (quote ~ns)) - (refer (this-ns-name) (the-ns (quote ~ns)) (quote ~as-nm)))) - -(defmacro ns [nm & body] - `(do (__ns__ ~nm) - ~@body)) - - -(defn symbol? [x] - (identical? Symbol (type x))) - - - - -(defmacro deftype [nm fields & body] - (let [ctor-name (symbol (str "->" (name nm))) - type-decl `(def ~nm (create-type ~(keyword (name nm)) ~fields)) - field-syms (transduce (map (comp symbol name)) conj fields) - inst (gensym) - ctor `(defn ~ctor-name ~field-syms - (let [~inst (new ~nm)] - ~@(transduce - (map (fn [field] - `(set-field! ~inst ~field ~(symbol (name field))))) - conj - fields) - ~inst)) - proto-bodies (transduce - (map (fn [body] - (cond - (symbol? body) `(satisfy ~body ~nm) - (seq? body) `(extend ~(first body) ~nm (fn ~@body)) - :else (assert false "Unknown body element in deftype, expected symbol or seq")))) - conj - body)] - `(do ~type-decl - ~ctor - ~@proto-bodies))) diff --git a/pixie/stdlib.pxi b/pixie/stdlib.pxi new file mode 100644 index 00000000..2501e2fe --- /dev/null +++ b/pixie/stdlib.pxi @@ -0,0 +1,3125 @@ +(in-ns :pixie.stdlib) + +(def libc (ffi-library pixie.platform/lib-c-name)) + + +(def exit (ffi-fn libc "exit" [CInt] CInt)) +(def puts (ffi-fn libc "puts" [CCharP] CInt)) + + +(def sh (ffi-fn libc "system" [CCharP] CInt)) +(def printf (ffi-fn libc "printf" [CCharP] CInt :variadic? true)) +(def getenv (ffi-fn libc "getenv" [CCharP] CCharP)) + +(def libedit (ffi-library (str "libedit." pixie.platform/so-ext))) +(def readline (ffi-fn libedit "readline" [CCharP] CCharP)) +(def rand (ffi-fn libc "rand" [] CInt)) +(def srand (ffi-fn libc "srand" [CInt] CInt)) +(def fopen (ffi-fn libc "fopen" [CCharP CCharP] CVoidP)) +(def fread (ffi-fn libc "fread" [CVoidP CInt CInt CVoidP] CInt)) +(def mkdtemp (ffi-fn libc "mkdtemp" [CCharP] CCharP)) +(def mkdir (ffi-fn libc "mkdir" [CCharP] CCharP)) +(def rmdir (ffi-fn libc "rmdir" [CCharP] CCharP)) +(def rm (ffi-fn libc "remove" [CCharP] CCharP)) + +(def libm (ffi-library (str "libm." pixie.platform/so-ext))) +(def atan2 (ffi-fn libm "atan2" [CDouble CDouble] CDouble)) +(def lround (ffi-fn libm "lround" [CDouble] CInt)) + + +(def reset! -reset!) + +(def program-arguments []) + +(def fn (fn* [& args] + (cons 'fn* args))) +(set-macro! fn) + +(def let (fn* [& args] + (cons 'let* args))) +(set-macro! let) + +(def loop (fn* [& args] + (cons 'loop* args))) +(set-macro! loop) + +(def identity + (fn ^{:doc "The identity function. Returns its argument." + :added "0.1"} + identity + [x] + x)) + +(def conj + (fn ^{:doc "Adds elements to the collection. Elements are added to the end except in the case of Cons lists." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + conj + ([] []) + ([coll] coll) + ([coll item] (-conj coll item)) + ([coll item & args] + (reduce -conj (-conj coll item) args)))) + +(def conj! + (fn ^{:doc "Adds elements to the transient collection. Elements are added to the end except in the case of Cons lists." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + conj! + ([] (-transient [])) + ([coll] (-persistent! coll)) + ([coll item] (-conj! coll item)) + ([coll item & args] + (reduce -conj! (-conj! coll item) args)))) + +(def disj (fn ^{:doc "Removes elements from the collection." + :signatures [[] [coll] [coll item]] + :added "0.1"} + disj + ([] []) + ([coll] coll) + ([coll item] (-disj coll item)) + ([coll item & items] + (reduce -disj (-disj coll item) items)))) + +(comment (def disj! (fn ^{:doc "Removes elements from the transient collection." + :signatures [[] [coll] [coll item] [coll item & items]] + :added "0.1"} + disj! + ([] (-transient [])) + ([result] (-persistent! result)) + ([result item] (-disj! result item)) + ([coll item & items] + (reduce -disj! (-disj! coll item) items))))) + +(def pop + (fn ^{:doc "Pops elements off a stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + pop + ([coll] (-pop coll)))) + +(def push + (fn ^{:doc "Pushes an element on to a stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + push + ([coll x] (-push coll x)))) + +(def pop! + (fn ^{:doc "Pops elements off a transient stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + pop! + ([coll] (-pop! coll)))) + +(def push! + (fn ^{:doc "Pushes an element on to a transient stack." + :signatures [[] [coll] [coll item] [coll item & args]] + :added "0.1"} + push! + ([coll x] (-push! coll x)))) + +(def transient (fn [coll] (-transient coll))) + +(def persistent! (fn [coll] (-persistent! coll))) + +(def transduce (fn transduce + ([f coll] + (let [result (-reduce coll f (f))] + (f result))) + + ([xform rf coll] + (let [f (xform rf) + result (-reduce coll f (f))] + (f result))) + ([xform rf init coll] + (let [f (xform rf) + result (-reduce coll f init)] + (f result))))) + +(def map (fn ^{:doc "Creates a transducer that applies f to every input element." + :signatures [[f] [f coll]] + :added "0.1"} + map + ([f] + (fn [xf] + (fn + ([] (xf)) + ([result] (xf result)) + ([result item] (xf result (f item)))))) + ([f coll] + (lazy-seq* + (fn [] + (let [s (seq coll)] + (if s + (cons (f (first s)) + (map f (rest s))) + nil))))) + ([f & colls] + (let [step (fn step [cs] + (lazy-seq* + (fn [] + (let [ss (map seq cs)] + (if (every? identity ss) + (cons (map first ss) (step (map rest ss))) + nil)))))] + (map (fn [args] (apply f args)) (step colls)))))) + +(def reduce (fn + ([rf col] + (reduce rf (rf) col)) + ([rf init col] + (-reduce col rf init)))) + +(def instance? (fn ^{:doc "Checks if x is an instance of t. + + When t is seqable, checks if x is an instance of + any of the types contained therein." + :signatures [[t x]]} + instance? [t x] + (if (-satisfies? ISeqable t) + (let [ts (seq t)] + (if (not ts) false + (if (-instance? (first ts) x) + true + (instance? (rest ts) x)))) + (-instance? t x)))) + +(def satisfies? (fn ^{:doc "Checks if x satisfies the protocol p. + + When p is seqable, checks if x satisfies all of + the protocols contained therein." + :signatures [[t x]]} + satisfies? [p x] + (if (-satisfies? ISeqable p) + (let [ps (seq p)] + (if (not ps) true + (if (not (-satisfies? (first ps) x)) + false + (satisfies? (rest ps) x)))) + (-satisfies? p x)))) + +(def into (fn ^{:doc "Add the elements of `from` to the collection `to`." + :signatures [[to from] [to xform from]] + :added "0.1"} + ([to from] + (if (satisfies? IToTransient to) + (persistent! (reduce conj! (transient to) from)) + (reduce conj to from))) + ([to xform from] + (if (satisfies? IToTransient to) + (transduce xform conj! (transient to) from) + (transduce xform conj to from))))) + +(def interpose + (fn ^{:doc "Returns a transducer that inserts `val` in between elements of a collection." + :signatures [[val] [val coll]] + :added "0.1"} + interpose + ([val] (fn [xf] + (let [first? (atom true)] + (fn + ([] (xf)) + ([result] (xf result)) + ([result item] (if @first? + (do (reset! first? false) + (xf result item)) + (xf (xf result val) item))))))) + ([val coll] + (transduce (interpose val) conj coll)))) + +(def preserving-reduced + (fn preserving-reduced [rf] + (fn pr-inner [a b] + (let [ret (rf a b)] + (if (reduced? ret) + (reduced ret) + ret))))) + +(def cat + (fn ^{:doc "A transducer that concatenates elements of a collection." + :added "0.1"} + cat + [rf] + (let [rrf (preserving-reduced rf)] + (fn cat-inner + ([] (rf)) + ([result] (rf result)) + ([result input] + (reduce rrf result input)))))) + +(def mapcat + (fn ^{:doc "Maps f over the elements of coll and concatenates the result." + :added "0.1"} + mapcat + ([f] + (comp (map f) cat)) + ([f coll] + (transduce (mapcat f) conj coll)))) + +(def seq-reduce (fn seq-reduce + [coll f init] + (loop [init init + coll (seq coll)] + (if (reduced? init) + @init + (if (seq coll) + (recur (f init (first coll)) + (seq (next coll))) + init))))) + +(def indexed-reduce + (fn indexed-reduce + [coll f init] + (let [max (count coll)] + (loop [init init + i 0] + (if (reduced? init) + @init + (if (-eq i max) + init + (recur (f init (nth coll i nil)) (+ i 1)))))))) + +(def rest (fn ^{:doc "Returns the elements after the first element, or () if there are no more elements." + :signatures [[coll]] + :added "0.1"} + [coll] + (let [next (next coll)] + (if next next '())))) + +;; Make all Function types extend IFn +(extend -invoke Code -invoke) +(extend -invoke NativeFn -invoke) +(extend -invoke VariadicCode -invoke) +(extend -invoke MultiArityFn -invoke) +(extend -invoke Closure -invoke) +(extend -invoke Var -invoke) +(extend -invoke PolymorphicFn -invoke) +(extend -invoke DoublePolymorphicFn -invoke) + +(extend -reduce Cons seq-reduce) +(extend -reduce PersistentList seq-reduce) +(extend -reduce LazySeq seq-reduce) + + +(comment (extend -reduce Array indexed-reduce)) + +(extend -reduce Buffer indexed-reduce) +(extend -reduce String indexed-reduce) + +(extend -str Bool + (fn [x] + (if (identical? x true) + "true" + "false"))) +(extend -repr Bool -str) + +(extend -str Nil (fn [x] "nil")) +(extend -repr Nil -str) +(extend -reduce Nil (fn [self f init] init)) +(extend -hash Nil (fn [self] 100000)) +(extend -with-meta Nil (fn [self _] nil)) +(extend -deref Nil (fn [_] nil)) +(extend -contains-key Nil (fn [_ _] false)) + +(extend -hash Integer hash-int) + +(extend -eq Integer -num-eq) +(extend -eq Float -num-eq) +(extend -eq Ratio -num-eq) + +(def ordered-hash-reducing-fn + (fn ordered-hash-reducing-fn + ([] (new-hash-state)) + ([state] (finish-hash-state state)) + ([state itm] (update-hash-ordered! state itm)))) + +(def unordered-hash-reducing-fn + (fn unordered-hash-reducing-fn + ([] (new-hash-state)) + ([state] (finish-hash-state state)) + ([state itm] (update-hash-unordered! state itm)))) + +(def string-builder + (fn ^{:doc "Creates a reducing function that builds a string based on calling str on the transduced collection."} + ([] (-string-builder)) + ([sb] (str sb)) + ([sb item] (conj! sb item)))) + +(extend -str PersistentVector + (fn [v] + (str "[" (transduce (interpose " ") string-builder v) "]"))) +(extend -repr PersistentVector + (fn [v] + (str "[" (transduce (comp (map -repr) (interpose " ")) string-builder v) "]"))) + +(extend -str Cons + (fn [v] + (str "(" (transduce (interpose " ") string-builder v) ")"))) +(extend -repr Cons + (fn [v] + (str "(" (transduce (comp (map -repr) (interpose " ")) string-builder v) ")"))) + +(extend -hash Cons + (fn [v] + (transduce ordered-hash-reducing-fn v))) + +(extend -str PersistentList + (fn [v] + (str "(" (transduce (interpose " ") string-builder v) ")"))) +(extend -repr PersistentList + (fn [v] + (str "(" (transduce (comp (map -repr) (interpose " ")) string-builder v) ")"))) + +(extend -hash PersistentList + (fn [v] + (transduce ordered-hash-reducing-fn v))) + + +(extend -str LazySeq + (fn [v] + (str "(" (transduce (interpose " ") string-builder v) ")"))) +(extend -repr LazySeq + (fn [v] + (str "(" (transduce (comp (map -repr) (interpose " ")) string-builder v) ")"))) + +(extend -hash PersistentVector + (fn [v] + (transduce ordered-hash-reducing-fn v))) + +(extend -hash PersistentHashSet + (fn [v] + (transduce ordered-hash-reducing-fn v))) + + + +(add-marshall-handlers PersistentHashSet + (fn [obj] (vec obj)) + (fn [obj] (apply hash-set obj))) + +(extend -hash PersistentHashMap + (fn [v] + (transduce ordered-hash-reducing-fn v))) + + + +(extend -hash EmptyList (fn [v] 5555555)) + +(extend -hash Bool + (fn [v] + (if v + 1111111 + 3333333))) + +(def = -eq) + +(extend -seq PersistentVector + (fn vector-seq + ([self] + (vector-seq self 0)) + ([self x] + (if (= x (count self)) + nil + (cons (nth self x) (lazy-seq* (fn [] (vector-seq self (+ x 1))))))))) + +(extend -seq String + (fn string-seq + ([self] + (string-seq self 0)) + ([self x] + (if (= x (count self)) + nil + (cons (nth self x) (lazy-seq* (fn [] (string-seq self (+ x 1))))))))) + +(def concat + (fn ^{:doc "Concatenates its arguments." + :signatures [[& args]] + :added "0.1"} + concat + [& args] (transduce cat conj args))) + +(def key (fn [x] (-key x))) +(def val (fn [x] (-val x))) + +(def defn (fn ^{:doc "Defines a new function." + :signatures [[nm doc? meta? & body]]} + defn + [nm & rest] + (let [meta (if (meta nm) (meta nm) {}) + meta (if (instance? String (first rest)) + (assoc meta :doc (first rest)) + meta) + rest (if (instance? String (first rest)) (next rest) rest) + meta (if (satisfies? IMap (first rest)) + (merge meta (first rest)) + meta) + rest (if (satisfies? IMap (first rest)) (next rest) rest) + meta (if (-contains-key meta :signatures) + meta + (merge meta {:signatures + (if (satisfies? IVector (first rest)) + [(first rest)] + (transduce (map first) conj rest))})) + nm (with-meta nm meta)] + `(def ~nm (fn ~nm ~@rest))))) +(set-macro! defn) + +(defn defmacro + {:doc "Defines a new macro." + :added "0.1"} + [nm & rest] + `(do (defn ~nm ~@rest) + (set-macro! ~nm) + ~nm)) +(set-macro! defmacro) + +(defmacro defn- + {:doc "Define a new non-public function. Otherwise the same as defn." + :signatures [[nm doc? meta? & body]] + :added "0.1"} + [nm & rest] + (let [nm (with-meta nm (assoc (meta nm) :private true))] + (cons `defn (cons nm rest)))) + +(defmacro letfn + "fnspec ==> (fname [params*] exprs) or (fname ([params*] exprs)+) + + Takes a vector of function specs and a body, and generates a set of + bindings of functions to their names. All of the names are available + in all of the definitions of the functions, as well as the body." + {:added "0.2" + :forms '[(letfn [fnspecs*] exprs*)]} + [fnspecs & body] + `(letfn* ~(vec (interleave (map first fnspecs) + (map #(cons `fn %) fnspecs))) + ~@body)) + +(defn not + {:doc "Inverts the input, if a truthy value is supplied, returns false, otherwise +returns true." + :signatures [[x]] + :added "0.1"} + [x] + (if x false true)) + +(defn + + {:doc "Adds the arguments, returning 0 if no arguments." + :signatures [[& args]] + :added "0.1"} + ([] 0) + ([x] x) + ([x y] (-add x y)) + ([x y & args] + (reduce -add (-add x y) args))) + +(defn - + ([] 0) + ([x] (-sub 0 x)) + ([x y] (-sub x y)) + ([x y & args] + (reduce -sub (-sub x y) args))) + +(defn * + ([] 1) + ([x] x) + ([x y] (-mul x y)) + ([x y & args] + (reduce -mul (-mul x y) args))) + +(defn unchecked-add + {:doc "Adds the arguments, returning 0 if no arguments." + :signatures [[& args]] + :added "0.1"} + ([] 0) + ([x] x) + ([x y] (-unchecked-add x y)) + ([x y & args] + (reduce -unchecked-add (-unchecked-add x y) args))) + +(defn unchecked-subtract + ([] 0) + ([x] (-unchecked-subtract 0 x)) + ([x y] (-unchecked-subtract x y)) + ([x y & args] + (reduce -unchecked-subtract (-unchecked-subtract x y) args))) + +(defn unchecked-multiply + ([] 1) + ([x] x) + ([x y] (-unchecked-multiply x y)) + ([x y & args] + (reduce -unchecked-multiply (-unchecked-multiply x y) args))) + +(defn / + ([x] (-div 1 x)) + ([x y] (-div x y)) + ([x y & args] + (reduce -div (-div x y) args))) + +(defn quot [num div] + (-quot num div)) + +(defn rem [num div] + (-rem num div)) + +(defn rand-int + {:doc "Returns a random integer between 0 (inclusive) and n (exclusive)."} + [n] + (if (zero? n) + 0 + (rem (rand) n))) + +(defn = + {:doc "Returns true if all the arguments are equivalent. Otherwise, returns false. Uses +-eq to perform equality checks." + :signatures [[& args]] + :added "0.1"} + ([x] true) + ([x y] (eq x y)) + ([x y & rest] (if (eq x y) + (apply = y rest) + false))) + +(defn not= + {:doc "Returns true if one (or more) of the arguments are not equivalent to the others. Uses +-eq to perform equality checks." + :signatures [[& args]] + :added "0.1"} + ([x] false) + ([x y] (not (eq x y))) + ([x y & rest] (not (apply = x y rest)))) + +(defn < + ([x] true) + ([x y] (-lt x y)) + ([x y & rest] (if (-lt x y) + (apply < y rest) + false))) + +(defn > + ([x] true) + ([x y] (-gt x y)) + ([x y & rest] (if (-gt x y) + (apply > y rest) + false))) + +(defn <= + ([x] true) + ([x y] (-lte x y)) + ([x y & rest] (if (-lte x y) + (apply <= y rest) + false))) + +(defn >= + ([x] true) + ([x y] (-gte x y)) + ([x y & rest] (if (-gte x y) + (apply >= y rest) + false))) + +(defn pos? + {:doc "Returns true if x is greater than zero." + :signatures [[x]] + :added "0.1"} + [x] + (> x 0)) + +(defn neg? + {:doc "Returns true if x is less than zero." + :signatures [[x]] + :added "0.1"} + [x] + (< x 0)) + +(defn zero? + {:doc "Returns true if x is equal to zero." + :signatures [[x]] + :added "0.1"} + [x] + (= x 0)) + +(defn inc + {:doc "Increments x by one." + :signatures [[x]] + :added "0.1"} + [x] + (+ x 1)) + +(defn dec + {:doc "Decrements x by one." + :signatures [[x]] + :added "0.1"} + [x] + (- x 1)) + +(defn unchecked-inc + {:doc "Increments x by one." + :signatures [[x]] + :added "0.1"} + [x] + (unchecked-add x 1)) + +(defn unchecked-dec + {:doc "Decrements x by one." + :signatures [[x]] + :added "0.1"} + [x] + (unchecked-subtract x 1)) + +(defn empty? + {:doc "Returns true if the collection has no items, otherwise false." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? ICounted coll) + (zero? (count coll)) + (not (seq coll)))) + +(defn not-empty? + {:doc "Returns true if the collection has items, otherwise false." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (seq coll) true false)) + +(defn even? + {:doc "Returns true if n is even." + :signatures [[n]] + :added "0.1"} + [n] + (zero? (rem n 2))) + +(defn odd? + {:doc "Returns true of n is odd." + :signatures [[n]] + :added "0.1"} + [n] + (= (rem n 2) 1)) + +(defn nth + {:doc "Returns the element at the idx. If the index is not found it will return an error. + However, if you specify a not-found parameter, it will substitute that instead." + :signatures [[coll idx] [coll idx not-found]] + :added "0.1"} + ([coll idx] (-nth coll idx)) + ([coll idx not-found] (-nth-not-found coll idx not-found))) + +(defn first + {:doc "Returns the first item in coll, if coll implements IIndexed nth will be used to retrieve + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 0 nil) + (-first coll))) + +(defn second + {:doc "Returns the second item in coll, if coll implements IIndexed nth will be used to retrieve + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 1 nil) + (first (next coll)))) + +(defn third + {:doc "Returns the third item in coll, if coll implements IIndexed nth will be used to retrieve + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 2 nil) + (first (next (next coll))))) + +(defn fourth + {:doc "Returns the fourth item in coll, if coll implements IIndexed nth will be used to retrieve + the item from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (if (satisfies? IIndexed coll) + (nth coll 3 nil) + (first (next (next (next coll)))))) + +(defn assoc + {:doc "Associates the key with the value in the collection." + :signatures [[m] [m k v] [m k v & kvs]] + :added "0.1"} + ([m] m) + ([m k v] + (-assoc m k v)) + ([m k v & rest] + (apply assoc (-assoc m k v) rest))) + +(defn dissoc + {:doc "Removes the value associated with the keys from the collection." + :signatures [[m] [m & ks]] + :added "0.1"} + ([m] m) + ([m & ks] + (reduce -dissoc m ks))) + +(defn contains? + {:doc "Checks if there is a value associated with key in the collection. + +Does *not* check for the presence of a value in the collection, only whether +there's a value associated with the key. Use `some` for checking for values." + :examples [["(contains? {:a 1} :a)" nil true] + ["(contains? {:a 1} :b)" nil false] + ["(contains? #{:a :b :c} :a)" nil true] + ["(contains? [:a :b :c] 0)" nil true] + ["(contains? [:a :b :c] 4)" nil false] + ["(contains? [:a :b :c] :a)" nil false]] + :signatures [[coll key]] + :added "0.1"} + [coll key] + (-contains-key coll key)) + +(defn hash-set [& args] + {:doc "Creates a hash-set from the arguments of the function." + :added "0.1"} + (set args)) + +(defn vec + {:doc "Converts a reducable collection into a vector using the (optional) transducer." + :signatures [[coll] [xform coll]] + :added "0.1"} + ([coll] + (transduce conj! coll)) + ([xform coll] + (transduce xform conj! coll))) + +(defn get-val [inst] + (get-field inst :val)) + +(defn comp + {:doc "Composes the given functions, applying the last function first." + :examples [["((comp inc first) [41 2 3])" nil 42]] + :signatures [[f] [f & fs]] + :added "0.1"} + ([] identity) + ([f] f) + ([f1 f2] + (fn [& args] + (f1 (apply f2 args)))) + ([f1 f2 f3] + (fn [& args] + (f1 (f2 (apply f3 args))))) + ([f1 f2 f3 & fs] + (fn [& args] + (apply (transduce comp (apply list f1 f2 f3 fs)) args)))) + +(defmacro cond + {:doc "Checks if any of the tests is truthy, if so, stops and returns the value of the corresponding body." + :signatures [[] [test then & clauses]] + :added "0.1"} + ([] nil) + ([test then & clauses] + `(if ~test + ~then + (cond ~@clauses)))) + +(defmacro try [& body] + (loop [catch nil + catch-sym nil + body-items [] + finally nil + body (seq body)] + (let [form (first body)] + (if form + (if (not (seq? form)) + (recur catch catch-sym (conj body-items form) finally (next body)) + (let [head (first form)] + (cond + (= head 'catch) (if catch + (throw [:pixie.stdlib/SyntaxException + "Can only have one catch clause per try"]) + (recur (next (next form)) (first (next form)) body-items finally (next body))) + (= head 'finally) (if finally + (throw [:pixie.stdlib/SyntaxException + "Can only have one finally clause per try"]) + (recur catch catch-sym body-items (next form) (next body))) + :else (recur catch catch-sym (conj body-items form) finally (next body))))) + (let [catch-data (cond + (keyword? catch-sym) (let [sym (first catch)] + (assert (symbol? sym) (str "Invalid catch binding form" + catch)) + [`[(if (= ~catch-sym (ex-data ~sym)) + (do ~@(next catch)) + (throw ~sym))] + sym]) + (seq? catch-sym) (let [sym (first catch)] + (assert (symbol? sym) (str "Invalid catch binding form" + catch)) + [`[(if ~catch-sym + (do ~@(next catch)) + (throw ~sym))] + sym]) + :else [catch catch-sym])] + `(-try-catch + (fn [] ~@body-items) + ~(if catch + `(fn [~(nth catch-data 1)] ~@(nth catch-data 0)) + `(fn [] nil)) + + (fn [] ~@finally))))))) + +#_(defn . + {:doc "Access the field named by the symbol. + +If further arguments are passed, invokes the method named by symbol, passing the object and arguments." + :signatures [[obj sym] [obj sym & args]] + :added "0.1"} + ([obj sym] + (get-field obj sym)) + ([obj sym & args] + (apply (get-field obj sym) obj args))) + +(defn true? [v] (identical? v true)) +(defn false? [v] (identical? v false)) + +(defn number? [v] (instance? Number v)) +(defn integer? [v] (instance? Integer v)) +(defn float? [v] (instance? Float v)) +(defn ratio? [v] (instance? Ratio v)) + +(defn char? [v] (instance? Character v)) +(defn string? [v] (instance? String v)) +(defn keyword? [v] (instance? Keyword v)) + +(defn list? [v] (instance? [PersistentList Cons] v)) +(defn set? [v] (instance? PersistentHashSet v)) +(defn map? [v] (satisfies? IMap v)) +(defn fn? [v] (satisfies? IFn v)) +(defn coll? [v] (satisfies? IPersistentCollection v)) + +(defn indexed? [v] (satisfies? IIndexed v)) +(defn counted? [v] (satisfies? ICounted v)) + +(defn map-entry? [v] (satisfies? IMapEntry v)) + +(defn last + {:doc "Returns the last element of the collection, or nil if none." + :signatures [[coll]] + :added "0.1"} + [coll] + (cond + (satisfies? IIndexed coll) + (when (pos? (count coll)) + (nth coll (dec (count coll)))) + + (satisfies? ISeq coll) + (if (next coll) + (recur (next coll)) + (first coll)) + + (satisfies? ISeqable coll) + (recur (seq coll)))) + +(defn butlast + {:doc "Returns all elements but the last from the collection." + :signatures [[coll]] + :added "0.1"} + [coll] + (loop [res [] + coll coll] + (if (next coll) + (recur (conj res (first coll)) (next coll)) + (seq res)))) + +(defn complement + {:doc "Given a function, returns a new function which takes the same arguments + but returns the opposite truth value."} + [f] + (assert (fn? f) "Complement must be passed a function") + (fn + ([] (not (f))) + ([x] (not (f x))) + ([x y] (not (f x y))) + ([x y & more] (not (apply f x y more))))) + +(defn constantly [x] + {:doc "Returns a function that always returns x, no matter what it is called with." + :examples [["(let [f (constantly :me)] [(f 1) (f \"foo\") (f :abc) (f nil)])" + nil [:me :me :me :me]]]} + (fn [& _] x)) + +(extend -count MapEntry (fn [self] 2)) +(extend -nth MapEntry (fn map-entry-nth [self idx] + (cond (= idx 0) (-key self) + (= idx 1) (-val self)))) +(extend -nth-not-found MapEntry (fn map-entry-nth [self idx not-found] + (cond (= idx 0) (-key self) + (= idx 1) (-val self) + :else not-found))) +(extend -eq MapEntry (fn [self other] + (cond + (map-entry? other) (and (= (-key self) + (-key other)) + (= (-val self) + (-val other))) + (vector? other) (= [(-key self) (-val self)] + other) + :else (= (seq self) other)))) + +(extend -reduce MapEntry indexed-reduce) + +(extend -str MapEntry + (fn [v] + (str "[" (transduce (interpose " ") string-builder v) "]"))) +(extend -repr MapEntry + (fn [v] + (str "[" (transduce (comp (map -repr) (interpose " ")) string-builder v) "]"))) + +(extend -hash MapEntry + (fn [v] + (transduce ordered-hash-reducing-fn v))) + +(extend -seq MapEntry + (fn [self] + (list (-key self) (-val self)))) + +(defn select-keys + {:doc "Returns a map containing only the entries of `m` where the entry's key is in `key-seq`."} + [m key-seq] + (with-meta + (transduce + (comp (filter (fn [k] (contains? m k))) + (map (fn [k] [k (get m k)]))) + conj {} key-seq) + (meta m))) + +(defn keys + {:doc "If called with no arguments, returns a transducer that will extract the key from each map entry. If passed + a collection, will assume that it is a hashmap and return a vector of all keys from the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (map key)) + ([m] + (transduce (map key) conj! m))) + +(defn vals + {:doc "If called with no arguments, returns a transducer that will extract the key from each map entry. If passed + a collection, will assume that it is a hashmap and return a vector of all keys from the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (map val)) + ([m] + (transduce (map val) conj! m))) + +(extend -seq PersistentHashMap + (fn [m] + (reduce conj nil m))) + +(extend -str PersistentHashMap + (fn [v] + (let [entry->str (map (fn [e] (vector (key e) " " (val e))))] + (str "{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) +(extend -repr PersistentHashMap + (fn [v] + (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] + (str "{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) + +(extend -hash PersistentHashMap + (fn [v] + (transduce cat unordered-hash-reducing-fn v))) + +(extend -seq PersistentHashSet + (fn [s] + (reduce conj nil s))) + +(extend -str PersistentHashSet + (fn [s] + (str "#{" (transduce (interpose " ") string-builder s) "}"))) +(extend -repr PersistentHashSet + (fn [s] + (str "#{" (transduce (comp (map -repr) (interpose " ")) string-builder s) "}"))) + +(extend -empty Cons (fn [_] '())) +(extend -empty LazySeq (fn [_] '())) +(extend -empty PersistentList (fn [_] '())) +(extend -empty EmptyList (fn [_] '())) +(extend -empty PersistentVector (fn [_] [])) +(extend -empty Array (fn [_] (make-array 0))) +(extend -empty PersistentHashMap (fn [_] {})) +(extend -empty PersistentHashSet (fn [_] #{})) + +(extend -conj PersistentHashMap + (fn [coll x] + (cond + (instance? MapEntry x) + (assoc coll (key x) (val x)) + + (instance? PersistentVector x) + (if (= (count x) 2) + (assoc coll (nth x 0 nil) (nth x 1 nil)) + (throw + [:pixie.stdlib/InvalidArgumentException + "Vector arg to map conj must be a pair"])) + + (satisfies? ISeqable x) + (reduce conj coll (-seq x)) + + :else + (throw + [:pixie.stdlib/InvalidArgumentException + (str (type x) " cannot be conjed to a map")])))) + +(extend -conj Cons + (fn [coll x] + (cons x coll))) + +(extend -conj LazySeq + (fn [coll x] + (cons x coll))) + +(defn empty + {:doc "Returns an empty collection of the same type, or nil." + :added "0.1"} + [coll] + (if (satisfies? IEmpty coll) + (-empty coll) + nil)) + +(extend -str Keyword + (fn [k] + (if (namespace k) + (str ":" (namespace k) "/" (name k)) + (str ":" (name k))))) +(extend -repr Keyword -str) + +(extend -repr Symbol -str) + +(defn get + {:doc "Gets an element from a collection implementing ILookup. Returns nil or the default value if not found." + :added "0.1"} + ([mp k] + (get mp k nil)) + ([mp k not-found] + (-val-at mp k not-found))) + +(extend -invoke Keyword (fn + ([k m not-found] + (-val-at m k not-found)) + ([k m] + (-val-at m k nil)))) +(extend -invoke PersistentHashMap get) +(extend -invoke PersistentHashSet get) + + +(defn get-in + {:doc "Gets a value from a nested collection at the \"path\" given by the keys." + :examples [["(get-in {:a [{:b 42}]} [:a 0 :b])" nil 42]] + :signatures [[m ks] [m ks not-found]] + :added "0.1"} + ([m ks] + (reduce get m ks)) + ([m ks not-found] + (loop [sentinel 'x + m m + ks (seq ks)] + (if ks + (let [m (get m (first ks) sentinel)] + (if (identical? sentinel m) + not-found + (recur sentinel m (next ks)))) + m)))) + +(defn assoc-in + {:doc "Associates a value in a nested collection given by the path. + +Creates new maps if the keys are not present." + :examples [["(assoc-in {} [:a :b :c] 42)" nil {:a {:b {:c 42}}}]] + :added "0.1"} + ([m ks v] + (let [ks (seq ks) + k (first ks) + ks (next ks)] + (if ks + (assoc m k (assoc-in (get m k) ks v)) + (assoc m k v))))) + +(defn update-in + {:doc "Updates a value in a nested collection." + :examples [["(update-in {:a {:b {:c 41}}} [:a :b :c] inc)" nil {:a {:b {:c 42}}}]] + :added "0.1"} + [m ks f & args] + (let [f (fn [m] (apply f m args)) + update-inner-f (fn update-inner-f + ([m f k] + (assoc m k (f (get m k)))) + ([m f k & ks] + (assoc m k (apply update-inner-f (get m k) f ks))))] + (apply update-inner-f m f ks))) + +(def subs pixie.string.internal/substring) + +(defmacro assert + ([test] + `(if ~test + nil + (throw [:pixie.stdlib/AssertionException + "Assert failed"]))) + ([test msg] + `(if ~test + nil + (throw [:pixie.stdlib/AssertionException + (str "Assert failed: " ~msg)])))) + +(defmacro resolve + {:doc "Resolves the var associated with the symbol in the current namespace." + :added "0.1"} + [sym] + `(resolve-in (this-ns-name) ~sym)) + +(defmacro binding [bindings & body] + (let [bindings (apply hashmap bindings) + set-vars (reduce (fn [res binding] + (conj res `(set! (resolve (quote ~(key binding))) ~(val binding)))) + [] + bindings)] + `(do (push-binding-frame!) + ~@set-vars + (let [ret (do ~@body)] + (pop-binding-frame!) + ret)))) + +(defmacro ns [nm & body] + (let [bmap (reduce (fn [m b] + (update-in m [(first b)] (fnil conj []) (rest b))) + {} + body) + requires + (do + (assert (>= 1 (count (:require bmap))) + "Only one :require block can be defined per namespace") + (map (fn [r] `(require ~@r)) (first (:require bmap)))) + + old-style-requires + (map (fn [r] `(require ~@r)) + (bmap 'require))] + `(do (in-ns ~(keyword (name nm))) + ~@requires + ~@old-style-requires))) + +(defn symbol? [x] + (identical? Symbol (type x))) + + +(defmacro lazy-seq [& body] + `(lazy-seq* (fn [] ~@body))) + +(def Protocol @(resolve (symbol "/Protocol"))) + +(defn protocol? [x] + (instance? Protocol x)) + +(defmacro deftype + {:doc "Defines a custom type." + :examples [["(deftype Person [name] + IObject + (-str [self] + (str \"\")))"] + ["(str (->Person \"James\"))" nil ""]] + :added "0.1"} + [nm fields & body] + (let [ctor-name (symbol (str "->" (name nm))) + fields (transduce (map (comp keyword name)) conj fields) + field-syms (transduce (map (comp symbol name)) conj fields) + mk-body (fn [body] + (let [fn-name (first body) + _ (assert (symbol? fn-name) "protocol override must have a name") + args (second body) + _ (assert (or (vector? args) + (seq? args)) "protocol override must have arguments") + self-arg (first args) + _ (assert (symbol? self-arg) "protocol override must have at least one `self' argument") + + rest (next (next body)) + body (reduce + (fn [body f] + `[(local-macro [~(symbol (name f)) + (get-field ~self-arg ~(keyword (name f)))] + ~@body)]) + rest + fields)] + `(fn ~(symbol (str fn-name "_" nm)) ~args ~@body))) + bodies (reduce + (fn [res body] + (cond + (symbol? body) (cond + (= body 'Object) [body (second res) (third res)] + (protocol? @(resolve-in *ns* body)) [@(resolve-in *ns* body) + (second res) + (conj (third res) body)] + :else (throw + [:pixie.stdlib/AssertionException + (str "can only extend protocols or Object, not " body " of type " (type body))])) + (seq? body) (let [proto (first res) tbs (second res) pbs (third res)] + (if (protocol? proto) + [proto tbs (conj pbs body)] + [proto (conj tbs body) pbs])))) + [nil [] []] + body) + type-bodies (second bodies) + proto-bodies (third bodies) + all-fields (reduce (fn [r tb] (conj r (keyword (name (first tb))))) fields type-bodies) + type-decl `(def ~nm (create-type ~(keyword (name nm)) ~all-fields)) + inst (gensym) + ctor `(defn ~ctor-name ~field-syms + (new ~nm + ~@field-syms + ~@(transduce (map (fn [type-body] + (mk-body type-body))) + conj + type-bodies))) + proto-bodies (transduce + (map (fn [body] + (cond + (symbol? body) `(satisfy ~body ~nm) + (seq? body) `(extend ~(first body) ~nm ~(mk-body body)) + :else (assert false "Unknown body element in deftype, expected symbol or seq")))) + conj + proto-bodies)] + `(do ~type-decl + ~ctor + ~@proto-bodies))) + +(defn -make-record-assoc-body [cname fields] + (let [k-sym (gensym "k") + v-sym (gensym "v") + this-sym (gensym "this") + result `(-assoc [~this-sym ~k-sym ~v-sym] + (case ~k-sym + ~@(mapcat + (fn [k] + [k `(~cname ~@(mapv (fn [x] + (if (= x k) + v-sym + `(get-field ~this-sym ~x))) + fields))]) + fields) + (throw [:pixie.stdlib/NotImplementedException + (str "Can't assoc to a unknown field: " ~k-sym)])))] + result)) + +(defmacro defrecord + {:doc "Defines a record type. + +Similar to `deftype`, but supports construction from a map using `map->Type` +and implements IAssociative, ILookup and IObject." + :added "0.1"} + [nm field-syms & body] + (let [ctor-name (symbol (str "->" (name nm))) + map-ctor-name (symbol (str "map" (name ctor-name))) + fields (transduce (map (comp keyword name)) conj field-syms) + type-from-map `(defn ~map-ctor-name [m] + (apply ~ctor-name (map #(get m %) ~fields))) + meta-gs (gensym "meta") + default-bodies ['IAssociative + (-make-record-assoc-body ctor-name fields) + + `(-contains-key [self k] + (contains? ~(set fields) k)) + `(-dissoc [self k] + (throw [:pixie.stdlib/NotImplementedException + "dissoc is not supported on defrecords"])) + 'ILookup + (let [self-nm (gensym "self") + k-nm (gensym "k")] + `(-val-at [~self-nm ~k-nm not-found#] + (case ~k-nm + ~@(mapcat + (fn [k] + [k `(get-field ~self-nm ~k-nm)]) + fields) + not-found#))) + + 'IReduce + `(-reduce [self# f# init#] + (loop [fields# ~fields + acc# init#] + (if-let [field# (first fields#)] + (let [acc# (f# acc# (map-entry field# + (get-field self# + field#)))] + (if (reduced? acc#) + @acc# + (recur (next fields#) acc#))) + acc#))) + 'ICounted + `(-count [self] ~(count fields)) + + 'ISeqable + `(-seq [self#] + (map #(map-entry % (get-field self# %)) + ~fields)) + + 'IPersistentCollection + `(-conj [self# x] + (cond + (instance? MapEntry x) + (assoc self# (key x) (val x)) + (instance? PersistentVector x) + (if (= (count x) 2) + (assoc self# (first x) (second x)) + (throw + [:pixie.stdlib/InvalidArgumentException + "Vector arg to record conj must be a pair"])))) + + `(-disj [self# x] + (throw [:pixie.stdlib/NotImplementedException + "disj is not supported on defrecords"])) + + 'IMeta + `(-with-meta [self# ~meta-gs] + (new ~nm + ~@(conj field-syms meta-gs))) + `(-meta [self#] __meta) + + 'IObject + `(-str [self#] + (str "<" ~(name nm) " " (reduce #(assoc %1 %2 (%2 self#)) {} ~fields) ">")) + `(-eq [self other] + (and (instance? ~nm other) + ~@(map (fn [field] + `(= (~field self) (~field other))) + fields))) + `(-hash [self] + (hash [~@field-syms])) + `IRecord] + deftype-decl `(deftype ~nm ~(conj fields '__meta) ~@default-bodies ~@body) + ctor `(defn ~ctor-name ~field-syms + (new ~nm + ~@(conj field-syms nil)))] + `(do ~type-from-map + ~deftype-decl + ~ctor))) + +(defn print + {:doc "Prints the arguments, seperated by spaces." + :added "0.1"} + [& args] + (printf (transduce (interpose " ") str args)) + nil) + +(defn println + {:doc "Prints the arguments, separated by spaces, with a newline at the end." + :added "0.1"} + [& args] + (puts (transduce (interpose " ") str args)) + nil) + +(defn pr-str + {:doc "Formats the arguments using -repr, separated by spaces, returning a string." + :added "0.1"} + [& args] + (transduce (comp (map -repr) (interpose " ")) str args)) + +(defn pr + {:doc "Prints the arguments using -repr, separated by spaces." + :added "0.1"} + [& args] + (printf (apply pr-str args)) + nil) + +(defn prn + {:doc "Prints the arguments using -repr, separated by spaces, with a newline at the end." + :added "0.1"} + [& args] + (puts (apply pr-str args)) + nil) + +(defn repeatedly + {:doc "Returns a lazy seq that contains the return values of repeated calls to f. + + Yields an infinite seq with one argument. + With two arguments n specifies the number of elements." + :examples [["(into '(:batman!) (repeatedly 8 (fn [] :na)))" + nil (:na :na :na :na :na :na :na :na :batman!)]] + :signatures [[f] [n f]]} + ([f] (lazy-seq (cons (f) (repeatedly f)))) + ([n f] (take n (repeatedly f)))) + +(defmacro doseq + {:doc "Evaluates all elements of the seq, presumably for side effects. Returns nil." + :added "0.1"} + [binding & body] + (assert (= (count binding) 2) "expected a binding and a collection") + (let [b (first binding) + s (second binding)] + `(loop [s# (seq ~s)] + (if s# + (let [~b (first s#)] + ~@body + (recur (next s#))))))) + +(defmacro doc + {:doc "Returns the documentation of the given value." + :added "0.1"} + [v] + + (let [vr (resolve-in *ns* v) + x (if vr @vr) + doc (get (meta x) :doc) + has-doc? (if doc true (get (meta x) :signatures))] + (cond + (satisfies? IDoc x) (-doc x) + has-doc? (let [sigs (get (meta x) :signatures) + examples (get (meta x) :examples) + indent (fn [s] + (if (>= (pixie.string.internal/index-of s "\n") 0) + (apply str "\n" (map #(str " " % "\n") (pixie.string.internal/split s "\n"))) + s))] + (println (str (namespace vr) "/" (name vr))) + (if sigs + (prn (seq sigs))) + (if doc + (do (println) + (println doc))) + (if examples + (do + (println) + (doseq [example examples] + (println (str " user => " (indent (first example)))) + (if (second example) + (print (indent (second example)))) + (if (contains? example 2) + (println (str " " (-repr (third example)))))))) + (println) + nil) + (the-ns v) (doc-ns v)))) + +(defn doc-ns + {:doc "Prints a summarizing documentation of the symbols in a namespace." + :added "0.1"} + [ns] + (let [ns (the-ns ns) + short-doc (fn [x] + (let [doc (get (meta x) :doc)] + (if doc + (let [newline (pixie.string.internal/index-of doc "\n")] + (pixie.string.internal/substring doc 0 (if (< newline 0) (count doc) newline))))))] + (println (str (name ns) ":")) + (vec (map (fn [sym] + (print (str " " (name sym))) + (let [doc (short-doc @(resolve-in ns sym))] + (if doc + (print (str (apply str (repeat (- 30 (count (name sym))) " ")) + doc)))) + (println)) + (keys (ns-map ns)))) + nil)) + +(defn swap! + {:doc "Swaps the value in the atom, by applying f to the current value. + +The new value is thus `(apply f current-value-of-atom args)`." + :signatures [[atom f & args]] + :added "0.1"} + [a f & args] + (reset! a (apply f @a args))) + +(defn nil? [x] + (identical? x nil)) + +(defn some? [x] + {:doc "Returns true if x is not nil."} + (not (nil? x))) + +(defn fnil [f else] + (fn [x & args] + (apply f (if (nil? x) else x) args))) + +(defmacro foreach [binding & body] + (assert (= 2 (count binding)) "binding and collection required") + `(reduce + (fn [_ ~(nth binding 0 nil)] + ~@body + nil) + nil + ~(nth binding 1 nil))) + +(defmacro dotimes + {:doc "Executes the expressions in the body n times." + :examples [["(dotimes [i 3] (println i))" "1\n2\n3\n"]] + :signatures [[[i n] & body]] + :added "0.1"} + [bind & body] + (let [b (nth bind 0 nil)] + `(let [max# ~(nth bind 1 nil)] + (loop [~b 0] + (if (= ~b max#) + nil + (do ~@body + (recur (inc ~b)))))))) + +(defmacro and + {:doc "Checks if the given expressions return truthy values, returning the last, or false." + :examples [["(and true false)" nil false] + ["(and 1 2 3)" nil 3] + ["(and 1 false 3)" nil false]] + :added "0.1"} + ([] true) + ([x] x) + ([x y] `(if ~x ~y false)) + ([x y & more] `(if ~x (and ~y ~@more) false))) + +(defmacro or + {:doc "Returns the value of the first expression that returns a truthy value, or false." + :examples [["(or 1 2 3)" nil 1] ["(or false 2)" nil 2] ["(or false nil)" nil nil]] + :added "0.1"} + ([] false) + ([x] x) + ([x y] `(let [r# ~x] + (if r# r# ~y))) + ([x y & more] `(let [r# ~x] + (if r# r# (or ~y ~@more))))) + +(defmacro when [test & body] + `(if ~test (do ~@body))) + +(defmacro when-not [test & body] + `(if (not ~test) (do ~@body))) + +(defmacro when-let [binding & body] + (let [bind (nth binding 0 nil) + test (nth binding 1 nil)] + `(let [tmp# ~test] + (when tmp# + (let [~bind tmp#] + ~@body))))) + +(defmacro if-not + ([test then] `(if-not ~test ~then nil)) + ([test then else] + `(if (not ~test) ~then ~else))) + +(defmacro if-let + ([binding then] `(if-let ~binding ~then nil)) + ([binding then else] + (let [bind (nth binding 0 nil) + test (nth binding 1 nil)] + `(let [tmp# ~test] + (if tmp# + (let [~bind tmp#] + ~then) + ~else))))) + +(defn some + {:doc "Returns the first true value of the predicate for the elements of the collection." + :signatures [[pred coll]] + :added "0.1"} + [pred coll] + (if (seq coll) + (or (pred (first coll)) + (some pred (next coll))) + false)) + +(defn nnext + {:doc "Equivalent to (next (next coll))." + :added "0.1"} + [coll] + (next (next coll))) + +(defn nthnext + {:doc "Returns the result of calling next n times on the collection." + :examples [["(nthnext [1 2 3 4 5] 2)" nil (3 4 5)] + ["(nthnext [1 2 3 4 5] 7)" nil nil]] + :added "0.1"} + [coll n] + (loop [n n + xs (seq coll)] + (if (and xs (pos? n)) + (recur (dec n) (next xs)) + xs))) + +(defn ith + {:doc "Returns the ith element of the collection, negative values count from the end. + If an index is out of bounds, will throw an Index out of Range exception. + However, if you specify a not-found parameter, it will substitute that instead." + :signatures [[coll i] [coll idx not-found]] + :added "0.1"} + ([coll i] + (when coll + (let [idx (if (neg? i) (+ i (count coll)) i)] + (nth coll idx)))) + ([coll i not-found] + (when coll + (let [idx (if (neg? i) (+ i (count coll)) i)] + (nth coll idx not-found))))) + +(defn ensure-reduced [x] + (if (reduced? x) + x + (reduced x))) + +(defn take + {:doc "Takes n elements from the collection, or fewer, if not enough." + :added "0.1"} + ([n] + (fn [rf] + (let [seen (atom 0)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [s (swap! seen inc)] + (cond (< s n) (rf result input) + (= s n) (ensure-reduced (rf result input)) + :else (reduced result)))))))) + ([n coll] + (lazy-seq + (when (pos? n) + (when-let [s (seq coll)] + (cons (first s) (take (dec n) (next s)))))))) + +(defn drop + {:doc "Drops n elements from the start of the collection." + :added "0.1"} + ([n] + (fn [rf] + (let [seen (atom 0)] + (fn + ([] (rf)) + ([result] + (rf result)) + ([result input] + (let [s (swap! seen inc)] + (if (> s n) + (rf result input) + result))))))) + ([n coll] + (let [s (seq coll)] + (if (and (pos? n) s) + (recur (dec n) (next s)) + s)))) + +(defn split-at + {:doc "Returns a vector of the first n elements of the collection, and the remaining elements." + :examples [["(split-at 2 [:a :b :c :d :e])" nil + [(:a :b) (:c :d :e)]]]} + [n coll] + [(take n coll) (drop n coll)]) + +(defmacro while + {:doc "Repeatedly executes body while test expression is true. Presumes + some side-effect will cause test to become false/nil. Returns nil." + :added "0.1"} + [test & body] + `(loop [] + (when ~test + ~@body + (recur)))) + +(defn take-while + {:doc "Returns a lazy sequence of successive items from coll while + (pred item) returns true. pred must be free of side-effects. + Returns a transducer when no collection is provided." + :added "0.1"} + ([pred] + (fn [rf] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (if (pred input) + (rf result input) + (reduced result)))))) + ([pred coll] + (lazy-seq + (when-let [s (seq coll)] + (when (pred (first s)) + (cons (first s) (take-while pred (rest s)))))))) + + +(defn drop-while + {:doc "Returns a lazy sequence of the items in coll starting from the + first item for which (pred item) returns logical false. Returns a + stateful transducer when no collection is provided." + :added "0.1"} + ([pred] + (fn [rf] + (let [dv (atom true)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [drop? @dv] + (if drop? + (if (pred input) + result + (do + (reset! dv nil) + (rf result input))) + (rf result input)))))))) + ([pred coll] + (let [step (fn [pred coll] + (let [s (seq coll)] + (if (and s (pred (first s))) + (recur pred (rest s)) + s)))] + (lazy-seq (step pred coll))))) + +(defn cycle + [coll] + (if (empty? coll) + () + (let [cycle' + (fn cycle' [current] + (lazy-seq + (cons + (first current) + (let [rst (rest current)] + (cycle' (if (empty? rst) coll rst))))))] + (cycle' coll)))) + +;; TODO: use a transient map in the future +(defn group-by + {:doc "Groups the collection into a map keyed by the result of applying f on each element. The value at each key is a vector of elements in order of appearance." + :examples [["(group-by even? [1 2 3 4 5])" nil {false [1 3 5] true [2 4]}] + ["(group-by (partial apply +) [[1 2 3] [2 4] [1 2]])" nil {6 [[1 2 3] [2 4]] 3 [[1 2]]}]] + :signatures [[f coll]] + :added "0.1"} + [f coll] + (reduce (fn [res elem] + (update-in res [(f elem)] (fnil conj []) elem)) + {} + coll)) + +;; TODO: use a transient map in the future +(defn frequencies + {:doc "Returns a map with distinct elements as keys and the number of occurences as values." + :added "0.1"} + [coll] + (reduce (fn [res elem] + (update-in res [elem] (fnil inc 0))) + {} + coll)) + +(defn partition + {:doc "Separates the collection into collections of size n, starting at the beginning, with an optional step size. + +The last element of the result contains the remaining element, not necessarily of size n if +not enough elements were present." + :examples [["(partition 2 [1 2 3 4 5 6])" nil ((1 2) (3 4) (5 6))] + ["(partition 2 [1 2 3 4 5])" nil ((1 2) (3 4) (5))] + ["(partition 2 1 [1 2 3 4 5])" nil ((1 2) (2 3) (3 4) (4 5) (5))]] + :signatures [[n coll] [n step coll]] + :added "0.1"} + ([n coll] (partition n n coll)) + ([n step coll] + (when-let [s (seq coll)] + (lazy-seq + (cons (take n s) (partition n step (drop step s))))))) + +(defn partitionf + {:doc "A generalized version of partition. Instead of taking a constant number of elements, + this function calls f with the remaining collection to determine how many elements to + take." + :examples [["(partitionf first [2 :a, 3 :a :b, 4 :a :b :c])" + nil ((2 :a) (3 :a :b) (4 :a :b :c))]]} + [f coll] + (when-let [s (seq coll)] + (lazy-seq + (let [n (f s)] + (cons (take n s) + (partitionf f (drop n s))))))) + +(defn map-indexed + {:doc "Returns a lazy sequence consisting of the + result of applying f to 0 and the first item of coll, followed by + applying f to 1 and the second item in coll, etc, until coll is + exhausted. Thus function f should accept 2 arguments, index and + item. Returns a stateful transducer when no collection is provided." + :added "0.1" + :signatures [[f] [f coll]]} + ([f] + (fn [rf] + (let [i (atom -1)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (rf result (f (swap! i inc) input))))))) + ([f coll] + (let [mapi (fn mapi [i coll] + (lazy-seq + (when-let [s (seq coll)] + (cons (f i (first s)) + (mapi (inc i) (rest s))))))] + (mapi 0 coll)))) + +(defn keep-indexed + {:doc "Returns a lazy sequence of the non-nil + results of (f index item). Note, this means false return values will + be included. f must be free of side-effects. Returns a stateful + transducer when no collection is provided." + :signatures [[f] [f coll]] + :added "0.1"} + ([f] + (fn [rf] + (let [iv (atom -1)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [i (swap! iv inc) + v (f i input)] + (if (nil? v) + result + (rf result v)))))))) + ([f coll] + (let [keepi (fn keepi [i coll] + (lazy-seq + (when-let [s (seq coll)] + (let [x (f i (first s))] + (if (nil? x) + (keepi (inc i) (rest s)) + (cons x (keepi (inc i) (rest s))))))))] + (keepi 0 coll)))) + +(defn reductions + {:doc "Returns a lazy seq of the intermediate values of the + reduction (as per reduce) of coll by f, starting with init." + :added "0.1" + :signatures [[f coll] [f init coll]]} + ([f coll] + (lazy-seq + (if-let [s (seq coll)] + (reductions f (first s) (rest s)) + (list (f))))) + ([f init coll] + (if (reduced? init) + (list @init) + (cons init + (lazy-seq + (when-let [s (seq coll)] + (reductions f (f init (first s)) (rest s)))))))) + +(defn completing + "Takes a reducing function f of 2 args and returns a fn suitable for + transduce by adding an arity-1 signature that calls cf (default - + identity) on the result argument." + ([f] (completing f identity)) + ([f cf] + (fn + ([] (f)) + ([x] (cf x)) + ([x y] (f x y))))) + +(deftype Eduction [xform coll] + IReduce + (-reduce [self f init] + ;; NB (completing f) isolates completion of inner rf from outer rf + (transduce xform (completing f) init coll)) + + ISeqable + (-seq [self] + (sequence xform coll))) + +(defn eduction + "Returns a reducible/iterable application of the transducers + to the items in coll. Transducers are applied in order as if + combined with comp. Note that these applications will be + performed every time reduce/iterator is called." + [& xforms] + (->Eduction (apply comp (butlast xforms)) (last xforms))) + +(defn destructure [binding expr] + (cond + (symbol? binding) [binding expr] + (vector? binding) (let [name (gensym "vec__")] + (reduce conj [name expr] + (destructure-vector binding name))) + (map? binding) (let [name (gensym "map__")] + (reduce conj [name expr] + (destructure-map binding name))) + :else (throw [:pixie.stdlib/AssertionException + (str "unsupported binding form: " binding)]))) + +(defn destructure-vector [binding-vector expr] + (loop [bindings (seq binding-vector) + i 0 + res []] + (if bindings + (let [binding (first bindings)] + (cond + (= binding '&) (recur (nnext bindings) + (inc (inc i)) + (reduce conj res (destructure (second bindings) `(nthnext ~expr ~i)))) + (= binding :as) (reduce conj res (destructure (second bindings) expr)) + :else (recur (next bindings) + (inc i) + (reduce conj res (destructure (first bindings) `(nth ~expr ~i nil)))))) + res))) + +(defn destructure-map [binding-map expr] + (let [defaults (or (:or binding-map) {}) + res + (loop [bindings (seq binding-map) + res []] + (if bindings + (let [binding (key (first bindings)) + binding-key (val (first bindings))] + (if (contains? #{:or :as :keys} binding) + (recur (next bindings) res) + (recur (next bindings) + (reduce conj res (destructure binding `(get ~expr ~binding-key ~(get defaults binding))))))) + res)) + expand-with (fn [convert] #(vector % `(get ~expr ~(convert %) ~(get defaults %)))) + res (if (contains? binding-map :keys) (transduce (map (expand-with (comp keyword name))) concat res (get binding-map :keys)) res) + res (if (contains? binding-map :as) + (reduce conj res [(get binding-map :as) expr]) + res)] + res)) + +(defmacro let + {:doc "Makes the bindings availlable in the body. + +The bindings must be a vector of binding-expr pairs. The binding can be a destructuring +binding, as below. + +Vector destructuring: + [x y z] binds the first three elements of the collection to x, y and z + [x y & rest] binds rest to the elements after the first two elements of the collection + [x y :as v] binds the value of the complete collection to v + +Map destructuring: + {a :a, b :b} binds a and b to the values associated with :a and :b + {a :a :as m} binds the value of the complete collection to m + {a :a :or {a 42}} binds a to the value associated with :a, or 42, if not present + {:keys [a b c]} binds a, b and c to the values associated with :a, :b and :c + +All these forms can be combined and nested, in the example below: + +(let [[x y [z :as iv] :as v] [1 2 [3 4 5] 6 7] + {a :a [b c {:keys [d]}] :more :or {a 42}} {:a 1, :more [1 2 {:d 3, :e 4}]}] + ...) + +For more information, see http://clojure.org/special_forms#binding-forms"} + [bindings & body] + (let* [destructured-bindings (transduce (map (fn [args] + (assert (= 2 (count args)) (str "Bindings must be in pairs, not " args + " " (meta (first args)))) + (apply destructure args))) + concat + [] + (partition 2 bindings))] + `(let* ~destructured-bindings + ~@body))) + +(extend -nth ISeq (fn [s n] + (when (empty? s) + (throw + [:pixie.stdlib/OutOfRangeException + "Index out of Range"])) + (if (and (pos? n) s) + (recur (next s) (dec n)) + (if (zero? n) + (first s) + (throw [:pixie.stdlib/OutOfRangeException + "Index out of Range"]))))) +(extend -nth-not-found ISeq (fn [s n not-found] + (if (and (pos? n) s) + (recur (next s) (dec n) not-found) + (or (first s) not-found)))) + +(defn abs + {:doc "Returns the absolute value of x." + :added "0.1"} + [x] + (if (< x 0) + (* -1 x) + x)) + +(deftype Repeat [n x] + IReduce + (-reduce [self f init] + (loop [i 0 + acc init] + (if (< i n) + (let [acc (f acc x)] + (if (reduced? acc) + @acc + (recur (inc i) acc))) + acc))) + ICounted + (-count [self] + n) + IIndexed + (-nth [self idx] + (if (and (>= idx 0) (< idx n)) + x + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"]))) + (-nth-not-found [self idx not-found] + (if (and (>= idx 0) (< idx n)) + x + not-found)) + ISeqable + (-seq [self] + (when (>= n 1) + (cons x (lazy-seq* (fn [] (->Repeat (dec n) x))))))) + +(extend -str Repeat + (fn [v] + (-str (seq v)))) + +(extend -repr Repeat -str) + +(defn repeat + {:doc "Returns a seqable of repetitions of a value." + :examples [["(repeat 3 :buffalo)" nil (:buffalo :buffalo :buffalo)] + ["(map vector '(1 2 3) (repeat :ahahah))" nil ([1 :ahahah] [2 :ahahah] [3 :ahahah])]] + :signatures [[x] [n x]] + :added "0.1"} + ([x] + (let [positive-infinity (/ 1.0 0)] + (repeat positive-infinity x))) + ([n x] + (->Repeat n x))) + +(deftype Range [start stop step] + IReduce + (-reduce [self f init] + (loop [i start + acc init] + (if (or (and (> step 0) (< i stop)) + (and (< step 0) (> i stop)) + (and (= step 0))) + (let [acc (f acc i)] + (if (reduced? acc) + @acc + (recur (+ i step) acc))) + acc))) + ICounted + (-count [self] + (if (or (and (< start stop) (< step 0)) + (and (> start stop) (> step 0)) + (= step 0)) + 0 + (abs (quot (- start stop) step)))) + IIndexed + (-nth [self idx] + (when (or (= start stop 0) (neg? idx)) + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"])) + (let [cmp (if (< start stop) < >) + val (+ start (* idx step))] + (if (cmp val stop) + val + (throw [:pixie.stdlib/OutOfRangeException "Index out of Range"])))) + (-nth-not-found [self idx not-found] + (let [cmp (if (< start stop) < >) + val (+ start (* idx step))] + (if (cmp val stop) + val + not-found))) + ISeq + (-first [this] + (when (not= start stop) + start)) + (-next [this] + (let [i (+ step start)] + (when (or (and (> step 0) (< i stop)) + (and (< step 0) (> i stop)) + (and (= step 0))) + (range i stop step)))) + ISeqable + (-seq [self] self)) + +(extend -str Range + (fn [v] + (str "(" (transduce (interpose " ") string-builder v) ")"))) + +(extend -repr Range + (fn [v] + (str "(" (transduce (interpose " ") string-builder v) ")"))) + +(defn range + {:doc "Returns a range of numbers." + :examples [["(seq (range 3))" nil (0 1 2)] + ["(seq (range 3 5))" nil (3 4)] + ["(seq (range 0 10 2))" nil (0 2 4 6 8)] + ["(seq (range 5 -1 -1))" nil (5 4 3 2 1 0)]] + :signatures [[] [stop] [start stop] [start stop step]] + :added "0.1"} + ([] (->Range 0 MAX-NUMBER 1)) + ([stop] (->Range 0 stop 1)) + ([start stop] (->Range start stop 1)) + ([start stop step] (->Range start stop step))) + +(extend -eq ISeqable -seq-eq) + +(deftype Unknown []) +(def unknown (->Unknown)) + +(extend -eq PersistentHashMap + (fn [self other] + (cond + (not (map? other)) false + (not= (count self) (count other)) false + :else (reduce (fn + ([_] true) + ([_ entry] + (let [other-val (get other (key entry) unknown)] + (if (not= other-val (val entry)) + (reduced false) + true)))) + true + self)))) + +(defn filter + {:doc "Filters the collection for elements matching the predicate." + :signatures [[pred] [pred coll]] + :added "0.1"} + ([pred] + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (if (pred i) + (xf acc i) + acc))))) + ([pred coll] + (lazy-seq + (when-let [s (seq coll)] + (let [[f & r] s] + (if (pred f) + (cons f (filter pred r)) + (filter pred r))))))) + +(defn remove + {:doc "Removes any element from the collection which matches the predicate. The complement of filter." + :signatures [[pred] [pred coll]] + :added "0.1"} + ([pred] + (filter (complement pred))) + ([pred coll] + (filter (complement pred) coll))) + +(defn sequence + "Returns a lazy sequence of `data`, optionally transforming it using `xform`" + ([coll] + (if (seq? coll) coll + (or (seq coll) ()))) + ([xform coll] + (let [step (fn step [xform acc xs] + (if-let [s (seq xs)] + (let [next-acc ((xform conj) acc (first s))] + (if (= acc next-acc) (step xform next-acc (next s)) + (concat (drop (count acc) next-acc) (step xform next-acc (next s))))) + nil))] + (lazy-seq (step xform [] coll))))) + +(defn distinct + {:doc "Returns the distinct elements in the collection." + :signatures [[] [coll]] + :added "0.1"} + ([] (fn [xf] + (let [seen (atom #{})] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] + (if (contains? @seen i) + acc + (do + (swap! seen conj i) + (xf acc i)))))))) + ([coll] + (let [step (fn step [xs seen] + (lazy-seq + ((fn [f seen] + (when-let [s (seq f)] + (let [xs (first s)] + (if (contains? seen xs) + (step (rest s) seen) + (cons xs (step (rest s) (conj seen xs))))))) + xs seen)))] + (step coll #{})))) + +(defn keep + ([f] + (fn [xf] + (fn + ([] (xf)) + ([acc] (xf acc)) + ([acc i] (let [result (f i)] + (if result + (xf acc result) + acc)))))) + ([f coll] + (lazy-seq + (when-let [s (seq coll)] + (let [[first & rest] s + result (f first)] + (if result + (cons result (keep f rest)) + (keep f rest))))))) + +(defn refer + {:doc "Refer to the specified vars from a namespace directly. + +Supported filters: + :rename refer to the given vars under a different name + :exclude don't refer the given vars + :refer + :all refer all vars + :refer refer only the given vars + :only same as refer + +user => (refer 'pixie.string :refer :all) +user => (refer 'pixie.string :only '(index-of starts-with? ends-with?)) +user => (refer 'pixie.string :rename '{index-of find}) +user => (refer 'pixie.string :exclude '(substring))" + :added "0.1"} + [ns-sym & filters] + (let [ns (or (the-ns ns-sym) (throw [:pixie.stdlib/NamespaceNotFoundException + (str "No such namespace: " ns-sym)])) + filters (apply hashmap filters) + nsmap (ns-map ns) + rename (or (:rename filters) {}) + exclude (set (:exclude filters)) + refers (if (= :all (:refer filters)) + (keys nsmap) + (or (:refer filters) (:only filters)))] + (when (and refers (not (satisfies? ISeqable refers))) + (throw [:pixie.stdlib/InvalidArgumentException + ":only/:refer must be a collection of symbols"])) + (when-let [as (:as filters)] + (refer-ns *ns* ns-sym as)) + (loop [syms (seq refers)] + (if (not syms) + nil + (do + (let [sym (first syms)] + (when-not (exclude sym) + (let [v (nsmap sym)] + (when-not v + (throw [:pixie.stdlib/SymbolNotFoundException + (str sym "does not exist")])) + (refer-symbol *ns* (or (rename sym) sym) v)))) + (recur (next syms))))) + nil)) + + + +(defmacro require [ns & args] + `(do (load-ns (quote ~ns)) + (assert (the-ns (quote ~ns)) + (str "Couldn't find the namespace " (quote ~ns) " after loading the file")) + + (apply refer (quote [~ns ~@args])))) + + + +(defn merge-with + {:doc "Returns a map consisting of each map merged onto the first. If a + map contains a key that already exists in the result, the + value will be f applied to the value in the result map and + the value from the map being merged in." + :examples [["(merge-with + {:a 1 :b 2} {:a 3 :c 5} {:c 3 :d 4})" nil {:a 4, :b 2, :c 8 :d 4}]]} + [f & maps] + (cond + (empty? maps) nil + (= (count maps) 1) (first maps) + :else (let [merge2 (fn [m1 m2] + (reduce (fn [res e] + (let [k (key e) v (val e)] + (if (contains? m1 k) + (assoc res k (f (get m1 k) v)) + (assoc res k v)))) + (or m1 {}) + m2))] + (reduce merge2 (first maps) (next maps))))) + +(defn every? + {:doc "Checks if every element of the collection satisfies the predicate." + :added "0.1"} + [pred coll] + (cond + (nil? (seq coll)) true + (pred (first coll)) (recur pred (next coll)) + :else false)) + +; If you want a fn that uses destructuring in its parameter list, place +; it after this definition. If you don't, you will get compile failures +; in unrelated files. +(defmacro fn + {:doc "Creates a function. + +The following two forms are allowed: + (fn name? [param*] & body) + (fn name? ([param*] & body)+) + +The params can be destructuring bindings, see `(doc let)` for details."} + [& decls] + (let [name (if (symbol? (first decls)) [(first decls)] nil) + decls (if name (next decls) decls) + decls (cond + (vector? (first decls)) (list decls) + ;(satisfies? ISeqable (first decls)) decls + ;:else (throw (str "expected a vector or a seq, got a " (type decls))) + :else decls) + decls (seq (map (fn* [decl] + (let [argv (first decl) + names (vec (map #(if (= % '&) '& (gensym "arg__")) argv)) + bindings (loop [i 0 bindings []] + (if (< i (count argv)) + (if (= (nth argv i) '&) + (recur (inc i) bindings) + (recur (inc i) (reduce conj bindings [(nth argv i) (nth names i)]))) + bindings)) + body (next decl) + conds (when (and (next body) (map? (first body))) + (first body)) + pre (:pre conds) + post (:post conds) + body (if conds (next body) body) + body (if post + `((let [~'% ~(if (> (count body) 1) + `(do ~@body) + (first body))] + ~@(map (fn* [c] `(assert ~c (str '~c))) post) + ~'%)) + body) + body (if pre + (seq (concat + (map (fn* [c] `(assert ~c (str '~c))) pre) + body)) + body)] + (if (every? symbol? argv) + `(~argv ~@body) + `(~names + (let ~bindings + ~@body))))) + decls))] + (if (= (count decls) 1) + `(fn* ~@name ~(first (first decls)) ~@(next (first decls))) + `(fn* ~@name ~@decls)))) + +;; TODO: implement :>> like in Clojure? +(defmacro condp + "Takes a binary predicate, an expression and a number of two-form clauses. +Calls the predicate on the first value of each clause and the expression. +If the result is truthy returns the second value of the clause. + +If the number of arguments is odd and no clause matches, the last argument is returned. +If the number of arguments is even and no clause matches, throws an exception." + [pred-form expr & clauses] + (let [x (gensym 'expr), pred (gensym 'pred)] + `(let [~x ~expr, ~pred ~pred-form] + (cond ~@(mapcat + (fn [[a b :as clause]] + (if (> (count clause) 1) + `((~pred ~a ~x) ~b) + `(:else ~a))) + (partition 2 clauses)) + :else (throw [:pixie.stdlib/MissingClauseException + "No matching clause!"]))))) + +(defmacro case + "Takes an expression and a number of two-form clauses. +Checks for each clause if the first part is equal to the expression. +If yes, returns the value of the second part. + +The first part of each clause can also be a set. If that is the case, the clause matches when the result of the expression is in the set. + +If the number of arguments is odd and no clause matches, the last argument is returned. +If the number of arguments is even and no clause matches, throws an exception." + [expr & args] + `(condp #(if (set? %1) (%1 %2) (= %1 %2)) + ~expr ~@args)) + + + + +(deftype MultiMethod [dispatch-fn default-val methods] + IMessageObject + (-get-attr [this kw] + (case kw + :methods methods + :else nil)) + IFn + (-invoke [self & args] + (let [dispatch-val (apply dispatch-fn args) + method (if (contains? @methods dispatch-val) + (get @methods dispatch-val) + (get @methods default-val)) + _ (assert method (str "no method defined for " dispatch-val))] + (apply method args)))) + + +(defmacro defmulti + {:doc "Defines a multimethod, which dispatches to its methods based on dispatch-fn." + :examples [["(defmulti greet first)"] + ["(defmethod greet :hi [[_ name]] (str \"Hi, \" name \"!\"))"] + ["(defmethod greet :hello [[_ name]] (str \"Hello, \" name \".\"))"] + ["(greet [:hi \"Jane\"])" nil "Hi, Jane!"]] + :signatures [[name dispatch-fn & options]] + :added "0.1"} + [name & args] + (let [[meta args] (if (string? (first args)) + [{:doc (first args)} (next args)] + [{} args]) + [meta args] (if (map? (first args)) + [(merge meta (first args)) (next args)] + [meta args]) + dispatch-fn (first args) + options (apply hashmap (next args))] + `(def ~name (->MultiMethod ~dispatch-fn ~(get options :default :default) (atom {}))))) + +(defmacro defmethod + {:doc "Defines a method of a multimethod. See `(doc defmulti)` for details." + :signatures [[name dispatch-val [param*] & body]] + :added "0.1"} + [name dispatch-val params & body] + `(do + (let [methods (.-methods ~name)] + (swap! methods + assoc + ~dispatch-val (fn ~params + ~@body)) + ~name))) + +(defmulti Foo :r) +(defmethod Foo :r + [x] x) + +(defmacro declare + {:doc "Forward declares the given variable names, setting them to nil." + :added "0.1"} + [& nms] + (let [defs (map (fn [nm] `(def ~nm)) (seq nms))] + `(do ~@defs))) + +(defmacro defprotocol + {:doc "Defines a new protocol." + :examples [["(defprotocol SayHi (hi [x]))"] + ["(extend hi String (fn [name] (str \"Hi, \" name \"!\")))"] + ["(hi \"Jane\")" nil "Hi, Jane!"]] + :added "0.1"} + [nm & sigs] + `(pixie.stdlib.internal/-defprotocol (quote ~nm) + ~(reduce (fn [r sig] + (conj r `(quote ~(first sig)))) + [] + sigs))) + +(defmacro extend-type + {:doc "Extends the protocols to the given type. + +Expands to calls to `extend`." + :examples [["(defprotocol SayHi (hi [x]))"] + ["(extend-type String SayHi (hi [name] (str \"Hi, \" name \"!\")))"] + ["(hi \"Jane\")" nil "Hi, Jane!"]] + :added "0.1"} + [tp & extensions] + (let [[_ extends] (reduce (fn [[proto res] extend] + (cond + (symbol? extend) [extend res] + :else [proto (conj res `(extend ~(first extend) ~tp (fn ~@(next extend))))])) + [] + extensions)] + `(do + ~@extends))) + +(defmacro extend-protocol + {:doc "Extend the protocol to the given types. + +Expands to calls to `extend-type`." + :examples [["(defprotocol SayHi (hi [x]))"] + ["(extend-protocol SayHi + + String + (hi [name] + (str \"Hi, \" name \"!\")) + + Integer + (hi [n] + (str \"Hi, #\" n \"!\")))"] + ["(hi \"Jane\")" nil "Hi, Jane!"] + ["(hi 42)" nil "Hi, #42!"]] + :added "0.1"} + [protocol & extensions] + ; tps is used to ensure protocols are extended in order + (let [[_ tps exts] (reduce (fn [[tp tps res] extend-body] + (cond + (symbol? extend-body) [extend-body (conj tps extend-body) (assoc res extend-body [])] + :else [tp tps (update-in res [tp] conj extend-body)])) + [nil [] {}] + extensions) + exts (reduce (fn [res tp] + (conj res `(extend-type ~tp ~protocol ~@(get exts tp)))) + [] + tps)] + `(do ~@exts))) + +(defprotocol IToFloat + (-float [this])) + +(defn float + {:doc "Converts a number to a float." + :since "0.1"} + [x] + (-float x)) + +(extend-type Number + IToFloat + (-float [x] (+ x 0.0))) + +(defprotocol IToInteger + (-int [x])) + +(extend-protocol IToInteger + Integer + (-int [x] x) + + Float + (-int [x] (floor x)) + + Ratio + (-int [x] + (int (/ (float (numerator x)) (float (denominator x))))) + + Character + (-int [x] + (+ x 0))) + +(defn int + {:doc "Converts a number to an integer." + :since "0.1"} + [x] + (-int x)) + +(defprotocol IRecord) + +(defn record? + {:doc "Returns true if x implements IRecord." + :since "0.1"} + [x] + (satisfies? IRecord x)) + +(defmacro for + {:doc "A list comprehension for the bindings." + :examples [["(for [x [1 2 3]] x)" nil [1 2 3]] + ["(for [x [1 2 3] y [:a :b :c]] [x y])" nil [[1 :a] [1 :b] [1 :c] [2 :a] [2 :b] [2 :c] [3 :a] [3 :b] [3 :c]]]] + :added "0.1"} + [bindings & body] + (assert (and (pos? (count bindings)) (even? (count bindings))) "for requires an even number of bindings") + (let [gen-loop (fn gen-loop [coll-bindings bindings] + (if (seq bindings) + (let [c (gensym "coll__") + binding (first bindings) + coll (second bindings)] + `(loop [res# [] + ~c (seq ~coll)] + (if ~c + (let [~binding (first ~c)] + (recur (into res# + ~(gen-loop (into coll-bindings + [binding `(first ~c)]) + (nnext bindings))) + (next ~c))) + res#))) + `[~@body]))] + `(or (seq ~(gen-loop [] bindings)) '()))) + +(defmacro doto + {:doc "Evaluate o, uses the value as the first argument in each form. Returns o."} + [o & forms] + (let [s (gensym o)] + `(let [~s ~o] + ~@(for [f forms] + (if (seq? f) + `(~(first f) ~s ~@(rest f)) + `(~f ~s))) + ~s))) + +(defn reverse + ; TODO: We should probably have a protocol IReversible, so we can e.g. + ; reverse vectors efficiently, etc.. + [coll] + "Returns a collection that contains all the elements of the argument in reverse order." + (into () coll)) + +(defmacro use + "Loads a namespace and refers all symbols from it." + [ns] + `(do + (load-ns ~ns) + (refer ~ns :refer :all))) + +(defn count-rf + "A Reducing function that counts the items reduced over." + ([] 0) + ([result] result) + ([result _] (inc result))) + +(defn dispose! + "Finalizes use of the object by cleaning up resources used by the object." + [x] + (-dispose! x) + nil) + +(defmacro using + "Evaluates body with the bindings available as with let, + calling -dispose! on each name afterwards. Returns the value of the + last expression in body." + [bindings & body] + (let [pairs (partition 2 bindings) + names (map first pairs)] + `(let [~@bindings + result# (do ~@body)] + ~@(map (fn [nm] + `(-dispose! ~nm)) + names) + result#))) + +(defn pst + {:doc "Prints the trace of a Runtime Exception if given, or the last Runtime Exception in *e." + :signatures [[] [e]] + :added "0.1"} + ([] (pst *e)) + ([e] (when e (print (str e))))) + +(defn trace + {:doc "Returns a seq of the trace of a Runtime Exception or the last Runtime Exception in *e." + :signatures [[] [e]] + :added "0.1"} + ([] (trace *e)) + ([e] (seq e))) + +(defn tree-seq + "Returns a lazy sequence of the nodes in a tree via a depth-first walk. +branch? - fn of node that should true when node has children +children - fn of node that should return a sequence of children (called if branch? true) +root - root node of the tree" + [branch? children root] + (let [walk (fn walk [node] + (lazy-seq + (cons node + (when (branch? node) + (mapcat walk (children node))))))] + (walk root))) + +(defn flatten + ; TODO: laziness? + {:doc "Takes any nested combination of ISeqable things, and return their contents as a single, flat sequence. + +Calling this function on something that is not ISeqable returns a seq with that value as its only element." + :examples [["(flatten [[1 2 [3 4] [5 6]] 7])" nil [1 2 3 4 5 6 7]] + ["(flatten :this)" nil [:this]]]} + [x] + (if (not (satisfies? ISeqable x)) [x] + (transduce (comp (map flatten) cat) + conj [] + (seq x)))) + +(defn juxt + {:doc "Returns a function that applies all fns to its arguments, and returns a vector of the results." + :examples [["((juxt + - *) 2 3)" nil [5 -1 6]]]} + [& fns] + (fn [& args] + (mapv #(apply % args) fns))) + +(defn map-invert + {:doc "Returns a map where the vals are mapped to the keys." + :examples [["(map-invert {:a :b, :c :d})" nil {:b :a, :d :c}]]} + [m] + (reduce (fn [m* ent] + (assoc m* (val ent) (key ent))) + {} m)) + +(defn mapv + {:doc "Returns a vector consisting of f applied to each element in col." + :examples [["(mapv inc '(1 2 3))" nil [2 3 4]]]} + ([f col] + (transduce (map f) conj col))) + +(defn macroexpand-1 + {:doc "If form is a macro call, returns the expanded form. Does nothing if not a macro call." + :signatures [[form]] + :examples [["(macroexpand-1 '(when condition this and-this))" + nil (if condition (do this and-this))] + ["(macroexpand-1 ())" nil ()] + ["(macroexpand-1 [1 2])" nil [1 2]]]} + [form] + (if (or (not (list? form)) + (= () form)) + form + (let [[sym & args] form + fvar (resolve-in *ns* sym)] + (if (and fvar (macro? @fvar)) + (apply @fvar args) + form)))) + +(def *1) +(def *2) +(def *3) +(defn -push-history [x] + (def *3 *2) + (def *2 *1) + (def *1 x)) + +(def *e) +(defn -set-*e [e] + (def *e e)) + +(def hash-map hashmap) + +(defn zipmap + "Returns a map with the elements of a mapped to the corresponding + elements of b." + [a b] + (into {} (map vector a b))) + +(extend -str Environment + (fn [v] + (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] + (str "#Environment{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) + +(extend -repr Environment + (fn [v] + (let [entry->str (map (fn [e] (vector (-repr (key e)) " " (-repr (val e)))))] + (str "#Environment{" (transduce (comp entry->str (interpose [", "]) cat) string-builder v) "}")))) + +(defn interleave + "Returns a seq of all the items in the input collections interleaved." + ([] ()) + ([c1] (seq c1)) + ([c1 c2] + (lazy-seq + (let [s1 (seq c1) + s2 (seq c2)] + (when (and s1 s2) + (cons (first s1) (cons (first s2) + (interleave (next s1) (next s2)))))))) + ([& colls] + (lazy-seq + (let [ss (map seq colls)] + (when (every? identity ss) + (concat (map first ss) + (apply interleave (map next ss)))))))) + +(defn min + "Returns the smallest of all the arguments to this function. Assumes arguments are numeric." + ([x] x) + ([x y] (if (< x y) x y)) + ([x y & zs] (apply min (min x y) zs))) + +(defn max + "Returns the largest of all the arguments to this function. Assumes arguments are numeric." + ([x] x) + ([x y] (if (> x y) x y)) + ([x y & zs] (apply max (max x y) zs))) + +(defn take-nth + "Returns a lazy seq of every nth item in coll. Returns a stateful + transducer when no collection is provided." + ([n] + (fn [rf] + (let [ia (atom -1)] + (fn + ([] (rf)) + ([result] (rf result)) + ([result input] + (let [i (swap! ia inc)] + (if (zero? (rem i n)) + (rf result input) + result))))))) + ([n coll] + (lazy-seq + (when-let [s (seq coll)] + (cons (first s) (take-nth n (drop n s))))))) + +(defmacro loop + [bindings & body] + (let [vals (take-nth 2 (drop 1 bindings)) + bindings (take-nth 2 bindings) + binding-syms (map (fn [b] (if (symbol? b) b (gensym))) bindings) + binding-forms (transduce + (map (fn [bind] + (let [[b v s] bind] + (if (symbol? b) + [b v] + [s v b s])))) + concat + [] + (map vector bindings vals binding-syms))] + `(let ~(vec binding-forms) + (loop* ~(vec (interleave binding-syms binding-syms)) + (let ~(vec (interleave bindings binding-syms)) + ~@body))))) + +(extend -str Namespace + (fn [v] (str ""))) + +(extend -repr Namespace -str) + + +(defn bool? + "Returns true if x is a Bool." + [x] + (instance? Bool x)) + +(defmacro -> + {:doc "Threads `x` through `forms`, passing the result of one step as the first argument of the next." + :examples [["(-> 3 inc inc)" nil 5] + ["(-> \"James\" (str \" is \" \"awesome \") (str \"(and stuff)\" \"!\"))" nil "James is awesome (and stuff)!"]] + :signatures [[x & forms]] + :added "0.1"} + [x & forms] + (loop [x x, forms forms] + (if forms + (let [form (first forms) + threaded (if (seq? form) + (with-meta `(~(first form) ~x ~@(next form)) (meta form)) + (list form x))] + (recur threaded (next forms))) + x))) + +(defmacro ->> + {:doc "Threads `x` through `forms`, passing the result of one step as the last argument of the next." + :examples [["(->> \"James\" (str \"we \" \"like \") (str \"you \" \"know \" \"what? \"))" nil "you know what? we like James"] + ["(->> 5 (range) (map inc) seq)" nil (1 2 3 4 5)]] + :signatures [[x & forms]] + :added "0.1"} + [x & forms] + (loop [x x, forms forms] + (if forms + (let [form (first forms) + threaded (if (seq? form) + (with-meta `(~(first form) ~@(next form) ~x) (meta form)) + (list form x))] + (recur threaded (next forms))) + x))) + +(defmacro some-> + {:doc "When expr is not nil, threads it into the first form (via ->), + and when that result is not nil, through the next etc." + :signatures [[expr & forms]] + :added "0.1"} + [expr & forms] + (let [g (gensym) + steps (map (fn [step] `(if (nil? ~g) nil (-> ~g ~step))) + forms)] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro some->> + {:doc "When expr is not nil, threads it into the first form (via ->>), + and when that result is not nil, through the next etc." + :signatures [[x & forms]] + :added "0,1"} + [expr & forms] + (let [g (gensym) + steps (map (fn [step] `(if (nil? ~g) nil (->> ~g ~step))) + forms)] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro cond-> + {:added "0.1" + :signatures [[expr & clauses]] + :doc "Takes an expression and a set of test/form pairs. Threads expr (via ->) + through each form for which the corresponding test + expression is true. Note that, unlike cond branching, cond-> threading does + not short circuit after the first true test expression."} + [expr & clauses] + (assert (even? (count clauses))) + (let [g (gensym) + steps (map (fn [[test step]] `(if ~test (-> ~g ~step) ~g)) + (partition 2 clauses))] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro cond->> + {:doc "Takes an expression and a set of test/form pairs. Threads expr (via ->>) + through each form for which the corresponding test expression + is true. Note that, unlike cond branching, cond->> threading does not short circuit + after the first true test expression." + :signatures [[expr & clauses]] + :added "0.1"} + [expr & clauses] + (assert (even? (count clauses))) + (let [g (gensym) + steps (map (fn [[test step]] `(if ~test (->> ~g ~step) ~g)) + (partition 2 clauses))] + `(let [~g ~expr + ~@(interleave (repeat g) (butlast steps))] + ~(if (empty? steps) + g + (last steps))))) + +(defmacro as-> + {:doc "Binds name to expr, evaluates the first form in the lexical context + of that binding, then binds name to that result, repeating for each + successive form, returning the result of the last form." + :signatures [[expr name & forms]] + :added "0,1"} + [expr name & forms] + `(let [~name ~expr + ~@(interleave (repeat name) (butlast forms))] + ~(if (empty? forms) + name + (last forms)))) + +(defprotocol IComparable + (-compare [x y] + "Compares two objects. Returns 0 when x is equal to y, -1 when x + is logically smaller than y, and 1 when x is logically larger.")) + +(defn compare-numbers + [x y] + (cond + (> x y) 1 + (< x y) -1 + :else 0)) + +(defn compare-counted + [x y] + (if (= x y) + 0 + (let [min-length (min (count x) (count y))] + (loop [n 0] + (if (not= min-length n) + (let [diff (-compare (nth x n) + (nth y n))] + (if-not (zero? diff) + diff + (recur (inc n)))) + ;; We have compared all characters of the smallest string + ;; against the largest string. + ;; If equal lengths 0 otherwise -1 or 1 + (compare-numbers (count x) (count y))))))) + +(defn compare-named + [x y] + (if (= x y) + 0 + (compare-counted (str x) (str y)))) + +(extend-protocol ISeq + ISeqable + (-first [coll] (-first (seq coll))) + (-next [coll] (-next (seq coll)))) + +(extend-protocol IComparable + Number + (-compare [x y] + (if (number? y) + (compare-numbers x y) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Character + (-compare [x y] + (if (char? y) + (compare-numbers (int x) (int y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + PersistentVector + (-compare [x y] + (if (vector? y) + (compare-counted x y) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + String + (-compare [x y] + (if (string? y) + (compare-counted (str x) (str y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Keyword + (-compare [x y] + (if (keyword? y) + (compare-counted (str x) (str y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Symbol + (-compare [x y] + (if (symbol? y) + (compare-counted (str x) (str y)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)]))) + + Bool + (-compare [x y] + (if (bool? y) + (cond + (= x y) 0 + (and (true? x) (false? y)) 1 + :else -1)) + (throw [::ComparisonError (str "Cannot compare: " x " to " y)])) + + Nil + (-compare [x y] + (if (nil? y) + 0 + (throw [::ComparisonError (str "Cannot compare: " x " to " y)])))) + +(defn compare + "Compares two objects. Returns 0 when x is equal to y, -1 when x is + logically smaller than y, and 1 when x is logically larger. x must + implement IComparable." + [x y] + (if (satisfies? IComparable x) + (-compare x y) + (throw [::ComparisonError (str x " does not satisfy IComparable")]))) + +(defn vary-meta + {:doc "Returns x with meta data updated with the application of f and args to it. +ex: (vary-meta x assoc :foo 42)" + :signatures [[x f & args]] + :added "0.1"} + [x f & args] + (with-meta x (apply f (meta x) args))) + +(defn memoize + {:doc "Returns a memoized version of function f. The first call will + realize the return value and subsequent calls get the same value + from its cache." + :signatures [[f]] + :added "0.1"} + [f] + (let [cache (atom {})] + (fn [& args] + (let [argsv (vec args) + val (get @cache argsv ::not-found)] + (if (= val ::not-found) + (let [ret (apply f args)] + (swap! cache assoc argsv ret) + ret) + val))))) + +(deftype Iterate [f x] + IReduce + (-reduce [self rf init] + (loop [next (f x) + acc (rf init x)] + (if (reduced? acc) + @acc + (recur (f next) (rf acc next))))) + ISeq + (-seq [self] + (cons x (lazy-seq* (fn [] (->Iterate f (f x))))))) + +(defn iterate + {:doc "Returns a lazy sequence of x, (f x), (f (f x)) etc. f must be free of + side-effects" + :signatures [[f x]] + :added "0.1"} + [f x] + (->Iterate f x)) diff --git a/pixie/streams.pxi b/pixie/streams.pxi new file mode 100644 index 00000000..1bdf358f --- /dev/null +++ b/pixie/streams.pxi @@ -0,0 +1,21 @@ +(ns pixie.streams) + +(defprotocol IFlushableStream + (flush [this] "Flushes all buffers in this stream and applies writes to any parent streams")) + +(defprotocol IInputStream + (read [this buffer len] "Reads multiple bytes into a buffer, returns the number of bytes read")) + +(defprotocol IOutputStream + (write [this buffer])) + +(defprotocol IByteInputStream + (read-byte [this])) + +(defprotocol IByteOutputStream + (write-byte [this byte])) + +(defprotocol ISeekableStream + (position [this]) + (seek [this position]) + (rewind [this])) diff --git a/pixie/streams/utf8.pxi b/pixie/streams/utf8.pxi new file mode 100644 index 00000000..8e8ab95f --- /dev/null +++ b/pixie/streams/utf8.pxi @@ -0,0 +1,86 @@ +(ns pixie.streams.utf8 + (require pixie.streams :refer :all)) + +(defprotocol IUTF8OutputStream + (write-char [this char] "Write a single character to the UTF8 stream")) + +(defprotocol IUTF8InputStream + (read-char [this] "Read a single character from the UTF8 stream")) + +(deftype UTF8OutputStream [out] + IUTF8OutputStream + (write-char [this ch] + (let [ch (int ch)] + (cond + (<= ch 0x7F) (write-byte out ch) + (<= ch 0x7FF) (do (write-byte out (bit-or 0xC0 (bit-shift-right ch 6))) + (write-byte out (bit-or 0x80 (bit-and ch 0x3F)))) + (<= ch 0xFFFF) (do (write-byte out (bit-or 0xE0 (bit-shift-right ch 12))) + (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 6) 0x3F))) + (write-byte out (bit-or 0x80 (bit-and ch 0x3F)))) + (<= ch 0x1FFFFF) (do (write-byte out (bit-or 0xE0 (bit-shift-right ch 18))) + (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 12) 0x3F))) + (write-byte out (bit-or 0x80 (bit-and (bit-shift-right ch 6) 0x3F))) + (write-byte out (bit-or 0x80 (bit-and ch 0x3F)))) + :else (assert false (str "Cannot encode a UTF8 character of code " ch))))) + IDisposable + (-dispose! [this] + (dispose! out))) + +(deftype UTF8InputStream [in bad-char] + IUTF8InputStream + (read-char [this] + (when-let [byte (read-byte in)] + (let [[n bytes error?] + (cond + (>= 0x7F byte) [byte 1 false] + (= 0xC0 (bit-and byte 0xE0)) [(bit-and byte 31) 2 false] + (= 0xE0 (bit-and byte 0xF0)) [(bit-and byte 15) 3 false] + (= 0xF0 (bit-and byte 0xF8)) [(bit-and byte 7) 4 false] + (= 0xF8 (bit-and byte 0xF8)) [(bit-and byte 3) 5 true] + (= 0xFC (bit-and byte 0xFE)) [(bit-and byte 1) 6 true] + :else [n 1 true])] + (if error? + (or bad-char + (throw [::invalid-character (str "Invalid UTF8 character decoded: " n)])) + (loop [i (dec bytes) n n] + (if (pos? i) + (recur (dec i) + (bit-or (bit-shift-left n 6) + (bit-and (read-byte in) 0x3F))) + (char n))))))) + IDisposable + (-dispose! [this] + (dispose! in)) + IReduce + (-reduce [this f init] + (let [rrf (preserving-reduced f)] + (loop [acc init] + (if-let [char (read-char this)] + (let [result (rrf acc char)] + (if (not (reduced? result)) + (recur result) + @result)) + acc))))) + +(defn utf8-input-stream + "Creates a UTF8 decoder that reads characters from the given IByteInputStream. If a bad character is found + an error will be thrown, unless an optional bad-character marker character is provided." + ([i] + (->UTF8InputStream i nil)) + ([i bad-char] + (->UTF8InputStream i bad-char))) + +(defn utf8-output-stream + "Creates a UTF8 encoder that writes characters to the given IByteOutputStream." + [o] + (->UTF8OutputStream o)) + +(defn utf8-output-stream-rf [output-stream] + (let [fp (utf8-output-stream output-stream)] + (fn ([] 0) + ([_] (dispose! fp)) + ([_ chr] + (assert (char? chr)) + (write-char fp chr) + nil)))) diff --git a/pixie/streams/zlib.pxi b/pixie/streams/zlib.pxi new file mode 100644 index 00000000..bae72ade --- /dev/null +++ b/pixie/streams/zlib.pxi @@ -0,0 +1,218 @@ +(ns pixie.streams.zlib + (:require [pixie.streams.zlib.ffi :as zlib.ffi] + [pixie.ffi :as ffi] + [pixie.streams :refer :all])) + +(defprotocol IZStream + (version [this]) + (set-input! [this] + "Should be called before deflate! to set a new chunk of input + to deflate") + + (full-output? [this] + "Returns true if the ouput buffer is full of deflated (compressed) data") + + (reset-output-buffer! [this] + "Make the output buffer ready to be refilled with data") + + (consumed-input? [this] + "Returns true if zlib has finished reading the input-buffer") + + (set-output-buffer-count! [this] + "Set the buffers count so down stream can safely read the buffer") + + ;; Deflation + (deflate-init! [this opts] + "Set up the compression with desired parameters") + + ;; Inflation + (inflate-init! [this opts] + "Set up decompression with desired parameters") + + ;; In/deflate depending on what the stream has been + ;; initialized as + (flate! [this down-stream mode] + "Compress/decompress") + + (flate-end! + "Cleanup")) + +(defn handle-errors! [status] + (cond + (= status zlib.ffi/Z_ERRNO) + (throw [::Error "Something went wrong"]) + + (= status zlib.ffi/Z_STREAM_ERROR) + (throw [::Error "The stream doesn't appear to be a valid zlib/gzip stream"]) + + (= status zlib.ffi/Z_DATA_ERROR) + (throw [::Error "There was something wrong with the data"]) + + + ;; TODO go through the different status and show + ;; appropriate messages + (neg? status) + (throw [::Error "There was something wrong with the data"]))) + + +;; This wraps the C data structure and stores some information about +;; how its been initialized: :deflate or :inflate. +(deftype ZStream [z-stream inited] + IZStream + (version [this] + (zlib.ffi/zlibVersion)) + + (full-output? [this] + (zero? (get z-stream :avail_out))) + + (consumed-input? [this] + (zero? (get z-stream :avail_in))) + + (reset-output-buffer! [this output-buffer] + (ffi/set! z-stream :next_out output-buffer) + (ffi/set! z-stream :avail_out (buffer-capacity output-buffer))) + + (set-output-buffer-count! [this output-buffer] + (let [fill-count (- (buffer-capacity output-buffer) + (get z-stream :avail_out))] + (set-buffer-count! output-buffer fill-count))) + + (set-input! [this input-buffer] + (ffi/set! z-stream :next_in input-buffer) + (ffi/set! z-stream :avail_in (count input-buffer))) + + (deflate-init! [this opts] + (assert (nil? inited) "ZStream can only be initialized once.") + (let [status (zlib.ffi/deflateInit2_ + z-stream + (get opts :level zlib.ffi/Z_BEST_COMPRESSION) ;level + zlib.ffi/Z_DEFLATED ;method + (+ 15 16) ;window (set for gz header) + 8 ;memlevel + zlib.ffi/Z_DEFAULT_STRATEGY ;strategy + (version this) ;version + (ffi/struct-size zlib.ffi/z_stream))] + (assert (= zlib.ffi/Z_OK status) "Failed to initiate zstream") + (set-field! this :inited :deflate))) + + (inflate-init! [this opts] + (assert (nil? inited) "ZStream can only be initialized once.") + (let [status (zlib.ffi/inflateInit2_ + z-stream + (+ 15 16) ;window (set for gz header) + (version this) ;version + (ffi/struct-size zlib.ffi/z_stream))] + (assert (= zlib.ffi/Z_OK status) "Failed to initiate zstream") + (set-field! this :inited :inflate))) + + (flate! [this output-buffer mode] + (case inited + :inflate + (let [status (zlib.ffi/inflate z-stream mode)] + (handle-errors! status) + (set-output-buffer-count! this output-buffer) + status) + + :deflate + (let [status (zlib.ffi/deflate z-stream mode)] + (handle-errors! status) + (set-output-buffer-count! this output-buffer) + status) + + (assert false "ZStream must be initialized before calling flate!"))) + + (flate-end! [this] + (case inited + :inflate + (zlib.ffi/inflateEnd z-stream) + + :deflate + (zlib.ffi/deflateEnd z-stream) + + (assert false "ZStream must be initialized before calling flate-end!")))) + +(defn z-stream [] + (let [z-stream (zlib.ffi/z_stream)] + ;; Set all the callbacks to NULL so zlib uses its default ones. + (ffi/set! z-stream :avail_in 0) + (ffi/set! z-stream :zalloc nil) + (ffi/set! z-stream :opaque nil) + (ffi/set! z-stream :zfree nil) + (->ZStream z-stream nil))) + +(deftype GZInputStream + [up-stream input-buffer z-stream] + IDisposable + (-dispose! [this] + (flate-end! z-stream)) + + IInputStream + (read [this buffer len] + (set-buffer-count! buffer 0) + (reset-output-buffer! z-stream buffer) + ;; We keep reading from upstream until we have filled our output buffer + (loop [] + (when (consumed-input? z-stream) + ;; If z-stream has finished reading the input-buffer we last gave it, + ;; give it a new one. + (read up-stream input-buffer (buffer-capacity input-buffer)) + (set-input! z-stream input-buffer)) + (flate! z-stream buffer zlib.ffi/Z_NO_FLUSH) + (if-not (empty? input-buffer) + (if (full-output? z-stream) + ;; The buffer is now filled up + (count buffer) + ;; We can still do some more de/compression + (recur)) + 0)))) + +(deftype GZOutputStream + [down-stream output-buffer z-stream] + IOutputStream + (write [this input-buffer] + (set-input! z-stream input-buffer) + (loop [] + (reset-output-buffer! z-stream output-buffer) + (flate! z-stream output-buffer zlib.ffi/Z_NO_FLUSH) + (when-not (empty? output-buffer) + (write down-stream output-buffer)) + ;; if there is still more 'flating to do, do it. + (when (full-output? z-stream) + (recur)))) + + IFlushableStream + (flush [this] + (loop [] + (reset-output-buffer! z-stream output-buffer) + (flate! z-stream output-buffer zlib.ffi/Z_FINISH) + (write down-stream output-buffer) + (when (full-output? z-stream) + (recur))) + (when (satisfies? IFlushableStream down-stream) + (flush down-stream))) + + IDisposable + (-dispose! [this] + (flate-end! z-stream) + (when (satisfies? IDisposable down-stream) + (-dispose! down-stream)))) + +(defn compressing-output-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [down-stream output-buffer opts] + (->GZOutputStream down-stream output-buffer (deflate-init! (z-stream) opts))) + +(defn decompressing-output-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [down-stream output-buffer opts] + (->GZOutputStream down-stream output-buffer (inflate-init! (z-stream) opts))) + +(defn decompressing-input-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [up-stream input-buffer opts] + (->GZInputStream up-stream input-buffer (inflate-init! (z-stream) opts))) + +(defn compressing-input-stream + "Takes a down-stream IInputStream and a buffer to store compressed chunks" + [up-stream input-buffer opts] + (->GZInputStream up-stream input-buffer (deflate-init! (z-stream) opts))) diff --git a/pixie/streams/zlib/ffi.pxi b/pixie/streams/zlib/ffi.pxi new file mode 100644 index 00000000..8c8bcdbb --- /dev/null +++ b/pixie/streams/zlib/ffi.pxi @@ -0,0 +1,59 @@ +(ns pixie.streams.zlib.ffi + (:require [pixie.ffi-infer :as f] + [pixie.ffi :as ffi])) + +(f/with-config {:library "z" + :includes ["zlib.h"]} + (f/defcstruct z_stream [:next_in + :avail_in + :total_in + :next_out + :avail_out + :total_out + + :msg + :state + + :zalloc + :zfree + :opaque + + :data_type + :adler + + :reserved]) + + (f/defcfn zError) + (f/defcfn zlibVersion) + + ;; Inflating (decompressing) + (f/defcfn inflate) + (f/defcfn inflateEnd) + (f/defcfn inflateInit2_) + + ;; Defalting (compressing) + (f/defcfn deflate) + (f/defcfn deflateInit_) + (f/defcfn deflateInit2_) + (f/defcfn deflateEnd)) + +(def Z_OK 0) +(def Z_NO_FLUSH 0) +(def Z_PARTIAL_FLUSH 1) +(def Z_SYNC_FLUSH 2) +(def Z_FULL_FLUSH 3) +(def Z_FINISH 4) +(def Z_BLOCK 5) + +(def Z_ERRNO -1) +(def Z_STREAM_ERROR -2) +(def Z_DATA_ERROR -3) + +(def Z_NO_COMPRESSION 0) +(def Z_BEST_SPEED 1) +(def Z_BEST_COMPRESSION 9) +(def Z_DEFAULT_COMPRESSION -1) + +(def Z_DEFLATED 8) + +(def Z_DEFAULT_STRATEGY 0) diff --git a/pixie/string.pxi b/pixie/string.pxi new file mode 100644 index 00000000..cc72b96f --- /dev/null +++ b/pixie/string.pxi @@ -0,0 +1,133 @@ +(ns pixie.string + (:require [pixie.stdlib :as std] + [pixie.string.internal :as si])) + +; reexport native string functions +(def substring si/substring) +(def index-of (comp #(if (not= -1 %) %) si/index-of)) +(def split si/split) + +(defn split-lines + "Splits on \\n or \\r\\n, the two typical line breaks." + [s] + (when s (apply concat (map #(split % "\n") (split s "\r\n"))))) + +(def ends-with? si/ends-with) +(def starts-with? si/starts-with) + +(def trim si/trim) +(def triml si/triml) +(def trimr si/trimr) + +(def capitalize si/capitalize) +(def lower-case si/lower-case) +(def upper-case si/upper-case) + +; TODO: There should be locale-aware variants of these values +(def lower "abcdefghijklmnopqrstuvwxyz") +(def upper "ABCDEFGHIJKLMNOPQRSTUVWXYZ") +(def digits "0123456789") +(def punctuation "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~") +(def whitespace (str \space \newline \tab \backspace \formfeed \return)) +(def letters (str lower upper)) +(def printable (str letters digits punctuation whitespace)) +(def hexdigits "0123456789abcdefABCDEF") +(def octdigits "012345678") + +(defn trim-newline + "Replace all trailing newline characters (\\r and \\n) from the end of a string." + [s] + (loop [index (count s)] + (if (zero? index) + "" + (let [ch (nth s (dec index))] + (if (or (= ch \newline) (= ch \return)) + (recur (dec index)) + (substring s 0 index)))))) + +(defn replace + "Replace all occurrences of x in s with r." + [s x r] + (let [offset (if (zero? (count x)) (+ 1 (count r)) (count r))] + (loop [start 0 + s s] + (if-let [i (index-of s x start)] + (recur (+ i offset) (str (substring s 0 i) r (substring s (+ i (count x))))) + s)))) + +(defn replace-first + "Replace the first occurrence of x in s with r." + [s x r] + (if-let [i (index-of s x)] + (str (substring s 0 i) r (substring s (+ i (count x)))) + s)) + +(defn reverse + "Returns s with its characters reversed." + [s] + (when s + (apply str (std/reverse s)))) + +(defn join + {:doc "Join the elements of the collection using an optional separator" + :examples [["(require pixie.string :as s)"] + ["(s/join [1 2 3])" nil "123"] + ["(s/join \", \" [1 2 3])" nil "1, 2, 3"]]} + ([coll] (join "" coll)) + ([separator coll] + (loop [s (seq coll) + res ""] + (cond + (nil? s) res + (nil? (next s)) (str res (first s)) + :else (recur (next s) (str res (first s) separator)))))) + +(defn blank? + "True if s is nil, empty, or contains only whitespace." + [s] + (if s + (let [white (set whitespace)] + (every? white s)) + true)) + +(defn escape + "Return a new string, using cmap to escape each character ch + from s as follows: + + If (cmap ch) is nil, append ch to the new string. + If (cmap ch) is non-nil, append (str (cmap ch)) instead." + [s cmap] + (if (or (nil? s) + (nil? cmap)) + s + (apply str (map #(if-let [c (cmap %)] c %) + (vec s))))) + +(defmacro interp + ; TODO: This might merit special read syntax + {:doc "String interpolation." + :examples [["(require pixie.string :refer [interp])"] + ["(interp \"2 plus 2 is $(+ 2 2)$!\")" nil "2 plus 2 is 4!"] + ["(let [x \"locals\"] (interp \"You can use arbitrary forms; for example $x$\"))" + nil "You can use arbitrary forms; for example locals"] + ["(interp \"$$$$ is the escape for a literal $$\")" + nil "$$ is the escape for a literal $"] + ]} + [txt] + (loop [forms [], txt txt] + (cond + (empty? txt) `(str ~@ forms) + (starts-with? txt "$") + (let [pos (or (index-of txt "$" 1) + (throw "Unmatched $ in interp argument!")) + form-str (subs txt 1 pos) + form (if (empty? form-str) "$" + (read-string form-str)) + rest-str (subs txt (inc pos))] + (recur (conj forms form) rest-str)) + :else + (let [pos (or (index-of txt "$") + (count txt)) + form (subs txt 0 pos) + rest-str (subs txt pos)] + (recur (conj forms form) rest-str))))) diff --git a/pixie/system.pxi b/pixie/system.pxi new file mode 100644 index 00000000..21306122 --- /dev/null +++ b/pixie/system.pxi @@ -0,0 +1,13 @@ +(ns pixie.system + (:require [pixie.ffi-infer :as i])) + +(i/with-config {:library "c" :imports ["stdio.h"]} + (i/defconst STDIN_FILENO) + (i/defconst STDOUT_FILENO) + (i/defconst STDERR_FILENO)) + +(def fdopen (ffi-fn libc "fdopen" [CInt CCharP] CVoidP)) + +(def stdin STDIN_FILENO) +(def stderr STDOUT_FILENO) +(def stdout STDERR_FILENO) diff --git a/pixie/test.pxi b/pixie/test.pxi new file mode 100644 index 00000000..2f622977 --- /dev/null +++ b/pixie/test.pxi @@ -0,0 +1,101 @@ +(ns pixie.test + (:require [pixie.string :as s] + [pixie.fs :as fs])) + +(def tests (atom {})) + +(def ^:dynamic *stats*) + +(def ^:dynamic *current-test*) + + +(defmacro deftest [nm & body] + `(do (defn ~nm [] + (println "Running:" (str (namespace (var ~nm)) "/" (name (var ~nm)))) + (try + ~@body + (swap! *stats* update-in [:pass] (fnil inc 0)) + (catch ex + (println "while running" ~(name nm) " " (quote (do ~@body))) + + (swap! *stats* update-in [:fail] (fnil inc 0)) + (println (str ex)) + (swap! *stats* update-in [:errors] (fnil conj []) ex)))) + (swap! tests assoc (symbol (str (namespace (var ~nm)) "/" (name (var ~nm)))) ~nm))) + + + +(defn run-tests [& args] + (push-binding-frame!) + (set! (var *stats*) (atom {:fail 0 :pass 0})) + + (let [match (or (first args) "") + tests (transduce (comp (filter #(>= (s/index-of (str (key %1)) match) 0)) + (map val)) + conj + @tests)] + (println "Running:" (count tests) "tests") + (foreach [test tests] + (test))) + + (let [stats @*stats*] + (println stats) + (pop-binding-frame!) + stats)) + +(defn load-all-tests [] + (println "Looking for tests...") + (let [dirs (distinct (map fs/dir @load-paths)) + pxi-files (->> dirs + (mapcat fs/walk-files) + (filter #(fs/extension? % "pxi")) + (filter #(s/starts-with? (fs/basename %) "test-")) + (distinct))] + (foreach [file pxi-files] + (println "Loading " file) + (load-file (fs/abs file))))) + + +(defmacro assert= [x y] + `(let* [xr# ~x + yr# ~y] + (assert (= xr# yr#) (str (show '~x xr#) " != " (show '~y yr#))))) + +(defmacro assert-throws? + ([body] + `(let [exn# (try (do ~body nil) (catch e# e#))] + (assert (not (nil? exn#)) + (str "Expected " (pr-str (quote ~body)) " to throw an exception")) + exn#)) + ([klass body] + `(let [exn# (assert-throws? ~body)] + (assert (= (type exn#) ~klass) + (str "Expected " (pr-str (quote ~body)) + " to throw exception of class " (pr-str ~klass) + " but got " (pr-str (type exn#)))) + exn#)) + ([klass msg body] + `(let [exn# (assert-throws? ~klass ~body)] + (assert (= (ex-msg exn#) ~msg) + (str "Expected " (pr-str (quote ~body)) + " to throw exception with message " (pr-str ~msg) + " but got " (pr-str (ex-msg exn#))))))) + +(defmacro assert [x] + `(let [x# ~x] + (assert x# (str '~x " is " x#)))) + + + +(defn show + ([orig res] + (if (= orig res) + (pr-str orig) + (str (pr-str orig) " = " (pr-str res))))) + + +(defmacro assert-table [pattern expr & vals] + (let [parted (partition (count pattern) vals)] + `(do ~@(for [fact parted] + `(let [~@(interleave pattern fact)] + ~expr))))) diff --git a/pixie/time.pxi b/pixie/time.pxi new file mode 100644 index 00000000..a045b14f --- /dev/null +++ b/pixie/time.pxi @@ -0,0 +1,9 @@ +(ns pixie.time + (:require [pixie.uv :as uv])) + +(defmacro time + [body] + `(let [start# (uv/uv_hrtime) + return# ~body] + (prn (str "Elapsed time: " (/ (- (uv/uv_hrtime) start#) 1000000.0) " ms")) + return#)) diff --git a/pixie/uv.pxi b/pixie/uv.pxi new file mode 100644 index 00000000..0bcf4fba --- /dev/null +++ b/pixie/uv.pxi @@ -0,0 +1,270 @@ +(ns pixie.uv + (:require [pixie.ffi :as ffi] + [pixie.ffi-infer :as f])) + +(f/with-config {:library "uv" + :includes ["uv.h"]} + (f/defconst UV_RUN_DEFAULT) + (f/defconst UV_RUN_ONCE) + (f/defconst UV_RUN_NOWAIT) + + (f/defcfn uv_close) + (f/defccallback uv_close_cb) + + + (f/defcstruct uv_loop_t []) + + (f/defcfn uv_loop_init) + (f/defcfn uv_loop_close) + (f/defcfn uv_default_loop) + + (f/defcfn uv_run) + (f/defcfn uv_loop_alive) + (f/defcfn uv_unref) + (f/defcfn uv_ref) + (f/defcfn uv_stop) + (f/defcfn uv_loop_size) + (f/defcfn uv_backend_fd) + (f/defcfn uv_backend_timeout) + (f/defcfn uv_now) + (f/defcfn uv_update_time) + (f/defcfn uv_walk) + + (f/defccallback uv_alloc_cb) + + (f/defcstruct uv_connect_t [:handle]) + (f/defcstruct uv_stream_t []) + + (f/defcfn uv_read_start) + (f/defccallback uv_read_cb) + + (f/defcfn uv_write) + (f/defcstruct uv_write_t []) + (f/defccallback uv_write_cb) + + + ;; Timer + + (f/defcstruct uv_timer_t []) + (f/defccallback uv_timer_cb) + (f/defcfn uv_timer_init) + (f/defcfn uv_timer_start) + (f/defcfn uv_timer_stop) + (f/defcfn uv_timer_again) + (f/defcfn uv_timer_set_repeat) + (f/defcfn uv_timer_get_repeat) + + ;; Time + (f/defcfn uv_hrtime) + + + ;; Filesystem + + (f/defcstruct uv_fs_t [:loop + :fs_type + :path + :result + :ptr + :statbuf.st_size + :statbuf.st_mode]) + (f/defcstruct uv_timespec_t [:tv_sec + :tv_nsec]) + (f/defcstruct uv_stat_t [:st_dev + :st_mode + :st_nlink + :st_uid + :st_gid + :st_rdev + :st_ino + :st_size + :st_blksize + :st_blocks + :st_flags + :st_gen + :st_atim.tv_sec + :st_atim.tv_nsec + :st_mtim.tv_sec + :st_mtim.tv_nsec + :st_ctim.tv_sec + :st_ctim.tv_nsec + :st_birthtim.tv_sec + :st_birthtim.tv_nsec]) + + (f/defconst UV_FS_UNKNOWN) + (f/defconst UV_FS_CUSTOM) + (f/defconst UV_FS_OPEN) + (f/defconst UV_FS_CLOSE) + (f/defconst UV_FS_READ) + (f/defconst UV_FS_WRITE) + (f/defconst UV_FS_SENDFILE) + (f/defconst UV_FS_STAT) + (f/defconst UV_FS_LSTAT) + (f/defconst UV_FS_FSTAT) + (f/defconst UV_FS_FTRUNCATE) + (f/defconst UV_FS_UTIME) + (f/defconst UV_FS_FUTIME) + (f/defconst UV_FS_ACCESS) + (f/defconst UV_FS_CHMOD) + (f/defconst UV_FS_FCHMOD) + (f/defconst UV_FS_FSYNC) + (f/defconst UV_FS_FDATASYNC) + (f/defconst UV_FS_UNLINK) + (f/defconst UV_FS_RMDIR) + (f/defconst UV_FS_MKDIR) + (f/defconst UV_FS_MKDTEMP) + (f/defconst UV_FS_RENAME) + (f/defconst UV_FS_SCANDIR) + (f/defconst UV_FS_LINK) + (f/defconst UV_FS_SYMLINK) + (f/defconst UV_FS_READLINK) + (f/defconst UV_FS_CHOWN) + (f/defconst UV_FS_FCHOWN) + + (f/defconst UV_DIRENT_UNKNOWN) + (f/defconst UV_DIRENT_FILE) + (f/defconst UV_DIRENT_DIR) + (f/defconst UV_DIRENT_LINK) + (f/defconst UV_DIRENT_FIFO) + (f/defconst UV_DIRENT_SOCKET) + (f/defconst UV_DIRENT_CHAR) + (f/defconst UV_DIRENT_BLOCK) + + (f/defconst UV_EOF) + + (f/defcstruct uv_dirent_t [:name + :type]) + + (f/defcstruct uv_buf_t [:base :len]) + + (f/defcfn uv_fs_req_cleanup) + (f/defcfn uv_fs_close) + (f/defcfn uv_fs_open) + + (f/defccallback uv_fs_cb) + + (f/defcfn uv_fs_unlink) + (f/defcfn uv_fs_write) + (f/defcfn uv_fs_read) + (f/defcfn uv_fs_mkdir) + (f/defcfn uv_fs_mkdtemp) + (f/defcfn uv_fs_rmdir) + (f/defcfn uv_fs_scandir) + (f/defcfn uv_fs_scandir_next) + (f/defcfn uv_fs_stat) + (f/defcfn uv_fs_fstat) + (f/defcfn uv_fs_lstat) + (f/defcfn uv_fs_rename) + (f/defcfn uv_fs_fsync) + (f/defcfn uv_fs_fdatasync) + (f/defcfn uv_fs_ftruncate) + (f/defcfn uv_fs_sendfile) + (f/defcfn uv_fs_access) + (f/defcfn uv_fs_chmod) + (f/defcfn uv_fs_fchmod) + (f/defcfn uv_fs_utime) + (f/defcfn uv_fs_futime) + (f/defcfn uv_fs_link) + (f/defcfn uv_fs_symlink) + (f/defcfn uv_fs_readlink) + (f/defcfn uv_fs_chown) + (f/defcfn uv_fs_fchown) + + (f/defconst O_RDONLY) + (f/defconst O_WRONLY) + (f/defconst O_RDWR) + + (f/defconst O_APPEND) + (f/defconst O_ASYNC) + (f/defconst O_CREAT) + + (f/defconst S_IRUSR) + (f/defconst S_IRWXU) + (f/defconst S_IWUSR) + + + ; ERRNO + (f/defconst UV_E2BIG) + (f/defconst UV_EACCES) + + (f/defcfn uv_err_name) + + ; async + (f/defcstruct uv_async_t []) + (f/defccallback uv_async_cb) + (f/defcfn uv_async_init) + (f/defcfn uv_async_send) + + ; TTY + (f/defcfn uv_tty_init) + (f/defconst UV_TTY_MODE_NORMAL) + (f/defcfn uv_guess_handle) + ;(f/defcfn uv_tty_set_mode) + (f/defcstruct uv_tty_t []) + (f/defconst UV_TTY) + + + ; TCP Networking + (f/defcstruct uv_tcp_t []) + (f/defc-raw-struct sockaddr_in []) + (f/defcfn uv_tcp_init) + (f/defcfn uv_ip4_addr) + (f/defcfn uv_tcp_bind) + (f/defcfn uv_listen) + (f/defcfn uv_accept) + (f/defcfn uv_tcp_connect) + (f/defcfn uv_tcp_keepalive) + (f/defcfn uv_read_start) + (f/defcfn uv_read_stop) + + (f/defccallback uv_connection_cb) + (f/defccallback uv_connect_cb) + + (f/defccallback uv_alloc_cb) + (f/defccallback uv_read_cb)) + + +(defn new-fs-buf [size] + (let [b (buffer size) + bt (uv_buf_t)] + (pixie.ffi/set! bt :base b) + (pixie.ffi/set! bt :len size) + bt)) + + +(defn throw-on-error [result] + (if (neg? result) + (throw [::UVException (str "UV Error: " (uv_err_name result))]) + result)) + +(defmacro defuvfsfn + ([nm args return] + (defuvfsfn nm (symbol (str "pixie.uv/uv_" (name nm))) args return)) + ([nm uv-fn args return] + `(defn ~nm ~args + (let [f (fn [k#] + (let [cb# (atom nil)] + (reset! cb# (ffi/ffi-prep-callback uv_fs_cb + (fn [req#] + (try + (pixie.stacklets/run-and-process k# (~return (pixie.ffi/cast req# uv_fs_t))) + (uv_fs_req_cleanup req#) + (-dispose! @cb#) + (catch e (println e)))))) + (~uv-fn + (uv_default_loop) + (uv_fs_t) + ~@args + @cb#)))] + (pixie.stacklets/call-cc f))))) + +(defn -prep-uv-buffer-fn [buf read-bytes] + (ffi/ffi-prep-callback + uv_alloc_cb + (fn [handle suggested-size uv-buf] + (try + (let [casted (ffi/cast uv-buf uv_buf_t)] + (ffi/set! casted :base buf) + (ffi/set! casted :len (min suggested-size + (buffer-capacity buf) + read-bytes))) + (catch ex (println ex)))))) diff --git a/pixie/vm/array.py b/pixie/vm/array.py index 0a9f17e0..4ca97813 100644 --- a/pixie/vm/array.py +++ b/pixie/vm/array.py @@ -1,18 +1,19 @@ import pixie.vm.rt as rt import pixie.vm.object as object +from pixie.vm.object import affirm from pixie.vm.code import extend, as_var from pixie.vm.numbers import Integer from pixie.vm.primitives import nil import pixie.vm.stdlib as proto import rpython.rlib.jit as jit +from rpython.rtyper.lltypesystem import lltype +from rpython.rlib.rarithmetic import intmask UNROLL_IF_SMALLER_THAN = 8 class Array(object.Object): _type = object.Type(u"pixie.stdlib.Array") - __immutable_fields__ = ["_list[*]"] - def type(self): - return Array._type + _immutable_fields_ = ["_list"] def __init__(self, lst): self._list = lst @@ -25,6 +26,8 @@ def reduce_small(self, f, init): init = f.invoke([init, self._list[x]]) return init + def list(self): + return self._list def reduce_large(self, f, init): for x in range(len(self._list)): @@ -33,30 +36,202 @@ def reduce_large(self, f, init): init = f.invoke([init, self._list[x]]) return init - - - - - @extend(proto._count, Array) def _count(self): assert isinstance(self, Array) - return rt.wrap(len(self._list)) + return rt.wrap(len(self.list())) @extend(proto._nth, Array) def _nth(self, idx): assert isinstance(self, Array) - return self._list[idx.int_val()] + ival = idx.int_val() + if ival < len(self.list()): + return self.list()[ival] + else: + affirm(False, u"Index out of Range") + +@extend(proto._nth_not_found, Array) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, Array) + ival = idx.int_val() + if ival < len(self.list()): + return self.list()[ival] + else: + return not_found @extend(proto._reduce, Array) def reduce(self, f, init): assert isinstance(self, Array) - if len(self._list) > UNROLL_IF_SMALLER_THAN: + if len(self.list()) > UNROLL_IF_SMALLER_THAN: return self.reduce_large(f, init) return self.reduce_small(f, init) +@extend(proto._seq, Array) +def _seq(self): + assert isinstance(self, Array) + if rt.count(self) > 0: + return ArraySeq(0, self) + else: + return nil + +class ArraySeq(object.Object): + _type = object.Type(u"pixie.stdlib.ArraySeq") + _immutable_fields_ = ["_idx", "_w_array"] + + def __init__(self, idx, array): + self._idx = idx + self._w_array = array + + def first(self): + return rt.nth(self._w_array, rt.wrap(self._idx)) + + def next(self): + if self._idx < rt.count(self._w_array) - 1: + return ArraySeq(self._idx + 1, self._w_array) + else: + return nil + + def reduce(self, f, init): + for x in range(self._idx, rt.count(self._w_array)): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, rt.nth(self._w_array, rt.wrap(x))]) + return init + +@extend(proto._first, ArraySeq) +def _first(self): + assert isinstance(self, ArraySeq) + return self.first() + +@extend(proto._next, ArraySeq) +def _next(self): + assert isinstance(self, ArraySeq) + return self.next() + +@extend(proto._seq, ArraySeq) +def _seq(self): + assert isinstance(self, ArraySeq) + return self + +@extend(proto._reduce, ArraySeq) +def _reduce(self, f, init): + assert isinstance(self, ArraySeq) + return self.reduce(f, init) def array(lst): assert isinstance(lst, list) return Array(lst) +@as_var("aget") +def aget(self, idx): + affirm(isinstance(self, Array), u"aget expects an Array as the first argument") + affirm(isinstance(idx, Integer), u"aget expects an Integer as the second argument") + return self.list()[idx.int_val()] + +@as_var("aset") +def aset(self, idx, val): + affirm(isinstance(self, Array), u"aset expects an Array as the first argument") + affirm(isinstance(idx, Integer), u"aset expects an Integer as the second argument") + self.list()[idx.int_val()] = val + return val + +@as_var("aslice") +def aslice(self, offset): + affirm(isinstance(self, Array), u"aset expects an Array as the first argument") + affirm(isinstance(offset, Integer), u"aset expects an Integer as the second argument") + + offset = offset.int_val() + if offset >= 0: + return Array(self.list()[offset:]) + else: + rt.throw(rt.wrap(u"offset must be an Integer >= 0")) + +@as_var("aconcat") +def aconcat(self, other): + affirm(isinstance(self, Array) and isinstance(other, Array), + u"aconcat expects 2 Arrays") + return Array(self.list() + other.list()) + +@as_var("alength") +def alength(self): + affirm(isinstance(self, Array), u"alength expects an Array") + + return rt.wrap(len(self.list())) + +@as_var("make-array") +def make_array(l): + affirm(isinstance(l, Integer), u"l must be an Integer") + return Array([nil] * l.int_val()) + + +# ByteArray +ARRAY_OF_UCHAR = lltype.Array(lltype.Char) + +class ByteArray(object.Object): + _type = object.Type(u"pixie.stdlib.ByteArray") + + def __init__(self, size): + self._cnt = size + self._buffer = lltype.malloc(ARRAY_OF_UCHAR, size, flavor="raw") + for x in range(size): + self._buffer[x] = chr(0) + + + def __del__(self): + lltype.free(self._buffer, flavor="raw") + + + @jit.unroll_safe + def reduce_small(self, f, init): + for x in range(self._cnt): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, rt.wrap(ord(self._buffer[x]))]) + return init + + + def reduce_large(self, f, init): + for x in range(self._cnt): + if rt.reduced_QMARK_(init): + return rt.deref(init) + init = f.invoke([init, rt.wrap(ord(self._buffer[x]))]) + return init + + +@as_var("byte-array") +def _byte_array(size): + assert isinstance(size, Integer) + v = size.r_uint_val() + return ByteArray(v) + +@extend(proto._reduce, ByteArray) +def _reduce(self, f, init): + assert isinstance(self, ByteArray) + if self._cnt > UNROLL_IF_SMALLER_THAN: + return self.reduce_large(f, init) + return self.reduce_small(f, init) + +@extend(proto._nth, ByteArray) +def _nth(self, idx): + assert isinstance(self, ByteArray) + affirm(isinstance(idx, Integer), u"Index must be an integer") + ival = idx.r_uint_val() + if 0 <= ival < self._cnt: + return rt.wrap(ord(self._buffer[ival])) + + return affirm(False, u"Index out of Range") + +@extend(proto._nth_not_found, ByteArray) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, ByteArray) + affirm(isinstance(idx, Integer), u"Index must be an integer") + ival = idx.r_uint_val() + if 0 <= ival < self._cnt: + return rt.wrap(ord(self._buffer[ival])) + + return not_found + +@extend(proto._count, ByteArray) +def _count(self): + assert isinstance(self, ByteArray) + return rt.wrap(intmask(self._cnt)) diff --git a/pixie/vm/atom.py b/pixie/vm/atom.py index 80a1ab9a..28254673 100644 --- a/pixie/vm/atom.py +++ b/pixie/vm/atom.py @@ -1,32 +1,45 @@ -import pixie.vm.rt as rt import pixie.vm.object as object from pixie.vm.code import extend, as_var from pixie.vm.primitives import nil import pixie.vm.stdlib as proto -import pixie.vm.stdlib as proto class Atom(object.Object): _type = object.Type(u"pixie.stdlib.Atom") - def type(self): - return Atom._type - def __init__(self, boxed_value): - self._boxed_value = boxed_value + def with_meta(self, meta): + return Atom(self._boxed_value, meta) + def meta(self): + return self._meta + + def __init__(self, boxed_value, meta=nil): + self._boxed_value = boxed_value + self._meta = meta @extend(proto._reset_BANG_, Atom) def _reset(self, v): assert isinstance(self, Atom) self._boxed_value = v - return self + return v + @extend(proto._deref, Atom) def _deref(self): assert isinstance(self, Atom) return self._boxed_value +@extend(proto._meta, Atom) +def _meta(self): + assert isinstance(self, Atom) + return self.meta() + +@extend(proto._with_meta, Atom) +def _with_meta(self, meta): + assert isinstance(self, Atom) + return self.with_meta(meta) + @as_var("atom") def atom(val=nil): return Atom(val) diff --git a/pixie/vm/bits.py b/pixie/vm/bits.py new file mode 100644 index 00000000..d0688672 --- /dev/null +++ b/pixie/vm/bits.py @@ -0,0 +1,91 @@ +from pixie.vm.code import as_var +from pixie.vm.object import affirm + +from pixie.vm.numbers import Integer +from rpython.rlib.rarithmetic import intmask + +import pixie.vm.rt as rt + +@as_var("bit-clear") +def bit_clear(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() & ~(1 << n.int_val())) + +@as_var("bit-set") +def bit_set(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() | (1 << n.int_val())) + +@as_var("bit-flip") +def bit_flip(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() ^ (1 << n.int_val())) + +@as_var("bit-not") +def bit_not(x): + affirm(isinstance(x, Integer), u"x must be an Integer") + return rt.wrap(~x.int_val()) + +@as_var("bit-test") +def bit_test(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap((x.int_val() & (1 << n.int_val())) != 0) + +@as_var("bit-and") +def bit_and(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() & y.int_val()) + +@as_var("bit-and-not") +def bit_and_not(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() & ~y.int_val()) + +@as_var("bit-or") +def bit_or(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() | y.int_val()) + +@as_var("bit-xor") +def bit_xor(x, y): + affirm(isinstance(x, Integer) and isinstance(y, Integer), u"x and y must be Integers") + return rt.wrap(x.int_val() ^ y.int_val()) + +@as_var("bit-shift-left") +def bit_shift_left(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() << n.int_val()) + +@as_var("bit-shift-right") +def bit_shift_right(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(x.int_val() >> n.int_val()) + +@as_var("unsigned-bit-shift-right") +def unsigned_bit_shift_right(x, n): + affirm(isinstance(x, Integer) and isinstance(n, Integer), u"x and n must be Integers") + return rt.wrap(intmask(x.r_uint_val() >> n.int_val())) + +digits = "0123456789abcdefghijklmnopqrstuvwxyz" + +@as_var("bit-str") +def bit_str(x, shift): + affirm(isinstance(x, Integer) and isinstance(shift, Integer), u"x and shift must be Integers") + x = x.int_val() + shift = shift.int_val() + + buf = ['_'] * 32 + char_pos = 32 + radix = 1 << shift + mask = radix - 1 + while True: + char_pos -= 1 + buf[char_pos] = digits[x & mask] + x = x >> shift + if x == 0: + break + + res = "" + for i in range(char_pos, char_pos + 32 - char_pos): + res += buf[i] + return rt.wrap(res) diff --git a/pixie/vm/bootstrap.py b/pixie/vm/bootstrap.py index 50831235..10b9f49b 100644 --- a/pixie/vm/bootstrap.py +++ b/pixie/vm/bootstrap.py @@ -1,16 +1,9 @@ -from pixie.vm.stacklet import with_stacklets -import pixie.vm.stacklet as stacklet from pixie.vm.code import wrap_fn @wrap_fn def bootstrap(): import pixie.vm.rt as rt - from pixie.vm.string import String assert False - rt.load_file(rt.wrap(u"pixie/stdlib.lisp")) + rt.load_ns(rt.wrap(u"pixie/stdlib.pxi")) -# run bootstrap -#with_stacklets(bootstrap) -# reset the stacklet state so we can translate with different settings -stacklet.global_state = stacklet.GlobalState() \ No newline at end of file diff --git a/pixie/vm/c_api.py b/pixie/vm/c_api.py new file mode 100644 index 00000000..a050a070 --- /dev/null +++ b/pixie/vm/c_api.py @@ -0,0 +1,20 @@ +from rpython.rlib.entrypoint import entrypoint_highlevel +from rpython.rtyper.lltypesystem import rffi, lltype +from rpython.rtyper.lltypesystem.lloperation import llop + +@entrypoint_highlevel('main', [rffi.CCHARP], c_name='pixie_init') +def pypy_execute_source(ll_progname): + from target import init_vm + progname = rffi.charp2str(ll_progname) + init_vm(progname) + res = 0 + return rffi.cast(rffi.INT, res) + +@entrypoint_highlevel('main', [rffi.CCHARP], c_name='pixie_execute_source') +def pypy_execute_source(ll_source): + from target import EvalFn, run_with_stacklets + source = rffi.charp2str(ll_source) + f = EvalFn(source) + run_with_stacklets.invoke([f]) + res = 0 + return rffi.cast(rffi.INT, res) diff --git a/pixie/vm/code.py b/pixie/vm/code.py index e3f9545f..5a54ea25 100644 --- a/pixie/vm/code.py +++ b/pixie/vm/code.py @@ -1,9 +1,11 @@ py_object = object import pixie.vm.object as object -from pixie.vm.object import affirm -from pixie.vm.primitives import nil, true, false +from pixie.vm.object import affirm, runtime_error +from pixie.vm.primitives import nil, false from rpython.rlib.rarithmetic import r_uint -from rpython.rlib.jit import elidable, elidable_promote, promote +from rpython.rlib.listsort import TimSort +from rpython.rlib.jit import elidable_promote, promote +from rpython.rlib.objectmodel import we_are_translated import rpython.rlib.jit as jit import pixie.vm.rt as rt @@ -23,13 +25,14 @@ "POP", "DEREF_VAR", "INSTALL", - "RECUR", "LOOP_RECUR", "ARG", "PUSH_SELF", "POP_UP_N", "MAKE_MULTI_ARITY", - "MAKE_VARIADIC"] + "MAKE_VARIADIC", + "YIELD", + "PUSH_NS"] for x in range(len(BYTECODES)): globals()[BYTECODES[x]] = r_uint(x) @@ -46,6 +49,7 @@ def resize_list(lst, new_size): i += 1 return new_list + @jit.unroll_safe def list_copy(from_lst, from_loc, to_list, to_loc, count): from_loc = r_uint(from_loc) @@ -54,10 +58,11 @@ def list_copy(from_lst, from_loc, to_list, to_loc, count): i = r_uint(0) while i < count: - to_list[to_loc + i] = from_lst[from_loc+i] + to_list[to_loc + i] = from_lst[from_loc + i] i += 1 return to_list + @jit.unroll_safe def slice_to_end(from_list, start_pos): start_pos = r_uint(start_pos) @@ -66,12 +71,14 @@ def slice_to_end(from_list, start_pos): list_copy(from_list, start_pos, new_lst, 0, items_to_copy) return new_lst + @jit.unroll_safe def slice_from_start(from_list, count, extra=r_uint(0)): new_lst = [None] * (count + extra) list_copy(from_list, 0, new_lst, 0, count) return new_lst + # class TailCall(object.Object): # _type = object.Type("TailCall") # __immutable_fields_ = ["_f", "_args"] @@ -84,19 +91,29 @@ def slice_from_start(from_list, count, extra=r_uint(0)): class BaseCode(object.Object): + _immutable_fields_ = ["_meta", "_name"] def __init__(self): assert isinstance(self, BaseCode) + self._name = u"unknown" self._is_macro = False + self._meta = nil + + def meta(self): + return self._meta + + def with_meta(self, meta): + assert false, "not implemented" + + def name(self): + return self._name def set_macro(self): self._is_macro = True def is_macro(self): + assert isinstance(self, BaseCode) return self._is_macro - def _invoke(self, args): - raise NotImplementedError() - def get_consts(self): raise NotImplementedError() @@ -107,24 +124,40 @@ def get_bytecode(self): def stack_size(self): return 0 - def invoke(self, args): - result = self._invoke(args) - return result - + def invoke_with(self, args, this_fn): + return self.invoke(args) + +def join_last(words, sep): + """ + Joins by commas and uses 'sep' on last word. + + Eg. join_last(['dog', 'cat', 'rat'] , 'and') = 'dog, cat and rat' + """ + if len(words) == 1: + return words[0] + else: + if len(words) == 2: + s = words[0] + u" " + sep + u" " + words[1] + else: + s = u", ".join(words[0:-1]) + s += u" " + sep + u" " + words[-1] + return s class MultiArityFn(BaseCode): _type = object.Type(u"pixie.stdlib.MultiArityFn") _immutable_fields_ = ["_arities[*]", "_required_arity", "_rest_fn"] - def type(self): - return MultiArityFn._type - - def __init__(self, arities, required_arity=0, rest_fn=None): + def __init__(self, name, arities, required_arity=0, rest_fn=None, meta=nil): BaseCode.__init__(self) + self._name = name self._arities = arities self._required_arity = required_arity self._rest_fn = rest_fn + self._meta = meta + + def with_meta(self, meta): + return MultiArityFn(self._name, self._arities, self._required_arity, self._rest_fn, meta) @elidable_promote() def get_fn(self, arity): @@ -135,63 +168,85 @@ def get_fn(self, arity): return self._rest_fn acc = [] - for x in self._arities: + sorted = TimSort(self.get_arities()) + sorted.sort() + for x in sorted.list: acc.append(unicode(str(x))) if self._rest_fn: - acc.append(u" or more") + acc.append(unicode(str(self._rest_fn.required_arity())) + u"+") - affirm(False, u"Wrong number of args to fn: got " + unicode(str(arity)) + u" expected " + u",".join(acc)) + runtime_error(u"Wrong number of arguments " + unicode(str(arity)) + u" for function '" + unicode(self._name) + u"'. Expected " + join_last(acc, u"or"), + u"pixie.stdlib/InvalidArityException") - def _invoke(self, args): - return self.get_fn(len(args)).invoke(args) + def get_arities(self): + return self._arities.keys() + def invoke(self, args): + return self.invoke_with(args, self) + def invoke_with(self, args, self_fn): + return self.get_fn(len(args)).invoke_with(args, self_fn) class NativeFn(BaseCode): """Wrapper for a native function""" - _type = object.Type(u"NativeFn") + _type = object.Type(u"pixie.stdlib.NativeFn") - def __init__(self): + def __init__(self, doc=None): BaseCode.__init__(self) - def type(self): - return NativeFn._type - - def _invoke(self, args): + def invoke(self, args): return self.inner_invoke(args) def inner_invoke(self, args): raise NotImplementedError() + def invoke_with(self, args, this_fn): + return self.invoke(args) + class Code(BaseCode): """Interpreted code block. Contains consts and """ - _type = object.Type(u"Code") - __immutable_fields__ = ["_consts[*]", "_bytecode", "_stack_size"] - - def type(self): - return Code._type + _type = object.Type(u"pixie.stdlib.Code") + _immutable_fields_ = ["_arity", "_consts[*]", "_bytecode", "_stack_size", "_meta", "_debug_points"] - def __init__(self, name, bytecode, consts, stack_size, debug_points): + def __init__(self, name, arity, bytecode, consts, stack_size, debug_points, meta=nil): BaseCode.__init__(self) + self._arity = arity self._bytecode = bytecode self._consts = consts self._name = name self._stack_size = stack_size self._debug_points = debug_points + self._meta = meta + + def with_meta(self, meta): + return Code(self._name, self._arity, self._bytecode, self._consts, self._stack_size, self._debug_points, meta=meta) def get_debug_points(self): return self._debug_points - def _invoke(self, args): + def invoke(self, args): + if len(args) == self.get_arity(): + return self.invoke_with(args, self) + else: + runtime_error(u"Invalid number of arguments " + unicode(str(len(args))) + + u" for function '" + unicode(str(self._name)) + u"'. Expected " + + unicode(str(self.get_arity())), + u":pixie.stdlib/InvalidArityException") + + def invoke_with(self, args, this_fn): try: - return interpret(self, args) + return interpret(self, args, self_obj=this_fn) except object.WrappedException as ex: ex._ex._trace.append(object.PixieCodeInfo(self._name)) raise + @elidable_promote() + def get_arity(self): + return self._arity + @elidable_promote() def get_consts(self): return self._consts @@ -209,52 +264,73 @@ def get_base_code(self): return self - class VariadicCode(BaseCode): - __immutable_fields__ = ["_required_arity", "_code"] + _immutable_fields_ = ["_required_arity", "_code", "_meta"] _type = object.Type(u"pixie.stdlib.VariadicCode") - def type(self): - return VariadicCode._type - - def __init__(self, code, required_arity): + def __init__(self, code, required_arity, meta=nil): BaseCode.__init__(self) self._required_arity = r_uint(required_arity) self._code = code + self._meta = meta - def _invoke(self, args): + def with_meta(self, meta): + return VariadicCode(self._code, self._required_arity, meta) + + def name(self): + return None + + def required_arity(self): + return self._required_arity + + def invoke(self, args): + return self.invoke_with(args, self) + + def invoke_with(self, args, self_fn): from pixie.vm.array import array argc = len(args) if self._required_arity == 0: - return self._code.invoke([array(args)]) + return self._code.invoke_with([array(args)], self_fn) if argc == self._required_arity: new_args = resize_list(args, len(args) + 1) new_args[len(args)] = array([]) - return self._code.invoke(new_args) + return self._code.invoke_with(new_args, self_fn) elif argc > self._required_arity: start = slice_from_start(args, self._required_arity, 1) rest = slice_to_end(args, self._required_arity) start[self._required_arity] = array(rest) - return self._code.invoke(start) + return self._code.invoke_with(start, self_fn) affirm(False, u"Got " + unicode(str(argc)) + u" arg(s) need at least " + unicode(str(self._required_arity))) -class Closure(BaseCode): - _type = object.Type(u"Closure") - __immutable_fields__ = ["_closed_overs[*]", "_code"] - def type(self): - return Closure._type - def __init__(self, code, closed_overs): +class Closure(BaseCode): + _type = object.Type(u"pixie.stdlib.Closure") + _immutable_fields_ = ["_closed_overs[*]", "_code", "_meta"] + + def __init__(self, code, closed_overs, meta=nil): BaseCode.__init__(self) affirm(isinstance(code, Code), u"Code argument to Closure must be an instance of Code") self._code = code self._closed_overs = closed_overs + self._meta = meta + + def with_meta(self, meta): + return Closure(self._code, self._closed_overs, meta) - def _invoke(self, args): + + def name(self): + return None + + def invoke(self, args): + return self.invoke_with(args, self) + + def invoke_with(self, args, self_fn): try: - return interpret(self, args) + return interpret(self, args, self_obj=self_fn) except object.WrappedException as ex: - ex._ex._trace.append(object.PixieCodeInfo(self._code._name)) + code = self._code + assert isinstance(code, Code) + ex._ex._trace.append(object.PixieCodeInfo(code._name)) raise def get_closed_over(self, idx): @@ -278,48 +354,58 @@ def get_base_code(self): def get_debug_points(self): return self._code.get_debug_points() -class Undefined(object.Object): - _type = object.Type(u"Undefined") - def type(self): - return Undefined._type +class Undefined(object.Object): + _type = object.Type(u"pixie.stdlib.Undefined") undefined = Undefined() + class DynamicVars(py_object): def __init__(self): - self._vars = [{}] + self._vars = rt.cons(rt.hashmap(), nil) def push_binding_frame(self): - self._vars.append(self._vars[-1].copy()) + self._vars = rt.cons(rt.first(self._vars), self._vars) def pop_binding_frame(self): - self._vars.pop() + self._vars = rt.next(self._vars) + + def current_frame(self): + return rt.first(self._vars) + + def get_current_frames(self): + return self._vars + + def set_current_frames(self, vars): + self._vars = vars def get_var_value(self, var, not_found): - return self._vars[-1].get(var, not_found) + return rt._val_at(self.current_frame(), var, not_found) def set_var_value(self, var, val): - self._vars[-1][var] = val + cur_frame = self.current_frame() + self.pop_binding_frame() + self._vars = rt.cons(rt._assoc(cur_frame, var, val), self._vars) -_dynamic_vars = DynamicVars() -class Var(BaseCode): - _type = object.Type(u"Var") - _immutable_fields_ = ["_rev?"] - def type(self): - return Var._type - def __init__(self, ns, name): +class Var(BaseCode): + _type = object.Type(u"pixie.stdlib.Var") + _immutable_fields_ = ["_ns_ref"] + + def __init__(self, ns_ref, ns, name): + BaseCode.__init__(self) + self._ns_ref = ns_ref self._ns = ns self._name = name - self._rev = 0 self._root = undefined self._dynamic = False def set_root(self, o): - self._rev += 1 + affirm(o is not None, u"Invalid var set") + self._ns_ref._rev += 1 self._root = o return self @@ -328,48 +414,54 @@ def set_value(self, val): _dynamic_vars.set_var_value(self, val) return self - def set_dynamic(self): self._dynamic = True - self._rev += 1 + self._ns_ref._rev += 1 + def get_dynamic_value(self): return _dynamic_vars.get_var_value(self, self._root) + + @elidable_promote() - def is_dynamic(self, rev): + def _is_dynamic(self, rev): return self._dynamic + def is_dynamic(self): + return self._is_dynamic(self.ns_ref()._rev) + @elidable_promote() def get_root(self, rev): return self._root + @elidable_promote() + def ns_ref(self): + return self._ns_ref def deref(self): - if self.is_dynamic(self._rev): - return self.get_dynamic_value() + if self.is_dynamic(): + if we_are_translated(): + return self.get_dynamic_value() + else: + ## NOT RPYTHON + if globals().has_key("_dynamic_vars"): + return self.get_dynamic_value() + else: + return self.get_root(self.ns_ref()._rev) else: - val = self.get_root(self._rev) + val = self.get_root(self.ns_ref()._rev) affirm(val is not undefined, u"Var " + self._name + u" is undefined") return val def is_defined(self): return self._root is not undefined - def _invoke(self, args): - return self.deref().invoke(args) - -class bindings(py_object): - def __init__(self, *args): - self._args = list(args) - - def __enter__(self): - _dynamic_vars.push_binding_frame() - for x in range(0, len(self._args), 2): - self._args[x].set_value(self._args[x + 1]) + def invoke_with(self, args, this_fn): + return self.invoke(args) - def __exit__(self, exc_type, exc_val, exc_tb): - _dynamic_vars.pop_binding_frame() + def invoke(self, args): + return self.deref().invoke(args) class Refer(py_object): @@ -382,10 +474,10 @@ def __init__(self, ns, refer_syms=[], refer_all=False): class Namespace(object.Object): _type = object.Type(u"pixie.stdlib.Namespace") - def type(self): - return Namespace._type + _immutable_fields_ = ["_rev?"] def __init__(self, name): + self._rev = 0 self._registry = {} self._name = name self._refers = {} @@ -395,7 +487,7 @@ def intern_or_make(self, name): affirm(isinstance(name, unicode), u"Var names must be unicode") v = self._registry.get(name, None) if v is None: - v = Var(self._name, name) + v = Var(self, self._name, name) self._registry[name] = v return v @@ -409,9 +501,31 @@ def add_refer(self, ns, as_nm=None, refer_all=False): self._refers[as_nm] = Refer(ns, refer_all=refer_all) + def add_refer_symbol(self, sym, var): + assert isinstance(self, Namespace) + + name = rt.name(sym) + prev_binding = self._registry.get(name, None) + if prev_binding is not None: + print rt.name(rt.str(rt.wrap(u"Warning: "), sym, rt.wrap(u" already refers to "), prev_binding)) + + self._registry[name] = var + return var + def include_stdlib(self): stdlib = _ns_registry.find_or_make(u"pixie.stdlib") self.add_refer(stdlib, refer_all=True) + + def resolve_ns(self, ns_alias): + refer = self._refers.get(ns_alias, None) + resolved_ns = None + if refer is not None: + resolved_ns = refer._namespace + if resolved_ns is None: + resolved_ns = _ns_registry.get(ns_alias, None) + if resolved_ns is None: + affirm(False, u"Unable to resolve namespace: " + ns_alias + u" inside namespace " + self._name) + return resolved_ns def resolve(self, s, use_refers=True): import pixie.vm.symbol as symbol @@ -420,17 +534,12 @@ def resolve(self, s, use_refers=True): name = rt.name(s) if ns is not None: - refer = self._refers.get(ns, None) - resolved_ns = None - if refer is not None: - resolved_ns = refer._namespace - if resolved_ns is None: - resolved_ns = _ns_registry.get(ns, None) - if resolved_ns is None: - affirm(False, u"Unable to resolve namespace: " + ns + u" inside namespace " + self._name) + resolved_ns = self.resolve_ns(ns) else: resolved_ns = self + assert isinstance(resolved_ns, Namespace) + var = resolved_ns._registry.get(name, None) if var is None and use_refers: for refer_nm in self._refers: @@ -442,11 +551,10 @@ def resolve(self, s, use_refers=True): return None return var - - def get(self, name, default): return self._registry.get(name, default) + class NamespaceRegistry(py_object): def __init__(self): self._registry = {} @@ -459,7 +567,6 @@ def find_or_make(self, name): self._registry[name] = v return v - def get(self, name, default): return self._registry.get(name, default) @@ -473,6 +580,7 @@ def intern_var(ns, name=None): return _ns_registry.find_or_make(ns).intern_or_make(name) + def get_var_if_defined(ns, name, els=None): w_ns = _ns_registry.get(ns, None) if w_ns is None: @@ -480,23 +588,29 @@ def get_var_if_defined(ns, name, els=None): return w_ns.get(name, els) - class DefaultProtocolFn(NativeFn): def __init__(self, pfn): + BaseCode.__init__(self) self._pfn = pfn - def _invoke(self, args): - from pixie.vm.string import String - tp = args[0].type()._name - affirm(False, u"No override for " + tp + u" on " + self._pfn._name + u" in protocol " + self._pfn._protocol._name) + def invoke(self, args): + tp = args[0].type() + assert isinstance(tp, object.Type) + pfn = self._pfn + if isinstance(pfn, PolymorphicFn): + protocol = pfn._protocol + elif isinstance(pfn, DoublePolymorphicFn): + protocol = pfn._protocol + else: + assert False + assert isinstance(protocol, Protocol) + affirm(False, u"No override for " + tp._name + u" on " + self._pfn._name + u" in protocol " + protocol._name) class Protocol(object.Object): - _type = object.Type(u"Protocol") + _type = object.Type(u"pixie.stdlib.Protocol") - __immutable_fields__ = ["_rev?"] - def type(self): - return Protocol._type + _immutable_fields_ = ["_rev?"] def __init__(self, name): self._name = name @@ -504,7 +618,6 @@ def __init__(self, name): self._satisfies = {} self._rev = 0 - def add_method(self, pfn): self._polyfns[pfn] = pfn @@ -521,35 +634,74 @@ def satisfies(self, tp): class PolymorphicFn(BaseCode): - _type = object.Type(u"PolymorphicFn") - def type(self): - return PolymorphicFn._type + _type = object.Type(u"pixie.stdlib.PolymorphicFn") + + _immutable_fields_ = ["_rev?"] - __immutable_fields__ = ["_rev?"] def __init__(self, name, protocol): BaseCode.__init__(self) self._name = name self._dict = {} + # stored separately to allow ordered extending (e.g. more general protocols later) + self._protos = [] self._rev = 0 self._protocol = protocol self._default_fn = DefaultProtocolFn(self) + self._fn_cache = {} protocol.add_method(self) def extend(self, tp, fn): self._dict[tp] = fn + if isinstance(tp, Protocol): + self._protos.append(tp) self._rev += 1 + self._fn_cache = {} self._protocol.add_satisfies(tp) + ## We have to special case this so that the GC doesn't go nuts trying to do a ton during + ## collection. + self.maybe_mark_finalizer(tp) + + def maybe_mark_finalizer(self, tp): + ## Gets overridden in stdlib + pass + + def _find_parent_fn(self, tp): + ## Search the entire object tree to find the function to execute + assert isinstance(tp, object.Type) + + find_tp = tp + while True: + result = self._dict.get(find_tp, None) + if result is not None: + return result + + for proto in self._protos: + if proto.satisfies(find_tp): + return self._dict[proto] + + find_tp = find_tp._parent + if find_tp is None: + break + + return self._default_fn + def set_default_fn(self, fn): self._default_fn = fn self._rev += 1 + self._fn_cache = {} @elidable_promote() def get_protocol_fn(self, tp, rev): - fn = self._dict.get(tp, self._default_fn) + fn = self._fn_cache.get(tp, None) + if fn is None: + fn = self._find_parent_fn(tp) + self._fn_cache[tp] = fn + return promote(fn) - def _invoke(self, args): + def invoke(self, args): + affirm(len(args) >= 1, u"Wrong number of args") a = args[0].type() fn = self.get_protocol_fn(a, self._rev) try: @@ -558,14 +710,13 @@ def _invoke(self, args): ex._ex._trace.append(object.PolymorphicCodeInfo(self._name, args[0].type())) raise + class DoublePolymorphicFn(BaseCode): """A function that is polymorphic on the first two arguments""" - _type = object.Type(u"DoublePolymorphicFn") + _type = object.Type(u"pixie.stdlib.DoublePolymorphicFn") - def type(self): - return DefaultProtocolFn._type + _immutable_fields_ = ["_rev?"] - __immutable_fields__ = ["_rev?"] def __init__(self, name, protocol): BaseCode.__init__(self) self._name = name @@ -596,17 +747,55 @@ def get_fn(self, tp1, tp2, _rev): fn = d1.get(tp2, self._default_fn) return promote(fn) - def _invoke(self, args): + def invoke(self, args): affirm(len(args) >= 2, u"DoublePolymorphicFunctions take at least two args") a = args[0].type() b = args[1].type() fn = self.get_fn(a, b, self._rev) return fn.invoke(args) + +# class ElidableFn(object.Object): +# _type = object.Type(u"pixie.stdlib.ElidableFn") +# __immutable_fields__ = ["_boxed_fn"] +# def type(self): +# return ElidableFn._type +# +# def __init__(self, boxed_fn): +# self._boxed_fn = boxed_fn +# +# @elidable +# def _elidable_invoke_0(self, fn): +# return self._boxed_fn.invoke([]) +# +# @elidable +# def _elidable_invoke_1(self, fn, arg0): +# return self._boxed_fn.invoke([arg0]) +# +# @elidable +# def _elidable_invoke_2(self, fn, arg0, arg1): +# return self._boxed_fn.invoke([arg0, arg1]) +# +# +# def invoke(self, args): +# largs = jit.promote(len(args)) +# fn = self._boxed_fn.promote() +# if largs == 0: +# return self._elidable_invoke_0(fn).promote() +# elif largs == 1: +# return self._elidable_invoke_1(fn, args[0].promote()).promote() +# elif largs == 2: +# return self._elidable_invoke_2(fn, args[0].promote(), args[1].promote()).promote() +# affirm(False, u"Too many args to Elidable Fn") + + def munge(s): return s.replace("-", "_").replace("?", "_QMARK_").replace("!", "_BANG_") + import inspect + + def defprotocol(ns, name, methods): """Define a protocol in the given namespace with the given name and methods, vars will be created in the namespace for the protocol and methods. This function will dump @@ -615,14 +804,15 @@ def defprotocol(ns, name, methods): name = unicode(name) methods = map(unicode, methods) gbls = inspect.currentframe().f_back.f_globals - proto = Protocol(name) + proto = Protocol(name) intern_var(ns, name).set_root(proto) gbls[munge(name)] = proto for method in methods: - poly = PolymorphicFn(method, proto) + poly = PolymorphicFn(method, proto) intern_var(ns, method).set_root(poly) gbls[munge(method)] = poly + def assert_type(x, tp): affirm(isinstance(x, tp), u"Fatal Error, this should never happen") return x @@ -630,13 +820,16 @@ def assert_type(x, tp): ## PYTHON FLAGS CO_VARARGS = 0x4 + + def wrap_fn(fn, tp=object.Object): """Converts a native Python function into a pixie function.""" + docstring = unicode(fn.__doc__) if fn.__doc__ else u"" def as_native_fn(f): - return type("W"+fn.__name__, (NativeFn,), {"inner_invoke": f})() + return type("W" + fn.__name__, (NativeFn,), {"inner_invoke": f, "_doc": docstring})() def as_variadic_fn(f): - return type("W"+fn.__name__[:len("__args")], (NativeFn,), {"inner_invoke": f})() + return type("W" + fn.__name__[:len("__args")], (NativeFn,), {"inner_invoke": f, "_doc": docstring})() code = fn.func_code if fn.__name__.endswith("__args"): @@ -688,6 +881,19 @@ def wrapped_fn(self, args): raise return as_native_fn(wrapped_fn) + if argc == 4: + def wrapped_fn(self, args): + affirm(len(args) == 4, u"Expected 4 arguments to " + fn_name) + + try: + return fn(args[0], args[1], args[2], args[3]) + except object.WrappedException as ex: + ex._ex._trace.append(object.NativeCodeInfo(fn_name)) + raise + return as_native_fn(wrapped_fn) + + assert False, "implement more" + def extend(pfn, tp1, tp2=None): """Extends a protocol to the given Type (not python type), with the decorated function @@ -709,7 +915,6 @@ def extend_inner(fn): return extend_inner - def as_var(ns, name=None): """Locates a var with the given name (defaulting to the namespace pixie.stdlib), sets the root to the decorated function. If the function is not an instance of BaseCode it will @@ -722,6 +927,7 @@ def as_var(ns, name=None): ns = ns if isinstance(ns, unicode) else unicode(ns) var = intern_var(ns, name) + def with_fn(fn): fn.__real_name__ = name if not isinstance(fn, object.Object): @@ -735,6 +941,23 @@ def returns(type): """Tags a var as for unwrapping in rt. When rt imports this var it will be automatically converted to this type""" def with_fn(fn): fn._returns = type - print "registered", fn return fn return with_fn + + + +class bindings(py_object): + def __init__(self, *args): + self._args = list(args) + + def __enter__(self): + _dynamic_vars.push_binding_frame() + for x in range(0, len(self._args), 2): + self._args[x].set_value(self._args[x + 1]) + + def __exit__(self, exc_type, exc_val, exc_tb): + _dynamic_vars.pop_binding_frame() + + +def init(): + globals()["_dynamic_vars"] = DynamicVars() diff --git a/pixie/vm/compiler.py b/pixie/vm/compiler.py index bd6e0ad5..60cc353a 100644 --- a/pixie/vm/compiler.py +++ b/pixie/vm/compiler.py @@ -1,18 +1,19 @@ -from pixie.vm.object import Object, _type_registry, affirm -from pixie.vm.primitives import nil, true, false, Bool +from pixie.vm.object import affirm +from pixie.vm.primitives import nil, true, Bool from pixie.vm.persistent_vector import EMPTY, PersistentVector +from pixie.vm.persistent_hash_set import PersistentHashSet import pixie.vm.numbers as numbers -from pixie.vm.cons import cons, Cons import pixie.vm.symbol as symbol import pixie.vm.code as code -from pixie.vm.keyword import Keyword -from pixie.vm.string import String +from pixie.vm.keyword import Keyword, keyword +from pixie.vm.string import Character, String from pixie.vm.atom import Atom -import pixie.vm.stdlib as proto -from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rarithmetic import r_uint, intmask +from pixie.vm.persistent_list import EmptyList +from pixie.vm.cons import cons +from pixie.vm.persistent_list import create_from_list import pixie.vm.rt as rt -from pixie.vm.util import * NS_VAR = code.intern_var(u"pixie.stdlib", u"*ns*") NS_VAR.set_dynamic() @@ -20,24 +21,33 @@ FN_NAME = code.intern_var(u"pixie.stdlib", u"*fn-name*") FN_NAME.set_dynamic() +DYNAMIC_KW = keyword(u"dynamic") + gensym_id = Atom(numbers.zero_int) +def gensym1(): + return gensym2(rt.wrap(u"gensym_")) -@code.as_var("gensym") -def gensym(): +def gensym2(prefix): rt.reset_BANG_(gensym_id, rt._add(rt.deref(gensym_id), rt.wrap(1))) i = rt.deref(gensym_id) - return rt.symbol(rt.str(rt.wrap(u"gensym_"), i)) + return rt.symbol(rt.str(prefix, i)) +gensym = code.intern_var(u"pixie.stdlib", u"gensym") +gensym.set_root(code.MultiArityFn(u"gensym", {0: code.wrap_fn(gensym1), 1: code.wrap_fn(gensym2)})) class with_ns(object): - def __init__(self, nm): + def __init__(self, nm, include_stdlib=False): assert isinstance(nm, unicode) self._ns = nm + self._include_stdlib=include_stdlib + def __enter__(self): code._dynamic_vars.push_binding_frame() NS_VAR.set_value(code._ns_registry.find_or_make(self._ns)) + if self._include_stdlib: + NS_VAR.deref().include_stdlib() def __exit__(self, exc_type, exc_val, exc_tb): code._dynamic_vars.pop_binding_frame() @@ -62,7 +72,7 @@ def __init__(self, name, argc, parent_ctx): locals[x] = Closure(locals[x], parent_ctx) else: locals = {} - + self.argc = argc self.bytecode = [] self.consts = [] self.locals = [locals] @@ -70,7 +80,10 @@ def __init__(self, name, argc, parent_ctx): self._max_sp = 0 self.can_tail_call = False self.closed_overs = [] - self.name = name + if name == default_fn_name and parent_ctx: + self.name = parent_ctx.name + u"_fn" + else: + self.name = name self.recur_points = [] self.debug_points = {} @@ -83,6 +96,7 @@ def add_sp(self, v): self._max_sp = self._sp def sub_sp(self, v): + assert self._sp >= v, (v, self._sp) if self._max_sp < self._sp: self._max_sp = self._sp self._sp -= v @@ -98,7 +112,7 @@ def pop_recur_point(self): self.recur_points.pop() def to_code(self, required_args=-1): - return code.Code(self.name, self.bytecode, clone(self.consts), self._max_sp + 1, self.debug_points) + return code.Code(self.name, self.argc, self.bytecode, clone(self.consts), self._max_sp + 1, self.debug_points) def push_arg(self, idx): self.bytecode.append(code.ARG) @@ -106,6 +120,10 @@ def push_arg(self, idx): self.add_sp(1) + def pop_locals(self, i=1): + for x in range(i): + self.locals.pop() + def add_local(self, name, arg): self.locals.append(self.locals[-1].copy()) self.locals[-1][name] = arg @@ -206,6 +224,7 @@ def __init__(self, sp): def emit(self, ctx): ctx.bytecode.append(code.DUP_NTH) + assert 0 <= ctx.sp() - self.sp < 100000 ctx.bytecode.append(r_uint(ctx.sp() - self.sp)) ctx.add_sp(1) @@ -231,10 +250,17 @@ def emit(self, ctx): ctx.bytecode.append(self.idx) ctx.add_sp(1) +class LocalMacro(LocalType): + def __init__(self, form): + self._from = form + + def emit(self, ctx): + compile_form(self._from, ctx) + -def resolve_var(ctx, name): +def resolve_var(name): return NS_VAR.deref().resolve(name) def resolve_local(ctx, name): @@ -243,10 +269,10 @@ def resolve_local(ctx, name): def is_macro_call(form, ctx): if rt.seq_QMARK_(form) is true and isinstance(rt.first(form), symbol.Symbol): - name = rt.first(form)._str + name = rt.name(rt.first(form)) if resolve_local(ctx, name): return None - var = resolve_var(ctx, rt.first(form)) + var = resolve_var(rt.first(form)) if var and var.is_defined(): val = var.deref() @@ -256,7 +282,7 @@ def is_macro_call(form, ctx): def call_macro(var, form, ctx): form = rt.next(form) - args = [None] * rt.count(form).int_val() + args = [None] * rt.count(form) i = 0 while form is not nil: args[i] = rt.first(form) @@ -267,7 +293,7 @@ def call_macro(var, form, ctx): class CompileMapRf(code.NativeFn): def __init__(self, ctx): self._ctx = ctx - def _invoke(self, args): + def invoke(self, args): map_entry = args[1] compile_form(rt.key(map_entry), self._ctx) compile_form(rt.val(map_entry), self._ctx) @@ -278,10 +304,58 @@ def compile_map_literal(form, ctx): rt.reduce(CompileMapRf(ctx), nil, form) - size = rt.count(form).int_val() * 2 + size = rt.count(form) * 2 ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(size) + 1) + if size > 0: + ctx.sub_sp(size) + + compile_meta(rt.meta(form), ctx) + +class ConsReduce(code.NativeFn): + def invoke(self, args): + return rt.cons(args[1], args[0]) + +def compile_set_literal(form, ctx): + vals = rt.reduce(ConsReduce(), nil, form) + vector_call = rt.cons(rt.symbol(rt.wrap(u"vector")), vals) + set_call = rt.cons(rt.symbol(rt.wrap(u"set")), rt.cons(vector_call, nil)) + compile_cons(set_call, ctx) + +def compile_meta(meta, ctx): + ctx.push_const(code.intern_var(u"pixie.stdlib", u'with-meta')) + ctx.bytecode.append(code.DUP_NTH) + ctx.bytecode.append(r_uint(1)) + ctx.add_sp(1) + ctx.push_const(meta) + ctx.bytecode.append(code.INVOKE) + ctx.bytecode.append(r_uint(3)) + ctx.sub_sp(2) + ctx.bytecode.append(code.POP_UP_N) + ctx.bytecode.append(1) + ctx.sub_sp(1) + +def maybe_oop_invoke(form): + head = rt.first(form) + if isinstance(rt.first(form), symbol.Symbol) and rt.name(head).startswith(".-"): + postfix = rt.next(form) + affirm(rt.count(postfix) == 1, u" Attribute lookups must only have one argument") + subject = rt.first(postfix) + kw = keyword(rt.name(head)[2:]) + fn = symbol.symbol(u"pixie.stdlib/-get-attr") + return create_from_list([fn, subject, kw]) + + elif isinstance(rt.first(form), symbol.Symbol) and rt.name(head).startswith("."): + subject = rt.first(rt.next(form)) + postfix = rt.next(rt.next(form)) + form = cons(keyword(rt.name(head)[1:]), postfix) + form = cons(subject, form) + form = cons(symbol.symbol(u"pixie.stdlib/-call-method"), form) + return form + + else: + return form def compile_form(form, ctx): @@ -289,26 +363,50 @@ def compile_form(form, ctx): ctx.push_const(nil) return - if rt.instance_QMARK_(rt.ISeq.deref(), form) and form is not nil: + if rt._satisfies_QMARK_(rt.ISeq.deref(), form) and form is not nil: + + form = maybe_oop_invoke(form) + return compile_cons(form, ctx) if isinstance(form, numbers.Integer): ctx.push_const(form) return + if isinstance(form, numbers.BigInteger): + ctx.push_const(form) + return + if isinstance(form, numbers.Float): + ctx.push_const(form) + return + if isinstance(form, numbers.Ratio): + ctx.push_const(form) + return if isinstance(form, symbol.Symbol): - name = form._str - loc = resolve_local(ctx, name) - if loc is None: - var = resolve_var(ctx, form) + name = rt.name(form) + ns = rt.namespace(form) - if var is None: - var = NS_VAR.deref().intern_or_make(name) + loc = resolve_local(ctx, name) + var = resolve_var(form) - ctx.push_const(var) + if var is None and loc: + loc.emit(ctx) + return - ctx.bytecode.append(code.DEREF_VAR) + if var and loc and ns is None: + loc.emit(ctx) return - loc.emit(ctx) + + if var is None: + name = rt.name(form) + var = NS_VAR.deref().intern_or_make(name) + + ctx.push_const(var) + + meta = rt.meta(form) + if meta is not nil: + ctx.debug_points[len(ctx.bytecode)] = rt.interpreter_code_info(meta) + + ctx.bytecode.append(code.DEREF_VAR) return if isinstance(form, Bool) or form is nil: @@ -320,8 +418,7 @@ def compile_form(form, ctx): return if isinstance(form, PersistentVector): - vector_var = rt.vector() - size = rt.count(form).int_val() + size = rt.count(form) #assert rt.count(form).int_val() == 0 ctx.push_const(code.intern_var(u"pixie.stdlib", u"vector")) for x in range(size): @@ -330,9 +427,16 @@ def compile_form(form, ctx): ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(r_uint(size + 1)) ctx.sub_sp(size) + + compile_meta(rt.meta(form), ctx) + return - if rt.instance_QMARK_(rt.IMap.deref(), form): + if isinstance(form, PersistentHashSet): + compile_set_literal(form, ctx) + return + + if rt._satisfies_QMARK_(rt.IMap.deref(), form): compile_map_literal(form, ctx) return @@ -340,6 +444,10 @@ def compile_form(form, ctx): ctx.push_const(form) return + if isinstance(form, Character): + ctx.push_const(form) + return + raise Exception("Can't compile ") def compile_platform_plus(form, ctx): @@ -356,32 +464,25 @@ def compile_platform_plus(form, ctx): ctx.enable_tail_call() return ctx -def compile_platform_eq(form, ctx): - form = form.next() - - affirm(rt.count(form).int_val() == 2, u"TODO: REMOVE") - while form is not nil: - compile_form(form.first(), ctx) - form = form.next() - - ctx.bytecode.append(code.EQ) - ctx.sub_sp(1) - return ctx - -def add_args(args, ctx): +def add_args(name, args, ctx): required_args = -1 local_idx = 0 - for x in range(rt.count(args).int_val()): + + if name != default_fn_name: + ctx.add_local(name, Self()) + + for x in range(rt.count(args)): arg = rt.nth(args, rt.wrap(x)) affirm(isinstance(arg, symbol.Symbol), u"Argument names must be symbols") - if arg._str == u"&": + if rt.name(arg) == u"&": - required_args = x + required_args = intmask(x) continue - ctx.add_local(arg._str, Arg(local_idx)) + ctx.add_local(rt.name(arg), Arg(local_idx)) local_idx += 1 return required_args +default_fn_name = u"some_long_name_unlikely_to_be_used" def compile_fn(form, ctx): form = rt.next(form) @@ -389,12 +490,12 @@ def compile_fn(form, ctx): name = rt.first(form) form = rt.next(form) else: - name = symbol.symbol(u"-fn") + name = symbol.symbol(default_fn_name) - if rt.instance_QMARK_(rt.ISeq.deref(), rt.first(form)): + if rt._satisfies_QMARK_(rt.ISeq.deref(), rt.first(form)): arities = [] while form is not nil: required_arity, argc = compile_fn_body(name, rt.first(rt.first(form)), rt.next(rt.first(form)), ctx) @@ -407,38 +508,43 @@ def compile_fn(form, ctx): for x in arities: ctx.bytecode.append(r_uint(x)) + ctx.add_sp(1) # result ctx.sub_sp(len(arities)) else: - compile_fn_body(name, rt.first(form), rt.next(form), ctx) + res = compile_fn_body(name, rt.first(form), rt.next(form), ctx) + if rt.meta(name) is not nil: + compile_meta(rt.meta(name), ctx) +LOOP = symbol.symbol(u"loop*") +def compile_fn_body(name, args, body, ctx): + new_ctx = Context(rt.name(name), rt.count(args), ctx) + required_args = add_args(rt.name(name), args, new_ctx) + affirm(isinstance(name, symbol.Symbol), u"Function names must be symbols") -def compile_fn_body(name, args, body, ctx): - new_ctx = Context(name._str, rt.count(args).int_val(), ctx) - required_args = add_args(args, new_ctx) - bc = 0 + arg_syms = EMPTY + for x in range(rt.count(args)): + sym = rt.nth(args, rt.wrap(x)) + if not rt.name(sym) == u"&": + arg_syms = rt.conj(rt.conj(arg_syms, sym), sym) - if name is not None: - affirm(isinstance(name, symbol.Symbol), u"Function names must be symbols") - #new_ctx.add_local(name._str, Self()) + body = rt.list(rt.cons(LOOP, rt.cons(arg_syms, body))) - new_ctx.push_recur_point(FunctionRecurPoint()) + #new_ctx.push_recur_point(FunctionRecurPoint()) new_ctx.disable_tail_call() if body is nil: compile_form(body, new_ctx) else: while body is not nil: - if rt.next(body) is nil: - new_ctx.enable_tail_call() + #if rt.next(body) is nil: + # new_ctx.enable_tail_call() compile_form(rt.first(body), new_ctx) - if rt.next(body) is not nil: - new_ctx.pop() - bc += 1 body = rt.next(body) - + if body is not nil: + new_ctx.pop() new_ctx.bytecode.append(code.RETURN) closed_overs = new_ctx.closed_overs if len(closed_overs) == 0: @@ -455,11 +561,11 @@ def compile_fn_body(name, args, body, ctx): ctx.bytecode.append(code.MAKE_VARIADIC) ctx.bytecode.append(r_uint(required_args)) - return required_args, rt.count(args).int_val() + return required_args, intmask(rt.count(args)) def compile_if(form, ctx): form = form.next() - affirm(2 <= rt.count(form).int_val() <= 3, u"If must have either 2 or 3 forms") + affirm(2 <= rt.count(form) <= 3, u"If must have either 2 or 3 forms") test = rt.first(form) form = rt.next(form) @@ -497,6 +603,12 @@ def compile_def(form, ctx): affirm(isinstance(name, symbol.Symbol), u"Def'd name must be a symbol") var = NS_VAR.deref().intern_or_make(rt.name(name)) + + if rt._val_at(rt.meta(name), DYNAMIC_KW, nil) is true: + assert isinstance(var, code.Var) + var.set_dynamic() + + ctx.push_const(var) compile_form(val, ctx) ctx.bytecode.append(code.SET_VAR) @@ -518,6 +630,9 @@ def compile_quote(form, ctx): data = rt.first(rt.next(form)) ctx.push_const(data) + if rt.meta(form) is not nil: + compile_meta(rt.meta(form), ctx) + def compile_recur(form, ctx): form = form.next() affirm(ctx.can_tail_call, u"Can't recur in non-tail position") @@ -534,20 +649,23 @@ def compile_recur(form, ctx): ctx.get_recur_point().emit(ctx, args) if ctc: ctx.enable_tail_call() - ctx.sub_sp(args - 1) + if args > 0: + ctx.sub_sp(r_uint(args - 1)) + else: + ctx.add_sp(r_uint(1)) def compile_let(form, ctx): - form = next(form) + form = rt.next(form) bindings = rt.first(form) affirm(isinstance(bindings, PersistentVector), u"Bindings must be a vector") - body = next(form) + body = rt.next(form) ctc = ctx.can_tail_call ctx.disable_tail_call() binding_count = 0 - for i in range(0, rt.count(bindings).int_val(), 2): + for i in range(0, rt.count(bindings), 2): binding_count += 1 name = rt.nth(bindings, rt.wrap(i)) affirm(isinstance(name, symbol.Symbol), u"Let locals must be symbols") @@ -555,7 +673,7 @@ def compile_let(form, ctx): compile_form(bind, ctx) - ctx.add_local(name._str, LetBinding(ctx.sp())) + ctx.add_local(rt.name(name), LetBinding(ctx.sp())) if ctc: ctx.enable_tail_call() @@ -572,26 +690,27 @@ def compile_let(form, ctx): ctx.bytecode.append(code.POP_UP_N) ctx.sub_sp(binding_count) ctx.bytecode.append(binding_count) + ctx.pop_locals(binding_count) def compile_loop(form, ctx): - form = next(form) + form = rt.next(form) bindings = rt.first(form) affirm(isinstance(bindings, PersistentVector), u"Loop bindings must be a vector") - body = next(form) - + body = rt.next(form) + ctx.enable_tail_call() ctc = ctx.can_tail_call ctx.disable_tail_call() binding_count = 0 - for i in range(0, rt.count(bindings).int_val(), 2): + for i in range(0, rt.count(bindings), 2): binding_count += 1 name = rt.nth(bindings, rt.wrap(i)) - affirm(isinstance(name, symbol.Symbol), u"Loop must bindings must be symbols") + affirm(isinstance(name, symbol.Symbol), u"Loop bindings must be symbols") bind = rt.nth(bindings, rt.wrap(i + 1)) compile_form(bind, ctx) - ctx.add_local(name._str, LetBinding(ctx.sp())) + ctx.add_local(rt.name(name), LetBinding(ctx.sp())) if ctc: ctx.enable_tail_call() @@ -599,7 +718,7 @@ def compile_loop(form, ctx): ctx.push_recur_point(LoopRecurPoint(binding_count, ctx)) while True: compile_form(rt.first(body), ctx) - body = next(body) + body = rt.next(body) if body is nil: break @@ -610,39 +729,79 @@ def compile_loop(form, ctx): ctx.bytecode.append(code.POP_UP_N) ctx.sub_sp(binding_count) ctx.bytecode.append(binding_count) + ctx.pop_locals(binding_count) def compile_comment(form, ctx): ctx.push_const(nil) -def compile_ns(form, ctx): - affirm(rt.count(form).int_val() == 2, u"ns only takes one argument, a symbol") +def compile_this_ns(form, ctx): + ctx.push_const(NS_VAR.deref()) - nm = rt.first(rt.next(form)) +def compile_var(form, ctx): + form = rt.next(form) + name = rt.first(form) - affirm(isinstance(nm, symbol.Symbol), u"Namespace name must be a symbol") + affirm(isinstance(name, symbol.Symbol), u"var name must be a symbol") - str_name = rt.name(nm) + if rt.namespace(name) is not None: + var = code._ns_registry.find_or_make(rt.namespace(name)) + else: + var = NS_VAR.deref().intern_or_make(rt.name(name)) - NS_VAR.set_value(code._ns_registry.find_or_make(str_name)) + ctx.push_const(var) + +def compile_catch(form, ctx): + affirm(False, u"Catch used outside of try") + +def compile_yield(form, ctx): + affirm(rt.count(form) == 2, u"yield takes a single argument") + arg = rt.first(rt.next(form)) + compile_form(arg, ctx) + ctx.bytecode.append(code.YIELD) + +def compile_in_ns(form, ctx): + affirm(rt.count(form) == 2, u"in-ns requires an argument") + arg = rt.first(rt.next(form)) + NS_VAR.set_value(code._ns_registry.find_or_make(rt.name(arg))) NS_VAR.deref().include_stdlib() - ctx.push_const(nil) + compile_fn_call(form, ctx) -def compile_this_ns(form, ctx): - ctx.push_const(NS_VAR.deref()) +def compile_local_macro(form, ctx): + form = rt.next(form) + binding = rt.first(form) + body = rt.next(form) + + sym = rt.nth(binding, rt.wrap(0)) + bind_form = rt.nth(binding, rt.wrap(1)) + ctx.add_local(rt.name(sym), LocalMacro(bind_form)) -builtins = {u"fn": compile_fn, + while True: + compile_form(rt.first(body), ctx) + body = rt.next(body) + + if body is nil: + break + else: + ctx.pop() + + ctx.pop_locals() + +builtins = {u"fn*": compile_fn, u"if": compile_if, - u"platform=": compile_platform_eq, u"def": compile_def, u"do": compile_do, u"quote": compile_quote, u"recur": compile_recur, - u"let": compile_let, - u"loop": compile_loop, + u"let*": compile_let, + u"loop*": compile_loop, u"comment": compile_comment, - u"__ns__": compile_ns, - u"this-ns-name": compile_this_ns} + u"var": compile_var, + u"catch": compile_catch, + u"this-ns-name": compile_this_ns, + u"in-ns": compile_in_ns, # yes, this is both a function and a compiler special form. + u"yield": compile_yield, + u"local-macro": compile_local_macro} def compiler_special(s): if isinstance(s, symbol.Symbol): @@ -656,11 +815,18 @@ def is_compiler_special(s): return True if compiler_special(s) is not None else False def compile_cons(form, ctx): + if isinstance(form, EmptyList): + ctx.push_const(form) + return + if isinstance(rt.first(form), symbol.Symbol): special = compiler_special(rt.first(form)) if special is not None: return special(form, ctx) + return compile_fn_call(form, ctx) + +def compile_fn_call(form, ctx): macro = is_macro_call(form, ctx) if macro: return compile_form(call_macro(macro, form, ctx), ctx) @@ -682,15 +848,15 @@ def compile_cons(form, ctx): # ctx.bytecode.append(code.TAIL_CALL) #else: if meta is not nil: - ctx.debug_points[len(ctx.bytecode)] = meta + ctx.debug_points[len(ctx.bytecode)] = rt.interpreter_code_info(meta) ctx.bytecode.append(code.INVOKE) ctx.bytecode.append(cnt) - ctx.sub_sp(cnt - 1) + ctx.sub_sp(r_uint(cnt - 1)) def compile(form): - ctx = Context(u"main", 0, None) + ctx = Context(u"_toplevel_", 0, None) compile_form(form, ctx) ctx.bytecode.append(code.RETURN) - return ctx.to_code() \ No newline at end of file + return ctx.to_code() diff --git a/pixie/vm/cons.py b/pixie/vm/cons.py index 3be07e04..27fcc109 100644 --- a/pixie/vm/cons.py +++ b/pixie/vm/cons.py @@ -1,5 +1,5 @@ import pixie.vm.object as object -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import nil import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var @@ -7,9 +7,6 @@ class Cons(object.Object): _type = object.Type(u"pixie.stdlib.Cons") - def type(self): - return Cons._type - def __init__(self, head, tail, meta=nil): self._first = head self._next = tail @@ -57,12 +54,5 @@ def _with_meta(self, meta): def cons(head, tail): return Cons(head, tail) -def count(self): - cnt = 0 - while self is not nil: - self = self.next() - cnt += 1 - return cnt - def cons(head, tail=nil): - return Cons(head, tail, nil) \ No newline at end of file + return Cons(head, tail, nil) diff --git a/pixie/vm/custom_types.py b/pixie/vm/custom_types.py index 9575b6b1..d5fb3047 100644 --- a/pixie/vm/custom_types.py +++ b/pixie/vm/custom_types.py @@ -1,61 +1,178 @@ -from pixie.vm.object import Object, Type, affirm -from pixie.vm.primitives import nil, true, false +from pixie.vm.object import Object, Type, affirm, runtime_error, finalizer_registry import rpython.rlib.jit as jit -from pixie.vm.numbers import Integer -from rpython.rlib.rarithmetic import r_uint from pixie.vm.code import as_var -from pixie.vm.symbol import Symbol -from pixie.vm.string import String +from pixie.vm.numbers import Integer, Float from pixie.vm.keyword import Keyword import pixie.vm.rt as rt +MAX_FIELDS = 32 + class CustomType(Type): - __immutable_fields__ = ["_slots"] + _immutable_fields_ = ["_slots[*]", "_rev?"] def __init__(self, name, slots): Type.__init__(self, name) self._slots = slots + self._mutable_slots = {} + self._rev = 0 @jit.elidable_promote() def get_slot_idx(self, nm): - return self._slots[nm] + return self._slots.get(nm, -1) + + def set_mutable(self, nm): + if not self.is_mutable(nm): + self._rev += 1 + self._mutable_slots[nm] = nm + + + @jit.elidable_promote() + def _is_mutable(self, nm, rev): + return nm in self._mutable_slots + + def is_mutable(self, nm): + return self._is_mutable(nm, self._rev) @jit.elidable_promote() def get_num_slots(self): return len(self._slots) class CustomTypeInstance(Object): - __immutable_fields__ = ["_type"] + _immutable_fields_ = ["_custom_type"] def __init__(self, type): affirm(isinstance(type, CustomType), u"Can't create a instance of a non custom type") - self._type = type - self._fields = [None] * self._type.get_num_slots() + self._custom_type = type def type(self): - return self._type + return self._custom_type def set_field(self, name, val): - idx = self._type.get_slot_idx(name) - self._fields[idx] = val + idx = self._custom_type.get_slot_idx(name) + if idx == -1: + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type())), + u"pixie.stdlib/InvalidFieldException") + + old_val = self.get_field_by_idx(idx) + if isinstance(old_val, AbstractMutableCell): + old_val.set_mutable_cell_value(self._custom_type, self, name, idx, val) + else: + self._custom_type.set_mutable(name) + self.set_field_by_idx(idx, val) return self + @jit.elidable + def _get_field_immutable(self, idx, rev): + return self.get_field_by_idx(idx) + + def get_field_immutable(self, idx): + tp = self._custom_type + assert isinstance(tp, CustomType) + return self._get_field_immutable(idx, tp._rev) + def get_field(self, name): - idx = self._type.get_slot_idx(name) - return self._fields[idx] + idx = self._custom_type.get_slot_idx(name) + if idx == -1: + runtime_error(u"Invalid field named " + rt.name(rt.str(name)) + u" on type " + rt.name(rt.str(self.type())), + u"pixie.stdlib/InvalidFieldException") + + if self._custom_type.is_mutable(name): + value = self.get_field_by_idx(idx) + else: + value = self.get_field_immutable(idx) + + if isinstance(value, AbstractMutableCell): + return value.get_mutable_cell_value() + else: + return value + + + def __del__(self): + if self.type().has_finalizer(): + finalizer_registry.register(self) + + +create_type_prefix = """ +def new_inst(tp, fields): + l = len(fields) + if l >= {max_c}: + runtime_error(u"Too many fields for type", u"pixie.stdlib/TooManyFields") +""" + +clause_template = """ + elif l == {c}: + return CustomType{c}(tp, fields) +""" + +def gen_ct_code(): + acc = create_type_prefix.format(max_c=MAX_FIELDS + 1) + for x in range(MAX_FIELDS + 1): + acc += clause_template.format(c=x) + + #print acc + return acc + +exec gen_ct_code() + + +type_template = """ +class CustomType{c}(CustomTypeInstance): + def __init__(self, tp, fields): + CustomTypeInstance.__init__(self, tp) + {self_fields_list} = fields + def get_field_by_idx(self, idx): + if idx >= {max}: + return None +""" + +get_field_by_idx_template = """ + elif idx == {c}: + return self._custom_field{c} +""" + +set_field_prefix_template = """ def set_field_by_idx(self, idx, val): - affirm(isinstance(idx, r_uint), u"idx must be a r_uint") - self._fields[idx] = val - return self + if idx >= {max}: + return +""" + +set_field_by_idx_template = """ + elif idx == {c}: + self._custom_field{c} = val +""" + +def gen_ct_class_code(): + acc = "" + for x in range(MAX_FIELDS + 1): + if x == 0: + self_fields_list = "_null_" + elif x == 1: + self_fields_list = "self._custom_field0," + else: + self_fields_list = ",".join(map(lambda x: "self._custom_field" + str(x), range(x))) + acc += type_template.format(c=x, self_fields_list=self_fields_list, max=x) + for y in range(x): + acc += get_field_by_idx_template.format(c=y) + + acc += set_field_prefix_template.format(max=x) + + for y in range(x): + acc += set_field_by_idx_template.format(c=y) + + + + #print acc + return acc + +exec gen_ct_class_code() @as_var("create-type") def create_type(type_name, fields): affirm(isinstance(type_name, Keyword), u"Type name must be a keyword") - field_count = rt.count(fields).int_val() acc = {} - for i in range(rt.count(fields).int_val()): + for i in range(rt.count(fields)): val = rt.nth(fields, rt.wrap(i)) affirm(isinstance(val, Keyword), u"Field names must be keywords") acc[val] = i @@ -64,9 +181,23 @@ def create_type(type_name, fields): return CustomType(rt.name(type_name), acc) @as_var("new") -def _new(tp): +@jit.unroll_safe +def _new__args(args): + affirm(len(args) >= 1, u"new takes at least one parameter") + tp = args[0] affirm(isinstance(tp, CustomType), u"Can only create a new instance of a custom type") - return CustomTypeInstance(tp) + cnt = len(args) - 1 + affirm(cnt - 1 != tp.get_num_slots(), u"Wrong number of initializer fields to custom type") + arr = [None] * cnt + for x in range(cnt): + val = args[x + 1] + if isinstance(val, Integer): + val = IntegerMutableCell(val.int_val()) + elif isinstance(val, Float): + val = FloatMutableCell(val.float_val()) + + arr[x] = val + return new_inst(tp, arr) @as_var("set-field!") def set_field(inst, field, val): @@ -81,3 +212,47 @@ def get_field(inst, field): affirm(isinstance(field, Keyword), u"Field must be a keyword") return inst.get_field(field) + + +class AbstractMutableCell(Object): + _type = Type(u"pixie.stdlib.AbstractMutableCell") + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + pass + + def get_mutable_cell_value(self): + pass + +class IntegerMutableCell(AbstractMutableCell): + def __init__(self, int_val): + self._mutable_integer_val = int_val + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + if not isinstance(value, Integer): + ct.set_mutable(nm) + if isinstance(value, Float): + fields.set_field_by_idx(idx, FloatMutableCell(value.float_val())) + else: + fields.set_field_by_idx(idx, value) + else: + self._mutable_integer_val = value.int_val() + + def get_mutable_cell_value(self): + return rt.wrap(self._mutable_integer_val) + +class FloatMutableCell(AbstractMutableCell): + def __init__(self, float_val): + self._mutable_float_val = float_val + + def set_mutable_cell_value(self, ct, fields, nm, idx, value): + if not isinstance(value, Float): + ct.set_mutable(nm) + if isinstance(value, Integer): + fields.set_field_by_idx(idx, IntegerMutableCell(value.int_val())) + else: + fields.set_field_by_idx(idx, value) + else: + self._mutable_float_val = value.float_val() + + def get_mutable_cell_value(self): + return rt.wrap(self._mutable_float_val) diff --git a/pixie/vm/interpreter.py b/pixie/vm/interpreter.py index ec102630..cb11508d 100644 --- a/pixie/vm/interpreter.py +++ b/pixie/vm/interpreter.py @@ -1,18 +1,17 @@ -from pixie.vm.object import Object, affirm, WrappedException +from pixie.vm.object import Object, affirm, WrappedException, Type, runtime_error import pixie.vm.code as code import pixie.vm.numbers as numbers -from pixie.vm.primitives import nil, true, false -from rpython.rlib.rarithmetic import r_uint, intmask -from rpython.rlib.jit import JitDriver, promote, elidable, elidable_promote, hint, unroll_safe +from pixie.vm.primitives import nil, false +from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.jit import JitDriver, promote, elidable, hint, unroll_safe import rpython.rlib.jit as jit import rpython.rlib.debug as debug -import pixie.vm.rt as rt -def get_location(ip, sp, bc, base_code): +def get_location(ip, sp, is_continuation, bc, base_code): return code.BYTECODES[bc[ip]] + " in " + str(base_code._name) -jitdriver = JitDriver(greens=["ip", "sp", "bc", "base_code"], reds=["frame"], virtualizables=["frame"], - get_printable_location=get_location) +jitdriver = JitDriver(greens=["ip", "sp", "is_continuation", "bc", "base_code"], reds=["frame"], virtualizables=["frame"], + get_printable_location=get_location, is_recursive=True) @elidable @@ -29,20 +28,33 @@ class Frame(object): "debug_points", "args[*]", "base_code", - "closed_overs[*]" -] - def __init__(self, code_obj, args): + "closed_overs[*]", + "finished", + "_is_continuation", + "self_obj" + ] + def __init__(self, code_obj, args, self_obj): self = hint(self, access_directly=True, fresh_virtualizable=True) self.code_obj = code_obj + self.self_obj = self_obj self.sp = r_uint(0) self.ip = r_uint(0) self.stack = [None] * code_obj.stack_size() self.args = debug.make_sure_not_resized(args) self.base_code = code_obj.get_base_code() self.debug_points = code_obj.get_debug_points() + self.finished = False + self._is_continuation = 0 if code_obj is not None: self.unpack_code_obj() + def set_continuation(self): + self._is_continuation = 1 + + def is_continuation(self): + return self._is_continuation == 1 + + def unpack_code_obj(self): self.bc = self.code_obj.get_bytecode() self.consts = self.code_obj.get_consts() @@ -67,23 +79,29 @@ def push(self, val): def pop(self): #print type(self.sp), self.sp self.sp -= 1 - assert 0 <= self.sp < len(self.stack), u"Stack out of range: " + unicode(str(self.sp)) + + if not 0 <= self.sp < len(self.stack): + runtime_error(u"Stack out of range: " + unicode(str(self.sp)), + u"pixie.vm.interpreter/InterpreterError") + v = self.stack[self.sp] self.stack[self.sp] = None return v def nth(self, delta): - assert delta >= 0 - assert self.sp - 1 >= delta + affirm(delta >= 0, u"Invalid nth value, (compiler error)") + if not self.sp - 1 >= delta: + runtime_error(u"Interpreter nth out of range: " + unicode(str(self.sp - 1)) + u", " + unicode(str(delta)), + u"pixie.vm.interpreter/InterpreterError") return self.stack[self.sp - delta - 1] def push_nth(self, delta): self.push(self.nth(delta)) def push_arg(self, idx): - assert 0 <= idx < len(self.args) - self.push(self.args[r_uint(idx)]) + if 0 <= idx < len(self.args): + self.push(self.args[r_uint(idx)]) @unroll_safe def push_n(self, args, argc): @@ -120,6 +138,7 @@ def make_multi_arity(frame, argc): d = {} required_arity = 0 rest_fn = None + fn_name = None for i in range(argc): a = frame.get_inst() if a & 256: @@ -128,24 +147,49 @@ def make_multi_arity(frame, argc): rest_fn = frame.pop() else: fn = frame.pop() + fn_name = fn.name() d[a] = fn + return code.MultiArityFn(fn_name, d, required_arity, rest_fn) + +class ShallowContinuation(Object): + _type = Type(u"pixie.stdlib.ShallowContinuation") + + def __init__(self, frame, val): + assert isinstance(frame, Frame) + self._frame = frame + self._val = val + + def is_finished(self): + return self._frame.finished + + def invoke(self, args): + affirm(len(args) == 1, u"Generators only take one argument") + self._frame.push(args[0]) + val = interpret(frame=self._frame) + self._val = val + return self._val + +def interpret(code_obj=None, args=[], self_obj = None, frame=None): + + if frame is None: + assert code_obj is not None + frame = Frame(code_obj, args, self_obj or code_obj) + - return code.MultiArityFn(d, required_arity, rest_fn) -def interpret(code_obj, args=[]): - frame = Frame(code_obj, args) while True: jitdriver.jit_merge_point(bc=frame.bc, ip=frame.ip, sp=frame.sp, base_code=frame.base_code, - frame=frame) + frame=frame, + is_continuation=frame._is_continuation) inst = frame.get_inst() - - #print code.BYTECODES[inst] + #print code.BYTECODES[inst], frame.consts, frame.ip if inst == code.LOAD_CONST: arg = frame.get_inst() + #print "load const", arg frame.push_const(arg) continue @@ -154,7 +198,7 @@ def interpret(code_obj, args=[]): argc = frame.get_inst() fn = frame.nth(argc - 1) - assert isinstance(fn, code.BaseCode), "Expected callable, got " + str(fn) + #assert isinstance(fn, code.BaseCode), "Expected callable, got " + str(fn) @@ -190,7 +234,7 @@ def interpret(code_obj, args=[]): if inst == code.RETURN: val = frame.pop() - + frame.finished = True return val if inst == code.COND_BR: @@ -246,27 +290,23 @@ def interpret(code_obj, args=[]): continue if inst == code.DEREF_VAR: + debug_ip = frame.ip var = frame.pop() if not isinstance(var, code.Var): - affirm(False, u"Can't deref " + var.type()._name) - frame.push(var.deref()) - continue - - if inst == code.RECUR: - argc = frame.get_inst() - args = frame.pop_n(argc) - - frame = Frame(frame.code_obj, args) - - jitdriver.can_enter_jit(bc=frame.bc, - ip=frame.ip, - sp=frame.sp, - base_code=frame.base_code, - frame=frame) - continue + tp = var.type() + assert isinstance(tp, Type) + affirm(False, u"Can't deref " + tp._name) + try: + frame.push(var.deref()) + continue + except WrappedException as ex: + dp = frame.debug_points.get(debug_ip - 1, None) + if dp: + ex._ex._trace.append(dp) + raise if inst == code.PUSH_SELF: - frame.push(frame.code_obj) + frame.push(frame.self_obj) continue if inst == code.DUP_NTH: @@ -297,7 +337,8 @@ def interpret(code_obj, args=[]): ip=frame.ip, sp=frame.sp, base_code=frame.base_code, - frame=frame) + frame=frame, + is_continuation=frame._is_continuation) continue if inst == code.MAKE_MULTI_ARITY: @@ -312,8 +353,22 @@ def interpret(code_obj, args=[]): continue + if inst == code.YIELD: + if not frame.is_continuation(): + frame.set_continuation() + #frame = jit.hint(frame, force_virtualizable=True) + k = ShallowContinuation(frame, frame.pop()) + return k + else: + return frame.pop() + + if inst == code.PUSH_NS: + from pixie.vm.compiler import NS_VAR + frame.push(NS_VAR.deref()) + continue + - print "NO DISPATCH FOR: " + code.BYTECODES[inst] + affirm(False, u"NO DISPATCH FOR: " + unicode(code.BYTECODES[inst])) raise Exception() diff --git a/pixie/vm/keyword.py b/pixie/vm/keyword.py index d23cddf8..7da9af14 100644 --- a/pixie/vm/keyword.py +++ b/pixie/vm/keyword.py @@ -1,17 +1,20 @@ -from pixie.vm.object import Object, Type, affirm -from pixie.vm.primitives import nil, true, false +from pixie.vm.object import Object, Type +from pixie.vm.primitives import nil from pixie.vm.string import String import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var import pixie.vm.rt as rt +import pixie.vm.util as util +from rpython.rlib.rarithmetic import intmask + class Keyword(Object): _type = Type(u"pixie.stdlib.Keyword") - def __init__(self, name): self._str = name self._w_name = None self._w_ns = None + self._hash = 0 def type(self): return Keyword._type @@ -45,7 +48,9 @@ def intern(self, nm): _kw_cache = KeywordCache() -def keyword(nm): +def keyword(nm, ns=None): + if ns: + nm = u"/".join([ns, nm]) return _kw_cache.intern(nm) @@ -61,7 +66,16 @@ def _namespace(self): self.init_names() return self._w_ns +@extend(proto._hash, Keyword) +def _hash(self): + assert isinstance(self, Keyword) + if self._hash == 0: + self._hash = util.hash_unencoded_chars(self._str) + return rt.wrap(intmask(self._hash)) + @as_var("keyword") def _keyword(s): - affirm(isinstance(s, String), u"Symbol name must be a string") - return keyword(s._str) \ No newline at end of file + if not isinstance(s, String): + from pixie.vm.object import runtime_error + runtime_error(u"Keyword name must be a string") + return keyword(s._str) diff --git a/pixie/vm/lazy_seq.py b/pixie/vm/lazy_seq.py index 0d6af515..65cb744c 100644 --- a/pixie/vm/lazy_seq.py +++ b/pixie/vm/lazy_seq.py @@ -1,21 +1,20 @@ import pixie.vm.object as object -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import nil import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var import pixie.vm.rt as rt +import rpython.rlib.jit as jit class LazySeq(object.Object): _type = object.Type(u"pixie.stdlib.LazySeq") - def type(self): - return LazySeq._type - def __init__(self, fn, meta=nil): self._fn = fn self._meta = meta self._s = nil + @jit.jit_callback("lazy_seq_sval") def sval(self): if self._fn is None: return self._s @@ -24,6 +23,21 @@ def sval(self): self._fn = None return self._s + @jit.dont_look_inside + def lazy_seq_seq(self): + self.sval() + if self._s is not nil: + ls = self._s + while True: + if isinstance(ls, LazySeq): + ls = ls.sval() + continue + else: + self._s = ls + return rt.seq(self._s) + else: + return nil + @extend(proto._first, LazySeq) @@ -41,19 +55,9 @@ def _next(self): @extend(proto._seq, LazySeq) def _seq(self): assert isinstance(self, LazySeq) - self.sval() - if self._s is not nil: - ls = self._s - while True: - if isinstance(ls, LazySeq): - ls = ls.sval() - continue - else: - self._s = ls - return rt.seq(self._s) - else: - return nil + return self.lazy_seq_seq() + @as_var("lazy-seq*") def lazy_seq(f): - return LazySeq(f) \ No newline at end of file + return LazySeq(f) diff --git a/pixie/vm/libs/c/uv_ffi.c b/pixie/vm/libs/c/uv_ffi.c new file mode 100644 index 00000000..3ff67de4 --- /dev/null +++ b/pixie/vm/libs/c/uv_ffi.c @@ -0,0 +1,19 @@ +#include "uv_ffi.h" +#include "stdlib.h" +#include "stdio.h" + +#define EXPORT __attribute__((visibility("default"))) + +void do_work(work_baton_t *req) +{ + ffi_call(req->cif, req->fn_addr, req->result, req->exb); +} + +EXPORT int uv_ffi_run(work_baton_t *w, uv_loop_t *loop, ffi_cif *cif, void* fn_addr, void *exb, void *result, uv_after_work_cb after_work) +{ + w->fn_addr = fn_addr; + w->cif = cif; + w->exb = exb; + w->result = result; + return uv_queue_work(loop, (uv_work_t *)w, (uv_work_cb)do_work, after_work); +} \ No newline at end of file diff --git a/pixie/vm/libs/c/uv_ffi.h b/pixie/vm/libs/c/uv_ffi.h new file mode 100644 index 00000000..ef0beac5 --- /dev/null +++ b/pixie/vm/libs/c/uv_ffi.h @@ -0,0 +1,25 @@ +#include "uv.h" +#include "ffi.h" + +typedef struct cif_desc_t { + ffi_cif *cif; + int abi; + int nargs; + ffi_type* rtype; + ffi_type** atypes; + int exchange_size; + int exchange_result; + int exchange_result_libffi; + int exchange_args[0]; +} cif_desc_t; + +void (*ffi_cb)(ffi_cif *cif); + +typedef struct work_baton_t +{ + uv_work_t work; + ffi_cif *cif; + void *fn_addr; + void *exb; + void *result; +} work_baton_t; \ No newline at end of file diff --git a/pixie/vm/libs/env.py b/pixie/vm/libs/env.py new file mode 100644 index 00000000..39b806df --- /dev/null +++ b/pixie/vm/libs/env.py @@ -0,0 +1,52 @@ +from pixie.vm.code import as_var +from pixie.vm.object import Object, Type, runtime_error +from pixie.vm.primitives import nil +from pixie.vm.string import String +import pixie.vm.stdlib as proto +from pixie.vm.code import extend, as_var +import pixie.vm.rt as rt +import os + +class Environment(Object): + _type = Type(u"pixie.stdlib.Environment") + + def val_at(self, key, not_found): + if not isinstance(key, String): + runtime_error(u"Environment variables are strings ") + key_str = str(rt.name(key)) + try: + var = os.environ[key_str] + return rt.wrap(var) + except KeyError: + return not_found + + # TODO: Implement me. + # def dissoc(self): + # def asssoc(self): + + def reduce_vars(self, f, init): + for k, v in os.environ.items(): + init = f.invoke([init, rt.map_entry(rt.wrap(k), rt.wrap(v))]) + if rt.reduced_QMARK_(init): + return init + return init + + +@extend(proto._val_at, Environment) +def _val_at(self, key, not_found): + assert isinstance(self, Environment) + v = self.val_at(key, not_found) + return v + +@extend(proto._reduce, Environment) +def _reduce(self, f, init): + assert isinstance(self, Environment) + val = self.reduce_vars(f, init) + if rt.reduced_QMARK_(val): + return rt.deref(val) + + return val + +@as_var("pixie.stdlib", "env") +def _env(): + return Environment() diff --git a/pixie/vm/libs/ffi.py b/pixie/vm/libs/ffi.py new file mode 100644 index 00000000..cdd670b4 --- /dev/null +++ b/pixie/vm/libs/ffi.py @@ -0,0 +1,993 @@ +py_object = object +import rpython.rlib.rdynload as dynload +import pixie.vm.object as object +from pixie.vm.object import runtime_error +from pixie.vm.keyword import Keyword +import pixie.vm.stdlib as proto +from pixie.vm.code import as_var, affirm, extend, wrap_fn +import pixie.vm.rt as rt +from rpython.rtyper.lltypesystem import rffi, lltype, llmemory +from pixie.vm.primitives import nil, true, false +from pixie.vm.numbers import to_float, Integer, Float, BigInteger, Ratio +from pixie.vm.string import String +from pixie.vm.keyword import Keyword +from pixie.vm.util import unicode_to_utf8 +from rpython.rlib import clibffi +from rpython.rlib.jit_libffi import jit_ffi_call, CIF_DESCRIPTION, CIF_DESCRIPTION_P, \ + FFI_TYPE_P, FFI_TYPE_PP, SIZE_OF_FFI_ARG +import rpython.rlib.jit_libffi as jit_libffi +from rpython.rlib.objectmodel import keepalive_until_here, we_are_translated +import rpython.rlib.jit as jit +from rpython.rlib.rarithmetic import intmask + + +""" +FFI interface for pixie. + +This code gets a bit interesting. We use the RPython rlib module jit_libffi to do the interfacing, you can find +good docs in that module. + +""" + +class PointerType(object.Object): + pass + + +class CType(object.Type): + def __init__(self, name): + object.Type.__init__(self, name) + + +class ExternalLib(object.Object): + _type = object.Type(u"pixie.stdlib.ExternalLib") + + def __init__(self, nm): + assert isinstance(nm, unicode) + self._name = nm + self._is_inited = False + self.load_lib() + + def load_lib(self): + if not self._is_inited: + load_paths = rt.deref(rt.deref(rt.load_paths)) + + for x in range(rt.count(load_paths)): + s = rffi.str2charp(str(rt.name(rt.nth(load_paths, rt.wrap(x)))) + "/" + str(self._name)) + try: + self._dyn_lib = dynload.dlopen(s) + self._is_inited = True + except dynload.DLOpenError as ex: + continue + finally: + rffi.free_charp(s) + break + + if not self._is_inited: + s = rffi.str2charp(str(self._name)) + try: + self._dyn_lib = dynload.dlopen(s) + self._is_inited = True + except dynload.DLOpenError as ex: + raise object.runtime_error(u"Couldn't Load Library: " + self._name, + u"pixie.stdlib/LibraryNotFoundException") + finally: + rffi.free_charp(s) + + + + + + + + + def get_fn_ptr(self, nm): + assert isinstance(nm, unicode) + s = rffi.str2charp(str(nm)) + sym = dynload.dlsym(self._dyn_lib, s) + rffi.free_charp(s) + return sym + +class FFIFn(object.Object): + _type = object.Type(u"pixie.stdlib.FFIFn") + _immutable_fields_ = ["_name", "_f_ptr", "_c_fn_type"] + + def __init__(self, name, fn_ptr, c_fn_type): + assert isinstance(c_fn_type, CFunctionType) + self._rev = 0 + self._name = name + self._f_ptr = fn_ptr + self._c_fn_type = c_fn_type + + @jit.unroll_safe + def prep_exb(self, args): + cd = self._c_fn_type.get_cd() + fn_tp = self._c_fn_type + + size = jit.promote(cd.exchange_size) + exb = rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, size, flavor="raw")) + tokens = [None] * len(args) + + for i, tp in enumerate(fn_tp._arg_types): + offset_p = rffi.ptradd(exb, jit.promote(cd.exchange_args[i])) + tokens[i] = tp.ffi_set_value(offset_p, args[i]) + + return exb, tokens + + def get_ret_val_from_buffer(self, exb): + cd = self._c_fn_type.get_cd() + offset_p = rffi.ptradd(exb, jit.promote(cd.exchange_result)) + ret_val = self._c_fn_type._ret_type.ffi_get_value(offset_p) + return ret_val + + @jit.unroll_safe + def _invoke(self, args): + arity = len(args) + tp_arity = len(self._c_fn_type._arg_types) + if self._c_fn_type._is_variadic: + if arity < tp_arity: + runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + + u", expected at least " + unicode(str(tp_arity))) + else: + if arity != tp_arity: + runtime_error(u"Wrong number of args to fn: got " + unicode(str(arity)) + + u", expected " + unicode(str(tp_arity))) + + exb, tokens = self.prep_exb(args) + cd = jit.promote(self._c_fn_type.get_cd()) + #fp = jit.promote(self._f_ptr) + jit_ffi_call(cd, + self._f_ptr, + exb) + ret_val = self.get_ret_val_from_buffer(exb) + + for x in range(len(args)): + t = tokens[x] + if t is not None: + t.finalize_token() + + lltype.free(exb, flavor="raw") + keepalive_until_here(args) + return ret_val + + def invoke(self, args): + self = jit.promote(self) + return self._invoke(args) + +@as_var("ffi-library") +def _ffi_library(ns): + nm = rt.name(ns) + return ExternalLib(nm) + +@as_var("ffi-fn") +def _ffi_fn__args(args): + affirm(len(args) >= 4, u"ffi-fn requires at least 4 arguments") + lib, nm, arg_types, ret_type = args[:4] + + affirm(isinstance(lib, ExternalLib), u"First argument must be an ExternalLib") + affirm(isinstance(ret_type, object.Type), u"Ret type must be a type") + affirm(rt.namespace(nm) is None, u"Name must not be namespaced") + + cnt = rt.count(arg_types) + new_args = [None] * cnt + for x in range(cnt): + t = rt.nth(arg_types, rt.wrap(x)) + affirm(isinstance(t, object.Type), u"Arg defs must be types") + new_args[x] = t + + kwargs = args[4:] + affirm(len(kwargs) & 0x1 == 0, u"ffi-fn requires even number of options") + + is_variadic = False + for i in range(0, len(kwargs)/2, 2): + key = kwargs[i] + val = kwargs[i+1] + + affirm(isinstance(key, Keyword), u"ffi-fn options should be keyword/bool pairs") + affirm(val is true or val is false, u"ffi-fn options should be keyword/bool pairs") + + k = rt.name(key) + if k == u"variadic?": + is_variadic = True if val is true else False + else: + affirm(False, u"unknown ffi-fn option: :" + k) + + tp = CFunctionType(new_args, ret_type, is_variadic) + nm = rt.name(nm) + f = FFIFn(nm, lib.get_fn_ptr(nm), tp) + return f + +@as_var("ffi-voidp") +def _ffi_voidp(lib, nm): + affirm(isinstance(lib, ExternalLib), u"First argument to ffi-voidp should be an external library") + name = rt.name(nm) + return VoidP(lib.get_fn_ptr(name)) + + + + + + + +class Buffer(PointerType): + """ Defines a byte buffer with non-gc'd (therefore non-movable) contents + """ + _type = object.Type(u"pixie.stdlib.Buffer") + + def __init__(self, size): + self._size = size + self._used_size = 0 + self._buffer = lltype.malloc(rffi.CCHARP.TO, size, flavor="raw") + + + def __del__(self): + #lltype.free(self._buffer, flavor="raw") + pass + + def set_used_size(self, size): + self._used_size = size + + def buffer(self): + return self._buffer + + def raw_data(self): + return rffi.cast(rffi.VOIDP, self._buffer) + + def count(self): + return self._used_size + + def nth_char(self, idx): + assert isinstance(idx, int) + return self._buffer[idx] + + def capacity(self): + return self._size + + def free_data(self): + lltype.free(self._buffer, flavor="raw") + + + +@extend(proto._dispose_BANG_, Buffer) +def _dispose_voidp(self): + self.free_data() + + +@extend(proto._nth, Buffer) +def _nth(self, idx): + return rt.wrap(ord(self.nth_char(idx.int_val()))) + +@extend(proto._nth_not_found, Buffer) +def _nth_not_found(self, idx, not_found): + return rt.wrap(ord(self.nth_char(idx.int_val()))) + + +@extend(proto._count, Buffer) +def _count(self): + return rt.wrap(intmask(self.count())) + +@as_var("buffer") +def buffer(size): + return Buffer(size.int_val()) + +@as_var("buffer-capacity") +def buffer_capacity(buffer): + return rt.wrap(intmask(buffer.capacity())) + +@as_var("set-buffer-count!") +def set_buffer_size(self, size): + self.set_used_size(size.int_val()) + return self + +def make_itype(name, ctype, llt): + lltp = lltype.Ptr(lltype.Array(llt, hints={'nolength': True})) + class GenericCInt(CType): + def __init__(self): + CType.__init__(self, name) + + def ffi_get_value(self, ptr): + casted = rffi.cast(lltp, ptr) + return Integer(rffi.cast(rffi.LONG, casted[0])) + + def ffi_set_value(self, ptr, val): + casted = rffi.cast(lltp, ptr) + casted[0] = rffi.cast(llt, val.int_val()) + + def ffi_size(self): + return rffi.sizeof(llt) + + def ffi_type(self): + return ctype + + return GenericCInt() + +from rpython.rlib.rarithmetic import build_int +for x in [8, 16, 32, 64]: + for s in [True, False]: + nm = "C" + ("" if s else "U") + "Int" + str(x) + int_tp = lltype.build_number(None, build_int(nm, s, x)) + ctype = clibffi.cast_type_to_ffitype(int_tp) + make_itype(unicode("pixie.stdlib." + nm), ctype, int_tp) + + + + + + + +class Token(py_object): + """ Tokens are returned by ffi_set_value and are called when ffi is ready to clean up resources + """ + def finalize_token(self): + pass + + + +class CInt(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CInt") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.INTP, ptr) + return Integer(rffi.cast(rffi.LONG, casted[0])) + + def ffi_set_value(self, ptr, val): + casted = rffi.cast(rffi.INTP, ptr) + casted[0] = rffi.cast(rffi.INT, val.int_val()) + + def ffi_size(self): + return rffi.sizeof(rffi.INT) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.INT) +CInt() + +class CFloat(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CFloat") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.FLOATP, ptr) + return Float(rffi.cast(rffi.DOUBLE, casted[0])) + + def ffi_set_value(self, ptr, val): + val = to_float(val) + casted = rffi.cast(rffi.FLOATP, ptr) + casted[0] = rffi.cast(rffi.FLOAT, val.float_val()) + + def ffi_size(self): + return rffi.sizeof(rffi.FLOAT) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.FLOAT) +CFloat() + +class CDouble(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CDouble") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.DOUBLEP, ptr) + return Float(casted[0]) + + def ffi_set_value(self, ptr, val): + val = to_float(val) + casted = rffi.cast(rffi.DOUBLEP, ptr) + casted[0] = rffi.cast(rffi.DOUBLE, val.float_val()) + + def ffi_size(self): + return rffi.sizeof(rffi.DOUBLE) + + def ffi_type(self): + return clibffi.cast_type_to_ffitype(rffi.DOUBLE) +CDouble() + +class CCharP(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CCharP") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.CCHARPP, ptr) + if casted[0] == lltype.nullptr(rffi.CCHARP.TO): + return nil + else: + return String(unicode(rffi.charp2str(casted[0]))) + + def ffi_set_value(self, ptr, val): + if isinstance(val, String): + pnt = rffi.cast(rffi.CCHARPP, ptr) + utf8 = unicode_to_utf8(rt.name(val)) + raw = rffi.str2charp(utf8) + pnt[0] = raw + return CCharPToken(raw) + elif isinstance(val, Buffer): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = val.buffer() + elif isinstance(val, VoidP): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = val.raw_data() + elif val is nil: + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = rffi.cast(rffi.VOIDP, 0) + elif isinstance(val, CStruct): + vpnt = rffi.cast(rffi.VOIDPP, ptr) + vpnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) + else: + frm_name = rt.name(rt.str(val.type())) + to_name = rt.name(rt.str(self)) + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) + + + def ffi_size(self): + return rffi.sizeof(rffi.CCHARP) + + def ffi_type(self): + return clibffi.ffi_type_pointer +CCharP() + +class CCharPToken(Token): + def __init__(self, raw): + self._raw = raw + + def finalize_token(self): + rffi.free_charp(self._raw) + + + +class CVoid(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CVoid") + + def ffi_get_value(self, ptr): + return nil + + def ffi_set_value(self, ptr, val): + runtime_error(u"Can't encode a Void") + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer + +cvoid = CVoid() + +class CVoidP(CType): + def __init__(self): + CType.__init__(self, u"pixie.stdlib.CVoidP") + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.VOIDPP, ptr) + if casted[0] == lltype.nullptr(rffi.VOIDP.TO): + return nil + else: + return VoidP(casted[0]) + + def ffi_set_value(self, ptr, val): + pnt = rffi.cast(rffi.VOIDPP, ptr) + if isinstance(val, String): + pnt = rffi.cast(rffi.CCHARPP, ptr) + utf8 = unicode_to_utf8(rt.name(val)) + raw = rffi.str2charp(utf8) + pnt[0] = raw + return CCharPToken(raw) + elif isinstance(val, Buffer): + pnt[0] = val.buffer() + elif isinstance(val, VoidP): + pnt[0] = val.raw_data() + elif val is nil: + pnt[0] = rffi.cast(rffi.VOIDP, 0) + elif isinstance(val, CStruct): + pnt[0] = rffi.cast(rffi.VOIDP, val.raw_data()) + else: + frm_name = rt.name(rt.str(val.type())) + to_name = rt.name(rt.str(self)) + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) + + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer +cvoidp = CVoidP() + +class VoidP(PointerType): + _type = cvoidp + + def __init__(self, raw_data): + self._raw_data = raw_data + + def raw_data(self): + return rffi.cast(rffi.VOIDP, self._raw_data) + + def free_data(self): + lltype.free(self._raw_data, flavor="raw") + +@extend(proto._dispose_BANG_, cvoidp) +def _dispose_voidp(self): + self.free_data() + + + +@as_var(u"pixie.ffi", u"prep-string") +def prep_string(s): + """Takes a Pixie string and returns a VoidP to that string. The string should be freed via dispose!, otherwise + memory leaks could result.""" + affirm(isinstance(s, String), u"Can only prep strings with prep-string") + utf8 = unicode_to_utf8(rt.name(s)) + raw = rffi.str2charp(utf8) + return VoidP(rffi.cast(rffi.VOIDP, raw)) + +@as_var(u"pixie.ffi", u"unpack") +def unpack(ptr, offset, tp): + """(unpack ptr offset tp) + Reads a value of type tp from offset of ptr.""" + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + affirm(isinstance(tp, CType), u"Packing type must be a CType") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + return tp.ffi_get_value(ptr) + +@as_var(u"pixie.ffi", u"pack!") +def pack(ptr, offset, tp, val): + """(pack! ptr offset tp val) + Writes val at offset of ptr with the format tp""" + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + affirm(isinstance(tp, CType), u"Packing type must be a CType") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + tp.ffi_set_value(ptr, val) + return nil + +@as_var(u"pixie.ffi", u"ptr-add") +def pack(ptr, offset): + affirm(isinstance(ptr, VoidP) or isinstance(ptr, Buffer) or isinstance(ptr, CStruct), u"Type is not unpackable") + ptr = rffi.ptradd(ptr.raw_data(), offset.int_val()) + return VoidP(ptr) + +class CStructType(object.Type): + base_type = object.Type(u"pixie.ffi.CStruct") + _immutable_fields_ = ["_desc", "_size"] + + def __init__(self, name, size, desc): + object.Type.__init__(self, name, CStructType.base_type) + self._desc = desc + self._size = size + #offsets is a dict of {nm, (type, offset)} + + def get_offset(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return offset + + def get_type(self, nm): + (tp, offset) = self._desc.get(nm, (None, 0)) + + assert tp is not None + + return tp + + def get_size(self): + return self._size + + def cast_to(self, frm): + return CStruct(self, frm.raw_data()) + + @jit.elidable_promote() + def get_desc(self, nm): + return self._desc.get(nm, (None, 0)) + + def invoke(self, args): + return CStruct(self, rffi.cast(rffi.VOIDP, lltype.malloc(rffi.CCHARP.TO, self._size, flavor="raw"))) + + +registered_callbacks = {} + +class CCallback(PointerType): + _type = object.Type(u"pixie.ffi.CCallback") + + def __init__(self, cft, raw_closure, id, fn): + self._fn = fn + self._cft = cft + self._raw_closure = raw_closure + self._is_invoked = False + self._unique_id = id + + def get_raw_closure(self): + return self._raw_closure + + def ll_invoke(self, llargs, llres): + cft = self._cft + assert isinstance(cft, CFunctionType) + + args = [None] * len(cft._arg_types) + for i, tp in enumerate(cft._arg_types): + args[i] = tp.ffi_get_value(llargs[i]) + + self._is_invoked = True + retval = self._fn.invoke(args) + if cft._ret_type is not cvoid: + cft._ret_type.ffi_set_value(llres, retval) + + + def cleanup(self): + del registered_callbacks[self._unique_id] + clibffi.closureHeap.free(self._raw_closure) + +@extend(proto._dispose_BANG_, CCallback) +def _dispose(self): + self.cleanup() + + + +@as_var(u"pixie.ffi", u"ffi-callback") +def ffi_callback(args, ret_type): + """(ffi-callback args ret-type) + Creates a ffi callback type. Args is a vector of CType args. Ret-type is the CType return + type of the callback. Returns a ffi callback type that can be used with ffi-prep-callback.""" + args_w = [None] * rt.count(args) + + for x in range(rt.count(args)): + arg = rt.nth(args, rt.wrap(x)) + if not isinstance(arg, object.Type): + runtime_error(u"Expected type, got " + rt.name(rt.str(arg))) + args_w[x] = arg + + if not isinstance(ret_type, object.Type): + runtime_error(u"Expected type, got " + rt.name(rt.str(ret_type))) + + return CFunctionType(args_w, ret_type) + + +class TranslatedIDGenerator(py_object): + def __init__(self): + self._val = 0 + + def get_next(self): + self._val += 1 + return self._val + +id_generator = TranslatedIDGenerator() + +@as_var(u"pixie.ffi", u"ffi-prep-callback") +def ffi_prep_callback(tp, f): + """(ffi-prep-callback callback-tp fn) + Prepares a Pixie function for use as a c callback. callback-tp is a ffi callback type, + fn is a pixie function (can be a closure, native fn or any object that implements -invoke. + Returns a function pointer that can be passed to c and invoked as a callback.""" + affirm(isinstance(tp, CFunctionType), u"First argument to ffi-prep-callback must be a CFunctionType") + raw_closure = rffi.cast(rffi.VOIDP, clibffi.closureHeap.alloc()) + + if not we_are_translated(): + unique_id = id_generator.get_next() + else: + unique_id = rffi.cast(lltype.Signed, raw_closure) + + res = clibffi.c_ffi_prep_closure(rffi.cast(clibffi.FFI_CLOSUREP, raw_closure), tp.get_cd().cif, + invoke_callback, + rffi.cast(rffi.VOIDP, unique_id)) + + + if rffi.cast(lltype.Signed, res) != clibffi.FFI_OK: + registered_callbacks[unique_id] = None + runtime_error(u"libffi failed to build this callback") + + cb = CCallback(tp, raw_closure, unique_id, f) + registered_callbacks[unique_id] = cb + + return cb + +# Callback Code + +@jit.jit_callback("CFFI") +def invoke_callback(ffi_cif, ll_res, ll_args, ll_userdata): + cb = registered_callbacks[rffi.cast(rffi.INT_real, ll_userdata)] + cb.ll_invoke(ll_args, ll_res) + +class FunctionTypeNameGenerator(py_object): + def __init__(self): + self._idx = 0 + def next(self): + self._idx += 1 + return unicode("CFunctionType" + str(self._idx)) + +name_gen = FunctionTypeNameGenerator() + +class CFunctionType(object.Type): + base_type = object.Type(u"pixie.ffi.CType") + _immutable_fields_ = ["_arg_types", "_ret_type", "_cd", "_is_variadic"] + + def __init__(self, arg_types, ret_type, is_variadic=False): + object.Type.__init__(self, name_gen.next(), CStructType.base_type) + self._arg_types = arg_types + self._ret_type = ret_type + self._is_variadic = is_variadic + self._cd = CifDescrBuilder(self._arg_types, self._ret_type).rawallocate() + + def ffi_get_value(self, ptr): + casted = rffi.cast(rffi.VOIDPP, ptr) + return FFIFn(u"", casted[0], self) + + def ffi_set_value(self, ptr, val): + if isinstance(val, CCallback): + casted = rffi.cast(rffi.VOIDPP, ptr) + casted[0] = val.get_raw_closure() + elif val is nil: + casted = rffi.cast(rffi.VOIDPP, ptr) + casted[0] = rffi.cast(rffi.VOIDP, 0) + else: + frm_name = rt.name(rt.str(val.type())) + to_name = rt.name(rt.str(self)) + affirm(False, u"Cannot encode " + frm_name + u" as " + to_name) + + return None + + def get_cd(self): + return self._cd + + def ffi_size(self): + return rffi.sizeof(rffi.VOIDP) + + def ffi_type(self): + return clibffi.ffi_type_pointer + +class CStruct(PointerType): + _immutable_fields_ = ["_type", "_buffer"] + def __init__(self, tp, buffer): + self._type = tp + self._buffer = buffer + + def type(self): + return self._type + + def raw_data(self): + return rffi.cast(rffi.VOIDP, self._buffer) + + def val_at(self, k, not_found): + (tp, offset) = self._type.get_desc(k) + + if tp is None: + return not_found + + offset = rffi.ptradd(self._buffer, offset) + return tp.ffi_get_value(rffi.cast(rffi.VOIDP, offset)) + + def set_val(self, k, v): + (tp, offset) = self._type.get_desc(k) + + if tp is None: + runtime_error(u"Invalid field name: " + rt.name(rt.str(k))) + + offset = rffi.ptradd(self._buffer, offset) + tp.ffi_set_value(rffi.cast(rffi.VOIDP, offset), v) + + return nil + + def free_data(self): + lltype.free(self._buffer, flavor="raw") + +@wrap_fn +def _dispose_cstruct(self): + self.free_data() + + + +@as_var("pixie.ffi", "c-struct") +def c_struct(name, size, spec): + """(c-struct name size spec) + Creates a CStruct named name, of size size, with the given spec. Spec is a vector + of vectors. Each row of the format [field-name type offset]""" + d = {} + for x in range(rt.count(spec)): + row = rt.nth(spec, rt.wrap(x)) + nm = rt.nth(row, rt.wrap(0)) + tp = rt.nth(row, rt.wrap(1)) + offset = rt.nth(row, rt.wrap(2)) + + affirm(isinstance(nm, Keyword), u"c-struct field names must be keywords") + #if not isinstance(tp, CType): + # runtime_error(u"c-struct field types must be c types, got: " + rt.name(rt.str(tp))) + + d[nm] = (tp, offset.int_val()) + + tp = CStructType(rt.name(name), size.int_val(), d) + proto._dispose_BANG_.extend(tp, _dispose_cstruct) + return tp + +@as_var("pixie.ffi", "cast") +def c_cast(frm, to): + """(cast from to) + Converts a VoidP to a CStruct. From is either a VoidP or a CStruct, to is a CStruct type.""" + if not isinstance(to, CStructType): + runtime_error(u"Expected a CStruct type to cast to, got " + rt.name(rt.str(to))) + + if not isinstance(frm, CStruct) and not isinstance(frm, VoidP): + runtime_error(u"From must be a CVoidP or a CStruct, got " + rt.name(rt.str(frm))) + + return to.cast_to(frm) + +@as_var("pixie.ffi", "struct-size") +def struct_size(tp): + """(struct-size tp) + Gives the size of the given CStruct type tp, in bytes.""" + if not isinstance(tp, CStructType): + runtime_error(u"Expected a CStruct type to get the size of, got " + rt.name(rt.str(tp))) + + return rt.wrap(tp.get_size()) + +@extend(proto._val_at, CStructType.base_type) +def val_at(self, k, not_found): + return self.val_at(k, not_found) + +@as_var("pixie.ffi", "set!") +def set_(self, k, val): + """(set! ptr k val) + Sets a field k of struct ptr to value val""" + return self.set_val(k, val) + + + +@as_var("pixie.ffi", "prep-ffi-call") +def prep_ffi_call__args(args): + fn = args[0] + affirm(isinstance(fn, CFunctionType), u"First arg must be a FFI function") + + +def comp_ptrs(a, b): + assert isinstance(a, PointerType) + if not isinstance(b, PointerType): + return false + if a.raw_data() == b.raw_data(): + return true + return false + +def hash_ptr(a): + assert isinstance(a, PointerType) + hashval = rffi.cast(lltype.Signed, a.raw_data()) + return rt.wrap(hashval) + +extend(proto._eq, Buffer)(comp_ptrs) +extend(proto._eq, CStructType.base_type)(comp_ptrs) +extend(proto._eq, VoidP)(comp_ptrs) + +extend(proto._hash, Buffer)(hash_ptr) +extend(proto._hash, CStructType.base_type)(hash_ptr) +extend(proto._hash, VoidP)(hash_ptr) + + +import sys + +USE_C_LIBFFI_MSVC = getattr(clibffi, 'USE_C_LIBFFI_MSVC', False) + + + +class CifDescrBuilder(py_object): + rawmem = lltype.nullptr(rffi.CCHARP.TO) + + def __init__(self, fargs, fresult): + self.fargs = fargs + self.fresult = fresult + + def fb_alloc(self, size): + size = llmemory.raw_malloc_usage(size) + if not self.bufferp: + self.nb_bytes += size + return lltype.nullptr(rffi.CCHARP.TO) + else: + result = self.bufferp + self.bufferp = rffi.ptradd(result, size) + return result + + def fb_fill_type(self, ctype, is_result_type): + return ctype.ffi_type() + + + def fb_build(self): + # Build a CIF_DESCRIPTION. Actually this computes the size and + # allocates a larger amount of data. It starts with a + # CIF_DESCRIPTION and continues with data needed for the CIF: + # + # - the argument types, as an array of 'ffi_type *'. + # + # - optionally, the result's and the arguments' ffi type data + # (this is used only for 'struct' ffi types; in other cases the + # 'ffi_type *' just points to static data like 'ffi_type_sint32'). + # + nargs = len(self.fargs) + + # start with a cif_description (cif and exchange_* fields) + self.fb_alloc(llmemory.sizeof(CIF_DESCRIPTION, nargs)) + + # next comes an array of 'ffi_type*', one per argument + atypes = self.fb_alloc(rffi.sizeof(FFI_TYPE_P) * nargs) + self.atypes = rffi.cast(FFI_TYPE_PP, atypes) + + # next comes the result type data + self.rtype = self.fb_fill_type(self.fresult, True) + + # next comes each argument's type data + for i, farg in enumerate(self.fargs): + atype = self.fb_fill_type(farg, False) + if self.atypes: + self.atypes[i] = atype + + def align_arg(self, n): + return (n + 7) & ~7 + + def fb_build_exchange(self, cif_descr): + nargs = len(self.fargs) + + # first, enough room for an array of 'nargs' pointers + exchange_offset = rffi.sizeof(rffi.VOIDP) * nargs + exchange_offset = self.align_arg(exchange_offset) + cif_descr.exchange_result = exchange_offset + + # then enough room for the result, rounded up to sizeof(ffi_arg) + exchange_offset += max(rffi.getintfield(self.rtype, 'c_size'), + SIZE_OF_FFI_ARG) + + # loop over args + for i, farg in enumerate(self.fargs): + #if isinstance(farg, W_CTypePointer): + # exchange_offset += 1 # for the "must free" flag + exchange_offset = self.align_arg(exchange_offset) + cif_descr.exchange_args[i] = exchange_offset + exchange_offset += rffi.getintfield(self.atypes[i], 'c_size') + + # store the exchange data size + cif_descr.exchange_size = exchange_offset + + def fb_extra_fields(self, cif_descr): + cif_descr.abi = clibffi.FFI_DEFAULT_ABI # XXX + cif_descr.nargs = len(self.fargs) + cif_descr.rtype = self.rtype + cif_descr.atypes = self.atypes + + @jit.dont_look_inside + def rawallocate(self): + + # compute the total size needed in the CIF_DESCRIPTION buffer + self.nb_bytes = 0 + self.bufferp = lltype.nullptr(rffi.CCHARP.TO) + self.fb_build() + + # allocate the buffer + if we_are_translated(): + rawmem = lltype.malloc(rffi.CCHARP.TO, self.nb_bytes, + flavor='raw') + rawmem = rffi.cast(CIF_DESCRIPTION_P, rawmem) + else: + # gross overestimation of the length below, but too bad + rawmem = lltype.malloc(CIF_DESCRIPTION_P.TO, self.nb_bytes, + flavor='raw') + + # the buffer is automatically managed from the W_CTypeFunc instance + # ctypefunc._cd = rawmem + + # call again fb_build() to really build the libffi data structures + self.bufferp = rffi.cast(rffi.CCHARP, rawmem) + self.fb_build() + assert self.bufferp == rffi.ptradd(rffi.cast(rffi.CCHARP, rawmem), + self.nb_bytes) + + # fill in the 'exchange_*' fields + self.fb_build_exchange(rawmem) + + # fill in the extra fields + self.fb_extra_fields(rawmem) + + # call libffi's ffi_prep_cif() function + res = jit_libffi.jit_ffi_prep_cif(rawmem) + if res != clibffi.FFI_OK: + runtime_error(u"libffi failed to build function type") + + return rawmem + + def get_item(self, nm): + (tp, offset) = self._type.get_desc(nm) + ptr = rffi.ptradd(self._buffer, offset) + return tp.ffi_load_from(ptr) + + +# Taken from PyPy + diff --git a/pixie/vm/libs/readline.py b/pixie/vm/libs/libedit.py similarity index 74% rename from pixie/vm/libs/readline.py rename to pixie/vm/libs/libedit.py index 3aa8ec0b..090eff3f 100644 --- a/pixie/vm/libs/readline.py +++ b/pixie/vm/libs/libedit.py @@ -1,7 +1,8 @@ import py +from pixie.vm.util import unicode_from_utf8 + from rpython.rtyper.lltypesystem import lltype, rffi -from rpython.rtyper.lltypesystem.lloperation import llop from rpython.translator import cdir from rpython.translator.tool.cbuild import ExternalCompilationInfo @@ -9,11 +10,11 @@ srcdir = py.path.local(cdir) / 'src' compilation_info = ExternalCompilationInfo( - includes=['readline/readline.h'], - libraries=["readline"]) + includes=['editline/readline.h'], + libraries=["edit"]) def llexternal(*args, **kwargs): - return rffi.llexternal(*args, compilation_info=compilation_info, **kwargs) + return rffi.llexternal(*args, compilation_info=compilation_info, releasegil=True, **kwargs) __readline = llexternal('readline', [rffi.CCHARP], rffi.CCHARP) @@ -22,4 +23,4 @@ def _readline(prompt): if result == lltype.nullptr(rffi.CCHARP.TO): return u"" else: - return unicode(rffi.charp2str(result)) + return unicode_from_utf8(rffi.charp2str(result)) + u"\n" diff --git a/pixie/vm/libs/path.py b/pixie/vm/libs/path.py new file mode 100644 index 00000000..e512efe8 --- /dev/null +++ b/pixie/vm/libs/path.py @@ -0,0 +1,87 @@ +import pixie.vm.rt as rt +from pixie.vm.code import as_var, extend +from pixie.vm.object import Object, Type +import pixie.vm.stdlib as proto +from pixie.vm.primitives import true, false +import os + +class Path(Object): + _type = Type(u"pixie.path.Path") + + def __init__(self, top): + self._path = rt.name(top) + + # keyword args don't seem to work nicely. + #def rel_path(self, other): + # "Returns the path relative to other path" + # return rt.wrap(str(os.path.relpath(self._path, start=other._path))) + + def abs_path(self): + "Returns the absolute path" + return rt.wrap(os.path.abspath(str(self._path))) + + # Basename doesn't play well with pypy... + #def basename(self): + # return rt.wrap(rt.name(os.path.basename("a"))) + + def exists(self): + return true if os.path.exists(str(self._path)) else false + + def is_file(self): + return true if os.path.isfile(str(self._path)) else false + + def is_dir(self): + return true if os.path.isdir(str(self._path)) else false + +@extend(proto._reduce, Path) +def _reduce(self, f, init): + assert isinstance(self, Path) + for dirpath, dirnames, filenames in os.walk(str(self._path)): + for dirname in dirnames: + init = f.invoke([init, Path(rt.wrap(dirpath + "/" + dirname))]) + if rt.reduced_QMARK_(init): + return rt.deref(init) + + for filename in filenames: + init = f.invoke([init, Path(rt.wrap(dirpath + "/" + filename))]) + if rt.reduced_QMARK_(init): + return rt.deref(init) + + return init + +# I have named prefixed all names with '-' to deal with the +# a namespace issue I was having. +# TODO: remove '-' and update calling functions when issue is fixed. + +@as_var("pixie.path", "-path") +def path(path): + return Path(path) + +# TODO: Implement this +@as_var("pixie.path", "-list-dir") +def list_dir(self): + assert isinstance(self, Path) + result = rt.vector() + for item in os.listdir(str(self._path)): + result = rt.conj(result, rt.wrap(str(self._path) + "/" + str(item))) + return result + +@as_var("pixie.path", "-abs") +def _abs(self): + assert isinstance(self, Path) + return self.abs_path() + +@as_var("pixie.path", "-exists?") +def exists_QMARK_(self): + assert isinstance(self, Path) + return self.exists() + +@as_var("pixie.path", "-file?") +def file_QMARK_(self): + assert isinstance(self, Path) + return self.is_file() + +@as_var("pixie.path", "-dir?") +def dir_QMARK_(self): + assert isinstance(self, Path) + return self.is_dir() diff --git a/pixie/vm/libs/platform.py b/pixie/vm/libs/platform.py new file mode 100644 index 00000000..ea3ec32a --- /dev/null +++ b/pixie/vm/libs/platform.py @@ -0,0 +1,23 @@ +from rpython.translator.platform import platform +from pixie.vm.persistent_list import create_from_list +from pixie.vm.string import String +from pixie.vm.code import as_var +from rpython.rlib.clibffi import get_libc_name +import os + + +as_var("pixie.platform", "os")(String(unicode(os.name))) +as_var("pixie.platform", "name")(String(unicode(platform.name))) +as_var("pixie.platform", "so-ext")(String(unicode(platform.so_ext))) +as_var("pixie.platform", "lib-c-name")(String(unicode(get_libc_name()))) + +c_flags = [] +for itm in platform.cflags: + c_flags.append(String(unicode(itm))) + +as_var("pixie.platform", "c-flags")(create_from_list(c_flags)) + +link_flags = [] +for itm in platform.link_flags: + c_flags.append(String(unicode(itm))) +as_var("pixie.platform", "link-flags")(create_from_list(link_flags)) diff --git a/pixie/vm/libs/pxic/__init__.py b/pixie/vm/libs/pxic/__init__.py new file mode 100644 index 00000000..f87606f0 --- /dev/null +++ b/pixie/vm/libs/pxic/__init__.py @@ -0,0 +1 @@ +__author__ = 'tim' diff --git a/pixie/vm/libs/pxic/reader.py b/pixie/vm/libs/pxic/reader.py new file mode 100644 index 00000000..2cc5d799 --- /dev/null +++ b/pixie/vm/libs/pxic/reader.py @@ -0,0 +1,215 @@ +from pixie.vm.libs.pxic.tags import * +from pixie.vm.object import runtime_error, get_type_by_name +from rpython.rlib.runicode import str_decode_utf_8 +from pixie.vm.string import String +from pixie.vm.keyword import Keyword, keyword +from pixie.vm.symbol import Symbol, symbol +from pixie.vm.numbers import Integer, Float, BigInteger +from pixie.vm.code import Code, Var, NativeFn, Namespace, intern_var +import pixie.vm.code as code +from pixie.vm.primitives import nil, true, false +from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP +from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR +from pixie.vm.persistent_list import create_from_list +from pixie.vm.reader import LinePromise +from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rbigint import rbigint +from pixie.vm.libs.pxic.util import read_handlers +import pixie.vm.rt as rt + + +class Reader(object): + def __init__(self, rdr): + self._rdr = rdr + self._obj_cache = {} + self._str_cache = {} + + def read(self, num=r_uint(1)): + return self._rdr.read(intmask(num)) + + def read_and_cache(self): + idx = len(self._obj_cache) + self._obj_cache[idx] = None # To match cache size growth of writer + obj = read_obj(self) + self._obj_cache[idx] = obj + return obj + + def read_cached_string(self): + sz = read_raw_integer(self) + if sz >= MAX_STRING_SIZE: + return self._str_cache[sz - MAX_STRING_SIZE] + else: + s, pos = str_decode_utf_8(self.read(sz), sz, "?") + self._str_cache[len(self._str_cache)] = s + return s + + def read_cached_obj(self): + idx = read_raw_integer(self) + return self._obj_cache[idx] + + +def read_tag(rdr): + tag = rdr.read() + return ord(tag[0]) + +def read_raw_integer(rdr): + return r_uint(ord(rdr.read()[0]) | (ord(rdr.read()[0]) << 8) | (ord(rdr.read()[0]) << 16) | (ord(rdr.read()[0]) << 24)) + +def read_raw_bigint(rdr): + nchars = read_raw_integer(rdr) + n = rbigint.fromint(0) + for i in range(nchars): + a = rbigint.fromint(ord(rdr.read()[0])) + a = a.lshift(8*i) + n = n.add(a) + return n + +def read_raw_string(rdr): + s = rdr.read_cached_string() + return s + +def read_code(rdr): + sz = read_raw_integer(rdr) + bytecode = [r_uint(0)] * sz + for x in range(sz): + bytecode[x] = read_raw_integer(rdr) + + sz = read_raw_integer(rdr) + consts = [None] * sz + for x in range(sz): + consts[x] = read_obj(rdr) + + stack_size = read_raw_integer(rdr) + + nm = read_raw_string(rdr) + + arity = read_raw_integer(rdr) + + dps = read_raw_integer(rdr) + debug_points = {} + for x in range(dps): + idx = read_raw_integer(rdr) + debug_points[idx] = read_obj(rdr) + + return Code(nm, arity, bytecode, consts, stack_size, debug_points) + + +def read_var(rdr): + ns = read_raw_string(rdr) + nm = read_raw_string(rdr) + is_dynamic = read_tag(rdr) + var = intern_var(ns, nm) + if is_dynamic is TRUE: + var.set_dynamic() + return var + +def read_map(rdr): + cnt = read_raw_integer(rdr) + acc = EMPTY_MAP + for x in range(cnt): + acc = rt._assoc(acc, read_obj(rdr), read_obj(rdr)) + + return acc + +def read_vector(rdr): + cnt = read_raw_integer(rdr) + acc = EMPTY_VECTOR + for x in range(cnt): + acc = rt._conj(acc, read_obj(rdr)) + + return acc + +def read_seq(rdr): + cnt = read_raw_integer(rdr) + lst = [None] * cnt + for x in range(cnt): + lst[x] = read_obj(rdr) + + return create_from_list(lst) + +def read_float(rdr): + return Float(float(str(read_raw_string(rdr)))) + +def read_namespace(rdr): + nm = read_raw_string(rdr) + return code._ns_registry.find_or_make(nm) + +def read_interpreter_code_info(rdr): + from pixie.vm.object import InterpreterCodeInfo + line = read_obj(rdr) + line_number = read_raw_integer(rdr) + column_number = read_raw_integer(rdr) + file = read_raw_string(rdr) + return InterpreterCodeInfo(line, intmask(line_number), intmask(column_number), file) + +def read_obj(rdr): + tag = read_tag(rdr) + + if tag == INT: + return Integer(intmask(read_raw_integer(rdr))) + elif tag == BIGINT: + return BigInteger(read_raw_bigint(rdr)) + elif tag == CODE: + return read_code(rdr) + elif tag == NIL: + return nil + elif tag == VAR: + return read_var(rdr) + elif tag == STRING: + return String(read_raw_string(rdr)) + elif tag == KEYWORD: + return keyword(read_raw_string(rdr)) + elif tag == SYMBOL: + return symbol(read_raw_string(rdr)) + elif tag == LINE_PROMISE: + lp = LinePromise() + lp._chrs = None + lp._str = read_raw_string(rdr) + return lp + elif tag == MAP: + return read_map(rdr) + elif tag == TRUE: + return true + elif tag == FALSE: + return false + elif tag == NIL: + return nil + elif tag == VECTOR: + return read_vector(rdr) + elif tag == SEQ: + return read_seq(rdr) + elif tag == FLOAT: + return read_float(rdr) + elif tag == NAMESPACE: + return read_namespace(rdr) + elif tag == INT_STRING: + return Integer(int(read_raw_string(rdr))) + elif tag == BIGINT_STRING: + return BigInteger(rbigint.fromstr(str(read_raw_string(rdr)))) + + + elif tag == NEW_CACHED_OBJ: + return rdr.read_and_cache() + elif tag == CACHED_OBJ: + return rdr.read_cached_obj() + + elif tag == EOF: + from pixie.vm.reader import eof + return eof + + elif tag == CODE_INFO: + return read_interpreter_code_info(rdr) + + elif tag == TAGGED: + tp_name = read_raw_string(rdr) + tp = get_type_by_name(tp_name) + handler = read_handlers.get(tp, None) + if handler is None: + runtime_error(u"No type handler for " + tp_name) + + obj = read_obj(rdr) + return handler.invoke([obj]) + else: + runtime_error(u"No dispatch for bytecode: " + unicode(tag_name[tag])) + + return nil diff --git a/pixie/vm/libs/pxic/tags.py b/pixie/vm/libs/pxic/tags.py new file mode 100644 index 00000000..eb7694f8 --- /dev/null +++ b/pixie/vm/libs/pxic/tags.py @@ -0,0 +1,38 @@ + +tag_name = ["INT", + "BIGINT", + "FLOAT", + "INT_STRING", + "BIGINT_STRING", + "STRING", + "CODE", + "TRUE", + "FALSE", + "NIL", + "VAR", + "MAP", + "VECTOR", + "SEQ", + "KEYWORD", + "SYMBOL", + "CACHED_OBJ", + "NEW_CACHED_OBJ", + "LINE_PROMISE", + "NAMESPACE", + "TAGGED", + "CODE_INFO", + "EOF"] + +tags = {} + + +SMALL_INT_START = 128 +SMALL_INT_END = 255 +SMALL_INT_MAX = SMALL_INT_END - SMALL_INT_END + +MAX_STRING_SIZE = 1 << 30 + +for idx, nm in enumerate(tag_name): + globals()[nm] = idx + tags[nm] = idx + diff --git a/pixie/vm/libs/pxic/util.py b/pixie/vm/libs/pxic/util.py new file mode 100644 index 00000000..01f6e5c1 --- /dev/null +++ b/pixie/vm/libs/pxic/util.py @@ -0,0 +1,9 @@ + + +read_handlers = {} +write_handlers = {} + +def add_marshall_handlers(tp, write, read): + read_handlers[tp] = read + write_handlers[tp] = write + diff --git a/pixie/vm/libs/pxic/writer.py b/pixie/vm/libs/pxic/writer.py new file mode 100644 index 00000000..0d2e511a --- /dev/null +++ b/pixie/vm/libs/pxic/writer.py @@ -0,0 +1,305 @@ +from pixie.vm.libs.pxic.tags import * +from pixie.vm.object import runtime_error, Object, Type, InterpreterCodeInfo +from rpython.rlib.runicode import unicode_encode_utf_8 +from pixie.vm.string import String +from pixie.vm.keyword import Keyword +from pixie.vm.symbol import Symbol +from pixie.vm.numbers import Integer, BigInteger, Float +from pixie.vm.code import Code, Var, NativeFn, Namespace +from pixie.vm.primitives import nil, true, false +from pixie.vm.reader import LinePromise +from rpython.rlib.objectmodel import specialize +from rpython.rlib.rarithmetic import r_uint +from rpython.rlib.rbigint import rbigint +import pixie.vm.rt as rt + +MAX_INT32 = r_uint(1 << 31) + +class Writer(object): + def __init__(self, wtr, with_cache=False): + self._wtr = wtr + self._obj_cache = {} + self._string_cache = {} + self._with_cache = with_cache + + def write(self, s): + assert isinstance(s, str) + self._wtr.write(s) + + def flush(self): + self._wtr.flush() + + def write_cached_obj(self, o, wfn): + if self._with_cache: + idx = self._obj_cache.get(o, -1) + if idx == -1: + idx = len(self._obj_cache) + self._obj_cache[o] = idx + write_tag(NEW_CACHED_OBJ, self) + wfn(o, self) + else: + write_tag(CACHED_OBJ, self) + write_int_raw(r_uint(idx), self) + else: + return wfn(o, self) + + def write_raw_cached_string(self, si): + assert isinstance(si, unicode) + if self._with_cache: + idx = self._string_cache.get(si, -1) + if idx == -1: + idx = len(self._string_cache) + self._string_cache[si] = idx + s = unicode_encode_utf_8(si, len(si), "?") + write_int_raw(len(s), self) + assert len(s) <= MAX_STRING_SIZE + self.write(s) + else: + write_int_raw(r_uint(MAX_STRING_SIZE + idx), self) + else: + errors = "?" + s = unicode_encode_utf_8(si, len(si), errors) + assert len(s) <= MAX_INT32 + write_int_raw(len(s), self) + self.write(s) + + + def write_object(self, o): + write_object(o, self) + + def finish(self): + write_tag(EOF, self) + self._wtr.flush() + +class WriterBox(Object): + _type = Type(u"pixie.stdlib.WriterBox") + + def __init__(self, wtr): + self._pxic_writer = wtr + + def get_pxic_writer(self): + return self._pxic_writer + +def write_tag(tag, wtr): + assert tag <= 0xFF + wtr.write(chr(tag)) + +def write_int_raw(i, wtr): + #if 0 <= i <= SMALL_INT_MAX: + # wtr.write(chr((i & 0xFF) + SMALL_INT_START)) + if 0 <= i <= MAX_INT32: + wtr.write(chr(i & 0xFF)) + wtr.write(chr((i >> 8) & 0xFF)) + wtr.write(chr((i >> 16) & 0xFF)) + wtr.write(chr((i >> 24) & 0xFF)) + else: + runtime_error(u"Raw int must be less than MAX_INT32, got: " + unicode(str(i))) + +def write_string_raw(si, wtr): + wtr.write_raw_cached_string(si) + +def write_bigint_raw(i, wtr): + bits = i.bit_length() + nchars = r_uint(bits / 8) + if (bits) % 8 != 0: + nchars += 1 + assert nchars <= MAX_INT32 + write_int_raw(nchars, wtr) # nchars used to represent the bigint + for j in range(nchars): + wtr.write(chr((i.rshift(j * 8).int_and_(0xFF).toint()))) + +def write_int(i, wtr): + if 0 <= i <= MAX_INT32: + wtr.write(chr(INT)) + write_int_raw(i, wtr) + else: + wtr.write(chr(INT_STRING)) + write_string_raw(unicode(str(i)), wtr) + +def write_bigint(i, wtr): + if i.int_ge(0): + wtr.write(chr(BIGINT)) + write_bigint_raw(i, wtr) + else: + wtr.write(chr(BIGINT_STRING)) + write_string_raw(unicode(i.str()), wtr) + +def write_float(f, wtr): + write_tag(FLOAT, wtr) + write_string_raw(unicode(str(f)), wtr) + +def write_string(s, wtr): + write_tag(STRING, wtr) + write_string_raw(s, wtr) + +def write_code(c, wtr): + assert isinstance(c, Code) + wtr.write(chr(CODE)) + + write_int_raw(len(c._bytecode), wtr) + for i in c._bytecode: + write_int_raw(i, wtr) + + write_int_raw(len(c._consts), wtr) + for const in c._consts: + write_object(const, wtr) + + write_int_raw(c._stack_size, wtr) + + + write_string_raw(c._name, wtr) + + write_int_raw(c._arity, wtr) + + write_int_raw(len(c._debug_points), wtr) + for k, v in c._debug_points.iteritems(): + write_int_raw(k, wtr) + write_object(v, wtr) + + + +class WriteParirFn(NativeFn): + def __init__(self, wtr): + self._wtr = wtr + + def invoke(self, args): + kv = args[1] + + write_object(rt._key(kv), self._wtr) + write_object(rt._val(kv), self._wtr) + + return nil + + +def write_map(mp, wtr): + write_tag(MAP, wtr) + write_int_raw(rt.count(mp), wtr) + + rt._reduce(mp, WriteParirFn(wtr), nil) + +class WriteItem(NativeFn): + def __init__(self, wtr): + self._wtr = wtr + + def invoke(self, args): + itm = args[1] + + write_object(itm, self._wtr) + + return nil + + +def write_vector(vec, wtr): + write_tag(VECTOR, wtr) + write_int_raw(rt.count(vec), wtr) + + rt._reduce(vec, WriteItem(wtr), nil) + +def write_seq(s, wtr): + write_tag(SEQ, wtr) + write_int_raw(rt.count(s), wtr) + + s = rt.seq(s) + + while s is not nil: + write_object(rt.first(s), wtr) + s = rt.next(s) + + + + +# def __init__(self, name, bytecode, consts, stack_size, debug_points, meta=nil): +# BaseCode.__init__(self) +# self._bytecode = bytecode +# self._consts = consts +# self._name = name +# self._stack_size = stack_size +# self._debug_points = debug_points +# self._meta = meta + +def write_var(var, wtr): + assert isinstance(var, Var) + write_tag(VAR, wtr) + write_string_raw(var._ns, wtr) + write_string_raw(var._name, wtr) + write_tag(TRUE if var.is_dynamic() else FALSE, wtr) + + +def write_keyword(kw, wtr): + assert isinstance(kw, Keyword) + write_tag(KEYWORD, wtr) + write_string_raw(kw._str, wtr) + +def write_symbol(sym, wtr): + assert isinstance(sym, Symbol) + write_tag(SYMBOL, wtr) + write_string_raw(sym._str, wtr) + +def write_line_promise(o, wtr): + assert isinstance(o, LinePromise) + write_tag(LINE_PROMISE, wtr) + o.finalize() + write_string_raw(o._str, wtr) + +def write_namespace(o, wtr): + assert isinstance(o, Namespace) + write_tag(NAMESPACE, wtr) + write_string_raw(o._name, wtr) + +def write_interpreter_code_info(obj, wtr): + line, line_number, column_number, file = obj.interpreter_code_info_state() + write_tag(CODE_INFO, wtr) + + write_object(line, wtr) + write_int_raw(r_uint(line_number), wtr) + write_int_raw(r_uint(column_number), wtr) + write_string_raw(file, wtr) + + +def write_object(obj, wtr): + wtr.flush() + if isinstance(obj, String): + write_string(rt.name(obj), wtr) + elif isinstance(obj, Integer): + write_int(obj.int_val(), wtr) + elif isinstance(obj, BigInteger): #TODO test + write_bigint(obj.bigint_val(), wtr) + elif isinstance(obj, Float): + write_float(obj.float_val(), wtr) + elif isinstance(obj, Code): + write_code(obj, wtr) + elif obj is nil: + wtr.write(chr(NIL)) + elif isinstance(obj, Var): + #wtr.write_cached_obj(obj, write_var) + write_var(obj, wtr) + elif rt._satisfies_QMARK_(rt.IMap.deref(), obj): + write_map(obj, wtr) + elif rt._satisfies_QMARK_(rt.IVector.deref(), obj): + write_vector(obj, wtr) + elif rt._satisfies_QMARK_(rt.ISeq.deref(), obj): + write_seq(obj, wtr) + elif isinstance(obj, Keyword): + wtr.write_cached_obj(obj, write_keyword) + elif isinstance(obj, LinePromise): + wtr.write_cached_obj(obj, write_line_promise) + elif obj is true: + write_tag(TRUE, wtr) + elif obj is false: + write_tag(FALSE, wtr) + elif isinstance(obj, Symbol): + write_symbol(obj, wtr) + elif isinstance(obj, Namespace): + wtr.write_cached_obj(obj, write_namespace) + elif isinstance(obj, InterpreterCodeInfo): + wtr.write_cached_obj(obj, write_interpreter_code_info) + else: + from pixie.vm.libs.pxic.util import write_handlers + handler = write_handlers.get(obj.type(), None) + if handler is None: + runtime_error(u"Object is not supported by pxic writer: " + rt.name(rt.str(obj.type()))) + else: + write_tag(TAGGED, wtr) + write_string_raw(obj.type().name(), wtr) + write_object(handler.invoke([obj]), wtr) + diff --git a/pixie/vm/libs/ring_buffer.py b/pixie/vm/libs/ring_buffer.py new file mode 100644 index 00000000..eab8ee8a --- /dev/null +++ b/pixie/vm/libs/ring_buffer.py @@ -0,0 +1,67 @@ +from rpython.rlib.rarithmetic import r_uint +from pixie.vm.primitives import nil + +empty_slot = (nil, nil) + +class RingBuffer(object): + def __init__(self, size): + assert isinstance(size, r_uint) + self._array = [empty_slot] * size + self._array_len = size + self._length = size + self._head = r_uint(0) + self._tail = r_uint(0) + + def pending(self): + return self._array_len - self._length + + def pop(self): + if not self._length == 0: + x = self._array[self._tail] + self._array[self._tail] = empty_slot + self._tail = (self._tail + 1) % self._array_len + self._length -= 1 + return x + return empty_slot + + def push(self, x): + self._array[self._head] = x + self._head = (self._head + 1) % self._array_len + self._length += 1 + + def unbounded_push(self, x): + if self._length == 0: + self.resize() + self.push(x) + + def resize(self): + new_arr_size = self._array_len * 2 + new_arr = [None] * new_arr_size + + if self._tail < self._head: + array_copy(self._array, self._tail, new_arr, 0, self._length) + self._tail = 0 + self._head = self._length + self._array = new_arr + self._array_len = new_arr_size + + elif self._tail > self._head: + array_copy(self._array, self._tail, new_arr, 0, self._array_len - self._tail) + array_copy(self._array, 0, new_arr, self._array_len - self._tail, self._head) + self._tail = 0 + self._head = self._length + self._array = new_arr + self._array_len = new_arr_size + + else: + self._tail = 0 + self._head = 0 + self._array = new_arr + self._array_len = new_arr_size + + +def array_copy(src, src_pos, dest, dest_pos, count): + x = r_uint(0) + while x < count: + dest[dest_pos + x] = src[src_pos + x] + x += 1 diff --git a/pixie/vm/libs/string.py b/pixie/vm/libs/string.py new file mode 100644 index 00000000..967046c8 --- /dev/null +++ b/pixie/vm/libs/string.py @@ -0,0 +1,123 @@ +import pixie.vm.rt as rt +from pixie.vm.string import String +from pixie.vm.code import as_var, intern_var, wrap_fn, MultiArityFn +from pixie.vm.object import affirm, runtime_error +from pixie.vm.numbers import Integer +from rpython.rlib.unicodedata import unicodedb_6_2_0 as unicodedb +import rpython.rlib.rstring as rstring +import pixie.vm.rt as rt + + + +@as_var("pixie.string.internal", "starts-with") +def startswith(a, b): + return rt.wrap(rt.name(a).startswith(rt.name(b))) + + +@as_var("pixie.string.internal", "ends-with") +def endswith(a, b): + return rt.wrap(rt.name(a).endswith(rt.name(b))) + +@as_var("pixie.string.internal", "split") +def split(a, b): + affirm(rt.count(b) > 0, u"separator can't be empty") + v = rt.vector() + for s in rstring.split(rt.name(a), rt.name(b)): + v = rt.conj(v, rt.wrap(s)) + return v + +def index_of2(a, sep): + return rt.wrap(rt.name(a).find(rt.name(sep))) + +def index_of3(a, sep, start): + affirm(isinstance(start, Integer), u"Third argument must be an integer") + start = start.int_val() + if start >= 0: + return rt.wrap(rt.name(a).find(rt.name(sep), start)) + else: + runtime_error(u"Third argument must be a non-negative integer") + +def index_of4(a, sep, start, end): + affirm(isinstance(start, Integer) and isinstance(end, Integer), u"Third and fourth argument must be integers") + start = start.int_val() + end = end.int_val() + if start >= 0 and end >= 0: + return rt.wrap(rt.name(a).find(rt.name(sep), start, end)) + else: + runtime_error(u"Third and fourth argument must be non-negative integers") + +index_of = intern_var(u"pixie.string.internal", u"index-of") +index_of.set_root(MultiArityFn(u"index-of", {2: wrap_fn(index_of2), 3: wrap_fn(index_of3), 4: wrap_fn(index_of4)}, + required_arity = 2)) + +def substring2(a, start): + return substring3(a, start, rt._count(a)) + +def substring3(a, start, end): + affirm(isinstance(a, String), u"First argument must be a string") + affirm(isinstance(start, Integer) and isinstance(end, Integer), u"Second and third argument must be integers") + start = start.int_val() + end = end.int_val() + if start >= 0 and end >= 0: + return rt.wrap(rt.name(a)[start:end]) + else: + runtime_error(u"Second and third argument must be non-negative integers") + +substring = intern_var(u"pixie.string.internal", u"substring") +substring.set_root(MultiArityFn(u"substring", {2: wrap_fn(substring2), 3: wrap_fn(substring3)}, + required_arity = 2)) + +@as_var("pixie.string.internal", "upper-case") +def upper_case(a): + a = rt.name(a) + res = "" + for ch in a: + res += chr(unicodedb.toupper(ord(ch))) + return rt.wrap(res) + +@as_var("pixie.string.internal", "lower-case") +def lower_case(a): + a = rt.name(a) + res = "" + for ch in a: + res += chr(unicodedb.tolower(ord(ch))) + return rt.wrap(res) + +@as_var("pixie.string.internal", "capitalize") +def capitalize(a): + a = rt.name(a) + res = u"" + res += unichr(unicodedb.toupper(ord(a[0]))) + res += a[1:] + return rt.wrap(res) + +@as_var("pixie.string.internal", "trim") +def trim(a): + a = rt.name(a) + i = 0 + while i < len(a) and unicodedb.isspace(ord(a[i])): + i += 1 + j = len(a) + while j > 0 and unicodedb.isspace(ord(a[j - 1])): + j -= 1 + if j <= i: + return rt.wrap(u"") + return rt.wrap(a[i:j]) + +@as_var("pixie.string.internal", "triml") +def triml(a): + a = rt.name(a) + i = 0 + while i < len(a) and unicodedb.isspace(ord(a[i])): + i += 1 + return rt.wrap(a[i:len(a)]) + +@as_var("pixie.string.internal", "trimr") +def trimr(a): + a = rt.name(a) + j = len(a) + while j > 0 and unicodedb.isspace(ord(a[j - 1])): + j -= 1 + if j <= 0: + return rt.wrap(u"") + return rt.wrap(a[0:j]) diff --git a/pixie/vm/map_entry.py b/pixie/vm/map_entry.py index 53fea618..3b14a6c4 100644 --- a/pixie/vm/map_entry.py +++ b/pixie/vm/map_entry.py @@ -1,21 +1,11 @@ py_object = object import pixie.vm.object as object -from pixie.vm.object import affirm -from pixie.vm.primitives import nil, true, false -from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint, intmask -import rpython.rlib.jit as jit -import pixie.vm.rt as rt - class MapEntry(object.Object): _type = object.Type(u"pixie.stdlib.MapEntry") - def type(self): - return MapEntry._type - def __init__(self, key, val): self._key = key self._val = val diff --git a/pixie/vm/numbers.py b/pixie/vm/numbers.py index a4bf13c6..8e637365 100644 --- a/pixie/vm/numbers.py +++ b/pixie/vm/numbers.py @@ -1,16 +1,20 @@ import pixie.vm.object as object -from pixie.vm.primitives import nil, true, false -from rpython.rlib.rarithmetic import r_uint +from pixie.vm.object import affirm +from pixie.vm.primitives import true, false +from rpython.rlib.rarithmetic import r_uint, ovfcheck +from rpython.rlib.rbigint import rbigint +import rpython.rlib.jit as jit from pixie.vm.code import DoublePolymorphicFn, extend, Protocol, as_var, wrap_fn +from pixie.vm.libs.pxic.util import add_marshall_handlers import pixie.vm.rt as rt -class Number(object.Object): - pass - +import math +class Number(object.Object): + _type = object.Type(u"pixie.stdlib.Number") class Integer(Number): - _type = object.Type(u"pixie.stdlib.Integer") + _type = object.Type(u"pixie.stdlib.Integer", Number._type) _immutable_fields_ = ["_int_val"] def __init__(self, i_val): @@ -22,47 +26,310 @@ def int_val(self): def r_uint_val(self): return r_uint(self._int_val) - def type(self): - return Integer._type + def promote(self): + return Integer(jit.promote(self._int_val)) zero_int = Integer(0) one_int = Integer(1) +class BigInteger(Number): + _type = object.Type(u"pixie.stdlib.BigInteger", Number._type) + _immutable_fields_ = ["_bigint_val"] + + def __init__(self, bi_val): + self._bigint_val = bi_val + + def bigint_val(self): + return self._bigint_val + +class Float(Number): + _type = object.Type(u"pixie.stdlib.Float", Number._type) + _immutable_fields_ = ["_float_val"] + + def __init__(self, f_val): + self._float_val = f_val + + def float_val(self): + return self._float_val + +class Ratio(Number): + _type = object.Type(u"pixie.stdlib.Ratio", Number._type) + _immutable_fields_ = ["_numerator", "_denominator"] + + def __init__(self, numerator, denominator): + assert numerator is not None and denominator is not None + self._numerator = numerator + self._denominator = denominator + + def numerator(self): + return self._numerator + + def denominator(self): + return self._denominator + +@wrap_fn +def ratio_write(obj): + assert isinstance(obj, Ratio) + return rt.vector(rt.wrap(obj.numerator()), rt.wrap(obj.denominator())) + +@wrap_fn +def ratio_read(obj): + return Ratio(rt.nth(obj, rt.wrap(0)).int_val(), rt.nth(obj, rt.wrap(1)).int_val()) + +add_marshall_handlers(Ratio._type, ratio_write, ratio_read) + IMath = as_var("IMath")(Protocol(u"IMath")) _add = as_var("-add")(DoublePolymorphicFn(u"-add", IMath)) _sub = as_var("-sub")(DoublePolymorphicFn(u"-sub", IMath)) _mul = as_var("-mul")(DoublePolymorphicFn(u"-mul", IMath)) _div = as_var("-div")(DoublePolymorphicFn(u"-div", IMath)) +_quot = as_var("-quot")(DoublePolymorphicFn(u"-quot", IMath)) +_rem = as_var("-rem")(DoublePolymorphicFn(u"-rem", IMath)) +_lt = as_var("-lt")(DoublePolymorphicFn(u"-lt", IMath)) +_gt = as_var("-gt")(DoublePolymorphicFn(u"-gt", IMath)) +_lte = as_var("-lte")(DoublePolymorphicFn(u"-lte", IMath)) +_gte = as_var("-gte")(DoublePolymorphicFn(u"-gte", IMath)) _num_eq = as_var("-num-eq")(DoublePolymorphicFn(u"-num-eq", IMath)) _num_eq.set_default_fn(wrap_fn(lambda a, b: false)) +IUncheckedMath = as_var("IUncheckedMath")(Protocol(u"IUncheckedMath")) +_unchecked_add = as_var("-unchecked-add")(DoublePolymorphicFn(u"-unchecked-add", IUncheckedMath)) +_unchecked_subtract = as_var("-unchecked-subtract")(DoublePolymorphicFn(u"-unchecked-subtract", IUncheckedMath)) +_unchecked_multiply = as_var("-unchecked-multiply")(DoublePolymorphicFn(u"-unchecked-multiply", IUncheckedMath)) + +as_var("MAX-NUMBER")(Integer(100000)) # TODO: set this to a real max number + +num_op_template = """@extend({pfn}, {ty1}._type, {ty2}._type) +def {pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + return {wrap_start}a.{conv1}() {op} b.{conv2}(){wrap_end} +""" + +ovf_op_template = """@extend({pfn}, {ty1}._type, {ty2}._type) +def {pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + try: + z = ovfcheck(a.{conv1}() {op} b.{conv2}()) + return {wrap_start}z{wrap_end} + except OverflowError: + z = rbigint.fromint(a.{conv1}()).{method_op}(rbigint.fromint(b.{conv2}())) + return {wrap_start}z{wrap_end} +""" + +def extend_num_op(pfn, ty1, ty2, conv1, op, conv2, wrap_start = "rt.wrap(", wrap_end = ")"): + tp = num_op_template.format(pfn=pfn, ty1=ty1.__name__, ty2=ty2.__name__, + conv1=conv1, op=op, conv2=conv2, + wrap_start=wrap_start, wrap_end=wrap_end) + exec tp + +def extend_ovf_op(pfn, ty1, ty2, conv1, op, method_op, conv2, wrap_start = "rt.wrap(", wrap_end = ")"): + tp = ovf_op_template.format(pfn=pfn, ty1=ty1.__name__, ty2=ty2.__name__, + conv1=conv1, op=op, method_op=method_op, conv2=conv2, + wrap_start=wrap_start, wrap_end=wrap_end) + exec tp + +extend_num_op("_quot", Integer, Integer, "int_val", "/", "int_val") +extend_num_op("_rem", Integer, Integer, "int_val", "%", "int_val") + +def define_num_ops(): + # maybe define get_val() instead of using tuples? + num_classes = [(Integer, "int_val"), (Float, "float_val")] + for (c1, conv1) in num_classes: + for (c2, conv2) in num_classes: + for (op, sym) in [("_add", "+"), ("_sub", "-"), ("_mul", "*"), ("_div", "/")]: + if op == "_div" and c1 == Integer and c2 == Integer: + continue + elif op == "_add" and c1 == Integer and c2 == Integer: + extend_ovf_op(op, c1, c2, conv1, sym, "add", conv2) + elif op == "_sub" and c1 == Integer and c2 == Integer: + extend_ovf_op(op, c1, c2, conv1, sym, "sub", conv2) + elif op == "_mul" and c1 == Integer and c2 == Integer: + extend_ovf_op(op, c1, c2, conv1, sym, "mul", conv2) + else: + extend_num_op(op, c1, c2, conv1, sym, conv2) + if c1 != Integer or c2 != Integer: + extend_num_op("_rem", c1, c2, conv1, ",", conv2, wrap_start = "rt.wrap(math.fmod(", wrap_end = "))") + extend_num_op("_quot", c1, c2, conv1, "/", conv2, wrap_start = "rt.wrap(math.floor(", wrap_end = "))") + for (op, sym) in [("_num_eq", "=="), ("_lt", "<"), ("_gt", ">"), ("_lte", "<="), ("_gte", ">=")]: + extend_num_op(op, c1, c2, conv1, sym, conv2, + wrap_start = "true if ", wrap_end = " else false") + +define_num_ops() + +def define_unchecked_num_ops(): + # maybe define get_val() instead of using tuples? + num_classes = [(Integer, "int_val"), (Float, "float_val")] + for (c1, conv1) in num_classes: + for (c2, conv2) in num_classes: + for (op, sym) in [("_unchecked_add", "+"), ("_unchecked_subtract", "-"), ("_unchecked_multiply", "*")]: + extend_num_op(op, c1, c2, conv1, sym, conv2) + +define_unchecked_num_ops() + +bigint_ops_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) +def _{pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + return rt.wrap({conv1}(a.{get1}()).{op}({conv2}(b.{get2}()))) +""" + +def define_bigint_ops(): + num_classes = [(Integer, "rbigint.fromint", "int_val"), (BigInteger, "", "bigint_val")] + for (c1, conv1, get1) in num_classes: + for (c2, conv2, get2) in num_classes: + if c1 == Integer and c2 == Integer: + continue + for (pfn, op) in [("_add", "add"), ("_sub", "sub"), ("_mul", "mul"), ("_div", "div"), + ("_num_eq", "eq"), ("_lt", "lt"), ("_gt", "gt"), ("_lte", "le"), ("_gte", "ge")]: + code = bigint_ops_tmpl.format(pfn=pfn, op=op, + ty1=c1.__name__, conv1=conv1, get1=get1, + ty2=c2.__name__, conv2=conv2, get2=get2) + exec code -@extend(_add, Integer._type, Integer._type) +define_bigint_ops() + +def gcd(u, v): + while v != 0: + r = u % v + u = v + v = r + return u + +@extend(_div, Integer._type, Integer._type) +def _div(n, d): + assert isinstance(n, Integer) and isinstance(d, Integer) + nv = n.int_val() + dv = d.int_val() + object.affirm(dv != 0, u"Divide by zero") + g = gcd(nv, dv) + if g == 0: + return rt.wrap(0) + nv = nv / g + dv = dv / g + if dv == 1: + return rt.wrap(nv) + elif dv == -1: + return rt.wrap(-1 * nv) + else: + if dv < 0: + nv = nv * -1 + dv = dv * -1 + return Ratio(nv, dv) + +@extend(_add, Ratio._type, Ratio._type) def _add(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return Integer(a.int_val() + b.int_val()) + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt._add(rt.wrap(b.numerator() * a.denominator()), + rt.wrap(a.numerator() * b.denominator())), + rt.wrap(a.denominator() * b.denominator())) -@extend(_sub, Integer._type, Integer._type) +@extend(_sub, Ratio._type, Ratio._type) def _sub(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return rt.wrap(a.int_val() - b.int_val()) + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt._add(rt.wrap(-1 * b.numerator() * a.denominator()), + rt.wrap(a.numerator() * b.denominator())), + rt.wrap(a.denominator() * b.denominator())) -@extend(_mul, Integer._type, Integer._type) +@extend(_mul, Ratio._type, Ratio._type) def _mul(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return rt.wrap(a.int_val() * b.int_val()) + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt.wrap(b.numerator() * a.numerator()), + rt.wrap(b.denominator() * a.denominator())) -@extend(_div, Integer._type, Integer._type) +@extend(_div, Ratio._type, Ratio._type) def _div(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return rt.wrap(a.int_val() / b.int_val()) + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._div(rt.wrap(b.denominator() * a.numerator()), + rt.wrap(b.numerator() * a.denominator())) -@extend(_num_eq, Integer._type, Integer._type) -def _div(a, b): - assert isinstance(a, Integer) and isinstance(b, Integer) - return true if a.int_val() == b.int_val() else false +@extend(_quot, Ratio._type, Ratio._type) +def _quot(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt.wrap((a.numerator() * b.denominator()) / (a.denominator() * b.numerator())) + +@extend(_rem, Ratio._type, Ratio._type) +def _rem(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + q = rt.wrap((a.numerator() * b.denominator()) / (a.denominator() * b.numerator())) + return rt._sub(a, rt._mul(q, b)) + +@extend(_lt, Ratio._type, Ratio._type) +def _lt(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if a.numerator() * b.denominator() < b.numerator() * a.denominator() else false + +@extend(_gt, Ratio._type, Ratio._type) +def _gt(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return rt._lt(b, a) + +@extend(_lte, Ratio._type, Ratio._type) +def _lte(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if rt._lt(b, a) is false else false + +@extend(_gte, Ratio._type, Ratio._type) +def gte(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if rt._lt(a, b) is false else false + +@extend(_num_eq, Ratio._type, Ratio._type) +def _num_eq(a, b): + assert isinstance(a, Ratio) and isinstance(b, Ratio) + return true if a.numerator() == b.numerator() and a.denominator() == b.denominator() else false + +mixed_op_tmpl = """@extend({pfn}, {ty1}._type, {ty2}._type) +def {pfn}_{ty1}_{ty2}(a, b): + assert isinstance(a, {ty1}) and isinstance(b, {ty2}) + return rt.{pfn}({conv1}(a), {conv2}(b)) +""" +def to_ratio(x): + if isinstance(x, Ratio): + return x + else: + return Ratio(x.int_val(), 1) + +def to_ratio_conv(c): + if c == Ratio: + return "" + else: + return "to_ratio" + +def to_float(x): + if isinstance(x, Float): + return x + if isinstance(x, Ratio): + return rt.wrap(x.numerator() / float(x.denominator())) + if isinstance(x, Integer): + return rt.wrap(float(x.int_val())) + if isinstance(x, BigInteger): + return rt.wrap(x.bigint_val().tofloat()) + object.runtime_error(u"Cannot convert %s to float" %x.type().name()) + +def to_float_conv(c): + if c == Float: + return "" + else: + return "to_float" + +def define_mixed_ops(): + for (c1, c2) in [(Integer, Ratio), (Ratio, Integer)]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_ratio_conv(c1), conv2=to_ratio_conv(c2)) + exec code + + for (c1, c2) in [(Float, Ratio), (Ratio, Float)]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) + exec code + + for (c1, c2) in [(Float, BigInteger), (BigInteger, Float)]: + for op in ["_add", "_sub", "_mul", "_div", "_quot", "_rem", "_lt", "_gt", "_lte", "_gte", "_num_eq"]: + code = mixed_op_tmpl.format(pfn=op, ty1=c1.__name__, ty2=c2.__name__, conv1=to_float_conv(c1), conv2=to_float_conv(c2)) + exec code + +define_mixed_ops() # def add(a, b): # if isinstance(a, Integer): @@ -91,4 +358,36 @@ def _str(i): def _repr(i): return rt.wrap(unicode(str(i.int_val()))) + @extend(proto._str, BigInteger._type) + def _str(b): + return rt.wrap(unicode(b.bigint_val().format('0123456789', suffix='N'))) + + @extend(proto._repr, BigInteger._type) + def _repr(b): + return rt.wrap(unicode(b.bigint_val().format('0123456789', suffix='N'))) + + @extend(proto._str, Float._type) + def _str(f): + return rt.wrap(unicode(str(f.float_val()))) + + @extend(proto._repr, Float._type) + def _repr(f): + return rt.wrap(unicode(str(f.float_val()))) + + @extend(proto._repr, Ratio._type) + def _repr(r): + return rt.wrap(unicode(str(r.numerator()) + "/" + str(r.denominator()))) + + @extend(proto._str, Ratio._type) + def _str(r): + return rt.wrap(unicode(str(r.numerator()) + "/" + str(r.denominator()))) + + @as_var("numerator") + def numerator(r): + affirm(isinstance(r, Ratio), u"First argument must be a Ratio") + return rt.wrap(r.numerator()) + @as_var("denominator") + def denominator(r): + affirm(isinstance(r, Ratio), u"First argument must be a Ratio") + return rt.wrap(r.denominator()) diff --git a/pixie/vm/object.py b/pixie/vm/object.py index bc682d22..2ef03de3 100644 --- a/pixie/vm/object.py +++ b/pixie/vm/object.py @@ -1,11 +1,60 @@ +from rpython.rlib.objectmodel import compute_identity_hash +import rpython.rlib.jit as jit +class FinalizerRegistry(object): + def __init__(self): + # TODO: PyPy uses a linked list, investigate if we need that too + self._registry = [] + + def register(self, o): + self._registry.append(o) + + def run_finalizers(self): + import pixie.vm.rt as rt + + vals = self._registry + self._registry = [] + for x in vals: + rt._finalize_BANG_(x) + +finalizer_registry = FinalizerRegistry() + class Object(object): """ Base Object for all VM objects """ + _attrs_ = () + + # Initialized after Type is defined. + _type = None def type(self): - affirm(False, u".type isn't overloaded") + assert(isinstance(self.__class__._type, Type)) + return self.__class__._type + + @jit.unroll_safe + def invoke(self, args): + import pixie.vm.stdlib as stdlib + return stdlib.invoke_other(self, args) + + def int_val(self): + affirm(False, u"Expected Number, not " + self.type().name()) + return 0 + + def r_uint_val(self): + affirm(False, u"Expected Number, not " + self.type().name()) + return 0 + + def hash(self): + import pixie.vm.rt as rt + return rt.wrap(compute_identity_hash(self)) + + def promote(self): + return self + + + + class TypeRegistry(object): def __init__(self): @@ -19,7 +68,6 @@ def register_type(self, nm, tp): self.var_for_type_and_name(nm, tp) def var_for_type_and_name(self, nm, tp): - from pixie.vm.symbol import symbol splits = nm.split(u".") size = len(splits) - 1 assert size >= 0 @@ -41,28 +89,74 @@ def get_by_name(self, nm, default=None): _type_registry = TypeRegistry() +def get_type_by_name(nm): + return _type_registry.get_by_name(nm) + class Type(Object): - def __init__(self, name): - #assert isinstance(name, unicode), u"Type names must be unicode" + _immutable_fields_ = ["_name", "_has_finalizer?"] + def __init__(self, name, parent=None, object_inited=True): + assert isinstance(name, unicode), u"Type names must be unicode" _type_registry.register_type(name, self) self._name = name - def type(self): - return Type._type + if object_inited: + if parent is None: + parent = Object._type + + parent.add_subclass(self) + + self._parent = parent + self._subclasses = [] + self._has_finalizer = False + + def name(self): + return self._name + + def parent(self): + return self._parent + + def add_subclass(self, tp): + self._subclasses.append(tp) + + def subclasses(self): + return self._subclasses + + @jit.elidable_promote() + def has_finalizer(self): + return self._has_finalizer + def set_finalizer(self): + self._has_finalizer = True -Type._type = Type(u"Type") +Object._type = Type(u"pixie.stdlib.Object", None, False) +Type._type = Type(u"pixie.stdlib.Type") + +@jit.elidable_promote() +def istypeinstance(obj, t): + obj_type = obj.type() + assert isinstance(obj_type, Type) + if obj_type is t: + return True + elif obj_type._parent is not None: + obj_type = obj_type._parent + while obj_type is not None: + if obj_type is t: + return True + obj_type = obj_type._parent + return False + else: + return False class RuntimeException(Object): _type = Type(u"pixie.stdlib.RuntimeException") - def __init__(self, data): + def __init__(self, msg, data): + assert data is not None + assert msg is not None + self._msg = msg self._data = data self._trace = [] - def type(self): - return RuntimeException._type - def __repr__(self): import pixie.vm.rt as rt s = [] @@ -71,9 +165,8 @@ def __repr__(self): for x in trace: s.append(x.__repr__()) s.append(u"\n") - - s.extend([u"RuntimeException: " + rt.str(self._data)._str + u"\n"]) - + s.extend([u"RuntimeException: " + rt.name(rt.str(self._data)) + u" " + + rt.name(rt.str(self._msg)) + u"\n"]) return u"".join(s) @@ -93,13 +186,29 @@ def affirm(val, msg): assert isinstance(msg, unicode) if not val: import pixie.vm.rt as rt - raise WrappedException(RuntimeException(rt.wrap(msg))) - + from pixie.vm.keyword import keyword + raise WrappedException(RuntimeException(rt.wrap(msg), keyword(u"pixie.stdlib/AssertionException"))) + +def runtime_error(msg, data=None): + import pixie.vm.rt as rt + from pixie.vm.keyword import keyword + if data is None: + data = u"pixie.stdlib/AssertionException" + raise WrappedException(RuntimeException(rt.wrap(msg), keyword(data))) + +def safe_invoke(f, args): + try: + f.invoke(args) + except Exception as ex: + if isinstance(ex, WrappedException): + print "UNSAFE EXCEPTION", ex._ex.__repr__() + else: + print "UNSAFE EXCEPTION", ex + return None class ErrorInfo(Object): _type = Type(u"pixie.stdlib.ErrorInfo") - def type(self): - return ErrorInfo._type + def __init__(self): pass @@ -122,6 +231,21 @@ def __repr__(self): + self._line.__repr__() + u"\n" \ + self.pad_chars() + u"^" + def interpreter_code_info_state(self): + return self._line, self._line_number, self._column_number, self._file + + def trace_map(self): + from pixie.vm.string import String + from pixie.vm.numbers import Integer + from pixie.vm.keyword import keyword + + tm = {keyword(u"type") : keyword(u"interpreter")} + tm[keyword(u"line")] = String(self._line.__repr__()) + tm[keyword(u"line-number")] = Integer(self._line_number) + tm[keyword(u"column-number")] = Integer(self._column_number) + tm[keyword(u"file")] = String(self._file) + return tm + class NativeCodeInfo(ErrorInfo): def __init__(self, name): self._name = name @@ -129,15 +253,35 @@ def __init__(self, name): def __repr__(self): return u"in internal function " + self._name + u"\n" + def trace_map(self): + from pixie.vm.string import String + from pixie.vm.numbers import Integer + from pixie.vm.keyword import keyword + + tm = {keyword(u"type") : keyword(u"native")} + tm[keyword(u"name")] = String(self._name) + return tm + class PolymorphicCodeInfo(ErrorInfo): def __init__(self, name, tp): self._name = name self._tp = tp def __repr__(self): - return u"in polymorphic function " + self._name + u" dispatching on " + self._tp._name + u"\n" + tp = self._tp + assert isinstance(tp, Type) + return u"in polymorphic function " + self._name + u" dispatching on " + tp._name + u"\n" + def trace_map(self): + from pixie.vm.string import String + from pixie.vm.keyword import keyword + tp = self._tp + assert isinstance(tp, Type) + tm = {keyword(u"type") : keyword(u"polymorphic")} + tm[keyword(u"name")] = String(self._name) + tm[keyword(u"tp")] = String(tp._name) + return tm class PixieCodeInfo(ErrorInfo): def __init__(self, name): @@ -145,3 +289,37 @@ def __init__(self, name): def __repr__(self): return u"in pixie function " + self._name + u"\n" + + def trace_map(self): + from pixie.vm.string import String + from pixie.vm.keyword import keyword + + tm = {keyword(u"type") : keyword(u"pixie")} + tm[keyword(u"name")] = String(self._name) + return tm + +class ExtraCodeInfo(ErrorInfo): + def __init__(self, str, data=None): + self._str = str + self._data = data + + def __repr__(self): + return self._str + + def trace_map(self): + import pixie.vm.rt as rt + from pixie.vm.keyword import keyword + + tm = {keyword(u"type"): keyword(u"extra"), + keyword(u"msg"): rt.wrap(self._str)} + + if self._data: + tm[keyword(u"data")] = self._data + + return tm + + +def add_info(ex, data): + assert isinstance(ex, WrappedException) + ex._ex._trace.append(ExtraCodeInfo(data)) + return ex diff --git a/pixie/vm/persistent_hash_map.py b/pixie/vm/persistent_hash_map.py index cd118bd7..7cab577c 100644 --- a/pixie/vm/persistent_hash_map.py +++ b/pixie/vm/persistent_hash_map.py @@ -2,14 +2,15 @@ import pixie.vm.object as object from pixie.vm.object import affirm from pixie.vm.primitives import nil, true, false -from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint, intmask +from rpython.rlib.rarithmetic import r_int, r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt -MASK_32 = 0xFFFFFFFF +MASK_32 = r_uint(0xFFFFFFFF) + +NOT_FOUND = object.Object() class Box(py_object): def __init__(self): @@ -18,19 +19,22 @@ def __init__(self): class PersistentHashMap(object.Object): _type = object.Type(u"pixie.stdlib.PersistentHashMap") - def type(self): - return PersistentHashMap._type - def __init__(self, cnt, root, meta=nil): self._cnt = cnt self._root = root self._meta = meta + def meta(self): + return self._meta + + def with_meta(self, meta): + return PersistentHashMap(self._cnt, self._root, meta) + def assoc(self, key, val): added_leaf = Box() new_root = (BitmapIndexedNode_EMPTY if self._root is None else self._root) \ - .assoc_inode(r_uint(0), rt.hash(key), key, val, added_leaf) + .assoc_inode(r_uint(0), rt.hash(key) & MASK_32, key, val, added_leaf) if new_root is self._root: return self @@ -38,18 +42,23 @@ def assoc(self, key, val): return PersistentHashMap(self._cnt if added_leaf._val is None else self._cnt + 1, new_root, self._meta) def val_at(self, key, not_found): - return not_found if self._root is None else self._root.find(r_uint(0), rt.hash(key), key, not_found) + return not_found if self._root is None else self._root.find(r_uint(0), rt.hash(key) & MASK_32, key, not_found) + def without(self, key): + if self._root is None: + return self + + new_root = self._root.without_inode(0, rt.hash(key) & MASK_32, key) + if new_root is self._root: + return self + return PersistentHashMap(self._cnt - 1, new_root, self._meta) class INode(object.Object): _type = object.Type(u"pixie.stdlib.INode") - def type(self): - return INode._type - def assoc_inode(self, shift, hash_val, key, val, added_leaf): pass @@ -59,6 +68,9 @@ def find(self, shift, hash_val, key, not_found): def reduce_inode(self, f, init): pass + def without(self, shift, hash, key): + pass + def mask(hash, shift): return (hash >> shift) & 0x01f @@ -87,13 +99,13 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): if key_or_null is None: assert isinstance(val_or_node, INode) - n = val_or_node.assoc_inode(shift + 5, hash_val, key, val, added_leaf) + n = val_or_node.assoc_inode(shift + 5, hash_val & MASK_32, key, val, added_leaf) if n is val_or_node: return self return BitmapIndexedNode(None, self._bitmap, clone_and_set(self._array, 2 * idx + 1, n)) - if key_or_null is None or rt.eq(key, key_or_null): + if rt.eq(key, key_or_null): if val is val_or_node: return self return BitmapIndexedNode(None, self._bitmap, clone_and_set(self._array, 2 * idx + 1, val)) @@ -102,7 +114,7 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): return BitmapIndexedNode(None, self._bitmap, clone_and_set2(self._array, 2 * idx, None, - 2 * idx + 1, create_node(shift+ 5, key_or_null, val_or_node, hash_val, key, val))) + 2 * idx + 1, create_node(shift + 5, key_or_null, val_or_node, hash_val, key, val))) else: n = bit_count(self._bitmap) if n >= 16: @@ -118,7 +130,7 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): else: nodes[i] = BitmapIndexedNode_EMPTY.assoc_inode(shift + 5, rt.hash(self._array[j]), self._array[j], self._array[j + 1], added_leaf) - j += 1 + j += 2 return ArrayNode(None, n + 1, nodes) else: @@ -143,21 +155,46 @@ def find(self, shift, hash_val, key, not_found): return val_or_node return not_found + def reduce_inode(self, f, init): for x in range(0, len(self._array), 2): key_or_none = self._array[x] val_or_node = self._array[x + 1] if key_or_none is None and val_or_node is not None: - init = init.reduce_inode(f, init) + init = val_or_node.reduce_inode(f, init) else: init = f.invoke([init, rt.map_entry(key_or_none, val_or_node)]) if rt.reduced_QMARK_(init): return init return init + def without_inode(self, shift, hash, key): + bit = bitpos(hash, shift) + if self._bitmap & bit == 0: + return self -BitmapIndexedNode_EMPTY = BitmapIndexedNode(None, r_uint(0), []) + idx = self.index(bit) + key_or_none = self._array[2 * idx] + val_or_node = self._array[2 * idx + 1] + + if key_or_none is None: + n = val_or_node.without_inode(shift + 5, hash, key) + if n is val_or_node: + return self + if n is not None: + return BitmapIndexedNode(None, self._bitmap, clone_and_set(self._array, 2 * idx + 1, n)) + + if self._bitmap == bit: + return None + + return BitmapIndexedNode(None, self._bitmap ^ bit, remove_pair(self._array, idx)) + + if rt.eq(key, key_or_none): + return BitmapIndexedNode(None, self._bitmap ^ bit, remove_pair(self._array, idx)) + + return self +BitmapIndexedNode_EMPTY = BitmapIndexedNode(None, r_uint(0), []) class ArrayNode(INode): def __init__(self, edit, cnt, array): @@ -177,6 +214,47 @@ def assoc_inode(self, shift, hash_val, key, val, added_leaf): return self return ArrayNode(None, self._cnt, clone_and_set(self._array, idx, n)) + def without_inode(self, shift, hash_val, key): + idx = r_uint(mask(hash_val, shift)) + node = self._array[idx] + if node is None: + return self + n = node.without_inode(shift + 5, hash_val, key) + if n is node: + return self + if n is None: + if self._cnt <= 8: # shrink + return self.pack(idx) + return ArrayNode(None, self._cnt - 1, clone_and_set(self._array, idx, n)) + else: + return ArrayNode(None, self._cnt, clone_and_set(self._array, idx, n)) + + def pack(self, idx): + new_array = [None] * (2 * (self._cnt - 1)) + j = r_uint(1) + bitmap = r_uint(0) + + i = r_uint(0) + while i < idx: + if self._array[i] is not None: + new_array[j] = self._array[i] + bitmap |= r_uint(1) << i + j += 2 + + i += 1 + + i = r_uint(idx) + 1 + while i < len(self._array): + if self._array[i] is not None: + new_array[j] = self._array[i] + bitmap |= r_uint(1) << i + j += 2 + + i += 1 + + return BitmapIndexedNode(None, bitmap, new_array) + + def find(self, shift, hash_val, key, not_found): idx = mask(hash_val, shift) node = self._array[idx] @@ -201,7 +279,22 @@ def __init__(self, edit, hash, array): self._array = array def assoc_inode(self, shift, hash_val, key, val, added_leaf): - assert False + if hash_val == self._hash: + count = len(self._array) + idx = self.find_index(key) + if idx != -1: + if self._array[idx + 1] == val: + return self; + return HashCollisionNode(None, hash_val, clone_and_set(self._array, r_uint(idx + 1), val)) + + new_array = [None] * (count + 2) + list_copy(self._array, 0, new_array, 0, count) + new_array[count] = key + added_leaf._val = added_leaf + new_array[count + 1] = val + return HashCollisionNode(self._edit, self._hash, new_array) + return BitmapIndexedNode(None, bitpos(self._hash, shift), [None, self]) \ + .assoc_inode(shift, hash_val, key, val, added_leaf) def find(self, shift, hash_val, key, not_found): for x in range(0, len(self._array), 2): @@ -218,26 +311,45 @@ def reduce_inode(self, f, init): continue val = self._array[x + 1] - init = f.invoke([init, val]) + init = f.invoke([init, rt.map_entry(key_or_nil, val)]) if rt.reduced_QMARK_(init): return init return init + def find_index(self, key): + i = r_int(0) + while i < len(self._array): + if rt.eq(key, self._array[i]): + return i + + i += 2 + + return r_int(-1) + + def without_inode(self, shift, hash, key): + idx = self.find_index(key) + if idx == -1: + return self + + if len(self._array) == 1: + return None + + return HashCollisionNode(None, self._hash, remove_pair(self._array, r_uint(idx) / 2)) def create_node(shift, key1, val1, key2hash, key2, val2): - key1hash = rt.hash(key1) + key1hash = rt.hash(key1) & MASK_32 if key1hash == key2hash: return HashCollisionNode(None, key1hash, [key1, val1, key2, val2]) added_leaf = Box() return BitmapIndexedNode_EMPTY.assoc_inode(shift, key1hash, key1, val1, added_leaf) \ - .assoc_inode(shift, key1hash, key1, val1, added_leaf) + .assoc_inode(shift, key2hash, key2, val2, added_leaf) def bit_count(i): assert isinstance(i, r_uint) - i = i - ((i >> 1) & 0x55555555) - i = (i & 0x33333333) + ((i >> 2) & 0x33333333) - return (((i + (i >> 4) & 0xF0F0F0F) * 0x1010101) & 0xffffffff) >> 24 + i = i - ((i >> 1) & r_uint(0x55555555)) + i = (i & r_uint(0x33333333)) + ((i >> 2) & r_uint(0x33333333)) + return (((i + (i >> 4) & r_uint(0xF0F0F0F)) * r_uint(0x1010101)) & r_uint(0xffffffff)) >> 24 @jit.unroll_safe def list_copy(from_lst, from_loc, to_list, to_loc, count): @@ -258,6 +370,7 @@ def clone_and_set(array, i, a): idx = r_uint(0) while idx < len(array): clone[idx] = array[idx] + idx += 1 clone[i] = a return clone @@ -269,11 +382,17 @@ def clone_and_set2(array, i, a, j, b): idx = r_uint(0) while idx < len(array): clone[idx] = array[idx] + idx += 1 clone[i] = a clone[j] = b return clone +def remove_pair(array, i): + new_array = [None] * (len(array) - 2) + list_copy(array, 0, new_array, 0, 2 * i) + list_copy(array, 2 * (i + 1), new_array, 2 * i, len(new_array) - (2 * i)) + return new_array ### hook into RT @@ -297,6 +416,11 @@ def hashmap__args(args): return acc +@extend(proto._count, PersistentHashMap) +def _count(self): + assert isinstance(self, PersistentHashMap) + return rt.wrap(self._cnt) + @extend(proto._val_at, PersistentHashMap) def _val_at(self, key, not_found): @@ -319,9 +443,33 @@ def _assoc(self, key, val): assert isinstance(self, PersistentHashMap) return self.assoc(key, val) +@extend(proto._dissoc, PersistentHashMap) +def _dissoc(self, key): + assert isinstance(self, PersistentHashMap) + return self.without(key) + proto.IMap.add_satisfies(PersistentHashMap._type) @extend(proto._count, PersistentHashMap) def _count(self): assert isinstance(self, PersistentHashMap) - return rt.wrap(intmask(self._cnt)) \ No newline at end of file + return rt.wrap(intmask(self._cnt)) + + +@extend(proto._meta, PersistentHashMap) +def _meta(self): + assert isinstance(self, PersistentHashMap) + return self.meta() + +@extend(proto._with_meta, PersistentHashMap) +def _with_meta(self, meta): + assert isinstance(self, PersistentHashMap) + return self.with_meta(meta) + +@extend(proto._contains_key, PersistentHashMap) +def _contains_key(self, key): + assert isinstance(self, PersistentHashMap) + if self._root is not None: + return true if self._root.find(r_uint(0), rt.hash(key), key, NOT_FOUND) is not NOT_FOUND else false + else: + return false diff --git a/pixie/vm/persistent_hash_set.py b/pixie/vm/persistent_hash_set.py new file mode 100644 index 00000000..f2bd5d4a --- /dev/null +++ b/pixie/vm/persistent_hash_set.py @@ -0,0 +1,97 @@ +py_object = object +import pixie.vm.object as object +from pixie.vm.primitives import nil, true, false +import pixie.vm.persistent_hash_map as persistent_hash_map +import pixie.vm.stdlib as proto +from pixie.vm.code import extend, as_var, intern_var +import pixie.vm.rt as rt + + +VAR_KEY = intern_var(u"pixie.stdlib", u"key") + +class PersistentHashSet(object.Object): + _type = object.Type(u"pixie.stdlib.PersistentHashSet") + + def __init__(self, meta, m): + self._meta = meta + self._map = m + + def conj(self, v): + return PersistentHashSet(self._meta, self._map.assoc(v, v)) + + def disj(self, k): + return PersistentHashSet(self._meta, self._map.without(k)) + + def meta(self): + return self._meta + + def with_meta(self, meta): + return PersistentHashSet(meta, self._map) + +EMPTY = PersistentHashSet(nil, persistent_hash_map.EMPTY) + +@as_var("set") +def _create(coll): + ret = EMPTY + coll = rt._seq(coll) + while coll is not nil: + ret = ret.conj(rt._first(coll)) + coll = rt._seq(rt._next(coll)) + return ret + +@extend(proto._count, PersistentHashSet) +def _count(self): + assert isinstance(self, PersistentHashSet) + return rt._count(self._map) + +@extend(proto._val_at, PersistentHashSet) +def _val_at(self, key, not_found): + assert isinstance(self, PersistentHashSet) + return rt._val_at(self._map, key, not_found) + +@extend(proto._contains_key, PersistentHashSet) +def _contains_key(self, key): + assert isinstance(self, PersistentHashSet) + return rt._contains_key(self._map, key) + +@extend(proto._eq, PersistentHashSet) +def _eq(self, obj): + assert isinstance(self, PersistentHashSet) + if self is obj: + return true + if not isinstance(obj, PersistentHashSet): + return false + if self._map._cnt != obj._map._cnt: + return false + + seq = rt.seq(obj) + while seq is not nil: + if rt._contains_key(self, rt.first(seq)) is false: + return false + seq = rt.next(seq) + return true + +@extend(proto._conj, PersistentHashSet) +def _conj(self, v): + assert isinstance(self, PersistentHashSet) + return self.conj(v) + +@extend(proto._disj, PersistentHashSet) +def _disj(self, v): + assert isinstance(self, PersistentHashSet) + return self.disj(v) + +@extend(proto._reduce, PersistentHashSet) +def _reduce(self, f, init): + assert isinstance(self, PersistentHashSet) + return rt._reduce(rt.keys(self._map), f, init) + +@extend(proto._meta, PersistentHashSet) +def _meta(self): + assert isinstance(self, PersistentHashSet) + return self.meta() + +@extend(proto._with_meta, PersistentHashSet) +def _with_meta(self, meta): + assert isinstance(self, PersistentHashSet) + return self.with_meta(meta) diff --git a/pixie/vm/persistent_list.py b/pixie/vm/persistent_list.py index 0b490256..10514b43 100644 --- a/pixie/vm/persistent_list.py +++ b/pixie/vm/persistent_list.py @@ -1,17 +1,13 @@ import pixie.vm.object as object -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import nil import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var -from pixie.vm.numbers import Integer from rpython.rlib.rarithmetic import r_uint, intmask import pixie.vm.rt as rt class PersistentList(object.Object): _type = object.Type(u"pixie.stdlib.PersistentList") - def type(self): - return PersistentList._type - def __init__(self, head, tail, cnt, meta=nil): self._first = head self._next = tail @@ -29,6 +25,7 @@ def meta(self): def with_meta(self, meta): return PersistentList(self._first, self._next, self._cnt, meta) + @extend(proto._first, PersistentList) @@ -69,11 +66,96 @@ def count(self): @as_var("list") def list__args(args): - i = r_uint(len(args)) + return create_from_list(args) + +def create_from_list(lst): + if len(lst) == 0: + return EmptyList() + + i = r_uint(len(lst)) acc = nil while i > 0: - acc = PersistentList(args[i - 1], acc, len(args) - i + 1, nil) + acc = PersistentList(lst[i - 1], acc, len(lst) - i + 1, nil) i -= 1 return acc +def create(*args): + return create_from_list(args) + +@extend(proto._meta, PersistentList) +def _meta(self): + assert isinstance(self, PersistentList) + return self.meta() + +@extend(proto._with_meta, PersistentList) +def _with_meta(self, meta): + assert isinstance(self, PersistentList) + return self.with_meta(meta) + + + + +class EmptyList(object.Object): + _type = object.Type(u"pixie.stdlib.EmptyList") + + def __init__(self, meta=nil): + self._meta = meta + + def meta(self): + return self._meta + + def with_meta(self, meta): + return EmptyList(meta) + + + + + +@extend(proto._first, EmptyList) +def _first(self): + assert isinstance(self, EmptyList) + return nil + +@extend(proto._next, EmptyList) +def _next(self): + assert isinstance(self, EmptyList) + return nil + +@extend(proto._seq, EmptyList) +def _seq(self): + assert isinstance(self, EmptyList) + return nil + +@extend(proto._count, EmptyList) +def _count(self): + assert isinstance(self, EmptyList) + return rt.wrap(0) + +@extend(proto._conj, EmptyList) +def _conj(self, itm): + assert isinstance(self, EmptyList) + return PersistentList(itm, nil, 1) + +@extend(proto._meta, EmptyList) +def _meta(self): + assert isinstance(self, EmptyList) + return self.meta() + +@extend(proto._with_meta, EmptyList) +def _with_meta(self, meta): + assert isinstance(self, EmptyList) + return self.with_meta(meta) + +@extend(proto._str, EmptyList) +def _str(self): + return rt.wrap(u"()") + +@extend(proto._repr, EmptyList) +def _str(self): + return rt.wrap(u"()") + +@extend(proto._reduce, EmptyList) +def _str(self, f, init): + return init + diff --git a/pixie/vm/persistent_vector.py b/pixie/vm/persistent_vector.py index 2508d9e3..2740caf5 100644 --- a/pixie/vm/persistent_vector.py +++ b/pixie/vm/persistent_vector.py @@ -4,17 +4,14 @@ from pixie.vm.primitives import nil, true, false from pixie.vm.numbers import Integer import pixie.vm.stdlib as proto -from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint, intmask, widen +from pixie.vm.code import extend, as_var +from rpython.rlib.rarithmetic import r_uint, intmask import rpython.rlib.jit as jit import pixie.vm.rt as rt - class Node(object.Object): _type = object.Type(u"pixie.stdlib.PersistentVectorNode") - def type(self): - return Node._type def __init__(self, edit, array=None): self._edit = edit @@ -27,9 +24,6 @@ def __init__(self, edit, array=None): class PersistentVector(object.Object): _type = object.Type(u"pixie.stdlib.PersistentVector") - def type(self): - return PersistentVector._type - def __init__(self, meta, cnt, shift, root, tail): self._meta = meta self._cnt = cnt @@ -59,33 +53,42 @@ def array_for(self, i): assert isinstance(node, Node) node = node._array[(i >> level) & 0x01f] level -= 5 + assert isinstance(node, Node) return node._array affirm(False, u"Index out of Range") - def nth(self, i, not_found=nil): + def nth(self, i, not_found=None): if 0 <= i < self._cnt: node = self.array_for(r_uint(i)) return node[i & 0x01f] - return not_found + if not_found is None: + affirm(False, u"Index out of Range") + else: + return not_found def conj(self, val): - assert self._cnt < 0xFFFFFFFF - i = self._cnt + assert self._cnt < r_uint(0xFFFFFFFF) if self._cnt - self.tailoff() < 32: new_tail = self._tail[:] new_tail.append(val) return PersistentVector(self._meta, self._cnt + 1, self._shift, self._root, new_tail) - tail_node = Node(self._root._edit, self._tail) + root = self._root + assert isinstance(root, Node) + tail_node = Node(root._edit, self._tail) new_shift = self._shift if (self._cnt >> 5) > (r_uint(1) << self._shift): - new_root = Node(self._root._edit) + root = self._root + assert isinstance(root, Node) + new_root = Node(root._edit) new_root._array[0] = self._root - new_root._array[1] = new_path(self._root._edit, self._shift, tail_node) + root = self._root + assert isinstance(root, Node) + new_root._array[1] = new_path(root._edit, self._shift, tail_node) new_shift += 5 else: @@ -95,7 +98,12 @@ def conj(self, val): def push_tail(self, level, parent, tail_node): subidx = ((self._cnt - 1) >> level) & 0x01f + assert isinstance(parent, Node) ret = Node(parent._edit, parent._array[:]) + + root = self._root + assert isinstance(root, Node) + if (level == 5): node_to_insert = tail_node else: @@ -103,13 +111,11 @@ def push_tail(self, level, parent, tail_node): if child is not None: node_to_insert = self.push_tail(level - 5, child, tail_node) else: - node_to_insert = new_path(self._root._edit, level - 5, tail_node) + node_to_insert = new_path(root._edit, level - 5, tail_node) ret._array[subidx] = node_to_insert return ret - - def pop(self): affirm(self._cnt != 0, u"Can't pop an empty vector") @@ -118,7 +124,7 @@ def pop(self): if self._cnt - self.tailoff() > 1: size = len(self._tail) - 1 - assert size >= 0 # for translation + assert size >= 0 # for translation new_tail = self._tail[:size] return PersistentVector(self._meta, self._cnt - 1, self._shift, self._root, new_tail) @@ -138,11 +144,14 @@ def pop(self): def pop_tail(self, level, node): sub_idx = ((self._cnt - 1) >> level) & 0x01f if level > 5: + assert isinstance(node, Node) new_child = self.pop_tail(level - 5, node._array[sub_idx]) if new_child is None or sub_idx == 0: return None else: - ret = Node(self._root._edit, node._array[:]) + root = self._root + assert isinstance(root, Node) + ret = Node(root._edit, node._array[:]) ret._array[sub_idx] = new_child return ret @@ -150,10 +159,38 @@ def pop_tail(self, level, node): return None else: - ret = Node(self._root._edit, node._array[:]) + root = self._root + assert isinstance(root, Node) + assert isinstance(node, Node) + ret = Node(root._edit, node._array[:]) ret._array[sub_idx] = None return ret + def assoc_at(self, idx, val): + if idx >= 0 and idx < self._cnt: + if idx >= self.tailoff(): + new_tail = self._tail[:] + new_tail[idx & 0x01f] = val + return PersistentVector(self._meta, self._cnt, self._shift, self._root, new_tail) + return PersistentVector(self._meta, self._cnt, self._shift, do_assoc(self._shift, self._root, idx, val), self._tail) + if idx == self._cnt: + return self.conj(val) + else: + object.runtime_error(u"index out of range", + u"pixie.stdlib/OutOfRangeException") + + +def do_assoc(lvl, node, idx, val): + assert isinstance(node, Node) + ret = Node(node._edit, node._array[:]) + if lvl == 0: + ret._array[idx & 0x01f] = val + else: + subidx = (idx >> lvl) & 0x01f + ret._array[subidx] = do_assoc(lvl - 5, node._array[subidx], idx, val) + return ret + + def new_path(edit, level, node): if level == 0: return node @@ -161,14 +198,13 @@ def new_path(edit, level, node): ret._array[0] = new_path(edit, level - 5, node) return ret + edited = u"edited" + class TransientVector(object.Object): _type = object.Type(u"pixie.stdlib.TransientVector") - def type(self): - return TransientVector._type - def __init__(self, cnt, shift, root, tail): self._cnt = cnt self._shift = shift @@ -177,17 +213,24 @@ def __init__(self, cnt, shift, root, tail): @staticmethod def editable_root(node): + assert isinstance(node, Node) return Node(edited, node._array[:]) def ensure_editable(self): - affirm(self._root._edit is not None, u"Transient used after call to persist!") + root = self._root + assert isinstance(root, Node) + affirm(root._edit is not None, u"Transient used after call to persist!") def ensure_node_editable(self, node): - if node._edit is self._root._edit: + assert isinstance(node, Node) + root = self._root + assert isinstance(root, Node) + if node._edit is root._edit: return node - return Node(self._root._edit, node._array[:]) - + root = self._root + assert isinstance(root, Node) + return Node(root._edit, node._array[:]) def tailoff(self): if self._cnt < 32: @@ -197,7 +240,10 @@ def tailoff(self): def persistent(self): self.ensure_editable() - self._root._edit = None + root = self._root + assert isinstance(root, Node) + + root._edit = None trimmed = [None] * (self._cnt - self.tailoff()) list_copy(self._tail, 0, trimmed, 0, len(trimmed)) return PersistentVector(nil, self._cnt, self._shift, self._root, trimmed) @@ -217,16 +263,19 @@ def conj(self, val): self._cnt += 1 return self - tail_node = Node(self._root._edit, self._tail) + root = self._root + assert isinstance(root, Node) + + tail_node = Node(root._edit, self._tail) self._tail = [None] * 32 self._tail[0] = val new_shift = self._shift if (self._cnt >> 5) > (r_uint(1) << self._shift): - new_root = Node(self._root._edit) + new_root = Node(root._edit) new_root._array[0] = self._root - new_root._array[1] = new_path(self._root._edit, self._shift, tail_node) - new_shift += 1 + new_root._array[1] = new_path(root._edit, self._shift, tail_node) + new_shift += 5 else: new_root = self.push_tail(self._shift, self._root, tail_node) @@ -239,6 +288,9 @@ def conj(self, val): def push_tail(self, level, parent, tail_node): parent = self.ensure_node_editable(parent) + root = self._root + assert isinstance(root, Node) + sub_idx = ((self._cnt - 1) >> level) & 0x01f ret = parent @@ -249,7 +301,7 @@ def push_tail(self, level, parent, tail_node): if child is not None: node_to_insert = self.push_tail(level - 5, child, tail_node) else: - node_to_insert = new_path(self._root._edit, level-5, tail_node) + node_to_insert = new_path(root._edit, level - 5, tail_node) ret._array[sub_idx] = node_to_insert return ret @@ -276,7 +328,7 @@ def editable_array_for(self, i): node = self._root level = self._shift while level > 0: - node = self.ensure_node_editable(node._array[(i >> self._level) & 0x1f]) + node = self.ensure_node_editable(node._array[(i >> level) & 0x1f]) level -= 5 return node._array @@ -293,11 +345,7 @@ def nth(self, i, not_found=nil): def pop(self): self.ensure_editable() - affirm(self._cnt != 0, u"Can't pop and empty vector") - - if self._cnt == 0: - self._cnt = 0 - return self + affirm(self._cnt != 0, u"Can't pop an empty vector") i = self._cnt - 1 @@ -305,13 +353,16 @@ def pop(self): self._cnt -= 1 return self - new_tail = self.editable_array_for(self._cnt - 2) + new_tail = self.editable_array_for(self._cnt - 1) new_root = self.pop_tail(self._shift, self._root) new_shift = self._shift + root = self._root + assert isinstance(root, Node) + if new_root is None: - new_root = Node(self._root._edit) + new_root = Node(root._edit) if self._shift > 5 and new_root._array[1] is None: new_root = self.ensure_node_editable(new_root._array[0]) @@ -345,9 +396,6 @@ def pop_tail(self, level, node): return ret - - - @jit.unroll_safe def list_copy(from_lst, from_loc, to_list, to_loc, count): from_loc = r_uint(from_loc) @@ -356,42 +404,102 @@ def list_copy(from_lst, from_loc, to_list, to_loc, count): i = r_uint(0) while i < count: - to_list[to_loc + i] = from_lst[from_loc+i] + to_list[to_loc + i] = from_lst[from_loc + i] i += 1 return to_list - @extend(proto._count, PersistentVector) def _count(self): assert isinstance(self, PersistentVector) return rt.wrap(intmask(self._cnt)) + @extend(proto._nth, PersistentVector) def _nth(self, idx): assert isinstance(self, PersistentVector) return self.nth(idx.int_val()) +@extend(proto._nth_not_found, PersistentVector) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, PersistentVector) + return self.nth(idx.int_val(), not_found) + + +@extend(proto._val_at, PersistentVector) +def _val_at(self, key, not_found): + assert isinstance(self, PersistentVector) + if isinstance(key, Integer): + return self.nth(key.int_val(), not_found) + else: + return not_found + + +@extend(proto._eq, PersistentVector) +def _eq(self, obj): + assert isinstance(self, PersistentVector) + if self is obj: + return true + elif isinstance(obj, PersistentVector): + if self._cnt != obj._cnt: + return false + for i in range(0, intmask(self._cnt)): + if not rt.eq(self.nth(i), obj.nth(i)): + return false + return true + else: + if obj is nil or not rt._satisfies_QMARK_(proto.ISeqable, obj): + return false + seq = rt.seq(obj) + for i in range(0, intmask(self._cnt)): + if seq is nil or not rt.eq(self.nth(i), rt.first(seq)): + return false + seq = rt.next(seq) + if seq is not nil: + return false + return true + + +@extend(proto._contains_key, PersistentVector) +def _contains_key(self, key): + assert isinstance(self, PersistentVector) + if not isinstance(key, Integer): + return false + else: + return true if key.int_val() >= 0 and key.int_val() < intmask(self._cnt) else false + + @extend(proto._conj, PersistentVector) def _conj(self, v): assert isinstance(self, PersistentVector) return self.conj(v) + @extend(proto._push, PersistentVector) def _push(self, v): assert isinstance(self, PersistentVector) return self.conj(v) + @extend(proto._pop, PersistentVector) -def _push(self): +def _pop(self): assert isinstance(self, PersistentVector) return self.pop() + +@extend(proto._assoc, PersistentVector) +def _assoc(self, idx, val): + assert isinstance(self, PersistentVector) + affirm(isinstance(idx, Integer), u"key must be an integer") + return self.assoc_at(r_uint(idx.int_val()), val) + + @extend(proto._meta, PersistentVector) def _meta(self): assert isinstance(self, PersistentVector) return self.meta() + @extend(proto._with_meta, PersistentVector) def _with_meta(self, meta): assert isinstance(self, PersistentVector) @@ -399,8 +507,9 @@ def _with_meta(self, meta): _reduce_driver = jit.JitDriver(name="pixie.stdlib.PersistentVector_reduce", - greens=["f"], - reds="auto") + greens=["f"], + reds="auto") + @extend(proto._reduce, PersistentVector) def _reduce(self, f, init): @@ -409,7 +518,6 @@ def _reduce(self, f, init): while i < self._cnt: array = self.array_for(i) for j in range(len(array)): - item = array[j] _reduce_driver.jit_merge_point(f=f) init = f.invoke([init, array[j]]) @@ -423,26 +531,47 @@ def _reduce(self, f, init): @as_var("vector") def vector__args(args): - acc = EMPTY + acc = rt._transient(EMPTY) for x in range(len(args)): - acc = acc.conj(args[x]) - return acc + acc = rt._conj_BANG_(acc, args[x]) + return rt._persistent_BANG_(acc) + @extend(proto._transient, PersistentVector) def _transient(self): assert isinstance(self, PersistentVector) return TransientVector(self._cnt, self._shift, TransientVector.editable_root(self._root), TransientVector.editable_tail(self._tail)) + @extend(proto._persistent_BANG_, TransientVector) def _persistent(self): assert isinstance(self, TransientVector) return self.persistent() + @extend(proto._conj_BANG_, TransientVector) def _conj(self, val): assert isinstance(self, TransientVector) return self.conj(val) + +@extend(proto._pop_BANG_, TransientVector) +def _pop(self): + assert isinstance(self, TransientVector) + return self.pop() + + +@extend(proto._push_BANG_, TransientVector) +def _push(self, val): + assert isinstance(self, TransientVector) + return self.conj(val) + + +@extend(proto._count, TransientVector) +def _count(self): + assert isinstance(self, TransientVector) + return rt.wrap(intmask(self._cnt)) + proto.IVector.add_satisfies(PersistentVector._type) EMPTY = PersistentVector(nil, r_uint(0), r_uint(5), EMPTY_NODE, []) diff --git a/pixie/vm/primitives.py b/pixie/vm/primitives.py index acb37963..5924b6a5 100644 --- a/pixie/vm/primitives.py +++ b/pixie/vm/primitives.py @@ -1,22 +1,15 @@ import pixie.vm.object as object - class Nil(object.Object): _type = object.Type(u"pixie.stdlib.Nil") - def type(self): - return Nil._type - + def __repr__(self): + return u"nil" nil = Nil() - class Bool(object.Object): _type = object.Type(u"pixie.stdlib.Bool") - def type(self): - return Bool._type - - true = Bool() -false = Bool() \ No newline at end of file +false = Bool() diff --git a/pixie/vm/reader.py b/pixie/vm/reader.py index 78644fd8..4a6ab292 100644 --- a/pixie/vm/reader.py +++ b/pixie/vm/reader.py @@ -1,21 +1,29 @@ py_object = object import pixie.vm.object as object -from pixie.vm.object import affirm +from pixie.vm.object import affirm, runtime_error import pixie.vm.code as code +from pixie.vm.code import as_var from pixie.vm.primitives import nil, true, false import pixie.vm.numbers as numbers from pixie.vm.cons import cons from pixie.vm.symbol import symbol, Symbol -from pixie.vm.keyword import keyword +from pixie.vm.keyword import keyword, Keyword import pixie.vm.rt as rt from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR -from pixie.vm.libs.readline import _readline -from pixie.vm.string import String -from pixie.vm.code import wrap_fn, extend +from pixie.vm.string import Character +from pixie.vm.code import wrap_fn from pixie.vm.persistent_hash_map import EMPTY as EMPTY_MAP -import pixie.vm.stdlib as proto +from pixie.vm.persistent_hash_set import EMPTY as EMPTY_SET +from pixie.vm.persistent_list import EmptyList import pixie.vm.compiler as compiler +from rpython.rlib.rbigint import rbigint +from rpython.rlib.rsre import rsre_re as re + +READING_FORM_VAR = code.intern_var(u"pixie.stdlib", u"*reading-form*") +READING_FORM_VAR.set_dynamic() +READING_FORM_VAR.set_root(false) + LINE_NUMBER_KW = keyword(u"line-number") COLUMN_NUMBER_KW = keyword(u"column-number") LINE_KW = keyword(u"line") @@ -23,7 +31,13 @@ GEN_SYM_ENV = code.intern_var(u"pixie.stdlib.reader", u"*gen-sym-env*") GEN_SYM_ENV.set_dynamic() -GEN_SYM_ENV.set_value(EMPTY_MAP) +GEN_SYM_ENV.set_root(EMPTY_MAP) + +ARG_AMP = symbol(u"&") +ARG_MAX = keyword(u"max-arg") +ARG_ENV = code.intern_var(u"pixie.stdlib.reader", u"*arg-env*") +ARG_ENV.set_dynamic() +ARG_ENV.set_root(nil) class PlatformReader(object.Object): _type = object.Type(u"PlatformReader") @@ -31,10 +45,14 @@ class PlatformReader(object.Object): def read(self): assert False - def unread(self, ch): + def unread(self): pass + def reset_line(self): + return self + class StringReader(PlatformReader): + _type = object.Type(u"pixie.stdlib.StringReader") def __init__(self, s): affirm(isinstance(s, unicode), u"StringReader requires unicode") @@ -48,20 +66,22 @@ def read(self): self._idx += 1 return ch - def unread(self, ch): + def unread(self): self._idx -= 1 -class PromptReader(PlatformReader): - def __init__(self): +class UserSpaceReader(PlatformReader): + def __init__(self, reader_fn): self._string_reader = None + self._reader_fn = reader_fn def read(self): if self._string_reader is None: - result = _readline(">>") + result = rt.name(self._reader_fn.invoke([])) + code._dynamic_vars.set_var_value(READING_FORM_VAR, rt.wrap(READING_FORM_VAR.deref().int_val() + 1)) if result == u"": raise EOFError() - self._string_reader = StringReader(result + u"\n") + self._string_reader = StringReader(result) try: return self._string_reader.read() @@ -69,14 +89,22 @@ def read(self): self._string_reader = None return self.read() - def unread(self, ch): + def reset_line(self): + self._string_reader = None + + def unread(self): assert self._string_reader is not None - self._string_reader.unread(ch) + self._string_reader.unread() + +@as_var(u"reader-fn") +def reader_fn(fn): + """(reader-fn f) + Creates a new reader that can be passed to read, that will call the given f when a new string + of input is needed.""" + return MetaDataReader(UserSpaceReader(fn)) class LinePromise(object.Object): _type = object.Type(u"pixie.stdlib.LinePromise") - def type(self): - return LinePromise._type def __init__(self): self._chrs = [] @@ -86,8 +114,9 @@ def add_char(self, ch): self._chrs.append(ch) def finalize(self): - self._str = u"".join(self._chrs) - self._chrs = None + if self._chrs is not None: + self._str = u"".join(self._chrs) + self._chrs = None def is_finalized(self): return self._chrs is None @@ -137,11 +166,21 @@ def read(self): self._cur_ch = ch return ch + def reset_line(self): + self._line.finalize() + self._line_number += 1 + self._column_number = 0 + self._parent_reader.reset_line() + return self + def get_metadata(self): - return object.InterpreterCodeInfo(self._line, self._line_number, self._column_number, self._filename) + return rt.hashmap(LINE_KW, rt.wrap(self._line), + LINE_NUMBER_KW, rt.wrap(self._line_number), + COLUMN_NUMBER_KW, rt.wrap(self._column_number), + FILE_KW, rt.wrap(self._filename)) - def unread(self, ch): + def unread(self): affirm(not self._has_unread, u"Can't unread twice") self._has_unread = True self._prev_chr = self._cur_ch @@ -164,15 +203,17 @@ def is_whitespace(ch): def is_digit(ch): return ch in u"0123456789" +def is_terminating_macro(ch): + return ch != u"#" and ch != u"'" and ch != u"%" and ch in handlers + def eat_whitespace(rdr): while True: ch = rdr.read() if is_whitespace(ch): continue - rdr.unread(ch) + rdr.unread() return - class ReaderHandler(py_object): def invoke(self, rdr, ch): pass @@ -181,67 +222,109 @@ class ListReader(ReaderHandler): def invoke(self, rdr, ch): lst = [] while True: - eat_whitespace(rdr) + try: + eat_whitespace(rdr) + except EOFError: + throw_syntax_error_with_data(rdr, u"Unmatched list open '('") ch = rdr.read() + if ch == u")": + if len(lst) == 0: + return EmptyList() acc = nil for x in range(len(lst) - 1, -1, -1): acc = cons(lst[x], acc) return acc + + rdr.unread() - rdr.unread(ch) - lst.append(read(rdr, True)) + itm = read_inner(rdr, True, always_return_form=False) + if itm != rdr: + lst.append(itm) -class UnmachedListReader(ReaderHandler): +class UnmatchedListReader(ReaderHandler): def invoke(self, rdr, ch): - raise SyntaxError() + throw_syntax_error_with_data(rdr, u"Unmatched list close ')'") class VectorReader(ReaderHandler): def invoke(self, rdr, ch): acc = EMPTY_VECTOR while True: - eat_whitespace(rdr) + try: + eat_whitespace(rdr) + except EOFError: + rdr + throw_syntax_error_with_data(rdr, u"Unmatched vector open '['") + ch = rdr.read() if ch == u"]": return acc - rdr.unread(ch) - acc = rt.conj(acc, read(rdr, True)) + rdr.unread() + itm = read_inner(rdr, True, always_return_form=False) + if itm != rdr: + acc = rt.conj(acc, itm) -class UnmachedVectorReader(ReaderHandler): + +class UnmatchedVectorReader(ReaderHandler): def invoke(self, rdr, ch): - raise SyntaxError() + throw_syntax_error_with_data(rdr, u"Unmatched vector close ']'") class MapReader(ReaderHandler): def invoke(self, rdr, ch): acc = EMPTY_MAP while True: - eat_whitespace(rdr) + try: + eat_whitespace(rdr) + except EOFError: + throw_syntax_error_with_data(rdr, u"Unmatched map open '{'") + ch = rdr.read() if ch == u"}": return acc - rdr.unread(ch) - k = read(rdr, True) - v = read(rdr, False) - acc = rt._assoc(acc, k, v) + rdr.unread() + itm = read_inner(rdr, True, always_return_form=False) + if itm != rdr: + k = itm + itm = rdr + while itm == rdr: + itm = read_inner(rdr, False, always_return_form=False) + v = itm + acc = rt._assoc(acc, k, v) + return acc -class UnmachedMapReader(ReaderHandler): +class UnmatchedMapReader(ReaderHandler): def invoke(self, rdr, ch): - affirm(False, u"Unmatched Map brace ") + affirm(False, u"Unmatched Map brace '}'") class QuoteReader(ReaderHandler): def invoke(self, rdr, ch): - itm = read(rdr, True) + itm = read_inner(rdr, True) return cons(symbol(u"quote"), cons(itm)) class KeywordReader(ReaderHandler): + def fqd(self, itm): + ns_alias = rt.namespace(itm) + current_nms = rt.ns.deref() + + if ns_alias is None: + return keyword(rt.name(itm), rt.name(current_nms)) + else: + ns_fqd = current_nms.resolve_ns(ns_alias) + return keyword(rt.name(itm), rt.name(ns_fqd)) + def invoke(self, rdr, ch): - itm = read(rdr, True) - affirm(isinstance(itm, Symbol), u"Can't keyword quote a non-symbol") + ch = rdr.read() + if ch == u":": + ch = rdr.read() + itm = read_symbol(rdr, ch, False) + return self.fqd(itm) + else: + itm = read_symbol(rdr, ch, False) - return keyword(itm._str) + return keyword(rt.name(itm), rt.namespace(itm)) class LiteralStringReader(ReaderHandler): def invoke(self, rdr, ch): @@ -252,14 +335,85 @@ def invoke(self, rdr, ch): try: v = rdr.read() except EOFError: - raise Exception("unmatched quote") + return throw_syntax_error_with_data(rdr, u"Unmatched string quote '\"'") + if v == "\"": return rt.wrap(u"".join(acc)) - acc.append(v) + elif v == "\\": + #inside escape... TODO handle everything. + try: + v = rdr.read() + if v == "\"": + acc.append("\"") + elif v == "\\": + acc.append("\\") + elif v == "n": + acc.append("\n") + elif v == "r": + acc.append("\r") + elif v == "t": + acc.append("\t") + else: + throw_syntax_error_with_data(rdr, u"unhandled escape character: " + v) + except EOFError: + throw_syntax_error_with_data(rdr, u"eof after escape character") + else: + acc.append(v) + +def read_token(rdr): + acc = rdr.read() + while True: + ch = rdr.read() + if is_whitespace(ch) or is_terminating_macro(ch): + rdr.unread() + return acc + acc += ch + +def read_unicode_char(rdr, token, offset, length, base): + if len(token) != offset + length: + throw_syntax_error_with_data(rdr, u"Invalid unicode character: \\" + token) + c = 0 + for i in range(offset, offset + length): + d = int(str(token[i:i+1]), base) + c = c * base + d + return c + +class LiteralCharacterReader(ReaderHandler): + def invoke(self, rdr, ch): + token = read_token(rdr) + if len(token) == 1: + return Character(ord(token[0])) + elif token == u"newline": + return Character(ord("\n")) + elif token == u"space": + return Character(ord(" ")) + elif token == u"tab": + return Character(ord("\t")) + elif token == u"backspace": + return Character(ord("\b")) + elif token == u"formfeed": + return Character(ord("\f")) + elif token == u"return": + return Character(ord("\r")) + elif token.startswith("u"): + c = read_unicode_char(rdr, token, 1, 4, 16) + if c >= 0xd800 and c <= 0xdfff: + throw_syntax_error_with_data(rdr, u"Invalid character constant: \\" + token) + return Character(c) + elif token.startswith("o"): + l = len(token) - 1 + if l > 3: + throw_syntax_error_with_data(rdr, u"Invalid octal escape sequence: \\" + token) + c = read_unicode_char(rdr, token, 1, l, 8) + if c > 0377: + throw_syntax_error_with_data(rdr, u"Octal escape sequences must be in range [0, 377]") + return Character(c) + else: + throw_syntax_error_with_data(rdr, u"Unsupported character: \\" + token) class DerefReader(ReaderHandler): def invoke(self, rdr, ch): - return rt.cons(symbol(u"-deref"), rt.cons(read(rdr, True), nil)) + return rt.cons(symbol(u"-deref"), rt.cons(read_inner(rdr, True), nil)) QUOTE = symbol(u"quote") @@ -269,20 +423,21 @@ def invoke(self, rdr, ch): CONCAT = symbol(u"concat") SEQ = symbol(u"seq") LIST = symbol(u"list") +HASHMAP = symbol(u"hashmap") def is_unquote(form): - return True if rt.instance_QMARK_(rt.ISeq.deref(), form) \ + return True if rt._satisfies_QMARK_(rt.ISeq.deref(), form) \ and rt.eq(rt.first(form), UNQUOTE) \ else False def is_unquote_splicing(form): - return True if rt.instance_QMARK_(rt.ISeq.deref(), form) \ + return True if rt._satisfies_QMARK_(rt.ISeq.deref(), form) \ and rt.eq(rt.first(form), UNQUOTE_SPLICING) \ else False class SyntaxQuoteReader(ReaderHandler): def invoke(self, rdr, ch): - form = read(rdr, True) + form = read_inner(rdr, True) with code.bindings(GEN_SYM_ENV, EMPTY_MAP): result = self.syntax_quote(form) @@ -293,7 +448,8 @@ def invoke(self, rdr, ch): def syntax_quote(form): if isinstance(form, Symbol) and compiler.is_compiler_special(form): ret = rt.list(QUOTE, form) - if isinstance(form, Symbol): + + elif isinstance(form, Symbol): if rt.namespace(form) is None and rt.name(form).endswith("#"): gmap = rt.deref(GEN_SYM_ENV) affirm(gmap is not nil, u"Gensym literal used outside a syntax quote") @@ -301,7 +457,7 @@ def syntax_quote(form): if gs is nil: gs = rt.symbol(rt.str(form, rt.wrap(u"__"), rt.gensym())) GEN_SYM_ENV.set_value(rt.assoc(gmap, form, gs)) - form = gs + form = gs else: var = rt.resolve_in(compiler.NS_VAR.deref(), form) if var is nil: @@ -312,19 +468,30 @@ def syntax_quote(form): elif is_unquote(form): ret = rt.first(rt.next(form)) elif is_unquote_splicing(form): - raise Exception("Unquote splicing not used inside list") + return runtime_error(u"Unquote splicing not used inside list") elif rt.vector_QMARK_(form) is true: ret = rt.list(APPLY, CONCAT, SyntaxQuoteReader.expand_list(form)) - elif rt.seq_QMARK_(form) is true: + elif rt.map_QMARK_(form) is true: + mes = SyntaxQuoteReader.flatten_map(form) + ret = rt.list(APPLY, HASHMAP, rt.list(APPLY, CONCAT, SyntaxQuoteReader.expand_list(mes))) + elif form is not nil and rt.seq_QMARK_(form) is true: ret = rt.list(APPLY, LIST, rt.cons(CONCAT, SyntaxQuoteReader.expand_list(rt.seq(form)))) else: ret = rt.list(QUOTE, form) return ret + @staticmethod + def flatten_map(form): + return rt.reduce(flatten_map_rfn, EMPTY_VECTOR, form) + @staticmethod def expand_list(form): return rt.reduce(expand_list_rfn, EMPTY_VECTOR, form) +@wrap_fn +def flatten_map_rfn(ret, item): + return rt.conj(rt.conj(ret, rt.first(item)), rt.first(rt.next(item))) + @wrap_fn def expand_list_rfn(ret, item): if is_unquote(item): @@ -342,23 +509,217 @@ def invoke(self, rdr, ch): if ch == "@": sym = UNQUOTE_SPLICING else: - rdr.unread(ch) + rdr.unread() - form = read(rdr, True) + form = read_inner(rdr, True) return rt.list(sym, form) +class MetaReader(ReaderHandler): + def invoke(self, rdr, ch): + meta = read_inner(rdr, True) + obj = read_inner(rdr, True) + + if isinstance(meta, Keyword): + meta = rt.hashmap(meta, true) + + if isinstance(meta, Symbol): + meta = rt.hashmap(keyword(u"tag"), meta) + + if rt._satisfies_QMARK_(rt.IMeta.deref(), obj): + return rt.with_meta(obj, rt.merge(meta, rt.meta(obj))) + + return obj + +class ArgReader(ReaderHandler): + def invoke(self, rdr, ch): + if ARG_ENV.deref() is nil: + return read_symbol(rdr, ch) + + ch = rdr.read() + rdr.unread() + if is_whitespace(ch) or is_terminating_macro(ch): + return ArgReader.register_next_arg(1) + + n = read_inner(rdr, True) + if rt.eq(n, ARG_AMP): + return ArgReader.register_next_arg(-1) + if not isinstance(n, numbers.Integer): + throw_syntax_error_with_data(rdr, u"%-arg must be %, % or %&") + return ArgReader.register_next_arg(n.int_val()) + + @staticmethod + def register_next_arg(n): + arg_env = ARG_ENV.deref() + max_arg = rt.get(arg_env, ARG_MAX) + if n > max_arg.int_val(): + arg_env = rt.assoc(arg_env, ARG_MAX, rt.wrap(n)) + + arg = rt.get(arg_env, rt.wrap(n), nil) + if arg is nil: + arg = ArgReader.gen_arg(n) + arg_env = rt.assoc(arg_env, rt.wrap(n), arg) + ARG_ENV.set_value(arg_env) + return arg + + @staticmethod + def gen_arg(n): + s = unicode(str(n)) + if n == -1: + s = u"_rest" + return rt.gensym(rt.wrap(u"arg" + s + u"__")) + +class FnReader(ReaderHandler): + def invoke(self, rdr, ch): + if ARG_ENV.deref() is not nil: + throw_syntax_error_with_data(rdr, u"Nested #()s are not allowed") + + try: + ARG_ENV.set_value(rt.assoc(EMPTY_MAP, ARG_MAX, rt.wrap(-1))) + + rdr.unread() + form = read_inner(rdr, True) + + args = EMPTY_VECTOR + percent_args = ARG_ENV.deref() + max_arg = rt.get(percent_args, ARG_MAX) + + for i in range(1, max_arg.int_val() + 1): + arg = rt.get(percent_args, rt.wrap(i)) + if arg is nil: + arg = ArgReader.gen_arg(i) + args = rt.conj(args, arg) + + rest_arg = rt.get(percent_args, rt.wrap(-1)) + if rest_arg is not nil: + args = rt.conj(args, ARG_AMP) + args = rt.conj(args, rest_arg) + + return rt.cons(symbol(u"fn*"), rt.cons(args, rt.cons(form, nil))) + finally: + ARG_ENV.set_value(nil) + +class SetReader(ReaderHandler): + def invoke(self, rdr, ch): + acc = EMPTY_SET + while True: + try: + eat_whitespace(rdr) + except EOFError: + throw_syntax_error_with_data(rdr, u"Unmatched set open '#{'") + ch = rdr.read() + if ch == u"}": + return acc + + rdr.unread() + itm = read_inner(rdr, True, always_return_form=False) + if itm != rdr: + acc = acc.conj(itm) + +class CommentReader(ReaderHandler): + def invoke(self, rdr, ch): + read_inner(rdr, True, always_return_form=True) + return rdr + +dispatch_handlers = { + u"{": SetReader(), + u"(": FnReader(), + u"_": CommentReader() +} + +class DispatchReader(ReaderHandler): + def invoke(self, rdr, ch): + ch = rdr.read() + handler = dispatch_handlers.get(ch, None) + if handler is None: + return throw_syntax_error_with_data(rdr, u"unknown dispatch #" + ch) + return handler.invoke(rdr, ch) + +class LineCommentReader(ReaderHandler): + def invoke(self, rdr, ch): + self.skip_line(rdr) + return rdr + + def skip_line(self, rdr): + while True: + ch = rdr.read() + if ch == u"\n": + return + handlers = {u"(": ListReader(), - u")": UnmachedListReader(), + u")": UnmatchedListReader(), u"[": VectorReader(), - u"]": UnmachedVectorReader(), + u"]": UnmatchedVectorReader(), u"{": MapReader(), - u"}": UnmachedMapReader(), + u"}": UnmatchedMapReader(), u"'": QuoteReader(), u":": KeywordReader(), u"\"": LiteralStringReader(), + u"\\": LiteralCharacterReader(), u"@": DerefReader(), u"`": SyntaxQuoteReader(), - u"~": UnquoteReader()} + u"~": UnquoteReader(), + u"^": MetaReader(), + u"#": DispatchReader(), + u";": LineCommentReader(), + u"%": ArgReader() +} + +# inspired by https://github.com/clojure/tools.reader/blob/9ee11ed/src/main/clojure/clojure/tools/reader/impl/commons.clj#L45 +# sign hex oct radix decimal biginteger +# 1 2 3 4 5 6 7 8 +int_matcher = re.compile(u'^([-+]?)(?:(0[xX])([0-9a-fA-F]+)|0([0-7]+)|([1-9][0-9]?)[rR]([0-9a-zA-Z]+)|([0-9]*))(N)?$') + +float_matcher = re.compile(u'^([-+]?[0-9]+(\.[0-9]*)?([eE][-+]?[0-9]+)?)$') +ratio_matcher = re.compile(u'^([-+]?[0-9]+)/([0-9]+)$') + +def parse_int(m): + sign = 1 + if m.group(1) == u'-': + sign = -1 + + radix = 10 + + if m.group(7): + num = m.group(7) + elif m.group(2): + radix = 16 + num = m.group(3) + elif m.group(4): + radix = 8 + num = m.group(4) + elif m.group(5): + radix = int(m.group(5)) + num = m.group(6) + else: + return None + + if m.group(8): + return rt.wrap(rbigint.fromstr(str(m.group(1) + num), radix)) + else: + return rt.wrap(sign * int(str(num), radix)) + +def parse_float(m): + return rt.wrap(float(str(m.group(0)))) + +def parse_ratio(m): + n = rt.wrap(int(str(m.group(1)))) + d = rt.wrap(int(str(m.group(2)))) + return rt._div(n, d) + +def parse_number(s): + m = int_matcher.match(s) + if m: + return parse_int(m) + else: + m = float_matcher.match(s) + if m: + return parse_float(m) + else: + m = ratio_matcher.match(s) + if m: + return parse_ratio(m) + else: + return None def read_number(rdr, ch): acc = [ch] @@ -366,33 +727,39 @@ def read_number(rdr, ch): while True: ch = rdr.read() if is_whitespace(ch) or ch in handlers: - rdr.unread(ch) + rdr.unread() break acc.append(ch) except EOFError: pass - return rt.wrap(int(u"".join(acc))) + joined = u"".join(acc) + parsed = parse_number(joined) + if parsed is not None: + return parsed + return Symbol(joined) -def read_symbol(rdr, ch): +def read_symbol(rdr, ch, convert_primitives=True): acc = [ch] try: while True: ch = rdr.read() - if is_whitespace(ch) or ch in handlers: - rdr.unread(ch) + if is_whitespace(ch) or is_terminating_macro(ch): + rdr.unread() break acc.append(ch) except EOFError: pass sym_str = u"".join(acc) - if sym_str == u"true": - return true - if sym_str == u"false": - return false - if sym_str == u"nil": - return nil + + if convert_primitives: + if sym_str == u"true": + return true + if sym_str == u"false": + return false + if sym_str == u"nil": + return nil return symbol(sym_str) class EOF(object.Object): @@ -401,19 +768,30 @@ class EOF(object.Object): eof = EOF() +code.intern_var(u"pixie.stdlib", u"eof").set_root(eof) +def throw_syntax_error_with_data(rdr, txt): + assert isinstance(txt, unicode) + if isinstance(rdr, MetaDataReader): + meta = rdr.get_metadata() + else: + meta = nil + data = rt.interpreter_code_info(meta) + err = object.runtime_error(txt) + err._trace.append(data) + raise object.WrappedException(err) - -def read(rdr, error_on_eof): +def read_inner(rdr, error_on_eof, always_return_form=True): try: eat_whitespace(rdr) except EOFError as ex: if error_on_eof: - raise ex + runtime_error(u"Unexpected EOF while reading", + u"pixie.stdlib/EOFWhileReadingException") return eof @@ -427,6 +805,9 @@ def read(rdr, error_on_eof): macro = handlers.get(ch, None) if macro is not None: itm = macro.invoke(rdr, ch) + if always_return_form and itm == rdr: + return read_inner(rdr, error_on_eof, always_return_form=always_return_form) + elif is_digit(ch): itm = read_number(rdr, ch) @@ -434,19 +815,42 @@ def read(rdr, error_on_eof): elif ch == u"-": ch2 = rdr.read() if is_digit(ch2): - rdr.unread(ch2) + rdr.unread() itm = read_number(rdr, ch) + else: - rdr.unread(ch2) + rdr.unread() itm = read_symbol(rdr, ch) else: itm = read_symbol(rdr, ch) - if rt.has_meta_QMARK_(itm): - itm = rt.with_meta(itm, meta) + if itm != rdr: + if rt.has_meta_QMARK_(itm): + itm = rt.with_meta(itm, rt.merge(meta, rt.meta(itm))) return itm +def read(rdr, error_on_eof): + code._dynamic_vars.push_binding_frame() + code._dynamic_vars.set_var_value(READING_FORM_VAR, rt.wrap(0)) + try: + form = read_inner(rdr, error_on_eof) + return form + finally: + code._dynamic_vars.pop_binding_frame() + +@as_var("read") +def _read_(rdr, error_on_eof): + """(read rdr error-on-eof) + Reads a single form from the input reader. If error-on-eof is true, an error will be thrown if eof is reached + and a valid form has not been parsed. Else will return eof""" + assert isinstance(rdr, PlatformReader) + return read(rdr, rt.is_true(error_on_eof)) + + + + + diff --git a/pixie/vm/reduced.py b/pixie/vm/reduced.py index 023cb406..c94739a6 100644 --- a/pixie/vm/reduced.py +++ b/pixie/vm/reduced.py @@ -1,14 +1,11 @@ -import pixie.vm.rt as rt import pixie.vm.object as object from pixie.vm.code import extend, as_var, returns -from pixie.vm.primitives import nil, true, false +from pixie.vm.primitives import true, false import pixie.vm.stdlib as proto class Reduced(object.Object): _type = object.Type(u"pixie.stdlib.Reduced") - def type(self): - return Reduced._type def __init__(self, boxed_value): self._boxed_value = boxed_value diff --git a/pixie/vm/rt.py b/pixie/vm/rt.py index 3259f9ac..c17389cb 100644 --- a/pixie/vm/rt.py +++ b/pixie/vm/rt.py @@ -1,21 +1,23 @@ __config__ = None py_list = list -from rpython.rlib.objectmodel import specialize - +py_str = str +from rpython.rlib.objectmodel import specialize, we_are_translated def init(): - import pixie.vm.code as code from pixie.vm.object import affirm, _type_registry from rpython.rlib.rarithmetic import r_uint + from rpython.rlib.rbigint import rbigint from pixie.vm.primitives import nil, true, false from pixie.vm.string import String + from pixie.vm.object import Object + from pixie.vm.compiler import NS_VAR _type_registry.set_registry(code._ns_registry) def unwrap(fn): - if isinstance(fn, code.Var) and hasattr(fn.deref(), "_returns"): + if isinstance(fn, code.Var) and fn.is_defined() and hasattr(fn.deref(), "_returns"): tp = fn.deref()._returns if tp is bool: def wrapper(*args): @@ -31,24 +33,26 @@ def wrapper(*args): ret = fn.invoke(py_list(args)) if ret is nil: return None - affirm(isinstance(ret, String), u"Invalid return value, expected String") + + if not isinstance(ret, String): + from pixie.vm.object import runtime_error + runtime_error(u"Invalid return value, expected String") return ret._str return wrapper else: assert False, "Don't know how to convert" + str(tp) return lambda *args: fn.invoke(py_list(args)) - - if globals().has_key("__inited__"): + if "__inited__" in globals(): return import sys - sys.setrecursionlimit(10000) # Yeah we blow the stack sometimes, we promise it's not a bug + #sys.setrecursionlimit(10000) # Yeah we blow the stack sometimes, we promise it's not a bug + import pixie.vm.code as code import pixie.vm.numbers as numbers - from pixie.vm.code import wrap_fn + import pixie.vm.bits import pixie.vm.interpreter - import pixie.vm.stacklet as stacklet import pixie.vm.atom import pixie.vm.reduced import pixie.vm.util @@ -56,38 +60,61 @@ def wrapper(*args): import pixie.vm.lazy_seq import pixie.vm.persistent_list import pixie.vm.persistent_hash_map + import pixie.vm.persistent_hash_set import pixie.vm.custom_types - import pixie.vm.compiler as compiler import pixie.vm.map_entry - import pixie.vm.reader as reader - - - - numbers.init() + import pixie.vm.libs.platform + import pixie.vm.libs.ffi + import pixie.vm.libs.env + import pixie.vm.symbol + import pixie.vm.libs.path + import pixie.vm.libs.string + import pixie.vm.threads + import pixie.vm.string_builder + import pixie.vm.stacklet + + import pixie.vm.c_api @specialize.argtype(0) def wrap(x): + if isinstance(x, bool): + return true if x else false if isinstance(x, int): return numbers.Integer(x) + if isinstance(x, rbigint): + return numbers.BigInteger(x) + if isinstance(x, float): + return numbers.Float(x) if isinstance(x, unicode): return String(x) - #if isinstance(x, str): - # return String(unicode(x)) + if isinstance(x, py_str): + return String(unicode(x)) + if isinstance(x, Object): + return x + if x is None: + return nil + + if not we_are_translated(): + print x, type(x) affirm(False, u"Bad wrap") globals()["wrap"] = wrap + def int_val(x): + affirm(isinstance(x, numbers.Number), u"Expected number") + return x.int_val() + + globals()["int_val"] = int_val + from pixie.vm.code import _ns_registry, BaseCode, munge for name, var in _ns_registry._registry[u"pixie.stdlib"]._registry.iteritems(): name = munge(name) - print name - if isinstance(var.deref(), BaseCode): + if var.is_defined() and isinstance(var.deref(), BaseCode): globals()[name] = unwrap(var) else: globals()[name] = var - import pixie.vm.bootstrap def reinit(): @@ -96,34 +123,46 @@ def reinit(): if name in globals(): continue - print "Found ->> ", name, var.deref() - if isinstance(var.deref(), BaseCode): + if var.is_defined() and isinstance(var.deref(), BaseCode): globals()[name] = unwrap(var) else: globals()[name] = var - f = open("pixie/stdlib.lisp") - data = f.read() - f.close() - rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") - result = nil - - @wrap_fn - def run_load_stdlib(): - with compiler.with_ns(u"pixie.stdlib"): - while True: - form = reader.read(rdr, False) - if form is reader.eof: - return result - result = compiler.compile(form).invoke([]) - reinit() - - stacklet.with_stacklets(run_load_stdlib) - + #f = open("pixie/stdlib.pxi") + #data = f.read() + #f.close() + #rdr = reader.MetaDataReader(reader.StringReader(unicode(data)), u"pixie/stdlib.pixie") + #result = nil + # + # @wrap_fn + # def run_load_stdlib(): + # with compiler.with_ns(u"pixie.stdlib"): + # while True: + # form = reader.read(rdr, False) + # if form is reader.eof: + # return result + # result = compiler.compile(form).invoke([]) + # reinit() + # + # stacklet.with_stacklets(run_load_stdlib) + + init_fns = [u"reduce", u"get", u"reset!", u"assoc", u"key", u"val", u"keys", u"vals", u"vec", u"load-file", u"compile-file", + u"load-ns", u"hashmap", u"cons", u"-assoc", u"-val-at"] + for x in init_fns: + globals()[py_str(code.munge(x))] = unwrap(code.intern_var(u"pixie.stdlib", x)) + + init_vars = [u"load-paths"] + for x in init_vars: + globals()[py_str(code.munge(x))] = code.intern_var(u"pixie.stdlib", x) + + globals()[py_str(code.munge(u"ns"))] = NS_VAR + globals()["__inited__"] = True + globals()["is_true"] = lambda x: False if x is false or x is nil or x is None else True - globals()["__inited__"] = True + numbers.init() + code.init() diff --git a/pixie/vm/stacklet.py b/pixie/vm/stacklet.py index 199179d6..d4e8e5c4 100644 --- a/pixie/vm/stacklet.py +++ b/pixie/vm/stacklet.py @@ -1,153 +1,53 @@ -py_object = object -import pixie.vm.object as object -from pixie.vm.object import affirm -from pixie.vm.primitives import nil, true, false -from pixie.vm.code import BaseCode -from pixie.vm.numbers import Integer -import pixie.vm.stdlib as proto -from pixie.vm.code import extend, as_var -from rpython.rlib.rarithmetic import r_uint as r_uint32, intmask, widen -import rpython.rlib.jit as jit -import pixie.vm.rt as rt import rpython.rlib.rstacklet as rstacklet -from rpython.rtyper.lltypesystem import lltype, rffi +from pixie.vm.object import Object, Type, affirm +from pixie.vm.code import as_var +import pixie.vm.rt as rt -OP_NEW = 0x01 -OP_SWITCH = 0x02 -OP_CONTINUE = 0x03 -OP_EXIT = 0x04 -class GlobalState(py_object): +class GlobalState(object): def __init__(self): - self.reset() - - def reset(self): - self._th = None + self._is_inited = False self._val = None - self._ex = None - self._to = None - self._op = 0x00 - self._fn = None - self._init_fn = None - self._from = None - - def switch_back(self): - tmp = self._from - self._from = self._to - self._to = tmp global_state = GlobalState() + def init(): - if global_state._th is None: + if not global_state._is_inited: global_state._th = rstacklet.StackletThread(rt.__config__) - global_state._h = global_state._th.get_null_handle() + global_state._is_inited = True -def shutdown(): - global_state._h = global_state._th.get_null_handle() - global_state.reset() - -class WrappedHandler(BaseCode): - _type = object.Type(u"Stacklet") +class StackletHandle(Object): + _type = Type(u"StackletHandle") def __init__(self, h): - self._h = h - - def type(self): - return WrappedHandler._type + self._stacklet_handle = h + self._used = False - def _invoke(self, args): - affirm(len(args) == 1, u"Only one arg to continuation allowed") - global_state._from = global_state._to - global_state._to = self - global_state._op = OP_SWITCH + def invoke(self, args): + affirm(not self._used, u"Can only call a given stacklet handle once.") + affirm(len(args) == 1, u"Only one arg should be handed to a stacklet handle") + self._used = True global_state._val = args[0] - global_state._h = global_state._th.switch(global_state._h) - - return global_state._val - + new_h = StackletHandle(global_state._th.switch(self._stacklet_handle)) + val = global_state._val + global_state._val = None + return rt.vector(new_h, val) -def new_stacklet(f): - global_state._op = OP_NEW - global_state._val = f - global_state.switch_back() - global_state._h = global_state._th.switch(global_state._h) - val = global_state._val - return val - - -def new_handler(h, o): - global_state._h = h - - affirm(global_state._val is not None, u"Internal Stacklet Error") - f = global_state._val +def new_handler(h, _): + fn = global_state._val global_state._val = None + h = global_state._th.switch(h) + val = global_state._val + fn.invoke([StackletHandle(h), val]) + affirm(False, u"TODO: What do we do now?") + return h - global_state._op = OP_SWITCH - global_state.switch_back() - global_state._h = global_state._th.switch(h) - - #try: - f.invoke([global_state._from]) - #except Exception as ex: - # print "Uncaught Exception" + str(ex) - - - - return global_state._h - - -def init_handler(h, o): - global_state._h = h - - affirm(global_state._init_fn is not None, u"Internal Stacklet error") - f = global_state._init_fn - global_state._init_fn = None - - #try: - f.invoke([]) - #except Exception as ex: - # print "Uncaught Exception" + str(ex) - - global_state._op = OP_EXIT - return global_state._h - - -def with_stacklets(f): - - init() - global_state._init_fn = f - - main_h = global_state._th.new(init_handler) - global_state._from = WrappedHandler(main_h) - - while True: - if global_state._op == OP_NEW: - wh = WrappedHandler(global_state._th.get_null_handle()) - global_state._to = wh - wh._h = global_state._th.new(new_handler) - global_state._val = wh - continue - - elif global_state._op == OP_SWITCH: - to = global_state._to - to._h = global_state._th.switch(global_state._to._h) - continue - - elif global_state._op == OP_EXIT: - shutdown() - return global_state._val - - else: - break - - shutdown() - return None - -@as_var("create-stacklet") -def _new_stacklet(f): - return new_stacklet(f) - +@as_var("new-stacklet") +def new_stacklet(fn): + global_state._val = fn + h = global_state._th.new(new_handler) + return StackletHandle(h) diff --git a/pixie/vm/stdlib.py b/pixie/vm/stdlib.py index fc835e86..d8611d72 100644 --- a/pixie/vm/stdlib.py +++ b/pixie/vm/stdlib.py @@ -1,24 +1,27 @@ # -*- coding: utf-8 -*- -from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm -from pixie.vm.code import BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ - resize_list, list_copy, returns, get_var_if_defined +from pixie.vm.object import Object, Type, _type_registry, WrappedException, RuntimeException, affirm, InterpreterCodeInfo, istypeinstance, \ + runtime_error, add_info, ExtraCodeInfo, finalizer_registry +from pixie.vm.code import Namespace, BaseCode, PolymorphicFn, wrap_fn, as_var, defprotocol, extend, Protocol, Var, \ + list_copy, returns, intern_var, _ns_registry import pixie.vm.code as code -from types import MethodType from pixie.vm.primitives import true, false, nil import pixie.vm.numbers as numbers import rpython.rlib.jit as jit -import rpython.rlib.rstacklet as rstacklet from rpython.rlib.rarithmetic import r_uint - +from rpython.rlib.objectmodel import we_are_translated +import os.path as path +import sys defprotocol("pixie.stdlib", "ISeq", ["-first", "-next"]) defprotocol("pixie.stdlib", "ISeqable", ["-seq"]) defprotocol("pixie.stdlib", "ICounted", ["-count"]) -defprotocol("pixie.stdlib", "IIndexed", ["-nth"]) +defprotocol("pixie.stdlib", "IIndexed", ["-nth", "-nth-not-found"]) + +defprotocol("pixie.stdlib", "IPersistentCollection", ["-conj", "-disj"]) -defprotocol("pixie.stdlib", "IPersistentCollection", ["-conj"]) +defprotocol("pixie.stdlib", "IEmpty", ["-empty"]) defprotocol("pixie.stdlib", "IObject", ["-hash", "-eq", "-str", "-repr"]) _eq.set_default_fn(wrap_fn(lambda a, b: false)) @@ -31,7 +34,7 @@ defprotocol("pixie.stdlib", "INamed", ["-namespace", "-name"]) -defprotocol("pixie.stdlib", "IAssociative", ["-assoc"]) +defprotocol("pixie.stdlib", "IAssociative", ["-assoc", "-contains-key", "-dissoc"]) defprotocol("pixie.stdlib", "ILookup", ["-val-at"]) @@ -39,6 +42,10 @@ defprotocol("pixie.stdlib", "IStack", ["-push", "-pop"]) +defprotocol("pixie.stdlib", "IFn", ["-invoke"]) + +defprotocol("pixie.stdlib", "IDoc", ["-doc"]) + IVector = as_var("pixie.stdlib", "IVector")(Protocol(u"IVector")) IMap = as_var("pixie.stdlib", "IMap")(Protocol(u"IMap")) @@ -49,15 +56,71 @@ defprotocol("pixie.stdlib", "IToTransient", ["-transient"]) defprotocol("pixie.stdlib", "ITransientCollection", ["-conj!"]) +defprotocol("pixie.stdlib", "ITransientStack", ["-push!", "-pop!"]) + +defprotocol("pixie.stdlib", "IDisposable", ["-dispose!"]) +defprotocol("pixie.stdlib", "IFinalize", ["-finalize!"]) + +defprotocol("pixie.stdlib", "IMessageObject", ["-call-method", "-get-attr"]) + +def maybe_mark_finalizer(self, tp): + if self is _finalize_BANG_: + print "MARKING ", tp + + tp.set_finalizer() + +code.PolymorphicFn.maybe_mark_finalizer = maybe_mark_finalizer + +@as_var("pixie.stdlib.internal", "-defprotocol") +def _defprotocol(name, methods): + from pixie.vm.compiler import NS_VAR + from pixie.vm.persistent_vector import PersistentVector + from pixie.vm.symbol import Symbol + affirm(isinstance(name, Symbol), u"protocol name must be a symbol") + affirm(isinstance(methods, PersistentVector), u"protocol methods must be a vector of symbols") + method_list = [] + for i in range(0, rt.count(methods)): + method_sym = rt.nth(methods, rt.wrap(i)) + affirm(isinstance(method_sym, Symbol), u"protocol methods must be a vector of symbols") + method_list.append(rt.name(method_sym)) + + proto = Protocol(rt.name(name)) + ns = rt.name(NS_VAR.deref()) + intern_var(ns, rt.name(name)).set_root(proto) + for method in method_list: + method = unicode(method) + poly = PolymorphicFn(method, proto) + intern_var(ns, method).set_root(poly) + + return name + +def __make_code_overrides(x): + @extend(_meta, x._type) + def __meta(self): + assert isinstance(self, x) + return self.meta() + + @extend(_with_meta, x._type) + def __meta(self, meta): + assert isinstance(self, x) + return self.with_meta(meta) + +for x in (code.Code, code.Closure, code.VariadicCode, code.MultiArityFn): + __make_code_overrides(x) + def default_str(x): - from pixie.vm.string import String + tp = x.type() + assert isinstance(tp, Type) + return rt.wrap(u"") - return rt.wrap(u"") +def default_hash(x): + return x.hash() _str.set_default_fn(wrap_fn(default_str)) _repr.set_default_fn(wrap_fn(default_str)) +_hash.set_default_fn(wrap_fn(default_hash)) _meta.set_default_fn(wrap_fn(lambda x: nil)) @@ -66,11 +129,23 @@ def default_str(x): @as_var("first") def first(x): - return rt._first(x) + if rt._satisfies_QMARK_(ISeq, x): + return rt._first(x) + + seq = rt.seq(x) + if seq is nil: + return nil + return rt._first(seq) @as_var("next") def next(x): - return rt.seq(rt._next(x)) + if rt._satisfies_QMARK_(ISeq, x): + return rt.seq(rt._next(x)) + seq = rt.seq(x) + if seq is nil: + return nil + else: + return rt.seq(rt._next(seq)) @as_var("seq") def seq(x): @@ -78,8 +153,25 @@ def seq(x): @as_var("seq?") def seq_QMARK_(x): - return true if rt.instance_QMARK_(rt.ISeq.deref(), x) else false + return true if rt._satisfies_QMARK_(rt.ISeq.deref(), x) else false +@as_var("-seq-eq") +def _seq_eq(a, b): + if a is b: + return true + if a is nil or b is nil: + return false + if not (rt._satisfies_QMARK_(rt.ISeqable.deref(), b) or rt._satisfies_QMARK_(rt.ISeq.deref(), b)): + return false + + a = rt.seq(a) + b = rt.seq(b) + while a is not nil: + if b is nil or not rt.eq(rt.first(a), rt.first(b)): + return false + a = rt.next(a) + b = rt.next(b) + return true if b is nil else false @as_var("type") def type(x): @@ -88,8 +180,14 @@ def type(x): @extend(_str, Type) def _str(tp): import pixie.vm.string as string + assert isinstance(tp, Type) return string.rt.wrap(u"") +@extend(_repr, Type) +def _repr(tp): + import pixie.vm.string as string + assert isinstance(tp, Type) + return string.rt.wrap(tp._name) @extend(_first, nil._type) def _first(_): @@ -107,6 +205,18 @@ def _seq(_): def _count(_): return numbers.zero_int +@extend(_assoc, nil._type) +def __assoc(_, k, v): + return rt.hashmap(k, v) + +@extend(_reduce, nil._type) +def __reduce(self, f, init): + return init + +@extend(_val_at, nil._type) +def __val_at(x, k, not_found): + return not_found + @extend(_name, Var) def __name(self): assert isinstance(self, Var) @@ -136,10 +246,10 @@ def __name(_): def __hash(x): return rt._hash(x) - _count_driver = jit.JitDriver(name="pixie.stdlib.count", greens=["tp"], reds="auto") +@returns(r_uint) @as_var("count") def count(x): acc = 0 @@ -148,9 +258,12 @@ def count(x): while True: _count_driver.jit_merge_point(tp=rt.type(x)) if ICounted.satisfies(rt.type(x)): - return rt._add(rt.wrap(acc), rt._count(x)) + return rt._add(rt.wrap(acc), rt._count(x)) + seq = rt.seq(x) + if seq is nil: + return rt.wrap(acc) acc += 1 - x = rt.next(rt.seq(x)) + x = rt._next(seq) @as_var("+") @@ -168,7 +281,7 @@ def __with_meta(a, b): @returns(bool) @as_var("has-meta?") def __has_meta(a): - return true if rt.instance_QMARK_(rt.IMeta.deref(), a) else false + return true if rt._satisfies_QMARK_(rt.IMeta.deref(), a) else false @as_var("conj") def conj(a, b): @@ -178,83 +291,255 @@ def conj(a, b): def nth(a, b): return rt._nth(a, b) +@as_var("nth-not-found") +def nth_not_found(a, b, c): + return rt._nth_not_found(a, b, c) + @as_var("str") def str__args(args): - from pixie.vm.string import String acc = [] for x in args: - acc.append(rt._str(x)._str) + acc.append(rt.name(rt._str(x))) return rt.wrap(u"".join(acc)) @as_var("apply") @jit.unroll_safe def apply__args(args): last_itm = args[len(args) - 1] - if not rt.instance_QMARK_(rt.IIndexed.deref(), last_itm) or \ - not rt.instance_QMARK_(rt.ICounted.deref(), last_itm): - raise ValueError("Last item to apply must be bost IIndexed and ICounted") + if not (rt._satisfies_QMARK_(rt.IIndexed.deref(), last_itm) and + rt._satisfies_QMARK_(rt.ICounted.deref(), last_itm)): + last_itm = rt.vec(last_itm) fn = args[0] argc = r_uint(len(args) - 2) - out_args = [None] * (argc + r_uint(rt.count(last_itm).int_val())) + out_args = [None] * (argc + r_uint(rt.count(last_itm))) list_copy(args, 1, out_args, 0, argc) - for x in range(rt.count(last_itm).int_val()): + for x in range(rt.count(last_itm)): out_args[argc + x] = rt.nth(last_itm, rt.wrap(x)) return fn.invoke(out_args) -@as_var("print") -def _print(a): - print rt._str(a)._str - return nil +#@as_var("print") +#def _print(a): +# print rt._str(a)._str +# return nil @returns(bool) -@as_var("instance?") -def _instance(proto, o): - affirm(isinstance(proto, Protocol), u"proto must be a Protocol") +@as_var("-instance?") +def _instance(c, o): + affirm(isinstance(c, Type), u"c must be a type") + + return true if istypeinstance(o, c) else false - return true if proto.satisfies(o.type()) else false +def type_satisfies(proto, type): + affirm(isinstance(type, Type), u"type must be a Type") + if proto.satisfies(type): + return true + elif type == Object._type: + # top level type do not recurse + return false + elif type.parent(): + return type_satisfies(proto, type.parent()) + else: + return false +@returns(bool) +@as_var("-satisfies?") +def _satisfies(proto, o): + affirm(isinstance(proto, Protocol), u"proto must be a Protocol") + return type_satisfies(proto, o.type()) import pixie.vm.rt as rt -@as_var("load-file") -def load_file(filename): +@as_var("read-string") +def _read_string(s): import pixie.vm.reader as reader - import pixie.vm.compiler as compiler + return reader.read(reader.StringReader(unicode(rt.name(s))), True) + +# XXX seems broken under jit. +@as_var("eval") +def eval(form): + from pixie.vm.compiler import compile + from pixie.vm.interpreter import interpret + val = interpret(compile(form)) + return val + +@as_var("undefined?") +def is_undefined(var): + return rt.wrap(not var.is_defined()) + +@as_var("load-ns") +def load_ns(filename): import pixie.vm.string as string import pixie.vm.symbol as symbol - import os.path as path + if isinstance(filename, symbol.Symbol): affirm(rt.namespace(filename) is None, u"load-file takes a un-namespaced symbol") - filename_str = rt.name(filename).replace(u".", u"/") + u".lisp" + filename_str = rt.name(filename).replace(u".", u"/") + u".pxi" loaded_ns = code._ns_registry.get(rt.name(filename), None) if loaded_ns is not None: return loaded_ns - else: affirm(isinstance(filename, string.String), u"Filename must be string") - filename_str = filename._str + filename_str = rt.name(filename) + + paths = rt.deref(rt.deref(rt.load_paths)) + f = None + for x in range(rt.count(paths)): + path_x = rt.nth(paths, rt.wrap(x)) + affirm(isinstance(path_x, string.String), u"Contents of load-paths must be strings") + full_path = path.join(str(rt.name(path_x)), str(filename_str)) + if path.isfile(full_path): + f = full_path + break + + if f is None: + affirm(False, u"File '" + rt.name(filename) + u"' does not exist in any directory found in load-paths") + else: + rt.load_file(rt.wrap(f)) + return nil + +PXIC_WRITER = intern_var(u"pixie.stdlib", u"*pxic-writer*") +PXIC_WRITER.set_root(nil) +PXIC_WRITER.set_dynamic() + +@as_var("load-file") +def load_file(filename): + return _load_file(filename, False) + +@as_var("compile-file") +def compile_file(filename): + return _load_file(filename, True) + +def _load_file(filename, compile=False): + from pixie.vm.string import String + from pixie.vm.util import unicode_from_utf8 + import pixie.vm.reader as reader + import pixie.vm.libs.pxic.writer as pxic_writer + + + affirm(isinstance(filename, String), u"filename must be a string") + filename = str(rt.name(filename)) + + if filename.endswith(".pxic"): + load_pxic_file(filename) + return nil - affirm(path.isfile(str(filename_str)), u"File does not exist on path") + if path.isfile(filename + "c") and not compile: + load_pxic_file(filename + "c") + return nil - f = open(str(filename_str)) + affirm(path.isfile(filename), unicode(filename) + u" does not exist") + + f = open(filename) data = f.read() f.close() - rdr = reader.StringReader(unicode(data)) + + + if data.startswith("#!"): + newline_pos = data.find("\n") + if newline_pos > 0: + data = data[newline_pos:] + + if compile: + pxic_f = open(filename + "c", "wb") + wtr = pxic_writer.Writer(pxic_f, True) + with code.bindings(PXIC_WRITER, pxic_writer.WriterBox(wtr)): + rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) + wtr.finish() + pxic_f.close() + else: + with code.bindings(PXIC_WRITER, nil): + rt.load_reader(reader.MetaDataReader(reader.StringReader(unicode_from_utf8(data)), unicode(filename))) + + + return nil + +def load_pxic_file(filename): + f = open(filename) + from pixie.vm.libs.pxic.reader import Reader, read_obj + from pixie.vm.reader import eof + import pixie.vm.compiler as compiler + + if not we_are_translated(): + print "Loading precompiled file while interpreted, this may take time" with compiler.with_ns(u"user"): + compiler.NS_VAR.deref().include_stdlib() + rdr = Reader(f) while True: + if not we_are_translated(): + sys.stdout.write(".") + sys.stdout.flush() + o = read_obj(rdr) + if o is eof: + break + o.invoke([]) + + if not we_are_translated(): + print "done" + + +@as_var("load-reader") +def load_reader(rdr): + import pixie.vm.reader as reader + import pixie.vm.compiler as compiler + + if not we_are_translated(): + print "Loading file while interpreted, this may take time" + + val = PXIC_WRITER.deref() + if val is nil: + pxic_writer = None + else: + pxic_writer = val.get_pxic_writer() + + with compiler.with_ns(u"user"): + compiler.NS_VAR.deref().include_stdlib() + while True: + if not we_are_translated(): + sys.stdout.write(".") + sys.stdout.flush() form = reader.read(rdr, False) if form is reader.eof: return nil - result = compiler.compile(form).invoke([]) + + try: + compiled = compiler.compile(form) + + except WrappedException as ex: + meta = rt.meta(form) + if meta is not nil: + ci = rt.interpreter_code_info(meta) + add_info(ex, ci.__repr__()) + add_info(ex, u"Compiling: " + rt.name(rt.str(form))) + raise ex + + try: + if pxic_writer is not None: + pxic_writer.write_object(compiled) + + compiled.invoke([]) + + except WrappedException as ex: + meta = rt.meta(form) + if meta is not nil: + ci = rt.interpreter_code_info(meta) + add_info(ex, ci.__repr__()) + add_info(ex, u"Running: " + rt.name(rt.str(form))) + raise ex + + + if not we_are_translated(): + print "done" + + return nil @as_var("the-ns") def the_ns(ns_name): @@ -262,22 +547,89 @@ def the_ns(ns_name): return code._ns_registry.get(rt.name(ns_name), nil) -@as_var("refer") +@as_var("in-ns") +def in_ns(ns_name): + from pixie.vm.compiler import NS_VAR + NS_VAR.set_value(code._ns_registry.find_or_make(rt.name(ns_name))) + NS_VAR.deref().include_stdlib() + + return nil + +@as_var("ns-map") +def ns_map(ns): + from pixie.vm.symbol import Symbol + affirm(isinstance(ns, Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") + + if isinstance(ns, Symbol): + ns = rt.the_ns(ns) + if ns is nil: + return nil + + if isinstance(ns, Namespace): + m = rt.hashmap() + for name in ns._registry: + var = ns._registry.get(name, nil) + m = rt.assoc(m, rt.symbol(rt.wrap(name)), var) + return m + + return nil + +@as_var("ns-aliases") +def ns_aliases(ns): + from pixie.vm.symbol import Symbol + affirm(isinstance(ns, Namespace) or isinstance(ns, Symbol), u"ns must be a symbol or a namespace") + + if isinstance(ns, Symbol): + ns = rt.the_ns(ns) + if ns is nil: + return nil + + if isinstance(ns, Namespace): + m = rt.hashmap() + for alias in ns._refers: + refered_ns = ns._refers[alias]._namespace + m = rt.assoc(m, rt.symbol(rt.wrap(alias)), refered_ns) + return m + + return nil + +@as_var("refer-ns") def refer(ns, refer, alias): from pixie.vm.symbol import Symbol + from pixie.vm.string import String + + if isinstance(ns, Symbol) or isinstance(ns, String): + ns = _ns_registry.find_or_make(rt.name(ns)) + + if isinstance(refer, Symbol) or isinstance(refer, String): + refer = _ns_registry.find_or_make(rt.name(refer)) affirm(isinstance(ns, code.Namespace), u"First argument must be a namespace") - affirm(isinstance(refer, code.Namespace), u"Second argument must be a namespace") + if not isinstance(refer, code.Namespace): + runtime_error(u"Second argument must be a namespace not a " + refer.type().name()) + affirm(isinstance(alias, Symbol), u"Third argument must be a symbol") ns.add_refer(refer, rt.name(alias)) return nil +@as_var("refer-symbol") +def refer_symbol(ns, sym, var): + from pixie.vm.symbol import Symbol + + affirm(isinstance(ns, Namespace), u"First argument must be a namespace") + affirm(isinstance(sym, Symbol) and rt.namespace(sym) is None, u"Second argument must be a non-namespaced symbol") + affirm(isinstance(var, Var), u"Third argument must be a var") + + ns.add_refer_symbol(sym, var) + return nil @as_var("extend") -def extend(proto_fn, tp, fn): - affirm(isinstance(proto_fn, PolymorphicFn), u"First argument to extend should be a PolymorphicFn") - affirm(isinstance(tp, Type), u"Second argument to extend must be a Type") +def _extend(proto_fn, tp, fn): + if not isinstance(proto_fn, PolymorphicFn): + runtime_error(u"Fist argument to extend should be a PolymorphicFn not a " + proto_fn.type().name()) + + affirm(isinstance(tp, Type) or isinstance(tp, Protocol), u"Second argument to extend must be a Type or Protocol") affirm(isinstance(fn, BaseCode), u"Last argument to extend must be a function") proto_fn.extend(tp, fn) return nil @@ -292,9 +644,15 @@ def satisfy(protocol, tp): @as_var("type-by-name") def type_by_name(nm): import pixie.vm.string as string - affirm(isinstance(nm, string.String), u"type name must be string") + if not isinstance(nm, string.String): + runtime_error(u"type name must be string") return _type_registry.get_by_name(nm._str, nil) +@as_var("add-marshall-handlers") +def _add_marshall_handlers(tp, write, read): + from pixie.vm.libs.pxic.util import add_marshall_handlers + add_marshall_handlers(tp, write, read) + return nil @as_var("deref") @@ -307,7 +665,11 @@ def identical(a, b): @as_var("vector?") def vector_QMARK_(a): - return true if rt.instance_QMARK_(rt.IVector.deref(), a) else false + return true if rt._satisfies_QMARK_(rt.IVector.deref(), a) else false + +@as_var("map?") +def map_QMARK_(a): + return true if rt._satisfies_QMARK_(rt.IMap.deref(), a) else false @returns(bool) @as_var("eq") @@ -323,6 +685,12 @@ def set_macro(f): f.set_macro() return f +@returns(bool) +@as_var("macro?") +def macro_QMARK_(v): + f = v.deref() if isinstance(v, Var) else v + return true if isinstance(f, BaseCode) and f.is_macro() else false + @returns(unicode) @as_var("name") def name(s): @@ -335,15 +703,18 @@ def namespace(s): @as_var("-try-catch") def _try_catch(main_fn, catch_fn, final): + from pixie.vm.keyword import keyword try: return main_fn.invoke([]) except Exception as ex: if not isinstance(ex, WrappedException): - from pixie.vm.string import String if isinstance(ex, Exception): - ex = RuntimeException(rt.wrap(u"Some error")) + if not we_are_translated(): + print "Python Error Info: ", ex.__dict__, ex + raise + ex = RuntimeException(rt.wrap(u"Internal error: " + unicode(str(ex))), keyword(u"pixie.stdlib/InternalError")) else: - ex = RuntimeException(nil) + ex = RuntimeException(u"No available message", keyword(u"pixie.stdlib/UnknownInternalError")) return catch_fn.invoke([ex]) else: return catch_fn.invoke([ex._ex]) @@ -353,13 +724,23 @@ def _try_catch(main_fn, catch_fn, final): @as_var("throw") def _throw(ex): + from pixie.vm.keyword import keyword if isinstance(ex, RuntimeException): raise WrappedException(ex) - raise WrappedException(RuntimeException(ex)) + if rt._satisfies_QMARK_(IVector, ex): + data = rt.nth(ex, rt.wrap(0)) + msg = rt.nth(ex, rt.wrap(1)) + elif rt._satisfies_QMARK_(ILookup, ex): + data = rt._val_at(ex, keyword(u"data"), nil) + msg = rt._val_at(ex, keyword(u"msg"), nil) + else: + affirm(False, u"Can only throw vectors, maps and exceptions") + return nil + raise WrappedException(RuntimeException(msg, data)) @as_var("resolve-in") def _var(ns, nm): - affirm(isinstance(ns, code.Namespace), u"First argument to resolve-in must be a namespace") + affirm(isinstance(ns, Namespace), u"First argument to resolve-in must be a namespace") var = ns.resolve(nm) return var if var is not None else nil @@ -371,7 +752,8 @@ def set_dynamic(var): @as_var("set!") def set(var, val): - affirm(isinstance(var, Var), u"Can only set the dynamic value of a var") + if not isinstance(var, Var): + runtime_error(u"Can only set the dynamic value of a var. Not a: " + var.type().name()) var.set_value(val) return var @@ -383,4 +765,186 @@ def push_binding_frame(): @as_var("pop-binding-frame!") def pop_binding_frame(): code._dynamic_vars.pop_binding_frame() - return nil \ No newline at end of file + return nil + +# @as_var("elidable-fn") +# def elidable_fn(fn): +# return code.ElidableFn(fn) + +@as_var("promote") +def promote(i): + return i.promote() + +def invoke_other(obj, args): + from pixie.vm.array import array + return rt.apply(_invoke, obj, array(args)) + +@as_var("interpreter_code_info") +def _ici(meta): + import pixie.vm.reader as reader + line = rt._val_at(meta, reader.LINE_KW, nil) + line_number = rt._val_at(meta, reader.LINE_NUMBER_KW, nil) + col_number = rt._val_at(meta, reader.COLUMN_NUMBER_KW, nil) + file = rt._val_at(meta, reader.FILE_KW, nil) + + return InterpreterCodeInfo(line, + line_number.int_val() if line_number is not nil else 0, + col_number.int_val() if col_number is not nil else 0, + rt.name(file) if file is not nil else u"") + + +# @wrap_fn +# def interpreter_code_info_reader(obj): +# line, line_number, column_number, file = obj.interpreter_code_info_state() +# return rt.vector(line, rt.wrap(line_number), rt.wrap(column_number), rt.wrap(file)) +# +# +# @wrap_fn +# def interpreter_code_info_writer(obj): +# line = rt.nth(obj, rt.wrap(0)) +# line_number = rt.nth(obj, rt.wrap(1)).int_val() +# column_number = rt.nth(obj, rt.wrap(2)).int_val() +# file = rt.name(rt.nth(obj, rt.wrap(3))) +# return InterpreterCodeInfo(line, line_number, column_number, file) +# +# from pixie.vm.libs.pxic.util import add_marshall_handlers +# add_marshall_handlers(InterpreterCodeInfo._type, interpreter_code_info_writer, interpreter_code_info_reader) + + + + + + +@wrap_fn +def merge_fn(acc, x): + return rt._assoc(acc, rt._key(x), rt._val(x)) + + +@as_var("merge") +@jit.unroll_safe +def _merge__args(args): + affirm(len(args) > 0, u"Merge takes at least one arg") + acc = args[0] + for x in range(1, len(args)): + acc = rt._reduce(args[x], merge_fn, acc) + return acc + + + +@extend(_str, RuntimeException) +def _str(self): + assert isinstance(self, RuntimeException) + return rt.wrap(self.__repr__()) + +@extend(_seq, RuntimeException) +def _seq(self): + import pixie.vm.persistent_vector as vector + import pixie.vm.persistent_hash_map as hmap + from pixie.vm.keyword import keyword + assert isinstance(self, RuntimeException) + trace = vector.EMPTY + trace_element = rt.hashmap(keyword(u"type"), keyword(u"runtime")) + trace_element = rt.assoc(trace_element, keyword(u"data"), rt.wrap(self._data)) + trace_element = rt.assoc(trace_element, keyword(u"msg"), rt.wrap(self._msg)) + trace = rt.conj(trace, trace_element) + for x in self._trace: + tmap = x.trace_map() + trace_element = hmap.EMPTY + for key in tmap: + val = tmap[key] + trace_element = rt.assoc(trace_element, key, val) + + trace = rt.conj(trace, trace_element) + + return rt._seq(trace) + +@as_var("ex-msg") +def ex_msg(e): + """Returns the message contained in an exception""" + affirm(isinstance(e, RuntimeException), u"Argument must be a RuntimeException") + assert isinstance(e, RuntimeException) + return e._msg + +@as_var("ex-data") +def ex_data(e): + """Returns the data contained in an exception""" + affirm(isinstance(e, RuntimeException), u"Argument must be a RuntimeException") + assert isinstance(e, RuntimeException) + return e._data + + +@extend(_doc, code.NativeFn._type) +def _doc(self): + assert isinstance(self, code.NativeFn) + return rt.wrap(self._doc) + + +class PartialFunction(code.NativeFn): + _immutable_fields_ = ["_partial_f", "_partial_args"] + def __init__(self, f, args): + code.NativeFn.__init__(self) + self._partial_f = f + self._partial_args = args + + @jit.unroll_safe + def invoke(self, args): + new_args = [None] * (len(args) + len(self._partial_args)) + + for x in range(len(self._partial_args)): + new_args[x] = self._partial_args[x] + + plen = len(self._partial_args) + + for x in range(len(args)): + new_args[plen + x] = args[x] + + + return self._partial_f.invoke(new_args) + + +@as_var("partial") +@jit.unroll_safe +def _partial__args(args): + """(partial f & args) + Creates a function that is a partial application of f. Thus ((partial + 1) 2) == 3""" + + f = args[0] + + new_args = [None] * (len(args) - 1) + + for x in range(len(new_args)): + new_args[x] = args[x + 1] + + return PartialFunction(f, new_args) + +@as_var("-get-current-var-frames") +def _get_current_var_frames(self): + """(-get-current-var-frames) + Returns the current var frames, will be a cons list of hash maps containing mappings of vars to dynamic values""" + return code._dynamic_vars.get_current_frames() + +@as_var("-set-current-var-frames") +def _set_current_var_frames(self, frames): + """(-set-current-var-frames frames) + Sets the current var frames. Frames should be a cons list of hashmaps containing mappings of vars to dynamic + values. Setting this value to anything but this data format will cause undefined errors.""" + code._dynamic_vars.set_current_frames(frames) + +@as_var("add-exception-info") +def _add_exception_info(ex, str, data): + affirm(isinstance(ex, RuntimeException), u"First argument must be an exception") + assert isinstance(ex, RuntimeException) + ex._trace.append(ExtraCodeInfo(rt.name(str), data)) + return ex + +@as_var("-run-finalizers") +def _run_finalizers(): + finalizer_registry.run_finalizers() + return nil + + +@as_var("floor") +def _floor(x): + affirm(isinstance(x, numbers.Float), u"floor expects a Float") + return numbers.Integer(int(x.float_val())) + diff --git a/pixie/vm/string.py b/pixie/vm/string.py index 13c050fc..17327fa9 100644 --- a/pixie/vm/string.py +++ b/pixie/vm/string.py @@ -1,18 +1,16 @@ import pixie.vm.rt as rt -from pixie.vm.object import Object, Type -from pixie.vm.code import extend, as_var -from pixie.vm.primitives import nil +from pixie.vm.object import Object, Type, affirm +from pixie.vm.code import extend, as_var, wrap_fn +from pixie.vm.primitives import nil, true, false +from pixie.vm.numbers import Integer, _add import pixie.vm.stdlib as proto -import pixie.vm.numbers as numbers import pixie.vm.util as util -from rpython.rlib.rarithmetic import intmask +from rpython.rlib.rarithmetic import intmask, r_uint +from pixie.vm.libs.pxic.util import add_marshall_handlers class String(Object): _type = Type(u"pixie.stdlib.String") - def type(self): - return String._type - def __init__(self, s): #assert isinstance(s, unicode) self._str = s @@ -24,28 +22,57 @@ def _str(x): @extend(proto._repr, String) def _repr(self): - return rt.wrap(u"\"" + self._str + u"\"") - + res = u"" + assert isinstance(self, String) + for c in self._str: + if c == "\"": + res += u"\\\"" + elif c == "\n": + res += u"\\n" + elif c == "\t": + res += u"\\t" + elif c == "\b": + res += u"\\b" + elif c == "\f": + res += u"\\f" + elif c == "\r": + res += u"\\r" + else: + res += c + return rt.wrap(u"\"" + res + u"\"") @extend(proto._count, String) def _count(self): + assert isinstance(self, String) return rt.wrap(len(self._str)) @extend(proto._nth, String) def _nth(self, idx): + assert isinstance(self, String) i = idx.int_val() if 0 <= i < len(self._str): return Character(ord(self._str[i])) - raise IndexError() + affirm(False, u"Index out of Range") +@extend(proto._nth_not_found, String) +def _nth_not_found(self, idx, not_found): + assert isinstance(self, String) + i = idx.int_val() + if 0 <= i < len(self._str): + return Character(ord(self._str[i])) + return not_found + +@extend(proto._eq, String) +def _eq(self, v): + assert isinstance(self, String) + if not isinstance(v, String): + return false + return true if self._str == v._str else false class Character(Object): _type = Type(u"pixie.stdlib.Character") _immutable_fields_ = ["_char_val"] - def type(self): - return Character._type - def __init__(self, i): assert isinstance(i, int) self._char_val = i @@ -53,15 +80,21 @@ def __init__(self, i): def char_val(self): return self._char_val +@wrap_fn +def write_char(obj): + assert isinstance(obj, Character) + return rt.wrap(obj._char_val) +@wrap_fn +def read_char(obj): + return Character(obj.int_val()) + +add_marshall_handlers(Character._type, write_char, read_char) @extend(proto._str, Character) def _str(self): assert isinstance(self, Character) - cv = self.char_val() - if cv < 128: - return rt.wrap(u"\\"+unicode(chr(cv))) - return rt.wrap(u"\\u"+unicode(str(cv))) + return rt.wrap(u"" + unichr(self.char_val())) @extend(proto._repr, Character) def _repr(self): @@ -69,7 +102,36 @@ def _repr(self): cv = self.char_val() if cv < 128: return rt.wrap(u"\\"+unicode(chr(cv))) - return rt.wrap(u"\\u"+unicode(str(cv))) + hexv = rt.name(rt.bit_str(rt.wrap(self.char_val()), rt.wrap(4))) + return rt.wrap(u"\\u" + u"0" * (4 - len(hexv)) + hexv) + +@extend(proto._eq, Character) +def _eq(self, obj): + assert isinstance(self, Character) + if self is obj: + return true + if not isinstance(obj, Character): + return false + return true if self.char_val() == obj.char_val() else false + +@extend(proto._hash, Character) +def _hash(self): + return rt.wrap(intmask(util.hash_int(r_uint(self.char_val())))) + +@as_var("char") +def char(val): + affirm(isinstance(val, Integer), u"First argument must be an Integer") + return Character(val.int_val()) + +@extend(_add, Character._type, Integer._type) +def _add(a, b): + assert isinstance(a, Character) and isinstance(b, Integer) + return rt._add(rt.wrap(a.char_val()), b) + +@extend(_add, Character._type, Character._type) +def _add(a, b): + assert isinstance(a, Character) and isinstance(b, Character) + return Character(a.char_val() + b.char_val()) @extend(proto._name, String) @@ -82,4 +144,5 @@ def _namespace(self): @extend(proto._hash, String) def _hash(self): - return rt.wrap(intmask(util.hash_unencoded_chars(self._str))) \ No newline at end of file + assert isinstance(self, String) + return rt.wrap(intmask(util.hash_unencoded_chars(self._str))) diff --git a/pixie/vm/string_builder.py b/pixie/vm/string_builder.py new file mode 100644 index 00000000..588b00a5 --- /dev/null +++ b/pixie/vm/string_builder.py @@ -0,0 +1,34 @@ +import pixie.vm.rt as rt +from pixie.vm.object import Object, Type +from pixie.vm.code import as_var, extend +import pixie.vm.stdlib as proto + +class StringBuilder(Object): + _type = Type(u"pixie.stdlib.StringBuilder") + + def __init__(self): + self._strs = [] + + def add_str(self, s): + self._strs.append(s) + return self + + def to_string(self): + return u"".join(self._strs) + + +@extend(proto._conj_BANG_, StringBuilder) +def _conj(self, val): + return self.add_str(rt.name(rt._str(val))) + +@extend(proto._persistent_BANG_, StringBuilder) +def _persistent(self): + return rt._str(self) + +@extend(proto._str, StringBuilder) +def _str(self): + return rt.wrap(self.to_string()) + +@as_var("-string-builder") +def _string_builder(): + return StringBuilder() diff --git a/pixie/vm/symbol.py b/pixie/vm/symbol.py index 8e58d39a..390805d0 100644 --- a/pixie/vm/symbol.py +++ b/pixie/vm/symbol.py @@ -1,19 +1,22 @@ import pixie.vm.object as object -from pixie.vm.object import affirm from pixie.vm.primitives import nil, true, false import pixie.vm.stdlib as proto from pixie.vm.code import extend, as_var from pixie.vm.string import String import pixie.vm.rt as rt +import pixie.vm.util as util +from rpython.rlib.rarithmetic import intmask + class Symbol(object.Object): _type = object.Type(u"pixie.stdlib.Symbol") - - def __init__(self, s): + def __init__(self, s, meta=nil): #assert isinstance(s, unicode) self._str = s self._w_name = None self._w_ns = None + self._hash = 0 + self._meta = meta def type(self): return Symbol._type @@ -31,6 +34,12 @@ def init_names(self): self._w_ns = rt.wrap(s[0]) self._w_name = rt.wrap(u"/".join(s[1:])) + def with_meta(self, meta): + return Symbol(self._str, meta) + + def meta(self): + return self._meta + def symbol(s): @@ -60,10 +69,28 @@ def _namespace(self): self.init_names() return self._w_ns +@extend(proto._hash, Symbol) +def _hash(self): + assert isinstance(self, Symbol) + if self._hash == 0: + self._hash = util.hash_unencoded_chars(self._str) + return rt.wrap(intmask(self._hash)) + @as_var("symbol") def _symbol(s): - affirm(isinstance(s, String), u"Symbol name must be a string") + if not isinstance(s, String): + from pixie.vm.object import runtime_error + runtime_error(u"Symbol name must be a string") return symbol(s._str) +@extend(proto._meta, Symbol) +def _meta(self): + assert isinstance(self, Symbol) + return self.meta() + +@extend(proto._with_meta, Symbol) +def _with_meta(self, meta): + assert isinstance(self, Symbol) + return self.with_meta(meta) diff --git a/pixie/vm/test/test_compile.py b/pixie/vm/test/test_compile.py index 40073b9b..6793e333 100644 --- a/pixie/vm/test/test_compile.py +++ b/pixie/vm/test/test_compile.py @@ -1,54 +1,53 @@ -from pixie.vm.reader import read, StringReader, eof -from pixie.vm.object import Object, Type +from pixie.vm.reader import read_inner, StringReader, eof +from pixie.vm.object import Type from pixie.vm.cons import Cons from pixie.vm.numbers import Integer -from pixie.vm.symbol import symbol, Symbol -from pixie.vm.keyword import Keyword -from pixie.vm.compiler import compile_form, compile +from pixie.vm.symbol import Symbol +from pixie.vm.compiler import compile, with_ns from pixie.vm.interpreter import interpret from pixie.vm.code import Code, Var from pixie.vm.primitives import nil, true, false -from pixie.vm.custom_types import CustomTypeInstance -import unittest - -import pixie.vm.libs.readline def read_code(s): - return read(StringReader(unicode(s)), False) + with with_ns(u"user"): + return read(StringReader(unicode(s)), False) def test_add_compilation(): - code = compile(read_code(u"(platform+ 1 2)")) - assert isinstance(code, Code) + with with_ns(u"user"): + code = compile(read_code(u"(platform+ 1 2)")) + assert isinstance(code, Code) #interpret(code) def eval_string(s): - rdr = StringReader(unicode(s)) - result = nil - while True: - form = read(rdr, False) - if form is eof: - return result + with with_ns(u"user", True): + rdr = StringReader(unicode(s)) + result = nil + while True: + form = read(rdr, False) + if form is eof: + return result - result = compile(form).invoke([]) + result = compile(form).invoke([]) def test_fn(): - code = compile(read_code("((fn [x y] (+ x y)) 1 2)")) - assert isinstance(code, Code) - retval = interpret(code) - assert isinstance(retval, Integer) and retval.int_val() == 3 + with with_ns(u"user", True): + code = compile(read_code("((fn* [x y] (-add x y)) 1 2)")) + assert isinstance(code, Code) + retval = interpret(code) + assert isinstance(retval, Integer) and retval.int_val() == 3 def test_multiarity_fn(): - retval = eval_string("""(let [v 1 - f (fn ([] v) + retval = eval_string("""(let* [v 1 + f (fn* ([] v) ([x] (+ v x)))] (f)) """) assert isinstance(retval, Integer) and retval.int_val() == 1 - retval = eval_string("""(let [v 1 - f (fn ([] v) + retval = eval_string("""(let* [v 1 + f (fn* ([] v) ([x] (+ v x)))] (f 2)) @@ -74,10 +73,10 @@ def test_if_eq(): assert eval_string("(if (platform= 1 2) true false)") is false def test_return_self(): - assert isinstance(eval_string("((fn r [] r))"), Code) + assert isinstance(eval_string("((fn* r [] r))"), Code) def test_recursive(): - retval = eval_string("""((fn rf [x] + retval = eval_string("""((fn* rf [x] (if (platform= x 10) x (recur (+ x 1)))) @@ -137,7 +136,7 @@ def test_loop(): assert retval.int_val() == 10 def test_closures(): - retval = eval_string("""((fn [x] ((fn [] x))) 42)""") + retval = eval_string("""((fn* [x] ((fn* [] x))) 42)""") assert isinstance(retval, Integer) assert retval.int_val() == 42 @@ -160,7 +159,7 @@ def test_native(): def test_build_list(): - retval = eval_string("""((fn [i lst] + retval = eval_string("""((fn* [i lst] (if (platform= i 10) (count lst) (recur (+ i 1) (cons i lst)))) 0 nil) @@ -169,7 +168,7 @@ def test_build_list(): assert isinstance(retval, Integer) and retval.int_val() == 10 def test_build_vector(): - retval = eval_string("""((fn [i lst] + retval = eval_string("""((fn* [i lst] (if (platform= i 10) (count lst) (recur (+ i 1) (conj lst i)))) 0 []) @@ -186,17 +185,17 @@ def test_build_vector(): # assert isinstance(retval, Integer) and retval.int_val() == 42 def test_let(): - retval = eval_string(""" (let [x 42] x) """) + retval = eval_string(""" (let* [x 42] x) """) assert isinstance(retval, Integer) and retval.int_val() == 42 - retval = eval_string(""" (let [x 42 y 1] (+ x y)) """) + retval = eval_string(""" (let* [x 42 y 1] (+ x y)) """) assert isinstance(retval, Integer) and retval.int_val() == 43 def test_variadic_fn(): from pixie.vm.array import Array - retval = eval_string(""" ((fn [& rest] rest) 1 2 3 4) """) + retval = eval_string(""" ((fn* [& rest] rest) 1 2 3 4) """) print retval assert isinstance(retval, Array) and len(retval._list) == 4 # diff --git a/pixie/vm/test/test_hashmaps.py b/pixie/vm/test/test_hashmaps.py index 14053462..26fb0306 100644 --- a/pixie/vm/test/test_hashmaps.py +++ b/pixie/vm/test/test_hashmaps.py @@ -1,6 +1,4 @@ -import unittest import pixie.vm.rt as rt -from pixie.vm.numbers import Integer from pixie.vm.primitives import nil rt.init() @@ -11,4 +9,4 @@ def test_hashmap_create(): val = rt._val_at(acc, rt.wrap(1), nil) - assert val.int_val() == 2 \ No newline at end of file + assert val.int_val() == 2 diff --git a/pixie/vm/test/test_reader.py b/pixie/vm/test/test_reader.py index fa67007b..8d02de43 100644 --- a/pixie/vm/test/test_reader.py +++ b/pixie/vm/test/test_reader.py @@ -1,9 +1,12 @@ -from pixie.vm.reader import read, StringReader +from pixie.vm.reader import read_inner, StringReader from pixie.vm.object import Object from pixie.vm.cons import Cons from pixie.vm.numbers import Integer from pixie.vm.symbol import symbol, Symbol +from pixie.vm.string import Character from pixie.vm.persistent_vector import PersistentVector +from pixie.vm.persistent_hash_map import PersistentHashMap +from pixie.vm.primitives import nil import pixie.vm.rt as rt import unittest @@ -15,7 +18,20 @@ u"2": 2, u"((42))": ((42,),), u"(platform+ 1 2)": (symbol(u"platform+"), 1, 2), - u"[42 43 44]": [42, 43, 44]} + u"[42 43 44]": [42, 43, 44], + u"(1 2 ; 7 8 9\n3)": (1, 2, 3,), + u"(1 2 ; 7 8 9\r\n3)": (1, 2, 3,), + u"(1 2 ; 7 8 9\r\n)": (1, 2,), + u"(1 2 ; 7 8 9\n)": (1, 2,), + u"[1 2 ; 7 8 9\n]": [1, 2,], + u"(1 2; 7 8 9\n)": (1, 2,), + u";foo\n(1 2; 7 8 9\n)": (1, 2,), + u"{\"foo\" 1 ;\"bar\" 2\n\"baz\" 3}": {"foo": 1, "baz": 3}, + u"{\"foo\" ; \"bar\" 2\n1 \"baz\" 3}": {"foo": 1, "baz": 3}, + u"(\\a)": (Character(ord("a")),), + u"(\\))": (Character(ord(")")),), + u"(\\()": (Character(ord("(")),), + u"(\\;)": (Character(ord(";")),)} class TestReader(unittest.TestCase): def _compare(self, frm, to): @@ -25,6 +41,7 @@ def _compare(self, frm, to): for x in to: self._compare(frm.first(), x) frm = frm.next() + assert frm == nil elif isinstance(to, int): assert isinstance(frm, Integer) assert frm._int_val == to @@ -36,9 +53,20 @@ def _compare(self, frm, to): elif isinstance(to, list): assert isinstance(frm, PersistentVector) - for x in range(len(to)): + for x in range(max(len(to), rt._count(frm)._int_val)): self._compare(rt.nth(frm, rt.wrap(x)), to[x]) + elif isinstance(to, dict): + assert isinstance(frm, PersistentHashMap) + for key in dict.keys(to): + self._compare(frm.val_at(rt.wrap(key), ""), to[key]) + + assert rt._count(frm)._int_val == len(dict.keys(to)) + + elif isinstance(to, Character): + assert isinstance(frm, Character) + assert to._char_val == frm._char_val + else: raise Exception("Don't know how to handle " + str(type(to))) diff --git a/pixie/vm/test/test_stacklet.py b/pixie/vm/test/test_stacklet.py deleted file mode 100644 index 0ff66442..00000000 --- a/pixie/vm/test/test_stacklet.py +++ /dev/null @@ -1,73 +0,0 @@ -import pixie.vm.stacklet as stacklet -import pixie.vm.code as code -import pixie.vm.rt as rt -import pixie.vm.numbers as numbers -from pixie.vm.primitives import nil - -class YieldingFn(code.BaseCode): - def _invoke(self, args): - assert len(args) == 2 - hdler = args[0] - arg = args[1] - - hdler.invoke([numbers.zero_int]) - hdler.invoke([numbers.one_int]) - hdler.invoke([rt.wrap(2)]) - - return - - -@code.wrap_fn -def yielding_fn(yld): - - yld.invoke([1]) - yld.invoke([2]) - yld.invoke([3]) - - return 42 - -class WrappingFn(code.NativeFn): - def __init__(self, cont): - self._cont = cont - def _invoke(self, args): - ret = args[0] - ret.invoke([self._cont.invoke([4])]) - -def test_stacklets(): - @code.wrap_fn - def main(): - cont = stacklet.new_stacklet(yielding_fn) - assert cont.invoke([None]) == 1 - assert cont.invoke([None]) == 2 - assert cont.invoke([None]) == 3 - - stacklet.with_stacklets(main) - - pass - -def test_multi_stacklets(): - @code.wrap_fn - def main(): - cont1 = stacklet.new_stacklet(yielding_fn) - cont2 = stacklet.new_stacklet(yielding_fn) - assert cont1.invoke([None]) == 1 - assert cont2.invoke([None]) == 1 - assert cont1.invoke([None]) == 2 - assert cont2.invoke([None]) == 2 - assert cont1.invoke([None]) == 3 - assert cont2.invoke([None]) == 3 - - stacklet.with_stacklets(main) - - pass - -def test_stacklets2(): - @code.wrap_fn - def main(): - cont = stacklet.new_stacklet(yielding_fn) - cont2 = stacklet.new_stacklet(WrappingFn(cont)) - cont2.invoke([44]) - - stacklet.with_stacklets(main) - - pass \ No newline at end of file diff --git a/pixie/vm/test/test_vars.py b/pixie/vm/test/test_vars.py index 902cb0f2..2f52a631 100644 --- a/pixie/vm/test/test_vars.py +++ b/pixie/vm/test/test_vars.py @@ -1,4 +1,3 @@ -import unittest from pixie.vm.code import intern_var, get_var_if_defined @@ -7,4 +6,4 @@ def test_intern(): assert intern_var(u"foo2", u"bar") is not intern_var(u"foo", u"bar") assert get_var_if_defined(u"foo", u"bar") - assert get_var_if_defined(u"foo2", u"bar") \ No newline at end of file + assert get_var_if_defined(u"foo2", u"bar") diff --git a/pixie/vm/test/test_vectors.py b/pixie/vm/test/test_vectors.py index c9456a0a..b5dc4bff 100644 --- a/pixie/vm/test/test_vectors.py +++ b/pixie/vm/test/test_vectors.py @@ -1,5 +1,4 @@ -import unittest -from pixie.vm.persistent_vector import PersistentVector, EMPTY +from pixie.vm.persistent_vector import EMPTY def test_vector_conj(): @@ -8,4 +7,4 @@ def test_vector_conj(): for x in range(1200): acc = acc.conj(x) for y in range(x): - assert acc.nth(y) == y, "Error at: " + str(x) + " and " + str(y) \ No newline at end of file + assert acc.nth(y) == y, "Error at: " + str(x) + " and " + str(y) diff --git a/pixie/vm/threads.py b/pixie/vm/threads.py new file mode 100644 index 00000000..e42d7d18 --- /dev/null +++ b/pixie/vm/threads.py @@ -0,0 +1,87 @@ +from pixie.vm.object import Object, Type, safe_invoke +from pixie.vm.primitives import true +import rpython.rlib.rthread as rthread +from pixie.vm.primitives import nil +import rpython.rlib.rgil as rgil +from pixie.vm.code import as_var +import pixie.vm.rt as rt + +class Bootstrapper(object): + def __init__(self): + self._is_inited = False + #_self.init() + + def init(self): + if not self._is_inited: + self._lock = rthread.allocate_lock() + self._is_inited = True + rgil.allocate() + + def aquire(self, fn): + self.init() + self._lock.acquire(True) + self._fn = fn + + def fn(self): + return self._fn + + def release(self): + self._fn = None + self._lock.release() + + + def _cleanup_(self): + self._lock = None + self._is_inited = False + + +def bootstrap(): + rthread.gc_thread_start() + fn = bootstrapper.fn() + bootstrapper.release() + safe_invoke(fn, []) + rthread.gc_thread_die() + +bootstrapper = Bootstrapper() + +@as_var("-thread") +def new_thread(fn): + bootstrapper.aquire(fn) + ident = rthread.start_new_thread(bootstrap, ()) + return nil + +@as_var("-yield-thread") +def yield_thread(): + rgil.yield_thread() + return nil + +# Locks + +class Lock(Object): + _type = Type(u"pixie.stdlib.Lock") + def __init__(self, ll_lock): + self._ll_lock = ll_lock + + +@as_var("-create-lock") +def _create_lock(): + return Lock(rthread.allocate_lock()) + +@as_var("-acquire-lock") +def _acquire_lock(self, no_wait): + assert isinstance(self, Lock) + return rt.wrap(self._ll_lock.acquire(no_wait == true)) + +@as_var("-acquire-lock-timed") +def _acquire_lock(self, ms): + assert isinstance(self, Lock) + return rt.wrap(self._ll_lock.acquire(ms.int_val())) + +@as_var("-release-lock") +def _release_lock(self): + assert isinstance(self, Lock) + return rt.wrap(self._ll_lock.release()) + + +# The *_external_call() functions are themselves called only from the rffi +# module from a helper function that also has this hint. diff --git a/pixie/vm/util.py b/pixie/vm/util.py index 00c1264e..cd97ed7b 100644 --- a/pixie/vm/util.py +++ b/pixie/vm/util.py @@ -1,9 +1,10 @@ -from rpython.rlib.rarithmetic import r_uint, LONG_BIT, intmask, LONG_MASK +from rpython.rlib.rarithmetic import r_uint, LONG_BIT, intmask +from rpython.rlib.runicode import str_decode_utf_8, unicode_encode_utf_8 from pixie.vm.object import affirm seed = 0 -C1 = 0xcc9e2d51 -C2 = 0x1b873593 +C1 = r_uint(0xcc9e2d51) +C2 = r_uint(0x1b873593) @@ -30,7 +31,7 @@ def mix_k1(k1): def mix_h1(h1, k1): h1 ^= k1 h1 = rotl(h1, 13) - h1 = h1 * 5 + 0xe6546b64 + h1 = h1 * 5 + r_uint(0xe6546b64) return h1 def hash_unencoded_chars(u): @@ -55,9 +56,9 @@ def hash_unencoded_chars(u): def fmix(h1, length): h1 ^= length h1 ^= h1 >> 16 - h1 *= 0x85ebca6b + h1 *= r_uint(0x85ebca6b) h1 ^= h1 >> 13 - h1 *= 0xc2b2ae35 + h1 *= r_uint(0xc2b2ae35) h1 ^= h1 >> 16 return h1 @@ -71,17 +72,12 @@ def mix_coll_hash(hash, count): from pixie.vm.object import Object, Type -import pixie.vm.code as code from pixie.vm.code import as_var import pixie.vm.rt as rt -import pixie.vm.numbers as numbers class HashingState(Object): _type = Type(u"pixie.stdlib.HashingState") - def type(self): - return HashingState._type - def __init__(self): self._n = r_uint(0) self._hash = r_uint(1) @@ -123,3 +119,14 @@ def finish_hash_state(acc): def _hash_int(acc): return rt.wrap(intmask(hash_int(acc.r_uint_val()))) + +# unicode utils + +def unicode_from_utf8(s): + """Converts a `str` value to a `unicode` value assuming it's encoded in UTF8.""" + res, _ = str_decode_utf_8(s, len(s), 'strict') + return res + +def unicode_to_utf8(s): + """Converts a `unicode` value to a UTF8 encoded `str` value.""" + return unicode_encode_utf_8(s, len(s), 'strict') diff --git a/pixie/walk.pxi b/pixie/walk.pxi new file mode 100644 index 00000000..cef60c93 --- /dev/null +++ b/pixie/walk.pxi @@ -0,0 +1,113 @@ +(ns pixie.walk) + +(defprotocol IWalk + (-walk [x f])) + +(extend-protocol IWalk + PersistentList + (-walk [x f] + (apply list (map f x))) + + Cons + (-walk [x f] + (cons (f (first x)) (map f (next x)))) + + IMapEntry + (-walk [x f] + (map-entry (f (key x)) (f (val x)))) + + PersistentVector + (-walk [x f] + (into [] (map f) x)) + + PersistentHashSet + (-walk [x f] + (into #{} (map f) x)) + + PersistentHashMap + (-walk [x f] + (into {} (map f) x)) + + IRecord + (-walk [x f] + (into x (map f) x)) + + ISeqable + (-walk [x f] + (map f x)) + + IObject + (-walk [x f] x) + + Nil + (-walk [x f] nil)) + +(defn walk + {:doc "Traverses form, an arbitrary data structure. f is a + function. Applies f to each element of form, building up a data + structure of the same type. Recognizes all Pixie data + structures. Consumes seqs." + :signatures [[f x]] + :added "0.1"} + [f x] + (-walk x f)) + +(defn postwalk + {:doc "Performs a depth-first, post-order traversal of form. Calls f on + each sub-form, uses f's return value in place of the original. + Recognizes all Pixie data structures." + :signatures [[f x]] + :added "0.1"} + [f x] + (f (walk (partial postwalk f) x))) + +(defn prewalk + {:doc "Like postwalk, but does pre-order traversal." + :added "0.1"} + [f x] + (walk (partial prewalk f) (f x))) + +(defn prewalk-replace + {:doc "Recursively transforms form by replacing + keys in smap with their values. Like `replace` but works on + any data structure. Does replacement at the root of the tree + first." + :signatures [[f x]] + :added "0.1"} + [smap x] + (prewalk (fn [x] (if (contains? smap x) (smap x) x)) x)) + +(defn postwalk-replace + {:doc "Recursively transforms form by replacing keys in smap with + their values. Like `replace` but works on any data structure. + Does replacement at the leaves of the tree first." + :signatures [[smap x]] + :added "0.1"} + [smap x] + (postwalk (fn [x] (if (contains? smap x) (smap x) x)) x)) + +(defn keywordize-keys + {:doc "Recursively transforms all map keys from strings to keywords." + :signatures [[m]] + :added "0.1"} + [m] + (let [f (fn [[k v]] (if (string? k) [(keyword k) v] [k v]))] + ;; only apply to maps + (postwalk (fn [x] (if (map? x) (into {} (map f x)) x)) m))) + +(defn stringify-keys + {:doc "Recursively transforms all map keys from keywords to strings." + :signatures [[m]] + :added "0.1"} + [m] + (let [f (fn [[k v]] (if (keyword? k) [(name k) v] [k v]))] + ;; only apply to maps + (postwalk (fn [x] (if (map? x) (into {} (map f x)) x)) m))) + +(defn macroexpand-all + {:doc "Recursively performs all possible macroexpansions in + form. For development use only." + :added "0.1" + :signatures [[x]]} + [x] + (prewalk (fn [x] (if (seq? x) (macroexpand-1 x) x)) x)) diff --git a/run-tests.pxi b/run-tests.pxi new file mode 100644 index 00000000..d2c373aa --- /dev/null +++ b/run-tests.pxi @@ -0,0 +1,15 @@ +(require pixie.test :as t) + + +(swap! load-paths conj "./tests/") + +(println @load-paths) + +(if (= 0 (count program-arguments)) + (t/load-all-tests) + (doseq [nm program-arguments] + (println "Loading: " nm) + (load-ns (symbol nm)))) + +(let [result (apply t/run-tests program-arguments)] + (exit (get result :fail))) diff --git a/target.py b/target.py index ea0f70ba..71be1c7d 100644 --- a/target.py +++ b/target.py @@ -1,20 +1,28 @@ -from pixie.vm.compiler import compile, with_ns, NS_VAR -from pixie.vm.reader import StringReader, read, eof, PromptReader, MetaDataReader -from pixie.vm.interpreter import interpret +from pixie.vm.compiler import with_ns, NS_VAR +from pixie.vm.reader import StringReader from rpython.jit.codewriter.policy import JitPolicy from rpython.rlib.jit import JitHookInterface, Counters +from rpython.rlib.rfile import create_stdio from rpython.annotator.policy import AnnotatorPolicy -from pixie.vm.code import wrap_fn -from pixie.vm.stacklet import with_stacklets -import pixie.vm.stacklet as stacklet -from pixie.vm.object import RuntimeException, WrappedException - +from pixie.vm.code import wrap_fn, NativeFn, intern_var, Var +from pixie.vm.object import WrappedException +from rpython.translator.platform import platform +from pixie.vm.primitives import nil +from pixie.vm.atom import Atom +from pixie.vm.persistent_vector import EMPTY as EMPTY_VECTOR +from pixie.vm.util import unicode_from_utf8, unicode_to_utf8 import sys - +import os +import os.path as path +import rpython.rlib.rpath as rpath +import rpython.rlib.rpath as rposix +from rpython.rlib.objectmodel import we_are_translated +from rpython.jit.codewriter.policy import log class DebugIFace(JitHookInterface): def on_abort(self, reason, jitdriver, greenkey, greenkey_repr, logops, operations): - print "Aborted Trace, reason: ", Counters.counter_names[reason], logops, greenkey_repr + # print "Aborted Trace, reason: ", Counters.counter_names[reason], logops, greenkey_repr + pass import sys, pdb @@ -22,72 +30,244 @@ class Policy(JitPolicy, AnnotatorPolicy): def __init__(self): JitPolicy.__init__(self, DebugIFace()) - #def event(pol, bookkeeper, what, x): - # pass - def jitpolicy(driver): return JitPolicy(jithookiface=DebugIFace()) -@wrap_fn -def repl(): - from pixie.vm.keyword import keyword - import pixie.vm.rt as rt - from pixie.vm.string import String +PROGRAM_ARGUMENTS = intern_var(u"pixie.stdlib", u"program-arguments") +PROGRAM_ARGUMENTS.set_root(nil) + +LOAD_PATHS = intern_var(u"pixie.stdlib", u"load-paths") +LOAD_PATHS.set_root(nil) +load_path = intern_var(u"pixie.stdlib", u"internal-load-path") + +class ReplFn(NativeFn): + def __init__(self, args): + self._argv = args + + def inner_invoke(self, args): + import pixie.vm.rt as rt + from pixie.vm.code import intern_var + rt.load_ns(rt.wrap(u"pixie/repl.pxi")) + + repl = intern_var(u"pixie.repl", u"repl") + with with_ns(u"user"): + repl.invoke([]) + +load_file = intern_var(u"pixie.stdlib", u"load-file") + +class BatchModeFn(NativeFn): + def __init__(self, args): + self._file = args[0] + self._argv = args[1:] - with with_ns(u"user"): - NS_VAR.deref().include_stdlib() + def inner_invoke(self, args): + import pixie.vm.rt as rt + import pixie.vm.persistent_vector as vector - rdr = MetaDataReader(PromptReader()) - with with_ns(u"user"): - while True: + with with_ns(u"user"): + NS_VAR.deref().include_stdlib() + + acc = vector.EMPTY + for x in self._argv: + acc = rt.conj(acc, rt.wrap(x)) + + PROGRAM_ARGUMENTS.set_root(acc) + + with with_ns(u"user"): try: - val = read(rdr, False) - if val is eof: - break - val = interpret(compile(val)) + f = None + if self._file == '-': + f, _, _ = create_stdio() + else: + if not path.isfile(self._file): + print "Error: Cannot open '" + self._file + "'" + os._exit(1) + load_file.invoke([rt.wrap(self._file)]) + return None + + data = f.read() + f.close() + + if data.startswith("#!"): + newline_pos = data.find("\n") + if newline_pos > 0: + data = data[newline_pos:] + + rt.load_reader(StringReader(unicode_from_utf8(data))) except WrappedException as ex: print "Error: ", ex._ex.__repr__() - continue - if val is keyword(u"exit-repl"): + os._exit(1) + +class EvalFn(NativeFn): + def __init__(self, expr): + self._expr = expr + + def inner_invoke(self, args): + import pixie.vm.rt as rt + + with with_ns(u"user"): + NS_VAR.deref().include_stdlib() + + rt.load_reader(StringReader(unicode_from_utf8(self._expr))) + +class CompileFileFn(NativeFn): + def __init__(self, filename): + self._filename = filename + + def inner_invoke(self, args): + import pixie.vm.rt as rt + try: + rt.compile_file(rt.wrap(self._filename)) + except WrappedException as ex: + print "Error: ", ex._ex.__repr__() + os._exit(1) + + +class IsPreloadFlag(object): + def __init__(self): + self._is_true = False + + def is_true(self): + return self._is_true + + def set_true(self): + self._is_true = True + + +stdlib_loaded = IsPreloadFlag() + +@wrap_fn +def run_load_stdlib(): + global stdlib_loaded + if stdlib_loaded.is_true(): + return + + rt.load_ns(rt.wrap(u"pixie/stdlib.pxi")) + rt.load_ns(rt.wrap(u"pixie/stacklets.pxi")) + + stdlib_loaded.set_true() + +def load_stdlib(): + run_load_stdlib.invoke([]) + +from pixie.vm.code import intern_var +run_with_stacklets = intern_var(u"pixie.stacklets", u"run-with-stacklets") + +def init_vm(progname): + import pixie.vm.stacklet + pixie.vm.stacklet.init() + + init_load_path(progname) + load_stdlib() + add_to_load_paths(".") + +def entry_point(args): + try: + + init_vm(args[0]) + + interactive = True + exit = False + script_args = [] + + i = 1 + while i < len(args): + arg = args[i] + + if arg.startswith('-') and arg != '-': + if arg == '-v' or arg == '--version': + print "Pixie 0.1" + return 0 + elif arg == '-h' or arg == '--help': + print args[0] + " [] []" + print " -h, --help print this help" + print " -v, --version print the version number" + print " -e, --eval= evaluate the given expression" + print " -l, --load-path= add to pixie.stdlib/load-paths" + print " -c, --compile= compile to a .pxic file" + return 0 + elif arg == '-e' or arg == '--eval': + i += 1 + if i < len(args): + expr = args[i] + run_with_stacklets.invoke([EvalFn(expr)]) + return 0 + else: + print "Expected argument for " + arg + return 1 + elif arg == '-l' or arg == '--load-path': + i += 1 + if i < len(args): + path = args[i] + add_to_load_paths(path) + else: + print "Expected argument for " + arg + return 1 + + elif arg == "-c" or arg == "--compile": + i += 1 + if i < len(args): + path = args[i] + print "Compiling ", path + run_with_stacklets.invoke([CompileFileFn(path)]) + exit = True + else: + print "Expected argument for " + arg + return 1 + else: + print "Unknown option " + arg + return 1 + else: + interactive = False + script_args = args[i:] break - val = rt.str(val) - assert isinstance(val, String), "str should always return a string" - print val._str - -def entry_point(foo=None): - print "Pixie 0.1 - Interactive REPL" - #try: - # code = argv[1] - #except IndexError: - # print "must provide a program" - # return 1 - # rdr = StreamReader(sys.stdin) - # while True: - # #val = read(rdr, True) - # #if val is eof: - # # break - # val = read(StringReader(raw_input("user>")), True) - # print interpret(compile(val)) - # interpret(compile(read(StringReader(""" - # (do (def foo (fn [h v] (h 42))) - # ((create-stacklet foo) 0)) - # """), True))) - #rt.load_file(String(u"pixie/stdlib.lisp")) - - - - with_stacklets(repl) + + i += 1 + + if not exit: + if interactive: + run_with_stacklets.invoke([ReplFn(args)]) + else: + run_with_stacklets.invoke([BatchModeFn(script_args)]) + except WrappedException as we: + print we._ex.__repr__() return 0 +def add_to_load_paths(path): + rt.reset_BANG_(LOAD_PATHS.deref(), rt.conj(rt.deref(LOAD_PATHS.deref()), rt.wrap(path))) + + +def init_load_path(self_path): + if not path.isfile(self_path): + self_path = find_in_path(self_path) + assert self_path is not None + if path.islink(self_path): + self_path = os.readlink(self_path) + self_path = dirname(rpath.rabspath(self_path)) + + # runtime is not loaded yet, so we have to do it manually + LOAD_PATHS.set_root(Atom(EMPTY_VECTOR.conj(rt.wrap(self_path)))) + # just for run_load_stdlib (global variables can't be assigned to) + load_path.set_root(rt.wrap(self_path)) + +def dirname(path): + return rpath.sep.join(path.split(rpath.sep)[0:-1]) +def find_in_path(exe_name): + paths = os.environ.get('PATH') + paths = paths.split(path.pathsep) + for p in paths: + exe_path = path.join(p, exe_name) + if path.isfile(exe_path): + return exe_path + return None from rpython.rtyper.lltypesystem import lltype from rpython.jit.metainterp import warmspot def run_child(glob, loc): - import sys, pdb interp = loc['interp'] graph = loc['graph'] interp.malloc_check = False @@ -132,7 +312,7 @@ def run_debug(argv): import pixie.vm.rt as rt rt.init() -stacklet.global_state = stacklet.GlobalState() +#stacklet.global_state = stacklet.GlobalState() def target(*args): import pixie.vm.rt as rt @@ -141,6 +321,7 @@ def target(*args): rt.__config__ = args[0].config + print "ARG INFO: ", args return entry_point, None @@ -149,6 +330,4 @@ def target(*args): print rpython.config.translationoption.get_combined_translation_config() if __name__ == "__main__": - #run_debug(sys.argv) - #with_stacklets(bootstrap) - entry_point() \ No newline at end of file + entry_point(sys.argv) diff --git a/tests/pixie/tests/collections/test-maps.pxi b/tests/pixie/tests/collections/test-maps.pxi new file mode 100644 index 00000000..246e4af5 --- /dev/null +++ b/tests/pixie/tests/collections/test-maps.pxi @@ -0,0 +1,56 @@ +(ns collections.test-maps + (require pixie.test :as t)) + +(t/deftest maps-contains + (let [m {:a 1, :b 2, :c 3} + c [:a :b :c] + n [:d 'a 1]] + (foreach [c c] + (t/assert= (contains? m c) true)) + (foreach [n n] + (t/assert= (contains? m c) false)))) + +(t/deftest map-equals + (let [m {:a 1, :b 2, :c 3}] + (t/assert= (-eq {} {}) true) + (t/assert= (-eq m m) true) + (t/assert= (-eq m {:a 1, :b 2, :c 3}) true) + + (t/assert= (-eq m {}) false) + (t/assert= (-eq m nil) false) + + (t/assert= (-eq m {:a 1, :b 2}) false) + (t/assert= (-eq m [[:a 1] [:b 2] [:c 3]]) false) + + (t/assert= (-eq m {:a 1, :b 2, :c 4}) false) + (t/assert= (-eq m {:a 3, :b 2, :c 1}) false))) + +(t/deftest map-val-at-and-invoke + (let [m {:a 1, :b 2, :c 3}] + (foreach [e m] + (t/assert= (get m (key e)) (val e)) + (t/assert= (m (key e)) (val e))) + (t/assert= (get m :d) nil) + (t/assert= (m :d) nil) + (t/assert= (m :d :foo) :foo) + (t/assert= (:d m :foo) :foo))) + +(t/deftest map-without + (let [m {:a 1 :b 2}] + (t/assert= m m) + (t/assert= (dissoc m :a) {:b 2}) + (t/assert= (dissoc m :a :b) {}))) + +(t/deftest map-conj + (let [m {:a 1 :b 2}] + (t/assert= m m) + ;; Should conj vector of length 2 + (t/assert= (conj m [:c 3]) {:a 1 :b 2 :c 3}) + (t/assert= (conj m [:b 4]) {:a 1 :b 4}) + (t/assert= (conj m [:b 4] [:c 5]) {:a 1 :b 4 :c 5}) + + ;; Should conj sequences of pairs + (t/assert= (conj {} '([:a 1] [:b 2] [:c 3])) {:a 1 :b 2 :c 3}) + + ;; Should conj sequences of MapEntries + (t/assert= (conj {} (seq {:a 1 :b 2 :c 3})) {:a 1 :b 2 :c 3}))) diff --git a/tests/pixie/tests/collections/test-seqables.pxi b/tests/pixie/tests/collections/test-seqables.pxi new file mode 100644 index 00000000..dcf79354 --- /dev/null +++ b/tests/pixie/tests/collections/test-seqables.pxi @@ -0,0 +1,45 @@ +(ns collections.test-seqables + (require pixie.test :as t)) + +(t/deftest test-seq + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (seq l) '(1 2 3)) + (t/assert= (seq v) [1 2 3]) + + (t/assert= (seq nil) nil) + (t/assert= (seq []) nil))) + +(t/deftest test-first + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (first l) 1) + (t/assert= (first v) 1) + (t/assert= (first (seq l)) 1) + (t/assert= (first (seq v)) 1))) + +(t/deftest test-next + (let [l '(1 2 3) + v [1 2 3]] + (t/assert= (next l) '(2 3)) + (t/assert= (next v) '(2 3)) + (t/assert= (next (seq l)) '(2 3)) + (t/assert= (next (seq v)) '(2 3)) + + (t/assert= (next (next (next l))) nil) + (t/assert= (next (next (next v))) nil))) + +(t/deftest test-equals + (let [l '(1 2 3)] + (t/assert= l l) + (t/assert= l '(1 2 3)) + (t/assert= l [1 2 3]) + + (t/assert= (= nil '()) false) + (t/assert= (= '() nil) false) + (t/assert= (= l '(1 2 3 4)) false) + (t/assert= (= l [1 2 3 4]) false))) + +(t/deftest test-conj + (t/assert= '(3 1 2) (conj '(1 2) 3)) + (t/assert= '(5 4 3 1 2) (conj '(1 2) 3 4 5))) diff --git a/tests/pixie/tests/collections/test-sets.pxi b/tests/pixie/tests/collections/test-sets.pxi new file mode 100644 index 00000000..7b91c342 --- /dev/null +++ b/tests/pixie/tests/collections/test-sets.pxi @@ -0,0 +1,73 @@ +(ns collections.test-sets + (require pixie.test :as t) + (require pixie.tests.utils :as u)) + +(def worst-hashers (vec (map u/->WorstHasher) + (range 100))) + +(t/deftest test-count + (t/assert= (count (set [])) 0) + (t/assert= (count (set [1 2 3])) 3) + (t/assert= (count (set [1 1 2 1])) 2) + (t/assert= (count (set worst-hashers)) 100)) + +(t/deftest test-contains + (let [s #{1 2 3} + c [1 2 3] + n [-1 0 4] + g (set worst-hashers)] + (foreach [c c] + (t/assert= (contains? s c) true)) + (foreach [n n] + (t/assert= (contains? s n) false)) + (foreach [n worst-hashers] + (t/assert= (contains? g n) true)))) + +(t/deftest test-conj + (t/assert= (conj #{}) #{}) + (t/assert= (conj #{1 2} 3) #{1 2 3}) + (t/assert= (reduce conj #{} (range 10)) (set (vec (range 10)))) + (t/assert= (reduce conj #{} worst-hashers) (set worst-hashers))) + +(t/deftest test-disj + (t/assert= (disj #{}) #{}) + (t/assert= (disj #{1 2} 3) #{1 2}) + (t/assert= (disj #{1 2} 2) #{1}) + (t/assert= (reduce disj (set (vec (range 10))) (range 10)) #{}) + (t/assert= (reduce disj (set worst-hashers) worst-hashers) #{})) + +(t/deftest test-eq + (let [s #{1 2 3}] + (t/assert= s s) + (t/assert= s #{1 2 3}) + (t/assert= #{1 2 3} s) + + (t/assert= (= #{} nil) false) + (t/assert= (= #{} []) false) + (t/assert= (= #{} '()) false) + + (t/assert= (= s [1 2 3]) false) + (t/assert= (= s '(1 2 3)) false) + (t/assert= (= s #{1 2}) false) + (t/assert= (= s #{1 2 3 4}) false))) + +(t/deftest test-invoke + (let [s #{1 2 3}] + (t/assert= (s 1) 1) + (t/assert= (s 2) 2) + (t/assert= (s 3) 3) + + (t/assert= (s -1) nil) + (t/assert= (s 4) nil) + (t/assert= (s :d :foo) :foo) + (t/assert= (:d s :foo) :foo))) + +(t/deftest test-has-meta + (let [m {:has-meta true} + s (with-meta #{} m)] + (t/assert= (meta #{}) nil) + (t/assert= (meta s) m))) + +(t/deftest test-conj + (t/assert= #{1 2} (conj #{1} 2)) + (t/assert= #{1 2 3 4} (conj #{1} 2 3 4))) diff --git a/tests/pixie/tests/collections/test-vectors.pxi b/tests/pixie/tests/collections/test-vectors.pxi new file mode 100644 index 00000000..e6e6f26a --- /dev/null +++ b/tests/pixie/tests/collections/test-vectors.pxi @@ -0,0 +1,77 @@ +(ns pixie.tests.collections.test-vectors + (require pixie.test :as t)) + +(def MAX-SIZE 1064) + +(comment +;; Takes forever in interpreted mode but useful for debugging +(t/deftest vector-creation + (loop [acc []] + (if (= (count acc) MAX-SIZE) + acc + (do (dotimes [j (count acc)] + (t/assert= j (nth acc j))) + (recur (conj acc (count acc))))))) +) +(t/deftest vector-contains + (let [v [1 2 3] + c [0 1 2] + n [-1 3]] + (foreach [c c] + (t/assert= (contains? v c) true)) + (foreach [n n] + (t/assert= (contains? v n) false)))) + +(t/deftest vector-equals + (let [v [1 2 3]] + (t/assert= [] '()) + (t/assert= v v) + (t/assert= v [1 2 3]) + (t/assert= v '(1 2 3)) + + (t/assert= (= [] nil) false) + (t/assert= (= v []) false) + (t/assert= (= v [1 2]) false) + (t/assert= (= v [1 2 3 4]) false) + (t/assert= (= v '(1 2)) false) + (t/assert= (= v '(1 2 3 4)) false))) + +(t/deftest vector-conj + (t/assert= [1 2] (conj [1] 2)) + (t/assert= [1 2 3 4] (conj [1] 2 3 4))) + +(t/deftest vector-conj! + (t/assert= [1 2] (persistent! (conj! (transient [1]) 2))) + (t/assert= [1 2 3] (persistent! (conj! (transient [1]) 2 3)))) + +(t/deftest vector-push + (t/assert= [1] (push [] 1)) + (t/assert= [1 2] (push [1] 2)) + (t/assert= [0 1 2 3 4] (reduce push [] (range 5)))) + +(t/deftest vector-pop + (t/assert= [] (pop [1])) + (t/assert= [1] (pop [1 2])) + (t/assert= [1 2] (pop [1 2 3])) + (try (pop []) + (catch e + (t/assert= "Can't pop an empty vector" (ex-msg e))))) + +(t/deftest vector-push! + (t/assert= [1] (persistent! (push! (transient []) 1))) + (t/assert= [1 2] (persistent! (push! (transient [1]) 2)))) + +(t/deftest vector-pop! + (t/assert= [] (persistent! (pop! (transient [1])))) + (t/assert= [1] (persistent! (pop! (transient [1 2])))) + (t/assert= [1 2] (persistent! (pop! (transient [1 2 3])))) + (try (persistent! (pop! (transient []))) + (catch e + (t/assert= "Can't pop an empty vector" (ex-msg e))))) + +(t/deftest transient-vector-count + (t/assert= 0 (count (transient []))) + (t/assert= 1 (count (transient [1]))) + (t/assert= 2 (count (transient [1 2]))) + (t/assert= 1 (count (pop! (transient [1 2])))) + (t/assert= 100 (count (reduce conj! (transient []) (range 0 100))))) diff --git a/tests/pixie/tests/data/test-json.pxi b/tests/pixie/tests/data/test-json.pxi new file mode 100644 index 00000000..098ad019 --- /dev/null +++ b/tests/pixie/tests/data/test-json.pxi @@ -0,0 +1,76 @@ +(ns pixie.tests.data.test-json + (:require [pixie.test :refer :all] + [pixie.data.json :as json])) + + + +;; This test just ensures that we can use read-string; more thorough testing is +;; done in pixie.tests.parser.test-json +(deftest test-read-string + (assert= (json/read-string "{\"foo\": 42, \"bar\": [\"baz\"]}") + {"foo" 42, "bar" ["baz"]})) + +(deftest test-strings + (assert-table [x y] (assert= (json/write-string x) y) + \a "\"a\"" + "foo" "\"foo\"" + :bar "\"bar\"" + "" "\"\"")) + +(deftest test-numbers + (assert-table [x y] (assert= (json/write-string x) y) + 1 "1" + 1.0 "1.000000" + 0.1 "0.100000" + 1.1 "1.100000" + 1234.5678 "1234.567800" + -1 "-1" + -0.1 "-0.100000" + -1.1 "-1.100000" + -1234.5678 "-1234.567800" + 1e1 "10.000000" + 3/1 "3")) + +(deftest test-vectors + (assert-table [x y] (assert= (json/write-string x) y) + [] "[]" + [nil] "[null]" + [1 2] "[1, 2]" + [1 1.0 nil] "[1, 1.000000, null]" + ["foo" 42] "[\"foo\", 42]")) + +(deftest test-seqs + (assert-table [x y] (assert= (json/write-string x) y) + (take 0 (range)) "[]" + (take 1 (repeat nil)) "[null]" + (map identity [1 2]) "[1, 2]" + (reduce + [1 2 3]) "6")) + +(deftest test-lists + (assert-table [x y] (assert= (json/write-string x) y) + '() "[]" + '(nil) "[null]" + '(1 2) "[1, 2]" + '(1 1.0 nil) "[1, 1.000000, null]" + '("foo" 42) "[\"foo\", 42]" + (list) "[]" + (list nil) "[null]" + (list 1 2) "[1, 2]" + (list 1 1.0 nil) "[1, 1.000000, null]" + (list "foo" 42) "[\"foo\", 42]")) + +(deftest test-maps + (assert-table [x y] (assert= (json/write-string x) y) + {} "{}" + {:foo 42} "{\"foo\": 42}" + {"foo" 42} "{\"foo\": 42}" + {"foo" 42, "bar" nil} "{\"foo\": 42, \"bar\": null}")) + +(deftest test-round-trip + (let [data {"foo" 1, "bar" [2 3]}] ; won't work with keywords because the parser doesn't keywordise them (yet) + (assert= (-> data json/write-string json/read-string) + data)) + + (let [string "{\"foo\": [1, 2, 3]}"] + (assert= (-> string json/read-string json/write-string) + string))) diff --git a/tests/pixie/tests/fs/parent/bar.txt b/tests/pixie/tests/fs/parent/bar.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/pixie/tests/fs/parent/child/bar.txt b/tests/pixie/tests/fs/parent/child/bar.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/pixie/tests/fs/parent/child/foo.txt b/tests/pixie/tests/fs/parent/child/foo.txt new file mode 100644 index 00000000..e69de29b diff --git a/tests/pixie/tests/fs/parent/foo.txt b/tests/pixie/tests/fs/parent/foo.txt new file mode 100644 index 00000000..257cc564 --- /dev/null +++ b/tests/pixie/tests/fs/parent/foo.txt @@ -0,0 +1 @@ +foo diff --git a/tests/pixie/tests/io/test-tcp.pxi b/tests/pixie/tests/io/test-tcp.pxi new file mode 100644 index 00000000..a86439a2 --- /dev/null +++ b/tests/pixie/tests/io/test-tcp.pxi @@ -0,0 +1,40 @@ +(ns pixie.test.io.test-tcp + (:require [pixie.io.tcp :refer :all] + [pixie.io :refer [buffered-input-stream buffered-output-stream read-byte write-byte]] + [pixie.streams :refer :all] + [pixie.stacklets :as st] + [pixie.async :as async] + [pixie.uv :as uv] + [pixie.test :refer :all])) + +(deftest test-echo-server + (let [client-done (async/promise) + on-client (fn on-client [conn] + (let [in (buffered-input-stream conn) + out (buffered-output-stream conn)] + (try + (loop [] + (let [val (read-byte in)] + (write-byte out val) + (flush out) + (recur))) + (catch ex + (dispose! in) + (dispose! out) + + (dispose! conn) + (client-done true))))) + + server (tcp-server "0.0.0.0" 4242 on-client)] + + (let [client-stream (tcp-client "127.0.0.1" 4242) + in (buffered-input-stream client-stream) + out (buffered-output-stream client-stream)] + + (dotimes [x 255] + (write-byte out x) + (flush out) + (assert= x (read-byte in))) + (dispose! client-stream) + (assert @client-done) + (dispose! server)))) diff --git a/tests/pixie/tests/parser/test-json.pxi b/tests/pixie/tests/parser/test-json.pxi new file mode 100644 index 00000000..e79ef6cd --- /dev/null +++ b/tests/pixie/tests/parser/test-json.pxi @@ -0,0 +1,33 @@ +(ns pixie.tests.parser.test-json + (:require [pixie.test :refer :all] + [pixie.parser.json :as json])) + + + +(deftest test-json-numbers + (assert-table [x y] (assert= (json/read-string x) y) + "1" 1 + "1.0" 1.0 + "0.1" 0.1 + "1.1" 1.1 + "1234.5678" 1234.5678 + + "-1" -1 + "-0.1" -0.1 + "-1.1" -1.1 + "-1234.5678" -1234.5678 + "1e1" 1e1)) + +(deftest test-vectors + (assert-table [x y] (assert= (json/read-string x) y) + "[]" [] + "[null]" [nil] + "[1, 2]" [1 2] + "[1, 1.0, null]" [1 1.0 nil] + "[\"foo\", 42]" ["foo" 42])) + +(deftest test-maps + (assert-table [x y] (assert= (json/read-string x) y) + "{\"foo\": 42}" {"foo", 42} + "{\"foo\": 42, \"bar\":null}" {"foo" 42 + "bar" nil})) diff --git a/tests/pixie/tests/streams/test-utf8.pxi b/tests/pixie/tests/streams/test-utf8.pxi new file mode 100644 index 00000000..ede7a1c5 --- /dev/null +++ b/tests/pixie/tests/streams/test-utf8.pxi @@ -0,0 +1,17 @@ +(ns pixie.streams.test-utf8 + (require pixie.streams.utf8 :refer :all) + (require pixie.io :as io) + (require pixie.test :refer :all)) + + +(deftest test-writing-ints + (using [os (-> (io/open-write "/tmp/pixie-utf-test.txt") + (io/buffered-output-stream 1024) + utf8-output-stream)] + (dotimes [x 32000] + (write-char os (char x)))) + (using [is (-> (io/open-read "/tmp/pixie-utf-test.txt") + (io/buffered-input-stream 1024) + utf8-input-stream)] + (dotimes [x 32000] + (assert= x (int (read-char is)))))) diff --git a/tests/pixie/tests/test-arrays.pxi b/tests/pixie/tests/test-arrays.pxi new file mode 100644 index 00000000..ecd651cb --- /dev/null +++ b/tests/pixie/tests/test-arrays.pxi @@ -0,0 +1,78 @@ +(ns pixie.test.test-arrays + (require pixie.test :as t)) + +(t/deftest test-array-creation + (let [a (make-array 10)] + (t/assert= (count a) 10) + (t/assert= (alength a) 10) + (foreach [x a] + (t/assert= x nil)))) + +(t/deftest test-alength + (let [a (make-array 10)] + (t/assert= (alength a) 10) + (t/assert-throws? RuntimeException + (alength [])))) + +(t/deftest test-aget-and-aset + (let [a (make-array 10)] + (dotimes [i 10] + (t/assert= (aget a i) nil)) + + (dotimes [i 10] + (aset a i i)) + + (dotimes [i 10] + (t/assert= (aget a i) i)) + + (t/assert-throws? RuntimeException + (aget a 1.0)) + + (t/assert-throws? RuntimeException + (aset a 1.0 :foo)))) + +(t/deftest test-aconcat + (let [a1 (make-array 10) + a2 (make-array 10)] + (t/assert= (alength (aconcat a1 a2)) (+ (alength a1) (alength a2))) + + (dotimes [i 10] + (aset a1 i i) + (aset a2 i (+ 10 i))) + + (let [a3 (aconcat a1 a2)] + (dotimes [i 20] + (t/assert= (aget a3 i) i))) + + (t/assert-throws? RuntimeException + (t/aconcat a1 [])) + + (t/assert-throws? RuntimeException + (t/aconcat a1 '())))) + +(t/deftest test-aslice + (let [a (make-array 10)] + (dotimes [i 10] + (aset a i i)) + + (let [a1 (aslice a 3) + a2 (aslice a 7)] + (foreach [i (range 0 7)] + (t/assert= (aget a1 i) (+ i 3))) + (foreach [i (range 0 3)] + (t/assert= (aget a2 i) (+ i 7)))) + + (t/assert-throws? RuntimeException + (aslice [1 2 3 4] 0 2)) + + (t/assert-throws? RuntimeException + (aslice '() 0 2)) + + (t/assert-throws? RuntimeException + (aslice a 1.0 2)))) + + +(t/deftest test-byte-array-creation + (let [ba (byte-array 10)] + (t/assert= (vec ba) [0 0 0 0 0 0 0 0 0 0]) + (t/assert= (count ba) 10))) diff --git a/tests/pixie/tests/test-async.pxi b/tests/pixie/tests/test-async.pxi new file mode 100644 index 00000000..19fa3731 --- /dev/null +++ b/tests/pixie/tests/test-async.pxi @@ -0,0 +1,29 @@ +(ns pixie.tests.test-async + (:require [pixie.stacklets :as st] + [pixie.async :as async :refer :all] + [pixie.test :as t :refer :all])) + + +(deftest test-future-deref + (let [f (future 42)] + (assert= @f 42))) + +(deftest test-future-deref-chain + (let [f1 (future 42) + f2 (future @f1) + f3 (future @f2)] + (assert= @f3 42))) + +(def *some-var* 0) +(set-dynamic! (var *some-var*)) + +(deftest test-dynamic-var-propagation + (set! (var *some-var*) 0) + (assert= *some-var* 0) + (let [fr @(future (do (println "running") + (let [old-val *some-var*] + (set! (var *some-var*) 42) + [old-val *some-var*])))] + + (assert= fr [0 42]) + (assert= *some-var* 0))) diff --git a/tests/pixie/tests/test-bits.pxi b/tests/pixie/tests/test-bits.pxi new file mode 100644 index 00000000..be044f9d --- /dev/null +++ b/tests/pixie/tests/test-bits.pxi @@ -0,0 +1,59 @@ +(ns pixie.test.test-bits + (require pixie.test :as t)) + +(t/deftest test-bit-clear + (t/assert= (bit-clear 0 0) 0) + (t/assert= (bit-clear 2r11111 7) 2r11111) + + (t/assert= (bit-clear 2r111 0) 2r110) + (t/assert= (bit-clear 2r111 1) 2r101) + (t/assert= (bit-clear 2r111 2) 2r011)) + +(t/deftest test-bit-set + (t/assert= (bit-set 2r111 0) 2r111) + (t/assert= (bit-set 2r000 1) 2r010)) + +(t/deftest test-bit-flip + (t/assert= (bit-flip 2r101 0) 2r100) + (t/assert= (bit-flip 2r101 1) 2r111)) + +(t/deftest test-bit-test + (t/assert= (bit-test 2r101 0) true) + (t/assert= (bit-test 2r101 1) false)) + +(t/deftest test-bit-and + (t/assert= (bit-and 0 0) 0) + (t/assert= (bit-and 2r101 2r101) 2r101) + (t/assert= (bit-and 2r101 2r101) 2r101) + (t/assert= (bit-and 2r101 0) 0)) + +(t/deftest test-bit-or + (t/assert= (bit-or 0 0) 0) + (t/assert= (bit-or 2r101 2r010) 2r111) + (t/assert= (bit-or 2r111 2r010) 2r111) + (t/assert= (bit-or 2r111 2r111) 2r111) + (t/assert= (bit-or 2r101 0) 2r101)) + +(t/deftest test-bit-xor + (t/assert= (bit-xor 0 0) 0) + (t/assert= (bit-xor 2r101 2r010) 2r111) + (t/assert= (bit-xor 2r111 2r010) 2r101) + (t/assert= (bit-xor 2r111 2r111) 2r000) + (t/assert= (bit-xor 2r101 0) 2r101)) + +(t/deftest test-bit-shift-left + (t/assert= (bit-shift-left 0 0) 0) + (t/assert= (bit-shift-left 2r101 0) 2r101) + + (t/assert= (bit-shift-left 0 7) 0) + (t/assert= (bit-shift-left 2r001 2) 2r100) + (t/assert= (bit-shift-left 2r111 2) 2r11100)) + +(t/deftest test-bit-shift-right + (t/assert= (bit-shift-right 0 0) 0) + (t/assert= (bit-shift-right 2r101 0) 2r101) + + (t/assert= (bit-shift-right 0 7) 0) + (t/assert= (bit-shift-right 2r001 2) 0) + (t/assert= (bit-shift-right 2r111 2) 2r001) + (t/assert= (bit-shift-right 2r1011010 2) 2r10110)) diff --git a/tests/pixie/tests/test-buffers.pxi b/tests/pixie/tests/test-buffers.pxi new file mode 100644 index 00000000..1afd5326 --- /dev/null +++ b/tests/pixie/tests/test-buffers.pxi @@ -0,0 +1,41 @@ +(ns pixie.tests.test-buffers + (:require [pixie.test :refer :all] + [pixie.buffers :refer :all])) + +(deftest test-adding-and-removing-from-buffer + (let [buffer (ring-buffer 10)] + (dotimes [x 100] + (add! buffer x) + (assert= x (remove! buffer))))) + +(deftest test-adding-multiple-items + (let [buffer (ring-buffer 10)] + (dotimes [x 10] + (add! buffer x)) + (dotimes [x 10] + (assert= x (remove! buffer))))) + +(deftest test-adding-multiple-items-with-resize + (dotimes [y 100] + (let [buffer (ring-buffer 2)] + (dotimes [x y] + (add-unbounded! buffer x)) + (dotimes [x y] + (assert= x (remove! buffer)))))) + + +(def drain-buffer (partial into [])) + +(deftest test-dropping-buffer + (let [buf (dropping-buffer 4)] + (dotimes [x 5] + (add! buf x)) + (assert= [0 1 2 3] (drain-buffer buf)) + (assert (not (full? buf))))) + +(deftest test-sliding-buffer + (let [buf (sliding-buffer 4)] + (dotimes [x 5] + (add! buf x)) + (assert= [1 2 3 4] (drain-buffer buf)) + (assert (not (full? buf))))) diff --git a/tests/pixie/tests/test-channels.pxi b/tests/pixie/tests/test-channels.pxi new file mode 100644 index 00000000..0cdebfd0 --- /dev/null +++ b/tests/pixie/tests/test-channels.pxi @@ -0,0 +1,87 @@ +(ns pixie.tests.test-channels + (:require [pixie.test :refer :all] + [pixie.channels :refer :all] + [pixie.async :refer :all] + [pixie.stacklets :as st])) + + +(deftest simple-read-and-write + (let [takep (promise) + putp (promise) + c (chan)] + (assert (-put! c 42 putp)) + (assert (-take! c takep)) + + (assert= @takep 42) + (assert= @putp true))) + +(deftest simple-take-before-put + (let [takep (promise) + putp (promise) + c (chan)] + (assert (-take! c takep)) + (assert (-put! c 42 putp)) + + (assert= @takep 42) + (assert= @putp true))) + +(deftest simple-take-before-put-with-buffers + (let [c (chan 2) + takesp (mapv (fn [_] (promise)) + (range 10)) + putsp (mapv (fn [_] (promise)) + (range 10))] + (doseq [p takesp] + (assert (-take! c p))) + (dotimes [x 10] + (assert (-put! c x (nth putsp x)))) + + (doseq [p putsp] + (assert= @p true)) + + (dotimes [x 10] + (assert= @(nth takesp x) x)))) + +(deftest closing-dispatches-future-takes-with-nil + (let [c (chan 1) + tp (promise)] + (-close! c) + (-take! c tp) + (assert= @tp nil))) + +(deftest closing-dispatches-past-takes-with-nil + (let [c (chan 1) + tp (promise)] + (-take! c tp) + (-close! c) + (assert= @tp nil))) + +(deftest closed-channels-return-false-on-future-puts + (let [c (chan 1) + pp (promise)] + (-close! c) + (assert= (-put! c 42 pp) false) + (assert= @pp false))) + +(deftest closing-allows-puts-to-flush + (let [c (chan) + tps (mapv (fn [_] (promise)) (range 3))] + (dotimes [x 2] + (-put! c x (fn [_] nil))) + (-close! c) + (assert= (mapv (partial -take! c) tps) [true true false]) + (assert= (mapv deref tps) [0 1 nil]))) + +(deftest alt-handlers-only-invoke-one-callback + (let [completed (atom 0) + c (chan 5) + [f1 f2] (alt-handlers + [(fn [_] (swap! completed inc)) + (fn [_] (swap! completed inc))])] + (-take! c f1) + (-take! c f2) + (-put! c 1 (fn [_])) + (-put! c 2 (fn [_])) + (st/yield-control) + + (assert= @completed 1))) diff --git a/tests/pixie/tests/test-compare.pxi b/tests/pixie/tests/test-compare.pxi new file mode 100644 index 00000000..3811e7c8 --- /dev/null +++ b/tests/pixie/tests/test-compare.pxi @@ -0,0 +1,59 @@ +(ns pixie.tests.test-compare + (require pixie.test :as t)) + +(t/deftest test-compare-numbers + (t/assert= (compare 1 1) 0) + (t/assert= (compare 1 2) -1) + (t/assert= (compare 1 -1) 1) + + (t/assert= (compare 1 1.0) 0) + (t/assert= (compare 1.0 1) 0) + (t/assert= (compare 1.0 2.0) -1) + (t/assert= (compare 1.0 -1.0) 1) + + (t/assert= (compare 1/2 1/2) 0) + (t/assert= (compare 1/3 1/2) -1) + (t/assert= (compare 1/2 1/3) 1)) + +(t/deftest test-compare-strings + (t/assert= (compare "a" "a") 0) + (t/assert= (compare "a" "b") -1) + (t/assert= (compare "b" "a") 1) + + (t/assert= (compare "aa" "a") 1) + (t/assert= (compare "a" "aa") -1) + + (t/assert= (compare "aa" "b") -1) + (t/assert= (compare "b" "aa") 1) + + (t/assert= (compare "aaaaaaaa" "azaaaaaa") -1) + (t/assert= (compare "azaaaaaa" "aaaaaaaa") 1)) + +(t/deftest test-compare-keywords + (t/assert= (compare :a :a) 0) + (t/assert= (compare :a :b) -1) + (t/assert= (compare :b :a) 1) + (t/assert= (compare :ns/a :ns/a) 0) + (t/assert= (compare :ns/a :ns/b) -1) + (t/assert= (compare :a :aa) -1) + (t/assert= (compare :aa :a) 1)) + +(t/deftest test-compare-symbols + (t/assert= (compare 'a 'a) 0) + (t/assert= (compare 'a 'b) -1) + (t/assert= (compare 'b 'a) 1) + (t/assert= (compare 'ns/a 'ns/a) 0) + (t/assert= (compare 'ns/a 'ns/b) -1) + (t/assert= (compare 'a 'aa) -1) + (t/assert= (compare 'aa 'a) 1)) + +(t/deftest test-compare-vectors + (t/assert= (compare [] []) 0) + (t/assert= (compare [1] []) 1) + (t/assert= (compare [1] [2]) -1) + (t/assert= (compare [1 2] [1 3]) -1) + (t/assert= (compare [:a] [:a]) 0) + (t/assert= (compare [:a] [:b]) -1) + (t/assert= (compare [:a] [:a :a]) -1) + (t/assert= (compare ["a"] ["a"]) 0) + (t/assert= (compare ["a"] ["a" "b"]) -1)) diff --git a/tests/pixie/tests/test-compiler.pxi b/tests/pixie/tests/test-compiler.pxi new file mode 100644 index 00000000..751ee95e --- /dev/null +++ b/tests/pixie/tests/test-compiler.pxi @@ -0,0 +1,73 @@ +(ns pixie.test.test-compiler + (require pixie.test :as t)) + +(t/deftest test-do + (t/assert= (do 1) 1) + (t/assert= (do 1 2) 2) + (t/assert= (do) nil) + (t/assert= (do 1 2 3 4 5 6) 6)) + +(t/deftest test-if + (t/assert= (if true 42 nil) 42) + (t/assert= (if false 42 nil) nil) + (t/assert= (if false 42) nil)) + +(t/deftest test-let + (t/assert= (let [] 1) 1) + (t/assert= (let [x 1]) nil) + (t/assert= (let []) nil)) + +(t/deftest test-lists + (t/assert= (vec '()) []) + (t/assert= (vec '()) ())) + + +(defprotocol IMutable + (mutate! [this])) + +(deftype Foo [x] + IMutable + (mutate! [this] + (let [xold x] + (set-field! this :x 42) + (t/assert (not (= xold x))) + (t/assert (= x 42))))) + +(t/deftest test-deftype-mutables + (mutate! (->Foo 0))) + +(t/deftest test-recur-must-tail + (t/assert-throws? + (eval + '(loop [n 0] + (inc (recur n))))) + + (t/assert-throws? + (eval + '(fn [n] + (if (zero? n) + 1 + (* n (recur (dec n))))))) + +(t/assert-throws? + (eval + '(fn [n] + (if (zero? n) + (* n (recur (dec n))) + 1)))) + + (t/assert= + (eval + '(loop [n 0] + (if (= 10 n) + n + (recur (inc n))))) + 10) + + (t/assert= + (eval + '(loop [n 0] + (if (not= 10 n) + (recur (inc n)) + n))) + 10)) diff --git a/tests/pixie/tests/test-csp.pxi b/tests/pixie/tests/test-csp.pxi new file mode 100644 index 00000000..94a241bd --- /dev/null +++ b/tests/pixie/tests/test-csp.pxi @@ -0,0 +1,48 @@ +(ns pixie.tests.test-buffers + (require pixie.test :refer :all) + (require pixie.csp :refer :all)) + + +(deftest test-go-blocks-return-values + (assert= (! c) (range 10)) + (close! c))] + (assert= (vec c) (range 10)) + (assert= (! c1 1) + (>! c2 2) + (assert (not (= c1 c2))) + (assert= (! c 1) + (assert= (alts! [c] :default 42) [c 1]))) + +(deftest test-timeout-channel + (let [ts [(timeout 300) + (timeout 200) + (timeout 100)]] + (-> (go + (loop [ts (set ts) + res []] + (if (empty? ts) + res + (let [[p _] (alts! ts)] + (recur (set (remove #{p} ts)) + (conj res p)))))) + Three 1 2 3)) +(def t2 (->Three 1 2 3)) +(def t3 (map->Three {:one 1, :two 2, :three 3})) +(def t4 (->Three 1 2 4)) +(def t5 (->Three 3 4 5)) + +(t/deftest test-satisfies + (foreach [t [t1 t2 t3 t4 t5]] + (t/assert= t t) + (t/assert (satisfies? IRecord t)) + (t/assert (satisfies? IAssociative t)) + (t/assert (satisfies? ILookup t)) + (t/assert (satisfies? IPersistentCollection t)))) + +(t/deftest test-record-pred + (t/assert (record? t1))) + +(t/deftest test-eq + (t/assert= t1 t2) + (t/assert= t2 t3) + (t/assert= t3 t1) + (foreach [t [t1 t2 t3]] + (t/assert (not (= t (assoc t :one 42)))) + (t/assert (not (= t t4))) + (t/assert (not (= t t5))))) + +(t/deftest test-ilookup + (foreach [t [t1 t2 t3]] + (t/assert (satisfies? ILookup t)) + (t/assert= (get t :one) 1) + (t/assert= (get t :two) 2) + (t/assert= (get t :three) 3) + (t/assert= (get t :oops) nil) + (t/assert= (get t :oops 'not-found) 'not-found))) + +(t/deftest test-iassociative + (foreach [t [t1 t2 t3]] + (t/assert (satisfies? IAssociative t)) + (t/assert= t (assoc t4 :three 3)) + (let [t' (assoc t :one 42) + t-oops (try (assoc t :oops 'never-found) + (catch ex t))] + (t/assert (not (= t t'))) + (t/assert= (get t' :one) 42) + (t/assert= t t-oops) + (t/assert (not (contains? t-oops :oops))) + (t/assert= (get t-oops :oops) nil)) + + (t/assert (contains? t :one)) + (t/assert (contains? t :two)) + (t/assert (contains? t :three)))) + +(t/deftest test-ipersistentcoll + (t/assert= 11 (-> t1 (conj [:one 11]) :one)) + (t/assert= 11 (-> t1 (conj (map-entry :one 11)) :one))) + +(t/deftest test-record-metadata + (t/assert= nil (meta t1)) + (t/assert= :foo (-> t1 (with-meta :foo) meta))) + + +(t/deftest ireduce [] + (t/assert= [[:one 1] [:two 2] [:three 3]] (reduce conj [] t1)) + (t/assert= [1 2 3] (vals t1)) + (t/assert= [:one :two :three] (keys t1))) + +(t/deftest icounted [] + (t/assert= (count t1) 3)) + +(t/deftest seqable [] + (t/assert= (key (first (seq t1))) :one) + (t/assert= (val (first (seq t1))) 1) + (t/assert (instance? LazySeq (seq t1)))) diff --git a/tests/pixie/tests/test-deftype.pxi b/tests/pixie/tests/test-deftype.pxi new file mode 100644 index 00000000..f6123aa0 --- /dev/null +++ b/tests/pixie/tests/test-deftype.pxi @@ -0,0 +1,80 @@ +(ns pixie.test.test-deftype + (require pixie.test :as t)) + +(deftype Simple [:val]) +(deftype Simple2 [val]) + +(t/deftest test-simple + (let [o1 (->Simple 1) + o2 (->Simple 2)] + (foreach [obj-and-val [[o1 1] [o2 2]]] + (let [o (first obj-and-val) + v (second obj-and-val)] + (t/assert= (get-field o :val) v))))) + +(deftype MagicalVectorMap [] IMap IVector) + +(t/deftest test-satisfies + (let [mvm (->MagicalVectorMap)] + (t/assert (satisfies? IVector mvm)) + (t/assert (satisfies? IMap mvm)))) + +(deftype Count [:val] + ICounted + (-count [self] val)) + +(deftype Count2 [val] + ICounted + (-count [self] val)) + +(t/deftest test-extend + (let [o1 (->Count 1) + o2 (->Count 2)] + (foreach [obj-and-val [[o1 1] [o2 2]]] + (let [o (first obj-and-val) + v (second obj-and-val)] + (t/assert= (get-field o :val) v) + (t/assert (satisfies? ICounted o)) + (t/assert= (-count o) v) + (t/assert= (count o) v))))) + +(defprotocol TestObject + (add [self x & args]) + (one-plus [self x & xs])) + +(deftype Three [:one :two :three] + TestObject + (add [self x & args] + (apply + x args)) + (one-plus [self x & xs] + (apply + one x xs)) + ICounted + (-count [self] (+ one two three))) + +(deftype Three2 [one two three] + TestObject + (add [self x & args] + (apply + x args)) + (one-plus [self x & xs] + (apply + one x xs)) + ICounted + (-count [self] (+ one two three))) + +(t/deftest test-complex + (let [o1 (->Three 1 2 3) + o2 (->Three2 3 4 5)] + (foreach [obj-and-vals [[o1 1 2 3] [o2 3 4 5]]] + (let [o (first obj-and-vals) + one (second obj-and-vals) + two (third obj-and-vals) + three (fourth obj-and-vals)] + (t/assert= (get-field o :one) one) + (t/assert= (get-field o :two) two) + (t/assert= (get-field o :three) three) + + (t/assert (satisfies? ICounted o)) + (t/assert= (-count o) (+ one two three)) + (t/assert= (count o) (+ one two three)) + + (t/assert= (add o 21 21) 42) + (t/assert= (one-plus o 9) (+ one 9)))))) diff --git a/tests/pixie/tests/test-destructuring.pxi b/tests/pixie/tests/test-destructuring.pxi new file mode 100644 index 00000000..13eb93a0 --- /dev/null +++ b/tests/pixie/tests/test-destructuring.pxi @@ -0,0 +1,76 @@ +(ns pixie.test.test-destructuring + (require pixie.test :as t)) + +(t/deftest test-let-simple + (t/assert= (let [x 1 y 2 z 3] [x y z]) [1 2 3]) + (t/assert= (let [x 1 y 2 z 3] [x y z]) (let* [x 1 y 2 z 3] [1 2 3])) + + (t/assert= (let [x 1 x 2] x) 2) + (t/assert= (let [x 1 y 2 x 3] x) 3) + + (t/assert= (let [x 1] (let [x 2] x)) 2)) + +(t/deftest test-let-vector-simple + (t/assert= (let [[x y z] [1 2 3]] [x y z]) [1 2 3]) + (t/assert= (let [[x y z] [1 2 3 4]] [x y z]) [1 2 3]) + + (t/assert= (let [[x y z & rest] [1 2 3 4]] + [x y z rest]) + [1 2 3 '(4)]) + (t/assert= (let [[x y z & rest] [1 2]] + [x y z rest]) + [1 2 nil nil])) + +(t/deftest test-let-vector-nested + (t/assert= (let [[[x y] z & rest] [[1 2] 3 4]] + [x y z rest]) + [1 2 3 '(4)]) + + (t/assert= (let [[[x [y]] z & rest] [[1 [2 3]] 4 5]] + [x y z rest]) + [1 2 4 '(5)])) + +(t/deftest test-let-vector-rest + (t/assert= (let [[x y & [z & rest]] [1 2 3 4 5]] + [x y z rest]) + [1 2 3 '(4 5)])) + +(t/deftest test-let-map + (t/assert= (let [{a :a, b :b, {c :c :as s} :d :as m} {:a 1, :b 2, :d {:c 3}}] + [a b c s m]) + [1 2 3 {:c 3} {:a 1, :b 2, :d {:c 3}}]) + + (t/assert= (let [{:keys [a b c] :as m} {:a 1, :b 2, :c 3, :d 4}] + [a b c (:d m)]) + [1 2 3 4])) + +(t/deftest test-let-map-defaults + (t/assert= (let [{a :a :or {a 42}} {:a 1}] a) 1) + (t/assert= (let [{a :a :or {a 42}} {}] a) 42) + + (t/assert= (let [{a :a :or {a 42}} {:a nil}] a) nil) + (t/assert= (let [{a :a :or {a 42}} {:a false}] a) false) + + (t/assert= (let [{:keys [a], :or {a 42}} {:a 1}] a) 1) + (t/assert= (let [{:keys [a], :or {a 42}} {}] a) 42)) + +(t/deftest test-fn-simple + (t/assert= ((fn [[x y & rest]] [x y rest]) [1 2 3 4 5]) [1 2 '(3 4 5)]) + (t/assert= ((fn [{a :a, b :b :as m}] [a b m]) {:a 1, :b 2, :answer 42}) [1 2 {:a 1, :b 2, :answer 42}]) + + (t/assert= ((fn [[[x y] z & rest]] [x y z rest]) [[1 2] 3 4 5]) [1 2 3 '(4 5)])) + +(t/deftest test-fn-multiple-args + (t/assert= ((fn [[x y z] {:keys [a b c]}] [x y z a b c]) [1 2 3] {:a 4, :b 5, :c 6}) + [1 2 3 4 5 6])) + +(t/deftest test-fn-rest-args + (let [f1 (fn [& [status]] (or status :yay))] + (t/assert= (f1) :yay) + (t/assert= (f1 :nay) :nay) + (t/assert= (f1 :nay :something-else :whatever) :nay)) + (let [f2 (fn [x & [y]] + (+ x (or y 1)))] + (t/assert= (f2 41) 42) + (t/assert= (f2 21 21) 42) + (t/assert= (f2 21 21 :something-else :whatever) 42))) diff --git a/tests/pixie/tests/test-docs.pxi b/tests/pixie/tests/test-docs.pxi new file mode 100644 index 00000000..dd6b96cf --- /dev/null +++ b/tests/pixie/tests/test-docs.pxi @@ -0,0 +1,24 @@ +(ns pixie.tests.test-docs + (require pixie.test :as t)) + +; validate the examples in the docs by checking whether the included +; results match the actual results you get by evaluating the examples. + +(defn check-examples [ns] + (let [ns (the-ns ns) + syms (keys (ns-map ns))] + (doseq [sym syms] + (let [meta (meta @(resolve-in ns sym)) + examples (get meta :examples)] + (doseq [example examples] + (if (contains? example 2) + (t/assert= (eval (read-string (first example))) + (third example)) + (eval (read-string (first example))))))))) + +(t/deftest test-stdlib-docs + (check-examples 'pixie.stdlib)) + +(t/deftest test-string-docs + (load-ns 'pixie.string) + (check-examples 'pixie.string)) diff --git a/tests/pixie/tests/test-errors.pxi b/tests/pixie/tests/test-errors.pxi new file mode 100644 index 00000000..1ef095b5 --- /dev/null +++ b/tests/pixie/tests/test-errors.pxi @@ -0,0 +1,15 @@ +(ns pixie.test.test-errors + (:require [pixie.test :refer :all])) + +(deftest test-add-exception-info + (try + (try + (+ 1 "foo") + (catch ex + (throw (add-exception-info ex "My Msg" :my-data)))) + (catch ex + (let [filter-fn (fn [mp] + (and (= (:type mp) :extra) + (= (:msg mp) "My Msg") + (= (:data mp) :my-data)))] + (assert= 1 (count (filter filter-fn ex))))))) diff --git a/tests/pixie/tests/test-ffi.pxi b/tests/pixie/tests/test-ffi.pxi new file mode 100644 index 00000000..4cc486dd --- /dev/null +++ b/tests/pixie/tests/test-ffi.pxi @@ -0,0 +1,83 @@ +(ns pixie.tests.test-ffi + (require pixie.test :as t) + (require pixie.math :as m) + (require pixie.ffi-infer :as i)) + + + +(t/deftest test-buffer-ffi + (let [fp (fopen "README.md" "r") + b (buffer 1024)] + (t/assert= 10 (fread b 1 10 fp)) + (t/assert= 91 (nth b 0)))) + +(t/deftest test-arity-check + (let [sscanf-2 (ffi-fn libc "sscanf" [CCharP CCharP] CInt) + sscanf-* (ffi-fn libc "sscanf" [CCharP CCharP] CInt :variadic? true)] + (try + (sscanf-2 "too few arguments") + (t/assert false) + (catch ex (t/assert= (type ex) RuntimeException))) + + (try + (sscanf-2 "too" "many" "arguments") + (t/assert false) + (catch ex (t/assert= (type ex) RuntimeException))) + + (try + (sscanf-* "too few arguments") + (t/assert false) + (catch ex (t/assert= (type ex) RuntimeException))) + + (sscanf-2 "string" "fmt") + (sscanf-* "string" "fmt") + (sscanf-* "string" "fmt" "optional arg1" "optional arg2") + (t/assert true))) + +(t/deftest test-ffi-infer + (t/assert= 0.5 (m/asin (m/sin 0.5)))) + +(t/deftest test-cdouble + (i/with-config {:library "m" + :cxx-flags ["-lm"] + :includes ["math.h"]} + (i/defcfn sinf) + (i/defcfn asinf) + (i/defcfn cosf) + (i/defcfn powf)) + (t/assert= 0.5 (asinf (sinf 0.5))) + (t/assert= 1.0 (+ (powf (sinf 0.5) 2.0) (powf (cosf 0.5) 2.0)))) + +(t/deftest test-invalid-float-argument + (t/assert-throws? (m/sin "nil")) + (t/assert-throws? (m/sin nil)) + ) + +(t/deftest test-ffi-callbacks + (let [MAX 255 + qsort-cb (pixie.ffi/ffi-callback [CVoidP CVoidP] CInt) + qsort (ffi-fn libc "qsort" [CVoidP CInt CInt qsort-cb] CInt) + + buf (buffer MAX)] + (using [cb (pixie.ffi/ffi-prep-callback qsort-cb (fn [x y] + (if (> (pixie.ffi/unpack x 0 CUInt8) + (pixie.ffi/unpack y 0 CUInt8)) + -1 + 1)))] + (dotimes [x MAX] + (pixie.ffi/pack! buf x CUInt8 x)) + (qsort buf MAX 1 cb) + + (dotimes [x (dec MAX)] + (t/assert (> (pixie.ffi/unpack buf x CUInt8) + (pixie.ffi/unpack buf (inc x) CUInt8))))))) + +(t/deftest test-size + (t/assert= (pixie.ffi/struct-size (pixie.ffi/c-struct "struct" 1234 [])) 1234)) + + +(t/deftest test-double-coercion + (t/assert= (m/sin 1) (m/sin 1.0)) + (let [big (reduce * 1 (range 1 100))] + (t/assert= (m/sin big) (m/sin (float big)))) + (t/assert= (m/sin (/ 1 2)) (m/sin (float (/ 1 2))))) diff --git a/tests/pixie/tests/test-fns.pxi b/tests/pixie/tests/test-fns.pxi new file mode 100644 index 00000000..23915d0c --- /dev/null +++ b/tests/pixie/tests/test-fns.pxi @@ -0,0 +1,60 @@ +(ns pixie.test.test-fns + (require pixie.test :as t)) + +(t/deftest test-fn-literals + (t/assert= (#(+ 3 4)) 7) + (t/assert= (#(+ 3 %) 4) 7) + (t/assert= (#(+ 3 %1) 4) 7) + (t/assert= (#(+ %1 3) 4) 7) + (t/assert= (#(+ %1 %1) 3.5) 7.0) + (t/assert= (#(+ %1 %2) 3 4) 7) + (t/assert= (#(- %2 %1) 3 4) 1) + (t/assert= (#(+ %1 %1 %2 %2) 1.5 2) 7.0) + (t/assert= (#(+ %1 %3) 3 'ignored 4) 7) + (t/assert= (#(- %3 %1) 3 'ignored 4) 1) + (t/assert= (#(apply + %1 %2 %&) 1 2 3 4 5) (+ 1 2 3 4 5))) + +;; Note these tests are for functions of type 'Code'. +(t/deftest test-code-arity-errors + (let [arity-0 (fn arity-0 []) + arity-1 (fn arity-1 [a]) + arity-2 (fn arity-2 [a b]) + arity-0-or-1 (fn arity-0-or-1 ([]) ([a])) + arity-1-or-3 (fn arity-1-or-3 ([a]) ([a b c])) + arity-0-or-1-or-3-or-more + (fn arity-0-or-1-or-3-or-more ([]) ([a]) ([a b c & more]))] + (t/assert-throws? RuntimeException + "Invalid number of arguments 1 for function 'arity-0'. Expected 0" + (arity-0 :foo)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 2 for function 'arity-0'. Expected 0" + (arity-0 :foo :bar)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 0 for function 'arity-1'. Expected 1" + (arity-1)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 2 for function 'arity-1'. Expected 1" + (arity-1 :foo :bar)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 0 for function 'arity-2'. Expected 2" + (arity-2)) + (t/assert-throws? RuntimeException + "Invalid number of arguments 1 for function 'arity-2'. Expected 2" + (arity-2 :foo)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 2 for function 'arity-0-or-1'. Expected 0 or 1" + (arity-0-or-1 :foo :bar)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 3 for function 'arity-0-or-1'. Expected 0 or 1" + (arity-0-or-1 :foo :bar :baz)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 2 for function 'arity-1-or-3'. Expected 1 or 3" + (arity-1-or-3 :foo :bar)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 0 for function 'arity-1-or-3'. Expected 1 or 3" + (arity-1-or-3)) + (t/assert-throws? RuntimeException + "Wrong number of arguments 2 for function 'arity-0-or-1-or-3-or-more'. Expected 0, 1 or 3+" + (arity-0-or-1-or-3-or-more :foo :bar)))) + +(t/deftest test-code-arities) diff --git a/tests/pixie/tests/test-forms.pxi b/tests/pixie/tests/test-forms.pxi new file mode 100644 index 00000000..b909869a --- /dev/null +++ b/tests/pixie/tests/test-forms.pxi @@ -0,0 +1,90 @@ +(ns pixie.tests.test-forms + (require pixie.test :as t)) + +(t/deftest test-when + (t/assert= (when false :never) nil) + (t/assert= (when nil :never) nil) + (t/assert= (when (= 3 4) :never) nil) + + (t/assert= (when true :always) :always) + (t/assert= (when (+ 3 4) :always) :always) + (t/assert= (when {} :always) :always) + + (let [c (atom 0)] + (when (= 3 3) + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-not + (t/assert= (when-not false :always) :always) + (t/assert= (when-not nil :always) :always) + (t/assert= (when-not (= 3 4) :always) :always) + + (t/assert= (when-not true :never) nil) + (t/assert= (when-not (+ 3 4) :never) nil) + (t/assert= (when-not {} :never) nil) + + (let [c (atom 0)] + (when-not (= 3 4) + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-let + (t/assert= (when-let [v false] :never) nil) + (t/assert= (when-let [v nil] :never) nil) + (t/assert= (when-let [v (= 3 4)] :never) nil) + + (t/assert= (when-let [v true] :always) :always) + (t/assert= (when-let [v (+ 3 4)] :always) :always) + (t/assert= (when-let [v {}] :always) :always) + + (let [c (atom 0)] + (when-let [v @c] + (swap! c inc) + (swap! c inc) + (swap! c inc)) + (t/assert= @c 3))) + +(t/deftest test-when-let-destructuring + (t/assert= (when-let [[x y & z] false] :yay) nil) + (t/assert= (when-let [[x y & z] nil] :yay) nil) + (t/assert= (when-let [{:keys [a b]} nil] :yay) nil) + + (t/assert= (when-let [[x y & z] [1 2 3]] :yay) :yay) + (t/assert= (when-let [[x y & z] [1 2 3]] [x y z]) [1 2 '(3)]) + (t/assert= (when-let [{:keys [a b]} {}] :yay) :yay) + (t/assert= (when-let [{:keys [a b]} {}] [a b]) [nil nil]) + (t/assert= (when-let [{:keys [a b]} {:a 1, :b 41}] [a b]) [1 41])) + +(t/deftest test-if-let + (t/assert= (if-let [v false] :yay :nay) :nay) + (t/assert= (if-let [v false] :yay) nil) + (t/assert= (if-let [v nil] :yay :nay) :nay) + (t/assert= (if-let [v nil] :yay) nil) + (t/assert= (if-let [v (= 3 4)] :yay :nay) :nay) + (t/assert= (if-let [v (= 3 4)] :yay) nil) + + (t/assert= (if-let [v true] :yay :nay) :yay) + (t/assert= (if-let [v true] :yay) :yay) + (t/assert= (if-let [v (+ 3 4)] v :nay) 7) + (t/assert= (if-let [v (+ 3 4)] v) 7) + (t/assert= (if-let [v {}] v :nay) {}) + (t/assert= (if-let [v {}] v) {})) + +(t/deftest test-if-let-destructuring + (t/assert= (if-let [[x y & z] false] :yay :nay) :nay) + (t/assert= (if-let [[x y & z] false] :yay) nil) + (t/assert= (if-let [[x y & z] nil] :yay :nay) :nay) + (t/assert= (if-let [[x y & z] nil] :yay) nil) + (t/assert= (if-let [{:keys [a b]} nil] :yay :nay) :nay) + (t/assert= (if-let [{:keys [a b]} nil] :yay) nil) + + (t/assert= (if-let [[x y & z] [1 2 3]] :yay :nay) :yay) + (t/assert= (if-let [[x y & z] [1 2 3]] [x y z] :nay) [1 2 '(3)]) + (t/assert= (if-let [{:keys [a b]} {}] :yay :nay) :yay) + (t/assert= (if-let [{:keys [a b]} {}] [a b] :nay) [nil nil]) + (t/assert= (if-let [{:keys [a b]} {:a 1, :b 41}] [a b] :nay) [1 41])) diff --git a/tests/pixie/tests/test-fs.pxi b/tests/pixie/tests/test-fs.pxi new file mode 100644 index 00000000..c02d664a --- /dev/null +++ b/tests/pixie/tests/test-fs.pxi @@ -0,0 +1,125 @@ +(ns pixie.tests.test-fs + (require pixie.test :as t) + (require pixie.fs :as fs)) + +(t/deftest test-file-comparisons + "All these paths are the same" + (let [dir-a "tests/pixie/tests/fs/parent" + dir-b "tests/pixie/tests/fs/../fs/parent" + dir-c "tests/pixie/tests/fs/../fs/parent/../../fs/parent" + file-a (str dir-a "/foo.txt") + file-b (str dir-b "/foo.txt") + file-c (str dir-c "/foo.txt")] + (t/assert= (= (fs/dir dir-a) + (fs/dir dir-b) + (fs/dir dir-c)) + true) + + (t/assert= (= (fs/file file-a) + (fs/file file-b) + (fs/file file-c)) + true) + + (t/assert= (count (distinct [(fs/dir dir-a) + (fs/dir dir-b) + (fs/dir dir-c)])) + 1) + (t/assert= (count (distinct [(fs/file file-a) + (fs/file file-b) + (fs/file file-c)])) + 1))) + +(comment + ; Although these pass locally, they don't appear to work on travis-ci (segfault) + (t/deftest test-walking + (let [dir-a "tests/pixie/tests/fs"] + (t/assert= (set (fs/walk (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/dir (str dir-a "/parent/child")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) + + (t/assert= (set (fs/walk-files (fs/dir dir-a))) + #{(fs/file (str dir-a "/parent/foo.txt")) + (fs/file (str dir-a "/parent/bar.txt")) + (fs/file (str dir-a "/parent/child/foo.txt")) + (fs/file (str dir-a "/parent/child/bar.txt"))}) + + (t/assert= (set (fs/walk-dirs (fs/dir dir-a))) + #{(fs/dir (str dir-a "/parent")) + (fs/dir (str dir-a "/parent/child"))}))) + + (t/deftest test-list + (let [dir-a "tests/pixie/tests/fs/parent"] + (t/assert= (set (fs/list (fs/dir dir-a))) + #{(fs/file (str dir-a "/foo.txt")) + (fs/file (str dir-a "/bar.txt")) + (fs/dir (str dir-a "/child"))}))) +) + +(t/deftest test-rel? + (let [dir-a (fs/dir "tests/pixie/tests/fs") + dir-b (fs/dir "tests/pixie/tests/fs/parent") + dir-c (fs/dir "tests/pixie/tests/fs/parent/child") + file-a (fs/file "tests/pixie/tests/fs/parent/foo.txt") + file-b (fs/file "tests/pixie/tests/fs/parent/bar.txt") + file-c (fs/file "tests/pixie/tests/fs/parent/child/foo.txt")] + ;; Test directory-directory comparisons + (t/assert= (fs/rel dir-a dir-a) ".") + (t/assert= (fs/rel dir-a dir-b) "..") + (t/assert= (fs/rel dir-a dir-c) "../..") + (t/assert= (fs/rel dir-c dir-b) "child") + (t/assert= (fs/rel dir-c dir-a) "parent/child") + ;; Test file-file comparisons + (t/assert= (fs/rel file-a file-a) ".") + (t/assert= (fs/rel file-a file-b) "../foo.txt") + (t/assert= (fs/rel file-a file-c) "../../foo.txt") + (t/assert= (fs/rel file-c file-a) "../child/foo.txt") + (t/assert= (fs/rel file-c file-b) "../child/foo.txt") + ;; Test file-directory comparisons + (t/assert= (fs/rel file-a dir-a) "parent/foo.txt") + (t/assert= (fs/rel file-a dir-b) "foo.txt") + (t/assert= (fs/rel file-a dir-c) "../foo.txt"))) + +(t/deftest test-basename? + (let [dir-a (fs/dir "tests/pixie/tests/fs") + dir-b (fs/dir "tests/pixie/tests/fs/parent") + dir-c (fs/dir "tests/pixie/tests/fs/parent/child") + file-a (fs/file "tests/pixie/tests/fs/parent/foo.txt") + file-b (fs/file "tests/pixie/tests/fs/parent/bar.txt") + file-c (fs/file "tests/pixie/tests/fs/parent/child/foo.txt")] + ;; Test directory-directory comparisons + (t/assert= (fs/basename dir-a) "fs") + (t/assert= (fs/basename dir-b) "parent") + (t/assert= (fs/basename dir-c) "child") + (t/assert= (fs/basename file-a) "foo.txt") + (t/assert= (fs/basename file-b) "bar.txt") + (t/assert= (fs/basename file-c) "foo.txt"))) + +(t/deftest test-exists? + (let [real-dir (fs/dir "tests/pixie/tests/fs/parent") + fake-dir (fs/dir "tests/pixie/tests/fs/parent/fake-dir") + fake-file (fs/dir "tests/pixie/tests/fs/parent/fake-file")] + (t/assert= (fs/exists? real-dir) true) + (t/assert= (fs/exists? fake-dir) false) + (t/assert= (fs/exists? fake-file) false))) + +(t/deftest test-size + (let [file-with-content (fs/file "tests/pixie/tests/fs/parent/foo.txt") + file-without-content (fs/file "tests/pixie/tests/fs/parent/bar.txt") + fake-file (fs/file "tests/pixie/tests/fs/parent/fake-file")] + (t/assert= (fs/size file-with-content) 4) + (t/assert= (fs/size file-without-content) 0) + (t/assert-throws? (fs/size fake-file)))) + +(t/deftest test-permissions + (let [file (fs/file "tests/pixie/tests/fs/parent/foo.txt") + fake-file (fs/file "tests/pixie/tests/fs/parent/fake-file")] + ; because Travis seems to change the permissions of some files... + (let [perms (fs/permissions file)] + (t/assert= (count perms) 3) + (t/assert (instance? String perms)) + (t/assert= (set (seq perms)) #{\6 \4})) + (t/assert-throws? (fs/permissions fake-file)))) diff --git a/tests/pixie/tests/test-io-utf8.txt b/tests/pixie/tests/test-io-utf8.txt new file mode 100644 index 00000000..37bf4044 --- /dev/null +++ b/tests/pixie/tests/test-io-utf8.txt @@ -0,0 +1 @@ +I love ๐Ÿบ . This is a thumbs up ๐Ÿ‘ diff --git a/tests/pixie/tests/test-io.pxi b/tests/pixie/tests/test-io.pxi new file mode 100644 index 00000000..07a3606b --- /dev/null +++ b/tests/pixie/tests/test-io.pxi @@ -0,0 +1,190 @@ +(ns pixie.tests.test-io + (require pixie.test :as t) + (require pixie.streams :as st :refer :all) + (require pixie.streams.utf8 :as utf8 :refer :all) + (require pixie.io :as io) + (require pixie.io-blocking :as blocking) + (require pixie.streams.zlib :as zlib)) + +(t/deftest test-file-reduction + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (map identity) + count-rf + f) + 91)) + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (comp (map char) (take 4)) string-builder f) + "This")) + (let [f (blocking/open-read "tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (map identity) + count-rf + f) + 91)) + (let [f (blocking/open-read "tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (comp (map char) (take 4)) string-builder f) + "This"))) + +(t/deftest test-process-reduction + (let [f (io/run-command "ls tests/pixie/tests/test-io.txt")] + (t/assert= f "tests/pixie/tests/test-io.txt\n")) + (let [pipe (blocking/popen-read "ls tests/pixie/tests/test-io.txt")] + (t/assert= (transduce (comp (map char) (take 6)) string-builder pipe) + "tests/"))) + +(t/deftest test-read-into-buffer + (let [f (io/open-read "tests/pixie/tests/test-io.txt")] + (let [buf (buffer 16)] + (io/read f buf 4) + (t/assert= (transduce (map char) string-builder buf) "This") + + (io/read f buf 4) + (t/assert= (transduce (map char) string-builder buf) " is ") + + + (io/read f buf 0) + (t/assert= (transduce (map char) string-builder buf) "") + + (t/assert-throws? (io/read f buf 17)) + (t/assert-throws? (io/read f buf -2))))) + +(t/deftest test-read-line + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] + (io/read-line f) + (t/assert= (io/read-line f) "Second line.") + (t/assert= (io/read-line f) nil))) + +(t/deftest test-line-reader + (let [lines (io/line-reader (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert= (transduce (map count) conj [] lines) + [77 12])) + (let [lines (io/line-reader (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert= (transduce (comp (map count) (take 1)) conj [] lines) + [77]))) + +(t/deftest test-line-seq + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt")) + s (io/line-seq f)] + (t/assert= (last s) "Second line."))) + +(t/deftest test-seek + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] + (io/read-line f) + (t/assert= (io/read-line f) "Second line.") + (io/rewind f) + (io/read-line f) + (t/assert= (io/read-line f) "Second line.") + (io/seek f (- (position f) 6)) + (t/assert= (io/read-line f) "line."))) + +(t/deftest test-buffered-input-streams + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert= (char (io/read-byte f)) \T) + (t/assert= (char (io/read-byte f)) \h) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (transduce (comp (map char) (take 5)) string-builder f) + " is a") + (t/assert= (transduce (comp (map char) (take 5)) string-builder f) + " test"))) + +(t/deftest test-buffered-input-streams-throws-on-non-input-streams + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt"))] + (t/assert-throws? (io/buffered-input-stream f)))) + +(t/deftest test-buffered-input-streams-seek + ;; We use a buffer size of 4 because the test file isn't huge and i am terrible at + ;; counting characters... + (let [f (io/buffered-input-stream (io/open-read "tests/pixie/tests/test-io.txt") 4)] + ;; Read the first word 'This' + (t/assert= (position f) 0) + (t/assert= (char (io/read-byte f)) \T) + (t/assert= (position f) 1) + (t/assert= (char (io/read-byte f)) \h) + (t/assert= (position f) 2) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 3) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (position f) 4) + + ;; Back to start of file (this is a seek with in the buffer) + (rewind f) + + ;; Should read the first word again + (t/assert= (char (io/read-byte f)) \T) + (t/assert= (position f) 1) + (t/assert= (char (io/read-byte f)) \h) + (t/assert= (position f) 2) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 3) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (position f) 4) + + ;; Skip the space (we will have caused a seek upstream) + (seek f 5) + + ;; Read 'is' + (t/assert= (position f) 5) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 6) + (t/assert= (char (io/read-byte f)) \s) + (t/assert= (position f) 7) + + ;; Jump ahead to 'file' (another seek upstream) + (seek f 15) + (t/assert= (position f) 15) + (t/assert= (char (io/read-byte f)) \f) + (t/assert= (position f) 16) + (t/assert= (char (io/read-byte f)) \i) + (t/assert= (position f) 17) + (t/assert= (char (io/read-byte f)) \l) + (t/assert= (position f) 18) + (t/assert= (char (io/read-byte f)) \e) + + ;; Another seek with in a buffer + (seek f 16) + (t/assert= (position f) 16) + (t/assert= (char (io/read-byte f)) \i))) + +(t/deftest test-slurp-spit + (let [val (vec (range 1280))] + (io/spit "test.tmp" val) + (t/assert= val (read-string (io/slurp "test.tmp")))) + (t/assert-throws? (io/slurp 1)) + (t/assert-throws? (io/slurp :foo)) + (t/assert= "I love ๐Ÿบ . This is a thumbs up ๐Ÿ‘\n" + (io/slurp "tests/pixie/tests/test-io-utf8.txt"))) + +(defn compress-content [output-stream content] + (transduce (map identity) + (-> output-stream + (zlib/compressing-output-stream (buffer 512) {}) + (io/buffered-output-stream 1024) + utf8/utf8-output-stream-rf) + (str content))) + +(defn compress-and-decompress-content [output-stream content] + (transduce (map identity) + (-> output-stream + (zlib/decompressing-output-stream (buffer 512) {}) + (zlib/compressing-output-stream (buffer 512) {}) + (io/buffered-output-stream 1024) + utf8/utf8-output-stream-rf) + (str content))) + +(t/deftest test-write-compressed + (io/run-command "rm compressed-output.gz") + (compress-content (io/open-write "compressed-output.gz") (range 1000)) + ;; decompress the file with zcat + (io/run-command "gunzip compressed-output.gz -c > compressed-output.txt") + (t/assert= (range 1000) (read-string (io/slurp "compressed-output.txt"))) + + ;; Wrapping an IInputStream in decompressing-stream should get the same result + (t/assert= (range 1000) (read-string (io/slurp + (zlib/decompressing-input-stream + (io/open-read "compressed-output.gz") + (buffer 512) {}))))) + +;; sticking a compressor into a decompressor should result in the original data +(t/deftest test-decompression + (compress-and-decompress-content (io/open-write "decompressed-output.txt") (range 1000)) + (t/assert= (range 1000) (read-string (io/slurp "decompressed-output.txt")))) diff --git a/tests/pixie/tests/test-io.txt b/tests/pixie/tests/test-io.txt new file mode 100644 index 00000000..ce2e9b4d --- /dev/null +++ b/tests/pixie/tests/test-io.txt @@ -0,0 +1,2 @@ +This is a test file used in testing the io routines. Please do not remove it. +Second line. diff --git a/tests/pixie/tests/test-ith.pxi b/tests/pixie/tests/test-ith.pxi new file mode 100644 index 00000000..0591165c --- /dev/null +++ b/tests/pixie/tests/test-ith.pxi @@ -0,0 +1,51 @@ +(ns pixie.tests.test-ith + (require pixie.test :as t)) + +(t/deftest test-ith + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (t/assert= (ith [1 2 3 4 5] 0) 1) + (t/assert= (ith v 0) 1) + (t/assert= (ith v 0) (nth v 0)))) + +(t/deftest test-ith-nil + (t/assert= (ith nil 0) nil) + (t/assert= (ith nil 1) nil) + (t/assert= (ith nil -1) nil)) + +(t/deftest test-ith-empty-always-oob + (t/assert= "Index out of Range" (try (ith [] 0) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith [] 1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith [] -1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith '() 0) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith '() 1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith '() -1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith (range 0 0) 0) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith (range 0 0) 1) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith (range 0 0) -1) (catch e (ex-msg e))))) + +(t/deftest test-ith-out-of-bounds + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (t/assert= "Index out of Range" (try (ith v 5) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith l 5) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith r 5) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith v -6) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith l -6) (catch e (ex-msg e)))) + (t/assert= "Index out of Range" (try (ith r -6) (catch e (ex-msg e)))))) + +(t/deftest test-ith-doseq + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (doseq [i (range 0 5)] + (t/assert= (ith v i) (nth v i)) + (t/assert= (ith l i) (nth l i)) + (t/assert= (ith r i) (nth r i))) + (doseq [i (range -5 0)] + (t/assert= (ith v i) (nth v (+ 5 i))) + (t/assert= (ith l i) (nth l (+ 5 i))) + (t/assert= (ith r i) (nth r (+ 5 i)))))) + diff --git a/tests/pixie/tests/test-keywords.pxi b/tests/pixie/tests/test-keywords.pxi new file mode 100644 index 00000000..1094cf45 --- /dev/null +++ b/tests/pixie/tests/test-keywords.pxi @@ -0,0 +1,69 @@ +(ns pixie.tests.test-keywords + (require pixie.test :as t)) + +(t/deftest keyword-invoke + (let [m {:a 1, :b 2, :c 3}] + (t/assert= (:a m) 1) + (t/assert= (:b m) 2) + (t/assert= (:c m) 3) + (t/assert= (:d m) nil) + (t/assert= (:d m :foo) :foo))) + +(t/deftest keyword-namespace + (t/assert= (namespace :foo/bar) "foo") + (t/assert= (namespace :cat/dog) "cat") + (t/assert= (namespace ::foo) "pixie.tests.test-keywords")) + +(t/deftest fqd-keywords + (t/assert-throws? (read-string "::x/bar")) + (t/assert-throws? (read-string "::a.b/foo")) + (refer-ns 'my.other.ns 'my.fake.core 'fake) + (binding [*ns* (the-ns 'my.other.ns)] + (t/assert= :my.fake.core/foo (read-string "::fake/foo")) + (t/assert= :my.fake.core/foo (read-string "::my.fake.core/foo")) + (t/assert-throws? (read-string "::f/foo")))) + +(t/deftest keyword-equality + (t/assert= :foo/bar :foo/bar) + (t/assert= (not= :foo/bar :cat/bar) true) + (t/assert= (not= :foo/cat :foo/dog) true)) + +(t/deftest keyword-reader + (t/assert= (read-string ":1") :1) + (t/assert= (read-string ":1") :1) + (t/assert= (read-string ":1.0") :1.0) + (t/assert= (read-string ":foo") :foo) + (t/assert= (read-string ":1foo") :1foo) + (t/assert= (read-string ":foo/bar") :foo/bar) + (t/assert= (read-string ":1foo/1bar") :1foo/1bar) + (t/assert= (read-string ":nil") :nil) + (t/assert= (read-string ":true") :true) + (t/assert= (read-string ":false") :false) + + ;; We are reading at runtime so the namespace isn't + ;; going to be pixie.test.test-keywords. Its probably + ;; 'user but lets explicitly set it. + ;; The refer-ns is to initialize a new space + (refer-ns 'my.other.ns 'my.fake.core 'fake) + (binding [*ns* (the-ns 'my.other.ns)] + (t/assert= (read-string "::1") :my.other.ns/1) + (t/assert= (read-string "::1.0") :my.other.ns/1.0) + (t/assert= (read-string "::foo") :my.other.ns/foo) + (t/assert= (read-string "::true") :my.other.ns/true))) + +(t/deftest string-to-keyword + (t/assert= (keyword "1") :1) + (t/assert= (keyword "1") :1) + (t/assert= (keyword "1.0") :1.0) + (t/assert= (keyword "foo") :foo) + (t/assert= (keyword "1foo") :1foo) + (t/assert= (keyword "foo/bar") :foo/bar) + (t/assert= (keyword "1foo/1bar") :1foo/1bar) + (t/assert= (keyword "nil") :nil) + (t/assert= (keyword "true") :true) + (t/assert= (keyword "false") :false) + (t/assert-throws? (keyword 1)) + (t/assert-throws? (keyword :a)) + (t/assert-throws? (keyword 'a)) + (t/assert-throws? (keyword nil)) + (t/assert-throws? (keyword true))) diff --git a/tests/pixie/tests/test-macros.pxi b/tests/pixie/tests/test-macros.pxi new file mode 100644 index 00000000..238408d3 --- /dev/null +++ b/tests/pixie/tests/test-macros.pxi @@ -0,0 +1,52 @@ +(ns collections.test-macros + (require pixie.test :as t)) + +(t/deftest hashmap-unquote + (let [x 10 k :boop] + (t/assert= (-eq `{:x ~x} {:x 10}) true) + (t/assert= (-eq `{~k ~x} {:boop 10}) true) + (t/assert= (-eq `{:x {:y ~x}} {:x {:y 10}}) true))) + +(def constantly-nil (constantly nil)) + +(t/deftest some->test + (t/assert (nil? (some-> nil))) + (t/assert (= 0 (some-> 0))) + (t/assert (= -1 (some-> 1 (- 2)))) + (t/assert (nil? (some-> 1 constantly-nil (- 2))))) + +(t/deftest some->>test + (t/assert (nil? (some->> nil))) + (t/assert (= 0 (some->> 0))) + (t/assert (= 1 (some->> 1 (- 2)))) + (t/assert (nil? (some->> 1 constantly-nil (- 2))))) + +(t/deftest cond->test + (t/assert (= 0 (cond-> 0))) + (t/assert (= -1 (cond-> 0 true inc true (- 2)))) + (t/assert (= 0 (cond-> 0 false inc))) + (t/assert (= -1 (cond-> 1 true (- 2) false inc)))) + +(t/deftest cond->>test + (t/assert (= 0 (cond->> 0))) + (t/assert (= 1 (cond->> 0 true inc true (- 2)))) + (t/assert (= 0 (cond->> 0 false inc))) + (t/assert (= 1 (cond->> 1 true (- 2) false inc)))) + +(t/deftest as->test + (t/assert (= 0 (as-> 0 x))) + (t/assert (= 1 (as-> 0 x (inc x)))) + (t/assert (= 2 (as-> [0 1] x + (map inc x) + (reverse x) + (first x))))) + +(t/deftest threading-loop-recur + (t/assert (nil? (loop [] + (as-> 0 x + (when-not (zero? x) + (recur)))))) + (t/assert (nil? (loop [x nil] (some-> x recur)))) + (t/assert (nil? (loop [x nil] (some->> x recur)))) + (t/assert (= 0 (loop [x 0] (cond-> x false recur)))) + (t/assert (= 0 (loop [x 0] (cond->> x false recur))))) diff --git a/tests/pixie/tests/test-numbers.pxi b/tests/pixie/tests/test-numbers.pxi new file mode 100644 index 00000000..270ba742 --- /dev/null +++ b/tests/pixie/tests/test-numbers.pxi @@ -0,0 +1,79 @@ +(ns pixie.tests.test-numbers + (require pixie.test :as t)) + +(t/deftest integer-literals + (t/assert= 0xa 10) + (t/assert= -0xa -10) + (t/assert= 012 10) + (t/assert= -012 -10) + (t/assert= 2r1010 10) + (t/assert= -2r1010 -10)) + +(t/deftest float-literals + (t/assert= 10. 10.0) + (t/assert= -10. -10.0) + (t/assert= 1e1 10.0) + (t/assert= -1e1 -10.0) + (t/assert= 1e-1 0.1) + (t/assert= -1e-1 -0.1)) + +(t/deftest mixed-float-ops + (t/assert= (+ 1/2 0.5) 1.0) + (t/assert= (+ 0 1.0) 1.0)) + +(t/deftest ratio-literals + (t/assert= 3/4 (/ 3 4)) + (t/assert= -3/4 (/ -3 4)) + (t/assert= 6/8 3/4) + (t/assert= 9/12 3/4) + (t/assert= 3/1 3)) + +(t/deftest ratio-from-divide + (t/assert= (/ 3 4) 3/4)) + +(t/deftest ratio-ops + (t/assert= (+ 1/2 1/2) 1) + (t/assert= (- 1/2 1/2) 0) + (t/assert= (* 1/2 1/2) 1/4) + (t/assert= (/ 1/2 1/2) 1)) + +(t/deftest ratio-accessors + (doseq [[r n d] [[3/2 3 2] [1/9 1 9] [-3/89 -3 89]]] + (t/assert= (numerator r) n) + (t/assert= (denominator r) d))) + +(t/deftest test-int + (doseq [[x i] [[1 1] [3.0 3] [3.5 3] [3.999 3] [3/2 1]]] + (t/assert= (int x) i))) + +(t/deftest test-float + (doseq [[x f] [[1 1.0] [3 3.0] [3.333 3.333] [3/2 1.5] [1/7 (/ 1.0 7.0)]]] + (t/assert= (float x) f))) + +(t/deftest rem-types + (t/assert= Integer (type (rem 5 3))) + (t/assert= Float (type (rem 5.0 3))) + (t/assert= Ratio (type (rem 7/2 3))) + (t/assert= Float (type (rem 7/2 3.0)))) + +(t/deftest quot-types + (t/assert= Integer (type (quot 5 3))) + (t/assert= Float (type (quot 5.0 3))) + (t/assert= Integer (type (quot 7/2 3/7))) + (t/assert= Float (type (quot 7/2 3.0)))) + +(t/deftest test-big-int-eq + (t/assert (-num-eq 1N 1)) + (t/assert (-num-eq 1 1N)) + ;(t/assert= 1N 1) this fails, should it? + (t/assert= 1 1N)) + +(t/deftest test-promotion + (t/assert= BigInteger (type (reduce * 1 (range 1 100)))) + (t/assert (-num-eq 1000000000000000000000N (* 10000000 + 10000000 + 10000000)))) + +(t/deftest test-wrap-around + (t/assert= Integer (type (unchecked-add 9223372036854775807 1))) + (t/assert (-num-eq -9223372036854775808 (unchecked-add 9223372036854775807 1)))) diff --git a/tests/pixie/tests/test-object.pxi b/tests/pixie/tests/test-object.pxi new file mode 100644 index 00000000..3a805042 --- /dev/null +++ b/tests/pixie/tests/test-object.pxi @@ -0,0 +1,15 @@ +(ns pixie.tests.test-object + (require pixie.test :as t)) + +(t/deftest test-hash + (t/assert= (hash (var foo)) (hash (var foo))) + (t/assert (not= (hash (var foo)) (hash (var bar))))) + + +(deftype FooType []) + +(t/deftest test-everything-is-an-object + (t/assert (instance? Object 42)) + (t/assert (instance? Object [])) + (t/assert (instance? Object "Foo")) + (t/assert (instance? Object FooType))) diff --git a/tests/pixie/tests/test-parser.pxi b/tests/pixie/tests/test-parser.pxi new file mode 100644 index 00000000..af21a1b7 --- /dev/null +++ b/tests/pixie/tests/test-parser.pxi @@ -0,0 +1,46 @@ +(ns pixie.tests.test-parser + (:require [pixie.test :refer :all] + [pixie.parser :refer :all])) + +(deftest test-and + (let [p (parser [] + ENTRY (and \a \b <- [\a \b])) + c (string-cursor "abc")] + (assert= ((:ENTRY p) c) [\a \b]) + (assert= (current c) \c) + (assert= 2 (snapshot c)))) + +(deftest test-or + (let [p (parser [] + ENTRY (or \b \a)) + c (string-cursor "abc")] + (assert= ((:ENTRY p) c) \a) + (assert= (current c) \b) + (assert= 1 (snapshot c)))) + + +(defparser as-and-bs [] + ENTRY (and S -> value + end + <- value) + S (one+ AB) <- `[:S ~@value] + AB (and A -> a + B -> b + <- [:AB a b]) + A (one+ \a) <- `[:A ~@value] + B (one+ \b) <- `[:B ~@value]) + +(deftest test-as-and-bs + (let [c (string-cursor "aabbaa")] + (assert= ((:S as-and-bs) c) [:S [:AB + [:A \a \a] + [:B \b \b]]]) + (assert (not (at-end? c))) + (assert= (snapshot c) 4) + + (assert (failure? ((:S as-and-bs) c)))) + + (let [c (string-cursor "aabbaa")] + (assert (failure? ((:ENTRY as-and-bs) c)) ) + (assert (not (at-end? c))) + (assert= (snapshot c) 0))) diff --git a/tests/pixie/tests/test-readeval.pxi b/tests/pixie/tests/test-readeval.pxi new file mode 100644 index 00000000..636dca23 --- /dev/null +++ b/tests/pixie/tests/test-readeval.pxi @@ -0,0 +1,73 @@ +(ns pixie.tests.test-readeval + (require pixie.test :as t)) + +(t/deftest test-read + (t/assert= (read-string "0xDEADBEEF") 3735928559) + (t/assert= (read-string "0xDeadBeef") 3735928559) + (t/assert= (read-string "0xdeadbeef") 3735928559) + (t/assert= (read-string "foo") 'foo) + (t/assert= (read-string "()") '()) + (t/assert= (read-string "(1 2 3)") '(1 2 3)) + (t/assert= (read-string "[1 2 3]") [1 2 3]) + (t/assert= (read-string "{:a 1 :b 2 :c 3}") {:a 1 :b 2 :c 3}) + (t/assert= (read-string "\"foo\"") "foo") + (t/assert= (read-string "\"fo\\\\o\"") "fo\\o") + (t/assert= (read-string "false") false) + (t/assert= (read-string "true") true) + (t/assert= (read-string "#{1 2 3}") #{1 2 3}) + (t/assert= (read-string "(foo (bar (baz)))") '(foo (bar (baz))))) + +(t/deftest test-list-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched list open '('" + (read-string "(")) + (t/assert-throws? RuntimeException + "Unmatched list open '('" + (read-string "((foo bar)"))) + +(t/deftest test-vector-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched vector open '['" + (read-string "[")) + (t/assert-throws? RuntimeException + "Unmatched vector open '['" + (read-string "[[foo bar]"))) + +(t/deftest test-map-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched map open '{'" + (read-string "{")) + (t/assert-throws? RuntimeException + "Unmatched map open '{'" + (read-string "{foo {a b}"))) + +(t/deftest test-set-unclosed-list-fail + (t/assert-throws? RuntimeException + "Unmatched set open '#{'" + (read-string "#{")) + (t/assert-throws? RuntimeException + "Unmatched set open '#{'" + (read-string "#{foo #{a}"))) + +(t/deftest test-string-unclosed-fail + (t/assert-throws? RuntimeException + "Unmatched string quote '\"'" + (read-string "\"")) + (t/assert-throws? RuntimeException + "Unmatched string quote '\"'" + (read-string "\"foo"))) + +(t/deftest test-comments-in-forms + (t/assert= (read-string "(foo ; a comment\n )") '(foo)) + (t/assert= (read-string "[foo ; a comment\n ]") '[foo]) + (t/assert= (read-string "{:foo :bar ; a comment\n }") '{:foo :bar}) + (t/assert= (read-string "#{:foo ; a comment\n }") '#{:foo}) + (t/assert= (read-string "{:foo ; a comment\n :bar }") '{:foo :bar})) + +(t/deftest test-comment-reader-macro + (t/assert= (read-string "(foo #_bar baz)") '(foo baz)) + (t/assert= (read-string "(foo #_ bar baz)") '(foo baz)) + (t/assert= (read-string "(foo #_ #_ bar baz)") '(foo)) + (t/assert= (read-string "(foo #_(bar goo) baz)") '(foo baz)) + (t/assert= (read-string "(foo bar #_baz)") '(foo bar)) + (t/assert= (read-string "(foo bar #_ ; comment \n baz)") '(foo bar))) diff --git a/tests/pixie/tests/test-sets.pxi b/tests/pixie/tests/test-sets.pxi new file mode 100644 index 00000000..bccc8041 --- /dev/null +++ b/tests/pixie/tests/test-sets.pxi @@ -0,0 +1,69 @@ +(ns pixie.test.test-sets + (require pixie.test :as t) + (require pixie.set :as s)) + +(let [magic #{:bibbidi :bobbidi :boo} + work #{:got :no :time :to :dilly-dally}] + + (t/deftest test-union + (let [dreams #{:the :dreams :that :i :wish} + magical-work #{:bibbidi :bobbidi :boo + :got :no :time :to :dilly-dally} + dream-of-magical-work #{:bibbidi :bobbidi :boo + :got :no :time :to :dilly-dally + :the :dreams :that :i :wish}] + (t/assert= (s/union) #{}) + (t/assert= (s/union magic) magic) + (t/assert= (s/union magic magic) magic) + (t/assert= (s/union magic magic magic) magic) + (t/assert= (s/union magic work) magical-work) + (t/assert= (s/union work magic) magical-work) + (t/assert= (s/union magic work dreams) dream-of-magical-work) + (t/assert-throws? (s/union [:i :only] [:love :sets])))) + + (t/deftest test-difference + (t/assert= (s/difference) #{}) + (t/assert= (s/difference magic) magic) + (t/assert= (s/difference magic #{:boo}) #{:bibbidi :bobbidi}) + (t/assert= (s/difference magic work) magic) + (t/assert= (s/difference magic #{:bibbidi} #{:bobbidi}) #{:boo}) + (t/assert-throws? (s/difference [:i :only] [:love :sets]))) + + (t/deftest test-intersection + (t/assert= (s/intersection) #{}) + (t/assert= (s/intersection magic) magic) + (t/assert= (s/intersection magic magic) magic) + (t/assert= (s/intersection magic work) #{}) + (t/assert= (s/intersection magic #{:boo}) #{:boo}) + (t/assert= (s/intersection #{:boo} magic) #{:boo}) + (t/assert= (s/intersection magic #{:bobbidi :boo}) #{:bobbidi :boo}) + (t/assert= (s/intersection magic #{:bibbidi :boo} #{:bobbidi :boo}) #{:boo}) + (t/assert-throws? (s/intersection [:i :only] [:love :sets]))) + + (t/deftest test-subset? + (t/assert (not (s/subset? magic work))) + (t/assert (s/subset? magic magic)) + (t/assert (not (s/subset? magic #{:foo}))) + (t/assert (s/subset? #{:boo} magic)) + (t/assert-throws? (s/subset? [:i :only] [:love :sets]))) + + (t/deftest test-strict-subset? + (t/assert (not (s/strict-subset? magic work))) + (t/assert (not (s/strict-subset? magic magic))) + (t/assert (not (s/strict-subset? magic #{:foo}))) + (t/assert (s/strict-subset? #{:boo} magic)) + (t/assert-throws? (s/strict-subset? [:i :only] [:love :sets]))) + + (t/deftest test-superset? + (t/assert (not (s/superset? magic work))) + (t/assert (s/superset? magic magic)) + (t/assert (not (s/superset? #{:foo} magic))) + (t/assert (s/superset? magic #{:boo})) + (t/assert-throws? (s/superset? [:i :only] [:love :sets]))) + + (t/deftest test-strict-superset? + (t/assert (not (s/strict-superset? magic work))) + (t/assert (not (s/strict-superset? magic magic))) + (t/assert (not (s/strict-superset? #{:foo} magic))) + (t/assert (s/strict-superset? magic #{:boo})) + (t/assert-throws? (s/strict-superset? [:i :only] [:love :sets])))) diff --git a/tests/pixie/tests/test-stdlib.pxi b/tests/pixie/tests/test-stdlib.pxi new file mode 100644 index 00000000..9b2174e0 --- /dev/null +++ b/tests/pixie/tests/test-stdlib.pxi @@ -0,0 +1,844 @@ +(ns pixie.tests.test-stdlib + (require pixie.test :as t)) + +(t/deftest test-identity + (let [vs [nil true false [1 2 3] #{1 2 3} :oops]] + (doseq [v vs] + (t/assert= (identity v) v)))) + +(t/deftest test-map + (t/assert= (map inc [1 2 3]) [2 3 4]) + (t/assert= (map + [1 2 3] [4 5 6]) [5 7 9]) + (t/assert= (map + [1 2 3] [4 5 6] [7 8 9]) [12 15 18]) + (t/assert= (map + [1 2 3] (repeat 7)) [8 9 10]) + (let [value (map identity [1 2 3])] + (t/assert= (seq value) [1 2 3]) + (t/assert= (seq value) [1 2 3]))) + +(t/deftest test-mapcat + (t/assert= (mapcat identity []) []) + (t/assert= (mapcat first [[[1 2]] [[3] [:not :present]] [[4 5 6]]]) [1 2 3 4 5 6])) + +(t/deftest test-indexed + (t/assert= (map-indexed (fn [& xs] xs) []) []) + (t/assert= (map-indexed (fn [& xs] xs) [:a :b]) [[0 :a] [1 :b]]) + (t/assert= (transduce (map-indexed (fn [& xs] xs)) conj [:a :b]) [[0 :a] [1 :b]]) + + (t/assert= (keep-indexed (constantly true) []) []) + (t/assert= (keep-indexed (constantly nil) []) []) + (t/assert= (keep-indexed (fn [i x] [i x]) [:a :b]) [[0 :a] [1 :b]]) + + (t/assert= (transduce (keep-indexed (constantly true)) conj [:a :b]) [true true]) + (t/assert= (transduce (keep-indexed (constantly nil)) conj [:a :b]) []) + (t/assert= (transduce (keep-indexed (fn [i x] [i x])) conj [:a :b]) [[0 :a] [1 :b]]) + + (let [even-index? (fn [i x] + (when (even? i) + x))] + (t/assert= (transduce (keep-indexed even-index?) conj [:a :b :c :d]) + [:a :c]) + (t/assert= (transduce (map-indexed even-index?) conj [:a :b :c :d]) + [:a nil :c nil]) + + (t/assert= (transduce (comp (keep-indexed even-index?) + (take 3)) + conj + [:a :b :c :d :e :f]) + [:a :c :e]) + (t/assert= (transduce (comp (map-indexed even-index?) + (take 3)) + conj + [:a :b :c :d]) + [:a nil :c]))) + +(t/deftest test-reductions + (t/assert= (reductions + nil) + [0]) + (t/assert= (reductions + [1 2 3 4 5]) + [1 3 6 10 15]) + (t/assert= (reductions + 10 [1 2 3 4 5]) + [10 11 13 16 20 25])) + +(t/deftest test-str + (t/assert= (str nil) "nil") + (t/assert= (str true) "true") + (t/assert= (str false) "false") + (t/assert= (str \a) "a") + (t/assert= (str \u269b) "โš›") + (t/assert= (str "hey") "hey") + (t/assert= (str :hey) ":hey") + (t/assert= (str 'hey) "hey") + + (t/assert= (str '()) "()") + (t/assert= (str '(1 2 3)) "(1 2 3)") + (t/assert= (str [1 2 3]) "[1 2 3]") + (t/assert= (str #{1}) "#{1}") + (t/assert= (str {}) "{}") + (t/assert= (str {:a 1}) "{:a 1}") + (t/assert= (str (type 3)) "") + (t/assert= (str [1 {:a 1} "hey"]) "[1 {:a 1} hey]") + (t/assert= (seq (map identity "iterable")) '(\i \t \e \r \a \b \l \e)) + (t/assert= (str (repeat 3 7)) "(7 7 7)") + (t/assert= (str (range 3)) "(0 1 2)")) + +(t/deftest test-repr + (t/assert= (-repr nil) "nil") + (t/assert= (-repr true) "true") + (t/assert= (-repr false) "false") + (t/assert= (-repr \a) "\\a") + (t/assert= (-repr \u00fa) "\\u00fa") + (t/assert= (-repr \u0fab) "\\u0fab") + (t/assert= (-repr \u269b) "\\u269b") + (t/assert= (-repr "hey") "\"hey\"") + (t/assert= (-repr :hey) ":hey") + (t/assert= (-repr 'hey) "hey") + + (t/assert= (-repr '()) "()") + (t/assert= (-repr '(1 2 3)) "(1 2 3)") + (t/assert= (-repr [1 2 3]) "[1 2 3]") + (t/assert= (-repr #{1}) "#{1}") + (t/assert= (-repr {}) "{}") + (t/assert= (-repr {:a 1}) "{:a 1}") + (t/assert= (-repr (type 3)) "pixie.stdlib.Integer") + + (t/assert= (-repr [1 {:a 1} "hey"]) "[1 {:a 1} \"hey\"]") + (t/assert= (-repr (repeat 3 7)) "(7 7 7)") + (t/assert= (-repr (range 3)) "(0 1 2)")) + +(t/deftest test-nth + ;; works if the index is found + (t/assert= (nth [1 2 3] 1) 2) + (t/assert= (nth '(1 2 3) 1) 2) + (t/assert= (nth (make-array 3) 2) nil) + (t/assert= (nth (repeat 4 1) 3) 1) + (t/assert= (nth (range 4) 1) 1) + (t/assert= (nth "hithere" 1) \i) + + ;; throws error for bad index + (try + (nth [1 2 3] 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth '(1 2 3) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (make-array 3) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (range 4) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth "hithere" 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth [] 0) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth '() 0) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (repeat 99 nil) 99) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + (try + (nth (range 0 0) 0) + (catch ex (t/assert= (ex-msg ex) "Index out of Range"))) + + ;; if not-found is specified, uses that for out of range + (t/assert= (nth [1 2 3] 99 :default) :default) + (t/assert= (nth '(1 2 3) 99 :default) :default) + (t/assert= (nth (make-array 3) 99 :default) :default) + (t/assert= (nth (repeat 4 1) 99 :default) :default) + (t/assert= (nth (range 4) 99 :default) :default) + (t/assert= (nth "hithere" 99 :default) :default) + + (t/assert= (nth [1 2 3] 1 :default) 2) + (t/assert= (nth '(1 2 3) 1 :default) 2) + (t/assert= (nth (make-array 3) 2 :default) nil) + (t/assert= (nth (repeat 4 1) 3 :default) 1) + (t/assert= (nth (range 4) 1 :default) 1) + (t/assert= (nth "hithere" 1 :deafult) \i)) + +(t/deftest test-get-on-vector + (t/assert= (get [1 2 3] 0) 1) + (t/assert= (get [1 2 3] 1) 2) + (t/assert= (get [1 2 3] 2) 3) + (t/assert= (get [1 2 3] 4) nil) + (t/assert= (get [1 2 3] 5) nil) + (t/assert= (get [1 2 3] 5 :not-found) :not-found)) + +(t/deftest test-first + (t/assert= (first []) nil) + (t/assert= (first '()) nil) + (t/assert= (first (make-array 0)) nil) + (t/assert= (first {}) nil) + (t/assert= (first #{}) nil) + + (t/assert= (first {:a 1}) (map-entry :a 1)) + (t/assert= (first #{:a}) :a) + + (t/assert= (first [1 2 3]) 1) + (t/assert= (first '(1 2 3)) 1) + (let [a (make-array 3)] + (aset a 0 1) + (aset a 1 2) + (aset a 2 3) + (t/assert= (first a) 1))) + +(t/deftest test-last + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6)] + (t/assert= (last nil) nil) + (t/assert= (last []) nil) + (t/assert= (last (repeat 3 nil)) nil) + (t/assert= (last (range 0 0)) nil) + (t/assert= (last v) 5) + (t/assert= (last l) 5) + (t/assert= (last r) 5))) + +(t/deftest test-butlast + (let [v [1 2 3 4 5] + l '(1 2 3 4 5) + r (range 1 6) + res '(1 2 3 4)] + (t/assert= (butlast nil) nil) + (t/assert= (butlast []) nil) + (t/assert= (butlast (range 0 0)) nil) + (t/assert= (butlast v) res) + (t/assert= (butlast l) res) + (t/assert= (butlast r) res))) + +(t/deftest test-empty? + (t/assert= (empty? []) true) + (t/assert= (empty? '()) true) + (t/assert= (empty? (make-array 0)) true) + (t/assert= (empty? {}) true) + (t/assert= (empty? #{}) true) + (t/assert= (empty? (repeat '())) false) + (t/assert= (empty? (range 1 5)) false) + + (t/assert= (empty? [1 2 3]) false) + (t/assert= (empty? '(1 2 3)) false) + (let [a (make-array 1)] + (aset a 0 1) + (t/assert= (empty? a) false)) + (t/assert= (empty? {:a 1}) false) + (t/assert= (empty? #{:a :b}) false)) + +(t/deftest test-not-empty? + (t/assert= (not-empty? []) false) + (t/assert= (not-empty? '()) false) + (t/assert= (not-empty? (make-array 0)) false) + (t/assert= (not-empty? {}) false) + (t/assert= (not-empty? #{}) false) + (t/assert= (not-empty? (repeat 0 'x)) false) + (t/assert= (not-empty? (range 1 5)) true) + + (t/assert= (not-empty? [1 2 3]) true) + (t/assert= (not-empty? '(1 2 3)) true) + (let [a (make-array 1)] + (aset a 0 1) + (t/assert= (not-empty? a) true)) + (t/assert= (not-empty? {:a 1}) true) + (t/assert= (not-empty? #{:a :b}) true)) + +(t/deftest test-keys + (let [v {:a 1 :b 2 :c 3}] + (t/assert= (set (keys v)) #{:a :b :c}) + (t/assert= (transduce (keys) conj! v) (keys v)))) + +(t/deftest test-vals + (let [v {:a 1 :b 2 :c 3}] + (t/assert= (set (vals v)) #{1 2 3}) + (t/assert= (transduce (vals) conj! v) (vals v)))) + +(t/deftest test-select-keys + (let [m ^{:k :v} {:a 1 :b 2}] + (t/assert= (select-keys m [:a :b]) m) + (t/assert= :v + (-> (select-keys m [:a]) + meta + :k)) + (t/assert= (select-keys m [:a :not-found]) {:a 1}) + (t/assert= (select-keys m nil) {}) + (t/assert= (select-keys {} [:a]) {}))) + +(t/deftest test-empty + (t/assert= (empty '(1 2 3)) '()) + (t/assert= (empty (list 1 2 3)) '()) + (t/assert= (empty (lazy-seq)) '()) + (t/assert= (empty '()) '()) + (t/assert= (empty [1 2 3]) []) + (t/assert= (empty (make-array 3)) (make-array 0)) + (t/assert= (empty {:a 1, :b 2, :c 3}) {}) + (t/assert= (empty #{1 2 3}) #{})) + +(t/deftest test-count + (t/assert= (count nil) 0) + (t/assert= (count '()) 0) + (t/assert= (count '(1 2 '(3 4))) 3) + (t/assert= (count '(nil nil nil)) 3) + (t/assert= (count (cons 1 [2 3])) 3) + (t/assert= (count (list)) 0) + (t/assert= (count (list 1 2 3)) 3) + (t/assert= (count []) 0) + (t/assert= (count [1 2 3]) 3) + (t/assert= (count {}) 0) + (t/assert= (count {:a 1, :b 2, :c 3}) 3) + (t/assert= (count (first (seq {:a 1, :b 2, :c 3}))) 2) + (t/assert= (count #{}) 0) + (t/assert= (count (conj #{1 2 3} 3)) 3) + (t/assert= (count (lazy-seq '())) 0) + (t/assert= (count (lazy-seq '(1 2 3))) 3) + (t/assert= (count (cons 1 (lazy-seq [2 3]))) 3) + (t/assert= (count (range 0 -9 -3)) 3) + (t/assert= (count "") 0) + (t/assert= (count "123") 3) + (t/assert= (count (make-array 3)) 3)) + +(t/deftest test-vec + (let [v '(1 2 3 4 5)] + (t/assert= (vec v) [1 2 3 4 5]) + (t/assert= (vec (map inc) v) [2 3 4 5 6]))) + + +(t/deftest test-keep + (let [v [-1 0 1 2 3 4 5]] + (t/assert= (vec (keep pos?) v) [true true true true true]) + (t/assert= (vec (keep pos? v)) (vec (keep pos?) v)))) + +(t/deftest test-assoc + (t/assert= (assoc {} :a 3) {:a 3}) + (t/assert= (assoc {:a 1} :a 3) {:a 3}) + + (t/assert= (assoc [] 0 :ok) [:ok]) + (t/assert= (assoc [1] 0 :ok) [:ok]) + (t/assert= (assoc [1 2 3] 1 :ok) [1 :ok 3])) + +(t/deftest test-get-in + (let [m {:a 1 :b 2 :x {:a 2 :x [1 2 3]}}] + (t/assert= (get-in m [:a]) 1) + (t/assert= (get-in m [:missing]) nil) + (t/assert= (get-in m [:missing] :not-found) :not-found) + (t/assert= (get-in m [:x :x 0] :not-found) 1))) + +(t/deftest test-assoc-in + (t/assert= (assoc-in {:a {:b 2}} [:a :b] 3) {:a {:b 3}}) + (t/assert= (assoc-in {:a [{:b 2}]} [:a 0 :b] 3) {:a [{:b 3}]}) + ; non existing keys create maps (not vectors, even if the keys are integers) + (t/assert= (assoc-in {} [:a :b] 3) {:a {:b 3}}) + (t/assert= (assoc-in {} [:a 0 :b] 3) {:a {0 {:b 3}}})) + +(t/deftest test-update-in + (t/assert= (update-in {} [:a :b] (fnil inc 0)) {:a {:b 1}}) + (t/assert= (update-in {:a {:b 2}} [:a :b] inc) {:a {:b 3}}) + (t/assert= (update-in {:a [{:b 2}]} [:a 0 :b] inc) {:a [{:b 3}]})) + +(t/deftest test-fn? + (t/assert= (fn? inc) true) + (t/assert= (fn? {}) true) + (t/assert= (fn? #(%)) true) + (t/assert= (fn? :foo) true) + (t/assert= (fn? 1) false) + (t/assert= (fn? and) true) + (t/assert= (fn? "foo") false) + (t/assert= (fn? (let [x 8] (fn [y] (+ x y)))) true)) + +(t/deftest test-coll? + (t/assert= (coll? '()) true) + (t/assert= (coll? []) true) + (t/assert= (coll? {:foo "bar"}) true) + (t/assert= (coll? #{:foo :bar}) true) + (t/assert= (coll? #(%)) false) + (t/assert= (coll? :foo) false) + (t/assert= (coll? "foo") false) + (t/assert= (coll? 1) false)) + +(t/deftest test-macro? + (t/assert= (macro? and) true) + (t/assert= (macro? or) true) + (t/assert= (macro? defn) true) + (t/assert= (macro? inc) false) + (t/assert= (macro? 1) false) + (t/assert= (macro? :foo) false) + (t/assert= (macro? "foo") false)) + +(def ^:dynamic *earmuffiness* :low) + +(t/deftest test-binding + (t/assert= *earmuffiness* :low) + (binding [*earmuffiness* :quite-high] + (t/assert= *earmuffiness* :quite-high)) + (t/assert= *earmuffiness* :low)) + +(t/deftest test-every? + (t/assert= (every? even? [2 4 6 8]) true) + (t/assert= (every? odd? [2 4 6 8]) false) + (t/assert= (every? even? [2 3 6 8]) false) + (t/assert= (every? even? []) true) + (t/assert= (every? odd? []) true)) + +(t/deftest test-rand-int + (let [vs (repeatedly 10 #(rand-int 4))] + (t/assert (every? #(and (>= % 0) (< % 4)) vs))) + (let [vs (repeatedly 10 #(rand-int 0))] + (t/assert (every? zero? vs)))) + +(t/deftest test-some + (t/assert= (some even? [2 4 6 8]) true) + (t/assert= (some odd? [2 4 6 8]) false) + (t/assert= (some even? [2 3 6 8]) true) + (t/assert= (some even? [1 3 5 8]) true) + (t/assert= (some even? []) false) + (t/assert= (some odd? [2]) false) + (t/assert= (some #{:x :y} [:a :b :x :y]) :x) + (t/assert= (some #{:x :y} [:a :b :c :y]) :y) + (t/assert= (some #{:x :y} [:a :b :c :d]) false)) + +(t/deftest test-filter + (t/assert= (vec (filter (fn [x] true) [])) []) + (t/assert= (vec (filter (fn [x] false) [])) []) + (t/assert= (vec (filter (fn [x] true) [1 2 3 4])) [1 2 3 4]) + (t/assert= (vec (filter (fn [x] false) [1 2 3 4])) []) + (t/assert= (vec (filter (fn [x] true)) + [1 2 3 4]) + [1 2 3 4]) + (t/assert= (vec (filter (fn [x] false)) + [1 2 3 4]) + []) + (t/assert= (seq (filter (fn [x] true) [])) nil) + (t/assert= (seq (filter (fn [x] false) [])) nil) + (t/assert= (seq (filter (fn [x] true) [1 2 3 4])) '(1 2 3 4)) + (t/assert= (seq (filter (fn [x] false) [1 2 3 4])) nil) + (t/assert= (into {} (filter (fn [[_ v]] (odd? v)) {:a 1, :b 2, :c 3, :d 4})) + {:a 1 :c 3})) + +(t/deftest test-remove + (t/assert= (remove even? [1 2 3 4 5]) '(1 3 5))) + +(t/deftest test-distinct + (t/assert= (seq (distinct [1 2 3 2 1])) '(1 2 3)) + (t/assert= (vec (distinct) [1 1 2 2 3 3]) [1 2 3]) + (t/assert= (vec (distinct) [nil nil nil]) [nil])) + +(t/deftest test-merge + (t/assert= (merge nil) nil) + (t/assert= (merge {}) {}) + (t/assert= (merge {:a 1} nil) {:a 1}) + + (t/assert= (merge {} {:a 1, :b 2}) {:a 1, :b 2}) + (t/assert= (merge {:a 1} {:b 2}) {:a 1, :b 2}) + (t/assert= (merge {} {:a 1} {:b 2}) {:a 1, :b 2}) + + (t/assert= (merge {:a 1} {:a 2, :b 3}) {:a 2, :b 3}) + (t/assert= (merge {:a 1, :b 4} {:a 2} {:a 3}) {:a 3, :b 4})) + +(t/deftest test-merge-with + (t/assert= (merge-with identity nil) nil) + (t/assert= (merge-with identity {}) {}) + + (t/assert= (merge-with identity {} {:a 1, :b 2}) {:a 1, :b 2}) + (t/assert= (merge-with identity {:a 1} {:b 2}) {:a 1, :b 2}) + + (t/assert= (merge-with (fn [a b] a) {:a 1} {:a 2}) {:a 1}) + (t/assert= (merge-with (fn [a b] a) {:a 1} {:a 2} {:a 3}) {:a 1}) + (t/assert= (merge-with (fn [a b] b) {:a 1} {:a 2}) {:a 2}) + + (t/assert= (merge-with + {:a 21} {:a 21}) {:a 42}) + (t/assert= (merge-with + {:a 21} {:a 21, :b 1}) {:a 42, :b 1})) + +(t/deftest test-for + (t/assert= (for [x [1 2 3]] x) [1 2 3]) + (t/assert= (for [x [1 2 3] y [:a :b :c]] [x y]) + [[1 :a] [1 :b] [1 :c] + [2 :a] [2 :b] [2 :c] + [3 :a] [3 :b] [3 :c]]) + (t/assert= (for [x [[1 2 3]] + y x] + y) + [1 2 3])) + +(t/deftest test-doto + (let [a (atom 0)] + (t/assert= a (doto a (swap! + 3) (swap! str))) + (t/assert= @a "3"))) + +(t/deftest test-into + (t/assert= [1 3] (into [] (comp (map inc) (filter odd?)) (range 3))) + (t/assert= {:a 1 :b 2} (into {} [[:a 1] [:b 2]]))) + +(t/deftest test-ex-msg + (try + (throw [::something "This is an exception"]) + (catch e + (t/assert= (ex-msg e) "This is an exception") + (t/assert= (ex-data e) ::something)))) + +(t/deftest test-ex-filtering + (let [f (fn [val] + (try + (try + (throw [val "Some failure"]) + (catch ::catch-this ex + :found)) + (catch ex + :not-found)))] + (t/assert= (f ::catch-this) :found) + (t/assert= (f :something-else) :not-found)) + + (let [f (fn [val] + (try + (try + (throw [val "Some failure"]) + (catch (= ::catch-this (ex-data ex)) ex + :found)) + (catch ex + :not-found)))] + (t/assert= (f ::catch-this) :found) + (t/assert= (f :something-else) :not-found))) + +(t/deftest test-repeat + (t/assert= (seq (repeat 3 1)) '(1 1 1)) + (t/assert= (seq (repeat 0 1)) nil) + (t/assert= (count (repeat 4096 'x)) 4096) + (t/assert= (count (repeat 0 'x)) 0)) + +(t/deftest test-range + (t/assert= (= (-seq (range 10)) + '(0 1 2 3 4 5 6 7 8 9)) + true)) + +(t/deftest test-ns + ;; Create a namespace called foo + (in-ns :foo) + (def bar :humbug) + (defn baz [x y] (+ x y)) + ;; Back into the text namespace + (in-ns :pixie.tests.test-stdlib) + (t/assert= (set (keys (ns-map 'foo))) + #{'bar 'baz})) + +(t/deftest test-ns-aliases + (in-ns :ns-to-require) + (in-ns :my-fake-ns) + (require ns-to-require :as some-alias) + (in-ns :pixie.tests.test-stdlib) + (t/assert= {'some-alias (the-ns 'ns-to-require) + 'pixie.stdlib (the-ns 'pixie.stdlib)} + (ns-aliases (the-ns 'my-fake-ns)))) + +(t/deftest test-while + (t/assert= (while (pos? 0) true ) nil) + (t/assert= (while (pos? 0) false) nil) + (t/assert= 0 (let [x (atom 10) + cnt (atom 0)] + (while (pos? @x) + (do (swap! x dec) + (swap! cnt inc))) + @x)) + (t/assert= 10 (let [x (atom 10) + cnt (atom 0)] + (while (pos? @x) + (do (swap! x dec) + (swap! cnt inc))) + @cnt))) + +(t/deftest test-loop + (t/assert= + [3 -3] + (loop [[a b :as vs] [0 0]] + (if (> a 2) + vs + (recur [(inc a) (dec b)])))) + (t/assert= + 3 + (loop [a 1] + (if (> a 2) + a + (recur (inc a)))))) + +(t/deftest test-take + (t/assert= (take 0 [1 2 3 4]) ()) + (t/assert= (take 1 [1 2 3 4]) [1]) + (t/assert= (take 2 [1 2 3 4]) [1 2]) + (t/assert= (take 3 [1 2 3 4]) [1 2 3]) + (t/assert= (transduce (take 0) conj [1 2 3 4]) []) + (t/assert= (transduce (take 1) conj [1 2 3 4]) [1]) + (t/assert= (transduce (take 2) conj [1 2 3 4]) [1 2]) + (t/assert= (transduce (take 3) conj [1 2 3 4]) [1 2 3]) + (t/assert= (transduce (take 10) conj [1 2 3 4]) [1 2 3 4]) + (t/assert= (transduce (comp (take 2) (take 1)) conj [1 2 3 4]) [1]) + (t/assert= (transduce (comp (take 1) (take 2)) conj [1 2 3 4]) [1]) + + (let [call-count (atom 0) + inc-call-count! (fn [x] + (swap! call-count inc) + x)] + (t/assert= (transduce (comp (map inc-call-count!) (take 2)) conj (range 10)) + [0 1]) + (t/assert= @call-count 2) + (t/assert= (transduce (comp (take 2) (map inc-call-count!)) conj (range 10)) + [0 1]) + (t/assert= @call-count 4))) + +(t/deftest test-drop + (t/assert= (drop 0 [1 2 3 4]) [1 2 3 4]) + (t/assert= (drop 1 [1 2 3 4]) [2 3 4]) + (t/assert= (drop 2 [1 2 3 4]) [3 4]) + (t/assert= (drop 3 [1 2 3 4]) [4]) + (t/assert= (transduce (drop 0) conj [1 2 3 4]) [1 2 3 4]) + (t/assert= (transduce (drop 1) conj [1 2 3 4]) [2 3 4]) + (t/assert= (transduce (drop 2) conj [1 2 3 4]) [3 4]) + (t/assert= (transduce (drop 3) conj [1 2 3 4]) [4]) + (t/assert= (transduce (drop 10) conj [1 2 3 4]) []) + (t/assert= (transduce (comp (drop 1) (take 2)) conj [1 2 3 4]) [2 3]) + (t/assert= (transduce (comp (take 2) (drop 1)) conj [1 2 3 4]) [2])) + +(t/deftest test-take-while + (t/assert= (take-while pos? [1 2 3 -1]) [1 2 3]) + (t/assert= (take-while pos? [-1 2]) ()) + (t/assert= (transduce (take-while even?) conj [2 4 6 7 8]) [2 4 6]) + (t/assert= (transduce (take-while even?) conj [0 2] [1 4 6]) [0 2]) + (t/assert= (transduce (take-while even?) conj [1 3] [2 4 6 7 8]) [1 3 2 4 6]) + (t/assert= (transduce (comp (take-while even?) (take 2)) conj [1 3] [2 4 6 7 8]) [1 3 2 4])) + +(t/deftest test-drop-while + (t/assert= (drop-while pos? [1 2 3 -1]) [-1]) + (t/assert= (drop-while pos? [-1 2]) [-1 2]) + (t/assert= (transduce (drop-while even?) conj [2 4 6 7 8]) [7 8]) + (t/assert= (transduce (drop-while even?) conj [0 2] [1 4 6]) [0 2 1 4 6]) + (t/assert= (transduce (drop-while even?) conj [0 2] [2 4 6 7 8]) [0 2 7 8]) + (t/assert= (transduce (comp (drop-while even?) (take 2)) conj [0 2] [2 4 6 7 8]) [0 2 7 8])) + +(t/deftest test-cycle + (t/assert= (cycle ()) ()) + (t/assert= (cycle nil) ()) + (t/assert= (take 5 (cycle '(1 2))) '(1 2 1 2 1)) + (t/assert= (take 3 (cycle [nil])) '(nil nil nil))) + +(t/deftest test-eduction + ;; one xform + (t/assert= [1 2 3 4 5] + (eduction (map inc) (range 5))) + ;; multiple xforms + (t/assert= ["2" "4"] + (eduction (map inc) (filter even?) (map str) (range 5))) + ;; materialize at the end + (t/assert= [1 1 2 1 2 3 1 2 3 4] + (vec (->> (range 5) + (eduction (mapcat range) (map inc))))) + (t/assert= {1 4, 2 3, 3 2, 4 1} + (->> (range 5) + (eduction (mapcat range) (map inc)) + frequencies)) + (t/assert= ["tac" "god" "hsif" "drib" "kravdraa"] + (->> ["cat" "dog" "fish" "bird" "aardvark"] + (eduction (map pixie.string/reverse)) + (seq))) + ;; expanding transducer with nils + (t/assert= '(1 2 3 nil 4 5 6 nil) + (seq (eduction cat [[1 2 3 nil] [4 5 6 nil]])))) + +(t/deftest test-trace + (try + (/ 0 0) + (catch e + (t/assert= (first (trace e)) {:type :runtime + :data :pixie.stdlib/AssertionException + :msg "Divide by zero"}) + (t/assert= (second (trace e)) {:type :native :name "_div"} )))) + +(t/deftest test-tree-seq + (t/assert= (vec (filter string? + (tree-seq map? + :ch + {:ch [{:ch ["a" "b"]} + {:ch ["c" "d"]} + {:ch [{:ch ["e" {:ch ["f"]}]}]}]}))) + ["a" "b" "c" "d" "e" "f"])) + +(t/deftest test-group-by + (t/assert= (group-by :age [{:name "banjo" :age 3} + {:name "mary" :age 3} + {:name "boris" :age 7}]) + {3 [{:name "banjo" :age 3} + {:name "mary" :age 3}] + 7 [{:name "boris" :age 7}]}) + (t/assert= (group-by even? (range 1 5)) + {true [2 4] + false [1 3]})) + + +(t/deftest test-frequencies + (t/assert= (frequencies [1 2 3 4 3 2 1]) + {1 2, 2 2, 3 2, 4 1})) + +(t/deftest test-condp + (t/assert-throws? RuntimeException + "No matching clause!" + (condp :dont-call-me :dont-use-me)) + (let [f (fn [x] + (condp = x + 1 :one + 2 :two + :whatever))] + (t/assert= (f 1) :one) + (t/assert= (f 2) :two) + (t/assert= (f 9) :whatever))) + +(t/deftest test-case + (t/assert-throws? RuntimeException + "No matching clause!" + (case :no-matter-what)) + (let [f (fn [x] + (case x + 1 :one + 2 :two + #{3 4} :large + :toolarge))] + (t/assert= (f 1) :one) + (t/assert= (f 2) :two) + (t/assert= (f 3) :large) + (t/assert= (f 4) :large) + (t/assert= (f 9) :toolarge))) + +(t/deftest test-instance? + (t/assert= (instance? Keyword :a) true) + (t/assert= (instance? Keyword 'a) false) + (t/assert= (instance? [Symbol Keyword] :a) true) + (t/assert= (instance? [Symbol Keyword] 'a) true) + (t/assert= (instance? [Symbol Keyword] 42) false) + (t/assert= (instance? [] :x) false) + (t/assert-throws? RuntimeException + "c must be a type" + (instance? :not-a-type 123)) + (t/assert-throws? RuntimeException + "c must be a type" + (instance? [Keyword :also-not-a-type] 123))) + +(t/deftest test-types-are-types + (t/assert= Type (type Keyword)) + (t/assert= Type (type Integer)) + (t/assert= Type (type Number)) + (t/assert= Type (type Object)) + (t/assert= Type (type Type))) + +(t/deftest test-satisfies? + (t/assert= (satisfies? IIndexed [1 2]) true) + (t/assert= (satisfies? IIndexed '(1 2)) false) + (t/assert= (satisfies? [] :xyz) true) + (t/assert= (satisfies? [ILookup IIndexed] [1 2]) true) + (t/assert= (satisfies? [ILookup IIndexed] {1 2}) false) + (t/assert-throws? RuntimeException + "proto must be a Protocol" + (satisfies? :not-a-proto 123)) + (t/assert-throws? RuntimeException + "proto must be a Protocol" + (satisfies? [IIndexed :also-not-a-proto] [1 2])) + (defprotocol IFoo (foo [this])) + (extend-protocol IFoo + Number + (foo [this] this)) + (t/assert= (satisfies? IFoo 1) true) + (t/assert= (satisfies? IFoo 1.0) true) + (t/assert= (satisfies? IFoo 1/2) true) + (t/assert= (satisfies? IFoo \a) false)) + +(t/deftest test-reduce + (t/assert= 300 (reduce + (repeat 100 3))) + (t/assert= 0 (reduce + (repeat 0 3))) + (t/assert= [9 9 9] (reduce (partial map +) (repeat 0) (repeat 3 (repeat 3 3)))) + (t/assert= 5050 (reduce + (range 101))) + (t/assert= 3628800 (reduce * (range 1 11))) + (t/assert= 5051 (reduce + 1 (range 101)))) + +(t/deftest test-comp + (t/assert= 5 ((comp inc inc inc inc) 1)) + (t/assert= :xyz ((comp) :xyz))) + +(t/deftest test-atom + (let [a (atom 0)] + (t/assert= 1 (swap! a inc)) + (t/assert= 2 (swap! a inc)) + (t/assert= 3 (reset! a 3)) + (t/assert= :bar (-> a (with-meta {:foo :bar}) meta :foo)))) + +(t/deftest pre-post-conds + (let [f (fn ([a] {:pre [(even? a)] :post [(= % 6)]} (/ a 2)) + ([a b] {:pre [(= (+ 1 a) b)] :post [(odd? %)]} (+ a b)))] + (t/assert= 6 (f 12)) + (t/assert-throws? RuntimeException + "Assert failed: (even? a)" + (f 13)) + (t/assert-throws? RuntimeException + "Assert failed: (= % 6)" + (f 14)) + (t/assert= 15 (f 7 8)) + (t/assert-throws? RuntimeException + "Assert failed: (= (+ 1 a) b)" + (f 1 1)))) + +(t/deftest test-lazyseq-conj + (t/assert= '(1 2 3) (conj (lazy-seq '(2 3)) 1))) + +(t/deftest map-entry-seq + (let [m (map-entry :a 1)] + (t/assert= (seq m) '(:a 1)) + ;; vectors are equal to seqs if their indexed values are equal + ;; thus, a vector of length 2 is equal to a map-entry's seq IFoo + ;; their values are equal + (t/assert= [:a 1] m) + (t/assert= '(:a 1) m))) + +(t/deftest map-entry-equals + (let [m (map-entry :a 1)] + (t/assert= m m) + (t/assert= m (map-entry :a 1)) + (t/assert= m [:a 1]) + (t/assert= m '(:a 1)))) + +(t/deftest test-vary-meta + (t/assert= 42 (-> {} (vary-meta assoc :foo 42) meta :foo))) + +(t/deftest test-memoize + (let [f (memoize rand)] + (t/assert= (f) (f)))) + +(t/deftest test-iterate + (t/assert= (take 5 (iterate inc 5)) '(5 6 7 8 9)) + (t/assert= (reduce (fn [a v] (reduced "foo")) 0 (iterate inc 1)) "foo") + (t/assert= (reduce (fn [a v] (if (< a 10) (+ a v) (reduced a))) 0 (iterate (partial + 2) 1)) 16)) + +(t/deftest test-sequence-empty-sequences + (t/assert= '() (take 1 (sequence (map inc) '()))) + (t/assert= '() (take 1 (sequence (map inc) []))) + (t/assert= '() (take 1 (sequence (map inc) #{}))) + (t/assert= '() (take 1 (sequence (map inc) {})))) + +(t/deftest test-sequence-non-empty-sequences + (t/assert= '(1 3) (take 2 (sequence (comp + (filter even?) + (map inc)) (range 3)))) + (t/assert= '(1) (take 1 (sequence (distinct) (repeat 4 1))))) + +(t/deftest test-sequence-early-terminating-sequences + (t/assert= '() (take 5 (sequence (filter (fn [x] false)) (repeat 8 8)))) + (t/assert= '(1 2) (take 3 (sequence (map identity) [1 2]))) + (t/assert= #{[:a 1] [:b 2]} (into #{} (take 3 (sequence (filter (fn [[k v]] + (keyword? k)) {:a 1 + :b 2 + "c" 3 + "d" 4})))))) + +(t/deftest test-letfn + (letfn [(hello [] "Hello") + (adder [x y] (+ x y))] + (t/assert= "Hello" (hello)) + (dotimes [i 10] + (dotimes [j 20] + (t/assert= (+ i j) (adder i j))))) + (letfn [(f [x n] (vec (repeat n x)))] + (t/assert= (f :x 3) [:x :x :x]) + (t/assert= (f 0 20) [0 0 0 0 0 + 0 0 0 0 0 + 0 0 0 0 0]))) diff --git a/tests/pixie/tests/test-strings.pxi b/tests/pixie/tests/test-strings.pxi new file mode 100644 index 00000000..0013f6e2 --- /dev/null +++ b/tests/pixie/tests/test-strings.pxi @@ -0,0 +1,189 @@ +(ns pixie.test.test-strings + (require pixie.test :as t) + (require pixie.string :as s)) + +(t/deftest test-starts-with? + (let [s "heyhohuh"] + (t/assert= (s/starts-with? s "") true) + (t/assert= (s/starts-with? s "hey") true) + (t/assert= (s/starts-with? s "heyho") true) + (t/assert= (s/starts-with? s s) true) + + (t/assert= (s/starts-with? s "ho") false) + (t/assert= (s/starts-with? s "foo") false))) + +(t/deftest test-ends-with? + (let [s "heyhohuh"] + (t/assert= (s/ends-with? s "") true) + (t/assert= (s/ends-with? s "huh") true) + (t/assert= (s/ends-with? s "hohuh") true) + (t/assert= (s/ends-with? s s) true) + + (t/assert= (s/ends-with? s "hey") false) + (t/assert= (s/ends-with? s "foo") false))) + +(t/deftest test-split + (let [s "hey,ho,huh"] + (t/assert= (s/split s ",") ["hey" "ho" "huh"]) + (t/assert= (s/split s "h") ["" "ey," "o," "u" ""]))) + +(t/deftest test-split-lines + ;; Splits unix-style lines. + (t/assert= (s/split-lines "bibbidi\nbobbidi\nboo") + ["bibbidi" "bobbidi" "boo"]) + ;; Splits windows-style lines. + (t/assert= (s/split-lines "bibbidi\r\nbobbidi\r\nboo") + ["bibbidi" "bobbidi" "boo"]) + ;; Splits mixed up stuff. + (t/assert= (s/split-lines "bibbidi\nbobbidi\r\nboo") + ["bibbidi" "bobbidi" "boo"]) + ;; Doesn't split lonely \r + (t/assert= (s/split-lines "bibbidi\nbobbidi\rboo") + ["bibbidi" "bobbidi\rboo"]) + ;; Works with a single line. + (t/assert= (s/split-lines "BibbidiBobbidiBoo") + ["BibbidiBobbidiBoo"]) + ;; Works with empty strings. + (t/assert= (s/split-lines "") [""]) + ;; Nil pass-through. + (t/assert= (s/split-lines nil) nil)) + +(t/deftest test-index-of + (let [s "heyhohuh"] + (t/assert= (s/index-of s "hey") 0) + (t/assert= (s/index-of s "ho") 3) + (t/assert= (s/index-of s "foo") nil) + + (t/assert= (s/index-of s "h" 2) 3) + (t/assert= (s/index-of s "h" 4) 5) + (t/assert= (s/index-of s "hey" 0) 0) + (t/assert= (s/index-of s "hey" 1) nil) + + (t/assert= (s/index-of s "h" 0 0) nil) + (t/assert= (s/index-of s "h" 1 2) nil))) + +(t/deftest test-substring + (let [s "heyhohuh"] + (t/assert= (s/substring s 0) s) + (t/assert= (s/substring s 3) (s/substring s 3 (count s))) + (t/assert= (s/substring s 0 0) "") + (t/assert= (s/substring s 0 3) "hey") + (t/assert= (s/substring s 3 5) "ho") + (t/assert= (s/substring s 5 8) "huh") + (t/assert= (s/substring s 3 10000) "hohuh"))) + +(t/deftest test-upper-case + (t/assert= (s/lower-case "") "") + (t/assert= (s/upper-case "hey") "HEY") + (t/assert= (s/upper-case "hEy") "HEY") + (t/assert= (s/upper-case "HEY") "HEY") + (t/assert= (s/upper-case "hey?!") "HEY?!")) + +(t/deftest test-lower-case + (t/assert= (s/lower-case "") "") + (t/assert= (s/lower-case "hey") "hey") + (t/assert= (s/lower-case "hEy") "hey") + (t/assert= (s/lower-case "HEY") "hey") + (t/assert= (s/lower-case "HEY?!") "hey?!")) + +(t/deftest test-capitalize + (t/assert= (s/capitalize "timothy") "Timothy") + (t/assert= (s/capitalize "Timothy") "Timothy")) + +(t/deftest test-trim + (t/assert= (s/trim "") "") + (t/assert= (s/trim " ") "") + (t/assert= (s/trim " hey ") "hey") + (t/assert= (s/trim " h ey ") "h ey")) + +(t/deftest test-triml + (t/assert= (s/triml "") "") + (t/assert= (s/triml " ") "") + (t/assert= (s/triml " hey") "hey") + (t/assert= (s/triml " hey ") "hey ") + (t/assert= (s/triml " h ey ") "h ey ")) + +(t/deftest test-trimr + (t/assert= (s/trimr "") "") + (t/assert= (s/trimr " ") "") + (t/assert= (s/trimr "hey ") "hey") + (t/assert= (s/trimr " hey ") " hey") + (t/assert= (s/trimr " h ey ") " h ey")) + +(t/deftest test-trim-newline + (t/assert= (s/trim-newline "") "") + (t/assert= (s/trim-newline "\r\n") "") + (t/assert= (s/trim-newline "hey\r\n") "hey") + (t/assert= (s/trim-newline "hey\n\r") "hey") + (t/assert= (s/trim-newline "hey\r") "hey") + (t/assert= (s/trim-newline "hey\n") "hey") + (t/assert= (s/trim-newline "hey\r\nthere\r\n") "hey\r\nthere") + (t/assert= (s/trim-newline "\r\nhey\r\n") "\r\nhey")) + +(t/deftest test-replace + (t/assert= (s/replace "hey,you,there" "," ", ") "hey, you, there") + (t/assert= (s/replace "hey,you,there" "," "") "heyyouthere") + (t/assert= (s/replace "&&&" "&" "&&") "&&&&&&") + (t/assert= (s/replace "oops" "" "WAT") "WAToWAToWATpWATsWAT")) + +(t/deftest test-replace-first + (t/assert= (s/replace-first "hey,you,there" "," ", ") "hey, you,there") + (t/assert= (s/replace-first "hey,you,there" "," "") "heyyou,there") + (t/assert= (s/replace-first "&&&" "&" "&&") "&&&&") + (t/assert= (s/replace-first "oops" "" "WAT") "WAToops")) + +(t/deftest test-reverse + (t/assert= (s/reverse "not a palindrome") "emordnilap a ton") + (t/assert= (s/reverse "tacocat") "tacocat") + (t/assert= (s/reverse "") "") + (t/assert= (s/reverse nil) nil)) + +(t/deftest test-join + (t/assert= (s/join []) "") + (t/assert= (s/join [1]) "1") + (t/assert= (s/join [1 2 3]) "123") + + (t/assert= (s/join ", " []) "") + (t/assert= (s/join ", " [1]) "1") + + (t/assert= (s/join ", " [1 2 3]) "1, 2, 3")) + +(t/deftest test-char-literals + (let [s "hey"] + (t/assert= (nth s 0) \h) + (t/assert= (nth s 0) \o150) + (t/assert= (nth s 0) \u0068) + + (t/assert= (nth s 1) \e) + (t/assert= (nth s 1) \o145) + (t/assert= (nth s 1) \u0065) + + (t/assert= (nth s 2) \y) + (t/assert= (nth s 2) \o171) + (t/assert= (nth s 2) \u0079))) + +(t/deftest test-char-conversions + (t/assert= (int \a) 97) + (t/assert= (char 97) \a) + + (t/assert= (int \u269b) 0x269b) + (t/assert= (char 0x269b) \u269b)) + +(t/deftest test-unicode + (t/assert= "hรขllo" "hรขllo")) + +(t/deftest test-blank? + (t/assert= (s/blank? nil) true) + (t/assert= (s/blank? "") true) + (t/assert= (s/blank? " ") true) + (t/assert= (s/blank? " \t \n \r ") true) + (t/assert= (s/blank? " foo ") false)) + +(t/deftest test-escape + (t/assert= (s/escape "foo" {\f \z}) "zoo") + (t/assert= (s/escape "foo" {\z \f}) "foo") + (t/assert= (s/escape "foobar" {\f \b \o \e \b \j}) "beejar") + (t/assert= (s/escape "foo" {}) "foo") + (t/assert= (s/escape "foo" nil) "foo") + (t/assert= (s/escape "" {\f \z}) "") + (t/assert= (s/escape nil {\f \z}) nil)) diff --git a/tests/pixie/tests/test-utf8.pxi b/tests/pixie/tests/test-utf8.pxi new file mode 100644 index 00000000..1c52ef01 --- /dev/null +++ b/tests/pixie/tests/test-utf8.pxi @@ -0,0 +1,9 @@ +(ns pixie.test.test-utf8 + (require pixie.test :as t)) + +(t/deftest test-utf8-string-val + (t/assert= "๐Ÿบ=๐Ÿ‘" "๐Ÿบ=๐Ÿ‘")) + +(t/deftest test-utf8-var-name + (let [๐Ÿบ "๐Ÿบ=๐Ÿ‘"] + (t/assert= ๐Ÿบ "๐Ÿบ=๐Ÿ‘"))) diff --git a/tests/pixie/tests/test-walk.pxi b/tests/pixie/tests/test-walk.pxi new file mode 100644 index 00000000..d7a0aa1a --- /dev/null +++ b/tests/pixie/tests/test-walk.pxi @@ -0,0 +1,59 @@ +(ns pixie.tests.test-walk + (:require [pixie.walk :as w] + [pixie.test :as t])) + +(t/deftest t-prewalk-replace + (t/assert (= (w/prewalk-replace {:a :b} [:a {:a :a} (list 3 :c :a)]) + [:b {:b :b} (list 3 :c :b)]))) + +(t/deftest t-postwalk-replace + (t/assert (= (w/postwalk-replace {:a :b} [:a {:a :a} (list 3 :c :a)]) + [:b {:b :b} (list 3 :c :b)]))) + +(t/deftest t-prewalk-order + (t/assert (= (let [a (atom [])] + (w/prewalk (fn [form] (swap! a conj form) form) + [1 2 {:a 3} (list 4 [5])]) + @a) + [[1 2 {:a 3} (list 4 [5])] + 1 2 {:a 3} [:a 3] :a 3 (list 4 [5]) + 4 [5] 5]))) + +(t/deftest t-postwalk-order + (t/assert (= (let [a (atom [])] + (w/postwalk (fn [form] (swap! a conj form) form) + [1 2 {:a 3} (list 4 [5])]) + @a) + [1 2 + :a 3 [:a 3] {:a 3} + 4 5 [5] (list 4 [5]) + [1 2 {:a 3} (list 4 [5])]]))) + +(defrecord Foo [a b c]) + +(t/deftest walk + "Checks that walk returns the correct result and type of collection" + (let [colls ['(1 2 3) + [1 2 3] + #{1 2 3} + {:a 1, :b 2, :c 3} + (->Foo 1 2 3)]] + (doseq [c colls] + (let [walked (w/walk identity c)] + (t/assert (= c walked)) + (t/assert (= (type c) (type walked))) + (if (or (map? c) + (record? c)) + (do + (t/assert (= (reduce + (vals (w/walk + #(map-entry + (key %) + (inc (val %))) + c))) + (reduce + (map (comp inc val) c))))) + (t/assert (= (reduce + (w/walk inc c)) + (reduce + (map inc c))))))))) + +(t/deftest t-stringify-keys + (t/assert (= (w/stringify-keys {:a 1, nil {:b 2 :c 3}, :d 4}) + {"a" 1, nil {"b" 2 "c" 3}, "d" 4}))) diff --git a/tests/pixie/tests/utils.pxi b/tests/pixie/tests/utils.pxi new file mode 100644 index 00000000..b7d6fce6 --- /dev/null +++ b/tests/pixie/tests/utils.pxi @@ -0,0 +1,13 @@ +(ns pixie.tests.utils) + +;; Here we create a new type which hashes poorly, in fact it's so bad we have a +;; hash space of only 1. +;; All members of WorstHasher return (hash "worst hasher") +;; when hash is called on them. +;; +;; This makes debugging, testing and benchmarking anything based off +;; PersistentHashMap trivial. + +;; X can be any thing you like. +(defrecord WorstHasher [x]) +(extend -hash WorstHasher (fn [self] (hash "worst hasher")))