Skip to content

Commit e5e9fc7

Browse files
authored
Refine embedded (#150)
* Refactored embedding code out of the base python library into it's own namespace. Refining jbridge a bit. * Finished up first round of embedded work.
1 parent b8477c5 commit e5e9fc7

5 files changed

Lines changed: 239 additions & 160 deletions

File tree

cljbridge.py

Lines changed: 167 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,167 @@
1+
"""Python bindings to start a Clojure repl from a python process. Relies on
2+
libpython-clj being in a deps.edn pathway as clojure is called from the command
3+
line to build the classpath. Expects javabridge to be installed and functional.
4+
5+
Javabridge will dynamically find the java library that corresponds with calling 'java'
6+
from the command line and load it. We then initialize Clojure and provide pathways
7+
to require namespaces, find symbols, and call functions.
8+
9+
There are two import initialization methods - init_clojure and init_clojure_repl - these
10+
take care of starting up everything in the correct order.
11+
"""
12+
import subprocess
13+
import javabridge
14+
15+
try:
16+
from collections.abc import Callable # noqa
17+
except ImportError:
18+
from collections import Callable
19+
20+
21+
def init_clojure_runtime():
22+
"""Initialize the clojure runtime. This needs to happen at least once before
23+
attempting to require a namespace or lookup a clojure var."""
24+
javabridge.static_call("clojure/lang/RT", "init", "()V")
25+
26+
27+
def find_clj_var(fn_ns, fn_name):
28+
"""Use the clojure runtime to find a var. Clojure vars are placeholders in namespaces that forward their operations to the data they point to. This allows someone to hold
29+
a var and but recompile a namespace to get different behavior. They implement both
30+
`clojure.lang.IFn` and `clojure.lang.IDeref` so they can act like a function and
31+
you can dereference them to get to their original value."""
32+
return javabridge.static_call("clojure/lang/RT",
33+
"var",
34+
"(Ljava/lang/String;Ljava/lang/String;)Lclojure/lang/Var;",
35+
fn_ns,
36+
fn_name)
37+
38+
39+
class CLJFn(Callable):
40+
"""Construct a python callable from a clojure object. This callable will forward
41+
function calls to it's Clojure object expecting a clojure.lang.IFn interface."""
42+
applyTo = javabridge.make_method("applyTo", "(clojure/lang/ISeq;)Ljava/lang/Object;")
43+
def __init__(self, ifn_obj):
44+
self.o = ifn_obj
45+
46+
def __call__(self, *args, **kw_args):
47+
if not kw_args:
48+
invoker = getattr(self, "invoke"+str(len(args)))
49+
return invoker(*args)
50+
else:
51+
raise Exception("Unable to handle kw_args for now")
52+
print(len(args), len(kw_args))
53+
54+
55+
for i in range(20):
56+
opargs = ""
57+
for j in range(i):
58+
opargs += "Ljava/lang/Object;"
59+
setattr(CLJFn, "invoke" + str(i),
60+
javabridge.make_method("invoke", "(" + opargs + ")Ljava/lang/Object;" ))
61+
62+
63+
def resolve_fn(namespaced_name):
64+
"""Resolve a clojure var given a fully qualified namespace name. The return value
65+
is callable. Note that the namespace itself needs to be required first."""
66+
ns_name, sym_name = namespaced_name.split("/")
67+
return CLJFn(find_clj_var(ns_name, sym_name))
68+
69+
70+
def resolve_call_fn(namespaced_fn_name, *args):
71+
"""Resolve a function given a fully qualified namespace name and call it."""
72+
return resolve_fn(namespaced_fn_name)(*args)
73+
74+
def symbol(sym_name):
75+
"""Create a clojure symbol from a string"""
76+
return javabridge.static_call("clojure/lang/Symbol", "intern",
77+
"(Ljava/lang/String;)Lclojure/lang/Symbol;", sym_name)
78+
79+
__REQUIRE_FN = None
80+
81+
def require(ns_name):
82+
"""Require a clojure namespace. This needs to happen before you find symbols
83+
in that namespace else you will be uninitialized var errors."""
84+
if not __REQUIRE_FN:
85+
_REQUIRE_FN = resolve_fn("clojure.core/require")
86+
return _REQUIRE_FN(symbol(ns_name))
87+
88+
89+
def init_libpy_embedded():
90+
"""Initialize libpy on a mode where it looks for symbols in the local process and
91+
it itself doesn't attempt to run the python intialization procedures but expects
92+
the python system to be previously initialized."""
93+
require("libpython-clj2.embedded")
94+
return resolve_call_fn("libpython-clj2.embedded/initialize!")
95+
96+
97+
def classpath(classpath_args=[]):
98+
"""Call clojure at the command line and return the classpath in as a list of
99+
strings. Clojure will pick up a local deps.edn or deps can be specified inline."""
100+
return subprocess.check_output(['clojure'] + list(classpath_args) + ['-Spath']).decode("utf-8").strip().split(':')
101+
102+
103+
DEFAULT_NREPL_VERSION = "0.8.3"
104+
DEFAULT_CIDER_NREPL_VERSION = "0.25.5"
105+
106+
107+
def repl_classpath(nrepl_version=DEFAULT_NREPL_VERSION,
108+
cider_nrepl_version=DEFAULT_CIDER_NREPL_VERSION,
109+
classpath_args=[]):
110+
"""Return the classpath with the correct deps to run nrepl and cider.
111+
Positional arguments are added after the -Sdeps argument to start the
112+
nrepl server."""
113+
return classpath(classpath_args=["-Sdeps", '{:deps {nrepl/nrepl {:mvn/version "%s"} cider/cider-nrepl {:mvn/version "%s"}}}' % (nrepl_version, cider_nrepl_version)]
114+
+ list(classpath_args))
115+
116+
117+
def init_clojure(classpath_args=[]):
118+
"""Initialize a vanilla clojure process using the clojure command line to output
119+
the classpath to use for the java vm. At the return of this function clojure is
120+
initialized and libpython-clj2.python's public functions will work.
121+
122+
* classpath_args - List of arguments that will be passed to the clojure command
123+
line process when building the classpath. """
124+
javabridge.start_vm(run_headless=True,
125+
class_path=classpath(classpath_args=classpath_args))
126+
init_clojure_runtime()
127+
init_libpy_embedded()
128+
return True
129+
130+
131+
def init_clojure_repl(**kw_args):
132+
"""Initialize clojure with extra arguments specifically for embedding a cider-nrepl
133+
server. Then start an nrepl server. The port will both be printed to stdout and
134+
output to a .nrepl_server file. This function does not return as it leaves the GIL
135+
released so that repl threads have access to Python. libpython-clj2.python is
136+
initialized 'require-python' pathways should work.
137+
138+
* classpath_args - List of additional arguments that be passed to the clojure
139+
process when building the classpath."""
140+
javabridge.start_vm(run_headless=True, class_path=repl_classpath(**kw_args))
141+
init_clojure_runtime()
142+
init_libpy_embedded()
143+
resolve_call_fn("libpython-clj2.embedded/start-repl!")
144+
145+
146+
class GenericJavaObj:
147+
__str__ = javabridge.make_method("toString", "()Ljava/lang/String;")
148+
get_class = javabridge.make_method("getClass", "()Ljava/lang/Class;")
149+
__repl__ = javabridge.make_method("toString", "()Ljava/lang/String;")
150+
def __init__(self, jobj):
151+
self.o = jobj
152+
153+
154+
def longCast(jobj):
155+
"Cast a java object to a primitive long value."
156+
return javabridge.static_call("clojure/lang/RT", "longCast",
157+
"(Ljava/lang/Object;)J", jobj)
158+
159+
160+
def to_ptr(pyobj):
161+
"""Create a tech.v3.datatype.ffi.Pointer java object from a python object. This
162+
allows you to pass python objects directly into libpython-clj2.python-derived
163+
pathways (such as ->jvm). If java is going to hold onto the python data for
164+
a long time and it will fall out of Python scope object should be
165+
'incref-tracked' - 'libpython-clj2.python.ffi/incref-track-pyobject'."""
166+
return javabridge.static_call("tech/v3/datatype/ffi/Pointer", "constructNonZero",
167+
"(J)Ltech/v3/datatype/ffi/Pointer;", id(pyobj))

deps.edn

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,6 +26,7 @@
2626
libpython-clj2.codegen
2727
libpython-clj2.python.np-array
2828
libpython-clj2.require
29+
libpython-clj2.embedded
2930
python.builtins
3031
python.numpy]}}
3132
:test

jbridge.py

Lines changed: 0 additions & 103 deletions
This file was deleted.

src/libpython_clj2/embedded.clj

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
(ns libpython-clj2.embedded
2+
"Tools for embedding clojure into a python host process.
3+
See jbridge.py for python details. This namespace relies on
4+
the classpath having nrepl and cider-nrepl on it. For example:
5+
6+
```console
7+
clojure -Sdeps '{:deps {nrepl/nrepl {:mvn/version \"0.8.3\"} cider/cider-nrepl {:mvn/version \"0.25.5\"}}}' ...
8+
```"
9+
(:require [libpython-clj2.python.ffi :as py-ffi]
10+
[nrepl.server :as server]
11+
[nrepl.cmdline :as cmdline]
12+
[clojure.tools.logging :as log]))
13+
14+
15+
(defn initialize!
16+
"Initialize python when this library is being called *from* a python program. In
17+
that case, unless libpath is explicitly provided the system will look for the
18+
python symbols in the current executable."
19+
([] (py-ffi/set-library! nil))
20+
([libpath] (py-ffi/set-library! libpath)))
21+
22+
23+
(defonce ^:private repl-server* (atom nil))
24+
25+
26+
(defn stop-repl!
27+
"If an existing repl has been started, stop it. This returns control to the
28+
python process."
29+
[]
30+
(swap! repl-server*
31+
(fn [server]
32+
(when server
33+
(try
34+
(locking #'repl-server*
35+
(server/stop-server server)
36+
(.notifyAll ^Object #'repl-server*))
37+
nil
38+
(catch Throwable e
39+
(log/errorf e "Failed to stop nrepl server!")
40+
nil))))))
41+
42+
43+
(defn start-repl!
44+
"This is called to start a clojure repl from python. Do not ever call this if
45+
not from a python thread as it starts with releasing the GIL. This function
46+
does not return until another thread calls stop-embedded-repl!.
47+
48+
If an existing repl server has been started this returns the port of the previous
49+
server else it returns the port of the new server.
50+
51+
Options are the same as the command line options found in nrepl.cmdline."
52+
([options]
53+
(when-not @repl-server*
54+
(let [tstate (when (== 1 (long (py-ffi/PyGILState_Check)))
55+
(py-ffi/PyEval_SaveThread))]
56+
(try
57+
(let [options (cmdline/server-opts
58+
(merge {:middleware '[cider.nrepl/cider-middleware]}
59+
options))
60+
server (cmdline/start-server options)
61+
_ (reset! repl-server* server)]
62+
(cmdline/ack-server server options)
63+
(cmdline/save-port-file server options)
64+
(log/info (cmdline/server-started-message server options))
65+
(locking #'repl-server*
66+
(.wait ^Object #'repl-server*)))
67+
(finally
68+
(when tstate
69+
(py-ffi/PyEval_RestoreThread tstate))))))
70+
(:port @repl-server*))
71+
([] (start-repl! nil)))

0 commit comments

Comments
 (0)