Skip to content

Commit cb515b4

Browse files
author
hartsantler
committed
testing fake threading module using webworkers (nodejs package workerjs)
1 parent e33e092 commit cb515b4

6 files changed

Lines changed: 231 additions & 30 deletions

File tree

‎pythonjs/python_to_pythonjs.py‎

Lines changed: 115 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -41,7 +41,13 @@ def log(txt):
4141
GLOBAL_VARIABLE_SCOPE = True
4242
log('not using python style variable scope')
4343

44-
writer = code_writer.Writer()
44+
writer = writer_main = code_writer.Writer()
45+
46+
__webworker_writers = dict()
47+
def get_webworker_writer( jsfile ):
48+
if jsfile not in __webworker_writers:
49+
__webworker_writers[ jsfile ] = code_writer.Writer()
50+
return __webworker_writers[ jsfile ]
4551

4652

4753

@@ -125,6 +131,8 @@ def __init__(self, source=None, module=None, module_path=None, dart=False, coffe
125131
self._with_dart = dart
126132
self._with_js = False
127133
self._in_lambda = False
134+
self._use_threading = False
135+
self._webworker_functions = dict()
128136

129137
self._source = source.splitlines()
130138
self._classes = dict() ## class name : [method names]
@@ -180,6 +188,11 @@ def __init__(self, source=None, module=None, module_path=None, dart=False, coffe
180188
else:
181189
self.visit(node)
182190

191+
def has_webworkers(self):
192+
return len(self._webworker_functions.keys())
193+
194+
def get_webworker_file_names(self):
195+
return set(self._webworker_functions.values())
183196

184197
def preprocess_custom_operators(self, data):
185198
'''
@@ -279,9 +292,12 @@ def _load_module(self, name): ## DEPRECATED
279292

280293
def visit_Import(self, node):
281294
for alias in node.names:
282-
writer.write( '## import: %s :: %s' %(alias.name, alias.asname) )
283-
## TODO namespaces: import x as y
284-
raise NotImplementedError('import, line %s' % node.lineno)
295+
if alias.name == 'threading':
296+
self._use_threading = True
297+
writer.write( 'Worker = require("workerjs")')
298+
else:
299+
#writer.write( '## import: %s :: %s' %(alias.name, alias.asname) )
300+
raise NotImplementedError('import, line %s' % node.lineno)
285301

286302
def visit_ImportFrom(self, node):
287303
if self._with_dart:
@@ -291,7 +307,6 @@ def visit_ImportFrom(self, node):
291307
else:
292308
lib = ministdlib.JS
293309

294-
295310
if node.module in lib:
296311
imported = False
297312
for n in node.names:
@@ -1649,7 +1664,19 @@ def visit_Call(self, node):
16491664
#raise SyntaxError("lambda functions must be assigned to a variable before being called")
16501665

16511666
name = self.visit(node.func)
1652-
if isinstance(node.func, ast.Attribute) and isinstance(node.func.value, Name) and node.func.value.id == 'pythonjs' and node.func.attr == 'configure':
1667+
if self._use_threading and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, Name) and node.func.value.id == 'threading':
1668+
if node.func.attr == 'start_new_thread':
1669+
return '__start_new_thread( %s, %s )' %(self.visit(node.args[0]), self.visit(node.args[1]))
1670+
else:
1671+
raise SyntaxError(node.func.attr)
1672+
1673+
elif self._use_threading and isinstance(node.func, ast.Attribute) and isinstance(node.func.value, ast.Attribute) and isinstance(node.func.value.value, Name) and node.func.value.value.id == 'threading':
1674+
assert node.func.value.attr == 'shared_list'
1675+
if node.func.attr == 'append':
1676+
return '__shared_list_append( %s )' %self.visit(node.args[0])
1677+
1678+
1679+
elif isinstance(node.func, ast.Attribute) and isinstance(node.func.value, Name) and node.func.value.id == 'pythonjs' and node.func.attr == 'configure':
16531680
for kw in node.keywords:
16541681
if kw.arg == 'javascript':
16551682
if kw.value.id == 'True':
@@ -2079,7 +2106,8 @@ def visit_Lambda(self, node):
20792106
return self.visit_FunctionDef(node)
20802107

20812108
def visit_FunctionDef(self, node):
2082-
log('-----------------')
2109+
global writer
2110+
20832111
if node in self._generator_function_nodes:
20842112
log('generator function: %s'%node.name)
20852113
self._generator_functions.add( node.name )
@@ -2090,6 +2118,7 @@ def visit_FunctionDef(self, node):
20902118
return
20912119
log('function: %s'%node.name)
20922120

2121+
threaded = False
20932122
property_decorator = None
20942123
decorators = []
20952124
with_js_decorators = []
@@ -2111,6 +2140,33 @@ def visit_FunctionDef(self, node):
21112140
inline = True
21122141
self._with_inline = True
21132142

2143+
elif isinstance(decorator, ast.Call) and isinstance(decorator.func, ast.Name) and decorator.func.id == 'webworker':
2144+
if not self._with_dart:
2145+
threaded = True
2146+
assert len(decorator.args) == 1
2147+
jsfile = decorator.args[0].s #self.visit(decorator.args[0])
2148+
writer_main.write('%s = "%s"' %(node.name, jsfile))
2149+
self._webworker_functions[ node.name ] = jsfile
2150+
2151+
writer = get_webworker_writer( jsfile )
2152+
writer.write( '__shared_list = []' )
2153+
writer.write('def __shared_list_append(v):')
2154+
writer.push()
2155+
writer.write( 'print("child sending message")' )
2156+
writer.write( '__shared_list.push(v)' )
2157+
writer.write( 'self.postMessage({"type":"append", "value":v})' )
2158+
writer.pull()
2159+
2160+
writer.write('def onmessage(e):')
2161+
writer.push()
2162+
writer.write( 'print("got message from parent")' )
2163+
writer.write( 'if e.data.type=="execute": %s.call(self, e.data.args, {}); self.postMessage({"type":"terminate"})' %node.name )
2164+
writer.write( 'elif e.data.type=="append": __shared_list.push(e.data.value)' )
2165+
writer.write( 'elif e.data.type=="setitem": __shared_list.insert(e.data.index, e.data.value)' )
2166+
writer.pull()
2167+
writer.write('self.onmessage = onmessage' )
2168+
2169+
21142170
elif self._with_dart:
21152171
with_dart_decorators.append( self.visit(decorator) )
21162172

@@ -2125,13 +2181,13 @@ def visit_FunctionDef(self, node):
21252181
elif isinstance(decorator, Name) and decorator.id == 'javascript':
21262182
javascript = True
21272183

2128-
elif isinstance(decorator, Name) and decorator.id in ('property', 'cached_property'):
2184+
elif isinstance(decorator, Name) and decorator.id == 'property':
21292185
property_decorator = decorator
21302186
n = node.name + '__getprop__'
21312187
self._decorator_properties[ node.original_name ] = dict( get=n, set=None )
21322188
node.name = n
2133-
if decorator.id == 'cached_property': ## TODO DEPRECATE
2134-
self._cached_property = node.original_name
2189+
#if decorator.id == 'cached_property': ## TODO DEPRECATE
2190+
# self._cached_property = node.original_name
21352191

21362192
elif isinstance(decorator, Attribute) and isinstance(decorator.value, Name) and decorator.value.id in self._decorator_properties:
21372193
if decorator.attr == 'setter':
@@ -2361,13 +2417,34 @@ def visit_FunctionDef(self, node):
23612417
log('(function has no arguments)')
23622418

23632419
################# function body #################
2364-
if self._cached_property:
2365-
writer.write('if self["__dict__"]["%s"]: return self["__dict__"]["%s"]' %(self._cached_property, self._cached_property))
23662420

2421+
#if self._cached_property: ## DEPRECATED
2422+
# writer.write('if self["__dict__"]["%s"]: return self["__dict__"]["%s"]' %(self._cached_property, self._cached_property))
23672423

23682424
self._return_type = None # tries to catch a return type in visit_Return
23692425

2370-
map(self.visit, node.body) ## write function body
2426+
## write function body ##
2427+
## if a new thread/webworker is started, the following function body must be wrapped in
2428+
## a closure callback and called later by setTimeout
2429+
timeouts = 0
2430+
for b in node.body:
2431+
2432+
if self._use_threading and isinstance(b, ast.Assign) and isinstance(b.value, ast.Call):
2433+
if isinstance(b.value.func, ast.Attribute) and isinstance(b.value.func.value, Name) and b.value.func.value.id == 'threading':
2434+
if b.value.func.attr == 'start_new_thread':
2435+
self.visit(b)
2436+
writer.write('def __callback%s():' %timeouts)
2437+
writer.push()
2438+
timeouts += 1
2439+
continue
2440+
2441+
self.visit(b)
2442+
2443+
2444+
for i in range(timeouts):
2445+
writer.pull()
2446+
## workerjs for nodejs requires at least 100ms to initalize onmessage/postMessage
2447+
writer.write('setTimeout(__callback%s, 500)' %i)
23712448

23722449
if self._return_type: ## check if a return type was caught
23732450
if return_type:
@@ -2397,12 +2474,13 @@ def visit_FunctionDef(self, node):
23972474
writer.pull()
23982475
writer.pull()
23992476

2400-
writer.pull()
2477+
writer.pull() ## end function body
24012478

24022479
if inline:
24032480
self._with_inline = False
24042481

24052482
if self._in_js_class:
2483+
writer = writer_main
24062484
return
24072485

24082486

@@ -2474,6 +2552,14 @@ def visit_FunctionDef(self, node):
24742552
else:
24752553
writer.write('%s = __get__(%s,"__call__")( [%s], JSObject() )' % (node.name, dec, node.name))
24762554

2555+
#if threaded:
2556+
# writer.write('%s()' %node.name)
2557+
# writer.write('self.termintate()')
2558+
2559+
2560+
writer = writer_main
2561+
2562+
24772563
#################### loops ###################
24782564
## the old-style for loop that puts a while loop inside a try/except and catches StopIteration,
24792565
## has a problem because at runtime if there is an error inside the loop, it will not show up in a strack trace,
@@ -2963,19 +3049,27 @@ def collect_generator_functions(node):
29633049

29643050

29653051
def main(script, dart=False, coffee=False, lua=False):
2966-
PythonToPythonJS(
3052+
translator = PythonToPythonJS(
29673053
source = script,
29683054
dart = dart or '--dart' in sys.argv,
29693055
coffee = coffee,
29703056
lua = lua
29713057
)
3058+
29723059
code = writer.getvalue()
2973-
if '--debug' in sys.argv:
2974-
try:
2975-
open('/tmp/python-to-pythonjs.debug.py', 'wb').write(code)
2976-
except:
2977-
pass
2978-
return code
3060+
3061+
if translator.has_webworkers():
3062+
res = {'main':code}
3063+
for jsfile in translator.get_webworker_file_names():
3064+
res[ jsfile ] = get_webworker_writer( jsfile ).getvalue()
3065+
return res
3066+
else:
3067+
if '--debug' in sys.argv:
3068+
try:
3069+
open('/tmp/python-to-pythonjs.debug.py', 'wb').write(code)
3070+
except:
3071+
pass
3072+
return code
29793073

29803074

29813075
def command():

‎pythonjs/pythonjs.js‎

Lines changed: 33 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
// PythonJS Runtime - regenerated on: Tue May 20 03:05:36 2014
1+
// PythonJS Runtime - regenerated on: Thu May 22 03:39:00 2014
22
__NULL_OBJECT__ = Object.create(null);
33
if (( "window" ) in this && ( "document" ) in this) {
44
__WEBWORKER__ = false;
@@ -3474,4 +3474,35 @@ __lambda__.NAME = "__lambda__";
34743474
__lambda__.args_signature = ["o"];
34753475
__lambda__.kwargs_signature = { };
34763476
__lambda__.types_signature = { };
3477-
json = __jsdict([["loads", null], ["dumps", null]]);
3477+
json = __jsdict([["loads", null], ["dumps", null]]);
3478+
threading = __jsdict([["shared_list", []]]);
3479+
__start_new_thread = function(f, args) {
3480+
var worker;
3481+
worker = new Worker(f);
3482+
var func = function(event) {
3483+
console.log("got signal from thread");
3484+
if (( event.data.type ) == "terminate") {
3485+
worker.terminate();
3486+
} else {
3487+
if (( event.data.type ) == "append") {
3488+
console.log("got append event");
3489+
threading.shared_list.push(event.data.value);
3490+
} else {
3491+
console.log("unknown event");
3492+
}
3493+
}
3494+
}
3495+
3496+
func.NAME = "func";
3497+
func.args_signature = ["event"];
3498+
func.kwargs_signature = { };
3499+
func.types_signature = { };
3500+
worker.onmessage = func;
3501+
worker.postMessage(__jsdict([["type", "execute"], ["args", args]]));
3502+
return worker;
3503+
}
3504+
3505+
__start_new_thread.NAME = "__start_new_thread";
3506+
__start_new_thread.args_signature = ["f", "args"];
3507+
__start_new_thread.kwargs_signature = { };
3508+
__start_new_thread.types_signature = { };

‎pythonjs/runtime/builtins.py‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1288,4 +1288,25 @@ def to_ascii(self):
12881288
'loads': lambda s: JSON.parse(s),
12891289
'dumps': lambda o: JSON.stringify(o)
12901290
}
1291+
threading = {
1292+
'shared_list' : []
1293+
}
1294+
1295+
1296+
def __start_new_thread(f, args):
1297+
worker = new(Worker(f))
1298+
1299+
def func(event):
1300+
print('got signal from thread')
1301+
if event.data.type == 'terminate':
1302+
worker.terminate()
1303+
elif event.data.type == 'append':
1304+
print('got append event')
1305+
threading.shared_list.push( event.data.value )
1306+
else:
1307+
print('unknown event')
1308+
1309+
worker.onmessage = func
1310+
worker.postMessage( {'type':'execute', 'args':args} )
1311+
return worker
12911312

‎pythonjs/translator.py‎

Lines changed: 12 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
#!/usr/bin/env python
2-
import sys, traceback
2+
import sys, traceback, json
33

44
from python_to_pythonjs import main as python_to_pythonjs
55
from pythonjs import main as pythonjs_to_javascript
@@ -39,7 +39,13 @@ def main(script):
3939
code = pythonjs_to_luajs( a )
4040
else:
4141
a = python_to_pythonjs(script)
42-
code = pythonjs_to_javascript( a )
42+
if isinstance(a, dict):
43+
res = {}
44+
for jsfile in a:
45+
res[ jsfile ] = pythonjs_to_javascript( a[jsfile] )
46+
return res
47+
else:
48+
code = pythonjs_to_javascript( a )
4349

4450
return code
4551

@@ -59,7 +65,10 @@ def command():
5965
data = sys.stdin.read()
6066

6167
js = main(data)
62-
print(js)
68+
if isinstance(js, dict):
69+
print( json.dumps(js) )
70+
else:
71+
print(js)
6372

6473

6574
if __name__ == '__main__':

0 commit comments

Comments
 (0)