From ff8a7011c50251c0d99a44b80e0bbce5148ddde6 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 16:06:28 -0700 Subject: [PATCH 01/69] drop in apply_request handler in zmq.ipkernel first step in merging kernel/streamkernel --- IPython/zmq/ipkernel.py | 143 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 142 insertions(+), 1 deletion(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index d57103ad4a4..6a1639ac96d 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -136,14 +136,22 @@ def __init__(self, **kwargs): # Build dict of handlers for message types msg_types = [ 'execute_request', 'complete_request', 'object_info_request', 'history_request', - 'connect_request', 'shutdown_request'] + 'connect_request', 'shutdown_request', + 'apply_request', + ] self.handlers = {} for msg_type in msg_types: self.handlers[msg_type] = getattr(self, msg_type) + control_msg_types = [ 'clear_request', 'abort_request' ] + self.control_handlers = {} + for msg_type in control_msg_types: + self.control_handlers[msg_type] = getattr(self, msg_type) + def do_one_iteration(self): """Do one iteration of the kernel's evaluation loop. """ + try: ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK) except Exception: @@ -435,6 +443,139 @@ def shutdown_request(self, ident, parent): parent['content'], parent) sys.exit(0) + #--------------------------------------------------------------------------- + # Engine methods + #--------------------------------------------------------------------------- + + def apply_request(self, stream, ident, parent): + # flush previous reply, so this request won't block it + stream.flush(zmq.POLLOUT) + try: + content = parent[u'content'] + bufs = parent[u'buffers'] + msg_id = parent['header']['msg_id'] + # bound = parent['header'].get('bound', False) + except: + self.log.error("Got bad msg: %s"%parent, exc_info=True) + return + # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) + # self.iopub_stream.send(pyin_msg) + # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent) + sub = {'dependencies_met' : True, 'engine' : self.ident, + 'started': datetime.now()} + try: + # allow for not overriding displayhook + if hasattr(sys.displayhook, 'set_parent'): + sys.displayhook.set_parent(parent) + sys.stdout.set_parent(parent) + sys.stderr.set_parent(parent) + # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns + working = self.user_ns + # suffix = + prefix = "_"+str(msg_id).replace("-","")+"_" + + f,args,kwargs = unpack_apply_message(bufs, working, copy=False) + # if bound: + # bound_ns = Namespace(working) + # args = [bound_ns]+list(args) + + fname = getattr(f, '__name__', 'f') + + fname = prefix+"f" + argname = prefix+"args" + kwargname = prefix+"kwargs" + resultname = prefix+"result" + + ns = { fname : f, argname : args, kwargname : kwargs , resultname : None } + # print ns + working.update(ns) + code = "%s=%s(*%s,**%s)"%(resultname, fname, argname, kwargname) + try: + exec code in working,working + result = working.get(resultname) + finally: + for key in ns.iterkeys(): + working.pop(key) + # if bound: + # working.update(bound_ns) + + packed_result,buf = serialize_object(result) + result_buf = [packed_result]+buf + except: + exc_content = self._wrap_exception('apply') + # exc_msg = self.session.msg(u'pyerr', exc_content, parent) + self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, + ident=asbytes('%s.pyerr'%self.prefix)) + reply_content = exc_content + result_buf = [] + + if exc_content['ename'] == 'UnmetDependency': + sub['dependencies_met'] = False + else: + reply_content = {'status' : 'ok'} + + # put 'ok'/'error' status in header, for scheduler introspection: + sub['status'] = reply_content['status'] + + # flush i/o + sys.stdout.flush() + sys.stderr.flush() + + reply_msg = self.session.send(stream, u'apply_reply', reply_content, + parent=parent, ident=ident,buffers=result_buf, subheader=sub) + + #--------------------------------------------------------------------------- + # Control messages + #--------------------------------------------------------------------------- + + def abort_queues(self): + for stream in self.shell_streams: + if stream: + self.abort_queue(stream) + + def abort_queue(self, stream): + while True: + idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) + if msg is None: + return + + self.log.info("Aborting:") + self.log.info(str(msg)) + msg_type = msg['header']['msg_type'] + reply_type = msg_type.split('_')[0] + '_reply' + # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) + # self.reply_socket.send(ident,zmq.SNDMORE) + # self.reply_socket.send_json(reply_msg) + reply_msg = self.session.send(stream, reply_type, + content={'status' : 'aborted'}, parent=msg, ident=idents) + self.log.debug(str(reply_msg)) + # We need to wait a bit for requests to come in. This can probably + # be set shorter for true asynchronous clients. + time.sleep(0.05) + + def abort_request(self, stream, ident, parent): + """abort a specifig msg by id""" + msg_ids = parent['content'].get('msg_ids', None) + if isinstance(msg_ids, basestring): + msg_ids = [msg_ids] + if not msg_ids: + self.abort_queues() + for mid in msg_ids: + self.aborted.add(str(mid)) + + content = dict(status='ok') + reply_msg = self.session.send(stream, 'abort_reply', content=content, + parent=parent, ident=ident) + self.log.debug(str(reply_msg)) + + def clear_request(self, stream, idents, parent): + """Clear our namespace.""" + self.user_ns = {} + msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, + content = dict(status='ok')) + self._initial_exec_lines() + + #--------------------------------------------------------------------------- # Protected interface #--------------------------------------------------------------------------- From 09cd42ad5671e0d5d4e87c6b14664ed7ee50eddc Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 16:09:04 -0700 Subject: [PATCH 02/69] lazy-formatting logs in ipkernel prevent unnecessary formatting of log messages not displayed. --- IPython/zmq/ipkernel.py | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 6a1639ac96d..1f5e9615de9 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -142,7 +142,7 @@ def __init__(self, **kwargs): self.handlers = {} for msg_type in msg_types: self.handlers[msg_type] = getattr(self, msg_type) - + control_msg_types = [ 'clear_request', 'abort_request' ] self.control_handlers = {} for msg_type in control_msg_types: @@ -151,7 +151,7 @@ def __init__(self, **kwargs): def do_one_iteration(self): """Do one iteration of the kernel's evaluation loop. """ - + try: ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK) except Exception: @@ -170,13 +170,13 @@ def do_one_iteration(self): # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. - self.log.debug('\n*** MESSAGE TYPE:'+str(msg_type)+'***') - self.log.debug(' Content: '+str(msg['content'])+'\n --->\n ') + self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) + self.log.debug(' Content: %s\n --->\n ', msg['content']) # Find and call actual handler for message handler = self.handlers.get(msg_type, None) if handler is None: - self.log.error("UNKNOWN MESSAGE TYPE:" +str(msg)) + self.log.error("UNKNOWN MESSAGE TYPE: %s", msg) else: handler(ident, msg) @@ -267,7 +267,7 @@ def execute_request(self, ident, parent): silent = content[u'silent'] except: self.log.error("Got bad msg: ") - self.log.error(str(Message(parent))) + self.log.error("%s", parent) return shell = self.shell # we'll need this a lot here @@ -365,7 +365,7 @@ def execute_request(self, ident, parent): reply_content = json_clean(reply_content) reply_msg = self.session.send(self.shell_socket, u'execute_reply', reply_content, parent, ident=ident) - self.log.debug(str(reply_msg)) + self.log.debug("%s", reply_msg) if reply_msg['content']['status'] == u'error': self._abort_queue() @@ -383,7 +383,7 @@ def complete_request(self, ident, parent): matches = json_clean(matches) completion_msg = self.session.send(self.shell_socket, 'complete_reply', matches, parent, ident) - self.log.debug(str(completion_msg)) + self.log.debug("%s", completion_msg) def object_info_request(self, ident, parent): content = parent['content'] @@ -394,7 +394,7 @@ def object_info_request(self, ident, parent): oinfo = json_clean(object_info) msg = self.session.send(self.shell_socket, 'object_info_reply', oinfo, parent, ident) - self.log.debug(msg) + self.log.debug("%s", msg) def history_request(self, ident, parent): # We need to pull these out, as passing **kwargs doesn't work with @@ -435,7 +435,7 @@ def connect_request(self, ident, parent): content = {} msg = self.session.send(self.shell_socket, 'connect_reply', content, parent, ident) - self.log.debug(msg) + self.log.debug("%s", msg) def shutdown_request(self, ident, parent): self.shell.exit_now = True @@ -456,7 +456,7 @@ def apply_request(self, stream, ident, parent): msg_id = parent['header']['msg_id'] # bound = parent['header'].get('bound', False) except: - self.log.error("Got bad msg: %s"%parent, exc_info=True) + self.log.error("Got bad msg: %s", parent, exc_info=True) return # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) # self.iopub_stream.send(pyin_msg) @@ -520,7 +520,7 @@ def apply_request(self, stream, ident, parent): # flush i/o sys.stdout.flush() sys.stderr.flush() - + reply_msg = self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident,buffers=result_buf, subheader=sub) @@ -540,7 +540,7 @@ def abort_queue(self, stream): return self.log.info("Aborting:") - self.log.info(str(msg)) + self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) @@ -548,7 +548,7 @@ def abort_queue(self, stream): # self.reply_socket.send_json(reply_msg) reply_msg = self.session.send(stream, reply_type, content={'status' : 'aborted'}, parent=msg, ident=idents) - self.log.debug(str(reply_msg)) + self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. time.sleep(0.05) @@ -566,7 +566,7 @@ def abort_request(self, stream, ident, parent): content = dict(status='ok') reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) - self.log.debug(str(reply_msg)) + self.log.debug("%s", reply_msg) def clear_request(self, stream, idents, parent): """Clear our namespace.""" @@ -593,12 +593,12 @@ def _abort_queue(self): assert ident is not None, \ "Unexpected missing message part." - self.log.debug("Aborting:\n"+str(Message(msg))) + self.log.debug("Aborting:\n%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' reply_msg = self.session.send(self.shell_socket, reply_type, {'status' : 'aborted'}, msg, ident=ident) - self.log.debug(reply_msg) + self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. time.sleep(0.1) @@ -631,7 +631,7 @@ def _raw_input(self, prompt, ident, parent): value = reply['content']['value'] except: self.log.error("Got bad raw_input reply: ") - self.log.error(str(Message(parent))) + self.log.error("%s", parent) value = '' if value == '\x04': # EOF @@ -688,7 +688,7 @@ def _at_shutdown(self): if self._shutdown_message is not None: self.session.send(self.shell_socket, self._shutdown_message) self.session.send(self.iopub_socket, self._shutdown_message) - self.log.debug(str(self._shutdown_message)) + self.log.debug("%s", self._shutdown_message) # A very short sleep to give zmq time to flush its message buffers # before Python truly shuts down. time.sleep(0.01) From 1901b79b52e8840c19d8d18a6d45b3295d5a17e4 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 16:46:13 -0700 Subject: [PATCH 03/69] allow multiple shell_sockets in ipkernel --- IPython/zmq/ipkernel.py | 100 ++++++++++++++++++++++++---------------- 1 file changed, 60 insertions(+), 40 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 1f5e9615de9..2075f56ab3f 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -42,7 +42,7 @@ from IPython.utils.frame import extract_module_locals from IPython.utils.jsonutil import json_clean from IPython.utils.traitlets import ( - Any, Instance, Float, Dict, CaselessStrEnum + Any, Instance, Float, Dict, CaselessStrEnum, List ) from entry_point import base_launch_kernel @@ -67,9 +67,9 @@ class Kernel(Configurable): shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') session = Instance(Session) profile_dir = Instance('IPython.core.profiledir.ProfileDir') - shell_socket = Instance('zmq.Socket') + shell_sockets = List() + control_socket = Instance('zmq.Socket') iopub_socket = Instance('zmq.Socket') - stdin_socket = Instance('zmq.Socket') log = Instance(logging.Logger) user_module = Instance('types.ModuleType') @@ -152,20 +152,38 @@ def do_one_iteration(self): """Do one iteration of the kernel's evaluation loop. """ - try: - ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK) - except Exception: - self.log.warn("Invalid Message:", exc_info=True) - return - if msg is None: - return + # always flush control socket first + while True: + if self.control_socket is None: + break + try: + idents,msg = self.session.recv(self.control_socket, zmq.NOBLOCK) + except Exception: + self.log.warn("Invalid Control Message:", exc_info=True) + continue + if msg is None: + break + self.dispatch_message(self.control_socket, idents, msg, self.control_handlers) + + for socket in self.shell_sockets: + try: + idents,msg = self.session.recv(socket, zmq.NOBLOCK) + except Exception: + self.log.warn("Invalid Message:", exc_info=True) + continue + + if msg is None: + continue + + self.dispatch_message(socket, idents, msg, self.handlers) + def dispatch_message(self, socket, idents, msg, handlers): msg_type = msg['header']['msg_type'] # This assert will raise in versions of zeromq 2.0.7 and lesser. # We now require 2.0.8 or above, so we can uncomment for safety. # print(ident,msg, file=sys.__stdout__) - assert ident is not None, "Missing message part." + assert idents is not None, "Missing message part." # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each @@ -174,11 +192,11 @@ def do_one_iteration(self): self.log.debug(' Content: %s\n --->\n ', msg['content']) # Find and call actual handler for message - handler = self.handlers.get(msg_type, None) + handler = handlers.get(msg_type, None) if handler is None: self.log.error("UNKNOWN MESSAGE TYPE: %s", msg) else: - handler(ident, msg) + handler(socket, idents, msg) # Check whether we should exit, in case the incoming message set the # exit flag on @@ -196,7 +214,11 @@ def start(self): # let's ignore (SIG_IGN) them until we're in a place to handle them properly signal(SIGINT,SIG_IGN) poller = zmq.Poller() - poller.register(self.shell_socket, zmq.POLLIN) + for socket in self.shell_sockets: + poller.register(socket, zmq.POLLIN) + if self.control_socket: + poller.register(self.control_socket, zmq.POLLIN) + # loop while self.eventloop has not been overridden while self.eventloop is None: try: @@ -254,7 +276,7 @@ def _publish_pyin(self, code, parent, execution_count): self.session.send(self.iopub_socket, u'pyin', {u'code':code, u'execution_count': execution_count}, parent=parent) - def execute_request(self, ident, parent): + def execute_request(self, socket, ident, parent): self.session.send(self.iopub_socket, u'status', @@ -363,7 +385,7 @@ def execute_request(self, ident, parent): # Send the reply. reply_content = json_clean(reply_content) - reply_msg = self.session.send(self.shell_socket, u'execute_reply', + reply_msg = self.session.send(socket, u'execute_reply', reply_content, parent, ident=ident) self.log.debug("%s", reply_msg) @@ -375,28 +397,28 @@ def execute_request(self, ident, parent): {u'execution_state':u'idle'}, parent=parent ) - def complete_request(self, ident, parent): + def complete_request(self, socket, ident, parent): txt, matches = self._complete(parent) matches = {'matches' : matches, 'matched_text' : txt, 'status' : 'ok'} matches = json_clean(matches) - completion_msg = self.session.send(self.shell_socket, 'complete_reply', + completion_msg = self.session.send(socket, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) - def object_info_request(self, ident, parent): + def object_info_request(self, socket, ident, parent): content = parent['content'] object_info = self.shell.object_inspect(content['oname'], detail_level = content.get('detail_level', 0) ) # Before we send this object over, we scrub it for JSON usage oinfo = json_clean(object_info) - msg = self.session.send(self.shell_socket, 'object_info_reply', + msg = self.session.send(socket, 'object_info_reply', oinfo, parent, ident) self.log.debug("%s", msg) - def history_request(self, ident, parent): + def history_request(self, socket, ident, parent): # We need to pull these out, as passing **kwargs doesn't work with # unicode keys before Python 2.6.5. hist_access_type = parent['content']['hist_access_type'] @@ -424,16 +446,16 @@ def history_request(self, ident, parent): hist = list(hist) content = {'history' : hist} content = json_clean(content) - msg = self.session.send(self.shell_socket, 'history_reply', + msg = self.session.send(socket, 'history_reply', content, parent, ident) self.log.debug("Sending history reply with %i entries", len(hist)) - def connect_request(self, ident, parent): + def connect_request(self, socket, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} - msg = self.session.send(self.shell_socket, 'connect_reply', + msg = self.session.send(socket, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) @@ -447,9 +469,7 @@ def shutdown_request(self, ident, parent): # Engine methods #--------------------------------------------------------------------------- - def apply_request(self, stream, ident, parent): - # flush previous reply, so this request won't block it - stream.flush(zmq.POLLOUT) + def apply_request(self, socket, ident, parent): try: content = parent[u'content'] bufs = parent[u'buffers'] @@ -504,7 +524,7 @@ def apply_request(self, stream, ident, parent): except: exc_content = self._wrap_exception('apply') # exc_msg = self.session.msg(u'pyerr', exc_content, parent) - self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, + self.session.send(self.iopub_socket, u'pyerr', exc_content, parent=parent, ident=asbytes('%s.pyerr'%self.prefix)) reply_content = exc_content result_buf = [] @@ -521,7 +541,7 @@ def apply_request(self, stream, ident, parent): sys.stdout.flush() sys.stderr.flush() - reply_msg = self.session.send(stream, u'apply_reply', reply_content, + reply_msg = self.session.send(socket, u'apply_reply', reply_content, parent=parent, ident=ident,buffers=result_buf, subheader=sub) #--------------------------------------------------------------------------- @@ -529,13 +549,13 @@ def apply_request(self, stream, ident, parent): #--------------------------------------------------------------------------- def abort_queues(self): - for stream in self.shell_streams: - if stream: - self.abort_queue(stream) + for socket in self.shell_sockets: + if socket: + self.abort_queue(socket) - def abort_queue(self, stream): + def abort_queue(self, socket): while True: - idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) + idents,msg = self.session.recv(socket, zmq.NOBLOCK, content=True) if msg is None: return @@ -546,14 +566,14 @@ def abort_queue(self, stream): # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) # self.reply_socket.send(ident,zmq.SNDMORE) # self.reply_socket.send_json(reply_msg) - reply_msg = self.session.send(stream, reply_type, + reply_msg = self.session.send(socket, reply_type, content={'status' : 'aborted'}, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. time.sleep(0.05) - def abort_request(self, stream, ident, parent): + def abort_request(self, socket, ident, parent): """abort a specifig msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, basestring): @@ -564,14 +584,14 @@ def abort_request(self, stream, ident, parent): self.aborted.add(str(mid)) content = dict(status='ok') - reply_msg = self.session.send(stream, 'abort_reply', content=content, + reply_msg = self.session.send(socket, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) - def clear_request(self, stream, idents, parent): + def clear_request(self, socket, idents, parent): """Clear our namespace.""" self.user_ns = {} - msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, + msg = self.session.send(socket, 'clear_reply', ident=idents, parent=parent, content = dict(status='ok')) self._initial_exec_lines() @@ -746,7 +766,7 @@ def initialize(self, argv=None): def init_kernel(self): kernel = Kernel(config=self.config, session=self.session, - shell_socket=self.shell_socket, + shell_sockets=[self.shell_socket], iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, From 1587a7f2eacf077ad99c0890e8fd7ddbea195fa2 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 16:56:03 -0700 Subject: [PATCH 04/69] allow aborting tasks --- IPython/zmq/ipkernel.py | 77 +++++++++++++++++++---------------------- 1 file changed, 35 insertions(+), 42 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 2075f56ab3f..8809ed6221d 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -42,7 +42,7 @@ from IPython.utils.frame import extract_module_locals from IPython.utils.jsonutil import json_clean from IPython.utils.traitlets import ( - Any, Instance, Float, Dict, CaselessStrEnum, List + Any, Instance, Float, Dict, CaselessStrEnum, List, Set ) from entry_point import base_launch_kernel @@ -84,7 +84,7 @@ def _user_ns_changed(self, name, old, new): self.shell.init_user_ns() # Private interface - + # Time to sleep after flushing the stdout/err buffers in each execute # cycle. While this introduces a hard limit on the minimal latency of the # execute cycle, it helps prevent output synchronization problems for @@ -109,6 +109,9 @@ def _user_ns_changed(self, name, old, new): # This is a dict of port number that the kernel is listening on. It is set # by record_ports and used by connect_request. _recorded_ports = Dict() + + # set of aborted msg_ids + aborted = Set() @@ -179,6 +182,7 @@ def do_one_iteration(self): def dispatch_message(self, socket, idents, msg, handlers): msg_type = msg['header']['msg_type'] + msg_id = msg['header']['msg_id'] # This assert will raise in versions of zeromq 2.0.7 and lesser. # We now require 2.0.8 or above, so we can uncomment for safety. @@ -191,6 +195,17 @@ def dispatch_message(self, socket, idents, msg, handlers): self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) + # check if request has been aborted + if msg_id in self.aborted: + self.aborted.remove(msg_id) + # is it safe to assume a msg_id will not be resubmitted? + reply_type = msg_type.split('_')[0] + '_reply' + status = {'status' : 'aborted'} + reply_msg = self.session.send(socket, reply_type, subheader=status, + content=status, parent=msg, ident=idents) + return + + # Find and call actual handler for message handler = handlers.get(msg_type, None) if handler is None: @@ -390,7 +405,7 @@ def execute_request(self, socket, ident, parent): self.log.debug("%s", reply_msg) if reply_msg['content']['status'] == u'error': - self._abort_queue() + self._abort_queues() self.session.send(self.iopub_socket, u'status', @@ -548,31 +563,6 @@ def apply_request(self, socket, ident, parent): # Control messages #--------------------------------------------------------------------------- - def abort_queues(self): - for socket in self.shell_sockets: - if socket: - self.abort_queue(socket) - - def abort_queue(self, socket): - while True: - idents,msg = self.session.recv(socket, zmq.NOBLOCK, content=True) - if msg is None: - return - - self.log.info("Aborting:") - self.log.info("%s", msg) - msg_type = msg['header']['msg_type'] - reply_type = msg_type.split('_')[0] + '_reply' - # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) - # self.reply_socket.send(ident,zmq.SNDMORE) - # self.reply_socket.send_json(reply_msg) - reply_msg = self.session.send(socket, reply_type, - content={'status' : 'aborted'}, parent=msg, ident=idents) - self.log.debug("%s", reply_msg) - # We need to wait a bit for requests to come in. This can probably - # be set shorter for true asynchronous clients. - time.sleep(0.05) - def abort_request(self, socket, ident, parent): """abort a specifig msg by id""" msg_ids = parent['content'].get('msg_ids', None) @@ -600,28 +590,31 @@ def clear_request(self, socket, idents, parent): # Protected interface #--------------------------------------------------------------------------- - def _abort_queue(self): + def _abort_queues(self): + for socket in self.shell_sockets: + if socket: + self._abort_queue(socket) + + def _abort_queue(self, socket): while True: - try: - ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK) - except Exception: - self.log.warn("Invalid Message:", exc_info=True) - continue + idents,msg = self.session.recv(socket, zmq.NOBLOCK, content=True) if msg is None: - break - else: - assert ident is not None, \ - "Unexpected missing message part." + return - self.log.debug("Aborting:\n%s", msg) + self.log.info("Aborting:") + self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' - reply_msg = self.session.send(self.shell_socket, reply_type, - {'status' : 'aborted'}, msg, ident=ident) + # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) + # self.reply_socket.send(ident,zmq.SNDMORE) + # self.reply_socket.send_json(reply_msg) + reply_msg = self.session.send(socket, reply_type, + content={'status' : 'aborted'}, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. - time.sleep(0.1) + time.sleep(0.05) + def _no_raw_input(self): """Raise StdinNotImplentedError if active frontend doesn't support From fbf9e2e3777a779bc9ae4103f1cf04353b47afff Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 20:59:30 -0700 Subject: [PATCH 05/69] respect copy flag in Session.recv --- IPython/zmq/session.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/zmq/session.py b/IPython/zmq/session.py index 46eedd71980..04d70d219c4 100644 --- a/IPython/zmq/session.py +++ b/IPython/zmq/session.py @@ -629,7 +629,7 @@ def recv(self, socket, mode=zmq.NOBLOCK, content=True, copy=True): if isinstance(socket, ZMQStream): socket = socket.socket try: - msg_list = socket.recv_multipart(mode) + msg_list = socket.recv_multipart(mode, copy=copy) except zmq.ZMQError as e: if e.errno == zmq.EAGAIN: # We can convert EAGAIN to None as we know in this case From d362026e1460889ed88706f7dd1d89affb9b6ca1 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 20:59:56 -0700 Subject: [PATCH 06/69] Don't copy shell requests --- IPython/zmq/ipkernel.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 8809ed6221d..820de8199f8 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -170,7 +170,7 @@ def do_one_iteration(self): for socket in self.shell_sockets: try: - idents,msg = self.session.recv(socket, zmq.NOBLOCK) + idents,msg = self.session.recv(socket, zmq.NOBLOCK, copy=False) except Exception: self.log.warn("Invalid Message:", exc_info=True) continue From ed57dafb2786bc2e15b761acc403a921f1502a89 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 21:11:56 -0700 Subject: [PATCH 07/69] move apply serialization into zmq.serialize --- IPython/parallel/util.py | 137 +----------------------------- IPython/zmq/ipkernel.py | 6 +- IPython/zmq/serialize.py | 179 +++++++++++++++++++++++++++++++++++++++ 3 files changed, 187 insertions(+), 135 deletions(-) create mode 100644 IPython/zmq/serialize.py diff --git a/IPython/parallel/util.py b/IPython/parallel/util.py index f96704af384..03c0cf91498 100644 --- a/IPython/parallel/util.py +++ b/IPython/parallel/util.py @@ -47,6 +47,9 @@ from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence from IPython.utils.newserialized import serialize, unserialize from IPython.zmq.log import EnginePUBHandler +from IPython.zmq.serialize import ( + unserialize_object, serialize_object, pack_apply_message, unpack_apply_message +) if py3compat.PY3: buffer = memoryview @@ -222,140 +225,6 @@ def disambiguate_url(url, location=None): return "%s://%s:%s"%(proto,ip,port) -def serialize_object(obj, threshold=64e-6): - """Serialize an object into a list of sendable buffers. - - Parameters - ---------- - - obj : object - The object to be serialized - threshold : float - The threshold for not double-pickling the content. - - - Returns - ------- - ('pmd', [bufs]) : - where pmd is the pickled metadata wrapper, - bufs is a list of data buffers - """ - databuffers = [] - if isinstance(obj, (list, tuple)): - clist = canSequence(obj) - slist = map(serialize, clist) - for s in slist: - if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold: - databuffers.append(s.getData()) - s.data = None - return pickle.dumps(slist,-1), databuffers - elif isinstance(obj, dict): - sobj = {} - for k in sorted(obj.iterkeys()): - s = serialize(can(obj[k])) - if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold: - databuffers.append(s.getData()) - s.data = None - sobj[k] = s - return pickle.dumps(sobj,-1),databuffers - else: - s = serialize(can(obj)) - if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold: - databuffers.append(s.getData()) - s.data = None - return pickle.dumps(s,-1),databuffers - - -def unserialize_object(bufs): - """reconstruct an object serialized by serialize_object from data buffers.""" - bufs = list(bufs) - sobj = pickle.loads(bufs.pop(0)) - if isinstance(sobj, (list, tuple)): - for s in sobj: - if s.data is None: - s.data = bufs.pop(0) - return uncanSequence(map(unserialize, sobj)), bufs - elif isinstance(sobj, dict): - newobj = {} - for k in sorted(sobj.iterkeys()): - s = sobj[k] - if s.data is None: - s.data = bufs.pop(0) - newobj[k] = uncan(unserialize(s)) - return newobj, bufs - else: - if sobj.data is None: - sobj.data = bufs.pop(0) - return uncan(unserialize(sobj)), bufs - -def pack_apply_message(f, args, kwargs, threshold=64e-6): - """pack up a function, args, and kwargs to be sent over the wire - as a series of buffers. Any object whose data is larger than `threshold` - will not have their data copied (currently only numpy arrays support zero-copy)""" - msg = [pickle.dumps(can(f),-1)] - databuffers = [] # for large objects - sargs, bufs = serialize_object(args,threshold) - msg.append(sargs) - databuffers.extend(bufs) - skwargs, bufs = serialize_object(kwargs,threshold) - msg.append(skwargs) - databuffers.extend(bufs) - msg.extend(databuffers) - return msg - -def unpack_apply_message(bufs, g=None, copy=True): - """unpack f,args,kwargs from buffers packed by pack_apply_message() - Returns: original f,args,kwargs""" - bufs = list(bufs) # allow us to pop - assert len(bufs) >= 3, "not enough buffers!" - if not copy: - for i in range(3): - bufs[i] = bufs[i].bytes - cf = pickle.loads(bufs.pop(0)) - sargs = list(pickle.loads(bufs.pop(0))) - skwargs = dict(pickle.loads(bufs.pop(0))) - # print sargs, skwargs - f = uncan(cf, g) - for sa in sargs: - if sa.data is None: - m = bufs.pop(0) - if sa.getTypeDescriptor() in ('buffer', 'ndarray'): - # always use a buffer, until memoryviews get sorted out - sa.data = buffer(m) - # disable memoryview support - # if copy: - # sa.data = buffer(m) - # else: - # sa.data = m.buffer - else: - if copy: - sa.data = m - else: - sa.data = m.bytes - - args = uncanSequence(map(unserialize, sargs), g) - kwargs = {} - for k in sorted(skwargs.iterkeys()): - sa = skwargs[k] - if sa.data is None: - m = bufs.pop(0) - if sa.getTypeDescriptor() in ('buffer', 'ndarray'): - # always use a buffer, until memoryviews get sorted out - sa.data = buffer(m) - # disable memoryview support - # if copy: - # sa.data = buffer(m) - # else: - # sa.data = m.buffer - else: - if copy: - sa.data = m - else: - sa.data = m.bytes - - kwargs[k] = uncan(unserialize(sa), g) - - return f,args,kwargs #-------------------------------------------------------------------------- # helpers for implementing old MEC API via view.apply diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 820de8199f8..b11fef8fedb 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -22,6 +22,9 @@ import time import traceback import logging +import uuid + +from datetime import datetime from signal import ( signal, default_int_handler, SIGINT, SIG_IGN ) @@ -47,6 +50,7 @@ from entry_point import base_launch_kernel from kernelapp import KernelApp, kernel_flags, kernel_aliases +from serialize import serialize_object, unpack_apply_message from session import Session, Message from zmqshell import ZMQInteractiveShell @@ -540,7 +544,7 @@ def apply_request(self, socket, ident, parent): exc_content = self._wrap_exception('apply') # exc_msg = self.session.msg(u'pyerr', exc_content, parent) self.session.send(self.iopub_socket, u'pyerr', exc_content, parent=parent, - ident=asbytes('%s.pyerr'%self.prefix)) + ident=py3compat.str_to_bytes('%s.pyerr'%self.prefix)) reply_content = exc_content result_buf = [] diff --git a/IPython/zmq/serialize.py b/IPython/zmq/serialize.py new file mode 100644 index 00000000000..efff2d68c41 --- /dev/null +++ b/IPython/zmq/serialize.py @@ -0,0 +1,179 @@ +"""serialization utilities for apply messages + +Authors: + +* Min RK +""" +#----------------------------------------------------------------------------- +# Copyright (C) 2010-2011 The IPython Development Team +# +# Distributed under the terms of the BSD License. The full license is in +# the file COPYING, distributed as part of this software. +#----------------------------------------------------------------------------- + +#----------------------------------------------------------------------------- +# Imports +#----------------------------------------------------------------------------- + +# Standard library imports +import logging +import os +import re +import socket +import sys + +try: + import cPickle + pickle = cPickle +except: + cPickle = None + import pickle + + +# IPython imports +from IPython.utils import py3compat +from IPython.utils.pickleutil import can, uncan, canSequence, uncanSequence +from IPython.utils.newserialized import serialize, unserialize + +if py3compat.PY3: + buffer = memoryview + +#----------------------------------------------------------------------------- +# Serialization Functions +#----------------------------------------------------------------------------- + +def serialize_object(obj, threshold=64e-6): + """Serialize an object into a list of sendable buffers. + + Parameters + ---------- + + obj : object + The object to be serialized + threshold : float + The threshold for not double-pickling the content. + + + Returns + ------- + ('pmd', [bufs]) : + where pmd is the pickled metadata wrapper, + bufs is a list of data buffers + """ + databuffers = [] + if isinstance(obj, (list, tuple)): + clist = canSequence(obj) + slist = map(serialize, clist) + for s in slist: + if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold: + databuffers.append(s.getData()) + s.data = None + return pickle.dumps(slist,-1), databuffers + elif isinstance(obj, dict): + sobj = {} + for k in sorted(obj.iterkeys()): + s = serialize(can(obj[k])) + if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold: + databuffers.append(s.getData()) + s.data = None + sobj[k] = s + return pickle.dumps(sobj,-1),databuffers + else: + s = serialize(can(obj)) + if s.typeDescriptor in ('buffer', 'ndarray') or s.getDataSize() > threshold: + databuffers.append(s.getData()) + s.data = None + return pickle.dumps(s,-1),databuffers + + +def unserialize_object(bufs): + """reconstruct an object serialized by serialize_object from data buffers.""" + bufs = list(bufs) + sobj = pickle.loads(bufs.pop(0)) + if isinstance(sobj, (list, tuple)): + for s in sobj: + if s.data is None: + s.data = bufs.pop(0) + return uncanSequence(map(unserialize, sobj)), bufs + elif isinstance(sobj, dict): + newobj = {} + for k in sorted(sobj.iterkeys()): + s = sobj[k] + if s.data is None: + s.data = bufs.pop(0) + newobj[k] = uncan(unserialize(s)) + return newobj, bufs + else: + if sobj.data is None: + sobj.data = bufs.pop(0) + return uncan(unserialize(sobj)), bufs + +def pack_apply_message(f, args, kwargs, threshold=64e-6): + """pack up a function, args, and kwargs to be sent over the wire + as a series of buffers. Any object whose data is larger than `threshold` + will not have their data copied (currently only numpy arrays support zero-copy)""" + msg = [pickle.dumps(can(f),-1)] + databuffers = [] # for large objects + sargs, bufs = serialize_object(args,threshold) + msg.append(sargs) + databuffers.extend(bufs) + skwargs, bufs = serialize_object(kwargs,threshold) + msg.append(skwargs) + databuffers.extend(bufs) + msg.extend(databuffers) + return msg + +def unpack_apply_message(bufs, g=None, copy=True): + """unpack f,args,kwargs from buffers packed by pack_apply_message() + Returns: original f,args,kwargs""" + bufs = list(bufs) # allow us to pop + assert len(bufs) >= 3, "not enough buffers!" + if not copy: + for i in range(3): + bufs[i] = bufs[i].bytes + cf = pickle.loads(bufs.pop(0)) + sargs = list(pickle.loads(bufs.pop(0))) + skwargs = dict(pickle.loads(bufs.pop(0))) + # print sargs, skwargs + f = uncan(cf, g) + for sa in sargs: + if sa.data is None: + m = bufs.pop(0) + if sa.getTypeDescriptor() in ('buffer', 'ndarray'): + # always use a buffer, until memoryviews get sorted out + sa.data = buffer(m) + # disable memoryview support + # if copy: + # sa.data = buffer(m) + # else: + # sa.data = m.buffer + else: + if copy: + sa.data = m + else: + sa.data = m.bytes + + args = uncanSequence(map(unserialize, sargs), g) + kwargs = {} + for k in sorted(skwargs.iterkeys()): + sa = skwargs[k] + if sa.data is None: + m = bufs.pop(0) + if sa.getTypeDescriptor() in ('buffer', 'ndarray'): + # always use a buffer, until memoryviews get sorted out + sa.data = buffer(m) + # disable memoryview support + # if copy: + # sa.data = buffer(m) + # else: + # sa.data = m.buffer + else: + if copy: + sa.data = m + else: + sa.data = m.bytes + + kwargs[k] = uncan(unserialize(sa), g) + + return f,args,kwargs + From 6174cb0a73e7186f9bb4323e15e851531bf5c4c9 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 21:21:47 -0700 Subject: [PATCH 08/69] add ids to ipkernel --- IPython/zmq/ipkernel.py | 17 +++++++++++++---- 1 file changed, 13 insertions(+), 4 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index b11fef8fedb..f9d27607423 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -15,7 +15,7 @@ #----------------------------------------------------------------------------- from __future__ import print_function -# Standard library imports. +# Standard library imports import __builtin__ import atexit import sys @@ -28,10 +28,11 @@ from signal import ( signal, default_int_handler, SIGINT, SIG_IGN ) -# System library imports. + +# System library imports import zmq -# Local imports. +# Local imports from IPython.core import pylabtools from IPython.config.configurable import Configurable from IPython.config.application import boolean_flag, catch_config_error @@ -45,7 +46,7 @@ from IPython.utils.frame import extract_module_locals from IPython.utils.jsonutil import json_clean from IPython.utils.traitlets import ( - Any, Instance, Float, Dict, CaselessStrEnum, List, Set + Any, Instance, Float, Dict, CaselessStrEnum, List, Set, Integer, Unicode ) from entry_point import base_launch_kernel @@ -87,6 +88,14 @@ def _user_ns_changed(self, name, old, new): self.shell.user_ns = new self.shell.init_user_ns() + # identities: + int_id = Integer(-1) + ident = Unicode() + + def _ident_default(self): + return unicode(uuid.uuid4()) + + # Private interface # Time to sleep after flushing the stdout/err buffers in each execute From 9440f9e68f5c63761392dedef3b167c8af076717 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 23:34:26 -0700 Subject: [PATCH 09/69] use IOLoop in ipkernel some eventloop integration updated to match --- IPython/zmq/eventloops.py | 5 +- IPython/zmq/ipkernel.py | 281 ++++++++++++++++++-------------------- IPython/zmq/kernelapp.py | 15 +- 3 files changed, 147 insertions(+), 154 deletions(-) diff --git a/IPython/zmq/eventloops.py b/IPython/zmq/eventloops.py index 3b8b89535ec..594c11c36d4 100644 --- a/IPython/zmq/eventloops.py +++ b/IPython/zmq/eventloops.py @@ -164,7 +164,10 @@ def doi(): # but still need a Poller for when there are no active windows, # during which time mainloop() returns immediately poller = zmq.Poller() - poller.register(kernel.shell_socket, zmq.POLLIN) + if kernel.control_stream: + poller.register(kernel.control_stream.socket, zmq.POLLIN) + for stream in kernel.shell_streams: + poller.register(stream.socket, zmq.POLLIN) while True: try: diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index f9d27607423..76a14c5c04a 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -26,11 +26,13 @@ from datetime import datetime from signal import ( - signal, default_int_handler, SIGINT, SIG_IGN + signal, getsignal, default_int_handler, SIGINT, SIG_IGN ) # System library imports import zmq +from zmq.eventloop import ioloop +from zmq.eventloop.zmqstream import ZMQStream # Local imports from IPython.core import pylabtools @@ -68,13 +70,18 @@ class Kernel(Configurable): # attribute to override with a GUI eventloop = Any(None) + def _eventloop_changed(self, name, old, new): + """schedule call to eventloop from IOLoop""" + loop = ioloop.IOLoop.instance() + loop.add_timeout(time.time()+0.1, self.enter_eventloop) shell = Instance('IPython.core.interactiveshell.InteractiveShellABC') session = Instance(Session) profile_dir = Instance('IPython.core.profiledir.ProfileDir') - shell_sockets = List() - control_socket = Instance('zmq.Socket') - iopub_socket = Instance('zmq.Socket') + shell_streams = List() + control_stream = Instance(ZMQStream) + iopub_stream = Instance(ZMQStream) + stdin_socket = Instance(zmq.Socket) log = Instance(logging.Logger) user_module = Instance('types.ModuleType') @@ -127,14 +134,9 @@ def _ident_default(self): aborted = Set() - def __init__(self, **kwargs): super(Kernel, self).__init__(**kwargs) - # Before we even start up the shell, register *first* our exit handlers - # so they come before the shell's - atexit.register(self._at_shutdown) - # Initialize the InteractiveShell subclass self.shell = ZMQInteractiveShell.instance(config=self.config, profile_dir = self.profile_dir, @@ -142,9 +144,9 @@ def __init__(self, **kwargs): user_ns = self.user_ns, ) self.shell.displayhook.session = self.session - self.shell.displayhook.pub_socket = self.iopub_socket + self.shell.displayhook.pub_socket = self.iopub_stream.socket self.shell.display_pub.session = self.session - self.shell.display_pub.pub_socket = self.iopub_socket + self.shell.display_pub.pub_socket = self.iopub_stream.socket # TMP - hack while developing self.shell._reply_content = None @@ -155,135 +157,111 @@ def __init__(self, **kwargs): 'connect_request', 'shutdown_request', 'apply_request', ] - self.handlers = {} + self.shell_handlers = {} for msg_type in msg_types: - self.handlers[msg_type] = getattr(self, msg_type) + self.shell_handlers[msg_type] = getattr(self, msg_type) control_msg_types = [ 'clear_request', 'abort_request' ] self.control_handlers = {} for msg_type in control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) - def do_one_iteration(self): - """Do one iteration of the kernel's evaluation loop. - """ - - # always flush control socket first - while True: - if self.control_socket is None: - break - try: - idents,msg = self.session.recv(self.control_socket, zmq.NOBLOCK) - except Exception: - self.log.warn("Invalid Control Message:", exc_info=True) - continue - if msg is None: - break - self.dispatch_message(self.control_socket, idents, msg, self.control_handlers) - - for socket in self.shell_sockets: - try: - idents,msg = self.session.recv(socket, zmq.NOBLOCK, copy=False) - except Exception: - self.log.warn("Invalid Message:", exc_info=True) - continue - - if msg is None: - continue - - self.dispatch_message(socket, idents, msg, self.handlers) + def dispatch_control(self, msg): + """dispatch control requests""" + idents,msg = self.session.feed_identities(msg, copy=False) + try: + msg = self.session.unserialize(msg, content=True, copy=False) + except: + self.log.error("Invalid Control Message", exc_info=True) + return - def dispatch_message(self, socket, idents, msg, handlers): - msg_type = msg['header']['msg_type'] - msg_id = msg['header']['msg_id'] + self.log.debug("Control received: %s", msg) - # This assert will raise in versions of zeromq 2.0.7 and lesser. - # We now require 2.0.8 or above, so we can uncomment for safety. - # print(ident,msg, file=sys.__stdout__) - assert idents is not None, "Missing message part." + header = msg['header'] + msg_id = header['msg_id'] + msg_type = header['msg_type'] + handler = self.control_handlers.get(msg_type, None) + if handler is None: + self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) + else: + handler(self.control_stream, idents, msg) + + def dispatch_shell(self, stream, msg): + """dispatch shell requests""" + # flush control requests first + if self.control_stream: + self.control_stream.flush() + + idents,msg = self.session.feed_identities(msg, copy=False) + try: + msg = self.session.unserialize(msg, content=True, copy=False) + except: + self.log.error("Invalid Message", exc_info=True) + return + + header = msg['header'] + msg_id = header['msg_id'] + msg_type = msg['header']['msg_type'] + # Print some info about this message and leave a '--->' marker, so it's # easier to trace visually the message chain when debugging. Each # handler prints its message at the end. self.log.debug('\n*** MESSAGE TYPE:%s***', msg_type) self.log.debug(' Content: %s\n --->\n ', msg['content']) - # check if request has been aborted if msg_id in self.aborted: self.aborted.remove(msg_id) # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status' : 'aborted'} - reply_msg = self.session.send(socket, reply_type, subheader=status, + reply_msg = self.session.send(stream, reply_type, subheader=status, content=status, parent=msg, ident=idents) return - - # Find and call actual handler for message - handler = handlers.get(msg_type, None) + handler = self.shell_handlers.get(msg_type, None) if handler is None: - self.log.error("UNKNOWN MESSAGE TYPE: %s", msg) + self.log.error("UNKNOWN MESSAGE TYPE: %r", msg_type) else: - handler(socket, idents, msg) + # ensure default_int_handler during handler call + sig = signal(SIGINT, default_int_handler) + try: + handler(stream, idents, msg) + finally: + signal(SIGINT, sig) + + def enter_eventloop(self): + """enter eventloop""" + self.log.critical("entering eventloop") + # restore default_int_handler + signal(SIGINT, default_int_handler) + self.eventloop(self) + self.log.critical("exiting eventloop") + # if eventloop exits, IOLoop should stop + ioloop.IOLoop.instance().stop() - # Check whether we should exit, in case the incoming message set the - # exit flag on - if self.shell.exit_now: - self.log.debug('\nExiting IPython kernel...') - # We do a normal, clean exit, which allows any actions registered - # via atexit (such as history saving) to take place. - sys.exit(0) + def start(self): + """register dispatchers for streams""" + if self.control_stream: + self.control_stream.on_recv(self.dispatch_control, copy=False) + def make_dispatcher(stream): + def dispatcher(msg): + return self.dispatch_shell(stream, msg) + return dispatcher - def start(self): - """ Start the kernel main loop. - """ - # a KeyboardInterrupt (SIGINT) can occur on any python statement, so - # let's ignore (SIG_IGN) them until we're in a place to handle them properly - signal(SIGINT,SIG_IGN) - poller = zmq.Poller() - for socket in self.shell_sockets: - poller.register(socket, zmq.POLLIN) - if self.control_socket: - poller.register(self.control_socket, zmq.POLLIN) - - # loop while self.eventloop has not been overridden - while self.eventloop is None: - try: - # scale by extra factor of 10, because there is no - # reason for this to be anything less than ~ 0.1s - # since it is a real poller and will respond - # to events immediately - - # double nested try/except, to properly catch KeyboardInterrupt - # due to pyzmq Issue #130 - try: - poller.poll(10*1000*self._poll_interval) - # restore raising of KeyboardInterrupt - signal(SIGINT, default_int_handler) - self.do_one_iteration() - except: - raise - finally: - # prevent raising of KeyboardInterrupt - signal(SIGINT,SIG_IGN) - except KeyboardInterrupt: - # Ctrl-C shouldn't crash the kernel - io.raw_print("KeyboardInterrupt caught in kernel") - # stop ignoring sigint, now that we are out of our own loop, - # we don't want to prevent future code from handling it - signal(SIGINT, default_int_handler) - while self.eventloop is not None: - try: - self.eventloop(self) - except KeyboardInterrupt: - # Ctrl-C shouldn't crash the kernel - io.raw_print("KeyboardInterrupt caught in kernel") - continue - else: - # eventloop exited cleanly, this means we should stop (right?) - self.eventloop = None - break + for s in self.shell_streams: + s.on_recv(make_dispatcher(s), copy=False) + + def do_one_iteration(self): + """step eventloop just once""" + if self.control_stream: + self.control_stream.flush() + for stream in self.shell_streams: + # handle at most one request per iteration + stream.flush(zmq.POLLIN, 1) + stream.flush(zmq.POLLOUT) + self.iopub_stream.flush() def record_ports(self, ports): @@ -301,12 +279,12 @@ def record_ports(self, ports): def _publish_pyin(self, code, parent, execution_count): """Publish the code request on the pyin stream.""" - self.session.send(self.iopub_socket, u'pyin', {u'code':code, + self.session.send(self.iopub_stream, u'pyin', {u'code':code, u'execution_count': execution_count}, parent=parent) - def execute_request(self, socket, ident, parent): + def execute_request(self, stream, ident, parent): - self.session.send(self.iopub_socket, + self.session.send(self.iopub_stream, u'status', {u'execution_state':u'busy'}, parent=parent ) @@ -413,40 +391,40 @@ def execute_request(self, socket, ident, parent): # Send the reply. reply_content = json_clean(reply_content) - reply_msg = self.session.send(socket, u'execute_reply', + reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent, ident=ident) self.log.debug("%s", reply_msg) if reply_msg['content']['status'] == u'error': self._abort_queues() - self.session.send(self.iopub_socket, + self.session.send(self.iopub_stream, u'status', {u'execution_state':u'idle'}, parent=parent ) - def complete_request(self, socket, ident, parent): + def complete_request(self, stream, ident, parent): txt, matches = self._complete(parent) matches = {'matches' : matches, 'matched_text' : txt, 'status' : 'ok'} matches = json_clean(matches) - completion_msg = self.session.send(socket, 'complete_reply', + completion_msg = self.session.send(stream, 'complete_reply', matches, parent, ident) self.log.debug("%s", completion_msg) - def object_info_request(self, socket, ident, parent): + def object_info_request(self, stream, ident, parent): content = parent['content'] object_info = self.shell.object_inspect(content['oname'], detail_level = content.get('detail_level', 0) ) # Before we send this object over, we scrub it for JSON usage oinfo = json_clean(object_info) - msg = self.session.send(socket, 'object_info_reply', + msg = self.session.send(stream, 'object_info_reply', oinfo, parent, ident) self.log.debug("%s", msg) - def history_request(self, socket, ident, parent): + def history_request(self, stream, ident, parent): # We need to pull these out, as passing **kwargs doesn't work with # unicode keys before Python 2.6.5. hist_access_type = parent['content']['hist_access_type'] @@ -474,30 +452,35 @@ def history_request(self, socket, ident, parent): hist = list(hist) content = {'history' : hist} content = json_clean(content) - msg = self.session.send(socket, 'history_reply', + msg = self.session.send(stream, 'history_reply', content, parent, ident) self.log.debug("Sending history reply with %i entries", len(hist)) - def connect_request(self, socket, ident, parent): + def connect_request(self, stream, ident, parent): if self._recorded_ports is not None: content = self._recorded_ports.copy() else: content = {} - msg = self.session.send(socket, 'connect_reply', + msg = self.session.send(stream, 'connect_reply', content, parent, ident) self.log.debug("%s", msg) - def shutdown_request(self, ident, parent): + def shutdown_request(self, stream, ident, parent): self.shell.exit_now = True self._shutdown_message = self.session.msg(u'shutdown_reply', - parent['content'], parent) - sys.exit(0) + parent['content'], parent + ) + # self.session.send(stream, self._shutdown_message, ident=ident) + + self._at_shutdown() + # call sys.exit after a short delay + ioloop.IOLoop.instance().add_timeout(time.time()+0.05, lambda : sys.exit(0)) #--------------------------------------------------------------------------- # Engine methods #--------------------------------------------------------------------------- - def apply_request(self, socket, ident, parent): + def apply_request(self, stream, ident, parent): try: content = parent[u'content'] bufs = parent[u'buffers'] @@ -552,7 +535,7 @@ def apply_request(self, socket, ident, parent): except: exc_content = self._wrap_exception('apply') # exc_msg = self.session.msg(u'pyerr', exc_content, parent) - self.session.send(self.iopub_socket, u'pyerr', exc_content, parent=parent, + self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, ident=py3compat.str_to_bytes('%s.pyerr'%self.prefix)) reply_content = exc_content result_buf = [] @@ -569,14 +552,14 @@ def apply_request(self, socket, ident, parent): sys.stdout.flush() sys.stderr.flush() - reply_msg = self.session.send(socket, u'apply_reply', reply_content, + reply_msg = self.session.send(stream, u'apply_reply', reply_content, parent=parent, ident=ident,buffers=result_buf, subheader=sub) #--------------------------------------------------------------------------- # Control messages #--------------------------------------------------------------------------- - def abort_request(self, socket, ident, parent): + def abort_request(self, stream, ident, parent): """abort a specifig msg by id""" msg_ids = parent['content'].get('msg_ids', None) if isinstance(msg_ids, basestring): @@ -587,14 +570,14 @@ def abort_request(self, socket, ident, parent): self.aborted.add(str(mid)) content = dict(status='ok') - reply_msg = self.session.send(socket, 'abort_reply', content=content, + reply_msg = self.session.send(stream, 'abort_reply', content=content, parent=parent, ident=ident) self.log.debug("%s", reply_msg) - def clear_request(self, socket, idents, parent): + def clear_request(self, stream, idents, parent): """Clear our namespace.""" self.user_ns = {} - msg = self.session.send(socket, 'clear_reply', ident=idents, parent=parent, + msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content = dict(status='ok')) self._initial_exec_lines() @@ -604,13 +587,13 @@ def clear_request(self, socket, idents, parent): #--------------------------------------------------------------------------- def _abort_queues(self): - for socket in self.shell_sockets: - if socket: - self._abort_queue(socket) + for stream in self.shell_streams: + if stream: + self._abort_queue(stream) - def _abort_queue(self, socket): + def _abort_queue(self, stream): while True: - idents,msg = self.session.recv(socket, zmq.NOBLOCK, content=True) + idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: return @@ -619,9 +602,9 @@ def _abort_queue(self, socket): msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) - # self.reply_socket.send(ident,zmq.SNDMORE) - # self.reply_socket.send_json(reply_msg) - reply_msg = self.session.send(socket, reply_type, + # self.reply_stream.send(ident,zmq.SNDMORE) + # self.reply_stream.send_json(reply_msg) + reply_msg = self.session.send(stream, reply_type, content={'status' : 'aborted'}, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably @@ -712,12 +695,9 @@ def _at_shutdown(self): """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: - self.session.send(self.shell_socket, self._shutdown_message) - self.session.send(self.iopub_socket, self._shutdown_message) + self.session.send(self.iopub_stream, self._shutdown_message) self.log.debug("%s", self._shutdown_message) - # A very short sleep to give zmq time to flush its message buffers - # before Python truly shuts down. - time.sleep(0.01) + [ s.flush(zmq.POLLOUT) for s in self.shell_streams + [self.iopub_stream] ] #----------------------------------------------------------------------------- # Aliases and Flags for the IPKernelApp @@ -770,10 +750,13 @@ def initialize(self, argv=None): self.init_code() def init_kernel(self): + + shell_stream = ZMQStream(self.shell_socket) + iopub_stream = ZMQStream(self.iopub_socket) kernel = Kernel(config=self.config, session=self.session, - shell_sockets=[self.shell_socket], - iopub_socket=self.iopub_socket, + shell_streams=[shell_stream], + iopub_stream=iopub_stream, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, diff --git a/IPython/zmq/kernelapp.py b/IPython/zmq/kernelapp.py index e7eec92d9e0..c3b60a3367b 100644 --- a/IPython/zmq/kernelapp.py +++ b/IPython/zmq/kernelapp.py @@ -15,15 +15,17 @@ # Imports #----------------------------------------------------------------------------- -# Standard library imports. +# Standard library imports import json import os import sys +import signal -# System library imports. +# System library imports import zmq +from zmq.eventloop import ioloop -# IPython imports. +# IPython imports from IPython.core.ultratb import FormattedTB from IPython.core.application import ( BaseIPythonApplication, base_flags, base_aliases, catch_config_error @@ -267,6 +269,9 @@ def init_io(self): displayhook_factory = import_item(str(self.displayhook_class)) sys.displayhook = displayhook_factory(self.session, self.iopub_socket) + def init_signal(self): + signal.signal(signal.SIGINT, signal.SIG_IGN) + def init_kernel(self): """Create the Kernel object itself""" kernel_factory = import_item(str(self.kernel_class)) @@ -289,6 +294,7 @@ def initialize(self, argv=None): # writing connection file must be *after* init_sockets self.write_connection_file() self.init_io() + self.init_signal() self.init_kernel() # flush stdout/stderr, so that anything written to these streams during # initialization do not get associated with the first execution request @@ -299,8 +305,9 @@ def start(self): self.heartbeat.start() if self.poller is not None: self.poller.start() + self.kernel.start() try: - self.kernel.start() + ioloop.IOLoop.instance().start() except KeyboardInterrupt: pass From 24fdfb12fffb3d59ba48d6e7729f55e620f660bf Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 23 Mar 2012 23:43:32 -0700 Subject: [PATCH 10/69] use IPython.zmq.kernel in parallel Engines --- IPython/parallel/engine/engine.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index fa0f8d46693..367666bbf78 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -34,8 +34,7 @@ from IPython.parallel.util import disambiguate_url, asbytes from IPython.zmq.session import Message - -from .streamkernel import Kernel +from IPython.zmq.ipkernel import Kernel class EngineFactory(RegistrationFactory): """IPython engine""" @@ -201,7 +200,7 @@ def complete_registration(self, msg, connect, maybe_tunnel): self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, - loop=loop, user_ns = self.user_ns, log=self.log) + loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.start() From 5c5118dffe4b537fcc437447ff6d67668c1fda66 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 00:18:11 -0700 Subject: [PATCH 11/69] allow exceptions in handlers without crashing --- IPython/zmq/ipkernel.py | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 76a14c5c04a..8ee9f960b18 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -185,7 +185,10 @@ def dispatch_control(self, msg): if handler is None: self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r", msg_type) else: - handler(self.control_stream, idents, msg) + try: + handler(self.control_stream, idents, msg) + except Exception: + self.log.error("Exception in control handler:", exc_info=True) def dispatch_shell(self, stream, msg): """dispatch shell requests""" @@ -227,6 +230,8 @@ def dispatch_shell(self, stream, msg): sig = signal(SIGINT, default_int_handler) try: handler(stream, idents, msg) + except Exception: + self.log.error("Exception in message handler:", exc_info=True) finally: signal(SIGINT, sig) @@ -363,9 +368,9 @@ def execute_request(self, stream, ident, parent): # or not. If it did, we proceed to evaluate user_variables/expressions if reply_content['status'] == 'ok': reply_content[u'user_variables'] = \ - shell.user_variables(content[u'user_variables']) + shell.user_variables(content.get(u'user_variables', [])) reply_content[u'user_expressions'] = \ - shell.user_expressions(content[u'user_expressions']) + shell.user_expressions(content.get(u'user_expressions', {})) else: # If there was an error, don't even try to compute variables or # expressions From 7f6086fec69520c4962f2b4f5acc302d2450d97b Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 00:22:54 -0700 Subject: [PATCH 12/69] use execute_request for parallel execute, instead of apply --- IPython/parallel/client/client.py | 62 ++++++++++++++++++++++++++++++- IPython/parallel/client/view.py | 26 +++++++++++-- 2 files changed, 83 insertions(+), 5 deletions(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index ef122fbcc87..a7862a2e82a 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -628,7 +628,30 @@ def _handle_execute_reply(self, msg): print ("got unknown result: %s"%msg_id) else: self.outstanding.remove(msg_id) - self.results[msg_id] = self._unwrap_exception(msg['content']) + + content = msg['content'] + header = msg['header'] + + # construct metadata: + md = self.metadata[msg_id] + md.update(self._extract_metadata(header, parent, content)) + # is this redundant? + self.metadata[msg_id] = md + + e_outstanding = self._outstanding_dict[md['engine_uuid']] + if msg_id in e_outstanding: + e_outstanding.remove(msg_id) + + # construct result: + if content['status'] == 'ok': + self.results[msg_id] = content + elif content['status'] == 'aborted': + self.results[msg_id] = error.TaskAborted(msg_id) + elif content['status'] == 'resubmitted': + # TODO: handle resubmission + pass + else: + self.results[msg_id] = self._unwrap_exception(content) def _handle_apply_reply(self, msg): """Save the reply to an apply_request into our results.""" @@ -1024,7 +1047,7 @@ def _maybe_raise(self, result): return result - def send_apply_message(self, socket, f, args=None, kwargs=None, subheader=None, track=False, + def send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, track=False, ident=None): """construct and send an apply message via a socket. @@ -1066,6 +1089,41 @@ def send_apply_message(self, socket, f, args=None, kwargs=None, subheader=None, return msg + def send_execute_request(self, socket, code, silent=False, subheader=None, ident=None): + """construct and send an execute request via a socket. + + """ + + assert not self._closed, "cannot use me anymore, I'm closed!" + # defaults: + subheader = subheader if subheader is not None else {} + + # validate arguments + if not isinstance(code, basestring): + raise TypeError("code must be text, not %s" % type(code)) + if not isinstance(subheader, dict): + raise TypeError("subheader must be dict, not %s" % type(subheader)) + + content = dict(code=code, silent=bool(silent), user_variables=[], user_expressions={}) + + + msg = self.session.send(socket, "execute_request", content=content, ident=ident, + subheader=subheader) + + msg_id = msg['header']['msg_id'] + self.outstanding.add(msg_id) + if ident: + # possibly routed to a specific engine + if isinstance(ident, list): + ident = ident[-1] + if ident in self._engines.values(): + # save for later, in case of engine death + self._outstanding_dict[ident].add(msg_id) + self.history.append(msg_id) + self.metadata[msg_id]['submitted'] = datetime.now() + + return msg + #-------------------------------------------------------------------------- # construct a View object #-------------------------------------------------------------------------- diff --git a/IPython/parallel/client/view.py b/IPython/parallel/client/view.py index dc1a078ce58..6b22c2c3a46 100644 --- a/IPython/parallel/client/view.py +++ b/IPython/parallel/client/view.py @@ -195,7 +195,7 @@ def temp_flags(self, **kwargs): @sync_results @save_ids def _really_apply(self, f, args, kwargs, block=None, **options): - """wrapper for client.send_apply_message""" + """wrapper for client.send_apply_request""" raise NotImplementedError("Implement in subclasses") def apply(self, f, *args, **kwargs): @@ -533,7 +533,7 @@ def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, tra msg_ids = [] trackers = [] for ident in _idents: - msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track, + msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track, ident=ident) if track: trackers.append(msg['tracker']) @@ -547,6 +547,7 @@ def _really_apply(self, f, args=None, kwargs=None, targets=None, block=None, tra pass return ar + @spin_after def map(self, f, *sequences, **kwargs): """view.map(f, *sequences, block=self.block) => list|AsyncMapResult @@ -590,6 +591,8 @@ def map(self, f, *sequences, **kwargs): pf = ParallelFunction(self, f, block=block, **kwargs) return pf.map(*sequences) + @sync_results + @save_ids def execute(self, code, targets=None, block=None): """Executes `code` on `targets` in blocking or nonblocking manner. @@ -604,6 +607,23 @@ def execute(self, code, targets=None, block=None): whether or not to wait until done to return default: self.block """ + block = self.block if block is None else block + targets = self.targets if targets is None else targets + + _idents = self.client._build_targets(targets)[0] + msg_ids = [] + trackers = [] + for ident in _idents: + msg = self.client.send_execute_request(self._socket, code, ident=ident) + msg_ids.append(msg['header']['msg_id']) + ar = AsyncResult(self.client, msg_ids, fname='execute', targets=targets) + if block: + try: + ar.get() + except KeyboardInterrupt: + pass + return ar + return self._really_apply(util._execute, args=(code,), block=block, targets=targets) def run(self, filename, targets=None, block=None): @@ -996,7 +1016,7 @@ def _really_apply(self, f, args=None, kwargs=None, block=None, track=None, follow = self._render_dependency(follow) subheader = dict(after=after, follow=follow, timeout=timeout, targets=idents, retries=retries) - msg = self.client.send_apply_message(self._socket, f, args, kwargs, track=track, + msg = self.client.send_apply_request(self._socket, f, args, kwargs, track=track, subheader=subheader) tracker = None if track is False else msg['tracker'] From 08b95f8c387147b55c772e70b76cccdfdf4e043c Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 00:23:07 -0700 Subject: [PATCH 13/69] scrub: ident on iopub --- IPython/zmq/ipkernel.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 8ee9f960b18..990604fb50a 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -284,15 +284,19 @@ def record_ports(self, ports): def _publish_pyin(self, code, parent, execution_count): """Publish the code request on the pyin stream.""" - self.session.send(self.iopub_stream, u'pyin', {u'code':code, - u'execution_count': execution_count}, parent=parent) + self.session.send(self.iopub_stream, u'pyin', + {u'code':code, u'execution_count': execution_count}, + parent=parent, + ) def execute_request(self, stream, ident, parent): self.session.send(self.iopub_stream, u'status', {u'execution_state':u'busy'}, - parent=parent ) + parent=parent, + ident='status', + ) try: content = parent[u'content'] From ccd2ca4c5496dde3f259bcde27e070a3a9bd2655 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 00:38:16 -0700 Subject: [PATCH 14/69] enable topics on IOPub messages --- IPython/zmq/ipkernel.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 990604fb50a..77679971669 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -286,7 +286,7 @@ def _publish_pyin(self, code, parent, execution_count): self.session.send(self.iopub_stream, u'pyin', {u'code':code, u'execution_count': execution_count}, - parent=parent, + parent=parent, ident=self._topic('pyin') ) def execute_request(self, stream, ident, parent): @@ -295,7 +295,7 @@ def execute_request(self, stream, ident, parent): u'status', {u'execution_state':u'busy'}, parent=parent, - ident='status', + ident=self._topic('status'), ) try: @@ -410,7 +410,8 @@ def execute_request(self, stream, ident, parent): self.session.send(self.iopub_stream, u'status', {u'execution_state':u'idle'}, - parent=parent ) + parent=parent, + ident=self._topic('status')) def complete_request(self, stream, ident, parent): txt, matches = self._complete(parent) @@ -545,7 +546,7 @@ def apply_request(self, stream, ident, parent): exc_content = self._wrap_exception('apply') # exc_msg = self.session.msg(u'pyerr', exc_content, parent) self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, - ident=py3compat.str_to_bytes('%s.pyerr'%self.prefix)) + ident=self._topic('pyerr')) reply_content = exc_content result_buf = [] @@ -595,6 +596,16 @@ def clear_request(self, stream, idents, parent): # Protected interface #--------------------------------------------------------------------------- + + def _topic(self, topic): + """prefixed topic for IOPub messages""" + if self.int_id >= 0: + base = "engine.%i" % self.int_id + else: + base = "kernel.%s" % self.ident + + return py3compat.cast_bytes("%s.%s" % (base, topic)) + def _abort_queues(self): for stream in self.shell_streams: if stream: @@ -704,7 +715,7 @@ def _at_shutdown(self): """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: - self.session.send(self.iopub_stream, self._shutdown_message) + self.session.send(self.iopub_stream, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) [ s.flush(zmq.POLLOUT) for s in self.shell_streams + [self.iopub_stream] ] From 02449c1400e1de8d504aff5c511e960f50c05cfe Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 00:56:15 -0700 Subject: [PATCH 15/69] use shell namespaces in apply_request --- IPython/zmq/ipkernel.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 77679971669..933820d455f 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -495,7 +495,6 @@ def apply_request(self, stream, ident, parent): content = parent[u'content'] bufs = parent[u'buffers'] msg_id = parent['header']['msg_id'] - # bound = parent['header'].get('bound', False) except: self.log.error("Got bad msg: %s", parent, exc_info=True) return @@ -510,15 +509,11 @@ def apply_request(self, stream, ident, parent): sys.displayhook.set_parent(parent) sys.stdout.set_parent(parent) sys.stderr.set_parent(parent) - # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns - working = self.user_ns - # suffix = + working = self.shell.user_ns + prefix = "_"+str(msg_id).replace("-","")+"_" f,args,kwargs = unpack_apply_message(bufs, working, copy=False) - # if bound: - # bound_ns = Namespace(working) - # args = [bound_ns]+list(args) fname = getattr(f, '__name__', 'f') @@ -530,15 +525,13 @@ def apply_request(self, stream, ident, parent): ns = { fname : f, argname : args, kwargname : kwargs , resultname : None } # print ns working.update(ns) - code = "%s=%s(*%s,**%s)"%(resultname, fname, argname, kwargname) + code = "%s = %s(*%s,**%s)" % (resultname, fname, argname, kwargname) try: - exec code in working,working + exec code in self.shell.user_global_ns, self.shell.user_ns result = working.get(resultname) finally: for key in ns.iterkeys(): working.pop(key) - # if bound: - # working.update(bound_ns) packed_result,buf = serialize_object(result) result_buf = [packed_result]+buf @@ -586,10 +579,9 @@ def abort_request(self, stream, ident, parent): def clear_request(self, stream, idents, parent): """Clear our namespace.""" - self.user_ns = {} + self.shell.reset(False) msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, content = dict(status='ok')) - self._initial_exec_lines() #--------------------------------------------------------------------------- @@ -597,6 +589,14 @@ def clear_request(self, stream, idents, parent): #--------------------------------------------------------------------------- + def _wrap_exception(self, method=None): + # import here, because _wrap_exception is only used in parallel, + # and parallel has higher min pyzmq version + from IPython.parallel.error import wrap_exception + e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method) + content = wrap_exception(e_info) + return content + def _topic(self, topic): """prefixed topic for IOPub messages""" if self.int_id >= 0: From c6717a52e8103777209db5a7af973dec10361420 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 12:12:51 -0700 Subject: [PATCH 16/69] don't abort queue on execution failure This causes many problems, and should be reconsidered. Disable for now, as it is necessary for the Kernel merge. --- IPython/zmq/ipkernel.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 933820d455f..c9574ae1d3a 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -405,7 +405,10 @@ def execute_request(self, stream, ident, parent): self.log.debug("%s", reply_msg) if reply_msg['content']['status'] == u'error': - self._abort_queues() + # FIXME: queue_abort on error is disabled + # A policy decision must be made + pass + # self._abort_queues() self.session.send(self.iopub_stream, u'status', From 716157fb0d6db653db571988234cb4da2d69dfe1 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 12:20:17 -0700 Subject: [PATCH 17/69] avoid passing code objects to execute, now that execute uses code-as-text. --- IPython/parallel/tests/test_view.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index 922019b4d88..8c524c105c0 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -421,16 +421,16 @@ def test_magic_autopx_blocking(self): sys.stdout = sio ip.magic_autopx() ip.run_cell('\n'.join(('a=5','b=10','c=0'))) - ip.run_cell('print b') + ip.run_cell('b*=2') + ip.run_cell('print (b)') ip.run_cell("b/c") - ip.run_code(compile('b*=2', '', 'single')) ip.magic_autopx() sys.stdout = savestdout output = sio.getvalue().strip() self.assertTrue(output.startswith('%autopx enabled')) self.assertTrue(output.endswith('%autopx disabled')) self.assertTrue('RemoteError: ZeroDivisionError' in output) - ar = v.get_result(-2) + ar = v.get_result(-1) self.assertEquals(v['a'], 5) self.assertEquals(v['b'], 20) self.assertRaisesRemote(ZeroDivisionError, ar.get) @@ -446,9 +446,9 @@ def test_magic_autopx_nonblocking(self): sys.stdout = sio ip.magic_autopx() ip.run_cell('\n'.join(('a=5','b=10','c=0'))) - ip.run_cell('print b') + ip.run_cell('print (b)') ip.run_cell("b/c") - ip.run_code(compile('b*=2', '', 'single')) + ip.run_cell('b*=2') ip.magic_autopx() sys.stdout = savestdout output = sio.getvalue().strip() From f9d70057460bddf1bad84ef6d73fdf15afc2485a Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 12:21:26 -0700 Subject: [PATCH 18/69] fixup shutdown/exit now that we use IOLoop various modes of shutting down should now result in exiting the loop. --- IPython/zmq/ipkernel.py | 15 ++++++++++----- IPython/zmq/zmqshell.py | 11 +++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index c9574ae1d3a..14d70d071e0 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -161,7 +161,7 @@ def __init__(self, **kwargs): for msg_type in msg_types: self.shell_handlers[msg_type] = getattr(self, msg_type) - control_msg_types = [ 'clear_request', 'abort_request' ] + control_msg_types = msg_types + [ 'clear_request', 'abort_request' ] self.control_handlers = {} for msg_type in control_msg_types: self.control_handlers[msg_type] = getattr(self, msg_type) @@ -480,14 +480,17 @@ def connect_request(self, stream, ident, parent): def shutdown_request(self, stream, ident, parent): self.shell.exit_now = True + content = dict(status='ok') + content.update(parent['content']) + self.session.send(stream, u'shutdown_reply', content, parent, ident=ident) + # same content, but different msg_id for broadcasting on IOPub self._shutdown_message = self.session.msg(u'shutdown_reply', - parent['content'], parent + content, parent ) - # self.session.send(stream, self._shutdown_message, ident=ident) self._at_shutdown() # call sys.exit after a short delay - ioloop.IOLoop.instance().add_timeout(time.time()+0.05, lambda : sys.exit(0)) + ioloop.IOLoop.instance().add_timeout(time.time()+0.1, lambda : sys.exit(0)) #--------------------------------------------------------------------------- # Engine methods @@ -615,6 +618,8 @@ def _abort_queues(self): self._abort_queue(stream) def _abort_queue(self, stream): + poller = zmq.Poller() + poller.register(stream.socket, zmq.POLLIN) while True: idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) if msg is None: @@ -632,7 +637,7 @@ def _abort_queue(self, stream): self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. - time.sleep(0.05) + poller.poll(50) def _no_raw_input(self): diff --git a/IPython/zmq/zmqshell.py b/IPython/zmq/zmqshell.py index 56d777fc493..d787dcf0053 100644 --- a/IPython/zmq/zmqshell.py +++ b/IPython/zmq/zmqshell.py @@ -19,8 +19,12 @@ import inspect import os import sys +import time from subprocess import Popen, PIPE +# System library imports +from zmq.eventloop import ioloop + # Our own from IPython.core.interactiveshell import ( InteractiveShell, InteractiveShellABC @@ -114,6 +118,12 @@ class ZMQInteractiveShell(InteractiveShell): exiter = Instance(ZMQExitAutocall) def _exiter_default(self): return ZMQExitAutocall(self) + + def _exit_now_changed(self, name, old, new): + """stop eventloop when exit_now fires""" + if new: + loop = ioloop.IOLoop.instance() + loop.add_timeout(time.time()+0.1, loop.stop) keepkernel_on_exit = None @@ -154,6 +164,7 @@ def auto_rewrite_input(self, cmd): def ask_exit(self): """Engage the exit actions.""" + self.exit_now = True payload = dict( source='IPython.zmq.zmqshell.ZMQInteractiveShell.ask_exit', exit=True, From 3724f43886bc84ee8c3bb1cdb4b752c69b1c0e69 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 13:02:28 -0700 Subject: [PATCH 19/69] remove now-obsolete kernel fork used in engine --- IPython/parallel/engine/streamkernel.py | 414 ------------------------ 1 file changed, 414 deletions(-) delete mode 100644 IPython/parallel/engine/streamkernel.py diff --git a/IPython/parallel/engine/streamkernel.py b/IPython/parallel/engine/streamkernel.py deleted file mode 100644 index 91fe845ebba..00000000000 --- a/IPython/parallel/engine/streamkernel.py +++ /dev/null @@ -1,414 +0,0 @@ -""" -Kernel adapted from kernel.py to use ZMQ Streams - -Authors: - -* Min RK -* Brian Granger -* Fernando Perez -* Evan Patterson -""" -#----------------------------------------------------------------------------- -# Copyright (C) 2010-2011 The IPython Development Team -# -# Distributed under the terms of the BSD License. The full license is in -# the file COPYING, distributed as part of this software. -#----------------------------------------------------------------------------- - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -# Standard library imports. -from __future__ import print_function - -import sys -import time - -from code import CommandCompiler -from datetime import datetime -from pprint import pprint - -# System library imports. -import zmq -from zmq.eventloop import ioloop, zmqstream - -# Local imports. -from IPython.utils.traitlets import Instance, List, Integer, Dict, Set, Unicode, CBytes -from IPython.zmq.completer import KernelCompleter - -from IPython.parallel.error import wrap_exception -from IPython.parallel.factory import SessionFactory -from IPython.parallel.util import serialize_object, unpack_apply_message, asbytes - -def printer(*args): - pprint(args, stream=sys.__stdout__) - - -class _Passer(zmqstream.ZMQStream): - """Empty class that implements `send()` that does nothing. - - Subclass ZMQStream for Session typechecking - - """ - def __init__(self, *args, **kwargs): - pass - - def send(self, *args, **kwargs): - pass - send_multipart = send - - -#----------------------------------------------------------------------------- -# Main kernel class -#----------------------------------------------------------------------------- - -class Kernel(SessionFactory): - - #--------------------------------------------------------------------------- - # Kernel interface - #--------------------------------------------------------------------------- - - # kwargs: - exec_lines = List(Unicode, config=True, - help="List of lines to execute") - - # identities: - int_id = Integer(-1) - bident = CBytes() - ident = Unicode() - def _ident_changed(self, name, old, new): - self.bident = asbytes(new) - - user_ns = Dict(config=True, help="""Set the user's namespace of the Kernel""") - - control_stream = Instance(zmqstream.ZMQStream) - task_stream = Instance(zmqstream.ZMQStream) - iopub_stream = Instance(zmqstream.ZMQStream) - client = Instance('IPython.parallel.Client') - - # internals - shell_streams = List() - compiler = Instance(CommandCompiler, (), {}) - completer = Instance(KernelCompleter) - - aborted = Set() - shell_handlers = Dict() - control_handlers = Dict() - - def _set_prefix(self): - self.prefix = "engine.%s"%self.int_id - - def _connect_completer(self): - self.completer = KernelCompleter(self.user_ns) - - def __init__(self, **kwargs): - super(Kernel, self).__init__(**kwargs) - self._set_prefix() - self._connect_completer() - - self.on_trait_change(self._set_prefix, 'id') - self.on_trait_change(self._connect_completer, 'user_ns') - - # Build dict of handlers for message types - for msg_type in ['execute_request', 'complete_request', 'apply_request', - 'clear_request']: - self.shell_handlers[msg_type] = getattr(self, msg_type) - - for msg_type in ['shutdown_request', 'abort_request']+self.shell_handlers.keys(): - self.control_handlers[msg_type] = getattr(self, msg_type) - - self._initial_exec_lines() - - def _wrap_exception(self, method=None): - e_info = dict(engine_uuid=self.ident, engine_id=self.int_id, method=method) - content=wrap_exception(e_info) - return content - - def _initial_exec_lines(self): - s = _Passer() - content = dict(silent=True, user_variable=[],user_expressions=[]) - for line in self.exec_lines: - self.log.debug("executing initialization: %s"%line) - content.update({'code':line}) - msg = self.session.msg('execute_request', content) - self.execute_request(s, [], msg) - - - #-------------------- control handlers ----------------------------- - def abort_queues(self): - for stream in self.shell_streams: - if stream: - self.abort_queue(stream) - - def abort_queue(self, stream): - while True: - idents,msg = self.session.recv(stream, zmq.NOBLOCK, content=True) - if msg is None: - return - - self.log.info("Aborting:") - self.log.info(str(msg)) - msg_type = msg['header']['msg_type'] - reply_type = msg_type.split('_')[0] + '_reply' - # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) - # self.reply_socket.send(ident,zmq.SNDMORE) - # self.reply_socket.send_json(reply_msg) - reply_msg = self.session.send(stream, reply_type, - content={'status' : 'aborted'}, parent=msg, ident=idents) - self.log.debug(str(reply_msg)) - # We need to wait a bit for requests to come in. This can probably - # be set shorter for true asynchronous clients. - time.sleep(0.05) - - def abort_request(self, stream, ident, parent): - """abort a specifig msg by id""" - msg_ids = parent['content'].get('msg_ids', None) - if isinstance(msg_ids, basestring): - msg_ids = [msg_ids] - if not msg_ids: - self.abort_queues() - for mid in msg_ids: - self.aborted.add(str(mid)) - - content = dict(status='ok') - reply_msg = self.session.send(stream, 'abort_reply', content=content, - parent=parent, ident=ident) - self.log.debug(str(reply_msg)) - - def shutdown_request(self, stream, ident, parent): - """kill ourself. This should really be handled in an external process""" - try: - self.abort_queues() - except: - content = self._wrap_exception('shutdown') - else: - content = dict(parent['content']) - content['status'] = 'ok' - msg = self.session.send(stream, 'shutdown_reply', - content=content, parent=parent, ident=ident) - self.log.debug(str(msg)) - dc = ioloop.DelayedCallback(lambda : sys.exit(0), 1000, self.loop) - dc.start() - - def dispatch_control(self, msg): - idents,msg = self.session.feed_identities(msg, copy=False) - try: - msg = self.session.unserialize(msg, content=True, copy=False) - except: - self.log.error("Invalid Message", exc_info=True) - return - else: - self.log.debug("Control received, %s", msg) - - header = msg['header'] - msg_id = header['msg_id'] - msg_type = header['msg_type'] - - handler = self.control_handlers.get(msg_type, None) - if handler is None: - self.log.error("UNKNOWN CONTROL MESSAGE TYPE: %r"%msg_type) - else: - handler(self.control_stream, idents, msg) - - - #-------------------- queue helpers ------------------------------ - - def check_dependencies(self, dependencies): - if not dependencies: - return True - if len(dependencies) == 2 and dependencies[0] in 'any all'.split(): - anyorall = dependencies[0] - dependencies = dependencies[1] - else: - anyorall = 'all' - results = self.client.get_results(dependencies,status_only=True) - if results['status'] != 'ok': - return False - - if anyorall == 'any': - if not results['completed']: - return False - else: - if results['pending']: - return False - - return True - - def check_aborted(self, msg_id): - return msg_id in self.aborted - - #-------------------- queue handlers ----------------------------- - - def clear_request(self, stream, idents, parent): - """Clear our namespace.""" - self.user_ns = {} - msg = self.session.send(stream, 'clear_reply', ident=idents, parent=parent, - content = dict(status='ok')) - self._initial_exec_lines() - - def execute_request(self, stream, ident, parent): - self.log.debug('execute request %s'%parent) - try: - code = parent[u'content'][u'code'] - except: - self.log.error("Got bad msg: %s"%parent, exc_info=True) - return - self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent, - ident=asbytes('%s.pyin'%self.prefix)) - started = datetime.now() - try: - comp_code = self.compiler(code, '') - # allow for not overriding displayhook - if hasattr(sys.displayhook, 'set_parent'): - sys.displayhook.set_parent(parent) - sys.stdout.set_parent(parent) - sys.stderr.set_parent(parent) - exec comp_code in self.user_ns, self.user_ns - except: - exc_content = self._wrap_exception('execute') - # exc_msg = self.session.msg(u'pyerr', exc_content, parent) - self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, - ident=asbytes('%s.pyerr'%self.prefix)) - reply_content = exc_content - else: - reply_content = {'status' : 'ok'} - - reply_msg = self.session.send(stream, u'execute_reply', reply_content, parent=parent, - ident=ident, subheader = dict(started=started)) - self.log.debug(str(reply_msg)) - if reply_msg['content']['status'] == u'error': - self.abort_queues() - - def complete_request(self, stream, ident, parent): - matches = {'matches' : self.complete(parent), - 'status' : 'ok'} - completion_msg = self.session.send(stream, 'complete_reply', - matches, parent, ident) - # print >> sys.__stdout__, completion_msg - - def complete(self, msg): - return self.completer.complete(msg.content.line, msg.content.text) - - def apply_request(self, stream, ident, parent): - # flush previous reply, so this request won't block it - stream.flush(zmq.POLLOUT) - try: - content = parent[u'content'] - bufs = parent[u'buffers'] - msg_id = parent['header']['msg_id'] - # bound = parent['header'].get('bound', False) - except: - self.log.error("Got bad msg: %s"%parent, exc_info=True) - return - # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) - # self.iopub_stream.send(pyin_msg) - # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent) - sub = {'dependencies_met' : True, 'engine' : self.ident, - 'started': datetime.now()} - try: - # allow for not overriding displayhook - if hasattr(sys.displayhook, 'set_parent'): - sys.displayhook.set_parent(parent) - sys.stdout.set_parent(parent) - sys.stderr.set_parent(parent) - # exec "f(*args,**kwargs)" in self.user_ns, self.user_ns - working = self.user_ns - # suffix = - prefix = "_"+str(msg_id).replace("-","")+"_" - - f,args,kwargs = unpack_apply_message(bufs, working, copy=False) - # if bound: - # bound_ns = Namespace(working) - # args = [bound_ns]+list(args) - - fname = getattr(f, '__name__', 'f') - - fname = prefix+"f" - argname = prefix+"args" - kwargname = prefix+"kwargs" - resultname = prefix+"result" - - ns = { fname : f, argname : args, kwargname : kwargs , resultname : None } - # print ns - working.update(ns) - code = "%s=%s(*%s,**%s)"%(resultname, fname, argname, kwargname) - try: - exec code in working,working - result = working.get(resultname) - finally: - for key in ns.iterkeys(): - working.pop(key) - # if bound: - # working.update(bound_ns) - - packed_result,buf = serialize_object(result) - result_buf = [packed_result]+buf - except: - exc_content = self._wrap_exception('apply') - # exc_msg = self.session.msg(u'pyerr', exc_content, parent) - self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, - ident=asbytes('%s.pyerr'%self.prefix)) - reply_content = exc_content - result_buf = [] - - if exc_content['ename'] == 'UnmetDependency': - sub['dependencies_met'] = False - else: - reply_content = {'status' : 'ok'} - - # put 'ok'/'error' status in header, for scheduler introspection: - sub['status'] = reply_content['status'] - - reply_msg = self.session.send(stream, u'apply_reply', reply_content, - parent=parent, ident=ident,buffers=result_buf, subheader=sub) - - # flush i/o - # should this be before reply_msg is sent, like in the single-kernel code, - # or should nothing get in the way of real results? - sys.stdout.flush() - sys.stderr.flush() - - def dispatch_queue(self, stream, msg): - self.control_stream.flush() - idents,msg = self.session.feed_identities(msg, copy=False) - try: - msg = self.session.unserialize(msg, content=True, copy=False) - except: - self.log.error("Invalid Message", exc_info=True) - return - else: - self.log.debug("Message received, %s", msg) - - - header = msg['header'] - msg_id = header['msg_id'] - msg_type = msg['header']['msg_type'] - if self.check_aborted(msg_id): - self.aborted.remove(msg_id) - # is it safe to assume a msg_id will not be resubmitted? - reply_type = msg_type.split('_')[0] + '_reply' - status = {'status' : 'aborted'} - reply_msg = self.session.send(stream, reply_type, subheader=status, - content=status, parent=msg, ident=idents) - return - handler = self.shell_handlers.get(msg_type, None) - if handler is None: - self.log.error("UNKNOWN MESSAGE TYPE: %r"%msg_type) - else: - handler(stream, idents, msg) - - def start(self): - #### stream mode: - if self.control_stream: - self.control_stream.on_recv(self.dispatch_control, copy=False) - - def make_dispatcher(stream): - def dispatcher(msg): - return self.dispatch_queue(stream, msg) - return dispatcher - - for s in self.shell_streams: - s.on_recv(make_dispatcher(s), copy=False) - - From cfdfd64a3941ea7c7e89ccee3ae821868756c603 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 24 Mar 2012 20:45:09 -0700 Subject: [PATCH 20/69] don't use history files in Engines --- IPython/parallel/engine/engine.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index 367666bbf78..002ff3d34a1 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -188,7 +188,10 @@ def complete_registration(self, msg, connect, maybe_tunnel): iopub_stream.setsockopt(zmq.IDENTITY, identity) connect(iopub_stream, iopub_addr) - # # Redirect input streams and set a display hook. + # disable history: + self.config.HistoryManager.hist_file = ':memory:' + + # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_stream, u'stdout') sys.stdout.topic = py3compat.cast_bytes('engine.%i.stdout' % self.id) From 99269aff9e6ca73dd03ed594894977521c99fc73 Mon Sep 17 00:00:00 2001 From: MinRK Date: Wed, 28 Mar 2012 15:39:44 -0700 Subject: [PATCH 21/69] add silent kwarg to run_cell This explicitly omits the actions described in the message spec: * history/logger * displayhook * post-exec --- IPython/core/interactiveshell.py | 24 ++++++--- IPython/core/tests/test_interactiveshell.py | 55 +++++++++++++++++++++ 2 files changed, 72 insertions(+), 7 deletions(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index ce69fa11c72..cd5e36e6d02 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -2429,7 +2429,7 @@ def safe_run_module(self, mod_name, where): self.showtraceback() warn('Unknown failure executing module: <%s>' % mod_name) - def run_cell(self, raw_cell, store_history=False): + def run_cell(self, raw_cell, store_history=False, silent=False): """Run a complete IPython cell. Parameters @@ -2440,9 +2440,15 @@ def run_cell(self, raw_cell, store_history=False): If True, the raw and translated cell will be stored in IPython's history. For user code calling back into IPython's machinery, this should be set to False. + silent : bool + If True, avoid side-effets, such as implicit displayhooks, history, + and logging. silent=True forces store_history=False. """ if (not raw_cell) or raw_cell.isspace(): return + + if silent: + store_history = False for line in raw_cell.splitlines(): self.input_splitter.push(line) @@ -2467,8 +2473,8 @@ def run_cell(self, raw_cell, store_history=False): if store_history: self.history_manager.store_inputs(self.execution_count, cell, raw_cell) - - self.logger.log(cell, raw_cell) + if not silent: + self.logger.log(cell, raw_cell) if not prefilter_failed: # don't run if prefilter failed @@ -2488,12 +2494,16 @@ def run_cell(self, raw_cell, store_history=False): if store_history: self.execution_count += 1 return None - + + interactivity = "none" if silent else "last_expr" self.run_ast_nodes(code_ast.body, cell_name, - interactivity="last_expr") - + interactivity=interactivity) + # Execute any registered post-execution functions. - for func, status in self._post_execute.iteritems(): + # unless we are silent + post_exec = [] if silent else self._post_execute.iteritems() + + for func, status in post_exec: if self.disable_failing_post_execute and not status: continue try: diff --git a/IPython/core/tests/test_interactiveshell.py b/IPython/core/tests/test_interactiveshell.py index aee855d3159..21008e77533 100644 --- a/IPython/core/tests/test_interactiveshell.py +++ b/IPython/core/tests/test_interactiveshell.py @@ -254,6 +254,61 @@ def test_bad_var_expand(self): self.assertEqual(ip.var_expand(u"{asdf}"), u"{asdf}") # ZeroDivisionError self.assertEqual(ip.var_expand(u"{1/0}"), u"{1/0}") + + def test_silent_nopostexec(self): + """run_cell(silent=True) doesn't invoke post-exec funcs""" + ip = get_ipython() + + d = dict(called=False) + def set_called(): + d['called'] = True + + ip.register_post_execute(set_called) + ip.run_cell("1", silent=True) + self.assertFalse(d['called']) + # double-check that non-silent exec did what we expected + # silent to avoid + ip.run_cell("1") + self.assertTrue(d['called']) + # remove post-exec + ip._post_execute.pop(set_called) + + def test_silent_noadvance(self): + """run_cell(silent=True) doesn't advance execution_count""" + ip = get_ipython() + + ec = ip.execution_count + # silent should force store_history=False + ip.run_cell("1", store_history=True, silent=True) + + self.assertEquals(ec, ip.execution_count) + # double-check that non-silent exec did what we expected + # silent to avoid + ip.run_cell("1", store_history=True) + self.assertEquals(ec+1, ip.execution_count) + + def test_silent_nodisplayhook(self): + """run_cell(silent=True) doesn't trigger displayhook""" + ip = get_ipython() + + d = dict(called=False) + + trap = ip.display_trap + save_hook = trap.hook + + def failing_hook(*args, **kwargs): + d['called'] = True + + try: + trap.hook = failing_hook + ip.run_cell("1", silent=True) + self.assertFalse(d['called']) + # double-check that non-silent exec did what we expected + # silent to avoid + ip.run_cell("1") + self.assertTrue(d['called']) + finally: + trap.hook = save_hook @skipif(sys.version_info[0] >= 3, "softspace removed in py3") def test_print_softspace(self): From de14aa101738f6af7c0dee500453e7c1da899b66 Mon Sep 17 00:00:00 2001 From: MinRK Date: Wed, 28 Mar 2012 15:44:08 -0700 Subject: [PATCH 22/69] use new run_cell(silent=True) instead of run_code in ipkernel --- IPython/parallel/client/client.py | 2 +- IPython/zmq/ipkernel.py | 19 +++++-------------- 2 files changed, 6 insertions(+), 15 deletions(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index a7862a2e82a..0f4a3e11cae 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -1089,7 +1089,7 @@ def send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, return msg - def send_execute_request(self, socket, code, silent=False, subheader=None, ident=None): + def send_execute_request(self, socket, code, silent=True, subheader=None, ident=None): """construct and send an execute request via a socket. """ diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 14d70d071e0..39840ca9986 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -334,14 +334,8 @@ def execute_request(self, stream, ident, parent): reply_content = {} try: - if silent: - # run_code uses 'exec' mode, so no displayhook will fire, and it - # doesn't call logging or history manipulations. Print - # statements in that code will obviously still execute. - shell.run_code(code) - else: - # FIXME: the shell calls the exception handler itself. - shell.run_cell(code, store_history=True) + # FIXME: the shell calls the exception handler itself. + shell.run_cell(code, store_history=not silent, silent=silent) except: status = u'error' # FIXME: this code right now isn't being used yet by default, @@ -359,7 +353,7 @@ def execute_request(self, stream, ident, parent): reply_content[u'status'] = status # Return the execution counter so clients can display prompts - reply_content['execution_count'] = shell.execution_count -1 + reply_content['execution_count'] = shell.execution_count - 1 # FIXME - fish exception info out of shell, possibly left there by # runlines. We'll need to clean up this logic later. @@ -404,11 +398,8 @@ def execute_request(self, stream, ident, parent): reply_content, parent, ident=ident) self.log.debug("%s", reply_msg) - if reply_msg['content']['status'] == u'error': - # FIXME: queue_abort on error is disabled - # A policy decision must be made - pass - # self._abort_queues() + if not silent and reply_msg['content']['status'] == u'error': + self._abort_queues() self.session.send(self.iopub_stream, u'status', From c5c7935c205f4e0a037a8cad25c9312c29e7baf7 Mon Sep 17 00:00:00 2001 From: MinRK Date: Wed, 18 Apr 2012 21:07:30 -0700 Subject: [PATCH 23/69] do not use ZMQStream for IOPub instead use regular zmq.Socket. it's unnecessarily complicated, and causes asynchronous/ordering issues. --- IPython/parallel/engine/engine.py | 14 +++++++------- IPython/zmq/ipkernel.py | 26 ++++++++++++-------------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index 002ff3d34a1..e9578cb88e4 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -184,25 +184,25 @@ def complete_registration(self, msg, connect, maybe_tunnel): # create iopub stream: iopub_addr = msg.content.iopub - iopub_stream = zmqstream.ZMQStream(ctx.socket(zmq.PUB), loop) - iopub_stream.setsockopt(zmq.IDENTITY, identity) - connect(iopub_stream, iopub_addr) + iopub_socket = ctx.socket(zmq.PUB) + iopub_socket.setsockopt(zmq.IDENTITY, identity) + connect(iopub_socket, iopub_addr) # disable history: self.config.HistoryManager.hist_file = ':memory:' # Redirect input streams and set a display hook. if self.out_stream_factory: - sys.stdout = self.out_stream_factory(self.session, iopub_stream, u'stdout') + sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') sys.stdout.topic = py3compat.cast_bytes('engine.%i.stdout' % self.id) - sys.stderr = self.out_stream_factory(self.session, iopub_stream, u'stderr') + sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') sys.stderr.topic = py3compat.cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: - sys.displayhook = self.display_hook_factory(self.session, iopub_stream) + sys.displayhook = self.display_hook_factory(self.session, iopub_socket) sys.displayhook.topic = py3compat.cast_bytes('engine.%i.pyout' % self.id) self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, - control_stream=control_stream, shell_streams=shell_streams, iopub_stream=iopub_stream, + control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) self.kernel.start() diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 39840ca9986..2a806101f2c 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -80,7 +80,7 @@ def _eventloop_changed(self, name, old, new): profile_dir = Instance('IPython.core.profiledir.ProfileDir') shell_streams = List() control_stream = Instance(ZMQStream) - iopub_stream = Instance(ZMQStream) + iopub_socket = Instance(zmq.Socket) stdin_socket = Instance(zmq.Socket) log = Instance(logging.Logger) @@ -144,9 +144,9 @@ def __init__(self, **kwargs): user_ns = self.user_ns, ) self.shell.displayhook.session = self.session - self.shell.displayhook.pub_socket = self.iopub_stream.socket + self.shell.displayhook.pub_socket = self.iopub_socket self.shell.display_pub.session = self.session - self.shell.display_pub.pub_socket = self.iopub_stream.socket + self.shell.display_pub.pub_socket = self.iopub_socket # TMP - hack while developing self.shell._reply_content = None @@ -266,7 +266,6 @@ def do_one_iteration(self): # handle at most one request per iteration stream.flush(zmq.POLLIN, 1) stream.flush(zmq.POLLOUT) - self.iopub_stream.flush() def record_ports(self, ports): @@ -284,14 +283,14 @@ def record_ports(self, ports): def _publish_pyin(self, code, parent, execution_count): """Publish the code request on the pyin stream.""" - self.session.send(self.iopub_stream, u'pyin', + self.session.send(self.iopub_socket, u'pyin', {u'code':code, u'execution_count': execution_count}, parent=parent, ident=self._topic('pyin') ) def execute_request(self, stream, ident, parent): - self.session.send(self.iopub_stream, + self.session.send(self.iopub_socket, u'status', {u'execution_state':u'busy'}, parent=parent, @@ -401,7 +400,7 @@ def execute_request(self, stream, ident, parent): if not silent and reply_msg['content']['status'] == u'error': self._abort_queues() - self.session.send(self.iopub_stream, + self.session.send(self.iopub_socket, u'status', {u'execution_state':u'idle'}, parent=parent, @@ -496,8 +495,8 @@ def apply_request(self, stream, ident, parent): self.log.error("Got bad msg: %s", parent, exc_info=True) return # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) - # self.iopub_stream.send(pyin_msg) - # self.session.send(self.iopub_stream, u'pyin', {u'code':code},parent=parent) + # self.iopub_socket.send(pyin_msg) + # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent) sub = {'dependencies_met' : True, 'engine' : self.ident, 'started': datetime.now()} try: @@ -535,7 +534,7 @@ def apply_request(self, stream, ident, parent): except: exc_content = self._wrap_exception('apply') # exc_msg = self.session.msg(u'pyerr', exc_content, parent) - self.session.send(self.iopub_stream, u'pyerr', exc_content, parent=parent, + self.session.send(self.iopub_socket, u'pyerr', exc_content, parent=parent, ident=self._topic('pyerr')) reply_content = exc_content result_buf = [] @@ -714,9 +713,9 @@ def _at_shutdown(self): """ # io.rprint("Kernel at_shutdown") # dbg if self._shutdown_message is not None: - self.session.send(self.iopub_stream, self._shutdown_message, ident=self._topic('shutdown')) + self.session.send(self.iopub_socket, self._shutdown_message, ident=self._topic('shutdown')) self.log.debug("%s", self._shutdown_message) - [ s.flush(zmq.POLLOUT) for s in self.shell_streams + [self.iopub_stream] ] + [ s.flush(zmq.POLLOUT) for s in self.shell_streams ] #----------------------------------------------------------------------------- # Aliases and Flags for the IPKernelApp @@ -771,11 +770,10 @@ def initialize(self, argv=None): def init_kernel(self): shell_stream = ZMQStream(self.shell_socket) - iopub_stream = ZMQStream(self.iopub_socket) kernel = Kernel(config=self.config, session=self.session, shell_streams=[shell_stream], - iopub_stream=iopub_stream, + iopub_socket=self.iopub_socket, stdin_socket=self.stdin_socket, log=self.log, profile_dir=self.profile_dir, From 2f3f0c88f98c31749b4ffedba7edd22c77fbc881 Mon Sep 17 00:00:00 2001 From: MinRK Date: Wed, 18 Apr 2012 21:28:54 -0700 Subject: [PATCH 24/69] remove disused pykernel --- IPython/zmq/pykernel.py | 282 ---------------------------------------- 1 file changed, 282 deletions(-) delete mode 100755 IPython/zmq/pykernel.py diff --git a/IPython/zmq/pykernel.py b/IPython/zmq/pykernel.py deleted file mode 100755 index 9bdaa9efa29..00000000000 --- a/IPython/zmq/pykernel.py +++ /dev/null @@ -1,282 +0,0 @@ -#!/usr/bin/env python -"""A simple interactive kernel that talks to a frontend over 0MQ. - -Things to do: - -* Implement `set_parent` logic. Right before doing exec, the Kernel should - call set_parent on all the PUB objects with the message about to be executed. -* Implement random port and security key logic. -* Implement control messages. -* Implement event loop and poll version. -""" - -#----------------------------------------------------------------------------- -# Imports -#----------------------------------------------------------------------------- - -# Standard library imports. -import __builtin__ -from code import CommandCompiler -import sys -import time -import traceback - -# System library imports. -import zmq - -# Local imports. -from IPython.utils import py3compat -from IPython.utils.traitlets import HasTraits, Instance, Dict, Float -from completer import KernelCompleter -from entry_point import base_launch_kernel -from session import Session, Message -from kernelapp import KernelApp - -#----------------------------------------------------------------------------- -# Main kernel class -#----------------------------------------------------------------------------- - -class Kernel(HasTraits): - - # Private interface - - # Time to sleep after flushing the stdout/err buffers in each execute - # cycle. While this introduces a hard limit on the minimal latency of the - # execute cycle, it helps prevent output synchronization problems for - # clients. - # Units are in seconds. The minimum zmq latency on local host is probably - # ~150 microseconds, set this to 500us for now. We may need to increase it - # a little if it's not enough after more interactive testing. - _execute_sleep = Float(0.0005, config=True) - - # This is a dict of port number that the kernel is listening on. It is set - # by record_ports and used by connect_request. - _recorded_ports = Dict() - - #--------------------------------------------------------------------------- - # Kernel interface - #--------------------------------------------------------------------------- - - session = Instance(Session) - shell_socket = Instance('zmq.Socket') - iopub_socket = Instance('zmq.Socket') - stdin_socket = Instance('zmq.Socket') - log = Instance('logging.Logger') - - def __init__(self, **kwargs): - super(Kernel, self).__init__(**kwargs) - self.user_ns = {} - self.history = [] - self.compiler = CommandCompiler() - self.completer = KernelCompleter(self.user_ns) - - # Build dict of handlers for message types - msg_types = [ 'execute_request', 'complete_request', - 'object_info_request', 'shutdown_request' ] - self.handlers = {} - for msg_type in msg_types: - self.handlers[msg_type] = getattr(self, msg_type) - - def start(self): - """ Start the kernel main loop. - """ - while True: - ident,msg = self.session.recv(self.shell_socket,0) - assert ident is not None, "Missing message part." - omsg = Message(msg) - self.log.debug(str(omsg)) - handler = self.handlers.get(omsg.msg_type, None) - if handler is None: - self.log.error("UNKNOWN MESSAGE TYPE: %s"%omsg) - else: - handler(ident, omsg) - - def record_ports(self, ports): - """Record the ports that this kernel is using. - - The creator of the Kernel instance must call this methods if they - want the :meth:`connect_request` method to return the port numbers. - """ - self._recorded_ports = ports - - #--------------------------------------------------------------------------- - # Kernel request handlers - #--------------------------------------------------------------------------- - - def execute_request(self, ident, parent): - try: - code = parent[u'content'][u'code'] - except: - self.log.error("Got bad msg: %s"%Message(parent)) - return - pyin_msg = self.session.send(self.iopub_socket, u'pyin',{u'code':code}, parent=parent) - - try: - comp_code = self.compiler(code, '') - - # Replace raw_input. Note that is not sufficient to replace - # raw_input in the user namespace. - raw_input = lambda prompt='': self._raw_input(prompt, ident, parent) - if py3compat.PY3: - __builtin__.input = raw_input - else: - __builtin__.raw_input = raw_input - - # Set the parent message of the display hook and out streams. - sys.displayhook.set_parent(parent) - sys.stdout.set_parent(parent) - sys.stderr.set_parent(parent) - - exec comp_code in self.user_ns, self.user_ns - except: - etype, evalue, tb = sys.exc_info() - tb = traceback.format_exception(etype, evalue, tb) - exc_content = { - u'status' : u'error', - u'traceback' : tb, - u'ename' : unicode(etype.__name__), - u'evalue' : unicode(evalue) - } - exc_msg = self.session.send(self.iopub_socket, u'pyerr', exc_content, parent) - reply_content = exc_content - else: - reply_content = { 'status' : 'ok', 'payload' : {} } - - # Flush output before sending the reply. - sys.stderr.flush() - sys.stdout.flush() - # FIXME: on rare occasions, the flush doesn't seem to make it to the - # clients... This seems to mitigate the problem, but we definitely need - # to better understand what's going on. - if self._execute_sleep: - time.sleep(self._execute_sleep) - - # Send the reply. - reply_msg = self.session.send(self.shell_socket, u'execute_reply', reply_content, parent, ident=ident) - self.log.debug(Message(reply_msg)) - if reply_msg['content']['status'] == u'error': - self._abort_queue() - - def complete_request(self, ident, parent): - matches = {'matches' : self._complete(parent), - 'status' : 'ok'} - completion_msg = self.session.send(self.shell_socket, 'complete_reply', - matches, parent, ident) - self.log.debug(completion_msg) - - def object_info_request(self, ident, parent): - context = parent['content']['oname'].split('.') - object_info = self._object_info(context) - msg = self.session.send(self.shell_socket, 'object_info_reply', - object_info, parent, ident) - self.log.debug(msg) - - def shutdown_request(self, ident, parent): - content = dict(parent['content']) - msg = self.session.send(self.shell_socket, 'shutdown_reply', - content, parent, ident) - msg = self.session.send(self.iopub_socket, 'shutdown_reply', - content, parent, ident) - self.log.debug(msg) - time.sleep(0.1) - sys.exit(0) - - #--------------------------------------------------------------------------- - # Protected interface - #--------------------------------------------------------------------------- - - def _abort_queue(self): - while True: - ident,msg = self.session.recv(self.shell_socket, zmq.NOBLOCK) - if msg is None: - # msg=None on EAGAIN - break - else: - assert ident is not None, "Missing message part." - self.log.debug("Aborting: %s"%Message(msg)) - msg_type = msg['header']['msg_type'] - reply_type = msg_type.split('_')[0] + '_reply' - reply_msg = self.session.send(self.shell_socket, reply_type, {'status':'aborted'}, msg, ident=ident) - self.log.debug(Message(reply_msg)) - # We need to wait a bit for requests to come in. This can probably - # be set shorter for true asynchronous clients. - time.sleep(0.1) - - def _raw_input(self, prompt, ident, parent): - # Flush output before making the request. - sys.stderr.flush() - sys.stdout.flush() - - # Send the input request. - content = dict(prompt=prompt) - msg = self.session.send(self.stdin_socket, u'input_request', content, parent, ident=ident) - - # Await a response. - ident,reply = self.session.recv(self.stdin_socket, 0) - try: - value = reply['content']['value'] - except: - self.log.error("Got bad raw_input reply: %s"%Message(parent)) - value = '' - return value - - def _complete(self, msg): - return self.completer.complete(msg.content.line, msg.content.text) - - def _object_info(self, context): - symbol, leftover = self._symbol_from_context(context) - if symbol is not None and not leftover: - doc = getattr(symbol, '__doc__', '') - else: - doc = '' - object_info = dict(docstring = doc) - return object_info - - def _symbol_from_context(self, context): - if not context: - return None, context - - base_symbol_string = context[0] - symbol = self.user_ns.get(base_symbol_string, None) - if symbol is None: - symbol = __builtin__.__dict__.get(base_symbol_string, None) - if symbol is None: - return None, context - - context = context[1:] - for i, name in enumerate(context): - new_symbol = getattr(symbol, name, None) - if new_symbol is None: - return symbol, context[i:] - else: - symbol = new_symbol - - return symbol, [] - -#----------------------------------------------------------------------------- -# Kernel main and launch functions -#----------------------------------------------------------------------------- - -def launch_kernel(*args, **kwargs): - """ Launches a simple Python kernel, binding to the specified ports. - - This function simply calls entry_point.base_launch_kernel with the right first - command to start a pykernel. See base_launch_kernel for arguments. - - Returns - ------- - A tuple of form: - (kernel_process, xrep_port, pub_port, req_port, hb_port) - where kernel_process is a Popen object and the ports are integers. - """ - return base_launch_kernel('from IPython.zmq.pykernel import main; main()', - *args, **kwargs) - -def main(): - """Run a PyKernel as an application""" - app = KernelApp.instance() - app.initialize() - app.start() - -if __name__ == '__main__': - main() From 40486214db46c3400019ae7328b098f55444f3d8 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 23 Apr 2012 12:30:14 -0700 Subject: [PATCH 25/69] remove references to removed pykernel and associated pure flag --- IPython/frontend/consoleapp.py | 3 --- IPython/frontend/qt/console/qtconsoleapp.py | 22 ++++++--------------- IPython/zmq/kernelapp.py | 4 ++-- IPython/zmq/kernelmanager.py | 8 +------- 4 files changed, 9 insertions(+), 28 deletions(-) diff --git a/IPython/frontend/consoleapp.py b/IPython/frontend/consoleapp.py index 2776eef688c..ab1beb75f2b 100644 --- a/IPython/frontend/consoleapp.py +++ b/IPython/frontend/consoleapp.py @@ -140,8 +140,6 @@ class IPythonConsoleApp(Configurable): frontend_flags = Any(app_flags) frontend_aliases = Any(app_aliases) - pure = CBool(False, config=True, - help="Use a pure Python kernel instead of an IPython kernel.") # create requested profiles by default, if they don't exist: auto_create = CBool(True) # connection info: @@ -330,7 +328,6 @@ def init_kernel_manager(self): ) # start the kernel if not self.existing: - kwargs = dict(ipython=not self.pure) kwargs['extra_arguments'] = self.kernel_argv self.kernel_manager.start_kernel(**kwargs) elif self.sshserver: diff --git a/IPython/frontend/qt/console/qtconsoleapp.py b/IPython/frontend/qt/console/qtconsoleapp.py index 89f0aa83935..80ccbd87870 100644 --- a/IPython/frontend/qt/console/qtconsoleapp.py +++ b/IPython/frontend/qt/console/qtconsoleapp.py @@ -101,9 +101,7 @@ def gui_excepthook(exctype, value, tb): # start with copy of flags flags = dict(flags) qt_flags = { - 'pure' : ({'IPythonQtConsoleApp' : {'pure' : True}}, - "Use a pure Python kernel instead of an IPython kernel."), - 'plain' : ({'ConsoleWidget' : {'kind' : 'plain'}}, + 'plain' : ({'IPythonQtConsoleApp' : {'plain' : True}}, "Disable rich text support."), } qt_flags.update(boolean_flag( @@ -180,18 +178,14 @@ class IPythonQtConsoleApp(BaseIPythonApplication, IPythonConsoleApp): plain = CBool(False, config=True, help="Use a plaintext widget instead of rich text (plain can't print/save).") - def _pure_changed(self, name, old, new): - kind = 'plain' if self.plain else 'rich' + def _plain_changed(self, name, old, new): + kind = 'plain' if new else 'rich' self.config.ConsoleWidget.kind = kind - if self.pure: - self.widget_factory = FrontendWidget - elif self.plain: + if new: self.widget_factory = IPythonWidget else: self.widget_factory = RichIPythonWidget - _plain_changed = _pure_changed - # the factory for creating a widget widget_factory = Any(RichIPythonWidget) @@ -210,7 +204,7 @@ def new_frontend_master(self): config=self.config, ) # start the kernel - kwargs = dict(ipython=not self.pure) + kwargs = dict() kwargs['extra_arguments'] = self.kernel_argv kernel_manager.start_kernel(**kwargs) kernel_manager.start_channels() @@ -273,17 +267,13 @@ def init_qt_elements(self): self.window.add_tab_with_frontend(self.widget) self.window.init_menu_bar() - self.window.setWindowTitle('Python' if self.pure else 'IPython') + self.window.setWindowTitle('IPython') def init_colors(self, widget): """Configure the coloring of the widget""" # Note: This will be dramatically simplified when colors # are removed from the backend. - if self.pure: - # only IPythonWidget supports styling - return - # parse the colors arg down to current known labels try: colors = self.config.ZMQInteractiveShell.colors diff --git a/IPython/zmq/kernelapp.py b/IPython/zmq/kernelapp.py index c3b60a3367b..50968a0061c 100644 --- a/IPython/zmq/kernelapp.py +++ b/IPython/zmq/kernelapp.py @@ -84,12 +84,12 @@ #----------------------------------------------------------------------------- class KernelApp(BaseIPythonApplication): - name='pykernel' + name='ipkernel' aliases = Dict(kernel_aliases) flags = Dict(kernel_flags) classes = [Session] # the kernel class, as an importstring - kernel_class = DottedObjectName('IPython.zmq.pykernel.Kernel') + kernel_class = DottedObjectName('IPython.zmq.ipkernel.Kernel') kernel = Any() poller = Any() # don't restrict this even though current pollers are all Threads heartbeat = Instance(Heartbeat) diff --git a/IPython/zmq/kernelmanager.py b/IPython/zmq/kernelmanager.py index a350866d22b..b175af54152 100644 --- a/IPython/zmq/kernelmanager.py +++ b/IPython/zmq/kernelmanager.py @@ -781,9 +781,6 @@ def start_kernel(self, **kw): Parameters: ----------- - ipython : bool, optional (default True) - Whether to use an IPython kernel instead of a plain Python kernel. - launcher : callable, optional (default None) A custom function for launching the kernel process (generally a wrapper around ``entry_point.base_launch_kernel``). In most cases, @@ -805,10 +802,7 @@ def start_kernel(self, **kw): self._launch_args = kw.copy() launch_kernel = kw.pop('launcher', None) if launch_kernel is None: - if kw.pop('ipython', True): - from ipkernel import launch_kernel - else: - from pykernel import launch_kernel + from ipkernel import launch_kernel self.kernel = launch_kernel(fname=self.connection_file, **kw) def shutdown_kernel(self, restart=False): From 5cc602be06145b29a5316674d6b1fe9d64ecbec5 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 23 Apr 2012 22:04:54 -0700 Subject: [PATCH 26/69] fix subheaders for execute_reply and aborted messages ref: #1647 --- IPython/parallel/controller/hub.py | 5 +++- IPython/zmq/ipkernel.py | 40 ++++++++++++++++++++++-------- 2 files changed, 34 insertions(+), 11 deletions(-) diff --git a/IPython/parallel/controller/hub.py b/IPython/parallel/controller/hub.py index c6e51a809e3..41ab5113062 100644 --- a/IPython/parallel/controller/hub.py +++ b/IPython/parallel/controller/hub.py @@ -721,13 +721,16 @@ def save_task_result(self, idents, msg): header = msg['header'] engine_uuid = header.get('engine', None) eid = self.by_ident.get(engine_uuid, None) + + status = header.get('status', None) if msg_id in self.pending: self.log.info("task::task %r finished on %s", msg_id, eid) self.pending.remove(msg_id) self.all_completed.add(msg_id) if eid is not None: - self.completed[eid].append(msg_id) + if status != 'aborted': + self.completed[eid].append(msg_id) if msg_id in self.tasks[eid]: self.tasks[eid].remove(msg_id) completed = header['date'] diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 2a806101f2c..192d4bdc606 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -218,7 +218,9 @@ def dispatch_shell(self, stream, msg): # is it safe to assume a msg_id will not be resubmitted? reply_type = msg_type.split('_')[0] + '_reply' status = {'status' : 'aborted'} - reply_msg = self.session.send(stream, reply_type, subheader=status, + sub = {'engine' : self.ident} + sub.update(status) + reply_msg = self.session.send(stream, reply_type, subheader=sub, content=status, parent=msg, ident=idents) return @@ -279,7 +281,15 @@ def record_ports(self, ports): #--------------------------------------------------------------------------- # Kernel request handlers #--------------------------------------------------------------------------- - + + def _make_subheader(self): + """init subheader dict, for execute/apply_reply""" + return { + 'dependencies_met' : True, + 'engine' : self.ident, + 'started': datetime.now(), + } + def _publish_pyin(self, code, parent, execution_count): """Publish the code request on the pyin stream.""" @@ -305,6 +315,8 @@ def execute_request(self, stream, ident, parent): self.log.error("Got bad msg: ") self.log.error("%s", parent) return + + sub = self._make_subheader() shell = self.shell # we'll need this a lot here @@ -393,8 +405,16 @@ def execute_request(self, stream, ident, parent): # Send the reply. reply_content = json_clean(reply_content) + + sub['status'] = reply_content['status'] + if reply_content['status'] == 'error' and \ + reply_content['ename'] == 'UnmetDependency': + sub['dependencies_met'] = False + reply_msg = self.session.send(stream, u'execute_reply', - reply_content, parent, ident=ident) + reply_content, parent, subheader=sub, + ident=ident) + self.log.debug("%s", reply_msg) if not silent and reply_msg['content']['status'] == u'error': @@ -497,8 +517,7 @@ def apply_request(self, stream, ident, parent): # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) # self.iopub_socket.send(pyin_msg) # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent) - sub = {'dependencies_met' : True, 'engine' : self.ident, - 'started': datetime.now()} + sub = self._make_subheader() try: # allow for not overriding displayhook if hasattr(sys.displayhook, 'set_parent'): @@ -619,11 +638,12 @@ def _abort_queue(self, stream): self.log.info("%s", msg) msg_type = msg['header']['msg_type'] reply_type = msg_type.split('_')[0] + '_reply' - # reply_msg = self.session.msg(reply_type, {'status' : 'aborted'}, msg) - # self.reply_stream.send(ident,zmq.SNDMORE) - # self.reply_stream.send_json(reply_msg) - reply_msg = self.session.send(stream, reply_type, - content={'status' : 'aborted'}, parent=msg, ident=idents) + + status = {'status' : 'aborted'} + sub = {'engine' : self.ident} + sub.update(status) + reply_msg = self.session.send(stream, reply_type, subheader=sub, + content=status, parent=msg, ident=idents) self.log.debug("%s", reply_msg) # We need to wait a bit for requests to come in. This can probably # be set shorter for true asynchronous clients. From 3531213400ae2716a0ae28e2cae6d6398ee7522f Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 23 Apr 2012 22:26:29 -0700 Subject: [PATCH 27/69] test resubmit of aborted tasks (#1647) --- IPython/parallel/tests/test_client.py | 39 ++++++++++++++++++++++++++- 1 file changed, 38 insertions(+), 1 deletion(-) diff --git a/IPython/parallel/tests/test_client.py b/IPython/parallel/tests/test_client.py index 48ba9e97bd4..2475cd97f9b 100644 --- a/IPython/parallel/tests/test_client.py +++ b/IPython/parallel/tests/test_client.py @@ -280,6 +280,25 @@ def test_hub_history(self): time.sleep(0.25) self.assertEquals(self.client.hub_history()[-1:],ar.msg_ids) + def _wait_for_idle(self): + """wait for an engine to become idle, according to the Hub""" + rc = self.client + + # timeout 2s, polling every 100ms + for i in range(20): + qs = rc.queue_status() + if qs['unassigned'] or any(qs[eid]['tasks'] for eid in rc.ids): + time.sleep(0.1) + else: + break + + # ensure Hub up to date: + qs = rc.queue_status() + self.assertEquals(qs['unassigned'], 0) + for eid in rc.ids: + self.assertEquals(qs[eid]['tasks'], 0) + + def test_resubmit(self): def f(): import random @@ -288,11 +307,29 @@ def f(): ar = v.apply_async(f) r1 = ar.get(1) # give the Hub a chance to notice: - time.sleep(0.5) + self._wait_for_idle() ahr = self.client.resubmit(ar.msg_ids) r2 = ahr.get(1) self.assertFalse(r1 == r2) + def test_resubmit_aborted(self): + def f(): + import random + return random.random() + v = self.client.load_balanced_view() + # restrict to one engine, so we can put a sleep + # ahead of the task, so it will get aborted + eid = self.client.ids[-1] + v.targets = [eid] + sleep = v.apply_async(time.sleep, 0.5) + ar = v.apply_async(f) + ar.abort() + self.assertRaises(error.TaskAborted, ar.get) + # Give the Hub a chance to get up to date: + self._wait_for_idle() + ahr = self.client.resubmit(ar.msg_ids) + r2 = ahr.get(1) + def test_resubmit_inflight(self): """ensure ValueError on resubmit of inflight task""" v = self.client.load_balanced_view() From 4cda2dc3b16c2e3ed7cdcd68d32367cf2c7ac3a8 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 8 May 2012 13:51:41 -0700 Subject: [PATCH 28/69] fix Kernel import in engineapp --- IPython/parallel/apps/ipengineapp.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/parallel/apps/ipengineapp.py b/IPython/parallel/apps/ipengineapp.py index 494bbd34084..65ab1652fb6 100755 --- a/IPython/parallel/apps/ipengineapp.py +++ b/IPython/parallel/apps/ipengineapp.py @@ -37,6 +37,7 @@ catch_config_error, ) from IPython.zmq.log import EnginePUBHandler +from IPython.zmq.ipkernel import Kernel from IPython.zmq.session import ( Session, session_aliases, session_flags ) @@ -44,7 +45,6 @@ from IPython.config.configurable import Configurable from IPython.parallel.engine.engine import EngineFactory -from IPython.parallel.engine.streamkernel import Kernel from IPython.parallel.util import disambiguate_url, asbytes from IPython.utils.importstring import import_item From d8f08af79584d7c9b0e55fba08734778961d3bda Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 8 May 2012 15:45:31 -0700 Subject: [PATCH 29/69] fix kernel_argv in qtconsole --- IPython/frontend/consoleapp.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/IPython/frontend/consoleapp.py b/IPython/frontend/consoleapp.py index ab1beb75f2b..16ae58d1b13 100644 --- a/IPython/frontend/consoleapp.py +++ b/IPython/frontend/consoleapp.py @@ -328,8 +328,7 @@ def init_kernel_manager(self): ) # start the kernel if not self.existing: - kwargs['extra_arguments'] = self.kernel_argv - self.kernel_manager.start_kernel(**kwargs) + self.kernel_manager.start_kernel(extra_arguments=self.kernel_argv) elif self.sshserver: # ssh, write new connection file self.kernel_manager.write_connection_file() From 788c73834aaf315f85c7d259042d4c7b387c20cc Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 8 May 2012 16:19:09 -0700 Subject: [PATCH 30/69] expose silent in view.execute --- IPython/parallel/client/view.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/parallel/client/view.py b/IPython/parallel/client/view.py index 6b22c2c3a46..ee5804e47a4 100644 --- a/IPython/parallel/client/view.py +++ b/IPython/parallel/client/view.py @@ -593,7 +593,7 @@ def map(self, f, *sequences, **kwargs): @sync_results @save_ids - def execute(self, code, targets=None, block=None): + def execute(self, code, silent=True, targets=None, block=None): """Executes `code` on `targets` in blocking or nonblocking manner. ``execute`` is always `bound` (affects engine namespace) @@ -614,7 +614,7 @@ def execute(self, code, targets=None, block=None): msg_ids = [] trackers = [] for ident in _idents: - msg = self.client.send_execute_request(self._socket, code, ident=ident) + msg = self.client.send_execute_request(self._socket, code, silent=silent, ident=ident) msg_ids.append(msg['header']['msg_id']) ar = AsyncResult(self.client, msg_ids, fname='execute', targets=targets) if block: From 60c884a3d92819a0bb9d153a5771c2db83062f1f Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 8 May 2012 16:19:58 -0700 Subject: [PATCH 31/69] track display data in the parallel Client --- IPython/parallel/client/client.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 0f4a3e11cae..3ef8bf05c14 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -97,6 +97,7 @@ def __init__(self, *args, **kwargs): 'pyerr' : None, 'stdout' : '', 'stderr' : '', + 'outputs' : [], } self.update(md) self.update(dict(*args, **kwargs)) @@ -773,8 +774,13 @@ def _flush_iopub(self, sock): md.update({'pyerr' : self._unwrap_exception(content)}) elif msg_type == 'pyin': md.update({'pyin' : content['code']}) + elif msg_type == 'display_data': + md['outputs'].append(content.get('data')) + elif msg_type == 'pyout': + md['pyout'] = content.get('data') else: - md.update({msg_type : content.get('data', '')}) + # unhandled msg_type (status, etc.) + pass # reduntant? self.metadata[msg_id] = md From a27adad3589d461aec35d9206de605f8929ce562 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 8 May 2012 18:07:00 -0700 Subject: [PATCH 32/69] discard parallel.util.asbytes in favor of py3compat.cast_bytes --- IPython/parallel/apps/ipengineapp.py | 5 +++-- IPython/parallel/client/client.py | 5 +++-- IPython/parallel/controller/heartmonitor.py | 9 +++++---- IPython/parallel/controller/hub.py | 13 +++++++------ IPython/parallel/controller/scheduler.py | 9 +++++---- IPython/parallel/engine/engine.py | 12 ++++++------ IPython/parallel/util.py | 6 ------ 7 files changed, 29 insertions(+), 30 deletions(-) diff --git a/IPython/parallel/apps/ipengineapp.py b/IPython/parallel/apps/ipengineapp.py index 65ab1652fb6..b249a74e0ad 100755 --- a/IPython/parallel/apps/ipengineapp.py +++ b/IPython/parallel/apps/ipengineapp.py @@ -45,9 +45,10 @@ from IPython.config.configurable import Configurable from IPython.parallel.engine.engine import EngineFactory -from IPython.parallel.util import disambiguate_url, asbytes +from IPython.parallel.util import disambiguate_url from IPython.utils.importstring import import_item +from IPython.utils.py3compat import cast_bytes from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float @@ -203,7 +204,7 @@ def load_connector_file(self): d = json.loads(f.read()) if 'exec_key' in d: - config.Session.key = asbytes(d['exec_key']) + config.Session.key = cast_bytes(d['exec_key']) try: config.EngineFactory.location diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 3ef8bf05c14..3a9bfcd00f0 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -36,6 +36,7 @@ from IPython.utils.jsonutil import rekey from IPython.utils.localinterfaces import LOCAL_IPS from IPython.utils.path import get_ipython_dir +from IPython.utils.py3compat import cast_bytes from IPython.utils.traitlets import (HasTraits, Integer, Instance, Unicode, Dict, List, Bool, Set, Any) from IPython.external.decorator import decorator @@ -370,7 +371,7 @@ def __init__(self, url_or_file=None, profile=None, profile_dir=None, ipython_dir if os.path.isfile(exec_key): extra_args['keyfile'] = exec_key else: - exec_key = util.asbytes(exec_key) + exec_key = cast_bytes(exec_key) extra_args['key'] = exec_key self.session = Session(**extra_args) @@ -468,7 +469,7 @@ def _build_targets(self, targets): if not isinstance(targets, (tuple, list, xrange)): raise TypeError("targets by int/slice/collection of ints only, not %s"%(type(targets))) - return [util.asbytes(self._engines[t]) for t in targets], list(targets) + return [cast_bytes(self._engines[t]) for t in targets], list(targets) def _connect(self, sshserver, ssh_kwargs, timeout): """setup all our socket connections to the cluster. This is called from diff --git a/IPython/parallel/controller/heartmonitor.py b/IPython/parallel/controller/heartmonitor.py index 83d967d0e79..927af4c0ac3 100755 --- a/IPython/parallel/controller/heartmonitor.py +++ b/IPython/parallel/controller/heartmonitor.py @@ -23,9 +23,10 @@ from zmq.eventloop import ioloop, zmqstream from IPython.config.configurable import LoggingConfigurable +from IPython.utils.py3compat import str_to_bytes from IPython.utils.traitlets import Set, Instance, CFloat, Integer -from IPython.parallel.util import asbytes, log_errors +from IPython.parallel.util import log_errors class Heart(object): """A basic heart object for responding to a HeartMonitor. @@ -123,7 +124,7 @@ def beat(self): self.responses = set() # print self.on_probation, self.hearts # self.log.debug("heartbeat::beat %.3f, %i beating hearts", self.lifetime, len(self.hearts)) - self.pingstream.send(asbytes(str(self.lifetime))) + self.pingstream.send(str_to_bytes(str(self.lifetime))) # flush stream to force immediate socket send self.pingstream.flush() @@ -151,8 +152,8 @@ def handle_heart_failure(self, heart): @log_errors def handle_pong(self, msg): "a heart just beat" - current = asbytes(str(self.lifetime)) - last = asbytes(str(self.last_ping)) + current = str_to_bytes(str(self.lifetime)) + last = str_to_bytes(str(self.last_ping)) if msg[1] == current: delta = time.time()-self.tic # self.log.debug("heartbeat::heart %r took %.2f ms to respond"%(msg[0], 1000*delta)) diff --git a/IPython/parallel/controller/hub.py b/IPython/parallel/controller/hub.py index 41ab5113062..259b0ea7459 100644 --- a/IPython/parallel/controller/hub.py +++ b/IPython/parallel/controller/hub.py @@ -28,6 +28,7 @@ # internal: from IPython.utils.importstring import import_item +from IPython.utils.py3compat import cast_bytes from IPython.utils.traitlets import ( HasTraits, Instance, Integer, Unicode, Dict, Set, Tuple, CBytes, DottedObjectName ) @@ -441,7 +442,7 @@ def _validate_targets(self, targets): for t in targets: # map raw identities to ids if isinstance(t, (str,unicode)): - t = self.by_ident.get(t, t) + t = self.by_ident.get(cast_bytes(t), t) _targets.append(t) targets = _targets bad_targets = [ t for t in targets if t not in self.ids ] @@ -719,8 +720,8 @@ def save_task_result(self, idents, msg): self.unassigned.remove(msg_id) header = msg['header'] - engine_uuid = header.get('engine', None) - eid = self.by_ident.get(engine_uuid, None) + engine_uuid = header.get('engine', u'') + eid = self.by_ident.get(cast_bytes(engine_uuid), None) status = header.get('status', None) @@ -763,7 +764,7 @@ def save_task_destination(self, idents, msg): # print (content) msg_id = content['msg_id'] engine_uuid = content['engine_id'] - eid = self.by_ident[util.asbytes(engine_uuid)] + eid = self.by_ident[cast_bytes(engine_uuid)] self.log.info("task::task %r arrived on %r", msg_id, eid) if msg_id in self.unassigned: @@ -853,13 +854,13 @@ def register_engine(self, reg, msg): """Register a new engine.""" content = msg['content'] try: - queue = util.asbytes(content['queue']) + queue = cast_bytes(content['queue']) except KeyError: self.log.error("registration::queue not specified", exc_info=True) return heart = content.get('heartbeat', None) if heart: - heart = util.asbytes(heart) + heart = cast_bytes(heart) """register a new engine, and create the socket(s) necessary""" eid = self._next_id # print (eid, queue, reg, heart) diff --git a/IPython/parallel/controller/scheduler.py b/IPython/parallel/controller/scheduler.py index 25c4a2c77bb..eea1043d901 100644 --- a/IPython/parallel/controller/scheduler.py +++ b/IPython/parallel/controller/scheduler.py @@ -42,10 +42,11 @@ from IPython.config.application import Application from IPython.config.loader import Config from IPython.utils.traitlets import Instance, Dict, List, Set, Integer, Enum, CBytes +from IPython.utils.py3compat import cast_bytes from IPython.parallel import error, util from IPython.parallel.factory import SessionFactory -from IPython.parallel.util import connect_logger, local_logger, asbytes +from IPython.parallel.util import connect_logger, local_logger from .dependency import Dependency @@ -262,7 +263,7 @@ def dispatch_notification(self, msg): self.log.error("Unhandled message type: %r"%msg_type) else: try: - handler(asbytes(msg['content']['queue'])) + handler(cast_bytes(msg['content']['queue'])) except Exception: self.log.error("task::Invalid notification msg: %r", msg, exc_info=True) @@ -316,7 +317,7 @@ def handle_stranded_tasks(self, engine): # prevent double-handling of messages continue - raw_msg = lost[msg_id][0] + raw_msg = lost[msg_id].raw_msg idents,msg = self.session.feed_identities(raw_msg, copy=False) parent = self.session.unpack(msg[1].bytes) idents = [engine, idents[0]] @@ -370,7 +371,7 @@ def dispatch_submission(self, raw_msg): # get targets as a set of bytes objects # from a list of unicode objects targets = header.get('targets', []) - targets = map(asbytes, targets) + targets = map(cast_bytes, targets) targets = set(targets) retries = header.get('retries', 0) diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index e9578cb88e4..b5f50625569 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -27,11 +27,11 @@ from IPython.utils.traitlets import ( Instance, Dict, Integer, Type, CFloat, Unicode, CBytes, Bool ) -from IPython.utils import py3compat +from IPython.utils.py3compat import cast_bytes from IPython.parallel.controller.heartmonitor import Heart from IPython.parallel.factory import RegistrationFactory -from IPython.parallel.util import disambiguate_url, asbytes +from IPython.parallel.util import disambiguate_url from IPython.zmq.session import Message from IPython.zmq.ipkernel import Kernel @@ -69,7 +69,7 @@ class EngineFactory(RegistrationFactory): bident = CBytes() ident = Unicode() def _ident_changed(self, name, old, new): - self.bident = asbytes(new) + self.bident = cast_bytes(new) using_ssh=Bool(False) @@ -194,12 +194,12 @@ def complete_registration(self, msg, connect, maybe_tunnel): # Redirect input streams and set a display hook. if self.out_stream_factory: sys.stdout = self.out_stream_factory(self.session, iopub_socket, u'stdout') - sys.stdout.topic = py3compat.cast_bytes('engine.%i.stdout' % self.id) + sys.stdout.topic = cast_bytes('engine.%i.stdout' % self.id) sys.stderr = self.out_stream_factory(self.session, iopub_socket, u'stderr') - sys.stderr.topic = py3compat.cast_bytes('engine.%i.stderr' % self.id) + sys.stderr.topic = cast_bytes('engine.%i.stderr' % self.id) if self.display_hook_factory: sys.displayhook = self.display_hook_factory(self.session, iopub_socket) - sys.displayhook.topic = py3compat.cast_bytes('engine.%i.pyout' % self.id) + sys.displayhook.topic = cast_bytes('engine.%i.pyout' % self.id) self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, diff --git a/IPython/parallel/util.py b/IPython/parallel/util.py index 03c0cf91498..5b3e5a6dcbb 100644 --- a/IPython/parallel/util.py +++ b/IPython/parallel/util.py @@ -124,12 +124,6 @@ def log_errors(f, self, *args, **kwargs): self.log.error("Uncaught exception in %r" % f, exc_info=True) -def asbytes(s): - """ensure that an object is ascii bytes""" - if isinstance(s, unicode): - s = s.encode('ascii') - return s - def is_url(url): """boolean check for whether a string is a zmq url""" if '://' not in url: From e0899e51609de680f530b32c1ade5366fcee7c69 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 12 May 2012 12:13:31 -0700 Subject: [PATCH 33/69] fix topic on displayhook in ZMQShell --- IPython/zmq/displayhook.py | 3 ++- IPython/zmq/ipkernel.py | 1 + IPython/zmq/zmqshell.py | 6 +++++- 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/IPython/zmq/displayhook.py b/IPython/zmq/displayhook.py index 358c979c0a2..a3f76abdd4d 100644 --- a/IPython/zmq/displayhook.py +++ b/IPython/zmq/displayhook.py @@ -43,6 +43,7 @@ class ZMQShellDisplayHook(DisplayHook): """A displayhook subclass that publishes data using ZeroMQ. This is intended to work with an InteractiveShell instance. It sends a dict of different representations of the object.""" + topic=None session = Instance(Session) pub_socket = Instance('zmq.Socket') @@ -67,6 +68,6 @@ def finish_displayhook(self): """Finish up all displayhook activities.""" sys.stdout.flush() sys.stderr.flush() - self.session.send(self.pub_socket, self.msg) + self.session.send(self.pub_socket, self.msg, ident=self.topic) self.msg = None diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 192d4bdc606..02b81c9dab4 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -145,6 +145,7 @@ def __init__(self, **kwargs): ) self.shell.displayhook.session = self.session self.shell.displayhook.pub_socket = self.iopub_socket + self.shell.displayhook.topic = self._topic('pyout') self.shell.display_pub.session = self.session self.shell.display_pub.pub_socket = self.iopub_socket diff --git a/IPython/zmq/zmqshell.py b/IPython/zmq/zmqshell.py index d787dcf0053..222a9658e74 100644 --- a/IPython/zmq/zmqshell.py +++ b/IPython/zmq/zmqshell.py @@ -183,7 +183,11 @@ def _showtraceback(self, etype, evalue, stb): dh = self.displayhook # Send exception info over pub socket for other clients than the caller # to pick up - exc_msg = dh.session.send(dh.pub_socket, u'pyerr', json_clean(exc_content), dh.parent_header) + topic = None + if dh.topic: + topic = dh.topic.replace(b'pyout', b'pyerr') + + exc_msg = dh.session.send(dh.pub_socket, u'pyerr', json_clean(exc_content), dh.parent_header, ident=topic) # FIXME - Hack: store exception info in shell object. Right now, the # caller is reading this info after the fact, we need to fix this logic From e7f402a01bfb2f81e04dd7fd2699717dc3629739 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 12 May 2012 12:14:33 -0700 Subject: [PATCH 34/69] fix message when trying new table due to mismatch in SQLiteDB also silence these messages in tests --- IPython/parallel/controller/sqlitedb.py | 5 +++-- IPython/parallel/tests/test_db.py | 5 ++++- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/IPython/parallel/controller/sqlitedb.py b/IPython/parallel/controller/sqlitedb.py index 0a85517e993..088ca971f16 100644 --- a/IPython/parallel/controller/sqlitedb.py +++ b/IPython/parallel/controller/sqlitedb.py @@ -226,15 +226,16 @@ def _init_db(self): # isolation_level = None)#, cached_statements=64) # print dir(self._db) - first_table = self.table + first_table = previous_table = self.table i=0 while not self._check_table(): i+=1 self.table = first_table+'_%i'%i self.log.warn( "Table %s exists and doesn't match db format, trying %s"% - (first_table,self.table) + (previous_table, self.table) ) + previous_table = self.table self._db.execute("""CREATE TABLE IF NOT EXISTS %s (msg_id text PRIMARY KEY, diff --git a/IPython/parallel/tests/test_db.py b/IPython/parallel/tests/test_db.py index 90a39025fe5..01cc523b71e 100644 --- a/IPython/parallel/tests/test_db.py +++ b/IPython/parallel/tests/test_db.py @@ -18,6 +18,7 @@ from __future__ import division +import logging import os import tempfile import time @@ -226,7 +227,9 @@ class TestSQLiteBackend(TestDictBackend): @dec.skip_without('sqlite3') def create_db(self): location, fname = os.path.split(temp_db) - return SQLiteDB(location=location, fname=fname) + log = logging.getLogger('test') + log.setLevel(logging.CRITICAL) + return SQLiteDB(location=location, fname=fname, log=log) def tearDown(self): self.db._db.close() From 4e5eb7d197c6dcdbbd4a46b07ac48a54680337a4 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 12 May 2012 12:16:13 -0700 Subject: [PATCH 35/69] soften messages for monitor messages with bad topics --- IPython/parallel/controller/hub.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/parallel/controller/hub.py b/IPython/parallel/controller/hub.py index 259b0ea7459..622dff840a6 100644 --- a/IPython/parallel/controller/hub.py +++ b/IPython/parallel/controller/hub.py @@ -468,13 +468,13 @@ def dispatch_monitor_traffic(self, msg): except ValueError: idents=[] if not idents: - self.log.error("Bad Monitor Message: %r", msg) + self.log.error("Monitor message without topic: %r", msg) return handler = self.monitor_handlers.get(switch, None) if handler is not None: handler(idents, msg) else: - self.log.error("Invalid monitor topic: %r", switch) + self.log.error("Unrecognized monitor topic: %r", switch) @util.log_errors From fa181ff0df17c855dff3e4dfafb6c3928daf981e Mon Sep 17 00:00:00 2001 From: MinRK Date: Sat, 12 May 2012 12:17:57 -0700 Subject: [PATCH 36/69] resubmitted tasks are now wholly separate (new msg_ids) This removes much of the fragility due to trying to cram a resubmit into the original ID. The `resubmit` record is now the resubmitted task ID, rather than the timestamp. --- IPython/parallel/client/client.py | 10 ++-- IPython/parallel/controller/hub.py | 70 ++++++++++++++----------- IPython/parallel/controller/sqlitedb.py | 4 +- IPython/parallel/tests/test_client.py | 5 +- docs/source/parallel/parallel_db.txt | 4 +- 5 files changed, 48 insertions(+), 45 deletions(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 3a9bfcd00f0..15a08eb642e 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -1286,12 +1286,6 @@ def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None): raise TypeError("indices must be str or int, not %r"%id) theids.append(id) - for msg_id in theids: - self.outstanding.discard(msg_id) - if msg_id in self.history: - self.history.remove(msg_id) - self.results.pop(msg_id, None) - self.metadata.pop(msg_id, None) content = dict(msg_ids = theids) self.session.send(self._query_socket, 'resubmit_request', content) @@ -1303,8 +1297,10 @@ def resubmit(self, indices_or_msg_ids=None, subheader=None, block=None): content = msg['content'] if content['status'] != 'ok': raise self._unwrap_exception(content) + mapping = content['resubmitted'] + new_ids = [ mapping[msg_id] for msg_id in theids ] - ar = AsyncHubResult(self, msg_ids=theids) + ar = AsyncHubResult(self, msg_ids=new_ids) if block: ar.wait() diff --git a/IPython/parallel/controller/hub.py b/IPython/parallel/controller/hub.py index 622dff840a6..5f558242a1c 100644 --- a/IPython/parallel/controller/hub.py +++ b/IPython/parallel/controller/hub.py @@ -1134,7 +1134,7 @@ def finish(reply): # validate msg_ids found_ids = [ rec['msg_id'] for rec in records ] - invalid_ids = filter(lambda m: m in self.pending, found_ids) + pending_ids = [ msg_id for msg_id in found_ids if msg_id in self.pending ] if len(records) > len(msg_ids): try: raise RuntimeError("DB appears to be in an inconsistent state." @@ -1147,40 +1147,46 @@ def finish(reply): raise KeyError("No such msg(s): %r" % missing) except KeyError: return finish(error.wrap_exception()) - elif invalid_ids: - msg_id = invalid_ids[0] + elif pending_ids: + pass + # no need to raise on resubmit of pending task, now that we + # resubmit under new ID, but do we want to raise anyway? + # msg_id = invalid_ids[0] + # try: + # raise ValueError("Task(s) %r appears to be inflight" % ) + # except Exception: + # return finish(error.wrap_exception()) + + # mapping of original IDs to resubmitted IDs + resubmitted = {} + + # send the messages + for rec in records: + header = rec['header'] + msg = self.session.msg(header['msg_type']) + msg_id = msg['msg_id'] + msg['content'] = rec['content'] + header.update(msg['header']) + msg['header'] = header + + self.session.send(self.resubmit, msg, buffers=rec['buffers']) + + resubmitted[rec['msg_id']] = msg_id + self.pending.add(msg_id) + msg['buffers'] = [] try: - raise ValueError("Task %r appears to be inflight" % msg_id) + self.db.add_record(msg_id, init_record(msg)) except Exception: - return finish(error.wrap_exception()) + self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True) - # clear the existing records - now = datetime.now() - rec = empty_record() - map(rec.pop, ['msg_id', 'header', 'content', 'buffers', 'submitted']) - rec['resubmitted'] = now - rec['queue'] = 'task' - rec['client_uuid'] = client_id[0] - try: - for msg_id in msg_ids: - self.all_completed.discard(msg_id) - self.db.update_record(msg_id, rec) - except Exception: - self.log.error('db::db error upating record', exc_info=True) - reply = error.wrap_exception() - else: - # send the messages - for rec in records: - header = rec['header'] - # include resubmitted in header to prevent digest collision - header['resubmitted'] = now - msg = self.session.msg(header['msg_type']) - msg['content'] = rec['content'] - msg['header'] = header - msg['header']['msg_id'] = rec['msg_id'] - self.session.send(self.resubmit, msg, buffers=rec['buffers']) - - finish(dict(status='ok')) + finish(dict(status='ok', resubmitted=resubmitted)) + + # store the new IDs in the Task DB + for msg_id, resubmit_id in resubmitted.iteritems(): + try: + self.db.update_record(msg_id, {'resubmitted' : resubmit_id}) + except Exception: + self.log.error("db::DB Error updating record: %s", msg_id, exc_info=True) def _extract_record(self, rec): diff --git a/IPython/parallel/controller/sqlitedb.py b/IPython/parallel/controller/sqlitedb.py index 088ca971f16..2b017d40a8a 100644 --- a/IPython/parallel/controller/sqlitedb.py +++ b/IPython/parallel/controller/sqlitedb.py @@ -138,7 +138,7 @@ class SQLiteDB(BaseDB): 'engine_uuid' : 'text', 'started' : 'timestamp', 'completed' : 'timestamp', - 'resubmitted' : 'timestamp', + 'resubmitted' : 'text', 'received' : 'timestamp', 'result_header' : 'dict text', 'result_content' : 'dict text', @@ -247,7 +247,7 @@ def _init_db(self): engine_uuid text, started timestamp, completed timestamp, - resubmitted timestamp, + resubmitted text, received timestamp, result_header dict text, result_content dict text, diff --git a/IPython/parallel/tests/test_client.py b/IPython/parallel/tests/test_client.py index 2475cd97f9b..99a00606805 100644 --- a/IPython/parallel/tests/test_client.py +++ b/IPython/parallel/tests/test_client.py @@ -331,13 +331,14 @@ def f(): r2 = ahr.get(1) def test_resubmit_inflight(self): - """ensure ValueError on resubmit of inflight task""" + """resubmit of inflight task""" v = self.client.load_balanced_view() ar = v.apply_async(time.sleep,1) # give the message a chance to arrive time.sleep(0.2) - self.assertRaisesRemote(ValueError, self.client.resubmit, ar.msg_ids) + ahr = self.client.resubmit(ar.msg_ids) ar.get(2) + ahr.get(2) def test_resubmit_badkey(self): """ensure KeyError on resubmit of nonexistant task""" diff --git a/docs/source/parallel/parallel_db.txt b/docs/source/parallel/parallel_db.txt index 648223fe39a..a742f57f945 100644 --- a/docs/source/parallel/parallel_db.txt +++ b/docs/source/parallel/parallel_db.txt @@ -34,7 +34,7 @@ TaskRecord keys: =============== =============== ============= Key Type Description =============== =============== ============= -msg_id uuid(bytes) The msg ID +msg_id uuid(ascii) The msg ID header dict The request header content dict The request content (likely empty) buffers list(bytes) buffers containing serialized request objects @@ -43,7 +43,7 @@ client_uuid uuid(bytes) IDENT of client's socket engine_uuid uuid(bytes) IDENT of engine's socket started datetime time task began execution on engine completed datetime time task finished execution (success or failure) on engine -resubmitted datetime time of resubmission (if applicable) +resubmitted uuid(ascii) msg_id of resubmitted task (if applicable) result_header dict header for result result_content dict content for result result_buffers list(bytes) buffers containing serialized request objects From 3c9c3671db77027f57a59abd104f7d5aca532589 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sun, 13 May 2012 13:06:11 -0700 Subject: [PATCH 37/69] remove old _really_apply call --- IPython/parallel/client/view.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/IPython/parallel/client/view.py b/IPython/parallel/client/view.py index ee5804e47a4..47bf0aa4d01 100644 --- a/IPython/parallel/client/view.py +++ b/IPython/parallel/client/view.py @@ -623,8 +623,6 @@ def execute(self, code, silent=True, targets=None, block=None): except KeyboardInterrupt: pass return ar - - return self._really_apply(util._execute, args=(code,), block=block, targets=targets) def run(self, filename, targets=None, block=None): """Execute contents of `filename` on my engine(s). From 6423e9d47a4fca5accb8abb6f4dc03ba655db9f2 Mon Sep 17 00:00:00 2001 From: MinRK Date: Sun, 13 May 2012 13:06:46 -0700 Subject: [PATCH 38/69] store execute_replies in a nice wrapper --- IPython/parallel/client/client.py | 60 +++++++++++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 15a08eb642e..daa0745adf2 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -72,6 +72,60 @@ def spin_first(f, self, *args, **kwargs): # Classes #-------------------------------------------------------------------------- + +class ExecuteReply(object): + """wrapper for finished Execute results""" + def __init__(self, msg_id, content, metadata): + self.msg_id = msg_id + self._content = content + self.execution_count = content['execution_count'] + self.metadata = metadata + + def __getitem__(self, key): + return self.metadata[key] + + def __getattr__(self, key): + if key not in self.metadata: + raise AttributeError(key) + return self.metadata[key] + + def __repr__(self): + pyout = self.metadata['pyout'] or {} + text_out = pyout.get('text/plain', '') + if len(text_out) > 32: + text_out = text_out[:29] + '...' + + return "" % (self.execution_count, text_out) + + def _repr_html_(self): + pyout = self.metadata['pyout'] or {} + return pyout.get("text/html") + + def _repr_latex_(self): + pyout = self.metadata['pyout'] or {} + return pyout.get("text/latex") + + def _repr_json_(self): + pyout = self.metadata['pyout'] or {} + return pyout.get("application/json") + + def _repr_javascript_(self): + pyout = self.metadata['pyout'] or {} + return pyout.get("application/javascript") + + def _repr_png_(self): + pyout = self.metadata['pyout'] or {} + return pyout.get("image/png") + + def _repr_jpeg_(self): + pyout = self.metadata['pyout'] or {} + return pyout.get("image/jpeg") + + def _repr_svg_(self): + pyout = self.metadata['pyout'] or {} + return pyout.get("image/svg+xml") + + class Metadata(dict): """Subclass of dict for initializing metadata values. @@ -646,7 +700,7 @@ def _handle_execute_reply(self, msg): # construct result: if content['status'] == 'ok': - self.results[msg_id] = content + self.results[msg_id] = ExecuteReply(msg_id, content, md) elif content['status'] == 'aborted': self.results[msg_id] = error.TaskAborted(msg_id) elif content['status'] == 'resubmitted': @@ -878,14 +932,14 @@ def spin(self): """ if self._notification_socket: self._flush_notifications() + if self._iopub_socket: + self._flush_iopub(self._iopub_socket) if self._mux_socket: self._flush_results(self._mux_socket) if self._task_socket: self._flush_results(self._task_socket) if self._control_socket: self._flush_control(self._control_socket) - if self._iopub_socket: - self._flush_iopub(self._iopub_socket) if self._query_socket: self._flush_ignored_hub_replies() From d34fb7b45f94925bcc88ea3c26f6b350bbe88f77 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 17:03:28 -0700 Subject: [PATCH 39/69] minor fixes to allow kernel to be re-entrant * user_mod doesn't typecheck ModuleType, because it can be a DummyMod * start hearbeat thread in initialize step, not start. * ensure shell.exit_now is False at Kernel.start --- IPython/zmq/ipkernel.py | 6 ++++-- IPython/zmq/kernelapp.py | 2 +- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 02b81c9dab4..084a3e6cfc5 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -84,7 +84,7 @@ def _eventloop_changed(self, name, old, new): stdin_socket = Instance(zmq.Socket) log = Instance(logging.Logger) - user_module = Instance('types.ModuleType') + user_module = Any() def _user_module_changed(self, name, old, new): if self.shell is not None: self.shell.user_module = new @@ -250,6 +250,7 @@ def enter_eventloop(self): def start(self): """register dispatchers for streams""" + self.shell.exit_now = False if self.control_stream: self.control_stream.on_recv(self.dispatch_control, copy=False) @@ -501,7 +502,8 @@ def shutdown_request(self, stream, ident, parent): self._at_shutdown() # call sys.exit after a short delay - ioloop.IOLoop.instance().add_timeout(time.time()+0.1, lambda : sys.exit(0)) + loop = ioloop.IOLoop.instance() + loop.add_timeout(time.time()+0.1, loop.stop) #--------------------------------------------------------------------------- # Engine methods diff --git a/IPython/zmq/kernelapp.py b/IPython/zmq/kernelapp.py index 50968a0061c..762401f4637 100644 --- a/IPython/zmq/kernelapp.py +++ b/IPython/zmq/kernelapp.py @@ -225,6 +225,7 @@ def init_sockets(self): self.heartbeat = Heartbeat(hb_ctx, (self.ip, self.hb_port)) self.hb_port = self.heartbeat.port self.log.debug("Heartbeat REP Channel on port: %i"%self.hb_port) + self.heartbeat.start() # Helper to make it easier to connect to an existing kernel. # set log-level to critical, to make sure it is output @@ -302,7 +303,6 @@ def initialize(self, argv=None): sys.stderr.flush() def start(self): - self.heartbeat.start() if self.poller is not None: self.poller.start() self.kernel.start() From f56f5a1c07b9dd49c80c87eb6426939322ff65f7 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 17:27:33 -0700 Subject: [PATCH 40/69] upstream change preventing kernel exit on SIGINT during eventloop integration. --- IPython/zmq/ipkernel.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 084a3e6cfc5..2c8f17d7278 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -243,7 +243,17 @@ def enter_eventloop(self): self.log.critical("entering eventloop") # restore default_int_handler signal(SIGINT, default_int_handler) - self.eventloop(self) + while self.eventloop is not None: + try: + self.eventloop(self) + except KeyboardInterrupt: + # Ctrl-C shouldn't crash the kernel + self.log.error("KeyboardInterrupt caught in kernel") + continue + else: + # eventloop exited cleanly, this means we should stop (right?) + self.eventloop = None + break self.log.critical("exiting eventloop") # if eventloop exits, IOLoop should stop ioloop.IOLoop.instance().stop() From 13736b908cab8c786d65376cc27b542e8e6b8724 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 18:21:17 -0700 Subject: [PATCH 41/69] InteractiveShell.restore_sys_module_state actually restores main previously restored it only as a string, rather than the module itself. --- IPython/core/interactiveshell.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/IPython/core/interactiveshell.py b/IPython/core/interactiveshell.py index cd5e36e6d02..460be44e13d 100644 --- a/IPython/core/interactiveshell.py +++ b/IPython/core/interactiveshell.py @@ -717,6 +717,7 @@ def save_sys_module_state(self): self._orig_sys_module_state['stderr'] = sys.stderr self._orig_sys_module_state['excepthook'] = sys.excepthook self._orig_sys_modules_main_name = self.user_module.__name__ + self._orig_sys_modules_main_mod = sys.modules.get(self.user_module.__name__) def restore_sys_module_state(self): """Restore the state of the sys module.""" @@ -726,7 +727,8 @@ def restore_sys_module_state(self): except AttributeError: pass # Reset what what done in self.init_sys_modules - sys.modules[self.user_module.__name__] = self._orig_sys_modules_main_name + if self._orig_sys_modules_main_mod is not None: + sys.modules[self._orig_sys_modules_main_name] = self._orig_sys_modules_main_mod #------------------------------------------------------------------------- # Things related to hooks From ea917e3e7412266cbb8f11086a1e87a6f73cc536 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 18:32:19 -0700 Subject: [PATCH 42/69] test re-entrant embed_kernel --- IPython/zmq/tests/test_embed_kernel.py | 30 ++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/IPython/zmq/tests/test_embed_kernel.py b/IPython/zmq/tests/test_embed_kernel.py index 6357c785dcf..6d426712fe5 100644 --- a/IPython/zmq/tests/test_embed_kernel.py +++ b/IPython/zmq/tests/test_embed_kernel.py @@ -156,3 +156,33 @@ def test_embed_kernel_namespace(): content = msg['content'] nt.assert_false(content['found']) +def test_embed_kernel_reentrant(): + """IPython.embed_kernel() can be called multiple times""" + cmd = '\n'.join([ + 'from IPython import embed_kernel', + 'count = 0', + 'def go():', + ' global count', + ' embed_kernel()', + ' count = count + 1', + '', + 'while True:' + ' go()', + '', + ]) + + with setup_kernel(cmd) as km: + shell = km.shell_channel + for i in range(5): + msg_id = shell.object_info('count') + msg = shell.get_msg(block=True, timeout=2) + content = msg['content'] + nt.assert_true(content['found']) + nt.assert_equals(content['string_form'], unicode(i)) + + # exit from embed_kernel + shell.execute("get_ipython().exit_now = True") + msg = shell.get_msg(block=True, timeout=2) + time.sleep(0.2) + + From 0d0437b5c9fcd277ec68b8ee8f1a260c808b33e5 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 18:34:18 -0700 Subject: [PATCH 43/69] undo initial clobbering done by init_sys_modules in embed_kernel revealed by test added in previous commit. --- IPython/zmq/ipkernel.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 2c8f17d7278..4c193a09b4c 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -887,6 +887,13 @@ def embed_kernel(module=None, local_ns=None, **kwargs): else: app = IPKernelApp.instance(**kwargs) app.initialize([]) + # Undo unnecessary sys module mangling from init_sys_modules. + # This would not be necessary if we could prevent it + # in the first place by using a different InteractiveShell + # subclass, as in the regular embed case. + main = app.kernel.shell._orig_sys_modules_main_mod + if main is not None: + sys.modules[app.kernel.shell._orig_sys_modules_main_name] = main # load the calling scope if not given (caller_module, caller_locals) = extract_module_locals(1) From 176eff9ac38b0ab72d2ae869e2579e8264971707 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 21:05:11 -0700 Subject: [PATCH 44/69] add @interactive decorator to IPython.parallel namespace --- IPython/parallel/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/IPython/parallel/__init__.py b/IPython/parallel/__init__.py index 4cb507d1230..c41aac246b6 100644 --- a/IPython/parallel/__init__.py +++ b/IPython/parallel/__init__.py @@ -35,6 +35,7 @@ from .client.client import Client from .client.remotefunction import * from .client.view import * +from .util import interactive from .controller.dependency import * From 05f0cd8a38ebe0525d8bd54bda46ca9b557ef4e8 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 21:30:09 -0700 Subject: [PATCH 45/69] add len(view) corresponds to the number of associated engines --- IPython/parallel/client/view.py | 11 +++++++++-- IPython/parallel/tests/test_view.py | 20 +++++++++++++++++++- 2 files changed, 28 insertions(+), 3 deletions(-) diff --git a/IPython/parallel/client/view.py b/IPython/parallel/client/view.py index 47bf0aa4d01..ca29e841906 100644 --- a/IPython/parallel/client/view.py +++ b/IPython/parallel/client/view.py @@ -128,13 +128,20 @@ def __init__(self, client=None, socket=None, **flags): assert not self.__class__ is View, "Don't use base View objects, use subclasses" - def __repr__(self): strtargets = str(self.targets) if len(strtargets) > 16: strtargets = strtargets[:12]+'...]' return "<%s %s>"%(self.__class__.__name__, strtargets) - + + def __len__(self): + if isinstance(self.targets, list): + return len(self.targets) + elif isinstance(self.targets, int): + return 1 + else: + return len(self.client) + def set_flags(self, **kwargs): """set my attribute flags by keyword. diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index 8c524c105c0..88caa9b6d67 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -25,6 +25,7 @@ from nose import SkipTest from IPython.testing import decorators as dec +from IPython.testing.ipunittest import ParametricTestCase from IPython import parallel as pmod from IPython.parallel import error @@ -39,7 +40,7 @@ def setup(): add_engines(3, total=True) -class TestView(ClusterTestCase): +class TestView(ClusterTestCase, ParametricTestCase): def test_z_crash_mux(self): """test graceful handling of engine death (direct)""" @@ -550,4 +551,21 @@ def test_single_engine_map(self): check = [ -1*i for i in r ] result = e0.map_sync(lambda x: -1*x, r) self.assertEquals(result, check) + + def test_len(self): + """len(view) makes sense""" + e0 = self.client[self.client.ids[0]] + yield self.assertEquals(len(e0), 1) + v = self.client[:] + yield self.assertEquals(len(v), len(self.client.ids)) + v = self.client.direct_view('all') + yield self.assertEquals(len(v), len(self.client.ids)) + v = self.client[:2] + yield self.assertEquals(len(v), 2) + v = self.client[:1] + yield self.assertEquals(len(v), 1) + v = self.client.load_balanced_view() + yield self.assertEquals(len(v), len(self.client.ids)) + # parametric tests seem to require manual closing? + self.client.close() From cf52d0635bee98a727e40c7500a48fba892ef699 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 21:55:15 -0700 Subject: [PATCH 46/69] add topic to display publisher, and fix set_parent for apply_requests --- IPython/parallel/engine/engine.py | 1 + IPython/zmq/ipkernel.py | 12 +++++++----- IPython/zmq/zmqshell.py | 7 ++++--- 3 files changed, 12 insertions(+), 8 deletions(-) diff --git a/IPython/parallel/engine/engine.py b/IPython/parallel/engine/engine.py index b5f50625569..136af6642f6 100644 --- a/IPython/parallel/engine/engine.py +++ b/IPython/parallel/engine/engine.py @@ -204,6 +204,7 @@ def complete_registration(self, msg, connect, maybe_tunnel): self.kernel = Kernel(config=self.config, int_id=self.id, ident=self.ident, session=self.session, control_stream=control_stream, shell_streams=shell_streams, iopub_socket=iopub_socket, loop=loop, user_ns=self.user_ns, log=self.log) + self.kernel.shell.display_pub.topic = cast_bytes('engine.%i.displaypub' % self.id) self.kernel.start() diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 4c193a09b4c..f374b418c81 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -527,16 +527,18 @@ def apply_request(self, stream, ident, parent): except: self.log.error("Got bad msg: %s", parent, exc_info=True) return + + # Set the parent message of the display hook and out streams. + self.shell.displayhook.set_parent(parent) + self.shell.display_pub.set_parent(parent) + sys.stdout.set_parent(parent) + sys.stderr.set_parent(parent) + # pyin_msg = self.session.msg(u'pyin',{u'code':code}, parent=parent) # self.iopub_socket.send(pyin_msg) # self.session.send(self.iopub_socket, u'pyin', {u'code':code},parent=parent) sub = self._make_subheader() try: - # allow for not overriding displayhook - if hasattr(sys.displayhook, 'set_parent'): - sys.displayhook.set_parent(parent) - sys.stdout.set_parent(parent) - sys.stderr.set_parent(parent) working = self.shell.user_ns prefix = "_"+str(msg_id).replace("-","")+"_" diff --git a/IPython/zmq/zmqshell.py b/IPython/zmq/zmqshell.py index 222a9658e74..b8d420f3088 100644 --- a/IPython/zmq/zmqshell.py +++ b/IPython/zmq/zmqshell.py @@ -43,7 +43,7 @@ from IPython.utils.jsonutil import json_clean from IPython.utils.path import get_py_filename from IPython.utils.process import arg_split -from IPython.utils.traitlets import Instance, Type, Dict, CBool +from IPython.utils.traitlets import Instance, Type, Dict, CBool, CBytes from IPython.utils.warn import warn, error from IPython.zmq.displayhook import ZMQShellDisplayHook, _encode_binary from IPython.zmq.session import extract_header @@ -60,6 +60,7 @@ class ZMQDisplayPublisher(DisplayPublisher): session = Instance(Session) pub_socket = Instance('zmq.Socket') parent_header = Dict({}) + topic = CBytes(b'displaypub') def set_parent(self, parent): """Set the parent for outbound messages.""" @@ -82,7 +83,7 @@ def publish(self, source, data, metadata=None): content['metadata'] = metadata self.session.send( self.pub_socket, u'display_data', json_clean(content), - parent=self.parent_header + parent=self.parent_header, ident=self.topic, ) def clear_output(self, stdout=True, stderr=True, other=True): @@ -97,7 +98,7 @@ def clear_output(self, stdout=True, stderr=True, other=True): self.session.send( self.pub_socket, u'clear_output', content, - parent=self.parent_header + parent=self.parent_header, ident=self.topic, ) class ZMQInteractiveShell(InteractiveShell): From 1805e539a24fdb1ec124b4ad54fbf800bb5847e5 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 14 May 2012 22:12:56 -0700 Subject: [PATCH 47/69] test new execute/output behaviors --- IPython/parallel/tests/test_view.py | 86 +++++++++++++++++++++++++++++ 1 file changed, 86 insertions(+) diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index 88caa9b6d67..d634a0b0e45 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -569,3 +569,89 @@ def test_len(self): # parametric tests seem to require manual closing? self.client.close() + + # begin execute tests + + def test_execute_reply(self): + e0 = self.client[self.client.ids[0]] + e0.block = True + ar = e0.execute("5", silent=False) + er = ar.get() + time.sleep(0.2) + self.assertEquals(str(er), "" % er.execution_count) + self.assertEquals(er.pyout['text/plain'], '5') + + def test_execute_reply_stdout(self): + e0 = self.client[self.client.ids[0]] + e0.block = True + ar = e0.execute("print (5)", silent=False) + er = ar.get() + time.sleep(0.2) + self.assertEquals(er.stdout.strip(), '5') + + def test_execute_pyout(self): + """execute triggers pyout with silent=False""" + view = self.client[:] + ar = view.execute("5", silent=False, block=True) + time.sleep(0.2) + expected = [{'text/plain' : '5'}] * len(view) + self.assertEquals(ar.pyout, expected) + + def test_execute_silent(self): + """execute does not trigger pyout with silent=True""" + view = self.client[:] + ar = view.execute("5", block=True) + expected = [None] * len(view) + self.assertEquals(ar.pyout, expected) + + def test_execute_magic(self): + """execute accepts IPython commands""" + view = self.client[:] + view.execute("a = 5") + ar = view.execute("%whos", block=True) + # this will raise, if that failed + ar.get(5) + time.sleep(0.2) + for stdout in ar.stdout: + lines = stdout.splitlines() + self.assertEquals(lines[0].split(), ['Variable', 'Type', 'Data/Info']) + found = False + for line in lines[2:]: + split = line.split() + if split == ['a', 'int', '5']: + found = True + break + self.assertTrue(found, "whos output wrong: %s" % stdout) + + def test_execute_displaypub(self): + """execute tracks display_pub output""" + view = self.client[:] + view.execute("from IPython.core.display import *") + ar = view.execute("[ display(i) for i in range(5) ]", block=True) + time.sleep(0.2) + outs = [ {u'text/plain' : unicode(i)} for i in range(5) ] + expected = [outs] * len(view) + self.assertEquals(ar.outputs, expected) + + def test_apply_displaypub(self): + """apply tracks display_pub output""" + view = self.client[:] + view.execute("from IPython.core.display import *") + + @interactive + def publish(): + [ display(i) for i in range(5) ] + + ar = view.apply_async(publish) + ar.get(5) + time.sleep(0.2) + outs = [ {u'text/plain' : unicode(j)} for j in range(5) ] + expected = [outs] * len(view) + self.assertEquals(ar.outputs, expected) + + def test_execute_raises(self): + """exceptions in execute requests raise appropriately""" + view = self.client[-1] + ar = view.execute("1/0") + self.assertRaisesRemote(ZeroDivisionError, ar.get, 2) + From efa86d9f35445d2c482a9aaf07701f367b3ddc2a Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 21 May 2012 17:12:55 -0700 Subject: [PATCH 48/69] clearer error when trying to use closed Client --- IPython/parallel/client/client.py | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index daa0745adf2..5519fafc8f9 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -1115,7 +1115,9 @@ def send_apply_request(self, socket, f, args=None, kwargs=None, subheader=None, This is the principal method with which all engine execution is performed by views. """ - assert not self._closed, "cannot use me anymore, I'm closed!" + if self._closed: + raise RuntimeError("Client cannot be used after its sockets have been closed") + # defaults: args = args if args is not None else [] kwargs = kwargs if kwargs is not None else {} @@ -1155,7 +1157,9 @@ def send_execute_request(self, socket, code, silent=True, subheader=None, ident= """ - assert not self._closed, "cannot use me anymore, I'm closed!" + if self._closed: + raise RuntimeError("Client cannot be used after its sockets have been closed") + # defaults: subheader = subheader if subheader is not None else {} From 04351cd7b56aaa3a15b0c22756cc8e486c498248 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 21 May 2012 17:22:44 -0700 Subject: [PATCH 49/69] Don't use asserts when checking for connection files --- IPython/parallel/client/client.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 5519fafc8f9..466de9114a9 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -364,15 +364,19 @@ def __init__(self, url_or_file=None, profile=None, profile_dir=None, ipython_dir if self._cd is not None: if url_or_file is None: url_or_file = pjoin(self._cd.security_dir, 'ipcontroller-client.json') - assert url_or_file is not None, "I can't find enough information to connect to a hub!"\ - " Please specify at least one of url_or_file or profile." + if url_or_file is None: + raise ValueError( + "I can't find enough information to connect to a hub!" + " Please specify at least one of url_or_file or profile." + ) if not util.is_url(url_or_file): # it's not a url, try for a file if not os.path.exists(url_or_file): if self._cd: url_or_file = os.path.join(self._cd.security_dir, url_or_file) - assert os.path.exists(url_or_file), "Not a valid connection file or url: %r"%url_or_file + if not os.path.exists(url_or_file): + raise IOError("Connection file not found: %r" % url_or_file) with open(url_or_file) as f: cfg = json.loads(f.read()) else: From 4cb7712d3702b77125506376e5f40fbc8058fd48 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 21 May 2012 22:07:47 -0700 Subject: [PATCH 50/69] fix date objects in _reserialize_reply --- IPython/frontend/html/notebook/handlers.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/IPython/frontend/html/notebook/handlers.py b/IPython/frontend/html/notebook/handlers.py index 62b3b82755b..92752a55573 100644 --- a/IPython/frontend/html/notebook/handlers.py +++ b/IPython/frontend/html/notebook/handlers.py @@ -30,6 +30,7 @@ from IPython.external.decorator import decorator from IPython.zmq.session import Session from IPython.lib.security import passwd_check +from IPython.utils.jsonutil import date_default try: from docutils.core import publish_string @@ -385,13 +386,13 @@ def _reserialize_reply(self, msg_list): except KeyError: pass msg.pop('buffers') - return jsonapi.dumps(msg) + return jsonapi.dumps(msg, default=date_default) def _on_zmq_reply(self, msg_list): try: msg = self._reserialize_reply(msg_list) - except: - self.application.log.critical("Malformed message: %r" % msg_list) + except Exception: + self.application.log.critical("Malformed message: %r" % msg_list, exc_info=True) else: self.write_message(msg) From 4afa36601f4d46807afd0c77b8d702e85845fab4 Mon Sep 17 00:00:00 2001 From: MinRK Date: Mon, 21 May 2012 22:58:02 -0700 Subject: [PATCH 51/69] disable save-thread when using in-memory history db. --- IPython/core/history.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/IPython/core/history.py b/IPython/core/history.py index df981e9fcb5..6c51f73299b 100644 --- a/IPython/core/history.py +++ b/IPython/core/history.py @@ -403,8 +403,9 @@ def __init__(self, shell=None, config=None, **traits): self.save_flag = threading.Event() self.db_input_cache_lock = threading.Lock() self.db_output_cache_lock = threading.Lock() - self.save_thread = HistorySavingThread(self) - self.save_thread.start() + if self.hist_file != ':memory:': + self.save_thread = HistorySavingThread(self) + self.save_thread.start() self.new_session() From ea84367e6e3406db887d597f492a3a20f5e27a68 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 00:02:51 -0700 Subject: [PATCH 52/69] fix run_cell signature in parallelmagic * accepts silent kwarg * store_history defaults to False --- IPython/extensions/parallelmagic.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/extensions/parallelmagic.py b/IPython/extensions/parallelmagic.py index 45bdb1ebbbe..2677d27a020 100644 --- a/IPython/extensions/parallelmagic.py +++ b/IPython/extensions/parallelmagic.py @@ -211,7 +211,7 @@ def _maybe_display_output(self, result): print '[stdout:%i]'%eid, stdout - def pxrun_cell(self, raw_cell, store_history=True): + def pxrun_cell(self, raw_cell, store_history=False, silent=False): """drop-in replacement for InteractiveShell.run_cell. This executes code remotely, instead of in the local namespace. @@ -258,7 +258,7 @@ def pxrun_cell(self, raw_cell, store_history=True): return False else: try: - result = self.active_view.execute(cell, block=False) + result = self.active_view.execute(cell, silent=False, block=False) except: ipself.showtraceback() return True From e4f5c292888dd0b56903854fe650c3a45c46a037 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 00:06:57 -0700 Subject: [PATCH 53/69] expose kernel at app-level, so EngineApp looks like KernelApp --- IPython/parallel/apps/ipengineapp.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/IPython/parallel/apps/ipengineapp.py b/IPython/parallel/apps/ipengineapp.py index b249a74e0ad..c901635e264 100755 --- a/IPython/parallel/apps/ipengineapp.py +++ b/IPython/parallel/apps/ipengineapp.py @@ -177,6 +177,11 @@ def _cluster_id_changed(self, name, old, new): aliases = Dict(aliases) flags = Dict(flags) + + @property + def kernel(self): + """allow access to the Kernel object, so I look like IPKernelApp""" + return self.engine.kernel def find_url_file(self): """Set the url file. From e9a0b8e20dd5d343f9ba0cc5a2d048679a8ec4fa Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 00:09:43 -0700 Subject: [PATCH 54/69] check for any Application with Kernel in zmq.eventloop rather than specifically IPKernelApp (%pylab now works in Engines) --- IPython/zmq/eventloops.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/IPython/zmq/eventloops.py b/IPython/zmq/eventloops.py index 594c11c36d4..4df3b262bc7 100644 --- a/IPython/zmq/eventloops.py +++ b/IPython/zmq/eventloops.py @@ -20,8 +20,10 @@ import zmq # Local imports. +from IPython.config.application import Application from IPython.utils import io + #------------------------------------------------------------------------------ # Eventloops for integrating the Kernel into different GUIs #------------------------------------------------------------------------------ @@ -206,11 +208,15 @@ def doi(): def enable_gui(gui, kernel=None): """Enable integration with a given GUI""" - if kernel is None: - from .ipkernel import IPKernelApp - kernel = IPKernelApp.instance().kernel if gui not in loop_map: raise ValueError("GUI %r not supported" % gui) + if kernel is None: + if Application.initialized(): + kernel = getattr(Application.instance(), 'kernel', None) + if kernel is None: + raise RuntimeError("You didn't specify a kernel," + " and no IPython Application with a kernel appears to be running." + ) loop = loop_map[gui] if kernel.eventloop is not None and kernel.eventloop is not loop: raise RuntimeError("Cannot activate multiple GUI eventloops") From 7576fb62d879a344ac50d59500400b3f7754db51 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 00:23:49 -0700 Subject: [PATCH 55/69] test %pylab on engines and update %autopx tests, now that %autopx uses silent=False --- IPython/parallel/tests/test_view.py | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index d634a0b0e45..1b40412b5c6 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -448,6 +448,7 @@ def test_magic_autopx_nonblocking(self): ip.magic_autopx() ip.run_cell('\n'.join(('a=5','b=10','c=0'))) ip.run_cell('print (b)') + ip.run_cell('import time; time.sleep(0.1)') ip.run_cell("b/c") ip.run_cell('b*=2') ip.magic_autopx() @@ -457,9 +458,12 @@ def test_magic_autopx_nonblocking(self): self.assertTrue(output.endswith('%autopx disabled')) self.assertFalse('ZeroDivisionError' in output) ar = v.get_result(-2) - self.assertEquals(v['a'], 5) - self.assertEquals(v['b'], 20) self.assertRaisesRemote(ZeroDivisionError, ar.get) + # prevent TaskAborted on pulls, due to ZeroDivisionError + time.sleep(0.5) + self.assertEquals(v['a'], 5) + # b*=2 will not fire, due to abort + self.assertEquals(v['b'], 10) def test_magic_result(self): ip = get_ipython() @@ -654,4 +658,19 @@ def test_execute_raises(self): view = self.client[-1] ar = view.execute("1/0") self.assertRaisesRemote(ZeroDivisionError, ar.get, 2) + + @dec.skipif_not_matplotlib + def test_amagic_pylab(self): + """%pylab works on engines""" + view = self.client[-1] + ar = view.execute("%pylab inline") + # at least check if this raised: + reply = ar.get(5) + # include imports, in case user config + ar = view.execute("plot(rand(100))", silent=False) + reply = ar.get(5) + self.assertEquals(len(reply.outputs), 1) + output = reply.outputs[0] + self.assertTrue("image/png" in output) + From 85e05e18939751c2bc1fb0988d70d6782af84c9e Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 00:34:01 -0700 Subject: [PATCH 56/69] demote enter/exit eventloop messages to INFO --- IPython/zmq/ipkernel.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index f374b418c81..2775d542251 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -240,7 +240,7 @@ def dispatch_shell(self, stream, msg): def enter_eventloop(self): """enter eventloop""" - self.log.critical("entering eventloop") + self.log.info("entering eventloop") # restore default_int_handler signal(SIGINT, default_int_handler) while self.eventloop is not None: @@ -254,7 +254,7 @@ def enter_eventloop(self): # eventloop exited cleanly, this means we should stop (right?) self.eventloop = None break - self.log.critical("exiting eventloop") + self.log.info("exiting eventloop") # if eventloop exits, IOLoop should stop ioloop.IOLoop.instance().stop() From d2f71ea524f1b6583d2b2ad114dfc61ec928ad59 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 10:52:26 -0700 Subject: [PATCH 57/69] move default log setup to _log_default from init_logging allows classes to be passed log attribute, avoiding duplicate setup and occasional errors if things are done in the wrong order. --- IPython/config/application.py | 42 +++++++++---------- .../frontend/html/notebook/clustermanager.py | 2 - IPython/frontend/html/notebook/notebookapp.py | 2 +- IPython/parallel/apps/baseapp.py | 5 ++- IPython/parallel/apps/ipcontrollerapp.py | 2 - IPython/parallel/apps/ipengineapp.py | 2 - 6 files changed, 26 insertions(+), 29 deletions(-) diff --git a/IPython/config/application.py b/IPython/config/application.py index ba26b571ede..59a21e59f00 100644 --- a/IPython/config/application.py +++ b/IPython/config/application.py @@ -132,6 +132,27 @@ def _log_level_changed(self, name, old, new): new = getattr(logging, new) self.log_level = new self.log.setLevel(new) + + log = Instance(logging.Logger) + def _log_default(self): + """Start logging for this application. + + The default is to log to stdout using a StreaHandler. The log level + starts at loggin.WARN, but this can be adjusted by setting the + ``log_level`` attribute. + """ + log = logging.getLogger(self.__class__.__name__) + log.setLevel(self.log_level) + if sys.executable.endswith('pythonw.exe'): + # this should really go to a file, but file-logging is only + # hooked up in parallel applications + _log_handler = logging.StreamHandler(open(os.devnull, 'w')) + else: + _log_handler = logging.StreamHandler() + _log_formatter = logging.Formatter("[%(name)s] %(message)s") + _log_handler.setFormatter(_log_formatter) + log.addHandler(_log_handler) + return log # the alias map for configurables aliases = Dict({'log-level' : 'Application.log_level'}) @@ -169,32 +190,11 @@ def __init__(self, **kwargs): if self.__class__ not in self.classes: self.classes.insert(0, self.__class__) - self.init_logging() - def _config_changed(self, name, old, new): SingletonConfigurable._config_changed(self, name, old, new) self.log.debug('Config changed:') self.log.debug(repr(new)) - def init_logging(self): - """Start logging for this application. - - The default is to log to stdout using a StreaHandler. The log level - starts at loggin.WARN, but this can be adjusted by setting the - ``log_level`` attribute. - """ - self.log = logging.getLogger(self.__class__.__name__) - self.log.setLevel(self.log_level) - if sys.executable.endswith('pythonw.exe'): - # this should really go to a file, but file-logging is only - # hooked up in parallel applications - self._log_handler = logging.StreamHandler(open(os.devnull, 'w')) - else: - self._log_handler = logging.StreamHandler() - self._log_formatter = logging.Formatter("[%(name)s] %(message)s") - self._log_handler.setFormatter(self._log_formatter) - self.log.addHandler(self._log_handler) - @catch_config_error def initialize(self, argv=None): """Do the basic steps to configure me. diff --git a/IPython/frontend/html/notebook/clustermanager.py b/IPython/frontend/html/notebook/clustermanager.py index 27e1e9bb8e0..7e4f428332e 100644 --- a/IPython/frontend/html/notebook/clustermanager.py +++ b/IPython/frontend/html/notebook/clustermanager.py @@ -45,8 +45,6 @@ class DummyIPClusterStart(IPClusterStart): def init_signal(self): pass - def init_logging(self): - pass def reinit_logging(self): pass diff --git a/IPython/frontend/html/notebook/notebookapp.py b/IPython/frontend/html/notebook/notebookapp.py index b157c97ff0d..22dbf2083ef 100644 --- a/IPython/frontend/html/notebook/notebookapp.py +++ b/IPython/frontend/html/notebook/notebookapp.py @@ -390,7 +390,6 @@ def init_configurables(self): self.cluster_manager.update_profiles() def init_logging(self): - super(NotebookApp, self).init_logging() # This prevents double log messages because tornado use a root logger that # self.log is a child of. The logging module dipatches log messages to a log # and all of its ancenstors until propagate is set to False. @@ -493,6 +492,7 @@ def _signal_stop(self, sig, frame): @catch_config_error def initialize(self, argv=None): + self.init_logging() super(NotebookApp, self).initialize(argv) self.init_configurables() self.init_webapp() diff --git a/IPython/parallel/apps/baseapp.py b/IPython/parallel/apps/baseapp.py index 6d6382a70b3..32b6d983f79 100644 --- a/IPython/parallel/apps/baseapp.py +++ b/IPython/parallel/apps/baseapp.py @@ -175,9 +175,12 @@ def reinit_logging(self): else: open_log_file = None if open_log_file is not None: - self.log.removeHandler(self._log_handler) + while self.log.handlers: + self.log.removeHandler(self.log.handlers[0]) self._log_handler = logging.StreamHandler(open_log_file) self.log.addHandler(self._log_handler) + else: + self._log_handler = self.log.handlers[0] # Add timestamps to log format: self._log_formatter = logging.Formatter("%(asctime)s.%(msecs).03d [%(name)s] %(message)s", datefmt="%Y-%m-%d %H:%M:%S") diff --git a/IPython/parallel/apps/ipcontrollerapp.py b/IPython/parallel/apps/ipcontrollerapp.py index ead40a71bc5..da9070a7af2 100755 --- a/IPython/parallel/apps/ipcontrollerapp.py +++ b/IPython/parallel/apps/ipcontrollerapp.py @@ -434,11 +434,9 @@ def forward_logging(self): lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) handler = PUBHandler(lsock) - self.log.removeHandler(self._log_handler) handler.root_topic = 'controller' handler.setLevel(self.log_level) self.log.addHandler(handler) - self._log_handler = handler @catch_config_error def initialize(self, argv=None): diff --git a/IPython/parallel/apps/ipengineapp.py b/IPython/parallel/apps/ipengineapp.py index c901635e264..f26750b412a 100755 --- a/IPython/parallel/apps/ipengineapp.py +++ b/IPython/parallel/apps/ipengineapp.py @@ -288,11 +288,9 @@ def forward_logging(self): context = self.engine.context lsock = context.socket(zmq.PUB) lsock.connect(self.log_url) - self.log.removeHandler(self._log_handler) handler = EnginePUBHandler(self.engine, lsock) handler.setLevel(self.log_level) self.log.addHandler(handler) - self._log_handler = handler def init_mpi(self): global mpi From 9fc6597aa9db758ad6d2587416f826a9a58ef5f8 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 11:12:24 -0700 Subject: [PATCH 58/69] Application.log_format is a configurable --- IPython/config/application.py | 5 ++++- IPython/parallel/apps/baseapp.py | 6 +++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/IPython/config/application.py b/IPython/config/application.py index 59a21e59f00..f9e94ee957e 100644 --- a/IPython/config/application.py +++ b/IPython/config/application.py @@ -133,6 +133,9 @@ def _log_level_changed(self, name, old, new): self.log_level = new self.log.setLevel(new) + log_format = Unicode("[%(name)s] %(message)s", config=True, + help="The Logging format template", + ) log = Instance(logging.Logger) def _log_default(self): """Start logging for this application. @@ -149,7 +152,7 @@ def _log_default(self): _log_handler = logging.StreamHandler(open(os.devnull, 'w')) else: _log_handler = logging.StreamHandler() - _log_formatter = logging.Formatter("[%(name)s] %(message)s") + _log_formatter = logging.Formatter(self.log_format) _log_handler.setFormatter(_log_formatter) log.addHandler(_log_handler) return log diff --git a/IPython/parallel/apps/baseapp.py b/IPython/parallel/apps/baseapp.py index 32b6d983f79..071a06de9d3 100644 --- a/IPython/parallel/apps/baseapp.py +++ b/IPython/parallel/apps/baseapp.py @@ -102,6 +102,10 @@ class BaseParallelApplication(BaseIPythonApplication): def _log_level_default(self): # temporarily override default_log_level to INFO return logging.INFO + + def _log_format_default(self): + """override default log format to include time""" + return u"%(asctime)s.%(msecs).03d [%(name)s] %(message)s" work_dir = Unicode(os.getcwdu(), config=True, help='Set the working dir for the process.' @@ -182,7 +186,7 @@ def reinit_logging(self): else: self._log_handler = self.log.handlers[0] # Add timestamps to log format: - self._log_formatter = logging.Formatter("%(asctime)s.%(msecs).03d [%(name)s] %(message)s", + self._log_formatter = logging.Formatter(self.log_format, datefmt="%Y-%m-%d %H:%M:%S") self._log_handler.setFormatter(self._log_formatter) # do not propagate log messages to root logger From f24fe970c909453376f48616367971e6becead53 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 11:37:52 -0700 Subject: [PATCH 59/69] split init_sockets into a few steps, so they can be better reused --- IPython/zmq/kernelapp.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/IPython/zmq/kernelapp.py b/IPython/zmq/kernelapp.py index 762401f4637..db0fdaaaab5 100644 --- a/IPython/zmq/kernelapp.py +++ b/IPython/zmq/kernelapp.py @@ -218,7 +218,9 @@ def init_sockets(self): self.stdin_socket = context.socket(zmq.ROUTER) self.stdin_port = self._bind_socket(self.stdin_socket, self.stdin_port) self.log.debug("stdin ROUTER Channel on port: %i"%self.stdin_port) - + + def init_heartbeat(self): + """start the heart beating""" # heartbeat doesn't share context, because it mustn't be blocked # by the GIL, which is accessed by libzmq when freeing zero-copy messages hb_ctx = zmq.Context() @@ -230,7 +232,9 @@ def init_sockets(self): # Helper to make it easier to connect to an existing kernel. # set log-level to critical, to make sure it is output self.log.critical("To connect another client to this kernel, use:") - + + def log_connection_info(self): + """display connection info, and store ports""" basename = os.path.basename(self.connection_file) if basename == self.connection_file or \ os.path.dirname(self.connection_file) == self.profile_dir.security_dir: @@ -292,7 +296,9 @@ def initialize(self, argv=None): self.init_session() self.init_poller() self.init_sockets() - # writing connection file must be *after* init_sockets + self.init_heartbeat() + # writing/displaying connection info must be *after* init_sockets/heartbeat + self.log_connection_info() self.write_connection_file() self.init_io() self.init_signal() From 3b1700a1566421c73f468b0c502d0de816dea942 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 12:07:14 -0700 Subject: [PATCH 60/69] reduce parentless IOPub message to warning level --- IPython/parallel/controller/hub.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/IPython/parallel/controller/hub.py b/IPython/parallel/controller/hub.py index 5f558242a1c..892e5e5d9c6 100644 --- a/IPython/parallel/controller/hub.py +++ b/IPython/parallel/controller/hub.py @@ -800,7 +800,7 @@ def save_iopub_message(self, topics, msg): parent = msg['parent_header'] if not parent: - self.log.error("iopub::invalid IOPub message: %r", msg) + self.log.warn("iopub::IOPub message lacks parent: %r", msg) return msg_id = parent['msg_id'] msg_type = msg['header']['msg_type'] From 23d0ff77505469156643e1855d32ccc6827d6cb8 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 12:08:00 -0700 Subject: [PATCH 61/69] add listen_kernel method to IPEngineApp --- IPython/parallel/apps/ipengineapp.py | 41 ++++++++++++++++++++++++++-- 1 file changed, 39 insertions(+), 2 deletions(-) diff --git a/IPython/parallel/apps/ipengineapp.py b/IPython/parallel/apps/ipengineapp.py index f26750b412a..e303e4af45c 100755 --- a/IPython/parallel/apps/ipengineapp.py +++ b/IPython/parallel/apps/ipengineapp.py @@ -37,7 +37,7 @@ catch_config_error, ) from IPython.zmq.log import EnginePUBHandler -from IPython.zmq.ipkernel import Kernel +from IPython.zmq.ipkernel import Kernel, IPKernelApp from IPython.zmq.session import ( Session, session_aliases, session_flags ) @@ -49,7 +49,7 @@ from IPython.utils.importstring import import_item from IPython.utils.py3compat import cast_bytes -from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float +from IPython.utils.traitlets import Bool, Unicode, Dict, List, Float, Instance #----------------------------------------------------------------------------- @@ -174,6 +174,9 @@ def _cluster_id_changed(self, name, old, new): log_url = Unicode('', config=True, help="""The URL for the iploggerapp instance, for forwarding logging to a central location.""") + + # an IPKernelApp instance, used to setup listening for shell frontends + kernel_app = Instance(IPKernelApp) aliases = Dict(aliases) flags = Dict(flags) @@ -226,7 +229,41 @@ def load_connector_file(self): config.EngineFactory.sshserver except AttributeError: config.EngineFactory.sshserver = d['ssh'] + + def listen_kernel(self): + """setup engine as listening Kernel, for frontends""" + if self.kernel_app is not None: + return + + self.log.info("Opening ports for direct connections") + + kernel = self.kernel + + app = self.kernel_app = IPKernelApp(log=self.log, config=self.config, + profile_dir = self.profile_dir, + session=self.engine.session, + ) + app.init_connection_file() + # relevant contents of init_sockets: + + app.shell_port = app._bind_socket(kernel.shell_streams[0], app.shell_port) + app.log.debug("shell ROUTER Channel on port: %i", app.shell_port) + app.iopub_port = app._bind_socket(kernel.iopub_socket, app.iopub_port) + app.log.debug("iopub PUB Channel on port: %i", app.iopub_port) + + kernel.stdin_socket = self.engine.context.socket(zmq.ROUTER) + app.stdin_port = app._bind_socket(kernel.stdin_socket, app.stdin_port) + app.log.debug("stdin ROUTER Channel on port: %i", app.stdin_port) + + # start the heartbeat, and log connection info: + + app.init_heartbeat() + + app.log_connection_info() + app.write_connection_file() + + def init_engine(self): # This is the working dir by now. sys.path.insert(0, '') From 83b4b12482328ee595687b000f4f9b314174af30 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 12:14:33 -0700 Subject: [PATCH 62/69] allow embed_kernel to be called from engine --- IPython/zmq/ipkernel.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 2775d542251..4889706eecd 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -37,7 +37,7 @@ # Local imports from IPython.core import pylabtools from IPython.config.configurable import Configurable -from IPython.config.application import boolean_flag, catch_config_error +from IPython.config.application import boolean_flag, catch_config_error, Application from IPython.core.application import ProfileDir from IPython.core.error import StdinNotImplementedError from IPython.core.shellapp import ( @@ -883,9 +883,21 @@ def embed_kernel(module=None, local_ns=None, **kwargs): on the first embed_kernel call for a given process. """ + try: + from IPython.parallel.apps.ipengineapp import IPEngineApp + except ImportError: + # IPython.parallel has higher zmq dependency that IPython.zmq + class IPEngineApp: + pass + # get the app if it exists, or set it up if it doesn't - if IPKernelApp.initialized(): - app = IPKernelApp.instance() + if Application.initialized(): + app = Application.instance() + if isinstance(app, IPEngineApp): + app.listen_kernel() + # return right away, because we embed in the Engine's + # namespace, not the calling one. + return else: app = IPKernelApp.instance(**kwargs) app.initialize([]) From 3fab9949a98ab8d4cec69885dc8e8753e74566d7 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 12:36:13 -0700 Subject: [PATCH 63/69] kernel app cleans up connection files that it wrote --- IPython/zmq/kernelapp.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/IPython/zmq/kernelapp.py b/IPython/zmq/kernelapp.py index db0fdaaaab5..feec06cda73 100644 --- a/IPython/zmq/kernelapp.py +++ b/IPython/zmq/kernelapp.py @@ -16,6 +16,7 @@ #----------------------------------------------------------------------------- # Standard library imports +import atexit import json import os import sys @@ -95,6 +96,7 @@ class KernelApp(BaseIPythonApplication): heartbeat = Instance(Heartbeat) session = Instance('IPython.zmq.session.Session') ports = Dict() + _full_connection_file = Unicode() # inherit config file name from parent: parent_appname = Unicode(config=True) @@ -165,6 +167,8 @@ def load_connection_file(self): fname = filefind(self.connection_file, ['.', self.profile_dir.security_dir]) except IOError: self.log.debug("Connection file not found: %s", self.connection_file) + # This means I own it, so I will clean it up: + atexit.register(self.cleanup_connection_file) return self.log.debug(u"Loading connection file %s", fname) with open(fname) as f: @@ -190,6 +194,16 @@ def write_connection_file(self): write_connection_file(cf, ip=self.ip, key=self.session.key, shell_port=self.shell_port, stdin_port=self.stdin_port, hb_port=self.hb_port, iopub_port=self.iopub_port) + + self._full_connection_file = cf + + def cleanup_connection_file(self): + cf = self._full_connection_file + self.log.debug("cleaning up connection file: %r", cf) + try: + os.remove(cf) + except (IOError, OSError): + pass def init_connection_file(self): if not self.connection_file: From 100a5e3ab25db851bde6b3f9c66e8f663d48ce6b Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 13:20:05 -0700 Subject: [PATCH 64/69] relax some timing constraints in parallel tests --- IPython/parallel/tests/__init__.py | 4 ++-- IPython/parallel/tests/test_asyncresult.py | 10 ++++----- IPython/parallel/tests/test_view.py | 25 ++++++++++++++++------ 3 files changed, 26 insertions(+), 13 deletions(-) diff --git a/IPython/parallel/tests/__init__.py b/IPython/parallel/tests/__init__.py index 70d88c61923..6e03b8c3564 100644 --- a/IPython/parallel/tests/__init__.py +++ b/IPython/parallel/tests/__init__.py @@ -65,7 +65,7 @@ def setup(): if cp.poll() is not None: print cp.poll() raise RuntimeError("The test controller failed to start.") - elif time.time()-tic > 10: + elif time.time()-tic > 15: raise RuntimeError("Timeout waiting for the test controller to start.") time.sleep(0.1) add_engines(1) @@ -93,7 +93,7 @@ def add_engines(n=1, profile='iptest', total=False): while len(rc) < base+n: if any([ ep.poll() is not None for ep in eps ]): raise RuntimeError("A test engine failed to start.") - elif time.time()-tic > 10: + elif time.time()-tic > 15: raise RuntimeError("Timeout waiting for engines to connect.") time.sleep(.1) rc.spin() diff --git a/IPython/parallel/tests/test_asyncresult.py b/IPython/parallel/tests/test_asyncresult.py index dfc3d368fe9..2a831e1e39c 100644 --- a/IPython/parallel/tests/test_asyncresult.py +++ b/IPython/parallel/tests/test_asyncresult.py @@ -154,7 +154,7 @@ def test_serial_time_single(self): ar = v.apply_async(time.sleep, 0.25) self.assertRaises(TimeoutError, getattr, ar, 'serial_time') ar.get(2) - self.assertTrue(ar.serial_time < 0.5) + self.assertTrue(ar.serial_time < 1.) self.assertTrue(ar.serial_time > 0.2) def test_serial_time_multi(self): @@ -171,8 +171,8 @@ def test_elapsed_single(self): ar = v.apply_async(time.sleep, 0.25) while not ar.ready(): time.sleep(0.01) - self.assertTrue(ar.elapsed < 0.3) - self.assertTrue(ar.elapsed < 0.3) + self.assertTrue(ar.elapsed < 1) + self.assertTrue(ar.elapsed < 1) ar.get(2) def test_elapsed_multi(self): @@ -180,8 +180,8 @@ def test_elapsed_multi(self): ar = v.apply_async(time.sleep, 0.25) while not ar.ready(): time.sleep(0.01) - self.assertTrue(ar.elapsed < 0.3) - self.assertTrue(ar.elapsed < 0.3) + self.assertTrue(ar.elapsed < 1) + self.assertTrue(ar.elapsed < 1) ar.get(2) def test_hubresult_timestamps(self): diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index 1b40412b5c6..965da8c60d0 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -575,13 +575,23 @@ def test_len(self): # begin execute tests + def _wait_for(self, f, timeout=10): + tic = time.time() + while time.time() <= tic + timeout: + if f(): + return + time.sleep(0.1) + self.client.spin() + if not f(): + print "Warning: Awaited condition never arrived" + def test_execute_reply(self): e0 = self.client[self.client.ids[0]] e0.block = True ar = e0.execute("5", silent=False) er = ar.get() - time.sleep(0.2) + self._wait_for(lambda : bool(er.pyout)) self.assertEquals(str(er), "" % er.execution_count) self.assertEquals(er.pyout['text/plain'], '5') @@ -590,14 +600,15 @@ def test_execute_reply_stdout(self): e0.block = True ar = e0.execute("print (5)", silent=False) er = ar.get() - time.sleep(0.2) + self._wait_for(lambda : bool(er.stdout)) self.assertEquals(er.stdout.strip(), '5') def test_execute_pyout(self): """execute triggers pyout with silent=False""" view = self.client[:] ar = view.execute("5", silent=False, block=True) - time.sleep(0.2) + self._wait_for(lambda : all(ar.pyout)) + expected = [{'text/plain' : '5'}] * len(view) self.assertEquals(ar.pyout, expected) @@ -615,7 +626,7 @@ def test_execute_magic(self): ar = view.execute("%whos", block=True) # this will raise, if that failed ar.get(5) - time.sleep(0.2) + self._wait_for(lambda : all(ar.stdout)) for stdout in ar.stdout: lines = stdout.splitlines() self.assertEquals(lines[0].split(), ['Variable', 'Type', 'Data/Info']) @@ -632,7 +643,8 @@ def test_execute_displaypub(self): view = self.client[:] view.execute("from IPython.core.display import *") ar = view.execute("[ display(i) for i in range(5) ]", block=True) - time.sleep(0.2) + + self._wait_for(lambda : all(len(er.outputs) >= 5 for er in ar)) outs = [ {u'text/plain' : unicode(i)} for i in range(5) ] expected = [outs] * len(view) self.assertEquals(ar.outputs, expected) @@ -648,7 +660,7 @@ def publish(): ar = view.apply_async(publish) ar.get(5) - time.sleep(0.2) + self._wait_for(lambda : all(len(out) >= 5 for out in ar.outputs)) outs = [ {u'text/plain' : unicode(j)} for j in range(5) ] expected = [outs] * len(view) self.assertEquals(ar.outputs, expected) @@ -669,6 +681,7 @@ def test_amagic_pylab(self): # include imports, in case user config ar = view.execute("plot(rand(100))", silent=False) reply = ar.get(5) + self._wait_for(lambda : all(ar.outputs)) self.assertEquals(len(reply.outputs), 1) output = reply.outputs[0] self.assertTrue("image/png" in output) From 46db02de8172b63a623a8822da896285133d2083 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 17:06:01 -0700 Subject: [PATCH 65/69] store whole content, instead of just display data in metadata.outputs --- IPython/parallel/client/client.py | 34 ++++++++++++++--------------- IPython/parallel/tests/test_view.py | 25 ++++++++++++--------- 2 files changed, 32 insertions(+), 27 deletions(-) diff --git a/IPython/parallel/client/client.py b/IPython/parallel/client/client.py index 466de9114a9..7770087874d 100644 --- a/IPython/parallel/client/client.py +++ b/IPython/parallel/client/client.py @@ -91,39 +91,39 @@ def __getattr__(self, key): def __repr__(self): pyout = self.metadata['pyout'] or {} - text_out = pyout.get('text/plain', '') + text_out = pyout.get('data', {}).get('text/plain', '') if len(text_out) > 32: text_out = text_out[:29] + '...' return "" % (self.execution_count, text_out) def _repr_html_(self): - pyout = self.metadata['pyout'] or {} - return pyout.get("text/html") + pyout = self.metadata['pyout'] or {'data':{}} + return pyout['data'].get("text/html") def _repr_latex_(self): - pyout = self.metadata['pyout'] or {} - return pyout.get("text/latex") + pyout = self.metadata['pyout'] or {'data':{}} + return pyout['data'].get("text/latex") def _repr_json_(self): - pyout = self.metadata['pyout'] or {} - return pyout.get("application/json") + pyout = self.metadata['pyout'] or {'data':{}} + return pyout['data'].get("application/json") def _repr_javascript_(self): - pyout = self.metadata['pyout'] or {} - return pyout.get("application/javascript") + pyout = self.metadata['pyout'] or {'data':{}} + return pyout['data'].get("application/javascript") def _repr_png_(self): - pyout = self.metadata['pyout'] or {} - return pyout.get("image/png") + pyout = self.metadata['pyout'] or {'data':{}} + return pyout['data'].get("image/png") def _repr_jpeg_(self): - pyout = self.metadata['pyout'] or {} - return pyout.get("image/jpeg") + pyout = self.metadata['pyout'] or {'data':{}} + return pyout['data'].get("image/jpeg") def _repr_svg_(self): - pyout = self.metadata['pyout'] or {} - return pyout.get("image/svg+xml") + pyout = self.metadata['pyout'] or {'data':{}} + return pyout['data'].get("image/svg+xml") class Metadata(dict): @@ -834,9 +834,9 @@ def _flush_iopub(self, sock): elif msg_type == 'pyin': md.update({'pyin' : content['code']}) elif msg_type == 'display_data': - md['outputs'].append(content.get('data')) + md['outputs'].append(content) elif msg_type == 'pyout': - md['pyout'] = content.get('data') + md['pyout'] = content else: # unhandled msg_type (status, etc.) pass diff --git a/IPython/parallel/tests/test_view.py b/IPython/parallel/tests/test_view.py index 965da8c60d0..6c1a7a0e816 100644 --- a/IPython/parallel/tests/test_view.py +++ b/IPython/parallel/tests/test_view.py @@ -593,7 +593,7 @@ def test_execute_reply(self): er = ar.get() self._wait_for(lambda : bool(er.pyout)) self.assertEquals(str(er), "" % er.execution_count) - self.assertEquals(er.pyout['text/plain'], '5') + self.assertEquals(er.pyout['data']['text/plain'], '5') def test_execute_reply_stdout(self): e0 = self.client[self.client.ids[0]] @@ -610,7 +610,8 @@ def test_execute_pyout(self): self._wait_for(lambda : all(ar.pyout)) expected = [{'text/plain' : '5'}] * len(view) - self.assertEquals(ar.pyout, expected) + mimes = [ out['data'] for out in ar.pyout ] + self.assertEquals(mimes, expected) def test_execute_silent(self): """execute does not trigger pyout with silent=True""" @@ -645,9 +646,10 @@ def test_execute_displaypub(self): ar = view.execute("[ display(i) for i in range(5) ]", block=True) self._wait_for(lambda : all(len(er.outputs) >= 5 for er in ar)) - outs = [ {u'text/plain' : unicode(i)} for i in range(5) ] - expected = [outs] * len(view) - self.assertEquals(ar.outputs, expected) + expected = [ {u'text/plain' : unicode(j)} for j in range(5) ] + for outputs in ar.outputs: + mimes = [ out['data'] for out in outputs ] + self.assertEquals(mimes, expected) def test_apply_displaypub(self): """apply tracks display_pub output""" @@ -661,9 +663,10 @@ def publish(): ar = view.apply_async(publish) ar.get(5) self._wait_for(lambda : all(len(out) >= 5 for out in ar.outputs)) - outs = [ {u'text/plain' : unicode(j)} for j in range(5) ] - expected = [outs] * len(view) - self.assertEquals(ar.outputs, expected) + expected = [ {u'text/plain' : unicode(j)} for j in range(5) ] + for outputs in ar.outputs: + mimes = [ out['data'] for out in outputs ] + self.assertEquals(mimes, expected) def test_execute_raises(self): """exceptions in execute requests raise appropriately""" @@ -672,7 +675,7 @@ def test_execute_raises(self): self.assertRaisesRemote(ZeroDivisionError, ar.get, 2) @dec.skipif_not_matplotlib - def test_amagic_pylab(self): + def test_magic_pylab(self): """%pylab works on engines""" view = self.client[-1] ar = view.execute("%pylab inline") @@ -684,6 +687,8 @@ def test_amagic_pylab(self): self._wait_for(lambda : all(ar.outputs)) self.assertEquals(len(reply.outputs), 1) output = reply.outputs[0] - self.assertTrue("image/png" in output) + self.assertTrue("data" in output) + data = output['data'] + self.assertTrue("image/png" in data) From e0b0d5e676053c4cac07cd27d01333649733f9fa Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 21:33:26 -0700 Subject: [PATCH 66/69] revert embed_kernel changes that implied bind_kernel in engine --- IPython/zmq/ipkernel.py | 18 +++--------------- 1 file changed, 3 insertions(+), 15 deletions(-) diff --git a/IPython/zmq/ipkernel.py b/IPython/zmq/ipkernel.py index 4889706eecd..2775d542251 100755 --- a/IPython/zmq/ipkernel.py +++ b/IPython/zmq/ipkernel.py @@ -37,7 +37,7 @@ # Local imports from IPython.core import pylabtools from IPython.config.configurable import Configurable -from IPython.config.application import boolean_flag, catch_config_error, Application +from IPython.config.application import boolean_flag, catch_config_error from IPython.core.application import ProfileDir from IPython.core.error import StdinNotImplementedError from IPython.core.shellapp import ( @@ -883,21 +883,9 @@ def embed_kernel(module=None, local_ns=None, **kwargs): on the first embed_kernel call for a given process. """ - try: - from IPython.parallel.apps.ipengineapp import IPEngineApp - except ImportError: - # IPython.parallel has higher zmq dependency that IPython.zmq - class IPEngineApp: - pass - # get the app if it exists, or set it up if it doesn't - if Application.initialized(): - app = Application.instance() - if isinstance(app, IPEngineApp): - app.listen_kernel() - # return right away, because we embed in the Engine's - # namespace, not the calling one. - return + if IPKernelApp.initialized(): + app = IPKernelApp.instance() else: app = IPKernelApp.instance(**kwargs) app.initialize([]) From 64f2f8d0388a742c51ceca4ec2cf026e3bea6213 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 22:05:01 -0700 Subject: [PATCH 67/69] add IPython.parallel.bind_kernel for using an Engine's Kernel with connecting frontends. --- IPython/lib/kernel.py | 6 +++--- IPython/parallel/__init__.py | 21 +++++++++++++++++++++ IPython/parallel/apps/ipengineapp.py | 20 +++++++++++++------- 3 files changed, 37 insertions(+), 10 deletions(-) diff --git a/IPython/lib/kernel.py b/IPython/lib/kernel.py index 345ecee1244..b7144c91a12 100644 --- a/IPython/lib/kernel.py +++ b/IPython/lib/kernel.py @@ -46,11 +46,11 @@ def get_connection_file(app=None): If unspecified, the currently running app will be used """ if app is None: - from IPython.zmq.kernelapp import KernelApp - if not KernelApp.initialized(): + from IPython.zmq.ipkernel import IPKernelApp + if not IPKernelApp.initialized(): raise RuntimeError("app not specified, and not in a running Kernel") - app = KernelApp.instance() + app = IPKernelApp.instance() return filefind(app.connection_file, ['.', app.profile_dir.security_dir]) def find_connection_file(filename, profile=None): diff --git a/IPython/parallel/__init__.py b/IPython/parallel/__init__.py index c41aac246b6..d265596894d 100644 --- a/IPython/parallel/__init__.py +++ b/IPython/parallel/__init__.py @@ -38,4 +38,25 @@ from .util import interactive from .controller.dependency import * +#----------------------------------------------------------------------------- +# Functions +#----------------------------------------------------------------------------- + + +def bind_kernel(**kwargs): + """Bind an Engine's Kernel to be used as a full IPython kernel. + + This allows a running Engine to be used simultaneously as a full IPython kernel + with the QtConsole or other frontends. + + This function returns immediately. + """ + from IPython.parallel.apps.ipengineapp import IPEngineApp + if IPEngineApp.initialized(): + app = IPEngineApp.instance() + else: + raise RuntimeError("Must be called from an IPEngineApp instance") + + return app.bind_kernel(**kwargs) + diff --git a/IPython/parallel/apps/ipengineapp.py b/IPython/parallel/apps/ipengineapp.py index e303e4af45c..d24f574ca1a 100755 --- a/IPython/parallel/apps/ipengineapp.py +++ b/IPython/parallel/apps/ipengineapp.py @@ -230,19 +230,25 @@ def load_connector_file(self): except AttributeError: config.EngineFactory.sshserver = d['ssh'] - def listen_kernel(self): - """setup engine as listening Kernel, for frontends""" + def bind_kernel(self, **kwargs): + """Promote engine to listening kernel, accessible to frontends.""" if self.kernel_app is not None: return - self.log.info("Opening ports for direct connections") + self.log.info("Opening ports for direct connections as an IPython kernel") kernel = self.kernel - app = self.kernel_app = IPKernelApp(log=self.log, config=self.config, - profile_dir = self.profile_dir, - session=self.engine.session, - ) + kwargs.setdefault('config', self.config) + kwargs.setdefault('log', self.log) + kwargs.setdefault('profile_dir', self.profile_dir) + kwargs.setdefault('session', self.engine.session) + + app = self.kernel_app = IPKernelApp(**kwargs) + + # allow IPKernelApp.instance(): + IPKernelApp._instance = app + app.init_connection_file() # relevant contents of init_sockets: From 45a4d04b7debe830518ea8ed23bc7255576211e5 Mon Sep 17 00:00:00 2001 From: MinRK Date: Tue, 22 May 2012 22:28:24 -0700 Subject: [PATCH 68/69] =?UTF-8?q?use=20[sys.exe,=20"-c",=20"=E2=80=A6launc?= =?UTF-8?q?h=5Fnew=5Finstance()"]=20in=20launchers?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit instead of [sys.exe, module_path], which resulted in the modules being `__main__`, which caused problems. --- IPython/parallel/apps/launcher.py | 18 +++++++----------- 1 file changed, 7 insertions(+), 11 deletions(-) diff --git a/IPython/parallel/apps/launcher.py b/IPython/parallel/apps/launcher.py index 3a5a12e90ae..0d8d2eb1f00 100644 --- a/IPython/parallel/apps/launcher.py +++ b/IPython/parallel/apps/launcher.py @@ -24,6 +24,7 @@ import os import re import stat +import sys import time # signal imports, handling various platforms, versions @@ -60,8 +61,8 @@ def check_output(*args, **kwargs): from IPython.utils.traitlets import ( Any, Integer, CFloat, List, Unicode, Dict, Instance, HasTraits, ) -from IPython.utils.path import get_ipython_module_path, get_home_dir -from IPython.utils.process import find_cmd, pycmd2argv, FindCmdError +from IPython.utils.path import get_home_dir +from IPython.utils.process import find_cmd, FindCmdError from .win32support import forward_read_events @@ -73,18 +74,13 @@ def check_output(*args, **kwargs): # Paths to the kernel apps #----------------------------------------------------------------------------- +cmd = "from IPython.parallel.apps.%s import launch_new_instance; launch_new_instance()" -ipcluster_cmd_argv = pycmd2argv(get_ipython_module_path( - 'IPython.parallel.apps.ipclusterapp' -)) +ipcluster_cmd_argv = [sys.executable, "-c", cmd % "ipclusterapp"] -ipengine_cmd_argv = pycmd2argv(get_ipython_module_path( - 'IPython.parallel.apps.ipengineapp' -)) +ipengine_cmd_argv = [sys.executable, "-c", cmd % "ipengineapp"] -ipcontroller_cmd_argv = pycmd2argv(get_ipython_module_path( - 'IPython.parallel.apps.ipcontrollerapp' -)) +ipcontroller_cmd_argv = [sys.executable, "-c", cmd % "ipcontrollerapp"] #----------------------------------------------------------------------------- # Base launchers and errors From 87af7410415b1b9f6f088a6e4b7e480e862f84e4 Mon Sep 17 00:00:00 2001 From: MinRK Date: Fri, 25 May 2012 16:28:30 -0700 Subject: [PATCH 69/69] switch order of failed-to-start errors in test_embed_kernel also increase timeout for kernel start to 10 seconds --- IPython/zmq/tests/test_embed_kernel.py | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/IPython/zmq/tests/test_embed_kernel.py b/IPython/zmq/tests/test_embed_kernel.py index 6d426712fe5..3453303c452 100644 --- a/IPython/zmq/tests/test_embed_kernel.py +++ b/IPython/zmq/tests/test_embed_kernel.py @@ -23,7 +23,7 @@ import nose.tools as nt from IPython.zmq.blockingkernelmanager import BlockingKernelManager -from IPython.utils import path +from IPython.utils import path, py3compat #------------------------------------------------------------------------------- @@ -68,17 +68,19 @@ def setup_kernel(cmd): ) # wait for connection file to exist, timeout after 5s tic = time.time() - while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + 5: + while not os.path.exists(connection_file) and kernel.poll() is None and time.time() < tic + 10: time.sleep(0.1) + if kernel.poll() is not None: + o,e = kernel.communicate() + e = py3compat.cast_unicode(e) + raise IOError("Kernel failed to start:\n%s" % e) + if not os.path.exists(connection_file): if kernel.poll() is None: kernel.terminate() raise IOError("Connection file %r never arrived" % connection_file) - if kernel.poll() is not None: - raise IOError("Kernel failed to start") - km = BlockingKernelManager(connection_file=connection_file) km.load_connection_file() km.start_channels()