gpoore · obtitus · Dec 10, 2011 · Jul 18, 2012 · Jul 18, 2012 · Jul 19, 2012
diff --git a/.gitignore b/.gitignore
@@ -0,0 +1,17 @@
+*~
+pythontex-files-pythontex_gallery/
+*.aux
+*.log
+*.pytxcode
+.DS_Store
+*.glo
+
+*.idx
+
+*.out
+
+*.toc
+
+*.fdb_latexmk
+
+*.fls
diff --git a/pythontex/async_pylab_save.py b/pythontex/async_pylab_save.py
@@ -0,0 +1,69 @@
+# -*- coding: utf-8 -*-
+import time
+import multiprocessing as mp
+import numpy as np
+import matplotlib
+#matplotlib.use('Agg')
+import matplotlib.pyplot as plt
+"""
+Asynchronous Plotting in Matplotlib: rather than call savefig directly, add plots to an asynchronous queue to avoid holding up the main program. Makes use of multiple processes to speed up the writing out.
+original author: astrofrog, https://gist.github.com/1453933
+minor modifications by: ob@cakebox.net
+"""
+
+class AsyncPylabSave():
+    def __init__(self, processes=mp.cpu_count()):
+        self.manager = mp.Manager()
+        self.nc = self.manager.Value('i', 0)
+        self.pids = []
+        self.processes = processes
+
+    def async_plotter(self, nc, fig, filename, processes, **kwargs):
+        while nc.value >= processes:
+            time.sleep(0.1)
+        nc.value += 1
+        fig.savefig(filename, **kwargs)
+        plt.close(fig)
+        nc.value -= 1
+
+    def savefig(self, filename, fig=None, **kwargs):
+        # Calls fig.savefig(filename) asynchronously, if fig is None (default) the current figure is saved.
+        # kwargs are sent directly to savefig.
+        if fig == None:
+            fig = plt.gcf()
+        p = mp.Process(target=self.async_plotter,
+                       args=(self.nc, fig, filename, self.processes),
+                       kwargs=kwargs)
+        p.start()
+        self.pids.append(p)
+
+    def join(self):
+        for p in self.pids:
+            p.join()
+
+"""
+Example usage:
+# Create instance of Asynchronous plotter
+a = AsyncPylabSave()
+
+for i in range(10):
+
+    print 'Preparing %04i.png' % i
+
+    # Generate random points
+    x = np.random.random(10000)
+    y = np.random.random(10000)
+
+    # Generate figure
+    fig = plt.figure()
+    ax = fig.add_subplot(1, 1, 1)
+    ax.set_xscale('log')
+    ax.set_yscale('log')
+    ax.scatter(x, y)
+
+    # Add figure to queue
+    a.savefig('%04i.png' % i, fig=fig, facecolor='r')
+
+# Wait for all plots to finish
+a.join()
+"""
diff --git a/pythontex/pythontex.dtx b/pythontex/pythontex.dtx
@@ -164,7 +164,7 @@
 % \item Installer file |pythontex.ins|
 % \item Documented \LaTeX\ source file |pythontex.dtx|, from which |pythontex.pdf| and |pythontex.sty| are generated
 % \item Main Python scripts |pythontex2.py| and |pythontex3.py|
-% \item Helper scripts |pythontex_utils2.py| and |pythontex_types2.py|, and |pythontex_utils3.py| and |pythontex_types3.py|
+% \item Helper scripts |pythontex_utils2.py|, |pythontex_types2.py|, |pythontex_utils3.py|, |pythontex_types3.py| and |async_pylab_save.py|
 % \item Installation script |pythontex_install_texlive| (for \TeX\ Live)
 % \item README
 % \item Optional batch file |pythontex.bat| for use in launching |pythontex*.py| under Windows 
@@ -183,6 +183,7 @@
 %     \item |pythontex2.py| and |pythontex3.py|
 %     \item |pythontex_types2.py| and |pythontex_types3.py|
 %     \item |pythontex_utils2.py| and |pythontex_utils3.py|
+%     \item |async_pylab_save.py|
 %     \end{itemize}
 % \item \meta{\TeX\ tree root}|/source/latex/pythontex/|
 %     \begin{itemize}
@@ -417,6 +418,20 @@
 % \meta{quoted~list} may \textbf{not} contain \LaTeX\ macros.  \meta{quoted~list} is interpreted as verbatim content, since in general the custom code will not be valid \LaTeX.
 %
 %
+% \subsubsection{Asynchronous saving of plots}
+% To avoid locking up the main program flow when saving figures the utility |async_pylab_save.py| is included.
+%
+% This works by making |asp = AsyncPylabSave()| available in every family and then calling |asp.join()| at the end. Calling 
+% |asp.savefig(...)|
+% will call |plt.savefig(...)|
+% without blocking the main program (using multiprocessing).
+% Or eqvivalently 
+% |asp.savefig(filename, fig=fig, **kwargs)|, where fig is a figure handle,
+% will work as |fig.savefig(filename, **kwargs)|.
+%
+% Please note that the figure will be closed after saving, so manipulation of the figure after calling save will not work. Saving multiple figures to the same filename will of course have unexpected results!
+% 
+% TODO: I guess this feature only makes sense in the pylab family\dots
 % \subsubsection{Formatting of typeset code}
 %
 % \DescribeMacro{\setpythontexfv\oarg{family}\marg{fancyvrb~settings}}

diff --git a/pythontex/pythontex.pdf b/pythontex/pythontex.pdf
diff --git a/pythontex/pythontex2.py b/pythontex/pythontex2.py
@@ -963,6 +963,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict):
     # Add in a Pygments process if applicable
     for key in update_pygments:
         if update_pygments[key] and not key.endswith('cons'):
+            print("* Pythontex pygment processing: " + str(key))            
             tasks.append(pool.apply_async(do_pygments, [outputdir,
                                                         jobname,
                                                         fvextfile,
@@ -975,6 +976,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict):
     # Add console processes
     for key in consoledict:
         if update_code[key] or update_pygments[key]:
+            print("* Pythontex console processing: " + str(key))            
             tasks.append(pool.apply_async(run_console, [outputdir,
                                                         jobname,
                                                         fvextfile,
@@ -990,6 +992,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict):
     # Add code processes.  Note that everything placed in the codedict 
     # needs to be executed, based on previous testing.
     for key in codedict:
+        print("* Pythontex code processing: " + str(key))
         [inputtype, inputsession, inputgroup] = key.split('#')
         tasks.append(pool.apply_async(run_code, [inputtype,
                                                  inputsession,
@@ -1141,7 +1144,7 @@ def run_code(inputtype, inputsession, inputgroup, outputdir, command,
     # Only work with files that have a nonzero size 
     if os.path.isfile(err_file_name) and os.stat(err_file_name).st_size != 0:
         # Reset the hash value, so that the code will be run next time
-        exit_status[currentkey] = ''
+        #exit_status[currentkey] = ''
         # Open error and code files.
         # We can't just use the code in memory, because the full script 
         # file was written but never fully assembled in memory.
@@ -1749,3 +1752,4 @@ def save_data(data):
     print('\n--------------------------------------------------')
     print('PythonTeX:  ' + data['raw_jobname'] + ' - ' + str(temp_data['errors']) + ' error(s), ' + str(temp_data['warnings']) + ' warning(s)')
 
+    exit(temp_data['errors'])
diff --git a/pythontex/pythontex3.py b/pythontex/pythontex3.py
@@ -963,6 +963,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict):
     # Add in a Pygments process if applicable
     for key in update_pygments:
         if update_pygments[key] and not key.endswith('cons'):
+            print("* Pythontex pygment processing: " + str(key))            
             tasks.append(pool.apply_async(do_pygments, [outputdir,
                                                         jobname,
                                                         fvextfile,
@@ -975,6 +976,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict):
     # Add console processes
     for key in consoledict:
         if update_code[key] or update_pygments[key]:
+            print("* Pythontex console processing: " + str(key))            
             tasks.append(pool.apply_async(run_console, [outputdir,
                                                         jobname,
                                                         fvextfile,
@@ -990,6 +992,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict):
     # Add code processes.  Note that everything placed in the codedict 
     # needs to be executed, based on previous testing.
     for key in codedict:
+        print("* Pythontex code processing: " + str(key))
         [inputtype, inputsession, inputgroup] = key.split('#')
         tasks.append(pool.apply_async(run_code, [inputtype,
                                                  inputsession,
@@ -1141,7 +1144,7 @@ def run_code(inputtype, inputsession, inputgroup, outputdir, command,
     # Only work with files that have a nonzero size 
     if os.path.isfile(err_file_name) and os.stat(err_file_name).st_size != 0:
         # Reset the hash value, so that the code will be run next time
-        exit_status[currentkey] = ''
+        #exit_status[currentkey] = ''
         # Open error and code files.
         # We can't just use the code in memory, because the full script 
         # file was written but never fully assembled in memory.
@@ -1749,3 +1752,4 @@ def save_data(data):
     print('\n--------------------------------------------------')
     print('PythonTeX:  ' + data['raw_jobname'] + ' - ' + str(temp_data['errors']) + ' error(s), ' + str(temp_data['warnings']) + ' warning(s)')
 
+    exit(temp_data['errors'])
diff --git a/pythontex/pythontex_install_texlive.py b/pythontex/pythontex_install_texlive.py
@@ -33,7 +33,7 @@
 # Imports
 import sys
 import platform
-from os import path, mkdir
+from os import path, mkdir, symlink, chmod
 from subprocess import check_call, check_output, CalledProcessError
 from shutil import copy
 
@@ -49,7 +49,7 @@
 needed_files = ['pythontex2.py', 'pythontex_types2.py', 'pythontex_utils2.py', 
                 'pythontex3.py', 'pythontex_types3.py', 'pythontex_utils3.py', 
                 'pythontex.sty', 'pythontex.ins', 'pythontex.dtx', 
-                'pythontex.pdf', 'README.rst']
+                'pythontex.pdf', 'README.rst', 'async_pylab_save.py']
 missing_files = False
 # Print a list of all files that are missing, and exit if any are
 for eachfile in needed_files:
@@ -60,11 +60,6 @@
     print('Exiting.')
     sys.exit(1)
 
-
-# Print starting message
-print('\nInstalling PythonTeX...')
-
-
 # Retrieve the location of a valid TeX tree
 # Attempt to use kpsewhich; otherwise, resort to manual input 
 try:
@@ -80,6 +75,10 @@
     print('\nkpsewhich is not happy with its arguments.')
     print('Cannot automatically find a valid texmf path.')
     texmf_path = input('Please enter a valid texmf path: ').rstrip('\r\n')
+
+# Print starting message
+print('\nInstalling PythonTeX into directory ' + texmf_path)
+
 # Make sure path slashes are compatible with the operating system 
 # This is only needed for Windows
 texmf_path = path.normcase(texmf_path)
@@ -127,6 +126,7 @@
 copy('pythontex3.py', scripts_path)
 copy('pythontex_types3.py', scripts_path)
 copy('pythontex_utils3.py', scripts_path)
+copy('async_pylab_save.py', scripts_path)
 # Install source
 if not path.exists(source_path):
     mkdir(source_path)
@@ -154,7 +154,27 @@
         print('The bin/ directory in your TeX distribution may be a good location.')
         print('The script pythontex.py is located in the following directory:')
         print('    ' + scripts_path)
-# If not under Windows, we alert the user regarding what is necessary to launch
+elif platform.system() in ['Linux', 'Darwin']: # todo: check for unix (maybe just check to see if os.symlink fails or not?)
+    root_path = path.split(texmf_path)[0]
+    bin_path = path.join(path.split(root_path)[0], 'bin')
+    if path.exists(bin_path):
+        for ver in [2, 3]:
+            link = path.join(bin_path, 'pythontex{0}.py'.format(ver))
+            try:
+                symlink(path.join(scripts_path, 'pythontex{0}.py'.format(ver)), link)
+            except OSError as e:
+                if e.errno == 17:
+                    pass # File exists
+                else:
+                    raise OSError(e)
+            chmod(link, 0775)
+            print('symlink created ' + link)
+    else:
+        print('\nCreating symlink failed, you may wish to create a symlink to pythontex.py.')
+        print('You may also want to make it executable via chmod.')
+        print('The script pythontex.py is located in the following directory:')
+        print('    ' + scripts_path)
+# If not under known system, we alert the user regarding what is necessary to launch
 # pythontex.py
 else:
     print('\nYou may wish to create a symlink to pythontex.py.')
@@ -179,5 +199,6 @@
 print('See the documentation for more information.')
 print('* * *\n')
 
-# Pause so that the user can see any errors or other messages
-input('\n[Press ENTER to exit]')
+if platform.system() == 'Windows':
+    # Pause so that the user can see any errors or other messages
+    input('\n[Press ENTER to exit]')
diff --git a/pythontex/pythontex_types2.py b/pythontex/pythontex_types2.py
@@ -138,6 +138,11 @@ def __init__(self, language, extension, command, default_code, utils_code,
     #        pytex = PythontexUtils()
     #        """
     #\\ End Python 3
+    # Add AsyncPylabSave import
+    utils_string_dict['python'] += """
+            from async_pylab_save import AsyncPylabSave
+            aps = AsyncPylabSave()
+            """
 
     inputs_string_const_dict['python'] = """
 
@@ -191,7 +196,10 @@ def __init__(self, language, extension, command, default_code, utils_code,
     #        """
     #\\ End Python 3
 
-    close_macrofile_string_dict['python'] = 'pytex.macrofile.close()\n'
+    close_macrofile_string_dict['python'] = """
+            pytex.macrofile.close()
+            aps.join()
+            """
 
     set_workingdir_string_dict['python'] = """
             if os.path.exists('{0}'):

diff --git a/pythontex/pythontex_types3.py b/pythontex/pythontex_types3.py
@@ -138,6 +138,11 @@ def __init__(self, language, extension, command, default_code, utils_code,
             pytex = PythontexUtils()
             """
     #\\ End Python 3
+    # Add AsyncPylabSave import
+    utils_string_dict['python'] += """
+            from async_pylab_save import AsyncPylabSave
+            aps = AsyncPylabSave()
+            """
 
     inputs_string_const_dict['python'] = """
 
@@ -191,7 +196,10 @@ def __init__(self, language, extension, command, default_code, utils_code,
             """
     #\\ End Python 3
 
-    close_macrofile_string_dict['python'] = 'pytex.macrofile.close()\n'
+    close_macrofile_string_dict['python'] = """
+            pytex.macrofile.close()
+            aps.join()
+            """
 
     set_workingdir_string_dict['python'] = """
             if os.path.exists('{0}'):

diff --git a/pythontex_gallery/pythontex_gallery.pdf b/pythontex_gallery/pythontex_gallery.pdf
diff --git a/pythontex_gallery/pythontex_gallery.tex b/pythontex_gallery/pythontex_gallery.tex
@@ -131,13 +131,18 @@ \section{Plots with matplotlib}
 xlabel(r'$x\mathrm{-axis}$')
 ylabel(r'$y\mathrm{-axis}$')
 legend(loc='lower right')
-savefig('myplot.pdf', bbox_inches='tight')
+# asynchronous version of plt.savefig()
+aps.savefig('myplot.pdf', bbox_inches='tight')
 \end{pylabblock}
 
 \begin{center}
 \includegraphics{myplot}
 \end{center}
 
+\subsection{Asynchronous saving of plots}
+The above example also shows the usage of the \pyv{AsyncPylabSave} package which is shipped with \pytex, it works
+the same way as \pyv{plt.savefig()} except that it does not block the main program. Please see the documentation for
+a few multiprocessing caveats.
 
 \section{Basic pylab interaction}