diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..2cc934a --- /dev/null +++ b/.gitignore @@ -0,0 +1,17 @@ +*~ +pythontex-files-pythontex_gallery/ +*.aux +*.log +*.pytxcode +.DS_Store +*.glo + +*.idx + +*.out + +*.toc + +*.fdb_latexmk + +*.fls diff --git a/pythontex/async_pylab_save.py b/pythontex/async_pylab_save.py new file mode 100644 index 0000000..18d1a12 --- /dev/null +++ b/pythontex/async_pylab_save.py @@ -0,0 +1,69 @@ +# -*- coding: utf-8 -*- +import time +import multiprocessing as mp +import numpy as np +import matplotlib +#matplotlib.use('Agg') +import matplotlib.pyplot as plt +""" +Asynchronous Plotting in Matplotlib: rather than call savefig directly, add plots to an asynchronous queue to avoid holding up the main program. Makes use of multiple processes to speed up the writing out. +original author: astrofrog, https://gist.github.com/1453933 +minor modifications by: ob@cakebox.net +""" + +class AsyncPylabSave(): + def __init__(self, processes=mp.cpu_count()): + self.manager = mp.Manager() + self.nc = self.manager.Value('i', 0) + self.pids = [] + self.processes = processes + + def async_plotter(self, nc, fig, filename, processes, **kwargs): + while nc.value >= processes: + time.sleep(0.1) + nc.value += 1 + fig.savefig(filename, **kwargs) + plt.close(fig) + nc.value -= 1 + + def savefig(self, filename, fig=None, **kwargs): + # Calls fig.savefig(filename) asynchronously, if fig is None (default) the current figure is saved. + # kwargs are sent directly to savefig. + if fig == None: + fig = plt.gcf() + p = mp.Process(target=self.async_plotter, + args=(self.nc, fig, filename, self.processes), + kwargs=kwargs) + p.start() + self.pids.append(p) + + def join(self): + for p in self.pids: + p.join() + +""" +Example usage: +# Create instance of Asynchronous plotter +a = AsyncPylabSave() + +for i in range(10): + + print 'Preparing %04i.png' % i + + # Generate random points + x = np.random.random(10000) + y = np.random.random(10000) + + # Generate figure + fig = plt.figure() + ax = fig.add_subplot(1, 1, 1) + ax.set_xscale('log') + ax.set_yscale('log') + ax.scatter(x, y) + + # Add figure to queue + a.savefig('%04i.png' % i, fig=fig, facecolor='r') + +# Wait for all plots to finish +a.join() +""" diff --git a/pythontex/pythontex.dtx b/pythontex/pythontex.dtx index b667573..24d2617 100644 --- a/pythontex/pythontex.dtx +++ b/pythontex/pythontex.dtx @@ -164,7 +164,7 @@ % \item Installer file |pythontex.ins| % \item Documented \LaTeX\ source file |pythontex.dtx|, from which |pythontex.pdf| and |pythontex.sty| are generated % \item Main Python scripts |pythontex2.py| and |pythontex3.py| -% \item Helper scripts |pythontex_utils2.py| and |pythontex_types2.py|, and |pythontex_utils3.py| and |pythontex_types3.py| +% \item Helper scripts |pythontex_utils2.py|, |pythontex_types2.py|, |pythontex_utils3.py|, |pythontex_types3.py| and |async_pylab_save.py| % \item Installation script |pythontex_install_texlive| (for \TeX\ Live) % \item README % \item Optional batch file |pythontex.bat| for use in launching |pythontex*.py| under Windows @@ -183,6 +183,7 @@ % \item |pythontex2.py| and |pythontex3.py| % \item |pythontex_types2.py| and |pythontex_types3.py| % \item |pythontex_utils2.py| and |pythontex_utils3.py| +% \item |async_pylab_save.py| % \end{itemize} % \item \meta{\TeX\ tree root}|/source/latex/pythontex/| % \begin{itemize} @@ -417,6 +418,20 @@ % \meta{quoted~list} may \textbf{not} contain \LaTeX\ macros. \meta{quoted~list} is interpreted as verbatim content, since in general the custom code will not be valid \LaTeX. % % +% \subsubsection{Asynchronous saving of plots} +% To avoid locking up the main program flow when saving figures the utility |async_pylab_save.py| is included. +% +% This works by making |asp = AsyncPylabSave()| available in every family and then calling |asp.join()| at the end. Calling +% |asp.savefig(...)| +% will call |plt.savefig(...)| +% without blocking the main program (using multiprocessing). +% Or eqvivalently +% |asp.savefig(filename, fig=fig, **kwargs)|, where fig is a figure handle, +% will work as |fig.savefig(filename, **kwargs)|. +% +% Please note that the figure will be closed after saving, so manipulation of the figure after calling save will not work. Saving multiple figures to the same filename will of course have unexpected results! +% +% TODO: I guess this feature only makes sense in the pylab family\dots % \subsubsection{Formatting of typeset code} % % \DescribeMacro{\setpythontexfv\oarg{family}\marg{fancyvrb~settings}} diff --git a/pythontex/pythontex.pdf b/pythontex/pythontex.pdf index 8c8bb11..a93e56b 100644 Binary files a/pythontex/pythontex.pdf and b/pythontex/pythontex.pdf differ diff --git a/pythontex/pythontex2.py b/pythontex/pythontex2.py index ab99932..95bf7d4 100644 --- a/pythontex/pythontex2.py +++ b/pythontex/pythontex2.py @@ -963,6 +963,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict): # Add in a Pygments process if applicable for key in update_pygments: if update_pygments[key] and not key.endswith('cons'): + print("* Pythontex pygment processing: " + str(key)) tasks.append(pool.apply_async(do_pygments, [outputdir, jobname, fvextfile, @@ -975,6 +976,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict): # Add console processes for key in consoledict: if update_code[key] or update_pygments[key]: + print("* Pythontex console processing: " + str(key)) tasks.append(pool.apply_async(run_console, [outputdir, jobname, fvextfile, @@ -990,6 +992,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict): # Add code processes. Note that everything placed in the codedict # needs to be executed, based on previous testing. for key in codedict: + print("* Pythontex code processing: " + str(key)) [inputtype, inputsession, inputgroup] = key.split('#') tasks.append(pool.apply_async(run_code, [inputtype, inputsession, @@ -1141,7 +1144,7 @@ def run_code(inputtype, inputsession, inputgroup, outputdir, command, # Only work with files that have a nonzero size if os.path.isfile(err_file_name) and os.stat(err_file_name).st_size != 0: # Reset the hash value, so that the code will be run next time - exit_status[currentkey] = '' + #exit_status[currentkey] = '' # Open error and code files. # We can't just use the code in memory, because the full script # file was written but never fully assembled in memory. @@ -1749,3 +1752,4 @@ def save_data(data): print('\n--------------------------------------------------') print('PythonTeX: ' + data['raw_jobname'] + ' - ' + str(temp_data['errors']) + ' error(s), ' + str(temp_data['warnings']) + ' warning(s)') + exit(temp_data['errors']) diff --git a/pythontex/pythontex3.py b/pythontex/pythontex3.py index c50b6dc..fe2e890 100644 --- a/pythontex/pythontex3.py +++ b/pythontex/pythontex3.py @@ -963,6 +963,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict): # Add in a Pygments process if applicable for key in update_pygments: if update_pygments[key] and not key.endswith('cons'): + print("* Pythontex pygment processing: " + str(key)) tasks.append(pool.apply_async(do_pygments, [outputdir, jobname, fvextfile, @@ -975,6 +976,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict): # Add console processes for key in consoledict: if update_code[key] or update_pygments[key]: + print("* Pythontex console processing: " + str(key)) tasks.append(pool.apply_async(run_console, [outputdir, jobname, fvextfile, @@ -990,6 +992,7 @@ def do_multiprocessing(data, temp_data, old_data, typedict): # Add code processes. Note that everything placed in the codedict # needs to be executed, based on previous testing. for key in codedict: + print("* Pythontex code processing: " + str(key)) [inputtype, inputsession, inputgroup] = key.split('#') tasks.append(pool.apply_async(run_code, [inputtype, inputsession, @@ -1141,7 +1144,7 @@ def run_code(inputtype, inputsession, inputgroup, outputdir, command, # Only work with files that have a nonzero size if os.path.isfile(err_file_name) and os.stat(err_file_name).st_size != 0: # Reset the hash value, so that the code will be run next time - exit_status[currentkey] = '' + #exit_status[currentkey] = '' # Open error and code files. # We can't just use the code in memory, because the full script # file was written but never fully assembled in memory. @@ -1749,3 +1752,4 @@ def save_data(data): print('\n--------------------------------------------------') print('PythonTeX: ' + data['raw_jobname'] + ' - ' + str(temp_data['errors']) + ' error(s), ' + str(temp_data['warnings']) + ' warning(s)') + exit(temp_data['errors']) diff --git a/pythontex/pythontex_install_texlive.py b/pythontex/pythontex_install_texlive.py index b30eb62..f3e0e89 100644 --- a/pythontex/pythontex_install_texlive.py +++ b/pythontex/pythontex_install_texlive.py @@ -33,7 +33,7 @@ # Imports import sys import platform -from os import path, mkdir +from os import path, mkdir, symlink, chmod from subprocess import check_call, check_output, CalledProcessError from shutil import copy @@ -49,7 +49,7 @@ needed_files = ['pythontex2.py', 'pythontex_types2.py', 'pythontex_utils2.py', 'pythontex3.py', 'pythontex_types3.py', 'pythontex_utils3.py', 'pythontex.sty', 'pythontex.ins', 'pythontex.dtx', - 'pythontex.pdf', 'README.rst'] + 'pythontex.pdf', 'README.rst', 'async_pylab_save.py'] missing_files = False # Print a list of all files that are missing, and exit if any are for eachfile in needed_files: @@ -60,11 +60,6 @@ print('Exiting.') sys.exit(1) - -# Print starting message -print('\nInstalling PythonTeX...') - - # Retrieve the location of a valid TeX tree # Attempt to use kpsewhich; otherwise, resort to manual input try: @@ -80,6 +75,10 @@ print('\nkpsewhich is not happy with its arguments.') print('Cannot automatically find a valid texmf path.') texmf_path = input('Please enter a valid texmf path: ').rstrip('\r\n') + +# Print starting message +print('\nInstalling PythonTeX into directory ' + texmf_path) + # Make sure path slashes are compatible with the operating system # This is only needed for Windows texmf_path = path.normcase(texmf_path) @@ -127,6 +126,7 @@ copy('pythontex3.py', scripts_path) copy('pythontex_types3.py', scripts_path) copy('pythontex_utils3.py', scripts_path) +copy('async_pylab_save.py', scripts_path) # Install source if not path.exists(source_path): mkdir(source_path) @@ -154,7 +154,27 @@ print('The bin/ directory in your TeX distribution may be a good location.') print('The script pythontex.py is located in the following directory:') print(' ' + scripts_path) -# If not under Windows, we alert the user regarding what is necessary to launch +elif platform.system() in ['Linux', 'Darwin']: # todo: check for unix (maybe just check to see if os.symlink fails or not?) + root_path = path.split(texmf_path)[0] + bin_path = path.join(path.split(root_path)[0], 'bin') + if path.exists(bin_path): + for ver in [2, 3]: + link = path.join(bin_path, 'pythontex{0}.py'.format(ver)) + try: + symlink(path.join(scripts_path, 'pythontex{0}.py'.format(ver)), link) + except OSError as e: + if e.errno == 17: + pass # File exists + else: + raise OSError(e) + chmod(link, 0775) + print('symlink created ' + link) + else: + print('\nCreating symlink failed, you may wish to create a symlink to pythontex.py.') + print('You may also want to make it executable via chmod.') + print('The script pythontex.py is located in the following directory:') + print(' ' + scripts_path) +# If not under known system, we alert the user regarding what is necessary to launch # pythontex.py else: print('\nYou may wish to create a symlink to pythontex.py.') @@ -179,5 +199,6 @@ print('See the documentation for more information.') print('* * *\n') -# Pause so that the user can see any errors or other messages -input('\n[Press ENTER to exit]') +if platform.system() == 'Windows': + # Pause so that the user can see any errors or other messages + input('\n[Press ENTER to exit]') diff --git a/pythontex/pythontex_types2.py b/pythontex/pythontex_types2.py index 1604149..c6a3438 100644 --- a/pythontex/pythontex_types2.py +++ b/pythontex/pythontex_types2.py @@ -138,6 +138,11 @@ def __init__(self, language, extension, command, default_code, utils_code, # pytex = PythontexUtils() # """ #\\ End Python 3 + # Add AsyncPylabSave import + utils_string_dict['python'] += """ + from async_pylab_save import AsyncPylabSave + aps = AsyncPylabSave() + """ inputs_string_const_dict['python'] = """ @@ -191,7 +196,10 @@ def __init__(self, language, extension, command, default_code, utils_code, # """ #\\ End Python 3 - close_macrofile_string_dict['python'] = 'pytex.macrofile.close()\n' + close_macrofile_string_dict['python'] = """ + pytex.macrofile.close() + aps.join() + """ set_workingdir_string_dict['python'] = """ if os.path.exists('{0}'): diff --git a/pythontex/pythontex_types3.py b/pythontex/pythontex_types3.py index 7604691..97af991 100644 --- a/pythontex/pythontex_types3.py +++ b/pythontex/pythontex_types3.py @@ -138,6 +138,11 @@ def __init__(self, language, extension, command, default_code, utils_code, pytex = PythontexUtils() """ #\\ End Python 3 + # Add AsyncPylabSave import + utils_string_dict['python'] += """ + from async_pylab_save import AsyncPylabSave + aps = AsyncPylabSave() + """ inputs_string_const_dict['python'] = """ @@ -191,7 +196,10 @@ def __init__(self, language, extension, command, default_code, utils_code, """ #\\ End Python 3 - close_macrofile_string_dict['python'] = 'pytex.macrofile.close()\n' + close_macrofile_string_dict['python'] = """ + pytex.macrofile.close() + aps.join() + """ set_workingdir_string_dict['python'] = """ if os.path.exists('{0}'): diff --git a/pythontex_gallery/pythontex_gallery.pdf b/pythontex_gallery/pythontex_gallery.pdf index e66c4f5..a3f51f2 100644 Binary files a/pythontex_gallery/pythontex_gallery.pdf and b/pythontex_gallery/pythontex_gallery.pdf differ diff --git a/pythontex_gallery/pythontex_gallery.tex b/pythontex_gallery/pythontex_gallery.tex index be95390..a834f06 100644 --- a/pythontex_gallery/pythontex_gallery.tex +++ b/pythontex_gallery/pythontex_gallery.tex @@ -131,13 +131,18 @@ \section{Plots with matplotlib} xlabel(r'$x\mathrm{-axis}$') ylabel(r'$y\mathrm{-axis}$') legend(loc='lower right') -savefig('myplot.pdf', bbox_inches='tight') +# asynchronous version of plt.savefig() +aps.savefig('myplot.pdf', bbox_inches='tight') \end{pylabblock} \begin{center} \includegraphics{myplot} \end{center} +\subsection{Asynchronous saving of plots} +The above example also shows the usage of the \pyv{AsyncPylabSave} package which is shipped with \pytex, it works +the same way as \pyv{plt.savefig()} except that it does not block the main program. Please see the documentation for +a few multiprocessing caveats. \section{Basic pylab interaction}