Skip to content

Commit 64ba062

Browse files
blepblep
committed
- doc is now generated in dist/doxygen
- makerelease now decompress the tarball, download and install scons, run scons check on provided platforms, decompress doc tarball and upload the doc on the project web
1 parent 35bdc07 commit 64ba062

2 files changed

Lines changed: 194 additions & 31 deletions

File tree

‎doxybuild.py‎

Lines changed: 24 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,19 @@
88
import shutil
99
from devtools import tarball
1010

11-
def find_program(filename):
11+
def find_program(*filenames):
1212
"""find a program in folders path_lst, and sets env[var]
13-
@param env: environmentA
14-
@param filename: name of the program to search for
15-
@param path_list: list of directories to search for filename
16-
@param var: environment value to be checked for in env or os.environ
17-
@return: either the value that is referenced with [var] in env or os.environ
18-
or the first occurrence filename or '' if filename could not be found
13+
@param filenames: a list of possible names of the program to search for
14+
@return: the full path of the filename if found, or '' if filename could not be found
1915
"""
2016
paths = os.environ.get('PATH', '').split(os.pathsep)
2117
suffixes = ('win32' in sys.platform ) and '.exe .com .bat .cmd' or ''
22-
for name in [filename+ext for ext in suffixes.split()]:
23-
for directory in paths:
24-
full_path = os.path.join(directory, name)
25-
if os.path.isfile(full_path):
26-
return full_path
18+
for filename in filenames:
19+
for name in [filename+ext for ext in suffixes.split()]:
20+
for directory in paths:
21+
full_path = os.path.join(directory, name)
22+
if os.path.isfile(full_path):
23+
return full_path
2724
return ''
2825

2926
def do_subst_in_file(targetfile, sourcefile, dict):
@@ -87,36 +84,38 @@ def build_doc( options, make_release=False ):
8784
options.silent = True
8885

8986
version = open('version','rt').read().strip()
90-
output_dir = '../build/doxygen' # relative to doc/doxyfile location.
87+
output_dir = 'dist/doxygen' # relative to doc/doxyfile location.
88+
if not os.path.isdir( output_dir ):
89+
os.makedirs( output_dir )
9190
top_dir = os.path.abspath( '.' )
9291
html_output_dirname = 'jsoncpp-api-html-' + version
9392
tarball_path = os.path.join( 'dist', html_output_dirname + '.tar.gz' )
9493
warning_log_path = os.path.join( output_dir, '../jsoncpp-doxygen-warning.log' )
94+
html_output_path = os.path.join( output_dir, html_output_dirname )
9595
def yesno( bool ):
9696
return bool and 'YES' or 'NO'
9797
subst_keys = {
9898
'%JSONCPP_VERSION%': version,
9999
'%DOC_TOPDIR%': '',
100100
'%TOPDIR%': top_dir,
101-
'%HTML_OUTPUT%': os.path.join( output_dir, html_output_dirname ),
101+
'%HTML_OUTPUT%': os.path.join( '..', output_dir, html_output_dirname ),
102102
'%HAVE_DOT%': yesno(options.with_dot),
103103
'%DOT_PATH%': os.path.split(options.dot_path)[0],
104104
'%HTML_HELP%': yesno(options.with_html_help),
105105
'%UML_LOOK%': yesno(options.with_uml_look),
106-
'%WARNING_LOG_PATH%': warning_log_path
106+
'%WARNING_LOG_PATH%': os.path.join( '..', warning_log_path )
107107
}
108108

109-
full_output_dir = os.path.join( 'doc', output_dir )
110-
if os.path.isdir( full_output_dir ):
111-
print 'Deleting directory:', full_output_dir
112-
shutil.rmtree( full_output_dir )
113-
if not os.path.isdir( full_output_dir ):
114-
os.makedirs( full_output_dir )
109+
if os.path.isdir( output_dir ):
110+
print 'Deleting directory:', output_dir
111+
shutil.rmtree( output_dir )
112+
if not os.path.isdir( output_dir ):
113+
os.makedirs( output_dir )
115114

116115
do_subst_in_file( 'doc/doxyfile', 'doc/doxyfile.in', subst_keys )
117116
ok = run_doxygen( options.doxygen_path, 'doc/doxyfile', 'doc', is_silent=options.silent )
118117
if not options.silent:
119-
print open(os.path.join('doc', warning_log_path), 'rb').read()
118+
print open(warning_log_path, 'rb').read()
120119
index_path = os.path.abspath(os.path.join(subst_keys['%HTML_OUTPUT%'], 'index.html'))
121120
print 'Generated documentation can be found in:'
122121
print index_path
@@ -126,12 +125,13 @@ def yesno( bool ):
126125
if options.make_tarball:
127126
print 'Generating doc tarball to', tarball_path
128127
tarball_sources = [
129-
full_output_dir,
128+
output_dir,
130129
'README.txt',
131130
'version'
132131
]
133-
tarball_basedir = os.path.join( full_output_dir, html_output_dirname )
132+
tarball_basedir = os.path.join( output_dir, html_output_dirname )
134133
tarball.make_tarball( tarball_path, tarball_sources, tarball_basedir, html_output_dirname )
134+
return tarball_path, html_output_dirname
135135

136136
def main():
137137
usage = """%prog

‎makerelease.py‎

Lines changed: 170 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
Requires Python 2.6
44
55
Example of invocation (use to test the script):
6-
python makerelease.py --force --retag 0.5.0 0.6.0-dev
6+
python makerelease.py --force --retag --platform=msvc6,msvc71,msvc80,mingw -ublep 0.5.0 0.6.0-dev
77
88
Example of invocation when doing a release:
99
python makerelease.py 0.5.0 0.6.0-dev
@@ -15,15 +15,25 @@
1515
import subprocess
1616
import xml.etree.ElementTree as ElementTree
1717
import shutil
18+
import urllib2
19+
import tempfile
20+
import os
21+
import time
1822
from devtools import antglob, fixeol, tarball
1923

2024
SVN_ROOT = 'https://jsoncpp.svn.sourceforge.net/svnroot/jsoncpp/'
2125
SVN_TAG_ROOT = SVN_ROOT + 'tags/jsoncpp'
26+
SCONS_LOCAL_URL = 'http://sourceforge.net/projects/scons/files/scons-local/1.2.0/scons-local-1.2.0.tar.gz/download'
27+
SOURCEFORGE_PROJECT = 'jsoncpp'
2228

2329
def set_version( version ):
2430
with open('version','wb') as f:
2531
f.write( version.strip() )
2632

33+
def rmdir_if_exist( dir_path ):
34+
if os.path.isdir( dir_path ):
35+
shutil.rmtree( dir_path )
36+
2737
class SVNError(Exception):
2838
pass
2939

@@ -89,8 +99,7 @@ def svn_export( tag_url, export_dir ):
8999
Target directory, including its parent is created if it does not exist.
90100
If the directory export_dir exist, it is deleted before export proceed.
91101
"""
92-
if os.path.isdir( export_dir ):
93-
shutil.rmtree( export_dir )
102+
rmdir_if_exist( export_dir )
94103
svn_command( 'export', tag_url, export_dir )
95104

96105
def fix_sources_eol( dist_dir ):
@@ -111,6 +120,114 @@ def fix_sources_eol( dist_dir ):
111120
for path in unix_sources:
112121
fixeol.fix_source_eol( path, is_dry_run = False, verbose = True, eol = '\n' )
113122

123+
def download( url, target_path ):
124+
"""Download file represented by url to target_path.
125+
"""
126+
f = urllib2.urlopen( url )
127+
try:
128+
data = f.read()
129+
finally:
130+
f.close()
131+
fout = open( target_path, 'wb' )
132+
try:
133+
fout.write( data )
134+
finally:
135+
fout.close()
136+
137+
def check_compile( distcheck_top_dir, platform ):
138+
cmd = [sys.executable, 'scons.py', 'platform=%s' % platform, 'check']
139+
print 'Running:', ' '.join( cmd )
140+
log_path = os.path.join( distcheck_top_dir, 'build-%s.log' % platform )
141+
flog = open( log_path, 'wb' )
142+
try:
143+
process = subprocess.Popen( cmd,
144+
stdout=flog,
145+
stderr=subprocess.STDOUT,
146+
cwd=distcheck_top_dir )
147+
stdout = process.communicate()[0]
148+
status = (process.returncode == 0)
149+
finally:
150+
flog.close()
151+
return (status, log_path)
152+
153+
def write_tempfile( content, **kwargs ):
154+
fd, path = tempfile.mkstemp( **kwargs )
155+
f = os.fdopen( fd, 'wt' )
156+
try:
157+
f.write( content )
158+
finally:
159+
f.close()
160+
return path
161+
162+
class SFTPError(Exception):
163+
pass
164+
165+
def run_sftp_batch( userhost, sftp, batch, retry=0 ):
166+
path = write_tempfile( batch, suffix='.sftp', text=True )
167+
# psftp -agent -C blep,jsoncpp@web.sourceforge.net -batch -b batch.sftp -bc
168+
cmd = [sftp, '-agent', '-C', '-batch', '-b', path, '-bc', userhost]
169+
error = None
170+
for retry_index in xrange(0, max(1,retry)):
171+
heading = retry_index == 0 and 'Running:' or 'Retrying:'
172+
print heading, ' '.join( cmd )
173+
process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT )
174+
stdout = process.communicate()[0]
175+
if process.returncode != 0:
176+
error = SFTPError( 'SFTP batch failed:\n' + stdout )
177+
else:
178+
break
179+
if error:
180+
raise error
181+
return stdout
182+
183+
def sourceforge_web_synchro( sourceforge_project, doc_dir,
184+
user=None, sftp='sftp' ):
185+
"""Notes: does not synchronize sub-directory of doc-dir.
186+
"""
187+
userhost = '%s,%s@web.sourceforge.net' % (user, sourceforge_project)
188+
stdout = run_sftp_batch( userhost, sftp, """
189+
cd htdocs
190+
dir
191+
exit
192+
""" )
193+
existing_paths = set()
194+
collect = 0
195+
for line in stdout.split('\n'):
196+
line = line.strip()
197+
if not collect and line.endswith('> dir'):
198+
collect = True
199+
elif collect and line.endswith('> exit'):
200+
break
201+
elif collect == 1:
202+
collect = 2
203+
elif collect == 2:
204+
path = line.strip().split()[-1:]
205+
if path and path[0] not in ('.', '..'):
206+
existing_paths.add( path[0] )
207+
upload_paths = set( [os.path.basename(p) for p in antglob.glob( doc_dir )] )
208+
paths_to_remove = existing_paths - upload_paths
209+
if paths_to_remove:
210+
print 'Removing the following file from web:'
211+
print '\n'.join( paths_to_remove )
212+
stdout = run_sftp_batch( userhost, sftp, """cd htdocs
213+
rm %s
214+
exit""" % ' '.join(paths_to_remove) )
215+
print 'Uploading %d files:' % len(upload_paths)
216+
batch_size = 10
217+
upload_paths = list(upload_paths)
218+
start_time = time.time()
219+
for index in xrange(0,len(upload_paths),batch_size):
220+
paths = upload_paths[index:index+batch_size]
221+
file_per_sec = (time.time() - start_time) / (index+1)
222+
remaining_files = len(upload_paths) - index
223+
remaining_sec = file_per_sec * remaining_files
224+
print '%d/%d, ETA=%.1fs' % (index+1, len(upload_paths), remaining_sec)
225+
run_sftp_batch( userhost, sftp, """cd htdocs
226+
lcd %s
227+
mput %s
228+
exit""" % (doc_dir, ' '.join(paths) ), retry=3 )
229+
230+
114231
def main():
115232
usage = """%prog release_version next_dev_version
116233
Update 'version' file to release_version and commit.
@@ -120,7 +237,9 @@ def main():
120237
121238
Performs an svn export of tag release version, and build a source tarball.
122239
123-
Must be started in the project top directory.
240+
Must be started in the project top directory.
241+
242+
Warning: --force should only be used when developping/testing the release script.
124243
"""
125244
from optparse import OptionParser
126245
parser = OptionParser(usage=usage)
@@ -133,13 +252,24 @@ def main():
133252
help="""Ignore pending commit. [Default: %default]""")
134253
parser.add_option('--retag', dest="retag_release", action='store_true', default=False,
135254
help="""Overwrite release existing tag if it exist. [Default: %default]""")
255+
parser.add_option('-p', '--platforms', dest="platforms", action='store', default='',
256+
help="""Comma separated list of platform passed to scons for build check.""")
257+
parser.add_option('--no-test', dest="no_test", action='store', default=False,
258+
help="""Skips build check.""")
259+
parser.add_option('-u', '--upload-user', dest="user", action='store',
260+
help="""Sourceforge user for SFTP documentation upload.""")
261+
parser.add_option('--sftp', dest='sftp', action='store', default=doxybuild.find_program('psftp', 'sftp'),
262+
help="""Path of the SFTP compatible binary used to upload the documentation.""")
136263
parser.enable_interspersed_args()
137264
options, args = parser.parse_args()
138265

139266
if len(args) < 1:
140267
parser.error( 'release_version missing on command-line.' )
141268
release_version = args[0]
142269

270+
if not options.platforms and not options.no_test:
271+
parser.error( 'You must specify either --platform or --no-test option.' )
272+
143273
if options.ignore_pending_commit:
144274
msg = ''
145275
else:
@@ -157,7 +287,12 @@ def main():
157287
svn_tag_sandbox( tag_url, 'Release ' + release_version )
158288

159289
print 'Generated doxygen document...'
160-
doxybuild.build_doc( options, make_release=True )
290+
## doc_dirname = r'jsoncpp-api-html-0.5.0'
291+
## doc_tarball_path = r'e:\prg\vc\Lib\jsoncpp-trunk\dist\jsoncpp-api-html-0.5.0.tar.gz'
292+
doc_tarball_path, doc_dirname = doxybuild.build_doc( options, make_release=True )
293+
doc_distcheck_dir = 'dist/doccheck'
294+
tarball.decompress( doc_tarball_path, doc_distcheck_dir )
295+
doc_distcheck_top_dir = os.path.join( doc_distcheck_dir, doc_dirname )
161296

162297
export_dir = 'dist/export'
163298
svn_export( tag_url, export_dir )
@@ -168,12 +303,40 @@ def main():
168303
print 'Generating source tarball to', source_tarball_path
169304
tarball.make_tarball( source_tarball_path, [export_dir], export_dir, prefix_dir=source_dir )
170305

306+
# Decompress source tarball, download and install scons-local
171307
distcheck_dir = 'dist/distcheck'
308+
distcheck_top_dir = distcheck_dir + '/' + source_dir
172309
print 'Decompressing source tarball to', distcheck_dir
310+
rmdir_if_exist( distcheck_dir )
173311
tarball.decompress( source_tarball_path, distcheck_dir )
312+
scons_local_path = 'dist/scons-local.tar.gz'
313+
print 'Downloading scons-local to', scons_local_path
314+
download( SCONS_LOCAL_URL, scons_local_path )
315+
print 'Decompressing scons-local to', distcheck_top_dir
316+
tarball.decompress( scons_local_path, distcheck_top_dir )
317+
318+
# Run compilation
319+
print 'Compiling decompressed tarball'
320+
all_build_status = True
321+
for platform in options.platforms.split(','):
322+
print 'Testing platform:', platform
323+
build_status, log_path = check_compile( distcheck_top_dir, platform )
324+
print 'see build log:', log_path
325+
print build_status and '=> ok' or '=> FAILED'
326+
all_build_status = all_build_status and build_status
327+
if not build_status:
328+
print 'Testing failed on at least one platform, aborting...'
329+
svn_remove_tag( tag_url, 'Removing tag due to failed testing' )
330+
sys.exit(1)
331+
if options.user:
332+
print 'Uploading documentation using user', options.user
333+
sourceforge_web_synchro( SOURCEFORGE_PROJECT, doc_distcheck_top_dir, user=options.user, sftp=options.sftp )
334+
print 'Completed documentatio upload'
335+
else:
336+
print 'No upload user specified. Documentation was not upload.'
337+
print 'Tarball can be found at:', doc_tarball_path
174338
#@todo:
175-
# ?compile & run & check
176-
# ?upload documentation
339+
#upload source & doc tarballs
177340
else:
178341
sys.stderr.write( msg + '\n' )
179342

0 commit comments

Comments
 (0)