|
| 1 | +import collections |
| 2 | +import itertools |
| 3 | +import json |
| 4 | +import os |
| 5 | +import os.path |
| 6 | +import re |
| 7 | +import shutil |
| 8 | +import string |
| 9 | +import subprocess |
| 10 | +import sys |
| 11 | +import cgi |
| 12 | + |
| 13 | +class BuildDesc: |
| 14 | + def __init__(self, prepend_envs=None, variables=None, build_type=None, generator=None): |
| 15 | + self.prepend_envs = prepend_envs or [] # [ { "var": "value" } ] |
| 16 | + self.variables = variables or [] |
| 17 | + self.build_type = build_type |
| 18 | + self.generator = generator |
| 19 | + |
| 20 | + def merged_with( self, build_desc ): |
| 21 | + """Returns a new BuildDesc by merging field content. |
| 22 | + Prefer build_desc fields to self fields for single valued field. |
| 23 | + """ |
| 24 | + return BuildDesc( self.prepend_envs + build_desc.prepend_envs, |
| 25 | + self.variables + build_desc.variables, |
| 26 | + build_desc.build_type or self.build_type, |
| 27 | + build_desc.generator or self.generator ) |
| 28 | + |
| 29 | + def env( self ): |
| 30 | + environ = os.environ.copy() |
| 31 | + for values_by_name in self.prepend_envs: |
| 32 | + for var, value in values_by_name.items(): |
| 33 | + var = var.upper() |
| 34 | + if type(value) is unicode: |
| 35 | + value = value.encode( sys.getdefaultencoding() ) |
| 36 | + if var in environ: |
| 37 | + environ[var] = value + os.pathsep + environ[var] |
| 38 | + else: |
| 39 | + environ[var] = value |
| 40 | + return environ |
| 41 | + |
| 42 | + def cmake_args( self ): |
| 43 | + args = ["-D%s" % var for var in self.variables] |
| 44 | + # skip build type for Visual Studio solution as it cause warning |
| 45 | + if self.build_type and 'Visual' not in self.generator: |
| 46 | + args.append( "-DCMAKE_BUILD_TYPE=%s" % self.build_type ) |
| 47 | + if self.generator: |
| 48 | + args.extend( ['-G', self.generator] ) |
| 49 | + return args |
| 50 | + |
| 51 | + def __repr__( self ): |
| 52 | + return "BuildDesc( %s, build_type=%s )" % (" ".join( self.cmake_args()), self.build_type) |
| 53 | + |
| 54 | +class BuildData: |
| 55 | + def __init__( self, desc, work_dir, source_dir ): |
| 56 | + self.desc = desc |
| 57 | + self.work_dir = work_dir |
| 58 | + self.source_dir = source_dir |
| 59 | + self.cmake_log_path = os.path.join( work_dir, 'batchbuild_cmake.log' ) |
| 60 | + self.build_log_path = os.path.join( work_dir, 'batchbuild_build.log' ) |
| 61 | + self.cmake_succeeded = False |
| 62 | + self.build_succeeded = False |
| 63 | + |
| 64 | + def execute_build(self): |
| 65 | + print 'Build %s' % self.desc |
| 66 | + self._make_new_work_dir( ) |
| 67 | + self.cmake_succeeded = self._generate_makefiles( ) |
| 68 | + if self.cmake_succeeded: |
| 69 | + self.build_succeeded = self._build_using_makefiles( ) |
| 70 | + return self.build_succeeded |
| 71 | + |
| 72 | + def _generate_makefiles(self): |
| 73 | + print ' Generating makefiles: ', |
| 74 | + cmd = ['cmake'] + self.desc.cmake_args( ) + [os.path.abspath( self.source_dir )] |
| 75 | + succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.cmake_log_path ) |
| 76 | + print 'done' if succeeded else 'FAILED' |
| 77 | + return succeeded |
| 78 | + |
| 79 | + def _build_using_makefiles(self): |
| 80 | + print ' Building:', |
| 81 | + cmd = ['cmake', '--build', self.work_dir] |
| 82 | + if self.desc.build_type: |
| 83 | + cmd += ['--config', self.desc.build_type] |
| 84 | + succeeded = self._execute_build_subprocess( cmd, self.desc.env(), self.build_log_path ) |
| 85 | + print 'done' if succeeded else 'FAILED' |
| 86 | + return succeeded |
| 87 | + |
| 88 | + def _execute_build_subprocess(self, cmd, env, log_path): |
| 89 | + process = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, cwd=self.work_dir, |
| 90 | + env=env ) |
| 91 | + stdout, _ = process.communicate( ) |
| 92 | + succeeded = (process.returncode == 0) |
| 93 | + with open( log_path, 'wb' ) as flog: |
| 94 | + log = ' '.join( cmd ) + '\n' + stdout + '\nExit code: %r\n' % process.returncode |
| 95 | + flog.write( fix_eol( log ) ) |
| 96 | + return succeeded |
| 97 | + |
| 98 | + def _make_new_work_dir(self): |
| 99 | + if os.path.isdir( self.work_dir ): |
| 100 | + print ' Removing work directory', self.work_dir |
| 101 | + shutil.rmtree( self.work_dir, ignore_errors=True ) |
| 102 | + if not os.path.isdir( self.work_dir ): |
| 103 | + os.makedirs( self.work_dir ) |
| 104 | + |
| 105 | +def fix_eol( stdout ): |
| 106 | + """Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). |
| 107 | + """ |
| 108 | + return re.sub( '\r*\n', os.linesep, stdout ) |
| 109 | + |
| 110 | +def load_build_variants_from_config( config_path ): |
| 111 | + with open( config_path, 'rb' ) as fconfig: |
| 112 | + data = json.load( fconfig ) |
| 113 | + variants = data[ 'cmake_variants' ] |
| 114 | + build_descs_by_axis = collections.defaultdict( list ) |
| 115 | + for axis in variants: |
| 116 | + axis_name = axis["name"] |
| 117 | + build_descs = [] |
| 118 | + if "generators" in axis: |
| 119 | + for generator_data in axis["generators"]: |
| 120 | + for generator in generator_data["generator"]: |
| 121 | + build_desc = BuildDesc( generator=generator, |
| 122 | + prepend_envs=generator_data.get("env_prepend") ) |
| 123 | + build_descs.append( build_desc ) |
| 124 | + elif "variables" in axis: |
| 125 | + for variables in axis["variables"]: |
| 126 | + build_desc = BuildDesc( variables=variables ) |
| 127 | + build_descs.append( build_desc ) |
| 128 | + elif "build_types" in axis: |
| 129 | + for build_type in axis["build_types"]: |
| 130 | + build_desc = BuildDesc( build_type=build_type ) |
| 131 | + build_descs.append( build_desc ) |
| 132 | + build_descs_by_axis[axis_name].extend( build_descs ) |
| 133 | + return build_descs_by_axis |
| 134 | + |
| 135 | +def generate_build_variants( build_descs_by_axis ): |
| 136 | + """Returns a list of BuildDesc generated for the partial BuildDesc for each axis.""" |
| 137 | + axis_names = build_descs_by_axis.keys() |
| 138 | + build_descs = [] |
| 139 | + for axis_name, axis_build_descs in build_descs_by_axis.items(): |
| 140 | + if len(build_descs): |
| 141 | + # for each existing build_desc and each axis build desc, create a new build_desc |
| 142 | + new_build_descs = [] |
| 143 | + for prototype_build_desc, axis_build_desc in itertools.product( build_descs, axis_build_descs): |
| 144 | + new_build_descs.append( prototype_build_desc.merged_with( axis_build_desc ) ) |
| 145 | + build_descs = new_build_descs |
| 146 | + else: |
| 147 | + build_descs = axis_build_descs |
| 148 | + return build_descs |
| 149 | + |
| 150 | +HTML_TEMPLATE = string.Template('''<html> |
| 151 | +<head> |
| 152 | + <title>$title</title> |
| 153 | + <style type="text/css"> |
| 154 | + td.failed {background-color:#f08080;} |
| 155 | + td.ok {background-color:#c0eec0;} |
| 156 | + </style> |
| 157 | +</head> |
| 158 | +<body> |
| 159 | +<table border="1"> |
| 160 | +<thead> |
| 161 | + <tr> |
| 162 | + <th>Variables</th> |
| 163 | + $th_vars |
| 164 | + </tr> |
| 165 | + <tr> |
| 166 | + <th>Build type</th> |
| 167 | + $th_build_types |
| 168 | + </tr> |
| 169 | +</thead> |
| 170 | +<tbody> |
| 171 | +$tr_builds |
| 172 | +</tbody> |
| 173 | +</table> |
| 174 | +</body></html>''') |
| 175 | + |
| 176 | +def generate_html_report( html_report_path, builds ): |
| 177 | + report_dir = os.path.dirname( html_report_path ) |
| 178 | + # Vertical axis: generator |
| 179 | + # Horizontal: variables, then build_type |
| 180 | + builds_by_generator = collections.defaultdict( list ) |
| 181 | + variables = set() |
| 182 | + build_types_by_variable = collections.defaultdict( set ) |
| 183 | + build_by_pos_key = {} # { (generator, var_key, build_type): build } |
| 184 | + for build in builds: |
| 185 | + builds_by_generator[build.desc.generator].append( build ) |
| 186 | + var_key = tuple(sorted(build.desc.variables)) |
| 187 | + variables.add( var_key ) |
| 188 | + build_types_by_variable[var_key].add( build.desc.build_type ) |
| 189 | + pos_key = (build.desc.generator, var_key, build.desc.build_type) |
| 190 | + build_by_pos_key[pos_key] = build |
| 191 | + variables = sorted( variables ) |
| 192 | + th_vars = [] |
| 193 | + th_build_types = [] |
| 194 | + for variable in variables: |
| 195 | + build_types = sorted( build_types_by_variable[variable] ) |
| 196 | + nb_build_type = len(build_types_by_variable[variable]) |
| 197 | + th_vars.append( '<th colspan="%d">%s</th>' % (nb_build_type, cgi.escape( ' '.join( variable ) ) ) ) |
| 198 | + for build_type in build_types: |
| 199 | + th_build_types.append( '<th>%s</th>' % cgi.escape(build_type) ) |
| 200 | + tr_builds = [] |
| 201 | + for generator in sorted( builds_by_generator ): |
| 202 | + tds = [ '<td>%s</td>\n' % cgi.escape( generator ) ] |
| 203 | + for variable in variables: |
| 204 | + build_types = sorted( build_types_by_variable[variable] ) |
| 205 | + for build_type in build_types: |
| 206 | + pos_key = (generator, variable, build_type) |
| 207 | + build = build_by_pos_key.get(pos_key) |
| 208 | + if build: |
| 209 | + cmake_status = 'ok' if build.cmake_succeeded else 'FAILED' |
| 210 | + build_status = 'ok' if build.build_succeeded else 'FAILED' |
| 211 | + cmake_log_url = os.path.relpath( build.cmake_log_path, report_dir ) |
| 212 | + build_log_url = os.path.relpath( build.build_log_path, report_dir ) |
| 213 | + td = '<td class="%s"><a href="%s" class="%s">CMake: %s</a>' % ( |
| 214 | + build_status.lower(), cmake_log_url, cmake_status.lower(), cmake_status) |
| 215 | + if build.cmake_succeeded: |
| 216 | + td += '<br><a href="%s" class="%s">Build: %s</a>' % ( |
| 217 | + build_log_url, build_status.lower(), build_status) |
| 218 | + td += '</td>' |
| 219 | + else: |
| 220 | + td = '<td></td>' |
| 221 | + tds.append( td ) |
| 222 | + tr_builds.append( '<tr>%s</tr>' % '\n'.join( tds ) ) |
| 223 | + html = HTML_TEMPLATE.substitute( |
| 224 | + title='Batch build report', |
| 225 | + th_vars=' '.join(th_vars), |
| 226 | + th_build_types=' '.join( th_build_types), |
| 227 | + tr_builds='\n'.join( tr_builds ) ) |
| 228 | + with open( html_report_path, 'wt' ) as fhtml: |
| 229 | + fhtml.write( html ) |
| 230 | + print 'HTML report generated in:', html_report_path |
| 231 | + |
| 232 | +def main(): |
| 233 | + usage = r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...] |
| 234 | +Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_run |
| 235 | +as described in CONFIG_JSON_PATH building in WORK_DIR. |
| 236 | +
|
| 237 | +Example of call: |
| 238 | +python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json |
| 239 | +""" |
| 240 | + from optparse import OptionParser |
| 241 | + parser = OptionParser(usage=usage) |
| 242 | + parser.allow_interspersed_args = True |
| 243 | +# parser.add_option('-v', '--verbose', dest="verbose", action='store_true', |
| 244 | +# help="""Be verbose.""") |
| 245 | + parser.enable_interspersed_args() |
| 246 | + options, args = parser.parse_args() |
| 247 | + if len(args) < 3: |
| 248 | + parser.error( "Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH." ) |
| 249 | + work_dir = args[0] |
| 250 | + source_dir = args[1].rstrip('/\\') |
| 251 | + config_paths = args[2:] |
| 252 | + for config_path in config_paths: |
| 253 | + if not os.path.isfile( config_path ): |
| 254 | + parser.error( "Can not read: %r" % config_path ) |
| 255 | + |
| 256 | + # generate build variants |
| 257 | + build_descs = [] |
| 258 | + for config_path in config_paths: |
| 259 | + build_descs_by_axis = load_build_variants_from_config( config_path ) |
| 260 | + build_descs.extend( generate_build_variants( build_descs_by_axis ) ) |
| 261 | + print 'Build variants (%d):' % len(build_descs) |
| 262 | + # assign build directory for each variant |
| 263 | + if not os.path.isdir( work_dir ): |
| 264 | + os.makedirs( work_dir ) |
| 265 | + builds = [] |
| 266 | + with open( os.path.join( work_dir, 'matrix-dir-map.txt' ), 'wt' ) as fmatrixmap: |
| 267 | + for index, build_desc in enumerate( build_descs ): |
| 268 | + build_desc_work_dir = os.path.join( work_dir, '%03d' % (index+1) ) |
| 269 | + builds.append( BuildData( build_desc, build_desc_work_dir, source_dir ) ) |
| 270 | + fmatrixmap.write( '%s: %s\n' % (build_desc_work_dir, build_desc) ) |
| 271 | + for build in builds: |
| 272 | + build.execute_build() |
| 273 | + html_report_path = os.path.join( work_dir, 'batchbuild-report.html' ) |
| 274 | + generate_html_report( html_report_path, builds ) |
| 275 | + print 'Done' |
| 276 | + |
| 277 | + |
| 278 | +if __name__ == '__main__': |
| 279 | + main() |
| 280 | + |
0 commit comments