from__future__importprint_functionimportcollectionsimportitertoolsimportjsonimportosimportos.pathimportreimportshutilimportstringimportsubprocessimportsysimportcgiclassBuildDesc:def__init__(self,prepend_envs=None,variables=None,build_type=None,generator=None):self.prepend_envs=prepend_envsor[]# [ { "var": "value" } ]self.variables=variablesor[]self.build_type=build_typeself.generator=generatordefmerged_with(self,build_desc):"""Returns a new BuildDesc by merging field content. Prefer build_desc fields to self fields for single valued field. """returnBuildDesc(self.prepend_envs+build_desc.prepend_envs,self.variables+build_desc.variables,build_desc.build_typeorself.build_type,build_desc.generatororself.generator)defenv(self):environ=os.environ.copy()forvalues_by_nameinself.prepend_envs:forvar,valueinlist(values_by_name.items()):var=var.upper()iftype(value)isunicode:value=value.encode(sys.getdefaultencoding())ifvarinenviron:environ[var]=value+os.pathsep+environ[var]else:environ[var]=valuereturnenvirondefcmake_args(self):args=["-D%s"%varforvarinself.variables]# skip build type for Visual Studio solution as it cause warningifself.build_typeand'Visual'notinself.generator:args.append("-DCMAKE_BUILD_TYPE=%s"%self.build_type)ifself.generator:args.extend(['-G',self.generator])returnargsdef__repr__(self):return"BuildDesc(%s, build_type=%s)"%(" ".join(self.cmake_args()),self.build_type)classBuildData:def__init__(self,desc,work_dir,source_dir):self.desc=descself.work_dir=work_dirself.source_dir=source_dirself.cmake_log_path=os.path.join(work_dir,'batchbuild_cmake.log')self.build_log_path=os.path.join(work_dir,'batchbuild_build.log')self.cmake_succeeded=Falseself.build_succeeded=Falsedefexecute_build(self):print('Build %s'%self.desc)self._make_new_work_dir()self.cmake_succeeded=self._generate_makefiles()ifself.cmake_succeeded:self.build_succeeded=self._build_using_makefiles()returnself.build_succeededdef_generate_makefiles(self):print(' Generating makefiles: ',end=' ')cmd=['cmake']+self.desc.cmake_args()+[os.path.abspath(self.source_dir)]succeeded=self._execute_build_subprocess(cmd,self.desc.env(),self.cmake_log_path)print('done'ifsucceededelse'FAILED')returnsucceededdef_build_using_makefiles(self):print(' Building:',end=' ')cmd=['cmake','--build',self.work_dir]ifself.desc.build_type:cmd+=['--config',self.desc.build_type]succeeded=self._execute_build_subprocess(cmd,self.desc.env(),self.build_log_path)print('done'ifsucceededelse'FAILED')returnsucceededdef_execute_build_subprocess(self,cmd,env,log_path):process=subprocess.Popen(cmd,stdout=subprocess.PIPE,stderr=subprocess.STDOUT,cwd=self.work_dir,env=env)stdout,_=process.communicate()succeeded=(process.returncode==0)withopen(log_path,'wb')asflog:log=' '.join(cmd)+'\n'+stdout+'\nExit code: %r\n'%process.returncodeflog.write(fix_eol(log))returnsucceededdef_make_new_work_dir(self):ifos.path.isdir(self.work_dir):print(' Removing work directory',self.work_dir)shutil.rmtree(self.work_dir,ignore_errors=True)ifnotos.path.isdir(self.work_dir):os.makedirs(self.work_dir)deffix_eol(stdout):"""Fixes wrong EOL produced by cmake --build on Windows (\r\r\n instead of \r\n). """returnre.sub('\r*\n',os.linesep,stdout)defload_build_variants_from_config(config_path):withopen(config_path,'rb')asfconfig:data=json.load(fconfig)variants=data['cmake_variants']build_descs_by_axis=collections.defaultdict(list)foraxisinvariants:axis_name=axis["name"]build_descs=[]if"generators"inaxis:forgenerator_datainaxis["generators"]:forgeneratoringenerator_data["generator"]:build_desc=BuildDesc(generator=generator,prepend_envs=generator_data.get("env_prepend"))build_descs.append(build_desc)elif"variables"inaxis:forvariablesinaxis["variables"]:build_desc=BuildDesc(variables=variables)build_descs.append(build_desc)elif"build_types"inaxis:forbuild_typeinaxis["build_types"]:build_desc=BuildDesc(build_type=build_type)build_descs.append(build_desc)build_descs_by_axis[axis_name].extend(build_descs)returnbuild_descs_by_axisdefgenerate_build_variants(build_descs_by_axis):"""Returns a list of BuildDesc generated for the partial BuildDesc for each axis."""axis_names=list(build_descs_by_axis.keys())build_descs=[]foraxis_name,axis_build_descsinlist(build_descs_by_axis.items()):iflen(build_descs):# for each existing build_desc and each axis build desc, create a new build_descnew_build_descs=[]forprototype_build_desc,axis_build_descinitertools.product(build_descs,axis_build_descs):new_build_descs.append(prototype_build_desc.merged_with(axis_build_desc))build_descs=new_build_descselse:build_descs=axis_build_descsreturnbuild_descsHTML_TEMPLATE=string.Template('''<html><head> <title>$title</title> <style type="text/css"> td.failed {background-color:#f08080;} td.ok {background-color:#c0eec0;} </style></head><body><table border="1"><thead> <tr> <th>Variables</th> $th_vars </tr> <tr> <th>Build type</th> $th_build_types </tr></thead><tbody>$tr_builds</tbody></table></body></html>''')defgenerate_html_report(html_report_path,builds):report_dir=os.path.dirname(html_report_path)# Vertical axis: generator# Horizontal: variables, then build_typebuilds_by_generator=collections.defaultdict(list)variables=set()build_types_by_variable=collections.defaultdict(set)build_by_pos_key={}# { (generator, var_key, build_type): build }forbuildinbuilds:builds_by_generator[build.desc.generator].append(build)var_key=tuple(sorted(build.desc.variables))variables.add(var_key)build_types_by_variable[var_key].add(build.desc.build_type)pos_key=(build.desc.generator,var_key,build.desc.build_type)build_by_pos_key[pos_key]=buildvariables=sorted(variables)th_vars=[]th_build_types=[]forvariableinvariables:build_types=sorted(build_types_by_variable[variable])nb_build_type=len(build_types_by_variable[variable])th_vars.append('<th colspan="%d">%s</th>'%(nb_build_type,cgi.escape(' '.join(variable))))forbuild_typeinbuild_types:th_build_types.append('<th>%s</th>'%cgi.escape(build_type))tr_builds=[]forgeneratorinsorted(builds_by_generator):tds=['<td>%s</td>\n'%cgi.escape(generator)]forvariableinvariables:build_types=sorted(build_types_by_variable[variable])forbuild_typeinbuild_types:pos_key=(generator,variable,build_type)build=build_by_pos_key.get(pos_key)ifbuild:cmake_status='ok'ifbuild.cmake_succeededelse'FAILED'build_status='ok'ifbuild.build_succeededelse'FAILED'cmake_log_url=os.path.relpath(build.cmake_log_path,report_dir)build_log_url=os.path.relpath(build.build_log_path,report_dir)td='<td class="%s"><a href="%s" class="%s">CMake: %s</a>'%(build_status.lower(),cmake_log_url,cmake_status.lower(),cmake_status)ifbuild.cmake_succeeded:td+='<br><a href="%s" class="%s">Build: %s</a>'%(build_log_url,build_status.lower(),build_status)td+='</td>'else:td='<td></td>'tds.append(td)tr_builds.append('<tr>%s</tr>'%'\n'.join(tds))html=HTML_TEMPLATE.substitute(title='Batch build report',th_vars=' '.join(th_vars),th_build_types=' '.join(th_build_types),tr_builds='\n'.join(tr_builds))withopen(html_report_path,'wt')asfhtml:fhtml.write(html)print('HTML report generated in:',html_report_path)defmain():usage=r"""%prog WORK_DIR SOURCE_DIR CONFIG_JSON_PATH [CONFIG2_JSON_PATH...]Build a given CMake based project located in SOURCE_DIR with multiple generators/options.dry_runas described in CONFIG_JSON_PATH building in WORK_DIR.Example of call:python devtools\batchbuild.py e:\buildbots\jsoncpp\build . devtools\agent_vmw7.json"""fromoptparseimportOptionParserparser=OptionParser(usage=usage)parser.allow_interspersed_args=True# parser.add_option('-v', '--verbose', dest="verbose", action='store_true',# help="""Be verbose.""")parser.enable_interspersed_args()options,args=parser.parse_args()iflen(args)<3:parser.error("Missing one of WORK_DIR SOURCE_DIR CONFIG_JSON_PATH.")work_dir=args[0]source_dir=args[1].rstrip('/\\')config_paths=args[2:]forconfig_pathinconfig_paths:ifnotos.path.isfile(config_path):parser.error("Can not read: %r"%config_path)# generate build variantsbuild_descs=[]forconfig_pathinconfig_paths:build_descs_by_axis=load_build_variants_from_config(config_path)build_descs.extend(generate_build_variants(build_descs_by_axis))print('Build variants (%d):'%len(build_descs))# assign build directory for each variantifnotos.path.isdir(work_dir):os.makedirs(work_dir)builds=[]withopen(os.path.join(work_dir,'matrix-dir-map.txt'),'wt')asfmatrixmap:forindex,build_descinenumerate(build_descs):build_desc_work_dir=os.path.join(work_dir,'%03d'%(index+1))builds.append(BuildData(build_desc,build_desc_work_dir,source_dir))fmatrixmap.write('%s: %s\n'%(build_desc_work_dir,build_desc))forbuildinbuilds:build.execute_build()html_report_path=os.path.join(work_dir,'batchbuild-report.html')generate_html_report(html_report_path,builds)print('Done')if__name__=='__main__':main()