Package madgraph :: Package interface :: Module madweight_interface
[hide private]
[frames] | no frames]

Source Code for Module madgraph.interface.madweight_interface

   1  ################################################################################ 
   2  # 
   3  # Copyright (c) 2011 The MadGraph Development team and Contributors 
   4  # 
   5  # This file is a part of the MadGraph 5 project, an application which  
   6  # automatically generates Feynman diagrams and matrix elements for arbitrary 
   7  # high-energy processes in the Standard Model and beyond. 
   8  # 
   9  # It is subject to the MadGraph license which should accompany this  
  10  # distribution. 
  11  # 
  12  # For more information, please visit: http://madgraph.phys.ucl.ac.be 
  13  # 
  14  ################################################################################ 
  15  """ 
  16  A user friendly interface to access all the function associated to MadWeight  
  17  """ 
  18   
  19  import logging 
  20  import os 
  21  import subprocess 
  22  import time 
  23  import glob 
  24  import math 
  25  import xml.sax.handler 
  26  import shutil 
  27  from cStringIO import StringIO 
  28   
  29  if __name__ == '__main__': 
  30      import sys 
  31      sys.path.append('/Users/omatt/Documents/eclipse/madweight/') 
  32   
  33  logger = logging.getLogger('cmdprint') 
  34   
  35  pjoin = os.path.join 
  36  try: 
  37      from madgraph import InvalidCmd, MadGraph5Error, MG5DIR 
  38      import madgraph.interface.extended_cmd as cmd 
  39      import madgraph.interface.common_run_interface as common_run 
  40       
  41       
  42      import madgraph.madweight.MW_info as MW_info 
  43      import madgraph.madweight.change_tf as change_tf 
  44      import madgraph.madweight.create_param as create_param 
  45      import madgraph.madweight.create_run as create_run 
  46      import madgraph.madweight.Cards as Cards 
  47      import madgraph.madweight.write_MadWeight as write_MadWeight 
  48      import madgraph.madweight.verif_event as verif_event 
  49      import madgraph.madweight.MW_driver as MW_driver 
  50       
  51      import madgraph.various.misc as misc 
  52      import madgraph.various.banner as banner 
  53      import madgraph.iolibs.files as files 
  54      MADEVENT = False 
  55  except ImportError, error: 
  56      logger.debug(error) 
  57      from internal import InvalidCmd, MadGraph5Error 
  58      import internal.extended_cmd as cmd 
  59      import internal.common_run_interface as common_run 
  60      import internal.madweight.MW_info as MW_info 
  61      import internal.madweight.change_tf as change_tf 
  62      import internal.madweight.create_param as create_param 
  63      import internal.madweight.create_run as create_run 
  64      import internal.madweight.Cards as Cards 
  65      import internal.madweight.write_MadWeight as write_MadWeight 
  66      import internal.madweight.verif_event as verif_event 
  67      import internal.madweight.MW_driver as MW_driver 
  68       
  69       
  70      import internal.misc as misc  
  71      import internal.banner as banner 
  72      import internal.files as files 
  73      MADEVENT = True 
  74   
  75   
  76  AlreadyRunning = common_run.AlreadyRunning 
  77   
  78  #=============================================================================== 
  79  # CmdExtended 
  80  #=============================================================================== 
81 -class CmdExtended(cmd.Cmd):
82 """Particularisation of the cmd command for MadEvent""" 83 84 #suggested list of command 85 next_possibility = { 86 'start': [], 87 } 88 89 debug_output = 'MW5_debug' 90 error_debug = 'Please report this bug on https://bugs.launchpad.net/mg5amcnlo\n' 91 error_debug += 'with MadWeight in the title of the bug report.' 92 error_debug += 'More information is found in \'%(debug)s\'.\n' 93 error_debug += 'Please attach this file to your report.' 94 95 config_debug = 'If you need help with this issue please contact us on https://answers.launchpad.net/mg5amcnlo\n' 96 97 98 keyboard_stop_msg = """stopping all operation 99 in order to quit madweight please enter exit""" 100 101 # Define the Error 102 InvalidCmd = InvalidCmd 103 ConfigurationError = MadGraph5Error 104
105 - def __init__(self, *arg, **opt):
106 """Init history and line continuation""" 107 108 # Tag allowing/forbiding question 109 self.force = False 110 111 # If possible, build an info line with current version number 112 # and date, from the VERSION text file 113 info = misc.get_pkg_info() 114 info_line = "" 115 if info and info.has_key('version') and info.has_key('date'): 116 len_version = len(info['version']) 117 len_date = len(info['date']) 118 if len_version + len_date < 30: 119 info_line = "#* VERSION %s %s %s *\n" % \ 120 (info['version'], 121 (30 - len_version - len_date) * ' ', 122 info['date']) 123 else: 124 root_path = pjoin(os.path.dirname(__file__), os.path.pardir,os.path.pardir) 125 version = open(pjoin(root_path,'MGMEVersion.txt')).readline().strip() 126 info_line = "#* VERSION %s %s *\n" % \ 127 (version, (24 - len(version)) * ' ') 128 129 # Create a header for the history file. 130 # Remember to fill in time at writeout time! 131 self.history_header = \ 132 '#************************************************************\n' + \ 133 '#* MadWeight 5 *\n' + \ 134 '#* *\n' + \ 135 "#* * * *\n" + \ 136 "#* * * * * *\n" + \ 137 "#* * * * * 5 * * * * *\n" + \ 138 "#* * * * * *\n" + \ 139 "#* * * *\n" + \ 140 "#* *\n" + \ 141 "#* *\n" + \ 142 info_line + \ 143 "#* *\n" + \ 144 "#* The MadGraph Development Team - Please visit us at *\n" + \ 145 "#* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \ 146 '#* *\n' + \ 147 '#************************************************************\n' + \ 148 '#* *\n' + \ 149 '#* Command File for MadWeight *\n' + \ 150 '#* *\n' + \ 151 '#* run as ./bin/madweight.py FILENAME *\n' + \ 152 '#* *\n' + \ 153 '#************************************************************\n' 154 155 if info_line: 156 info_line = info_line[1:] 157 158 logger.info(\ 159 "************************************************************\n" + \ 160 "* *\n" + \ 161 "* W E L C O M E to M A D G R A P H 5 *\n" + \ 162 "* M A D W E I G H T *\n" + \ 163 "* *\n" + \ 164 "* * * *\n" + \ 165 "* * * * * *\n" + \ 166 "* * * * * 5 * * * * *\n" + \ 167 "* * * * * *\n" + \ 168 "* * * *\n" + \ 169 "* *\n" + \ 170 info_line + \ 171 "* *\n" + \ 172 "* The MadGraph Development Team - Please visit us at *\n" + \ 173 "* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \ 174 "* *\n" + \ 175 "* Type 'help' for in-line help. *\n" + \ 176 "* *\n" + \ 177 "************************************************************") 178 179 cmd.Cmd.__init__(self, *arg, **opt)
180
181 -class HelpToCmd(object):
182
183 - def help_collect(self):
184 logger.info('collect [-refine]') 185 logger.info(' combine the results of the jobs launched on the cluster.') 186 logger.info(' This creates three type of output files:') 187 logger.info(' - weights.out [weight for event/card]') 188 logger.info(' - unnormalize-likelihood.out [-\sum ln(Weight)]') 189 logger.info(' - output.xml [additional information]') 190 logger.info('') 191 logger.info(' The option \'-refine\' is to be added if this is not the first') 192 logger.info(' cluster submission. Otherwise previous run submission will be lost.')
193
194 - def help_define_transfer_fct(self):
195 """help for define transfer_fct""" 196 logger.info(' Modify the current transfer functions') 197 logger.info(' If no argument provided a question showing the list of possibilities.') 198 logger.info(' will be ask. If the TF is provided as argument, no question is asked.')
199 200
201 -class CompleteForCmd(object):
202 203
204 - def complete_collect(self,text, line, begidx, endidx):
205 """ complete the collect command""" 206 args = self.split_arg(line[0:begidx]) 207 208 return self.list_completion(text,['-refine','-f','--refine'], line)
209
210 - def complete_define_transfer_fct(self, text, line, begidx, endidx):
211 """ complete the define_transfer_fct """ 212 213 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data') 214 listdir=os.listdir(path) 215 args = self.split_arg(line[0:begidx]) 216 if len(args) == 1: 217 218 possibilities = [content[3:-4] for content in listdir \ 219 if (content.startswith('TF') and content.endswith('dat'))] 220 return self.list_completion(text, possibilities, line)
221 222 #=============================================================================== 223 # MadWeightCmd 224 #===============================================================================
225 -class MadWeightCmd(CmdExtended, HelpToCmd, CompleteForCmd, common_run.CommonRunCmd):
226 227 _set_options = [] 228 prompt = 'MadWeight5>' 229 helporder = ['MadWeight Function', 'Documented commands', 'Advanced commands'] 230
231 - def remove_fct(self):
232 """Not in help: remove fct""" 233 return None
234 235 do_decay_events = remove_fct 236 do_delphes = remove_fct 237 do_pgs = remove_fct 238 239 240 ############################################################################
241 - def __init__(self, me_dir = None, options={}, *completekey, **stdin):
242 """ add information to the cmd """ 243 244 CmdExtended.__init__(self, *completekey, **stdin) 245 common_run.CommonRunCmd.__init__(self, me_dir, options) 246 self.configured = 0 # time at which the last option configuration occur
247
248 - def do_quit(self, *args, **opts):
249 common_run.CommonRunCmd.do_quit(self, *args, **opts) 250 CmdExtended.do_quit(self, *args, **opts) 251 return True
252
253 - def configure(self):
254 os.chdir(pjoin(self.me_dir)) 255 self.__CMD__initpos = self.me_dir 256 257 time_mod = max([os.path.getctime(pjoin(self.me_dir,'Cards','run_card.dat')), 258 os.path.getctime(pjoin(self.me_dir,'Cards','MadWeight_card.dat'))]) 259 260 261 if self.configured > time_mod and \ 262 hasattr(self,'MWparam') and hasattr(self,'run_card'): 263 return 264 self.configured = time_mod 265 266 267 268 self.MWparam = MW_info.MW_info(pjoin(self.me_dir,'Cards','MadWeight_card.dat')) 269 run_card = pjoin(self.me_dir, 'Cards','run_card.dat') 270 self.run_card = banner.RunCard(run_card) 271 272 if self.options['run_mode'] == 0: 273 self.exec_cmd('set run_mode 2 --no_save') 274 self.exec_cmd('set nb_core 1 --no_save') 275 if not self.options['cluster_temp_path']: 276 if self.options['run_mode'] == 2: 277 logger.info('Options cluster_temp_path is required for MW run. Trying to run with /tmp', 278 '$MG:BOLD') 279 self.exec_cmd('set cluster_temp_path /tmp --no_save') 280 elif self.options['cluster_type'] != 'condor': 281 raise Exception, 'cluster_temp_path needs to be define for MW. Please retry.'
282
283 - def do_define_transfer_fct(self, line):
284 """MadWeight Function:Define the current transfer function""" 285 286 with misc.chdir(self.me_dir): 287 self.configure() 288 args = self.split_arg(line) 289 290 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data') 291 listdir=os.listdir(path) 292 question = 'Please choose your transfer_function between\n' 293 possibilities = [content[3:-4] for content in listdir \ 294 if (content.startswith('TF') and content.endswith('dat'))] 295 for i, tfname in enumerate(possibilities): 296 question += ' %s / %s\n' % (i, tfname) 297 possibilities += range(len(possibilities)) 298 299 if args and args[0] in possibilities: 300 tfname = args[0] 301 else: 302 tfname = self.ask(question, 'dbl_gauss_pt_jet', possibilities) 303 if tfname.isdigit(): 304 tfname = possibilities[int(tfname)] 305 306 P_dir, MW_dir = MW_info.detect_SubProcess(P_mode=1) 307 os.chdir('./Source/MadWeight/transfer_function') 308 change_tf.create_TF_main(tfname,0, MW_dir)
309 310
311 - def do_treatcards(self, line):
312 """MadWeight Function:create the various param_card // compile input for the run_card""" 313 self.configure() 314 args = self.split_arg(line) 315 316 create_param.Param_card(run_name=self.MWparam) 317 self.MWparam.update_nb_card() 318 Cards.create_include_file(self.MWparam) 319 create_run.update_cuts_status(self.MWparam)
320
321 - def do_get_integration_channel(self, line):
322 """MadWeight Function:analyze the cards/diagram to find an way to integrate efficiently""" 323 self.configure() 324 args = self.split_arg(line) 325 326 write_MadWeight.create_all_fortran_code(self.MWparam)
327
328 - def do_compile(self, line, refine=False):
329 """MadWeight Function:compile the code""" 330 self.configure() 331 332 misc.compile(arg=["../lib/libtools.a"], cwd=pjoin(self.me_dir,'Source')) 333 misc.compile(arg=["../lib/libblocks.a"], cwd=pjoin(self.me_dir,'Source')) 334 misc.compile(arg=["../lib/libTF.a"], cwd=pjoin(self.me_dir,'Source')) 335 misc.compile(arg=["../lib/libpdf.a"], cwd=pjoin(self.me_dir,'Source')) 336 misc.compile(arg=["../lib/libdhelas.a"], cwd=pjoin(self.me_dir,'Source')) 337 misc.compile(arg=["../lib/libmodel.a"], cwd=pjoin(self.me_dir,'Source')) 338 misc.compile(arg=["../lib/libgeneric.a"], cwd=pjoin(self.me_dir,'Source')) 339 misc.compile(arg=["../lib/libcernlib.a"], cwd=pjoin(self.me_dir,'Source')) 340 341 # 342 # here check validity of some parameters 343 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['montecarlo_perm']=='t': 344 raise Exception, 'Cannot use mint if monte carlo over permutations' 345 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['use_sobol']=='t': 346 raise Exception, 'sobol generator with mint not implemented' 347 348 349 350 for MW_dir in self.MWparam.MW_listdir: 351 logger.info('compile %s' %MW_dir) 352 pdir = pjoin(self.me_dir,'SubProcesses',MW_dir) 353 if refine and os.path.exists(pjoin(pdir, 'initialization.o')): 354 os.remove(pjoin(pdir, 'initialization.o')) 355 if refine and os.path.exists(pjoin(pdir, 'comp_madweight')): 356 os.remove(pjoin(pdir, 'comp_madweight')) 357 misc.compile(cwd=pdir) 358 if not os.path.exists(pjoin(pdir, 'comp_madweight')): 359 raise Exception, 'compilation fails' 360 logger.info('MadWeight code has been compiled.')
361 362
363 - def do_check_events(self, line):
364 """MadWeight Function: check that the events are valid 365 and write the events to MG mapping""" 366 self.configure() 367 evt_file = pjoin(self.me_dir,'Events','input.lhco') 368 if not os.path.exists(evt_file): 369 question = 'Which LHCO file do you want to use?' 370 default = '' 371 if os.path.exists('%s.gz' % evt_file): 372 input_file = '%s.gz' % evt_file 373 else: 374 input_file = self.ask(question, default, path_msg='valid path') 375 376 if not input_file: 377 raise self.InvalidCmd('Please specify a valid LHCO File') 378 379 if input_file.endswith('.gz'): 380 misc.gunzip(input_file, keep=True, stdout=evt_file) 381 else: 382 files.cp(input_file, evt_file) 383 384 verif_event.verif_event(self.MWparam)
385 386 387 388 389 390
391 - def check_launch_jobs(self, args):
392 """format the argument to retrun a list with two argument, 393 The first corresponding to the fact if we need te create the output dir 394 The second if we can launch the job on the cluster.""" 395 396 if not args: 397 #use default 398 args[:] = [True, True] 399 return 400 else: 401 create_dir = True 402 launch = True 403 for arg in args: 404 if arg.count('=') !=1 : 405 logger.warning('command launch_jobs does not recognized argument %s. This argument is ignored' % arg) 406 restrict, value = arg.split('=') 407 if restrict == '--create_dir=': 408 if value in self.True: 409 create_dir = True 410 else: 411 create_dir = False 412 elif restrict == '--submit=': 413 if value in self.True: 414 launch = True 415 else: 416 launch = False 417 args[:] = [create_dir, launch] 418 return
419
420 - def do_submit_jobs(self, line):
421 """MadWeight Function:Submitting the jobs to the cluster""" 422 423 424 self.configure() 425 self.clean_old_run(keep_event=True) 426 args = self.split_arg(line) 427 self.check_launch_jobs(args) 428 # now args is of the type [True True] 429 create_dir, launch_jobs = args[0], args[1] 430 431 for nb_card in self.MWparam.actif_param: 432 for dirname in self.MWparam.MW_listdir: 433 nb_job = self.MWparam.nb_event_MW[dirname] 434 if self.MWparam['mw_run']['nb_event_by_node'] > 1: 435 nb_job = 1+ (nb_job-1) // self.MWparam['mw_run']['nb_event_by_node'] 436 437 for event_sample in range(nb_job): 438 self.submit_job(dirname, nb_card, event_sample) 439 440 starttime = time.time() 441 #logger.info(' Waiting for submitted jobs to complete') 442 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'), 443 starttime=starttime, level='madweight', update_results=False) 444 445 try: 446 self.cluster.wait(self.me_dir, update_status) 447 except Exception: 448 self.cluster.remove() 449 raise 450 except KeyboardInterrupt: 451 if not self.force: 452 ans = self.ask('Error detected. Do you want to clean the queue?', 453 default = 'y', choices=['y','n']) 454 else: 455 ans = 'y' 456 if ans == 'y': 457 self.cluster.remove() 458 raise
459
460 - def submit_job(self, dirname, nb_card, sample_nb, evt_file=None, restrict_evt=[]):
461 """launch on the cluster the job which creates the computation""" 462 463 input_files = [pjoin(self.me_dir, 'SubProcesses', dirname, 'comp_madweight'), 464 pjoin(self.me_dir, 'Cards', 'param_card_%i.dat' % nb_card), 465 self.get_pdf_input_filename(), 466 pjoin(self.me_dir, 'Cards', 'ident_card.dat'), 467 pjoin(self.me_dir, 'Cards', 'run_card.dat') 468 ] 469 470 # add event_file: 471 if not evt_file: 472 evt_file = (sample_nb // self.MWparam['mw_run']['event_packing']) 473 evt = 'verif_%i.lhco' % evt_file 474 first_event = (sample_nb % self.MWparam['mw_run']['event_packing']) * self.MWparam['mw_run']['nb_event_by_node'] 475 name = self.MWparam.name 476 input_files.append(pjoin(self.me_dir, 'SubProcesses', dirname, name, evt)) 477 478 if restrict_evt: 479 restrict_path = pjoin(self.me_dir, 'SubProcesses', dirname, name, 480 'restrict%i_%i.dat' % (nb_card,evt_file)) 481 input_files.append(restrict_path) 482 #open(restrict_path, 'w').write(' '.join(map(str, restrict_evt))) 483 484 # Need to add PDF (maybe also symfact, ...) ? 485 486 output_file = ['output_%s_%s.xml' % (nb_card, sample_nb)] 487 exe = pjoin(self.me_dir, 'bin', 'internal', 'madweight', 'MW_driver.py') 488 489 # expected args: card_nb, first_event, nb_event, evt, mw_int_points, log_level 490 args = [str(nb_card), str(first_event), 491 str(self.MWparam['mw_run']['nb_event_by_node']) ,evt, 492 str(self.MWparam['mw_run']['mw_int_points']), 493 self.MWparam['mw_run']['log_level'], str(sample_nb)] 494 cwd = pjoin(self.me_dir, 'SubProcesses', dirname, name) 495 # Ensure that the code is working ONLY if TEMP_CLUSTER_PATH is define 496 if self.options['run_mode'] == 0: 497 raise Exception , 'need to check the validity' 498 else: 499 # ensure that this is running with NO central disk !!! 500 if not self.options['cluster_temp_path'] and not self.options['cluster_type'] == 'condor': 501 raise self.ConfigurationError, 'MadWeight requires temp_cluster_path options to be define' 502 self.cluster.submit2(exe, args, cwd, input_files=input_files, output_files=output_file)
503 504 505
506 - def check_collect(self, args):
507 """ """ 508 509 if len(args) >1: 510 self.help_collect() 511 raise self.InvalidCmd, 'Invalid Command format' 512 elif len(args) == 1: 513 if args not in ['-refine', '--refine']: 514 args[0] = '-refine' 515 else: 516 self.help_collect() 517 raise self.InvalidCmd, 'Invalid Command format'
518
519 - def do_collect(self, line):
520 """MadWeight Function: making the collect of the results""" 521 522 self.configure() 523 args = self.split_arg(line) 524 self.check_collect(args) 525 xml_reader = MWParserXML() 526 527 name = self.MWparam.name 528 # 1. Concatanate the file. ############################################# 529 out_dir = pjoin(self.me_dir, 'Events', name) 530 if '-refine' in args: 531 out_path = pjoin(out_dir, 'refine.xml') 532 else: 533 out_path = pjoin(out_dir, 'output.xml') 534 if os.path.exists(out_path): 535 logger.warning('Output file already exists. Current one will be tagged with _old suffix') 536 logger.warning('Run "collect -refine to instead update your current results."') 537 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 538 files.mv(pjoin(out_dir, 'weights.out'), pjoin(out_dir, 'weights.out')) 539 for MWdir in self.MWparam.MW_listdir: 540 out_dir = pjoin(self.me_dir, 'Events', name, MWdir) 541 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 542 out_dir = pjoin(self.me_dir, 'Events', name) 543 544 fsock = open(out_path, 'w') 545 fsock.write('<madweight>\n<banner>\n') 546 # BANNER 547 for card in ['proc_card_mg5.dat','MadWeight_card.dat','transfer_card.dat','param_card.dat','run_card.dat']: 548 cname = card[:-4] 549 fsock.write('<%s>\n' % cname) 550 fsock.write(open(pjoin(self.me_dir,'Cards',card)).read().replace('<','!>')) 551 fsock.write('</%s>\n' % cname) 552 fsock.write('</banner>\n') 553 at_least_one = False 554 for MWdir in self.MWparam.MW_listdir: 555 out_dir = pjoin(self.me_dir, 'Events', name, MWdir) 556 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, name) 557 if not os.path.exists(out_dir): 558 os.mkdir(out_dir) 559 if '-refine' in args: 560 out_path = pjoin(out_dir, 'refine.xml') 561 else: 562 out_path = pjoin(out_dir, 'output.xml') 563 fsock2 = open(out_path,'w') 564 fsock.write('<subprocess id=\'%s\'>\n' % MWdir) 565 fsock2.write('<subprocess id=\'%s\'>\n' % MWdir) 566 for output in misc.glob('output_*_*.xml', input_dir): 567 at_least_one = True 568 text = open(output).read() 569 fsock2.write(text) 570 fsock.write(text) 571 os.remove(output) 572 fsock.write('</subprocess>\n') 573 fsock2.write('</subprocess>\n') 574 fsock2.close() 575 fsock.write('\n</madweight>\n') 576 fsock.close() 577 # 2. Special treatment for refine mode 578 if '-refine' in args: 579 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) 580 for MWdir in self.MWparam.MW_listdir: 581 out_dir = pjoin(self.me_dir, 'Events',name, MWdir) 582 ref_output = xml_reader2.read_file(pjoin(out_dir, 'refine.xml')) 583 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level']) 584 base_output = xml_reader2.read_file(pjoin(out_dir, 'output.xml')) 585 586 base_output.refine(ref_output) 587 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 588 base_output.write(pjoin(out_dir, 'output.xml'), MWdir) 589 elif not at_least_one: 590 logger.warning("Nothing to collect restore _old file as current.") 591 out_dir = pjoin(self.me_dir, 'Events', name) 592 files.mv(pjoin(out_dir, 'output_old.xml'), pjoin(out_dir, 'output.xml')) 593 files.mv(pjoin(out_dir, 'weights_old.out'), pjoin(out_dir, 'weights.out')) 594 for MWdir in self.MWparam.MW_listdir: 595 out_dir = pjoin(self.me_dir, 'Events', name, MWdir) 596 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml')) 597 598 599 600 # 3. Read the (final) log file for extracting data 601 total = {} 602 likelihood = {} 603 err_likelihood = {} 604 cards = set() 605 events = set() 606 tf_sets = set() 607 for MW_dir in self.MWparam.MW_listdir: 608 out_dir = pjoin(self.me_dir, 'Events', name, MW_dir) 609 xml_reader = MWParserXML() 610 data = xml_reader.read_file(pjoin(out_dir, 'output.xml')) 611 # 612 log_level = self.MWparam['mw_run']['log_level'] 613 generator = ((int(i),j,int(k),data[i][j][k]) for i in data 614 for j in data[i] 615 for k in data[i][j]) 616 for card, event, tf_set, obj in generator: 617 # update the full list of events/cards 618 cards.add(card) 619 events.add(event) 620 tf_sets.add(tf_set) 621 # now compute the associate value, error[square] 622 if (card,event, tf_set) in total: 623 value, error = total[(card, event, tf_set)] 624 else: 625 value, error = 0, 0 626 obj.calculate_total() 627 value, error = (value + obj.value, error + obj.error**2) 628 total[(card, event, tf_set)] = (value, error) 629 if tf_set == 1: 630 if value: 631 if card not in likelihood: 632 likelihood[card], err_likelihood[card] = 0, 0 633 likelihood[card] -= math.log(value) 634 err_likelihood[card] += error / value 635 else: 636 likelihood[card] = float('Inf') 637 err_likelihood[card] = float('nan') 638 639 640 # write the weights file: 641 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'w') 642 logger.info('Write output file with weight information: %s' % fsock.name) 643 fsock.write('# Weight (un-normalize) for each card/event/set of transfer fct\n') 644 fsock.write('# format: LHCO_event_number card_id tf_set_id value integration_error\n') 645 events = list(events) 646 events.sort() 647 cards = list(cards) 648 cards.sort() 649 tf_sets = list(tf_sets) 650 tf_sets.sort() 651 for event in events: 652 for card in cards: 653 for tf_set in tf_sets: 654 try: 655 value, error = total[(card, event,tf_set)] 656 except KeyError: 657 continue 658 error = math.sqrt(error) 659 fsock.write('%s %s %s %s %s \n' % (event.replace('@', ' '), card, tf_set, value, error)) 660 661 # write the likelihood file: 662 fsock = open(pjoin(self.me_dir, 'Events', name, 'un-normalized_likelihood.out'), 'w') 663 fsock.write('# Warning: this Likelihood needs a bin by bin normalization !\n') 664 fsock.write('# IF more than one set of transfer function are define. ONLY the first one is ') 665 fsock.write('# include in this file.') 666 fsock.write('# format: card_id value integration_error\n') 667 for card in cards: 668 value, error = likelihood[card], err_likelihood[card] 669 error = math.sqrt(error) 670 fsock.write('%s %s %s \n' % (card, value, error))
671 672
673 - def do_clean(self, line):
674 """MadWeight Function: syntax: clean [XXX] 675 clean the previous run XXX (the last one by default)""" 676 677 args = self.split_arg(line) 678 self.configure() 679 if len(args) == 0: 680 name = self.MWparam.name 681 else: 682 name = args[0] 683 684 ans = self.ask('Do you want to remove Events/%s directory?', 'y',['y','n']) 685 if ans == 'y': 686 try: 687 shutil.rmtree(pjoin(self.me_dir, 'Events', name)) 688 except Exception, error: 689 logger.warning(error) 690 for Pdir in self.MWparam.MW_listdir: 691 try: 692 shutil.rmtree(pjoin(self.me_dir, 'SubProcesses', Pdir, name)) 693 except Exception, error: 694 logger.warning(error)
695
696 - def ask_edit_cards(self, cards, *arg, **opts):
697 super(MadWeightCmd, self).ask_edit_cards(cards, *arg, **opts) 698 self.configured = 0 699 self.configure()
700
701 - def do_launch(self, line):
702 """MadWeight Function:run the full suite of commands""" 703 704 args = self.split_arg(line) 705 706 #if not os.path.exists(pjoin(self.me_dir, 'Cards','transfer_card.dat')): 707 # self.exec_cmd('define_transfer_fct') 708 709 cards = ['param_card.dat', 'run_card.dat', 'madweight_card.dat', 710 'transfer_card.dat', 'input.lhco'] 711 if not self.force: 712 self.ask_edit_cards(cards, mode='fixed', plot=False) 713 else: 714 self.configured = 0 715 self.configure() 716 with misc.chdir(self.me_dir): 717 if not os.path.exists(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile'])): 718 raise self.InvalidCmd('Please specify a valid LHCO File') 719 if pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']) not in \ 720 [pjoin(self.me_dir, 'Events', 'input.lhco'), pjoin(self.me_dir, 'Events', 'input.lhco.gz')]: 721 zipped = self.MWparam['mw_run']['inputfile'].endswith('.gz') 722 if zipped: 723 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']), 724 pjoin(self.me_dir, 'Events', 'input.lhco.gz')) 725 if os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')): 726 os.remove(pjoin(self.me_dir, 'Events', 'input.lhco')) 727 else: 728 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']), 729 pjoin(self.me_dir, 'Events', 'input.lhco')) 730 731 if not (os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')) or \ 732 os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco.gz'))): 733 raise self.InvalidCmd('Please specify a valid LHCO File') 734 735 self.exec_cmd('treatcards') 736 self.exec_cmd('get_integration_channel') 737 self.exec_cmd('compile') 738 self.exec_cmd('check_events') 739 self.exec_cmd('submit_jobs') 740 self.exec_cmd('collect')
741 742
743 - def check_refine(self, args):
744 """check the argument validity""" 745 746 if len(args) != 1: 747 raise self.InvalidCmd('refine requires a single argument') 748 749 try: 750 args[0] = float(args[0]) 751 except Exception: 752 raise self.InvalidCmd('First argument of refine command should be a number') 753 754 if args[0] < 0 or args[0]>1: 755 raise self.InvalidCmd('The first argument should be a number between 0 and 1.')
756
757 - def clean_old_run(self, keep_event=False):
758 759 for MWdir in self.MWparam.MW_listdir: 760 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, self.MWparam.name) 761 if not os.path.exists(input_dir): 762 continue 763 for filename in os.listdir(input_dir): 764 if keep_event and filename.startswith('verif'): 765 continue 766 os.remove(pjoin(input_dir, filename))
767 768 769
770 - def do_refine(self, line):
771 """MadWeight Function:syntax: refine X 772 relaunch the computation of the weight which have a precision lower than X""" 773 args = self.split_arg(line) 774 self.check_refine(args) 775 self.configure() 776 777 self.clean_old_run(keep_event=True) 778 if not self.force: 779 cards = ['madweight_card.dat'] 780 self.ask_edit_cards(cards, mode='fixed', plot=False) 781 self.exec_cmd("treatcards") 782 self.do_compile('', refine=True) # force re-compilation 783 784 nb_events_by_file = self.MWparam['mw_run']['nb_event_by_node'] * self.MWparam['mw_run']['event_packing'] 785 asked_events = self.MWparam['mw_run']['nb_exp_events'] 786 787 precision = args[0] 788 name = self.MWparam.name 789 allow_refine = [] 790 # events/cards to refine 791 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'r') 792 for line in fsock: 793 if '#' in line: 794 line = line.split('#')[0] 795 line = line.split() 796 if len(line) == 5: 797 lhco_nb, card_nb, tf_set, value, error = line 798 else: 799 continue 800 if tf_set != '1': 801 continue 802 if float(value) * precision < float(error): 803 allow_refine.append((int(card_nb), lhco_nb)) 804 logger.info("%s selected jobs for the refinment." % len(allow_refine)) 805 for MWdir in self.MWparam.MW_listdir: 806 # We need to know in which file are written all the relevant event 807 event_to_file = {} 808 for evt_nb in range(asked_events//nb_events_by_file +1): 809 evt = 'verif_%i.lhco' % evt_nb 810 for line in open(pjoin(self.me_dir, 'SubProcesses', MWdir, name, evt)): 811 split = line.split() 812 if len(split) == 3: 813 event_to_file[split[1]] = evt_nb 814 to_refine = {} 815 out_dir = pjoin(self.me_dir, 'Events',name, MWdir) 816 xml_reader = MWParserXML(keep_level='weight') 817 data = xml_reader.read_file(pjoin(out_dir, 'output.xml')) 818 generator = ((int(i),j,int(k),data[i][j][k]) for i in data 819 for j in data[i] 820 for k in data[i][j]) 821 for card, event, tf_set, obj in generator: 822 if tf_set != 1: 823 continue 824 value, error = obj.value, obj.error 825 if value * precision < error: 826 if card not in to_refine: 827 to_refine[card] = [] 828 if (card,event) in allow_refine: 829 to_refine[card].append(event) 830 if to_refine: 831 self.resubmit(MWdir, to_refine, event_to_file) 832 833 # control 834 starttime = time.time() 835 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'), 836 starttime=starttime, level='madweight', update_results=False) 837 try: 838 self.cluster.wait(self.me_dir, update_status) 839 except Exception: 840 self.cluster.remove() 841 raise 842 except KeyboardInterrupt: 843 if not self.force: 844 ans = self.ask('Error detected. Do you want to clean the queue?', 845 default = 'y', choices=['y','n']) 846 else: 847 ans = 'y' 848 if ans == 'y': 849 self.cluster.remove() 850 raise 851 852 self.do_collect('-refine')
853
854 - def resubmit(self, M_path, to_refine, event_to_file):
855 """resubmit various jobs""" 856 857 for card, event_list in to_refine.items(): 858 packets = {} 859 for event in event_list: 860 evt_nb_file = event_to_file[event] 861 if evt_nb_file in packets: 862 packets[evt_nb_file].append(event) 863 else: 864 packets[evt_nb_file] = [event] 865 866 # evt_file = (sample_nb // self.MWparam['mw_run']['event_packing']) 867 # open(restrict_path, 'w').write(' '.join(map(str, restrict_evt))) 868 869 max_evts = self.MWparam['mw_run']['nb_event_by_node'] 870 for evt_nb, evt_list in packets.items(): 871 restrict_path = pjoin(self.me_dir, 'SubProcesses', M_path, self.MWparam.name, 872 'restrict%i_%i.dat' % (card,evt_nb_file)) 873 open(restrict_path, 'w').write(' '.join(map(str, evt_list))) 874 875 nb_weights = len(evt_list) 876 for i in range(1+ (nb_weights-1)//max_evts): 877 #sub_list = evt_list[max_evts * i: max_evts * (i+1)] 878 self.submit_job(M_path, card, i, evt_file=evt_nb, 879 restrict_evt=True)
880 881 882 883 #=============================================================================== 884 # MadEventCmd 885 #===============================================================================
886 -class MadWeightCmdShell(MadWeightCmd, cmd.CmdShell):
887 """The command line processor of MadGraph""" 888 pass
889 890
891 -class CollectObj(dict):
892 pass 893 894 #2 #############################################################################
895 - def refine(self, other):
896 897 for card, CardDATA in other.items(): 898 if card not in self: 899 self[card] = other[card] 900 continue 901 902 for event, obj2 in CardDATA.items(): 903 self[card][event] = obj2
904 905 906
907 - def write(self, out_path, MWdir):
908 """ """ 909 fsock = open(out_path, 'w') 910 fsock.write('<subprocess id=\'%s\'>\n' % MWdir) 911 for card in self: 912 fsock.write('<card id=\'%s\'>\n' % card) 913 for event in self[card].values(): 914 event.write(fsock) 915 fsock.write('</card>\n') 916 917 fsock.write('</subprocess>') 918 fsock.close()
919 920 921 #1 #################################################################################
922 -class MWParserXML(xml.sax.handler.ContentHandler):
923 """ This class will organize in python obect the TF_param.dat file 924 (written in xml) 925 """ 926 927 #2 #############################################################################
928 - def __init__(self, keep_level='weight'):
929 self.in_el = {'process': '', 'card':'', 'event':'', 'tfset':'' , 930 'permutation':'', 'channel':'','full':''} 931 if keep_level in ['weight', 'event', 'debug']: 932 keep_level = 'tfset' 933 self.all_event = {} 934 self.keep_level = keep_level 935 self.buffer='' 936 self.output = CollectObj()
937 938 939 #2 #############################################################################
940 - def startElement(self, name, attributes):
941 942 # if in lower level than needed skip the collection of data 943 if self.in_el[self.keep_level] != '' or name == 'log': 944 return 945 946 obj_class = {'event': MW_driver.Weight, 'permutation':MW_driver.Permutation, 947 'channel': MW_driver.Channel, 'tfset':MW_driver.TFsets} 948 949 950 if name == "process": 951 pass 952 elif name == 'card': 953 id = attributes['id'] 954 if id not in self.output: 955 self.output[id] = {} 956 self.in_el[name] = self.output[id] 957 self.curr_card_id = id 958 elif name == 'event': 959 id = attributes['id'] 960 value = float(attributes['value']) 961 error = float(attributes['error']) 962 card = self.curr_card_id 963 if ((card,id) in self.all_event): 964 data = self.all_event[(card,id)] 965 else: 966 data = MW_driver.Weight(id, self.keep_level) 967 self.all_event[(card,id)] = data 968 data.value = value 969 data.error = error 970 self.in_el['event'] = data 971 # assign it in the mother: 972 card = self.in_el['card'] 973 card[id] = data 974 elif name == 'tfset': 975 id = attributes['id'] 976 value = float(attributes['value']) 977 error = float(attributes['error']) 978 data = MW_driver.TFsets(id) 979 data.value = value 980 data.error = error 981 data.add('0', '', value, error, '') 982 #assign it to the mother: 983 event = self.in_el['event'] 984 event[id] = data 985 self.in_el['tfset'] = data 986 elif name == 'permutation': 987 tfset = self.in_el['tfset'] 988 989 id = attributes['id'] 990 value = float(attributes['value']) 991 error = float(attributes['error']) 992 data = MW_driver.Permutation(id, '') 993 data.value = value 994 data.error = error 995 self.in_el['permutation'] = data 996 # assign it in the mother: 997 tfset[id] = data 998 elif name == 'channel': 999 id = attributes['id'] 1000 value = float(attributes['value']) 1001 error = float(attributes['error']) 1002 data = MW_driver.Channel(id, value, error) 1003 self.in_el['channel'] = data 1004 # assign it in the mother: 1005 permutation = self.in_el['permutation'] 1006 permutation[id] = data 1007 elif name in ['log','subprocess','br']: 1008 pass 1009 else: 1010 raise Exception, name 1011 if name != 'br': 1012 self.text = StringIO()
1013
1014 - def characters(self, content):
1015 self.text.write(content)
1016
1017 - def endElement(self, name):
1018 if name == 'log': 1019 data = self.in_el['event'] 1020 data.log = self.text.getvalue() 1021 1022 self.in_el[name] = ''
1023 1024 #2 #############################################################################
1025 - def read_file(self,filepos):
1026 """ parse the file and fulfill the object """ 1027 self.output = CollectObj() 1028 parser = xml.sax.make_parser( ) 1029 parser.setContentHandler(self) 1030 parser.parse(filepos) 1031 return self.output
1032