1
2
3
4
5
6
7
8
9
10
11
12
13
14
15 """
16 A user friendly interface to access all the function associated to MadWeight
17 """
18
19 import logging
20 import os
21 import subprocess
22 import time
23 import glob
24 import math
25 import xml.sax.handler
26 import shutil
27 from cStringIO import StringIO
28
29 if __name__ == '__main__':
30 import sys
31 sys.path.append('/Users/omatt/Documents/eclipse/madweight/')
32
33 logger = logging.getLogger('cmdprint')
34
35 pjoin = os.path.join
36 try:
37 from madgraph import InvalidCmd, MadGraph5Error, MG5DIR
38 import madgraph.interface.extended_cmd as cmd
39 import madgraph.interface.common_run_interface as common_run
40
41
42 import madgraph.madweight.MW_info as MW_info
43 import madgraph.madweight.change_tf as change_tf
44 import madgraph.madweight.create_param as create_param
45 import madgraph.madweight.create_run as create_run
46 import madgraph.madweight.Cards as Cards
47 import madgraph.madweight.write_MadWeight as write_MadWeight
48 import madgraph.madweight.verif_event as verif_event
49 import madgraph.madweight.MW_driver as MW_driver
50
51 import madgraph.various.misc as misc
52 import madgraph.various.banner as banner
53 import madgraph.iolibs.files as files
54 MADEVENT = False
55 except ImportError, error:
56 logger.debug(error)
57 from internal import InvalidCmd, MadGraph5Error
58 import internal.extended_cmd as cmd
59 import internal.common_run_interface as common_run
60 import internal.madweight.MW_info as MW_info
61 import internal.madweight.change_tf as change_tf
62 import internal.madweight.create_param as create_param
63 import internal.madweight.create_run as create_run
64 import internal.madweight.Cards as Cards
65 import internal.madweight.write_MadWeight as write_MadWeight
66 import internal.madweight.verif_event as verif_event
67 import internal.madweight.MW_driver as MW_driver
68
69
70 import internal.misc as misc
71 import internal.banner as banner
72 import internal.files as files
73 MADEVENT = True
74
75
76 AlreadyRunning = common_run.AlreadyRunning
77
78
79
80
82 """Particularisation of the cmd command for MadEvent"""
83
84
85 next_possibility = {
86 'start': [],
87 }
88
89 debug_output = 'MW5_debug'
90 error_debug = 'Please report this bug on https://bugs.launchpad.net/mg5amcnlo\n'
91 error_debug += 'with MadWeight in the title of the bug report.'
92 error_debug += 'More information is found in \'%(debug)s\'.\n'
93 error_debug += 'Please attach this file to your report.'
94
95 config_debug = 'If you need help with this issue please contact us on https://answers.launchpad.net/mg5amcnlo\n'
96
97
98 keyboard_stop_msg = """stopping all operation
99 in order to quit madweight please enter exit"""
100
101
102 InvalidCmd = InvalidCmd
103 ConfigurationError = MadGraph5Error
104
106 """Init history and line continuation"""
107
108
109 self.force = False
110
111
112
113 info = misc.get_pkg_info()
114 info_line = ""
115 if info and info.has_key('version') and info.has_key('date'):
116 len_version = len(info['version'])
117 len_date = len(info['date'])
118 if len_version + len_date < 30:
119 info_line = "#* VERSION %s %s %s *\n" % \
120 (info['version'],
121 (30 - len_version - len_date) * ' ',
122 info['date'])
123 else:
124 root_path = pjoin(os.path.dirname(__file__), os.path.pardir,os.path.pardir)
125 version = open(pjoin(root_path,'MGMEVersion.txt')).readline().strip()
126 info_line = "#* VERSION %s %s *\n" % \
127 (version, (24 - len(version)) * ' ')
128
129
130
131 self.history_header = \
132 '#************************************************************\n' + \
133 '#* MadWeight 5 *\n' + \
134 '#* *\n' + \
135 "#* * * *\n" + \
136 "#* * * * * *\n" + \
137 "#* * * * * 5 * * * * *\n" + \
138 "#* * * * * *\n" + \
139 "#* * * *\n" + \
140 "#* *\n" + \
141 "#* *\n" + \
142 info_line + \
143 "#* *\n" + \
144 "#* The MadGraph Development Team - Please visit us at *\n" + \
145 "#* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \
146 '#* *\n' + \
147 '#************************************************************\n' + \
148 '#* *\n' + \
149 '#* Command File for MadWeight *\n' + \
150 '#* *\n' + \
151 '#* run as ./bin/madweight.py FILENAME *\n' + \
152 '#* *\n' + \
153 '#************************************************************\n'
154
155 if info_line:
156 info_line = info_line[1:]
157
158 logger.info(\
159 "************************************************************\n" + \
160 "* *\n" + \
161 "* W E L C O M E to M A D G R A P H 5 *\n" + \
162 "* M A D W E I G H T *\n" + \
163 "* *\n" + \
164 "* * * *\n" + \
165 "* * * * * *\n" + \
166 "* * * * * 5 * * * * *\n" + \
167 "* * * * * *\n" + \
168 "* * * *\n" + \
169 "* *\n" + \
170 info_line + \
171 "* *\n" + \
172 "* The MadGraph Development Team - Please visit us at *\n" + \
173 "* https://server06.fynu.ucl.ac.be/projects/madgraph *\n" + \
174 "* *\n" + \
175 "* Type 'help' for in-line help. *\n" + \
176 "* *\n" + \
177 "************************************************************")
178
179 cmd.Cmd.__init__(self, *arg, **opt)
180
182
184 logger.info('collect [-refine]')
185 logger.info(' combine the results of the jobs launched on the cluster.')
186 logger.info(' This creates three type of output files:')
187 logger.info(' - weights.out [weight for event/card]')
188 logger.info(' - unnormalize-likelihood.out [-\sum ln(Weight)]')
189 logger.info(' - output.xml [additional information]')
190 logger.info('')
191 logger.info(' The option \'-refine\' is to be added if this is not the first')
192 logger.info(' cluster submission. Otherwise previous run submission will be lost.')
193
195 """help for define transfer_fct"""
196 logger.info(' Modify the current transfer functions')
197 logger.info(' If no argument provided a question showing the list of possibilities.')
198 logger.info(' will be ask. If the TF is provided as argument, no question is asked.')
199
200
202
203
205 """ complete the collect command"""
206 args = self.split_arg(line[0:begidx])
207
208 return self.list_completion(text,['-refine','-f','--refine'], line)
209
211 """ complete the define_transfer_fct """
212
213 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data')
214 listdir=os.listdir(path)
215 args = self.split_arg(line[0:begidx])
216 if len(args) == 1:
217
218 possibilities = [content[3:-4] for content in listdir \
219 if (content.startswith('TF') and content.endswith('dat'))]
220 return self.list_completion(text, possibilities, line)
221
222
223
224
225 -class MadWeightCmd(CmdExtended, HelpToCmd, CompleteForCmd, common_run.CommonRunCmd):
226
227 _set_options = []
228 prompt = 'MadWeight5>'
229 helporder = ['MadWeight Function', 'Documented commands', 'Advanced commands']
230
232 """Not in help: remove fct"""
233 return None
234
235 do_decay_events = remove_fct
236 do_delphes = remove_fct
237 do_pgs = remove_fct
238
239
240
241 - def __init__(self, me_dir = None, options={}, *completekey, **stdin):
247
252
282
284 """MadWeight Function:Define the current transfer function"""
285
286 with misc.chdir(self.me_dir):
287 self.configure()
288 args = self.split_arg(line)
289
290 path = pjoin(self.me_dir, 'Source', 'MadWeight', 'transfer_function', 'data')
291 listdir=os.listdir(path)
292 question = 'Please choose your transfer_function between\n'
293 possibilities = [content[3:-4] for content in listdir \
294 if (content.startswith('TF') and content.endswith('dat'))]
295 for i, tfname in enumerate(possibilities):
296 question += ' %s / %s\n' % (i, tfname)
297 possibilities += range(len(possibilities))
298
299 if args and args[0] in possibilities:
300 tfname = args[0]
301 else:
302 tfname = self.ask(question, 'dbl_gauss_pt_jet', possibilities)
303 if tfname.isdigit():
304 tfname = possibilities[int(tfname)]
305
306 P_dir, MW_dir = MW_info.detect_SubProcess(P_mode=1)
307 os.chdir('./Source/MadWeight/transfer_function')
308 change_tf.create_TF_main(tfname,0, MW_dir)
309
310
320
327
329 """MadWeight Function:compile the code"""
330 self.configure()
331
332 misc.compile(arg=["../lib/libtools.a"], cwd=pjoin(self.me_dir,'Source'))
333 misc.compile(arg=["../lib/libblocks.a"], cwd=pjoin(self.me_dir,'Source'))
334 misc.compile(arg=["../lib/libTF.a"], cwd=pjoin(self.me_dir,'Source'))
335 misc.compile(arg=["../lib/libpdf.a"], cwd=pjoin(self.me_dir,'Source'))
336 misc.compile(arg=["../lib/libdhelas.a"], cwd=pjoin(self.me_dir,'Source'))
337 misc.compile(arg=["../lib/libmodel.a"], cwd=pjoin(self.me_dir,'Source'))
338 misc.compile(arg=["../lib/libgeneric.a"], cwd=pjoin(self.me_dir,'Source'))
339 misc.compile(arg=["../lib/libcernlib.a"], cwd=pjoin(self.me_dir,'Source'))
340
341
342
343 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['montecarlo_perm']=='t':
344 raise Exception, 'Cannot use mint if monte carlo over permutations'
345 if self.MWparam['mw_run']['integrator']=='m' and self.MWparam['mw_run']['use_sobol']=='t':
346 raise Exception, 'sobol generator with mint not implemented'
347
348
349
350 for MW_dir in self.MWparam.MW_listdir:
351 logger.info('compile %s' %MW_dir)
352 pdir = pjoin(self.me_dir,'SubProcesses',MW_dir)
353 if refine and os.path.exists(pjoin(pdir, 'initialization.o')):
354 os.remove(pjoin(pdir, 'initialization.o'))
355 if refine and os.path.exists(pjoin(pdir, 'comp_madweight')):
356 os.remove(pjoin(pdir, 'comp_madweight'))
357 misc.compile(cwd=pdir)
358 if not os.path.exists(pjoin(pdir, 'comp_madweight')):
359 raise Exception, 'compilation fails'
360 logger.info('MadWeight code has been compiled.')
361
362
364 """MadWeight Function: check that the events are valid
365 and write the events to MG mapping"""
366 self.configure()
367 evt_file = pjoin(self.me_dir,'Events','input.lhco')
368 if not os.path.exists(evt_file):
369 question = 'Which LHCO file do you want to use?'
370 default = ''
371 if os.path.exists('%s.gz' % evt_file):
372 input_file = '%s.gz' % evt_file
373 else:
374 input_file = self.ask(question, default, path_msg='valid path')
375
376 if not input_file:
377 raise self.InvalidCmd('Please specify a valid LHCO File')
378
379 if input_file.endswith('.gz'):
380 misc.gunzip(input_file, keep=True, stdout=evt_file)
381 else:
382 files.cp(input_file, evt_file)
383
384 verif_event.verif_event(self.MWparam)
385
386
387
388
389
390
392 """format the argument to retrun a list with two argument,
393 The first corresponding to the fact if we need te create the output dir
394 The second if we can launch the job on the cluster."""
395
396 if not args:
397
398 args[:] = [True, True]
399 return
400 else:
401 create_dir = True
402 launch = True
403 for arg in args:
404 if arg.count('=') !=1 :
405 logger.warning('command launch_jobs does not recognized argument %s. This argument is ignored' % arg)
406 restrict, value = arg.split('=')
407 if restrict == '--create_dir=':
408 if value in self.True:
409 create_dir = True
410 else:
411 create_dir = False
412 elif restrict == '--submit=':
413 if value in self.True:
414 launch = True
415 else:
416 launch = False
417 args[:] = [create_dir, launch]
418 return
419
421 """MadWeight Function:Submitting the jobs to the cluster"""
422
423
424 self.configure()
425 self.clean_old_run(keep_event=True)
426 args = self.split_arg(line)
427 self.check_launch_jobs(args)
428
429 create_dir, launch_jobs = args[0], args[1]
430
431 for nb_card in self.MWparam.actif_param:
432 for dirname in self.MWparam.MW_listdir:
433 nb_job = self.MWparam.nb_event_MW[dirname]
434 if self.MWparam['mw_run']['nb_event_by_node'] > 1:
435 nb_job = 1+ (nb_job-1) // self.MWparam['mw_run']['nb_event_by_node']
436
437 for event_sample in range(nb_job):
438 self.submit_job(dirname, nb_card, event_sample)
439
440 starttime = time.time()
441
442 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'),
443 starttime=starttime, level='madweight', update_results=False)
444
445 try:
446 self.cluster.wait(self.me_dir, update_status)
447 except Exception:
448 self.cluster.remove()
449 raise
450 except KeyboardInterrupt:
451 if not self.force:
452 ans = self.ask('Error detected. Do you want to clean the queue?',
453 default = 'y', choices=['y','n'])
454 else:
455 ans = 'y'
456 if ans == 'y':
457 self.cluster.remove()
458 raise
459
460 - def submit_job(self, dirname, nb_card, sample_nb, evt_file=None, restrict_evt=[]):
461 """launch on the cluster the job which creates the computation"""
462
463 input_files = [pjoin(self.me_dir, 'SubProcesses', dirname, 'comp_madweight'),
464 pjoin(self.me_dir, 'Cards', 'param_card_%i.dat' % nb_card),
465 self.get_pdf_input_filename(),
466 pjoin(self.me_dir, 'Cards', 'ident_card.dat'),
467 pjoin(self.me_dir, 'Cards', 'run_card.dat')
468 ]
469
470
471 if not evt_file:
472 evt_file = (sample_nb // self.MWparam['mw_run']['event_packing'])
473 evt = 'verif_%i.lhco' % evt_file
474 first_event = (sample_nb % self.MWparam['mw_run']['event_packing']) * self.MWparam['mw_run']['nb_event_by_node']
475 name = self.MWparam.name
476 input_files.append(pjoin(self.me_dir, 'SubProcesses', dirname, name, evt))
477
478 if restrict_evt:
479 restrict_path = pjoin(self.me_dir, 'SubProcesses', dirname, name,
480 'restrict%i_%i.dat' % (nb_card,evt_file))
481 input_files.append(restrict_path)
482
483
484
485
486 output_file = ['output_%s_%s.xml' % (nb_card, sample_nb)]
487 exe = pjoin(self.me_dir, 'bin', 'internal', 'madweight', 'MW_driver.py')
488
489
490 args = [str(nb_card), str(first_event),
491 str(self.MWparam['mw_run']['nb_event_by_node']) ,evt,
492 str(self.MWparam['mw_run']['mw_int_points']),
493 self.MWparam['mw_run']['log_level'], str(sample_nb)]
494 cwd = pjoin(self.me_dir, 'SubProcesses', dirname, name)
495
496 if self.options['run_mode'] == 0:
497 raise Exception , 'need to check the validity'
498 else:
499
500 if not self.options['cluster_temp_path'] and not self.options['cluster_type'] == 'condor':
501 raise self.ConfigurationError, 'MadWeight requires temp_cluster_path options to be define'
502 self.cluster.submit2(exe, args, cwd, input_files=input_files, output_files=output_file)
503
504
505
518
520 """MadWeight Function: making the collect of the results"""
521
522 self.configure()
523 args = self.split_arg(line)
524 self.check_collect(args)
525 xml_reader = MWParserXML()
526
527 name = self.MWparam.name
528
529 out_dir = pjoin(self.me_dir, 'Events', name)
530 if '-refine' in args:
531 out_path = pjoin(out_dir, 'refine.xml')
532 else:
533 out_path = pjoin(out_dir, 'output.xml')
534 if os.path.exists(out_path):
535 logger.warning('Output file already exists. Current one will be tagged with _old suffix')
536 logger.warning('Run "collect -refine to instead update your current results."')
537 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
538 files.mv(pjoin(out_dir, 'weights.out'), pjoin(out_dir, 'weights.out'))
539 for MWdir in self.MWparam.MW_listdir:
540 out_dir = pjoin(self.me_dir, 'Events', name, MWdir)
541 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
542 out_dir = pjoin(self.me_dir, 'Events', name)
543
544 fsock = open(out_path, 'w')
545 fsock.write('<madweight>\n<banner>\n')
546
547 for card in ['proc_card_mg5.dat','MadWeight_card.dat','transfer_card.dat','param_card.dat','run_card.dat']:
548 cname = card[:-4]
549 fsock.write('<%s>\n' % cname)
550 fsock.write(open(pjoin(self.me_dir,'Cards',card)).read().replace('<','!>'))
551 fsock.write('</%s>\n' % cname)
552 fsock.write('</banner>\n')
553 at_least_one = False
554 for MWdir in self.MWparam.MW_listdir:
555 out_dir = pjoin(self.me_dir, 'Events', name, MWdir)
556 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, name)
557 if not os.path.exists(out_dir):
558 os.mkdir(out_dir)
559 if '-refine' in args:
560 out_path = pjoin(out_dir, 'refine.xml')
561 else:
562 out_path = pjoin(out_dir, 'output.xml')
563 fsock2 = open(out_path,'w')
564 fsock.write('<subprocess id=\'%s\'>\n' % MWdir)
565 fsock2.write('<subprocess id=\'%s\'>\n' % MWdir)
566 for output in misc.glob('output_*_*.xml', input_dir):
567 at_least_one = True
568 text = open(output).read()
569 fsock2.write(text)
570 fsock.write(text)
571 os.remove(output)
572 fsock.write('</subprocess>\n')
573 fsock2.write('</subprocess>\n')
574 fsock2.close()
575 fsock.write('\n</madweight>\n')
576 fsock.close()
577
578 if '-refine' in args:
579 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level'])
580 for MWdir in self.MWparam.MW_listdir:
581 out_dir = pjoin(self.me_dir, 'Events',name, MWdir)
582 ref_output = xml_reader2.read_file(pjoin(out_dir, 'refine.xml'))
583 xml_reader2 = MWParserXML(self.MWparam['mw_run']['log_level'])
584 base_output = xml_reader2.read_file(pjoin(out_dir, 'output.xml'))
585
586 base_output.refine(ref_output)
587 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
588 base_output.write(pjoin(out_dir, 'output.xml'), MWdir)
589 elif not at_least_one:
590 logger.warning("Nothing to collect restore _old file as current.")
591 out_dir = pjoin(self.me_dir, 'Events', name)
592 files.mv(pjoin(out_dir, 'output_old.xml'), pjoin(out_dir, 'output.xml'))
593 files.mv(pjoin(out_dir, 'weights_old.out'), pjoin(out_dir, 'weights.out'))
594 for MWdir in self.MWparam.MW_listdir:
595 out_dir = pjoin(self.me_dir, 'Events', name, MWdir)
596 files.mv(pjoin(out_dir, 'output.xml'), pjoin(out_dir, 'output_old.xml'))
597
598
599
600
601 total = {}
602 likelihood = {}
603 err_likelihood = {}
604 cards = set()
605 events = set()
606 tf_sets = set()
607 for MW_dir in self.MWparam.MW_listdir:
608 out_dir = pjoin(self.me_dir, 'Events', name, MW_dir)
609 xml_reader = MWParserXML()
610 data = xml_reader.read_file(pjoin(out_dir, 'output.xml'))
611
612 log_level = self.MWparam['mw_run']['log_level']
613 generator = ((int(i),j,int(k),data[i][j][k]) for i in data
614 for j in data[i]
615 for k in data[i][j])
616 for card, event, tf_set, obj in generator:
617
618 cards.add(card)
619 events.add(event)
620 tf_sets.add(tf_set)
621
622 if (card,event, tf_set) in total:
623 value, error = total[(card, event, tf_set)]
624 else:
625 value, error = 0, 0
626 obj.calculate_total()
627 value, error = (value + obj.value, error + obj.error**2)
628 total[(card, event, tf_set)] = (value, error)
629 if tf_set == 1:
630 if value:
631 if card not in likelihood:
632 likelihood[card], err_likelihood[card] = 0, 0
633 likelihood[card] -= math.log(value)
634 err_likelihood[card] += error / value
635 else:
636 likelihood[card] = float('Inf')
637 err_likelihood[card] = float('nan')
638
639
640
641 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'w')
642 logger.info('Write output file with weight information: %s' % fsock.name)
643 fsock.write('# Weight (un-normalize) for each card/event/set of transfer fct\n')
644 fsock.write('# format: LHCO_event_number card_id tf_set_id value integration_error\n')
645 events = list(events)
646 events.sort()
647 cards = list(cards)
648 cards.sort()
649 tf_sets = list(tf_sets)
650 tf_sets.sort()
651 for event in events:
652 for card in cards:
653 for tf_set in tf_sets:
654 try:
655 value, error = total[(card, event,tf_set)]
656 except KeyError:
657 continue
658 error = math.sqrt(error)
659 fsock.write('%s %s %s %s %s \n' % (event.replace('@', ' '), card, tf_set, value, error))
660
661
662 fsock = open(pjoin(self.me_dir, 'Events', name, 'un-normalized_likelihood.out'), 'w')
663 fsock.write('# Warning: this Likelihood needs a bin by bin normalization !\n')
664 fsock.write('# IF more than one set of transfer function are define. ONLY the first one is ')
665 fsock.write('# include in this file.')
666 fsock.write('# format: card_id value integration_error\n')
667 for card in cards:
668 value, error = likelihood[card], err_likelihood[card]
669 error = math.sqrt(error)
670 fsock.write('%s %s %s \n' % (card, value, error))
671
672
674 """MadWeight Function: syntax: clean [XXX]
675 clean the previous run XXX (the last one by default)"""
676
677 args = self.split_arg(line)
678 self.configure()
679 if len(args) == 0:
680 name = self.MWparam.name
681 else:
682 name = args[0]
683
684 ans = self.ask('Do you want to remove Events/%s directory?', 'y',['y','n'])
685 if ans == 'y':
686 try:
687 shutil.rmtree(pjoin(self.me_dir, 'Events', name))
688 except Exception, error:
689 logger.warning(error)
690 for Pdir in self.MWparam.MW_listdir:
691 try:
692 shutil.rmtree(pjoin(self.me_dir, 'SubProcesses', Pdir, name))
693 except Exception, error:
694 logger.warning(error)
695
700
702 """MadWeight Function:run the full suite of commands"""
703
704 args = self.split_arg(line)
705
706
707
708
709 cards = ['param_card.dat', 'run_card.dat', 'madweight_card.dat',
710 'transfer_card.dat', 'input.lhco']
711 if not self.force:
712 self.ask_edit_cards(cards, mode='fixed', plot=False)
713 else:
714 self.configured = 0
715 self.configure()
716 with misc.chdir(self.me_dir):
717 if not os.path.exists(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile'])):
718 raise self.InvalidCmd('Please specify a valid LHCO File')
719 if pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']) not in \
720 [pjoin(self.me_dir, 'Events', 'input.lhco'), pjoin(self.me_dir, 'Events', 'input.lhco.gz')]:
721 zipped = self.MWparam['mw_run']['inputfile'].endswith('.gz')
722 if zipped:
723 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']),
724 pjoin(self.me_dir, 'Events', 'input.lhco.gz'))
725 if os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')):
726 os.remove(pjoin(self.me_dir, 'Events', 'input.lhco'))
727 else:
728 files.cp(pjoin(self.me_dir, self.MWparam['mw_run']['inputfile']),
729 pjoin(self.me_dir, 'Events', 'input.lhco'))
730
731 if not (os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco')) or \
732 os.path.exists(pjoin(self.me_dir, 'Events', 'input.lhco.gz'))):
733 raise self.InvalidCmd('Please specify a valid LHCO File')
734
735 self.exec_cmd('treatcards')
736 self.exec_cmd('get_integration_channel')
737 self.exec_cmd('compile')
738 self.exec_cmd('check_events')
739 self.exec_cmd('submit_jobs')
740 self.exec_cmd('collect')
741
742
744 """check the argument validity"""
745
746 if len(args) != 1:
747 raise self.InvalidCmd('refine requires a single argument')
748
749 try:
750 args[0] = float(args[0])
751 except Exception:
752 raise self.InvalidCmd('First argument of refine command should be a number')
753
754 if args[0] < 0 or args[0]>1:
755 raise self.InvalidCmd('The first argument should be a number between 0 and 1.')
756
758
759 for MWdir in self.MWparam.MW_listdir:
760 input_dir = pjoin(self.me_dir, 'SubProcesses', MWdir, self.MWparam.name)
761 if not os.path.exists(input_dir):
762 continue
763 for filename in os.listdir(input_dir):
764 if keep_event and filename.startswith('verif'):
765 continue
766 os.remove(pjoin(input_dir, filename))
767
768
769
771 """MadWeight Function:syntax: refine X
772 relaunch the computation of the weight which have a precision lower than X"""
773 args = self.split_arg(line)
774 self.check_refine(args)
775 self.configure()
776
777 self.clean_old_run(keep_event=True)
778 if not self.force:
779 cards = ['madweight_card.dat']
780 self.ask_edit_cards(cards, mode='fixed', plot=False)
781 self.exec_cmd("treatcards")
782 self.do_compile('', refine=True)
783
784 nb_events_by_file = self.MWparam['mw_run']['nb_event_by_node'] * self.MWparam['mw_run']['event_packing']
785 asked_events = self.MWparam['mw_run']['nb_exp_events']
786
787 precision = args[0]
788 name = self.MWparam.name
789 allow_refine = []
790
791 fsock = open(pjoin(self.me_dir, 'Events', name, 'weights.out'), 'r')
792 for line in fsock:
793 if '#' in line:
794 line = line.split('#')[0]
795 line = line.split()
796 if len(line) == 5:
797 lhco_nb, card_nb, tf_set, value, error = line
798 else:
799 continue
800 if tf_set != '1':
801 continue
802 if float(value) * precision < float(error):
803 allow_refine.append((int(card_nb), lhco_nb))
804 logger.info("%s selected jobs for the refinment." % len(allow_refine))
805 for MWdir in self.MWparam.MW_listdir:
806
807 event_to_file = {}
808 for evt_nb in range(asked_events//nb_events_by_file +1):
809 evt = 'verif_%i.lhco' % evt_nb
810 for line in open(pjoin(self.me_dir, 'SubProcesses', MWdir, name, evt)):
811 split = line.split()
812 if len(split) == 3:
813 event_to_file[split[1]] = evt_nb
814 to_refine = {}
815 out_dir = pjoin(self.me_dir, 'Events',name, MWdir)
816 xml_reader = MWParserXML(keep_level='weight')
817 data = xml_reader.read_file(pjoin(out_dir, 'output.xml'))
818 generator = ((int(i),j,int(k),data[i][j][k]) for i in data
819 for j in data[i]
820 for k in data[i][j])
821 for card, event, tf_set, obj in generator:
822 if tf_set != 1:
823 continue
824 value, error = obj.value, obj.error
825 if value * precision < error:
826 if card not in to_refine:
827 to_refine[card] = []
828 if (card,event) in allow_refine:
829 to_refine[card].append(event)
830 if to_refine:
831 self.resubmit(MWdir, to_refine, event_to_file)
832
833
834 starttime = time.time()
835 update_status = lambda i, r, f: self.update_status((i, r, f, 'madweight'),
836 starttime=starttime, level='madweight', update_results=False)
837 try:
838 self.cluster.wait(self.me_dir, update_status)
839 except Exception:
840 self.cluster.remove()
841 raise
842 except KeyboardInterrupt:
843 if not self.force:
844 ans = self.ask('Error detected. Do you want to clean the queue?',
845 default = 'y', choices=['y','n'])
846 else:
847 ans = 'y'
848 if ans == 'y':
849 self.cluster.remove()
850 raise
851
852 self.do_collect('-refine')
853
854 - def resubmit(self, M_path, to_refine, event_to_file):
855 """resubmit various jobs"""
856
857 for card, event_list in to_refine.items():
858 packets = {}
859 for event in event_list:
860 evt_nb_file = event_to_file[event]
861 if evt_nb_file in packets:
862 packets[evt_nb_file].append(event)
863 else:
864 packets[evt_nb_file] = [event]
865
866
867
868
869 max_evts = self.MWparam['mw_run']['nb_event_by_node']
870 for evt_nb, evt_list in packets.items():
871 restrict_path = pjoin(self.me_dir, 'SubProcesses', M_path, self.MWparam.name,
872 'restrict%i_%i.dat' % (card,evt_nb_file))
873 open(restrict_path, 'w').write(' '.join(map(str, evt_list)))
874
875 nb_weights = len(evt_list)
876 for i in range(1+ (nb_weights-1)//max_evts):
877
878 self.submit_job(M_path, card, i, evt_file=evt_nb,
879 restrict_evt=True)
880
881
882
883
884
885
887 """The command line processor of MadGraph"""
888 pass
889
890
892 pass
893
894
896
897 for card, CardDATA in other.items():
898 if card not in self:
899 self[card] = other[card]
900 continue
901
902 for event, obj2 in CardDATA.items():
903 self[card][event] = obj2
904
905
906
907 - def write(self, out_path, MWdir):
908 """ """
909 fsock = open(out_path, 'w')
910 fsock.write('<subprocess id=\'%s\'>\n' % MWdir)
911 for card in self:
912 fsock.write('<card id=\'%s\'>\n' % card)
913 for event in self[card].values():
914 event.write(fsock)
915 fsock.write('</card>\n')
916
917 fsock.write('</subprocess>')
918 fsock.close()
919
920
921
923 """ This class will organize in python obect the TF_param.dat file
924 (written in xml)
925 """
926
927
928 - def __init__(self, keep_level='weight'):
929 self.in_el = {'process': '', 'card':'', 'event':'', 'tfset':'' ,
930 'permutation':'', 'channel':'','full':''}
931 if keep_level in ['weight', 'event', 'debug']:
932 keep_level = 'tfset'
933 self.all_event = {}
934 self.keep_level = keep_level
935 self.buffer=''
936 self.output = CollectObj()
937
938
939
941
942
943 if self.in_el[self.keep_level] != '' or name == 'log':
944 return
945
946 obj_class = {'event': MW_driver.Weight, 'permutation':MW_driver.Permutation,
947 'channel': MW_driver.Channel, 'tfset':MW_driver.TFsets}
948
949
950 if name == "process":
951 pass
952 elif name == 'card':
953 id = attributes['id']
954 if id not in self.output:
955 self.output[id] = {}
956 self.in_el[name] = self.output[id]
957 self.curr_card_id = id
958 elif name == 'event':
959 id = attributes['id']
960 value = float(attributes['value'])
961 error = float(attributes['error'])
962 card = self.curr_card_id
963 if ((card,id) in self.all_event):
964 data = self.all_event[(card,id)]
965 else:
966 data = MW_driver.Weight(id, self.keep_level)
967 self.all_event[(card,id)] = data
968 data.value = value
969 data.error = error
970 self.in_el['event'] = data
971
972 card = self.in_el['card']
973 card[id] = data
974 elif name == 'tfset':
975 id = attributes['id']
976 value = float(attributes['value'])
977 error = float(attributes['error'])
978 data = MW_driver.TFsets(id)
979 data.value = value
980 data.error = error
981 data.add('0', '', value, error, '')
982
983 event = self.in_el['event']
984 event[id] = data
985 self.in_el['tfset'] = data
986 elif name == 'permutation':
987 tfset = self.in_el['tfset']
988
989 id = attributes['id']
990 value = float(attributes['value'])
991 error = float(attributes['error'])
992 data = MW_driver.Permutation(id, '')
993 data.value = value
994 data.error = error
995 self.in_el['permutation'] = data
996
997 tfset[id] = data
998 elif name == 'channel':
999 id = attributes['id']
1000 value = float(attributes['value'])
1001 error = float(attributes['error'])
1002 data = MW_driver.Channel(id, value, error)
1003 self.in_el['channel'] = data
1004
1005 permutation = self.in_el['permutation']
1006 permutation[id] = data
1007 elif name in ['log','subprocess','br']:
1008 pass
1009 else:
1010 raise Exception, name
1011 if name != 'br':
1012 self.text = StringIO()
1013
1015 self.text.write(content)
1016
1018 if name == 'log':
1019 data = self.in_el['event']
1020 data.log = self.text.getvalue()
1021
1022 self.in_el[name] = ''
1023
1024
1032