1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16 """Methods and classes dealing with file access."""
17
18 import logging
19 import os
20 import shutil
21
22
23 logger = logging.getLogger('madgraph.files')
24
25
26
27
29 """Open a file, apply the function myfunct (with sock as an arg)
30 on its content and return the result. Deals properly with errors and
31 returns None if something goes wrong.
32 """
33 try:
34 sock = open(filename, 'r')
35 try:
36 ret_value = myfunct(sock, *args)
37 finally:
38 sock.close()
39 except IOError, (errno, strerror):
40 if opt.has_key('print_error'):
41 if not opt['print_error']:
42 return None
43 logger.error("I/O error on file %s (%s): %s" % (filename,errno, strerror))
44 return None
45
46 return ret_value
47
48
49
50
52 """Open a file for writing, apply the function myfunct (with sock as an arg)
53 on its content and return the result. Deals properly with errors and
54 returns None if something goes wrong.
55 """
56
57 try:
58 sock = open(filename, 'w')
59 try:
60 ret_value = myfunct(sock, *args)
61 finally:
62 sock.close()
63 except IOError, (errno, strerror):
64 if 'log' not in opts or opts['log']:
65 logger.error("I/O error (%s): %s" % (errno, strerror))
66 return None
67
68 return ret_value
69
70
71
72
74 """Open a file for appending, apply the function myfunct (with
75 sock as an arg) on its content and return the result. Deals
76 properly with errors and returns None if something goes wrong.
77 """
78
79 try:
80 sock = open(filename, 'a')
81 try:
82 ret_value = myfunct(sock, *args)
83 finally:
84 sock.close()
85 except IOError, (errno, strerror):
86 logger.error("I/O error (%s): %s" % (errno, strerror))
87 return None
88
89 return ret_value
90
91
92
93
94 -def is_uptodate(picklefile, path_list=None, min_time=1343682423):
95 """Check if the pickle files is uptodate compare to a list of files.
96 If no files are given, the pickle files is checked against it\' current
97 directory"""
98
99 if not os.path.exists(picklefile):
100 return False
101
102 if path_list is None:
103 dirpath = os.path.dirname(picklefile)
104 path_list = [ os.path.join(dirpath, file) for file in \
105 os.listdir(dirpath)]
106
107 assert type(path_list) == list, 'is_update expect a list of files'
108
109 pickle_date = os.path.getctime(picklefile)
110 if pickle_date < min_time:
111 return False
112
113 for path in path_list:
114 try:
115 if os.path.getmtime(path) > pickle_date:
116 return False
117 except Exception:
118 continue
119
120 return True
121
122
123
124
125
132
133 -def cp(path1, path2, log=True, error=False):
134 """ simple cp taking linux or mix entry"""
135 path1 = format_path(path1)
136 path2 = format_path(path2)
137 try:
138 shutil.copy(path1, path2)
139 except IOError, why:
140 import madgraph.various.misc as misc
141 try:
142 if os.path.exists(path2):
143 path2 = os.path.join(path2, os.path.split(path1)[1])
144 shutil.copytree(path1, path2)
145 except IOError, why:
146 if error:
147 raise
148 if log:
149 logger.warning(why)
150 else:
151 misc.sprint("fail to cp", why)
152 except shutil.Error:
153
154 pass
155
156 -def rm(path, log=True):
157 """removes path, that can be a single element or a list"""
158 if type(path) == list:
159 for p in path:
160 rm(p, log)
161 else:
162 path = format_path(path)
163 try:
164 os.remove(path)
165 except OSError:
166 shutil.rmtree(path, ignore_errors = True)
167
168
169
170 -def mv(path1, path2):
171 """simple mv taking linux or mix format entry"""
172 path1 = format_path(path1)
173 path2 = format_path(path2)
174 try:
175 shutil.move(path1, path2)
176 except Exception:
177
178 if os.path.isfile(path2):
179 os.remove(path2)
180 shutil.move(path1, path2)
181 return
182 elif os.path.isdir(path2) and os.path.exists(
183 os.path.join(path2, os.path.basename(path1))):
184 path2 = os.path.join(path2, os.path.basename(path1))
185 os.remove(path2)
186 shutil.move(path1, path2)
187 else:
188 raise
189
191
192 with open(src,'ab') as wfd:
193 for f in add:
194 with open(f,'rb') as fd:
195 shutil.copyfileobj(fd, wfd, 1024*1024*100)
196
197
198
199 -def ln(file_pos, starting_dir='.', name='', log=True, cwd=None, abspath=False):
200 """a simple way to have a symbolic link without to have to change directory
201 starting_point is the directory where to write the link
202 file_pos is the file to link
203 WARNING: not the linux convention
204 """
205 file_pos = format_path(file_pos)
206 starting_dir = format_path(starting_dir)
207 if not name:
208 name = os.path.split(file_pos)[1]
209
210 if cwd:
211 if not os.path.isabs(file_pos):
212 file_pos = os.path.join(cwd, file_pos)
213 if not os.path.isabs(starting_dir):
214 starting_dir = os.path.join(cwd, starting_dir)
215
216
217 path = os.path.join(starting_dir, name)
218 if os.path.exists(path):
219 if os.path.realpath(path) != os.path.realpath(file_pos):
220 os.remove(os.path.join(starting_dir, name))
221 else:
222 return
223
224 if not abspath:
225 target = os.path.relpath(file_pos, starting_dir)
226 else:
227 target = file_pos
228
229 try:
230 os.symlink(target, os.path.join(starting_dir, name))
231 except Exception, error:
232 if log:
233 logger.debug(error)
234 logger.warning('Could not link %s at position: %s' % (file_pos, \
235 os.path.realpath(starting_dir)))
236
238 if not os.path.exists(dst):
239 os.makedirs(dst)
240 for item in os.listdir(src):
241 s = os.path.join(src, item)
242 d = os.path.join(dst, item)
243 if os.path.isdir(s):
244 copytree(s, d, symlinks, ignore)
245 else:
246 shutil.copy2(s, d)
247