1
2 """Python modules manipulation utility functions.
3
4 :author: Logilab
5 :copyright: 2000-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
6 :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr
7 :license: General Public License version 2 - http://www.gnu.org/licenses
8
9 :type PY_SOURCE_EXTS: tuple(str)
10 :var PY_SOURCE_EXTS: list of possible python source file extension
11
12 :type STD_LIB_DIR: str
13 :var STD_LIB_DIR: directory where standard modules are located
14
15 :type BUILTIN_MODULES: dict
16 :var BUILTIN_MODULES: dictionary with builtin module names has key
17 """
18 __docformat__ = "restructuredtext en"
19
20 import sys
21 import os
22 from os.path import walk, splitext, join, abspath, isdir, dirname, exists
23 from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY
24
25 try:
26 import zipimport
27 except ImportError:
28 zipimport = None
29
30 ZIPFILE = object()
31
32 from logilab.common import STD_BLACKLIST
33
34 if sys.platform.startswith('win'):
35 PY_SOURCE_EXTS = ('py', 'pyw')
36 PY_COMPILED_EXTS = ('dll', 'pyd')
37 STD_LIB_DIR = join(sys.prefix, 'lib')
38 else:
39 PY_SOURCE_EXTS = ('py',)
40 PY_COMPILED_EXTS = ('so',)
41 STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3])
42
43 BUILTIN_MODULES = dict(zip(sys.builtin_module_names,
44 [1]*len(sys.builtin_module_names)))
45
46
48 """exception raised when we are not able to get a python
49 source file for a precompiled file
50 """
51
54 self.module = module
55 self.obj = obj
56 self._imported = None
57
63
65 try:
66 return super(LazyObject, self).__getattribute__(attr)
67 except AttributeError, ex:
68 return getattr(self.__getobj(), attr)
69
71 return self.__getobj()(*args, **kwargs)
72
73
75 """Load a Python module from it's name.
76
77 :type dotted_name: str
78 :param dotted_name: python name of a module or package
79
80 :type path: list or None
81 :param path:
82 optional list of path where the module or package should be
83 searched (use sys.path if nothing or None is given)
84
85 :type use_sys: bool
86 :param use_sys:
87 boolean indicating whether the sys.modules dictionary should be
88 used or not
89
90
91 :raise ImportError: if the module or package is not found
92
93 :rtype: module
94 :return: the loaded module
95 """
96 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
97
98
100 """Load a python module from it's splitted name.
101
102 :type parts: list(str) or tuple(str)
103 :param parts:
104 python name of a module or package splitted on '.'
105
106 :type path: list or None
107 :param path:
108 optional list of path where the module or package should be
109 searched (use sys.path if nothing or None is given)
110
111 :type use_sys: bool
112 :param use_sys:
113 boolean indicating whether the sys.modules dictionary should be used or not
114
115 :raise ImportError: if the module or package is not found
116
117 :rtype: module
118 :return: the loaded module
119 """
120 if use_sys:
121 try:
122 return sys.modules['.'.join(parts)]
123 except KeyError:
124 pass
125 modpath = []
126 prevmodule = None
127 for part in parts:
128 modpath.append(part)
129 curname = ".".join(modpath)
130 module = None
131 if len(modpath) != len(parts):
132
133 module = sys.modules.get(curname)
134 if module is None:
135 mp_file, mp_filename, mp_desc = find_module(part, path)
136 module = load_module(curname, mp_file, mp_filename, mp_desc)
137 if prevmodule:
138 setattr(prevmodule, part, module)
139 _file = getattr(module, "__file__", "")
140 if not _file and len(modpath) != len(parts):
141 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) )
142 path = [dirname( _file )]
143 prevmodule = module
144 return module
145
146
148 """Load a Python module from it's path.
149
150 :type filepath: str
151 :param filepath: path to the python module or package
152
153 :type path: list or None
154 :param path:
155 optional list of path where the module or package should be
156 searched (use sys.path if nothing or None is given)
157
158 :type use_sys: bool
159 :param use_sys:
160 boolean indicating whether the sys.modules dictionary should be
161 used or not
162
163
164 :raise ImportError: if the module or package is not found
165
166 :rtype: module
167 :return: the loaded module
168 """
169 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
170
171
173 """check there are some __init__.py all along the way"""
174 for part in mod_path:
175 path = join(path, part)
176 if not _has_init(path):
177 return False
178 return True
179
180
182 """given a file path return the corresponding splitted module's name
183 (i.e name of a module or package splitted on '.')
184
185 :type filename: str
186 :param filename: file's path for which we want the module's name
187
188 :type extrapath: dict
189 :param extrapath:
190 optional extra search path, with path as key and package name for the path
191 as value. This is usually useful to handle package splitted in multiple
192 directories using __path__ trick.
193
194
195 :raise ImportError:
196 if the corresponding module's name has not been found
197
198 :rtype: list(str)
199 :return: the corresponding splitted module's name
200 """
201 base = splitext(abspath(filename))[0]
202 if extrapath is not None:
203 for path_ in extrapath:
204 path = abspath(path_)
205 if path and base[:len(path)] == path:
206 submodpath = [pkg for pkg in base[len(path):].split(os.sep)
207 if pkg]
208 if _check_init(path, submodpath[:-1]):
209 return extrapath[path_].split('.') + submodpath
210 for path in sys.path:
211 path = abspath(path)
212 if path and base[:len(path)] == path:
213 if filename.find('site-packages') != -1 and \
214 path.find('site-packages') == -1:
215 continue
216 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg]
217 if _check_init(path, modpath[:-1]):
218 return modpath
219 raise ImportError('Unable to find module for %s in %s' % (
220 filename, ', \n'.join(sys.path)))
221
222
223
225 """given a mod path (i.e. splitted module / package name), return the
226 corresponding file, giving priority to source file over precompiled
227 file if it exists
228
229 :type modpath: list or tuple
230 :param modpath:
231 splitted module's name (i.e name of a module or package splitted
232 on '.')
233 (this means explicit relative imports that start with dots have
234 empty strings in this list!)
235
236 :type path: list or None
237 :param path:
238 optional list of path where the module or package should be
239 searched (use sys.path if nothing or None is given)
240
241 :type context_file: str or None
242 :param context_file:
243 context file to consider, necessary if the identifier has been
244 introduced using a relative import unresolvable in the actual
245 context (i.e. modutils)
246
247 :raise ImportError: if there is no such module in the directory
248
249 :rtype: str or None
250 :return:
251 the path to the module's file or None if it's an integrated
252 builtin module such as 'sys'
253 """
254 if context_file is not None:
255 context = dirname(context_file)
256 else:
257 context = context_file
258 if modpath[0] == 'xml':
259
260 try:
261 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context)
262 except ImportError:
263 return _file_from_modpath(modpath, path, context)
264 elif modpath == ['os', 'path']:
265
266 return os.path.__file__
267 return _file_from_modpath(modpath, path, context)
268
269
270
272 """given a dotted name return the module part of the name :
273
274 >>> get_module_part('logilab.common.modutils.get_module_part')
275 'logilab.common.modutils'
276
277
278 :type dotted_name: str
279 :param dotted_name: full name of the identifier we are interested in
280
281 :type context_file: str or None
282 :param context_file:
283 context file to consider, necessary if the identifier has been
284 introduced using a relative import unresolvable in the actual
285 context (i.e. modutils)
286
287
288 :raise ImportError: if there is no such module in the directory
289
290 :rtype: str or None
291 :return:
292 the module part of the name or None if we have not been able at
293 all to import the given name
294
295 XXX: deprecated, since it doesn't handle package precedence over module
296 (see #10066)
297 """
298
299 if dotted_name.startswith('os.path'):
300 return 'os.path'
301 parts = dotted_name.split('.')
302 if context_file is not None:
303
304
305 if parts[0] in BUILTIN_MODULES:
306 if len(parts) > 2:
307 raise ImportError(dotted_name)
308 return parts[0]
309
310 path = None
311 starti = 0
312 if parts[0] == '':
313 assert context_file is not None, \
314 'explicit relative import, but no context_file?'
315 path = []
316 starti = 1
317 while parts[starti] == '':
318 starti += 1
319 context_file = dirname(context_file)
320 for i in range(starti, len(parts)):
321 try:
322 file_from_modpath(parts[starti:i+1],
323 path=path, context_file=context_file)
324 except ImportError:
325 if not i >= max(1, len(parts) - 2):
326 raise
327 return '.'.join(parts[:i])
328 return dotted_name
329
330
331
333 """given a package directory return a list of all available python
334 modules in the package and its subpackages
335
336 :type package: str
337 :param package: the python name for the package
338
339 :type src_directory: str
340 :param src_directory:
341 path of the directory corresponding to the package
342
343 :type blacklist: list or tuple
344 :param blacklist:
345 optional list of files or directory to ignore, default to
346 the value of `logilab.common.STD_BLACKLIST`
347
348 :rtype: list
349 :return:
350 the list of all available python modules in the package and its
351 subpackages
352 """
353 def func(modules, directory, fnames):
354 """walk handler"""
355
356 for norecurs in blacklist:
357 try:
358 fnames.remove(norecurs)
359 except ValueError:
360 continue
361
362 if not '__init__.py' in fnames:
363 while fnames:
364 fnames.pop()
365 elif directory != src_directory:
366
367 dir_package = directory[len(src_directory):].replace(os.sep, '.')
368 modules.append(package + dir_package)
369 for filename in fnames:
370 src = join(directory, filename)
371 if isdir(src):
372 continue
373 if _is_python_file(filename) and filename != '__init__.py':
374 module = package + src[len(src_directory):-3]
375 modules.append(module.replace(os.sep, '.'))
376 modules = []
377 walk(src_directory, func, modules)
378 return modules
379
380
381
383 """given a package directory return a list of all available python
384 module's files in the package and its subpackages
385
386 :type src_directory: str
387 :param src_directory:
388 path of the directory corresponding to the package
389
390 :type blacklist: list or tuple
391 :param blacklist:
392 optional list of files or directory to ignore, default to the value of
393 `logilab.common.STD_BLACKLIST`
394
395 :rtype: list
396 :return:
397 the list of all available python module's files in the package and
398 its subpackages
399 """
400 def func(files, directory, fnames):
401 """walk handler"""
402
403 for norecurs in blacklist:
404 try:
405 fnames.remove(norecurs)
406 except ValueError:
407 continue
408
409 if not '__init__.py' in fnames:
410 while fnames:
411 fnames.pop()
412 for filename in fnames:
413 src = join(directory, filename)
414 if isdir(src):
415 continue
416 if _is_python_file(filename):
417 files.append(src)
418 files = []
419 walk(src_directory, func, files)
420 return files
421
422
424 """given a python module's file name return the matching source file
425 name (the filename will be returned identically if it's a already an
426 absolute path to a python source file...)
427
428 :type filename: str
429 :param filename: python module's file name
430
431
432 :raise NoSourceFile: if no source file exists on the file system
433
434 :rtype: str
435 :return: the absolute path of the source file if it exists
436 """
437 base, orig_ext = splitext(abspath(filename))
438 for ext in PY_SOURCE_EXTS:
439 source_path = '%s.%s' % (base, ext)
440 if exists(source_path):
441 return source_path
442 if include_no_ext and not orig_ext and exists(base):
443 return base
444 raise NoSourceFile(filename)
445
446
448 """remove submodules of `directories` from `sys.modules`"""
449 for modname, module in sys.modules.items():
450 modfile = getattr(module, '__file__', None)
451 if modfile:
452 for directory in directories:
453 if modfile.startswith(directory):
454 del sys.modules[modname]
455 break
456
457
459 """
460 rtype: bool
461 return: True if the filename is a python source file
462 """
463 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
464
465
466
468 """try to guess if a module is a standard python module (by default,
469 see `std_path` parameter's description)
470
471 :type modname: str
472 :param modname: name of the module we are interested in
473
474 :type std_path: list(str) or tuple(str)
475 :param std_path: list of path considered has standard
476
477
478 :rtype: bool
479 :return:
480 true if the module:
481 - is located on the path listed in one of the directory in `std_path`
482 - is a built-in module
483 """
484 modpath = modname.split('.')
485 modname = modpath[0]
486 try:
487 filename = file_from_modpath(modpath)
488 except ImportError:
489
490
491 return 0
492
493
494 if filename is None:
495 return 1
496 filename = abspath(filename)
497 for path in std_path:
498 path = abspath(path)
499 if filename.startswith(path):
500 pfx_len = len(path)
501 if filename[pfx_len+1:pfx_len+14] != 'site-packages':
502 return 1
503 return 0
504 return False
505
506
507
509 """return true if the given module name is relative to the given
510 file name
511
512 :type modname: str
513 :param modname: name of the module we are interested in
514
515 :type from_file: str
516 :param from_file:
517 path of the module from which modname has been imported
518
519 :rtype: bool
520 :return:
521 true if the module has been imported relatively to `from_file`
522 """
523 if not isdir(from_file):
524 from_file = dirname(from_file)
525 if from_file in sys.path:
526 return False
527 try:
528 find_module(modname.split('.')[0], [from_file])
529 return True
530 except ImportError:
531 return False
532
533
534
535
537 """given a mod path (i.e. splitted module / package name), return the
538 corresponding file
539
540 this function is used internally, see `file_from_modpath`'s
541 documentation for more information
542 """
543 assert len(modpath) > 0
544 if context is not None:
545 try:
546 mtype, mp_filename = _module_file(modpath, [context])
547 except ImportError:
548 mtype, mp_filename = _module_file(modpath, path)
549 else:
550 mtype, mp_filename = _module_file(modpath, path)
551 if mtype == PY_COMPILED:
552 try:
553 return get_source_file(mp_filename)
554 except NoSourceFile:
555 return mp_filename
556 elif mtype == C_BUILTIN:
557
558 return None
559 elif mtype == PKG_DIRECTORY:
560 mp_filename = _has_init(mp_filename)
561 return mp_filename
562
564 for filepath, importer in pic.items():
565 if importer is not None:
566 if importer.find_module(modpath[0]):
567 if not importer.find_module('/'.join(modpath)):
568 raise ImportError('No module named %s in %s/%s' % (
569 '.'.join(modpath[1:]), file, modpath))
570 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath
571 raise ImportError('No module named %s' % '.'.join(modpath))
572
574 """get a module type / file path
575
576 :type modpath: list or tuple
577 :param modpath:
578 splitted module's name (i.e name of a module or package splitted
579 on '.'), with leading empty strings for explicit relative import
580
581 :type path: list or None
582 :param path:
583 optional list of path where the module or package should be
584 searched (use sys.path if nothing or None is given)
585
586
587 :rtype: tuple(int, str)
588 :return: the module type flag and the file path for a module
589 """
590
591 try:
592 pic = sys.path_importer_cache
593 _path = (path is None and sys.path or path)
594 for __path in _path:
595 if not __path in pic:
596 try:
597 pic[__path] = zipimport.zipimporter(__path)
598 except zipimport.ZipImportError:
599 pic[__path] = None
600 checkeggs = True
601 except AttributeError:
602 checkeggs = False
603 imported = []
604 while modpath:
605 try:
606 _, mp_filename, mp_desc = find_module(modpath[0], path)
607 except ImportError:
608 if checkeggs:
609 return _search_zip(modpath, pic)[:2]
610 raise
611 else:
612 if checkeggs:
613 fullabspath = [abspath(x) for x in _path]
614 try:
615 pathindex = fullabspath.index(dirname(abspath(mp_filename)))
616 emtype, emp_filename, zippath = _search_zip(modpath, pic)
617 if pathindex > _path.index(zippath):
618
619 return emtype, emp_filename
620 except ValueError:
621
622 pass
623 except ImportError:
624 pass
625 checkeggs = False
626 imported.append(modpath.pop(0))
627 mtype = mp_desc[2]
628 if modpath:
629 if mtype != PKG_DIRECTORY:
630 raise ImportError('No module %s in %s' % ('.'.join(modpath),
631 '.'.join(imported)))
632 path = [mp_filename]
633 return mtype, mp_filename
634
636 """return true if the given filename should be considered as a python file
637
638 .pyc and .pyo are ignored
639 """
640 for ext in ('.py', '.so', '.pyd', '.pyw'):
641 if filename.endswith(ext):
642 return True
643 return False
644
645
647 """if the given directory has a valid __init__ file, return its path,
648 else return None
649 """
650 mod_or_pack = join(directory, '__init__')
651 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'):
652 if exists(mod_or_pack + '.' + ext):
653 return mod_or_pack + '.' + ext
654 return None
655