Package logilab :: Package common :: Module modutils
[frames] | no frames]

Source Code for Module logilab.common.modutils

  1  # -*- coding: utf-8 -*- 
  2  """Python modules manipulation utility functions. 
  3   
  4  :author: Logilab 
  5  :copyright: 2000-2008 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  6  :contact: http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  7  :license: General Public License version 2 - http://www.gnu.org/licenses 
  8   
  9  :type PY_SOURCE_EXTS: tuple(str) 
 10  :var PY_SOURCE_EXTS: list of possible python source file extension 
 11   
 12  :type STD_LIB_DIR: str 
 13  :var STD_LIB_DIR: directory where standard modules are located 
 14   
 15  :type BUILTIN_MODULES: dict 
 16  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 17  """ 
 18  __docformat__ = "restructuredtext en" 
 19   
 20  import sys 
 21  import os 
 22  from os.path import walk, splitext, join, abspath, isdir, dirname, exists 
 23  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 24   
 25  try: 
 26      import zipimport 
 27  except ImportError: 
 28      zipimport = None 
 29   
 30  ZIPFILE = object() 
 31   
 32  from logilab.common import STD_BLACKLIST 
 33   
 34  if sys.platform.startswith('win'): 
 35      PY_SOURCE_EXTS = ('py', 'pyw') 
 36      PY_COMPILED_EXTS = ('dll', 'pyd') 
 37      STD_LIB_DIR = join(sys.prefix, 'lib') 
 38  else: 
 39      PY_SOURCE_EXTS = ('py',) 
 40      PY_COMPILED_EXTS = ('so',) 
 41      STD_LIB_DIR = join(sys.prefix, 'lib', 'python%s' % sys.version[:3]) 
 42   
 43  BUILTIN_MODULES = dict(zip(sys.builtin_module_names, 
 44                             [1]*len(sys.builtin_module_names))) 
 45   
 46   
47 -class NoSourceFile(Exception):
48 """exception raised when we are not able to get a python 49 source file for a precompiled file 50 """
51
52 -class LazyObject(object):
53 - def __init__(self, module, obj):
54 self.module = module 55 self.obj = obj 56 self._imported = None
57
58 - def __getobj(self):
59 if self._imported is None: 60 self._imported = getattr(load_module_from_name(self.module), 61 self.obj) 62 return self._imported
63
64 - def __getattribute__(self, attr):
65 try: 66 return super(LazyObject, self).__getattribute__(attr) 67 except AttributeError, ex: 68 return getattr(self.__getobj(), attr)
69
70 - def __call__(self, *args, **kwargs):
71 return self.__getobj()(*args, **kwargs)
72 73
74 -def load_module_from_name(dotted_name, path=None, use_sys=1):
75 """Load a Python module from it's name. 76 77 :type dotted_name: str 78 :param dotted_name: python name of a module or package 79 80 :type path: list or None 81 :param path: 82 optional list of path where the module or package should be 83 searched (use sys.path if nothing or None is given) 84 85 :type use_sys: bool 86 :param use_sys: 87 boolean indicating whether the sys.modules dictionary should be 88 used or not 89 90 91 :raise ImportError: if the module or package is not found 92 93 :rtype: module 94 :return: the loaded module 95 """ 96 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
97 98
99 -def load_module_from_modpath(parts, path=None, use_sys=1):
100 """Load a python module from it's splitted name. 101 102 :type parts: list(str) or tuple(str) 103 :param parts: 104 python name of a module or package splitted on '.' 105 106 :type path: list or None 107 :param path: 108 optional list of path where the module or package should be 109 searched (use sys.path if nothing or None is given) 110 111 :type use_sys: bool 112 :param use_sys: 113 boolean indicating whether the sys.modules dictionary should be used or not 114 115 :raise ImportError: if the module or package is not found 116 117 :rtype: module 118 :return: the loaded module 119 """ 120 if use_sys: 121 try: 122 return sys.modules['.'.join(parts)] 123 except KeyError: 124 pass 125 modpath = [] 126 prevmodule = None 127 for part in parts: 128 modpath.append(part) 129 curname = ".".join(modpath) 130 module = None 131 if len(modpath) != len(parts): 132 # even with use_sys=False, should try to get outer packages from sys.modules 133 module = sys.modules.get(curname) 134 if module is None: 135 mp_file, mp_filename, mp_desc = find_module(part, path) 136 module = load_module(curname, mp_file, mp_filename, mp_desc) 137 if prevmodule: 138 setattr(prevmodule, part, module) 139 _file = getattr(module, "__file__", "") 140 if not _file and len(modpath) != len(parts): 141 raise ImportError("no module in %s" % ".".join(parts[len(modpath):]) ) 142 path = [dirname( _file )] 143 prevmodule = module 144 return module
145 146
147 -def load_module_from_file(filepath, path=None, use_sys=1):
148 """Load a Python module from it's path. 149 150 :type filepath: str 151 :param filepath: path to the python module or package 152 153 :type path: list or None 154 :param path: 155 optional list of path where the module or package should be 156 searched (use sys.path if nothing or None is given) 157 158 :type use_sys: bool 159 :param use_sys: 160 boolean indicating whether the sys.modules dictionary should be 161 used or not 162 163 164 :raise ImportError: if the module or package is not found 165 166 :rtype: module 167 :return: the loaded module 168 """ 169 return load_module_from_modpath(modpath_from_file(filepath), path, use_sys)
170 171
172 -def _check_init(path, mod_path):
173 """check there are some __init__.py all along the way""" 174 for part in mod_path: 175 path = join(path, part) 176 if not _has_init(path): 177 return False 178 return True
179 180
181 -def modpath_from_file(filename, extrapath=None):
182 """given a file path return the corresponding splitted module's name 183 (i.e name of a module or package splitted on '.') 184 185 :type filename: str 186 :param filename: file's path for which we want the module's name 187 188 :type extrapath: dict 189 :param extrapath: 190 optional extra search path, with path as key and package name for the path 191 as value. This is usually useful to handle package splitted in multiple 192 directories using __path__ trick. 193 194 195 :raise ImportError: 196 if the corresponding module's name has not been found 197 198 :rtype: list(str) 199 :return: the corresponding splitted module's name 200 """ 201 base = splitext(abspath(filename))[0] 202 if extrapath is not None: 203 for path_ in extrapath: 204 path = abspath(path_) 205 if path and base[:len(path)] == path: 206 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 207 if pkg] 208 if _check_init(path, submodpath[:-1]): 209 return extrapath[path_].split('.') + submodpath 210 for path in sys.path: 211 path = abspath(path) 212 if path and base[:len(path)] == path: 213 if filename.find('site-packages') != -1 and \ 214 path.find('site-packages') == -1: 215 continue 216 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 217 if _check_init(path, modpath[:-1]): 218 return modpath 219 raise ImportError('Unable to find module for %s in %s' % ( 220 filename, ', \n'.join(sys.path)))
221 222 223
224 -def file_from_modpath(modpath, path=None, context_file=None):
225 """given a mod path (i.e. splitted module / package name), return the 226 corresponding file, giving priority to source file over precompiled 227 file if it exists 228 229 :type modpath: list or tuple 230 :param modpath: 231 splitted module's name (i.e name of a module or package splitted 232 on '.') 233 (this means explicit relative imports that start with dots have 234 empty strings in this list!) 235 236 :type path: list or None 237 :param path: 238 optional list of path where the module or package should be 239 searched (use sys.path if nothing or None is given) 240 241 :type context_file: str or None 242 :param context_file: 243 context file to consider, necessary if the identifier has been 244 introduced using a relative import unresolvable in the actual 245 context (i.e. modutils) 246 247 :raise ImportError: if there is no such module in the directory 248 249 :rtype: str or None 250 :return: 251 the path to the module's file or None if it's an integrated 252 builtin module such as 'sys' 253 """ 254 if context_file is not None: 255 context = dirname(context_file) 256 else: 257 context = context_file 258 if modpath[0] == 'xml': 259 # handle _xmlplus 260 try: 261 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 262 except ImportError: 263 return _file_from_modpath(modpath, path, context) 264 elif modpath == ['os', 'path']: 265 # FIXME: currently ignoring search_path... 266 return os.path.__file__ 267 return _file_from_modpath(modpath, path, context)
268 269 270
271 -def get_module_part(dotted_name, context_file=None):
272 """given a dotted name return the module part of the name : 273 274 >>> get_module_part('logilab.common.modutils.get_module_part') 275 'logilab.common.modutils' 276 277 278 :type dotted_name: str 279 :param dotted_name: full name of the identifier we are interested in 280 281 :type context_file: str or None 282 :param context_file: 283 context file to consider, necessary if the identifier has been 284 introduced using a relative import unresolvable in the actual 285 context (i.e. modutils) 286 287 288 :raise ImportError: if there is no such module in the directory 289 290 :rtype: str or None 291 :return: 292 the module part of the name or None if we have not been able at 293 all to import the given name 294 295 XXX: deprecated, since it doesn't handle package precedence over module 296 (see #10066) 297 """ 298 # os.path trick 299 if dotted_name.startswith('os.path'): 300 return 'os.path' 301 parts = dotted_name.split('.') 302 if context_file is not None: 303 # first check for builtin module which won't be considered latter 304 # in that case (path != None) 305 if parts[0] in BUILTIN_MODULES: 306 if len(parts) > 2: 307 raise ImportError(dotted_name) 308 return parts[0] 309 # don't use += or insert, we want a new list to be created ! 310 path = None 311 starti = 0 312 if parts[0] == '': 313 assert context_file is not None, \ 314 'explicit relative import, but no context_file?' 315 path = [] # prevent resolving the import non-relatively 316 starti = 1 317 while parts[starti] == '': # for all further dots: change context 318 starti += 1 319 context_file = dirname(context_file) 320 for i in range(starti, len(parts)): 321 try: 322 file_from_modpath(parts[starti:i+1], 323 path=path, context_file=context_file) 324 except ImportError: 325 if not i >= max(1, len(parts) - 2): 326 raise 327 return '.'.join(parts[:i]) 328 return dotted_name
329 330 331
332 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
333 """given a package directory return a list of all available python 334 modules in the package and its subpackages 335 336 :type package: str 337 :param package: the python name for the package 338 339 :type src_directory: str 340 :param src_directory: 341 path of the directory corresponding to the package 342 343 :type blacklist: list or tuple 344 :param blacklist: 345 optional list of files or directory to ignore, default to 346 the value of `logilab.common.STD_BLACKLIST` 347 348 :rtype: list 349 :return: 350 the list of all available python modules in the package and its 351 subpackages 352 """ 353 def func(modules, directory, fnames): 354 """walk handler""" 355 # remove files/directories in the black list 356 for norecurs in blacklist: 357 try: 358 fnames.remove(norecurs) 359 except ValueError: 360 continue 361 # check for __init__.py 362 if not '__init__.py' in fnames: 363 while fnames: 364 fnames.pop() 365 elif directory != src_directory: 366 #src = join(directory, file) 367 dir_package = directory[len(src_directory):].replace(os.sep, '.') 368 modules.append(package + dir_package) 369 for filename in fnames: 370 src = join(directory, filename) 371 if isdir(src): 372 continue 373 if _is_python_file(filename) and filename != '__init__.py': 374 module = package + src[len(src_directory):-3] 375 modules.append(module.replace(os.sep, '.'))
376 modules = [] 377 walk(src_directory, func, modules) 378 return modules 379 380 381
382 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
383 """given a package directory return a list of all available python 384 module's files in the package and its subpackages 385 386 :type src_directory: str 387 :param src_directory: 388 path of the directory corresponding to the package 389 390 :type blacklist: list or tuple 391 :param blacklist: 392 optional list of files or directory to ignore, default to the value of 393 `logilab.common.STD_BLACKLIST` 394 395 :rtype: list 396 :return: 397 the list of all available python module's files in the package and 398 its subpackages 399 """ 400 def func(files, directory, fnames): 401 """walk handler""" 402 # remove files/directories in the black list 403 for norecurs in blacklist: 404 try: 405 fnames.remove(norecurs) 406 except ValueError: 407 continue 408 # check for __init__.py 409 if not '__init__.py' in fnames: 410 while fnames: 411 fnames.pop() 412 for filename in fnames: 413 src = join(directory, filename) 414 if isdir(src): 415 continue 416 if _is_python_file(filename): 417 files.append(src)
418 files = [] 419 walk(src_directory, func, files) 420 return files 421 422
423 -def get_source_file(filename, include_no_ext=False):
424 """given a python module's file name return the matching source file 425 name (the filename will be returned identically if it's a already an 426 absolute path to a python source file...) 427 428 :type filename: str 429 :param filename: python module's file name 430 431 432 :raise NoSourceFile: if no source file exists on the file system 433 434 :rtype: str 435 :return: the absolute path of the source file if it exists 436 """ 437 base, orig_ext = splitext(abspath(filename)) 438 for ext in PY_SOURCE_EXTS: 439 source_path = '%s.%s' % (base, ext) 440 if exists(source_path): 441 return source_path 442 if include_no_ext and not orig_ext and exists(base): 443 return base 444 raise NoSourceFile(filename)
445 446
447 -def cleanup_sys_modules(directories):
448 """remove submodules of `directories` from `sys.modules`""" 449 for modname, module in sys.modules.items(): 450 modfile = getattr(module, '__file__', None) 451 if modfile: 452 for directory in directories: 453 if modfile.startswith(directory): 454 del sys.modules[modname] 455 break
456 457
458 -def is_python_source(filename):
459 """ 460 rtype: bool 461 return: True if the filename is a python source file 462 """ 463 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
464 465 466
467 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
468 """try to guess if a module is a standard python module (by default, 469 see `std_path` parameter's description) 470 471 :type modname: str 472 :param modname: name of the module we are interested in 473 474 :type std_path: list(str) or tuple(str) 475 :param std_path: list of path considered has standard 476 477 478 :rtype: bool 479 :return: 480 true if the module: 481 - is located on the path listed in one of the directory in `std_path` 482 - is a built-in module 483 """ 484 modpath = modname.split('.') 485 modname = modpath[0] 486 try: 487 filename = file_from_modpath(modpath) 488 except ImportError: 489 # import failed, i'm probably not so wrong by supposing it's 490 # not standard... 491 return 0 492 # modules which are not living in a file are considered standard 493 # (sys and __builtin__ for instance) 494 if filename is None: 495 return 1 496 filename = abspath(filename) 497 for path in std_path: 498 path = abspath(path) 499 if filename.startswith(path): 500 pfx_len = len(path) 501 if filename[pfx_len+1:pfx_len+14] != 'site-packages': 502 return 1 503 return 0 504 return False
505 506 507
508 -def is_relative(modname, from_file):
509 """return true if the given module name is relative to the given 510 file name 511 512 :type modname: str 513 :param modname: name of the module we are interested in 514 515 :type from_file: str 516 :param from_file: 517 path of the module from which modname has been imported 518 519 :rtype: bool 520 :return: 521 true if the module has been imported relatively to `from_file` 522 """ 523 if not isdir(from_file): 524 from_file = dirname(from_file) 525 if from_file in sys.path: 526 return False 527 try: 528 find_module(modname.split('.')[0], [from_file]) 529 return True 530 except ImportError: 531 return False
532 533 534 # internal only functions ##################################################### 535
536 -def _file_from_modpath(modpath, path=None, context=None):
537 """given a mod path (i.e. splitted module / package name), return the 538 corresponding file 539 540 this function is used internally, see `file_from_modpath`'s 541 documentation for more information 542 """ 543 assert len(modpath) > 0 544 if context is not None: 545 try: 546 mtype, mp_filename = _module_file(modpath, [context]) 547 except ImportError: 548 mtype, mp_filename = _module_file(modpath, path) 549 else: 550 mtype, mp_filename = _module_file(modpath, path) 551 if mtype == PY_COMPILED: 552 try: 553 return get_source_file(mp_filename) 554 except NoSourceFile: 555 return mp_filename 556 elif mtype == C_BUILTIN: 557 # integrated builtin module 558 return None 559 elif mtype == PKG_DIRECTORY: 560 mp_filename = _has_init(mp_filename) 561 return mp_filename
562
563 -def _search_zip(modpath, pic):
564 for filepath, importer in pic.items(): 565 if importer is not None: 566 if importer.find_module(modpath[0]): 567 if not importer.find_module('/'.join(modpath)): 568 raise ImportError('No module named %s in %s/%s' % ( 569 '.'.join(modpath[1:]), file, modpath)) 570 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 571 raise ImportError('No module named %s' % '.'.join(modpath))
572
573 -def _module_file(modpath, path=None):
574 """get a module type / file path 575 576 :type modpath: list or tuple 577 :param modpath: 578 splitted module's name (i.e name of a module or package splitted 579 on '.'), with leading empty strings for explicit relative import 580 581 :type path: list or None 582 :param path: 583 optional list of path where the module or package should be 584 searched (use sys.path if nothing or None is given) 585 586 587 :rtype: tuple(int, str) 588 :return: the module type flag and the file path for a module 589 """ 590 # egg support compat 591 try: 592 pic = sys.path_importer_cache 593 _path = (path is None and sys.path or path) 594 for __path in _path: 595 if not __path in pic: 596 try: 597 pic[__path] = zipimport.zipimporter(__path) 598 except zipimport.ZipImportError: 599 pic[__path] = None 600 checkeggs = True 601 except AttributeError: 602 checkeggs = False 603 imported = [] 604 while modpath: 605 try: 606 _, mp_filename, mp_desc = find_module(modpath[0], path) 607 except ImportError: 608 if checkeggs: 609 return _search_zip(modpath, pic)[:2] 610 raise 611 else: 612 if checkeggs: 613 fullabspath = [abspath(x) for x in _path] 614 try: 615 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 616 emtype, emp_filename, zippath = _search_zip(modpath, pic) 617 if pathindex > _path.index(zippath): 618 # an egg takes priority 619 return emtype, emp_filename 620 except ValueError: 621 # XXX not in _path 622 pass 623 except ImportError: 624 pass 625 checkeggs = False 626 imported.append(modpath.pop(0)) 627 mtype = mp_desc[2] 628 if modpath: 629 if mtype != PKG_DIRECTORY: 630 raise ImportError('No module %s in %s' % ('.'.join(modpath), 631 '.'.join(imported))) 632 path = [mp_filename] 633 return mtype, mp_filename
634
635 -def _is_python_file(filename):
636 """return true if the given filename should be considered as a python file 637 638 .pyc and .pyo are ignored 639 """ 640 for ext in ('.py', '.so', '.pyd', '.pyw'): 641 if filename.endswith(ext): 642 return True 643 return False
644 645
646 -def _has_init(directory):
647 """if the given directory has a valid __init__ file, return its path, 648 else return None 649 """ 650 mod_or_pack = join(directory, '__init__') 651 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 652 if exists(mod_or_pack + '.' + ext): 653 return mod_or_pack + '.' + ext 654 return None
655