Package common :: Module modutils
[frames] | no frames]

Source Code for Module common.modutils

  1  # -*- coding: utf-8 -*- 
  2  # copyright 2003-2012 LOGILAB S.A. (Paris, FRANCE), all rights reserved. 
  3  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr 
  4  # 
  5  # This file is part of logilab-common. 
  6  # 
  7  # logilab-common is free software: you can redistribute it and/or modify it under 
  8  # the terms of the GNU Lesser General Public License as published by the Free 
  9  # Software Foundation, either version 2.1 of the License, or (at your option) any 
 10  # later version. 
 11  # 
 12  # logilab-common is distributed in the hope that it will be useful, but WITHOUT 
 13  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS 
 14  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more 
 15  # details. 
 16  # 
 17  # You should have received a copy of the GNU Lesser General Public License along 
 18  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>. 
 19  """Python modules manipulation utility functions. 
 20   
 21  :type PY_SOURCE_EXTS: tuple(str) 
 22  :var PY_SOURCE_EXTS: list of possible python source file extension 
 23   
 24  :type STD_LIB_DIR: str 
 25  :var STD_LIB_DIR: directory where standard modules are located 
 26   
 27  :type BUILTIN_MODULES: dict 
 28  :var BUILTIN_MODULES: dictionary with builtin module names has key 
 29  """ 
 30  __docformat__ = "restructuredtext en" 
 31   
 32  import sys 
 33  import os 
 34  from os.path import splitext, join, abspath, isdir, dirname, exists, basename 
 35  from imp import find_module, load_module, C_BUILTIN, PY_COMPILED, PKG_DIRECTORY 
 36  from distutils.sysconfig import get_config_var, get_python_lib, get_python_version 
 37  from distutils.errors import DistutilsPlatformError 
 38   
 39  try: 
 40      import zipimport 
 41  except ImportError: 
 42      zipimport = None 
 43   
 44  ZIPFILE = object() 
 45   
 46  from logilab.common import STD_BLACKLIST, _handle_blacklist 
 47   
 48  # Notes about STD_LIB_DIR 
 49  # Consider arch-specific installation for STD_LIB_DIR definition 
 50  # :mod:`distutils.sysconfig` contains to much hardcoded values to rely on 
 51  # 
 52  # :see: `Problems with /usr/lib64 builds <http://bugs.python.org/issue1294959>`_ 
 53  # :see: `FHS <http://www.pathname.com/fhs/pub/fhs-2.3.html#LIBLTQUALGTALTERNATEFORMATESSENTIAL>`_ 
 54  if sys.platform.startswith('win'): 
 55      PY_SOURCE_EXTS = ('py', 'pyw') 
 56      PY_COMPILED_EXTS = ('dll', 'pyd') 
 57  else: 
 58      PY_SOURCE_EXTS = ('py',) 
 59      PY_COMPILED_EXTS = ('so',) 
 60   
 61  try: 
 62      STD_LIB_DIR = get_python_lib(standard_lib=1) 
 63  # get_python_lib(standard_lib=1) is not available on pypy, set STD_LIB_DIR to 
 64  # non-valid path, see https://bugs.pypy.org/issue1164 
 65  except DistutilsPlatformError: 
 66      STD_LIB_DIR = '//' 
 67   
 68  EXT_LIB_DIR = get_python_lib() 
 69   
 70  BUILTIN_MODULES = dict(list(zip(sys.builtin_module_names, 
 71                             [1]*len(sys.builtin_module_names)))) 
 72   
 73   
74 -class NoSourceFile(Exception):
75 """exception raised when we are not able to get a python 76 source file for a precompiled file 77 """
78
79 -class LazyObject(object):
80 - def __init__(self, module, obj):
81 self.module = module 82 self.obj = obj 83 self._imported = None
84
85 - def _getobj(self):
86 if self._imported is None: 87 self._imported = getattr(load_module_from_name(self.module), 88 self.obj) 89 return self._imported
90
91 - def __getattribute__(self, attr):
92 try: 93 return super(LazyObject, self).__getattribute__(attr) 94 except AttributeError as ex: 95 return getattr(self._getobj(), attr)
96
97 - def __call__(self, *args, **kwargs):
98 return self._getobj()(*args, **kwargs)
99 100
101 -def load_module_from_name(dotted_name, path=None, use_sys=1):
102 """Load a Python module from its name. 103 104 :type dotted_name: str 105 :param dotted_name: python name of a module or package 106 107 :type path: list or None 108 :param path: 109 optional list of path where the module or package should be 110 searched (use sys.path if nothing or None is given) 111 112 :type use_sys: bool 113 :param use_sys: 114 boolean indicating whether the sys.modules dictionary should be 115 used or not 116 117 118 :raise ImportError: if the module or package is not found 119 120 :rtype: module 121 :return: the loaded module 122 """ 123 return load_module_from_modpath(dotted_name.split('.'), path, use_sys)
124 125
126 -def load_module_from_modpath(parts, path=None, use_sys=1):
127 """Load a python module from its splitted name. 128 129 :type parts: list(str) or tuple(str) 130 :param parts: 131 python name of a module or package splitted on '.' 132 133 :type path: list or None 134 :param path: 135 optional list of path where the module or package should be 136 searched (use sys.path if nothing or None is given) 137 138 :type use_sys: bool 139 :param use_sys: 140 boolean indicating whether the sys.modules dictionary should be used or not 141 142 :raise ImportError: if the module or package is not found 143 144 :rtype: module 145 :return: the loaded module 146 """ 147 if use_sys: 148 try: 149 return sys.modules['.'.join(parts)] 150 except KeyError: 151 pass 152 modpath = [] 153 prevmodule = None 154 for part in parts: 155 modpath.append(part) 156 curname = '.'.join(modpath) 157 module = None 158 if len(modpath) != len(parts): 159 # even with use_sys=False, should try to get outer packages from sys.modules 160 module = sys.modules.get(curname) 161 elif use_sys: 162 # because it may have been indirectly loaded through a parent 163 module = sys.modules.get(curname) 164 if module is None: 165 mp_file, mp_filename, mp_desc = find_module(part, path) 166 module = load_module(curname, mp_file, mp_filename, mp_desc) 167 if prevmodule: 168 setattr(prevmodule, part, module) 169 _file = getattr(module, '__file__', '') 170 if not _file and len(modpath) != len(parts): 171 raise ImportError('no module in %s' % '.'.join(parts[len(modpath):]) ) 172 path = [dirname( _file )] 173 prevmodule = module 174 return module
175 176
177 -def load_module_from_file(filepath, path=None, use_sys=1, extrapath=None):
178 """Load a Python module from it's path. 179 180 :type filepath: str 181 :param filepath: path to the python module or package 182 183 :type path: list or None 184 :param path: 185 optional list of path where the module or package should be 186 searched (use sys.path if nothing or None is given) 187 188 :type use_sys: bool 189 :param use_sys: 190 boolean indicating whether the sys.modules dictionary should be 191 used or not 192 193 194 :raise ImportError: if the module or package is not found 195 196 :rtype: module 197 :return: the loaded module 198 """ 199 modpath = modpath_from_file(filepath, extrapath) 200 return load_module_from_modpath(modpath, path, use_sys)
201 202
203 -def _check_init(path, mod_path):
204 """check there are some __init__.py all along the way""" 205 for part in mod_path: 206 path = join(path, part) 207 if not _has_init(path): 208 return False 209 return True
210 211
212 -def modpath_from_file(filename, extrapath=None):
213 """given a file path return the corresponding splitted module's name 214 (i.e name of a module or package splitted on '.') 215 216 :type filename: str 217 :param filename: file's path for which we want the module's name 218 219 :type extrapath: dict 220 :param extrapath: 221 optional extra search path, with path as key and package name for the path 222 as value. This is usually useful to handle package splitted in multiple 223 directories using __path__ trick. 224 225 226 :raise ImportError: 227 if the corresponding module's name has not been found 228 229 :rtype: list(str) 230 :return: the corresponding splitted module's name 231 """ 232 base = splitext(abspath(filename))[0] 233 if extrapath is not None: 234 for path_ in extrapath: 235 path = abspath(path_) 236 if path and base[:len(path)] == path: 237 submodpath = [pkg for pkg in base[len(path):].split(os.sep) 238 if pkg] 239 if _check_init(path, submodpath[:-1]): 240 return extrapath[path_].split('.') + submodpath 241 for path in sys.path: 242 path = abspath(path) 243 if path and base.startswith(path): 244 modpath = [pkg for pkg in base[len(path):].split(os.sep) if pkg] 245 if _check_init(path, modpath[:-1]): 246 return modpath 247 raise ImportError('Unable to find module for %s in %s' % ( 248 filename, ', \n'.join(sys.path)))
249 250 251
252 -def file_from_modpath(modpath, path=None, context_file=None):
253 """given a mod path (i.e. splitted module / package name), return the 254 corresponding file, giving priority to source file over precompiled 255 file if it exists 256 257 :type modpath: list or tuple 258 :param modpath: 259 splitted module's name (i.e name of a module or package splitted 260 on '.') 261 (this means explicit relative imports that start with dots have 262 empty strings in this list!) 263 264 :type path: list or None 265 :param path: 266 optional list of path where the module or package should be 267 searched (use sys.path if nothing or None is given) 268 269 :type context_file: str or None 270 :param context_file: 271 context file to consider, necessary if the identifier has been 272 introduced using a relative import unresolvable in the actual 273 context (i.e. modutils) 274 275 :raise ImportError: if there is no such module in the directory 276 277 :rtype: str or None 278 :return: 279 the path to the module's file or None if it's an integrated 280 builtin module such as 'sys' 281 """ 282 if context_file is not None: 283 context = dirname(context_file) 284 else: 285 context = context_file 286 if modpath[0] == 'xml': 287 # handle _xmlplus 288 try: 289 return _file_from_modpath(['_xmlplus'] + modpath[1:], path, context) 290 except ImportError: 291 return _file_from_modpath(modpath, path, context) 292 elif modpath == ['os', 'path']: 293 # FIXME: currently ignoring search_path... 294 return os.path.__file__ 295 return _file_from_modpath(modpath, path, context)
296 297 298
299 -def get_module_part(dotted_name, context_file=None):
300 """given a dotted name return the module part of the name : 301 302 >>> get_module_part('logilab.common.modutils.get_module_part') 303 'logilab.common.modutils' 304 305 :type dotted_name: str 306 :param dotted_name: full name of the identifier we are interested in 307 308 :type context_file: str or None 309 :param context_file: 310 context file to consider, necessary if the identifier has been 311 introduced using a relative import unresolvable in the actual 312 context (i.e. modutils) 313 314 315 :raise ImportError: if there is no such module in the directory 316 317 :rtype: str or None 318 :return: 319 the module part of the name or None if we have not been able at 320 all to import the given name 321 322 XXX: deprecated, since it doesn't handle package precedence over module 323 (see #10066) 324 """ 325 # os.path trick 326 if dotted_name.startswith('os.path'): 327 return 'os.path' 328 parts = dotted_name.split('.') 329 if context_file is not None: 330 # first check for builtin module which won't be considered latter 331 # in that case (path != None) 332 if parts[0] in BUILTIN_MODULES: 333 if len(parts) > 2: 334 raise ImportError(dotted_name) 335 return parts[0] 336 # don't use += or insert, we want a new list to be created ! 337 path = None 338 starti = 0 339 if parts[0] == '': 340 assert context_file is not None, \ 341 'explicit relative import, but no context_file?' 342 path = [] # prevent resolving the import non-relatively 343 starti = 1 344 while parts[starti] == '': # for all further dots: change context 345 starti += 1 346 context_file = dirname(context_file) 347 for i in range(starti, len(parts)): 348 try: 349 file_from_modpath(parts[starti:i+1], 350 path=path, context_file=context_file) 351 except ImportError: 352 if not i >= max(1, len(parts) - 2): 353 raise 354 return '.'.join(parts[:i]) 355 return dotted_name
356 357
358 -def get_modules(package, src_directory, blacklist=STD_BLACKLIST):
359 """given a package directory return a list of all available python 360 modules in the package and its subpackages 361 362 :type package: str 363 :param package: the python name for the package 364 365 :type src_directory: str 366 :param src_directory: 367 path of the directory corresponding to the package 368 369 :type blacklist: list or tuple 370 :param blacklist: 371 optional list of files or directory to ignore, default to 372 the value of `logilab.common.STD_BLACKLIST` 373 374 :rtype: list 375 :return: 376 the list of all available python modules in the package and its 377 subpackages 378 """ 379 modules = [] 380 for directory, dirnames, filenames in os.walk(src_directory): 381 _handle_blacklist(blacklist, dirnames, filenames) 382 # check for __init__.py 383 if not '__init__.py' in filenames: 384 dirnames[:] = () 385 continue 386 if directory != src_directory: 387 dir_package = directory[len(src_directory):].replace(os.sep, '.') 388 modules.append(package + dir_package) 389 for filename in filenames: 390 if _is_python_file(filename) and filename != '__init__.py': 391 src = join(directory, filename) 392 module = package + src[len(src_directory):-3] 393 modules.append(module.replace(os.sep, '.')) 394 return modules
395 396 397
398 -def get_module_files(src_directory, blacklist=STD_BLACKLIST):
399 """given a package directory return a list of all available python 400 module's files in the package and its subpackages 401 402 :type src_directory: str 403 :param src_directory: 404 path of the directory corresponding to the package 405 406 :type blacklist: list or tuple 407 :param blacklist: 408 optional list of files or directory to ignore, default to the value of 409 `logilab.common.STD_BLACKLIST` 410 411 :rtype: list 412 :return: 413 the list of all available python module's files in the package and 414 its subpackages 415 """ 416 files = [] 417 for directory, dirnames, filenames in os.walk(src_directory): 418 _handle_blacklist(blacklist, dirnames, filenames) 419 # check for __init__.py 420 if not '__init__.py' in filenames: 421 dirnames[:] = () 422 continue 423 for filename in filenames: 424 if _is_python_file(filename): 425 src = join(directory, filename) 426 files.append(src) 427 return files
428 429
430 -def get_source_file(filename, include_no_ext=False):
431 """given a python module's file name return the matching source file 432 name (the filename will be returned identically if it's a already an 433 absolute path to a python source file...) 434 435 :type filename: str 436 :param filename: python module's file name 437 438 439 :raise NoSourceFile: if no source file exists on the file system 440 441 :rtype: str 442 :return: the absolute path of the source file if it exists 443 """ 444 base, orig_ext = splitext(abspath(filename)) 445 for ext in PY_SOURCE_EXTS: 446 source_path = '%s.%s' % (base, ext) 447 if exists(source_path): 448 return source_path 449 if include_no_ext and not orig_ext and exists(base): 450 return base 451 raise NoSourceFile(filename)
452 453
454 -def cleanup_sys_modules(directories):
455 """remove submodules of `directories` from `sys.modules`""" 456 for modname, module in list(sys.modules.items()): 457 modfile = getattr(module, '__file__', None) 458 if modfile: 459 for directory in directories: 460 if modfile.startswith(directory): 461 del sys.modules[modname] 462 break
463 464
465 -def is_python_source(filename):
466 """ 467 rtype: bool 468 return: True if the filename is a python source file 469 """ 470 return splitext(filename)[1][1:] in PY_SOURCE_EXTS
471 472 473
474 -def is_standard_module(modname, std_path=(STD_LIB_DIR,)):
475 """try to guess if a module is a standard python module (by default, 476 see `std_path` parameter's description) 477 478 :type modname: str 479 :param modname: name of the module we are interested in 480 481 :type std_path: list(str) or tuple(str) 482 :param std_path: list of path considered has standard 483 484 485 :rtype: bool 486 :return: 487 true if the module: 488 - is located on the path listed in one of the directory in `std_path` 489 - is a built-in module 490 """ 491 modname = modname.split('.')[0] 492 try: 493 filename = file_from_modpath([modname]) 494 except ImportError as ex: 495 # import failed, i'm probably not so wrong by supposing it's 496 # not standard... 497 return 0 498 # modules which are not living in a file are considered standard 499 # (sys and __builtin__ for instance) 500 if filename is None: 501 return 1 502 filename = abspath(filename) 503 if filename.startswith(EXT_LIB_DIR): 504 return 0 505 for path in std_path: 506 if filename.startswith(abspath(path)): 507 return 1 508 return False
509 510 511
512 -def is_relative(modname, from_file):
513 """return true if the given module name is relative to the given 514 file name 515 516 :type modname: str 517 :param modname: name of the module we are interested in 518 519 :type from_file: str 520 :param from_file: 521 path of the module from which modname has been imported 522 523 :rtype: bool 524 :return: 525 true if the module has been imported relatively to `from_file` 526 """ 527 if not isdir(from_file): 528 from_file = dirname(from_file) 529 if from_file in sys.path: 530 return False 531 try: 532 find_module(modname.split('.')[0], [from_file]) 533 return True 534 except ImportError: 535 return False
536 537 538 # internal only functions ##################################################### 539
540 -def _file_from_modpath(modpath, path=None, context=None):
541 """given a mod path (i.e. splitted module / package name), return the 542 corresponding file 543 544 this function is used internally, see `file_from_modpath`'s 545 documentation for more information 546 """ 547 assert len(modpath) > 0 548 if context is not None: 549 try: 550 mtype, mp_filename = _module_file(modpath, [context]) 551 except ImportError: 552 mtype, mp_filename = _module_file(modpath, path) 553 else: 554 mtype, mp_filename = _module_file(modpath, path) 555 if mtype == PY_COMPILED: 556 try: 557 return get_source_file(mp_filename) 558 except NoSourceFile: 559 return mp_filename 560 elif mtype == C_BUILTIN: 561 # integrated builtin module 562 return None 563 elif mtype == PKG_DIRECTORY: 564 mp_filename = _has_init(mp_filename) 565 return mp_filename
566
567 -def _search_zip(modpath, pic):
568 for filepath, importer in list(pic.items()): 569 if importer is not None: 570 if importer.find_module(modpath[0]): 571 if not importer.find_module('/'.join(modpath)): 572 raise ImportError('No module named %s in %s/%s' % ( 573 '.'.join(modpath[1:]), file, modpath)) 574 return ZIPFILE, abspath(filepath) + '/' + '/'.join(modpath), filepath 575 raise ImportError('No module named %s' % '.'.join(modpath))
576
577 -def _module_file(modpath, path=None):
578 """get a module type / file path 579 580 :type modpath: list or tuple 581 :param modpath: 582 splitted module's name (i.e name of a module or package splitted 583 on '.'), with leading empty strings for explicit relative import 584 585 :type path: list or None 586 :param path: 587 optional list of path where the module or package should be 588 searched (use sys.path if nothing or None is given) 589 590 591 :rtype: tuple(int, str) 592 :return: the module type flag and the file path for a module 593 """ 594 # egg support compat 595 try: 596 pic = sys.path_importer_cache 597 _path = (path is None and sys.path or path) 598 for __path in _path: 599 if not __path in pic: 600 try: 601 pic[__path] = zipimport.zipimporter(__path) 602 except zipimport.ZipImportError: 603 pic[__path] = None 604 checkeggs = True 605 except AttributeError: 606 checkeggs = False 607 imported = [] 608 while modpath: 609 try: 610 _, mp_filename, mp_desc = find_module(modpath[0], path) 611 except ImportError: 612 if checkeggs: 613 return _search_zip(modpath, pic)[:2] 614 raise 615 else: 616 if checkeggs: 617 fullabspath = [abspath(x) for x in _path] 618 try: 619 pathindex = fullabspath.index(dirname(abspath(mp_filename))) 620 emtype, emp_filename, zippath = _search_zip(modpath, pic) 621 if pathindex > _path.index(zippath): 622 # an egg takes priority 623 return emtype, emp_filename 624 except ValueError: 625 # XXX not in _path 626 pass 627 except ImportError: 628 pass 629 checkeggs = False 630 imported.append(modpath.pop(0)) 631 mtype = mp_desc[2] 632 if modpath: 633 if mtype != PKG_DIRECTORY: 634 raise ImportError('No module %s in %s' % ('.'.join(modpath), 635 '.'.join(imported))) 636 path = [mp_filename] 637 return mtype, mp_filename
638
639 -def _is_python_file(filename):
640 """return true if the given filename should be considered as a python file 641 642 .pyc and .pyo are ignored 643 """ 644 for ext in ('.py', '.so', '.pyd', '.pyw'): 645 if filename.endswith(ext): 646 return True 647 return False
648 649
650 -def _has_init(directory):
651 """if the given directory has a valid __init__ file, return its path, 652 else return None 653 """ 654 mod_or_pack = join(directory, '__init__') 655 for ext in PY_SOURCE_EXTS + ('pyc', 'pyo'): 656 if exists(mod_or_pack + '.' + ext): 657 return mod_or_pack + '.' + ext 658 return None
659