Package CedarBackup2 :: Module util
[hide private]
[frames] | no frames]

Source Code for Module CedarBackup2.util

   1  # -*- coding: iso-8859-1 -*- 
   2  # vim: set ft=python ts=3 sw=3 expandtab: 
   3  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
   4  # 
   5  #              C E D A R 
   6  #          S O L U T I O N S       "Software done right." 
   7  #           S O F T W A R E 
   8  # 
   9  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  10  # 
  11  # Copyright (c) 2004-2008,2010 Kenneth J. Pronovici. 
  12  # All rights reserved. 
  13  # 
  14  # Portions copyright (c) 2001, 2002 Python Software Foundation. 
  15  # All Rights Reserved. 
  16  # 
  17  # This program is free software; you can redistribute it and/or 
  18  # modify it under the terms of the GNU General Public License, 
  19  # Version 2, as published by the Free Software Foundation. 
  20  # 
  21  # This program is distributed in the hope that it will be useful, 
  22  # but WITHOUT ANY WARRANTY; without even the implied warranty of 
  23  # MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. 
  24  # 
  25  # Copies of the GNU General Public License are available from 
  26  # the Free Software Foundation website, http://www.gnu.org/. 
  27  # 
  28  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  29  # 
  30  # Author   : Kenneth J. Pronovici <pronovic@ieee.org> 
  31  # Language : Python (>= 2.5) 
  32  # Project  : Cedar Backup, release 2 
  33  # Revision : $Id: util.py 1023 2011-10-11 23:44:50Z pronovic $ 
  34  # Purpose  : Provides general-purpose utilities. 
  35  # 
  36  # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # # 
  37   
  38  ######################################################################## 
  39  # Module documentation 
  40  ######################################################################## 
  41   
  42  """ 
  43  Provides general-purpose utilities.  
  44   
  45  @sort: AbsolutePathList, ObjectTypeList, RestrictedContentList, RegexMatchList, 
  46         RegexList, _Vertex, DirectedGraph, PathResolverSingleton,  
  47         sortDict, convertSize, getUidGid, changeOwnership, splitCommandLine, 
  48         resolveCommand, executeCommand, calculateFileAge, encodePath, nullDevice, 
  49         deriveDayOfWeek, isStartOfWeek, buildNormalizedPath,  
  50         ISO_SECTOR_SIZE, BYTES_PER_SECTOR,  
  51         BYTES_PER_KBYTE, BYTES_PER_MBYTE, BYTES_PER_GBYTE, KBYTES_PER_MBYTE, MBYTES_PER_GBYTE,  
  52         SECONDS_PER_MINUTE, MINUTES_PER_HOUR, HOURS_PER_DAY, SECONDS_PER_DAY,  
  53         UNIT_BYTES, UNIT_KBYTES, UNIT_MBYTES, UNIT_GBYTES, UNIT_SECTORS 
  54   
  55  @var ISO_SECTOR_SIZE: Size of an ISO image sector, in bytes. 
  56  @var BYTES_PER_SECTOR: Number of bytes (B) per ISO sector. 
  57  @var BYTES_PER_KBYTE: Number of bytes (B) per kilobyte (kB). 
  58  @var BYTES_PER_MBYTE: Number of bytes (B) per megabyte (MB). 
  59  @var BYTES_PER_GBYTE: Number of bytes (B) per megabyte (GB). 
  60  @var KBYTES_PER_MBYTE: Number of kilobytes (kB) per megabyte (MB). 
  61  @var MBYTES_PER_GBYTE: Number of megabytes (MB) per gigabyte (GB). 
  62  @var SECONDS_PER_MINUTE: Number of seconds per minute. 
  63  @var MINUTES_PER_HOUR: Number of minutes per hour. 
  64  @var HOURS_PER_DAY: Number of hours per day. 
  65  @var SECONDS_PER_DAY: Number of seconds per day. 
  66  @var UNIT_BYTES: Constant representing the byte (B) unit for conversion. 
  67  @var UNIT_KBYTES: Constant representing the kilobyte (kB) unit for conversion. 
  68  @var UNIT_MBYTES: Constant representing the megabyte (MB) unit for conversion. 
  69  @var UNIT_GBYTES: Constant representing the gigabyte (GB) unit for conversion. 
  70  @var UNIT_SECTORS: Constant representing the ISO sector unit for conversion. 
  71   
  72  @author: Kenneth J. Pronovici <pronovic@ieee.org> 
  73  """ 
  74   
  75   
  76  ######################################################################## 
  77  # Imported modules 
  78  ######################################################################## 
  79   
  80  import sys 
  81  import math 
  82  import os 
  83  import re 
  84  import time 
  85  import logging 
  86  import string  # pylint: disable=W0402 
  87  from subprocess import Popen, STDOUT, PIPE 
  88   
  89  from CedarBackup2.release import VERSION, DATE 
  90   
  91  try: 
  92     import pwd 
  93     import grp 
  94     _UID_GID_AVAILABLE = True    
  95  except ImportError: 
  96     _UID_GID_AVAILABLE = False    
  97   
  98   
  99  ######################################################################## 
 100  # Module-wide constants and variables 
 101  ######################################################################## 
 102   
 103  logger = logging.getLogger("CedarBackup2.log.util") 
 104  outputLogger = logging.getLogger("CedarBackup2.output") 
 105   
 106  ISO_SECTOR_SIZE    = 2048.0   # in bytes 
 107  BYTES_PER_SECTOR   = ISO_SECTOR_SIZE 
 108   
 109  BYTES_PER_KBYTE    = 1024.0 
 110  KBYTES_PER_MBYTE   = 1024.0 
 111  MBYTES_PER_GBYTE   = 1024.0 
 112  BYTES_PER_MBYTE    = BYTES_PER_KBYTE * KBYTES_PER_MBYTE 
 113  BYTES_PER_GBYTE    = BYTES_PER_MBYTE * MBYTES_PER_GBYTE 
 114   
 115  SECONDS_PER_MINUTE = 60.0 
 116  MINUTES_PER_HOUR   = 60.0 
 117  HOURS_PER_DAY      = 24.0 
 118  SECONDS_PER_DAY    = SECONDS_PER_MINUTE * MINUTES_PER_HOUR * HOURS_PER_DAY 
 119   
 120  UNIT_BYTES         = 0 
 121  UNIT_KBYTES        = 1 
 122  UNIT_MBYTES        = 2 
 123  UNIT_GBYTES        = 4 
 124  UNIT_SECTORS       = 3 
 125   
 126  MTAB_FILE          = "/etc/mtab" 
 127   
 128  MOUNT_COMMAND      = [ "mount", ] 
 129  UMOUNT_COMMAND     = [ "umount", ] 
 130   
 131  DEFAULT_LANGUAGE   = "C" 
 132  LANG_VAR           = "LANG" 
 133  LOCALE_VARS        = [ "LC_ADDRESS", "LC_ALL", "LC_COLLATE", 
 134                         "LC_CTYPE", "LC_IDENTIFICATION",  
 135                         "LC_MEASUREMENT", "LC_MESSAGES",  
 136                         "LC_MONETARY", "LC_NAME", "LC_NUMERIC", 
 137                         "LC_PAPER", "LC_TELEPHONE", "LC_TIME", ] 
138 139 140 ######################################################################## 141 # UnorderedList class definition 142 ######################################################################## 143 144 -class UnorderedList(list):
145 146 """ 147 Class representing an "unordered list". 148 149 An "unordered list" is a list in which only the contents matter, not the 150 order in which the contents appear in the list. 151 152 For instance, we might be keeping track of set of paths in a list, because 153 it's convenient to have them in that form. However, for comparison 154 purposes, we would only care that the lists contain exactly the same 155 contents, regardless of order. 156 157 I have come up with two reasonable ways of doing this, plus a couple more 158 that would work but would be a pain to implement. My first method is to 159 copy and sort each list, comparing the sorted versions. This will only work 160 if two lists with exactly the same members are guaranteed to sort in exactly 161 the same order. The second way would be to create two Sets and then compare 162 the sets. However, this would lose information about any duplicates in 163 either list. I've decided to go with option #1 for now. I'll modify this 164 code if I run into problems in the future. 165 166 We override the original C{__eq__}, C{__ne__}, C{__ge__}, C{__gt__}, 167 C{__le__} and C{__lt__} list methods to change the definition of the various 168 comparison operators. In all cases, the comparison is changed to return the 169 result of the original operation I{but instead comparing sorted lists}. 170 This is going to be quite a bit slower than a normal list, so you probably 171 only want to use it on small lists. 172 """ 173
174 - def __eq__(self, other):
175 """ 176 Definition of C{==} operator for this class. 177 @param other: Other object to compare to. 178 @return: True/false depending on whether C{self == other}. 179 """ 180 if other is None: 181 return False 182 selfSorted = self[:] 183 otherSorted = other[:] 184 selfSorted.sort() 185 otherSorted.sort() 186 return selfSorted.__eq__(otherSorted)
187
188 - def __ne__(self, other):
189 """ 190 Definition of C{!=} operator for this class. 191 @param other: Other object to compare to. 192 @return: True/false depending on whether C{self != other}. 193 """ 194 if other is None: 195 return True 196 selfSorted = self[:] 197 otherSorted = other[:] 198 selfSorted.sort() 199 otherSorted.sort() 200 return selfSorted.__ne__(otherSorted)
201
202 - def __ge__(self, other):
203 """ 204 Definition of S{>=} operator for this class. 205 @param other: Other object to compare to. 206 @return: True/false depending on whether C{self >= other}. 207 """ 208 if other is None: 209 return True 210 selfSorted = self[:] 211 otherSorted = other[:] 212 selfSorted.sort() 213 otherSorted.sort() 214 return selfSorted.__ge__(otherSorted)
215
216 - def __gt__(self, other):
217 """ 218 Definition of C{>} operator for this class. 219 @param other: Other object to compare to. 220 @return: True/false depending on whether C{self > other}. 221 """ 222 if other is None: 223 return True 224 selfSorted = self[:] 225 otherSorted = other[:] 226 selfSorted.sort() 227 otherSorted.sort() 228 return selfSorted.__gt__(otherSorted)
229
230 - def __le__(self, other):
231 """ 232 Definition of S{<=} operator for this class. 233 @param other: Other object to compare to. 234 @return: True/false depending on whether C{self <= other}. 235 """ 236 if other is None: 237 return False 238 selfSorted = self[:] 239 otherSorted = other[:] 240 selfSorted.sort() 241 otherSorted.sort() 242 return selfSorted.__le__(otherSorted)
243
244 - def __lt__(self, other):
245 """ 246 Definition of C{<} operator for this class. 247 @param other: Other object to compare to. 248 @return: True/false depending on whether C{self < other}. 249 """ 250 if other is None: 251 return False 252 selfSorted = self[:] 253 otherSorted = other[:] 254 selfSorted.sort() 255 otherSorted.sort() 256 return selfSorted.__lt__(otherSorted)
257
258 259 ######################################################################## 260 # AbsolutePathList class definition 261 ######################################################################## 262 263 -class AbsolutePathList(UnorderedList):
264 265 """ 266 Class representing a list of absolute paths. 267 268 This is an unordered list. 269 270 We override the C{append}, C{insert} and C{extend} methods to ensure that 271 any item added to the list is an absolute path. 272 273 Each item added to the list is encoded using L{encodePath}. If we don't do 274 this, we have problems trying certain operations between strings and unicode 275 objects, particularly for "odd" filenames that can't be encoded in standard 276 ASCII. 277 """ 278
279 - def append(self, item):
280 """ 281 Overrides the standard C{append} method. 282 @raise ValueError: If item is not an absolute path. 283 """ 284 if not os.path.isabs(item): 285 raise ValueError("Not an absolute path: [%s]" % item) 286 list.append(self, encodePath(item))
287
288 - def insert(self, index, item):
289 """ 290 Overrides the standard C{insert} method. 291 @raise ValueError: If item is not an absolute path. 292 """ 293 if not os.path.isabs(item): 294 raise ValueError("Not an absolute path: [%s]" % item) 295 list.insert(self, index, encodePath(item))
296
297 - def extend(self, seq):
298 """ 299 Overrides the standard C{insert} method. 300 @raise ValueError: If any item is not an absolute path. 301 """ 302 for item in seq: 303 if not os.path.isabs(item): 304 raise ValueError("Not an absolute path: [%s]" % item) 305 for item in seq: 306 list.append(self, encodePath(item))
307
308 309 ######################################################################## 310 # ObjectTypeList class definition 311 ######################################################################## 312 313 -class ObjectTypeList(UnorderedList):
314 315 """ 316 Class representing a list containing only objects with a certain type. 317 318 This is an unordered list. 319 320 We override the C{append}, C{insert} and C{extend} methods to ensure that 321 any item added to the list matches the type that is requested. The 322 comparison uses the built-in C{isinstance}, which should allow subclasses of 323 of the requested type to be added to the list as well. 324 325 The C{objectName} value will be used in exceptions, i.e. C{"Item must be a 326 CollectDir object."} if C{objectName} is C{"CollectDir"}. 327 """ 328
329 - def __init__(self, objectType, objectName):
330 """ 331 Initializes a typed list for a particular type. 332 @param objectType: Type that the list elements must match. 333 @param objectName: Short string containing the "name" of the type. 334 """ 335 super(ObjectTypeList, self).__init__() 336 self.objectType = objectType 337 self.objectName = objectName
338
339 - def append(self, item):
340 """ 341 Overrides the standard C{append} method. 342 @raise ValueError: If item does not match requested type. 343 """ 344 if not isinstance(item, self.objectType): 345 raise ValueError("Item must be a %s object." % self.objectName) 346 list.append(self, item)
347
348 - def insert(self, index, item):
349 """ 350 Overrides the standard C{insert} method. 351 @raise ValueError: If item does not match requested type. 352 """ 353 if not isinstance(item, self.objectType): 354 raise ValueError("Item must be a %s object." % self.objectName) 355 list.insert(self, index, item)
356
357 - def extend(self, seq):
358 """ 359 Overrides the standard C{insert} method. 360 @raise ValueError: If item does not match requested type. 361 """ 362 for item in seq: 363 if not isinstance(item, self.objectType): 364 raise ValueError("All items must be %s objects." % self.objectName) 365 list.extend(self, seq)
366
367 368 ######################################################################## 369 # RestrictedContentList class definition 370 ######################################################################## 371 372 -class RestrictedContentList(UnorderedList):
373 374 """ 375 Class representing a list containing only object with certain values. 376 377 This is an unordered list. 378 379 We override the C{append}, C{insert} and C{extend} methods to ensure that 380 any item added to the list is among the valid values. We use a standard 381 comparison, so pretty much anything can be in the list of valid values. 382 383 The C{valuesDescr} value will be used in exceptions, i.e. C{"Item must be 384 one of values in VALID_ACTIONS"} if C{valuesDescr} is C{"VALID_ACTIONS"}. 385 386 @note: This class doesn't make any attempt to trap for nonsensical 387 arguments. All of the values in the values list should be of the same type 388 (i.e. strings). Then, all list operations also need to be of that type 389 (i.e. you should always insert or append just strings). If you mix types -- 390 for instance lists and strings -- you will likely see AttributeError 391 exceptions or other problems. 392 """ 393
394 - def __init__(self, valuesList, valuesDescr, prefix=None):
395 """ 396 Initializes a list restricted to containing certain values. 397 @param valuesList: List of valid values. 398 @param valuesDescr: Short string describing list of values. 399 @param prefix: Prefix to use in error messages (None results in prefix "Item") 400 """ 401 super(RestrictedContentList, self).__init__() 402 self.prefix = "Item" 403 if prefix is not None: self.prefix = prefix 404 self.valuesList = valuesList 405 self.valuesDescr = valuesDescr
406
407 - def append(self, item):
408 """ 409 Overrides the standard C{append} method. 410 @raise ValueError: If item is not in the values list. 411 """ 412 if item not in self.valuesList: 413 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 414 list.append(self, item)
415
416 - def insert(self, index, item):
417 """ 418 Overrides the standard C{insert} method. 419 @raise ValueError: If item is not in the values list. 420 """ 421 if item not in self.valuesList: 422 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 423 list.insert(self, index, item)
424
425 - def extend(self, seq):
426 """ 427 Overrides the standard C{insert} method. 428 @raise ValueError: If item is not in the values list. 429 """ 430 for item in seq: 431 if item not in self.valuesList: 432 raise ValueError("%s must be one of the values in %s." % (self.prefix, self.valuesDescr)) 433 list.extend(self, seq)
434
435 436 ######################################################################## 437 # RegexMatchList class definition 438 ######################################################################## 439 440 -class RegexMatchList(UnorderedList):
441 442 """ 443 Class representing a list containing only strings that match a regular expression. 444 445 If C{emptyAllowed} is passed in as C{False}, then empty strings are 446 explicitly disallowed, even if they happen to match the regular expression. 447 (C{None} values are always disallowed, since string operations are not 448 permitted on C{None}.) 449 450 This is an unordered list. 451 452 We override the C{append}, C{insert} and C{extend} methods to ensure that 453 any item added to the list matches the indicated regular expression. 454 455 @note: If you try to put values that are not strings into the list, you will 456 likely get either TypeError or AttributeError exceptions as a result. 457 """ 458
459 - def __init__(self, valuesRegex, emptyAllowed=True, prefix=None):
460 """ 461 Initializes a list restricted to containing certain values. 462 @param valuesRegex: Regular expression that must be matched, as a string 463 @param emptyAllowed: Indicates whether empty or None values are allowed. 464 @param prefix: Prefix to use in error messages (None results in prefix "Item") 465 """ 466 super(RegexMatchList, self).__init__() 467 self.prefix = "Item" 468 if prefix is not None: self.prefix = prefix 469 self.valuesRegex = valuesRegex 470 self.emptyAllowed = emptyAllowed 471 self.pattern = re.compile(self.valuesRegex)
472
473 - def append(self, item):
474 """ 475 Overrides the standard C{append} method. 476 @raise ValueError: If item is None 477 @raise ValueError: If item is empty and empty values are not allowed 478 @raise ValueError: If item does not match the configured regular expression 479 """ 480 if item is None or (not self.emptyAllowed and item == ""): 481 raise ValueError("%s cannot be empty." % self.prefix) 482 if not self.pattern.search(item): 483 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 484 list.append(self, item)
485
486 - def insert(self, index, item):
487 """ 488 Overrides the standard C{insert} method. 489 @raise ValueError: If item is None 490 @raise ValueError: If item is empty and empty values are not allowed 491 @raise ValueError: If item does not match the configured regular expression 492 """ 493 if item is None or (not self.emptyAllowed and item == ""): 494 raise ValueError("%s cannot be empty." % self.prefix) 495 if not self.pattern.search(item): 496 raise ValueError("%s is not valid [%s]" % (self.prefix, item)) 497 list.insert(self, index, item)
498
499 - def extend(self, seq):
500 """ 501 Overrides the standard C{insert} method. 502 @raise ValueError: If any item is None 503 @raise ValueError: If any item is empty and empty values are not allowed 504 @raise ValueError: If any item does not match the configured regular expression 505 """ 506 for item in seq: 507 if item is None or (not self.emptyAllowed and item == ""): 508 raise ValueError("%s cannot be empty.", self.prefix) 509 if not self.pattern.search(item): 510 raise ValueError("%s is not valid: [%s]" % (self.prefix, item)) 511 list.extend(self, seq)
512
513 514 ######################################################################## 515 # RegexList class definition 516 ######################################################################## 517 518 -class RegexList(UnorderedList):
519 520 """ 521 Class representing a list of valid regular expression strings. 522 523 This is an unordered list. 524 525 We override the C{append}, C{insert} and C{extend} methods to ensure that 526 any item added to the list is a valid regular expression. 527 """ 528
529 - def append(self, item):
530 """ 531 Overrides the standard C{append} method. 532 @raise ValueError: If item is not an absolute path. 533 """ 534 try: 535 re.compile(item) 536 except re.error: 537 raise ValueError("Not a valid regular expression: [%s]" % item) 538 list.append(self, item)
539
540 - def insert(self, index, item):
541 """ 542 Overrides the standard C{insert} method. 543 @raise ValueError: If item is not an absolute path. 544 """ 545 try: 546 re.compile(item) 547 except re.error: 548 raise ValueError("Not a valid regular expression: [%s]" % item) 549 list.insert(self, index, item)
550
551 - def extend(self, seq):
552 """ 553 Overrides the standard C{insert} method. 554 @raise ValueError: If any item is not an absolute path. 555 """ 556 for item in seq: 557 try: 558 re.compile(item) 559 except re.error: 560 raise ValueError("Not a valid regular expression: [%s]" % item) 561 for item in seq: 562 list.append(self, item)
563
564 565 ######################################################################## 566 # Directed graph implementation 567 ######################################################################## 568 569 -class _Vertex(object):
570 571 """ 572 Represents a vertex (or node) in a directed graph. 573 """ 574
575 - def __init__(self, name):
576 """ 577 Constructor. 578 @param name: Name of this graph vertex. 579 @type name: String value. 580 """ 581 self.name = name 582 self.endpoints = [] 583 self.state = None
584
585 -class DirectedGraph(object):
586 587 """ 588 Represents a directed graph. 589 590 A graph B{G=(V,E)} consists of a set of vertices B{V} together with a set 591 B{E} of vertex pairs or edges. In a directed graph, each edge also has an 592 associated direction (from vertext B{v1} to vertex B{v2}). A C{DirectedGraph} 593 object provides a way to construct a directed graph and execute a depth- 594 first search. 595 596 This data structure was designed based on the graphing chapter in 597 U{The Algorithm Design Manual<http://www2.toki.or.id/book/AlgDesignManual/>}, 598 by Steven S. Skiena. 599 600 This class is intended to be used by Cedar Backup for dependency ordering. 601 Because of this, it's not quite general-purpose. Unlike a "general" graph, 602 every vertex in this graph has at least one edge pointing to it, from a 603 special "start" vertex. This is so no vertices get "lost" either because 604 they have no dependencies or because nothing depends on them. 605 """ 606 607 _UNDISCOVERED = 0 608 _DISCOVERED = 1 609 _EXPLORED = 2 610
611 - def __init__(self, name):
612 """ 613 Directed graph constructor. 614 615 @param name: Name of this graph. 616 @type name: String value. 617 """ 618 if name is None or name == "": 619 raise ValueError("Graph name must be non-empty.") 620 self._name = name 621 self._vertices = {} 622 self._startVertex = _Vertex(None) # start vertex is only vertex with no name
623
624 - def __repr__(self):
625 """ 626 Official string representation for class instance. 627 """ 628 return "DirectedGraph(%s)" % self.name
629
630 - def __str__(self):
631 """ 632 Informal string representation for class instance. 633 """ 634 return self.__repr__()
635
636 - def __cmp__(self, other):
637 """ 638 Definition of equals operator for this class. 639 @param other: Other object to compare to. 640 @return: -1/0/1 depending on whether self is C{<}, C{=} or C{>} other. 641 """ 642 # pylint: disable=W0212 643 if other is None: 644 return 1 645 if self.name != other.name: 646 if self.name < other.name: 647 return -1 648 else: 649 return 1 650 if self._vertices != other._vertices: 651 if self._vertices < other._vertices: 652 return -1 653 else: 654 return 1 655 return 0
656
657 - def _getName(self):
658 """ 659 Property target used to get the graph name. 660 """ 661 return self._name
662 663 name = property(_getName, None, None, "Name of the graph.") 664
665 - def createVertex(self, name):
666 """ 667 Creates a named vertex. 668 @param name: vertex name 669 @raise ValueError: If the vertex name is C{None} or empty. 670 """ 671 if name is None or name == "": 672 raise ValueError("Vertex name must be non-empty.") 673 vertex = _Vertex(name) 674 self._startVertex.endpoints.append(vertex) # so every vertex is connected at least once 675 self._vertices[name] = vertex
676
677 - def createEdge(self, start, finish):
678 """ 679 Adds an edge with an associated direction, from C{start} vertex to C{finish} vertex. 680 @param start: Name of start vertex. 681 @param finish: Name of finish vertex. 682 @raise ValueError: If one of the named vertices is unknown. 683 """ 684 try: 685 startVertex = self._vertices[start] 686 finishVertex = self._vertices[finish] 687 startVertex.endpoints.append(finishVertex) 688 except KeyError, e: 689 raise ValueError("Vertex [%s] could not be found." % e)
690
691 - def topologicalSort(self):
692 """ 693 Implements a topological sort of the graph. 694 695 This method also enforces that the graph is a directed acyclic graph, 696 which is a requirement of a topological sort. 697 698 A directed acyclic graph (or "DAG") is a directed graph with no directed 699 cycles. A topological sort of a DAG is an ordering on the vertices such 700 that all edges go from left to right. Only an acyclic graph can have a 701 topological sort, but any DAG has at least one topological sort. 702 703 Since a topological sort only makes sense for an acyclic graph, this 704 method throws an exception if a cycle is found. 705 706 A depth-first search only makes sense if the graph is acyclic. If the 707 graph contains any cycles, it is not possible to determine a consistent 708 ordering for the vertices. 709 710 @note: If a particular vertex has no edges, then its position in the 711 final list depends on the order in which the vertices were created in the 712 graph. If you're using this method to determine a dependency order, this 713 makes sense: a vertex with no dependencies can go anywhere (and will). 714 715 @return: Ordering on the vertices so that all edges go from left to right. 716 717 @raise ValueError: If a cycle is found in the graph. 718 """ 719 ordering = [] 720 for key in self._vertices: 721 vertex = self._vertices[key] 722 vertex.state = self._UNDISCOVERED 723 for key in self._vertices: 724 vertex = self._vertices[key] 725 if vertex.state == self._UNDISCOVERED: 726 self._topologicalSort(self._startVertex, ordering) 727 return ordering
728
729 - def _topologicalSort(self, vertex, ordering):
730 """ 731 Recursive depth first search function implementing topological sort. 732 @param vertex: Vertex to search 733 @param ordering: List of vertices in proper order 734 """ 735 vertex.state = self._DISCOVERED 736 for endpoint in vertex.endpoints: 737 if endpoint.state == self._UNDISCOVERED: 738 self._topologicalSort(endpoint, ordering) 739 elif endpoint.state != self._EXPLORED: 740 raise ValueError("Cycle found in graph (found '%s' while searching '%s')." % (vertex.name, endpoint.name)) 741 if vertex.name is not None: 742 ordering.insert(0, vertex.name) 743 vertex.state = self._EXPLORED
744
745 746 ######################################################################## 747 # PathResolverSingleton class definition 748 ######################################################################## 749 750 -class PathResolverSingleton(object):
751 752 """ 753 Singleton used for resolving executable paths. 754 755 Various functions throughout Cedar Backup (including extensions) need a way 756 to resolve the path of executables that they use. For instance, the image 757 functionality needs to find the C{mkisofs} executable, and the Subversion 758 extension needs to find the C{svnlook} executable. Cedar Backup's original 759 behavior was to assume that the simple name (C{"svnlook"} or whatever) was 760 available on the caller's C{$PATH}, and to fail otherwise. However, this 761 turns out to be less than ideal, since for instance the root user might not 762 always have executables like C{svnlook} in its path. 763 764 One solution is to specify a path (either via an absolute path or some sort 765 of path insertion or path appending mechanism) that would apply to the 766 C{executeCommand()} function. This is not difficult to implement, but it 767 seem like kind of a "big hammer" solution. Besides that, it might also 768 represent a security flaw (for instance, I prefer not to mess with root's 769 C{$PATH} on the application level if I don't have to). 770 771 The alternative is to set up some sort of configuration for the path to 772 certain executables, i.e. "find C{svnlook} in C{/usr/local/bin/svnlook}" or 773 whatever. This PathResolverSingleton aims to provide a good solution to the 774 mapping problem. Callers of all sorts (extensions or not) can get an 775 instance of the singleton. Then, they call the C{lookup} method to try and 776 resolve the executable they are looking for. Through the C{lookup} method, 777 the caller can also specify a default to use if a mapping is not found. 778 This way, with no real effort on the part of the caller, behavior can neatly 779 degrade to something equivalent to the current behavior if there is no 780 special mapping or if the singleton was never initialized in the first 781 place. 782 783 Even better, extensions automagically get access to the same resolver 784 functionality, and they don't even need to understand how the mapping 785 happens. All extension authors need to do is document what executables 786 their code requires, and the standard resolver configuration section will 787 meet their needs. 788 789 The class should be initialized once through the constructor somewhere in 790 the main routine. Then, the main routine should call the L{fill} method to 791 fill in the resolver's internal structures. Everyone else who needs to 792 resolve a path will get an instance of the class using L{getInstance} and 793 will then just call the L{lookup} method. 794 795 @cvar _instance: Holds a reference to the singleton 796 @ivar _mapping: Internal mapping from resource name to path. 797 """ 798 799 _instance = None # Holds a reference to singleton instance 800
801 - class _Helper:
802 """Helper class to provide a singleton factory method."""
803 - def __init__(self):
804 pass
805 - def __call__(self, *args, **kw):
806 # pylint: disable=W0212,R0201 807 if PathResolverSingleton._instance is None: 808 obj = PathResolverSingleton() 809 PathResolverSingleton._instance = obj 810 return PathResolverSingleton._instance
811 812 getInstance = _Helper() # Method that callers will use to get an instance 813
814 - def __init__(self):
815 """Singleton constructor, which just creates the singleton instance.""" 816 if PathResolverSingleton._instance is not None: 817 raise RuntimeError("Only one instance of PathResolverSingleton is allowed!") 818 PathResolverSingleton._instance = self 819 self._mapping = { }
820
821 - def lookup(self, name, default=None):
822 """ 823 Looks up name and returns the resolved path associated with the name. 824 @param name: Name of the path resource to resolve. 825 @param default: Default to return if resource cannot be resolved. 826 @return: Resolved path associated with name, or default if name can't be resolved. 827 """ 828 value = default 829 if name in self._mapping.keys(): 830 value = self._mapping[name] 831 logger.debug("Resolved command [%s] to [%s]." % (name, value)) 832 return value
833
834 - def fill(self, mapping):
835 """ 836 Fills in the singleton's internal mapping from name to resource. 837 @param mapping: Mapping from resource name to path. 838 @type mapping: Dictionary mapping name to path, both as strings. 839 """ 840 self._mapping = { } 841 for key in mapping.keys(): 842 self._mapping[key] = mapping[key]
843
844 845 ######################################################################## 846 # Pipe class definition 847 ######################################################################## 848 849 -class Pipe(Popen):
850 """ 851 Specialized pipe class for use by C{executeCommand}. 852 853 The L{executeCommand} function needs a specialized way of interacting 854 with a pipe. First, C{executeCommand} only reads from the pipe, and 855 never writes to it. Second, C{executeCommand} needs a way to discard all 856 output written to C{stderr}, as a means of simulating the shell 857 C{2>/dev/null} construct. 858 """
859 - def __init__(self, cmd, bufsize=-1, ignoreStderr=False):
860 stderr = STDOUT 861 if ignoreStderr: 862 devnull = nullDevice() 863 stderr = os.open(devnull, os.O_RDWR) 864 Popen.__init__(self, shell=False, args=cmd, bufsize=bufsize, stdin=None, stdout=PIPE, stderr=stderr)
865
866 867 ######################################################################## 868 # Diagnostics class definition 869 ######################################################################## 870 871 -class Diagnostics(object):
872 873 """ 874 Class holding runtime diagnostic information. 875 876 Diagnostic information is information that is useful to get from users for 877 debugging purposes. I'm consolidating it all here into one object. 878 879 @sort: __init__, __repr__, __str__ 880 """ 881 # pylint: disable=R0201 882
883 - def __init__(self):
884 """ 885 Constructor for the C{Diagnostics} class. 886 """
887
888 - def __repr__(self):
889 """ 890 Official string representation for class instance. 891 """ 892 return "Diagnostics()"
893
894 - def __str__(self):
895 """ 896 Informal string representation for class instance. 897 """ 898 return self.__repr__()
899
900 - def getValues(self):
901 """ 902 Get a map containing all of the diagnostic values. 903 @return: Map from diagnostic name to diagnostic value. 904 """ 905 values = {} 906 values['version'] = self.version 907 values['interpreter'] = self.interpreter 908 values['platform'] = self.platform 909 values['encoding'] = self.encoding 910 values['locale'] = self.locale 911 values['timestamp'] = self.timestamp 912 return values
913
914 - def printDiagnostics(self, fd=sys.stdout, prefix=""):
915 """ 916 Pretty-print diagnostic information to a file descriptor. 917 @param fd: File descriptor used to print information. 918 @param prefix: Prefix string (if any) to place onto printed lines 919 @note: The C{fd} is used rather than C{print} to facilitate unit testing. 920 """ 921 lines = self._buildDiagnosticLines(prefix) 922 for line in lines: 923 fd.write("%s\n" % line)
924
925 - def logDiagnostics(self, method, prefix=""):
926 """ 927 Pretty-print diagnostic information using a logger method. 928 @param method: Logger method to use for logging (i.e. logger.info) 929 @param prefix: Prefix string (if any) to place onto printed lines 930 """ 931 lines = self._buildDiagnosticLines(prefix) 932 for line in lines: 933 method("%s" % line)
934
935 - def _buildDiagnosticLines(self, prefix=""):
936 """ 937 Build a set of pretty-printed diagnostic lines. 938 @param prefix: Prefix string (if any) to place onto printed lines 939 @return: List of strings, not terminated by newlines. 940 """ 941 values = self.getValues() 942 keys = values.keys() 943 keys.sort() 944 tmax = Diagnostics._getMaxLength(keys) + 3 # three extra dots in output 945 lines = [] 946 for key in keys: 947 title = key.title() 948 title += (tmax - len(title)) * '.' 949 value = values[key] 950 line = "%s%s: %s" % (prefix, title, value) 951 lines.append(line) 952 return lines
953 954 @staticmethod
955 - def _getMaxLength(values):
956 """ 957 Get the maximum length from among a list of strings. 958 """ 959 tmax = 0 960 for value in values: 961 if len(value) > tmax: 962 tmax = len(value) 963 return tmax
964
965 - def _getVersion(self):
966 """ 967 Property target to get the Cedar Backup version. 968 """ 969 return "Cedar Backup %s (%s)" % (VERSION, DATE)
970
971 - def _getInterpreter(self):
972 """ 973 Property target to get the Python interpreter version. 974 """ 975 version = sys.version_info 976 return "Python %d.%d.%d (%s)" % (version[0], version[1], version[2], version[3])
977
978 - def _getEncoding(self):
979 """ 980 Property target to get the filesystem encoding. 981 """ 982 return sys.getfilesystemencoding() or sys.getdefaultencoding()
983
984 - def _getPlatform(self):
985 """ 986 Property target to get the operating system platform. 987 """ 988 try: 989 if sys.platform.startswith("win"): 990 windowsPlatforms = [ "Windows 3.1", "Windows 95/98/ME", "Windows NT/2000/XP", "Windows CE", ] 991 wininfo = sys.getwindowsversion() # pylint: disable=E1101 992 winversion = "%d.%d.%d" % (wininfo[0], wininfo[1], wininfo[2]) 993 winplatform = windowsPlatforms[wininfo[3]] 994 wintext = wininfo[4] # i.e. "Service Pack 2" 995 return "%s (%s %s %s)" % (sys.platform, winplatform, winversion, wintext) 996 else: 997 uname = os.uname() 998 sysname = uname[0] # i.e. Linux 999 release = uname[2] # i.e. 2.16.18-2 1000 machine = uname[4] # i.e. i686 1001 return "%s (%s %s %s)" % (sys.platform, sysname, release, machine) 1002 except: 1003 return sys.platform
1004
1005 - def _getLocale(self):
1006 """ 1007 Property target to get the default locale that is in effect. 1008 """ 1009 try: 1010 import locale 1011 return locale.getdefaultlocale()[0] 1012 except: 1013 return "(unknown)"
1014
1015 - def _getTimestamp(self):
1016 """ 1017 Property target to get a current date/time stamp. 1018 """ 1019 try: 1020 import datetime 1021 return datetime.datetime.utcnow().ctime() + " UTC" 1022 except: 1023 return "(unknown)"
1024 1025 version = property(_getVersion, None, None, "Cedar Backup version.") 1026 interpreter = property(_getInterpreter, None, None, "Python interpreter version.") 1027 platform = property(_getPlatform, None, None, "Platform identifying information.") 1028 encoding = property(_getEncoding, None, None, "Filesystem encoding that is in effect.") 1029 locale = property(_getLocale, None, None, "Locale that is in effect.") 1030 timestamp = property(_getTimestamp, None, None, "Current timestamp.")
1031
1032 1033 ######################################################################## 1034 # General utility functions 1035 ######################################################################## 1036 1037 ###################### 1038 # sortDict() function 1039 ###################### 1040 1041 -def sortDict(d):
1042 """ 1043 Returns the keys of the dictionary sorted by value. 1044 1045 There are cuter ways to do this in Python 2.4, but we were originally 1046 attempting to stay compatible with Python 2.3. 1047 1048 @param d: Dictionary to operate on 1049 @return: List of dictionary keys sorted in order by dictionary value. 1050 """ 1051 items = d.items() 1052 items.sort(lambda x, y: cmp(x[1], y[1])) 1053 return [key for key, value in items]
1054
1055 1056 ######################## 1057 # removeKeys() function 1058 ######################## 1059 1060 -def removeKeys(d, keys):
1061 """ 1062 Removes all of the keys from the dictionary. 1063 The dictionary is altered in-place. 1064 Each key must exist in the dictionary. 1065 @param d: Dictionary to operate on 1066 @param keys: List of keys to remove 1067 @raise KeyError: If one of the keys does not exist 1068 """ 1069 for key in keys: 1070 del d[key]
1071
1072 1073 ######################### 1074 # convertSize() function 1075 ######################### 1076 1077 -def convertSize(size, fromUnit, toUnit):
1078 """ 1079 Converts a size in one unit to a size in another unit. 1080 1081 This is just a convenience function so that the functionality can be 1082 implemented in just one place. Internally, we convert values to bytes and 1083 then to the final unit. 1084 1085 The available units are: 1086 1087 - C{UNIT_BYTES} - Bytes 1088 - C{UNIT_KBYTES} - Kilobytes, where 1 kB = 1024 B 1089 - C{UNIT_MBYTES} - Megabytes, where 1 MB = 1024 kB 1090 - C{UNIT_GBYTES} - Gigabytes, where 1 GB = 1024 MB 1091 - C{UNIT_SECTORS} - Sectors, where 1 sector = 2048 B 1092 1093 @param size: Size to convert 1094 @type size: Integer or float value in units of C{fromUnit} 1095 1096 @param fromUnit: Unit to convert from 1097 @type fromUnit: One of the units listed above 1098 1099 @param toUnit: Unit to convert to 1100 @type toUnit: One of the units listed above 1101 1102 @return: Number converted to new unit, as a float. 1103 @raise ValueError: If one of the units is invalid. 1104 """ 1105 if size is None: 1106 raise ValueError("Cannot convert size of None.") 1107 if fromUnit == UNIT_BYTES: 1108 byteSize = float(size) 1109 elif fromUnit == UNIT_KBYTES: 1110 byteSize = float(size) * BYTES_PER_KBYTE 1111 elif fromUnit == UNIT_MBYTES: 1112 byteSize = float(size) * BYTES_PER_MBYTE 1113 elif fromUnit == UNIT_GBYTES: 1114 byteSize = float(size) * BYTES_PER_GBYTE 1115 elif fromUnit == UNIT_SECTORS: 1116 byteSize = float(size) * BYTES_PER_SECTOR 1117 else: 1118 raise ValueError("Unknown 'from' unit %s." % fromUnit) 1119 if toUnit == UNIT_BYTES: 1120 return byteSize 1121 elif toUnit == UNIT_KBYTES: 1122 return byteSize / BYTES_PER_KBYTE 1123 elif toUnit == UNIT_MBYTES: 1124 return byteSize / BYTES_PER_MBYTE 1125 elif toUnit == UNIT_GBYTES: 1126 return byteSize / BYTES_PER_GBYTE 1127 elif toUnit == UNIT_SECTORS: 1128 return byteSize / BYTES_PER_SECTOR 1129 else: 1130 raise ValueError("Unknown 'to' unit %s." % toUnit)
1131
1132 1133 ########################## 1134 # displayBytes() function 1135 ########################## 1136 1137 -def displayBytes(bytes, digits=2): # pylint: disable=W0622
1138 """ 1139 Format a byte quantity so it can be sensibly displayed. 1140 1141 It's rather difficult to look at a number like "72372224 bytes" and get any 1142 meaningful information out of it. It would be more useful to see something 1143 like "69.02 MB". That's what this function does. Any time you want to display 1144 a byte value, i.e.:: 1145 1146 print "Size: %s bytes" % bytes 1147 1148 Call this function instead:: 1149 1150 print "Size: %s" % displayBytes(bytes) 1151 1152 What comes out will be sensibly formatted. The indicated number of digits 1153 will be listed after the decimal point, rounded based on whatever rules are 1154 used by Python's standard C{%f} string format specifier. (Values less than 1 1155 kB will be listed in bytes and will not have a decimal point, since the 1156 concept of a fractional byte is nonsensical.) 1157 1158 @param bytes: Byte quantity. 1159 @type bytes: Integer number of bytes. 1160 1161 @param digits: Number of digits to display after the decimal point. 1162 @type digits: Integer value, typically 2-5. 1163 1164 @return: String, formatted for sensible display. 1165 """ 1166 if(bytes is None): 1167 raise ValueError("Cannot display byte value of None.") 1168 bytes = float(bytes) 1169 if math.fabs(bytes) < BYTES_PER_KBYTE: 1170 fmt = "%.0f bytes" 1171 value = bytes 1172 elif math.fabs(bytes) < BYTES_PER_MBYTE: 1173 fmt = "%." + "%d" % digits + "f kB" 1174 value = bytes / BYTES_PER_KBYTE 1175 elif math.fabs(bytes) < BYTES_PER_GBYTE: 1176 fmt = "%." + "%d" % digits + "f MB" 1177 value = bytes / BYTES_PER_MBYTE 1178 else: 1179 fmt = "%." + "%d" % digits + "f GB" 1180 value = bytes / BYTES_PER_GBYTE 1181 return fmt % value 1182
1183 1184 ################################## 1185 # getFunctionReference() function 1186 ################################## 1187 1188 -def getFunctionReference(module, function):
1189 """ 1190 Gets a reference to a named function. 1191 1192 This does some hokey-pokey to get back a reference to a dynamically named 1193 function. For instance, say you wanted to get a reference to the 1194 C{os.path.isdir} function. You could use:: 1195 1196 myfunc = getFunctionReference("os.path", "isdir") 1197 1198 Although we won't bomb out directly, behavior is pretty much undefined if 1199 you pass in C{None} or C{""} for either C{module} or C{function}. 1200 1201 The only validation we enforce is that whatever we get back must be 1202 callable. 1203 1204 I derived this code based on the internals of the Python unittest 1205 implementation. I don't claim to completely understand how it works. 1206 1207 @param module: Name of module associated with function. 1208 @type module: Something like "os.path" or "CedarBackup2.util" 1209 1210 @param function: Name of function 1211 @type function: Something like "isdir" or "getUidGid" 1212 1213 @return: Reference to function associated with name. 1214 1215 @raise ImportError: If the function cannot be found. 1216 @raise ValueError: If the resulting reference is not callable. 1217 1218 @copyright: Some of this code, prior to customization, was originally part 1219 of the Python 2.3 codebase. Python code is copyright (c) 2001, 2002 Python 1220 Software Foundation; All Rights Reserved. 1221 """ 1222 parts = [] 1223 if module is not None and module != "": 1224 parts = module.split(".") 1225 if function is not None and function != "": 1226 parts.append(function) 1227 copy = parts[:] 1228 while copy: 1229 try: 1230 module = __import__(string.join(copy, ".")) 1231 break 1232 except ImportError: 1233 del copy[-1] 1234 if not copy: raise 1235 parts = parts[1:] 1236 obj = module 1237 for part in parts: 1238 obj = getattr(obj, part) 1239 if not callable(obj): 1240 raise ValueError("Reference to %s.%s is not callable." % (module, function)) 1241 return obj
1242
1243 1244 ####################### 1245 # getUidGid() function 1246 ####################### 1247 1248 -def getUidGid(user, group):
1249 """ 1250 Get the uid/gid associated with a user/group pair 1251 1252 This is a no-op if user/group functionality is not available on the platform. 1253 1254 @param user: User name 1255 @type user: User name as a string 1256 1257 @param group: Group name 1258 @type group: Group name as a string 1259 1260 @return: Tuple C{(uid, gid)} matching passed-in user and group. 1261 @raise ValueError: If the ownership user/group values are invalid 1262 """ 1263 if _UID_GID_AVAILABLE: 1264 try: 1265 uid = pwd.getpwnam(user)[2] 1266 gid = grp.getgrnam(group)[2] 1267 return (uid, gid) 1268 except Exception, e: 1269 logger.debug("Error looking up uid and gid for [%s:%s]: %s" % (user, group, e)) 1270 raise ValueError("Unable to lookup up uid and gid for passed in user/group.") 1271 else: 1272 return (0, 0)
1273
1274 1275 ############################# 1276 # changeOwnership() function 1277 ############################# 1278 1279 -def changeOwnership(path, user, group):
1280 """ 1281 Changes ownership of path to match the user and group. 1282 1283 This is a no-op if user/group functionality is not available on the 1284 platform, or if the either passed-in user or group is C{None}. Further, we 1285 won't even try to do it unless running as root, since it's unlikely to work. 1286 1287 @param path: Path whose ownership to change. 1288 @param user: User which owns file. 1289 @param group: Group which owns file. 1290 """ 1291 if _UID_GID_AVAILABLE: 1292 if user is None or group is None: 1293 logger.debug("User or group is None, so not attempting to change owner on [%s]." % path) 1294 elif not isRunningAsRoot(): 1295 logger.debug("Not root, so not attempting to change owner on [%s]." % path) 1296 else: 1297 try: 1298 (uid, gid) = getUidGid(user, group) 1299 os.chown(path, uid, gid) 1300 except Exception, e: 1301 logger.error("Error changing ownership of [%s]: %s" % (path, e))
1302
1303 1304 ############################# 1305 # isRunningAsRoot() function 1306 ############################# 1307 1308 -def isRunningAsRoot():
1309 """ 1310 Indicates whether the program is running as the root user. 1311 """ 1312 return os.getuid() == 0
1313
1314 1315 ############################## 1316 # splitCommandLine() function 1317 ############################## 1318 1319 -def splitCommandLine(commandLine):
1320 """ 1321 Splits a command line string into a list of arguments. 1322 1323 Unfortunately, there is no "standard" way to parse a command line string, 1324 and it's actually not an easy problem to solve portably (essentially, we 1325 have to emulate the shell argument-processing logic). This code only 1326 respects double quotes (C{"}) for grouping arguments, not single quotes 1327 (C{'}). Make sure you take this into account when building your command 1328 line. 1329 1330 Incidentally, I found this particular parsing method while digging around in 1331 Google Groups, and I tweaked it for my own use. 1332 1333 @param commandLine: Command line string 1334 @type commandLine: String, i.e. "cback --verbose stage store" 1335 1336 @return: List of arguments, suitable for passing to C{popen2}. 1337 1338 @raise ValueError: If the command line is None. 1339 """ 1340 if commandLine is None: 1341 raise ValueError("Cannot split command line of None.") 1342 fields = re.findall('[^ "]+|"[^"]+"', commandLine) 1343 fields = map(lambda field: field.replace('"', ''), fields) 1344 return fields
1345
1346 1347 ############################ 1348 # resolveCommand() function 1349 ############################ 1350 1351 -def resolveCommand(command):
1352 """ 1353 Resolves the real path to a command through the path resolver mechanism. 1354 1355 Both extensions and standard Cedar Backup functionality need a way to 1356 resolve the "real" location of various executables. Normally, they assume 1357 that these executables are on the system path, but some callers need to 1358 specify an alternate location. 1359 1360 Ideally, we want to handle this configuration in a central location. The 1361 Cedar Backup path resolver mechanism (a singleton called 1362 L{PathResolverSingleton}) provides the central location to store the 1363 mappings. This function wraps access to the singleton, and is what all 1364 functions (extensions or standard functionality) should call if they need to 1365 find a command. 1366 1367 The passed-in command must actually be a list, in the standard form used by 1368 all existing Cedar Backup code (something like C{["svnlook", ]}). The 1369 lookup will actually be done on the first element in the list, and the 1370 returned command will always be in list form as well. 1371 1372 If the passed-in command can't be resolved or no mapping exists, then the 1373 command itself will be returned unchanged. This way, we neatly fall back on 1374 default behavior if we have no sensible alternative. 1375 1376 @param command: Command to resolve. 1377 @type command: List form of command, i.e. C{["svnlook", ]}. 1378 1379 @return: Path to command or just command itself if no mapping exists. 1380 """ 1381 singleton = PathResolverSingleton.getInstance() 1382 name = command[0] 1383 result = command[:] 1384 result[0] = singleton.lookup(name, name) 1385 return result
1386
1387 1388 ############################ 1389 # executeCommand() function 1390 ############################ 1391 1392 -def executeCommand(command, args, returnOutput=False, ignoreStderr=False, doNotLog=False, outputFile=None):
1393 """ 1394 Executes a shell command, hopefully in a safe way. 1395 1396 This function exists to replace direct calls to C{os.popen} in the Cedar 1397 Backup code. It's not safe to call a function such as C{os.popen()} with 1398 untrusted arguments, since that can cause problems if the string contains 1399 non-safe variables or other constructs (imagine that the argument is 1400 C{$WHATEVER}, but C{$WHATEVER} contains something like C{"; rm -fR ~/; 1401 echo"} in the current environment). 1402 1403 Instead, it's safer to pass a list of arguments in the style supported bt 1404 C{popen2} or C{popen4}. This function actually uses a specialized C{Pipe} 1405 class implemented using either C{subprocess.Popen} or C{popen2.Popen4}. 1406 1407 Under the normal case, this function will return a tuple of C{(status, 1408 None)} where the status is the wait-encoded return status of the call per 1409 the C{popen2.Popen4} documentation. If C{returnOutput} is passed in as 1410 C{True}, the function will return a tuple of C{(status, output)} where 1411 C{output} is a list of strings, one entry per line in the output from the 1412 command. Output is always logged to the C{outputLogger.info()} target, 1413 regardless of whether it's returned. 1414 1415 By default, C{stdout} and C{stderr} will be intermingled in the output. 1416 However, if you pass in C{ignoreStderr=True}, then only C{stdout} will be 1417 included in the output. 1418 1419 The C{doNotLog} parameter exists so that callers can force the function to 1420 not log command output to the debug log. Normally, you would want to log. 1421 However, if you're using this function to write huge output files (i.e. 1422 database backups written to C{stdout}) then you might want to avoid putting 1423 all that information into the debug log. 1424 1425 The C{outputFile} parameter exists to make it easier for a caller to push 1426 output into a file, i.e. as a substitute for redirection to a file. If this 1427 value is passed in, each time a line of output is generated, it will be 1428 written to the file using C{outputFile.write()}. At the end, the file 1429 descriptor will be flushed using C{outputFile.flush()}. The caller 1430 maintains responsibility for closing the file object appropriately. 1431 1432 @note: I know that it's a bit confusing that the command and the arguments 1433 are both lists. I could have just required the caller to pass in one big 1434 list. However, I think it makes some sense to keep the command (the 1435 constant part of what we're executing, i.e. C{"scp -B"}) separate from its 1436 arguments, even if they both end up looking kind of similar. 1437 1438 @note: You cannot redirect output via shell constructs (i.e. C{>file}, 1439 C{2>/dev/null}, etc.) using this function. The redirection string would be 1440 passed to the command just like any other argument. However, you can 1441 implement the equivalent to redirection using C{ignoreStderr} and 1442 C{outputFile}, as discussed above. 1443 1444 @note: The operating system environment is partially sanitized before 1445 the command is invoked. See L{sanitizeEnvironment} for details. 1446 1447 @param command: Shell command to execute 1448 @type command: List of individual arguments that make up the command 1449 1450 @param args: List of arguments to the command 1451 @type args: List of additional arguments to the command 1452 1453 @param returnOutput: Indicates whether to return the output of the command 1454 @type returnOutput: Boolean C{True} or C{False} 1455 1456 @param doNotLog: Indicates that output should not be logged. 1457 @type doNotLog: Boolean C{True} or C{False} 1458 1459 @param outputFile: File object that all output should be written to. 1460 @type outputFile: File object as returned from C{open()} or C{file()}. 1461 1462 @return: Tuple of C{(result, output)} as described above. 1463 """ 1464 logger.debug("Executing command %s with args %s." % (command, args)) 1465 outputLogger.info("Executing command %s with args %s." % (command, args)) 1466 if doNotLog: 1467 logger.debug("Note: output will not be logged, per the doNotLog flag.") 1468 outputLogger.info("Note: output will not be logged, per the doNotLog flag.") 1469 output = [] 1470 fields = command[:] # make sure to copy it so we don't destroy it 1471 fields.extend(args) 1472 try: 1473 sanitizeEnvironment() # make sure we have a consistent environment 1474 try: 1475 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1476 except OSError: 1477 # On some platforms (i.e. Cygwin) this intermittently fails the first time we do it. 1478 # So, we attempt it a second time and if that works, we just go on as usual. 1479 # The problem appears to be that we sometimes get a bad stderr file descriptor. 1480 pipe = Pipe(fields, ignoreStderr=ignoreStderr) 1481 while True: 1482 line = pipe.stdout.readline() 1483 if not line: break 1484 if returnOutput: output.append(line) 1485 if outputFile is not None: outputFile.write(line) 1486 if not doNotLog: outputLogger.info(line[:-1]) # this way the log will (hopefully) get updated in realtime 1487 if outputFile is not None: 1488 try: # note, not every file-like object can be flushed 1489 outputFile.flush() 1490 except: pass 1491 if returnOutput: 1492 return (pipe.wait(), output) 1493 else: 1494 return (pipe.wait(), None) 1495 except OSError, e: 1496 try: 1497 if returnOutput: 1498 if output != []: 1499 return (pipe.wait(), output) 1500 else: 1501 return (pipe.wait(), [ e, ]) 1502 else: 1503 return (pipe.wait(), None) 1504 except UnboundLocalError: # pipe not set 1505 if returnOutput: 1506 return (256, []) 1507 else: 1508 return (256, None)
1509
1510 1511 ############################## 1512 # calculateFileAge() function 1513 ############################## 1514 1515 -def calculateFileAge(path):
1516 """ 1517 Calculates the age (in days) of a file. 1518 1519 The "age" of a file is the amount of time since the file was last used, per 1520 the most recent of the file's C{st_atime} and C{st_mtime} values. 1521 1522 Technically, we only intend this function to work with files, but it will 1523 probably work with anything on the filesystem. 1524 1525 @param path: Path to a file on disk. 1526 1527 @return: Age of the file in days (possibly fractional). 1528 @raise OSError: If the file doesn't exist. 1529 """ 1530 currentTime = int(time.time()) 1531 fileStats = os.stat(path) 1532 lastUse = max(fileStats.st_atime, fileStats.st_mtime) # "most recent" is "largest" 1533 ageInSeconds = currentTime - lastUse 1534 ageInDays = ageInSeconds / SECONDS_PER_DAY 1535 return ageInDays
1536
1537 1538 ################### 1539 # mount() function 1540 ################### 1541 1542 -def mount(devicePath, mountPoint, fsType):
1543 """ 1544 Mounts the indicated device at the indicated mount point. 1545 1546 For instance, to mount a CD, you might use device path C{/dev/cdrw}, mount 1547 point C{/media/cdrw} and filesystem type C{iso9660}. You can safely use any 1548 filesystem type that is supported by C{mount} on your platform. If the type 1549 is C{None}, we'll attempt to let C{mount} auto-detect it. This may or may 1550 not work on all systems. 1551 1552 @note: This only works on platforms that have a concept of "mounting" a 1553 filesystem through a command-line C{"mount"} command, like UNIXes. It 1554 won't work on Windows. 1555 1556 @param devicePath: Path of device to be mounted. 1557 @param mountPoint: Path that device should be mounted at. 1558 @param fsType: Type of the filesystem assumed to be available via the device. 1559 1560 @raise IOError: If the device cannot be mounted. 1561 """ 1562 if fsType is None: 1563 args = [ devicePath, mountPoint ] 1564 else: 1565 args = [ "-t", fsType, devicePath, mountPoint ] 1566 command = resolveCommand(MOUNT_COMMAND) 1567 result = executeCommand(command, args, returnOutput=False, ignoreStderr=True)[0] 1568 if result != 0: 1569 raise IOError("Error [%d] mounting [%s] at [%s] as [%s]." % (result, devicePath, mountPoint, fsType))
1570
1571 1572 ##################### 1573 # unmount() function 1574 ##################### 1575 1576 -def unmount(mountPoint, removeAfter=False, attempts=1, waitSeconds=0):
1577 """ 1578 Unmounts whatever device is mounted at the indicated mount point. 1579 1580 Sometimes, it might not be possible to unmount the mount point immediately, 1581 if there are still files open there. Use the C{attempts} and C{waitSeconds} 1582 arguments to indicate how many unmount attempts to make and how many seconds 1583 to wait between attempts. If you pass in zero attempts, no attempts will be 1584 made (duh). 1585 1586 If the indicated mount point is not really a mount point per 1587 C{os.path.ismount()}, then it will be ignored. This seems to be a safer 1588 check then looking through C{/etc/mtab}, since C{ismount()} is already in 1589 the Python standard library and is documented as working on all POSIX 1590 systems. 1591 1592 If C{removeAfter} is C{True}, then the mount point will be removed using 1593 C{os.rmdir()} after the unmount action succeeds. If for some reason the 1594 mount point is not a directory, then it will not be removed. 1595 1596 @note: This only works on platforms that have a concept of "mounting" a 1597 filesystem through a command-line C{"mount"} command, like UNIXes. It 1598 won't work on Windows. 1599 1600 @param mountPoint: Mount point to be unmounted. 1601 @param removeAfter: Remove the mount point after unmounting it. 1602 @param attempts: Number of times to attempt the unmount. 1603 @param waitSeconds: Number of seconds to wait between repeated attempts. 1604 1605 @raise IOError: If the mount point is still mounted after attempts are exhausted. 1606 """ 1607 if os.path.ismount(mountPoint): 1608 for attempt in range(0, attempts): 1609 logger.debug("Making attempt %d to unmount [%s]." % (attempt, mountPoint)) 1610 command = resolveCommand(UMOUNT_COMMAND) 1611 result = executeCommand(command, [ mountPoint, ], returnOutput=False, ignoreStderr=True)[0] 1612 if result != 0: 1613 logger.error("Error [%d] unmounting [%s] on attempt %d." % (result, mountPoint, attempt)) 1614 elif os.path.ismount(mountPoint): 1615 logger.error("After attempt %d, [%s] is still mounted." % (attempt, mountPoint)) 1616 else: 1617 logger.debug("Successfully unmounted [%s] on attempt %d." % (mountPoint, attempt)) 1618 break # this will cause us to skip the loop else: clause 1619 if attempt+1 < attempts: # i.e. this isn't the last attempt 1620 if waitSeconds > 0: 1621 logger.info("Sleeping %d second(s) before next unmount attempt." % waitSeconds) 1622 time.sleep(waitSeconds) 1623 else: 1624 if os.path.ismount(mountPoint): 1625 raise IOError("Unable to unmount [%s] after %d attempts." % (mountPoint, attempts)) 1626 logger.info("Mount point [%s] seems to have finally gone away." % mountPoint) 1627 if os.path.isdir(mountPoint) and removeAfter: 1628 logger.debug("Removing mount point [%s]." % mountPoint) 1629 os.rmdir(mountPoint)
1630
1631 1632 ########################### 1633 # deviceMounted() function 1634 ########################### 1635 1636 -def deviceMounted(devicePath):
1637 """ 1638 Indicates whether a specific filesystem device is currently mounted. 1639 1640 We determine whether the device is mounted by looking through the system's 1641 C{mtab} file. This file shows every currently-mounted filesystem, ordered 1642 by device. We only do the check if the C{mtab} file exists and is readable. 1643 Otherwise, we assume that the device is not mounted. 1644 1645 @note: This only works on platforms that have a concept of an mtab file 1646 to show mounted volumes, like UNIXes. It won't work on Windows. 1647 1648 @param devicePath: Path of device to be checked 1649 1650 @return: True if device is mounted, false otherwise. 1651 """ 1652 if os.path.exists(MTAB_FILE) and os.access(MTAB_FILE, os.R_OK): 1653 realPath = os.path.realpath(devicePath) 1654 lines = open(MTAB_FILE).readlines() 1655 for line in lines: 1656 (mountDevice, mountPoint, remainder) = line.split(None, 2) 1657 if mountDevice in [ devicePath, realPath, ]: 1658 logger.debug("Device [%s] is mounted at [%s]." % (devicePath, mountPoint)) 1659 return True 1660 return False
1661
1662 1663 ######################## 1664 # encodePath() function 1665 ######################## 1666 1667 -def encodePath(path):
1668 1669 r""" 1670 Safely encodes a filesystem path. 1671 1672 Many Python filesystem functions, such as C{os.listdir}, behave differently 1673 if they are passed unicode arguments versus simple string arguments. For 1674 instance, C{os.listdir} generally returns unicode path names if it is passed 1675 a unicode argument, and string pathnames if it is passed a string argument. 1676 1677 However, this behavior often isn't as consistent as we might like. As an example, 1678 C{os.listdir} "gives up" if it finds a filename that it can't properly encode 1679 given the current locale settings. This means that the returned list is 1680 a mixed set of unicode and simple string paths. This has consequences later, 1681 because other filesystem functions like C{os.path.join} will blow up if they 1682 are given one string path and one unicode path. 1683 1684 On comp.lang.python, Martin v. Löwis explained the C{os.listdir} behavior 1685 like this:: 1686 1687 The operating system (POSIX) does not have the inherent notion that file 1688 names are character strings. Instead, in POSIX, file names are primarily 1689 byte strings. There are some bytes which are interpreted as characters 1690 (e.g. '\x2e', which is '.', or '\x2f', which is '/'), but apart from 1691 that, most OS layers think these are just bytes. 1692 1693 Now, most *people* think that file names are character strings. To 1694 interpret a file name as a character string, you need to know what the 1695 encoding is to interpret the file names (which are byte strings) as 1696 character strings. 1697 1698 There is, unfortunately, no operating system API to carry the notion of a 1699 file system encoding. By convention, the locale settings should be used 1700 to establish this encoding, in particular the LC_CTYPE facet of the 1701 locale. This is defined in the environment variables LC_CTYPE, LC_ALL, 1702 and LANG (searched in this order). 1703 1704 If LANG is not set, the "C" locale is assumed, which uses ASCII as its 1705 file system encoding. In this locale, '\xe2\x99\xaa\xe2\x99\xac' is not a 1706 valid file name (at least it cannot be interpreted as characters, and 1707 hence not be converted to Unicode). 1708 1709 Now, your Python script has requested that all file names *should* be 1710 returned as character (ie. Unicode) strings, but Python cannot comply, 1711 since there is no way to find out what this byte string means, in terms 1712 of characters. 1713 1714 So we have three options: 1715 1716 1. Skip this string, only return the ones that can be converted to Unicode. 1717 Give the user the impression the file does not exist. 1718 2. Return the string as a byte string 1719 3. Refuse to listdir altogether, raising an exception (i.e. return nothing) 1720 1721 Python has chosen alternative 2, allowing the application to implement 1 1722 or 3 on top of that if it wants to (or come up with other strategies, 1723 such as user feedback). 1724 1725 As a solution, he suggests that rather than passing unicode paths into the 1726 filesystem functions, that I should sensibly encode the path first. That is 1727 what this function accomplishes. Any function which takes a filesystem path 1728 as an argument should encode it first, before using it for any other purpose. 1729 1730 I confess I still don't completely understand how this works. On a system 1731 with filesystem encoding "ISO-8859-1", a path C{u"\xe2\x99\xaa\xe2\x99\xac"} 1732 is converted into the string C{"\xe2\x99\xaa\xe2\x99\xac"}. However, on a 1733 system with a "utf-8" encoding, the result is a completely different string: 1734 C{"\xc3\xa2\xc2\x99\xc2\xaa\xc3\xa2\xc2\x99\xc2\xac"}. A quick test where I 1735 write to the first filename and open the second proves that the two strings 1736 represent the same file on disk, which is all I really care about. 1737 1738 @note: As a special case, if C{path} is C{None}, then this function will 1739 return C{None}. 1740 1741 @note: To provide several examples of encoding values, my Debian sarge box 1742 with an ext3 filesystem has Python filesystem encoding C{ISO-8859-1}. User 1743 Anarcat's Debian box with a xfs filesystem has filesystem encoding 1744 C{ANSI_X3.4-1968}. Both my iBook G4 running Mac OS X 10.4 and user Dag 1745 Rende's SuSE 9.3 box both have filesystem encoding C{UTF-8}. 1746 1747 @note: Just because a filesystem has C{UTF-8} encoding doesn't mean that it 1748 will be able to handle all extended-character filenames. For instance, 1749 certain extended-character (but not UTF-8) filenames -- like the ones in the 1750 regression test tar file C{test/data/tree13.tar.gz} -- are not valid under 1751 Mac OS X, and it's not even possible to extract them from the tarfile on 1752 that platform. 1753 1754 @param path: Path to encode 1755 1756 @return: Path, as a string, encoded appropriately 1757 @raise ValueError: If the path cannot be encoded properly. 1758 """ 1759 if path is None: 1760 return path 1761 try: 1762 if isinstance(path, unicode): 1763 encoding = sys.getfilesystemencoding() or sys.getdefaultencoding() 1764 path = path.encode(encoding) 1765 return path 1766 except UnicodeError: 1767 raise ValueError("Path could not be safely encoded as %s." % encoding)
1768
1769 1770 ######################## 1771 # nullDevice() function 1772 ######################## 1773 1774 -def nullDevice():
1775 """ 1776 Attempts to portably return the null device on this system. 1777 1778 The null device is something like C{/dev/null} on a UNIX system. The name 1779 varies on other platforms. 1780 """ 1781 return os.devnull
1782
1783 1784 ############################## 1785 # deriveDayOfWeek() function 1786 ############################## 1787 1788 -def deriveDayOfWeek(dayName):
1789 """ 1790 Converts English day name to numeric day of week as from C{time.localtime}. 1791 1792 For instance, the day C{monday} would be converted to the number C{0}. 1793 1794 @param dayName: Day of week to convert 1795 @type dayName: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1796 1797 @returns: Integer, where Monday is 0 and Sunday is 6; or -1 if no conversion is possible. 1798 """ 1799 if dayName.lower() == "monday": 1800 return 0 1801 elif dayName.lower() == "tuesday": 1802 return 1 1803 elif dayName.lower() == "wednesday": 1804 return 2 1805 elif dayName.lower() == "thursday": 1806 return 3 1807 elif dayName.lower() == "friday": 1808 return 4 1809 elif dayName.lower() == "saturday": 1810 return 5 1811 elif dayName.lower() == "sunday": 1812 return 6 1813 else: 1814 return -1 # What else can we do?? Thrown an exception, I guess.
1815
1816 1817 ########################### 1818 # isStartOfWeek() function 1819 ########################### 1820 1821 -def isStartOfWeek(startingDay):
1822 """ 1823 Indicates whether "today" is the backup starting day per configuration. 1824 1825 If the current day's English name matches the indicated starting day, then 1826 today is a starting day. 1827 1828 @param startingDay: Configured starting day. 1829 @type startingDay: string, i.e. C{"monday"}, C{"tuesday"}, etc. 1830 1831 @return: Boolean indicating whether today is the starting day. 1832 """ 1833 value = time.localtime().tm_wday == deriveDayOfWeek(startingDay) 1834 if value: 1835 logger.debug("Today is the start of the week.") 1836 else: 1837 logger.debug("Today is NOT the start of the week.") 1838 return value
1839
1840 1841 ################################# 1842 # buildNormalizedPath() function 1843 ################################# 1844 1845 -def buildNormalizedPath(path):
1846 """ 1847 Returns a "normalized" path based on a path name. 1848 1849 A normalized path is a representation of a path that is also a valid file 1850 name. To make a valid file name out of a complete path, we have to convert 1851 or remove some characters that are significant to the filesystem -- in 1852 particular, the path separator and any leading C{'.'} character (which would 1853 cause the file to be hidden in a file listing). 1854 1855 Note that this is a one-way transformation -- you can't safely derive the 1856 original path from the normalized path. 1857 1858 To normalize a path, we begin by looking at the first character. If the 1859 first character is C{'/'} or C{'\\'}, it gets removed. If the first 1860 character is C{'.'}, it gets converted to C{'_'}. Then, we look through the 1861 rest of the path and convert all remaining C{'/'} or C{'\\'} characters 1862 C{'-'}, and all remaining whitespace characters to C{'_'}. 1863 1864 As a special case, a path consisting only of a single C{'/'} or C{'\\'} 1865 character will be converted to C{'-'}. 1866 1867 @param path: Path to normalize 1868 1869 @return: Normalized path as described above. 1870 1871 @raise ValueError: If the path is None 1872 """ 1873 if path is None: 1874 raise ValueError("Cannot normalize path None.") 1875 elif len(path) == 0: 1876 return path 1877 elif path == "/" or path == "\\": 1878 return "-" 1879 else: 1880 normalized = path 1881 normalized = re.sub(r"^\/", "", normalized) # remove leading '/' 1882 normalized = re.sub(r"^\\", "", normalized) # remove leading '\' 1883 normalized = re.sub(r"^\.", "_", normalized) # convert leading '.' to '_' so file won't be hidden 1884 normalized = re.sub(r"\/", "-", normalized) # convert all '/' characters to '-' 1885 normalized = re.sub(r"\\", "-", normalized) # convert all '\' characters to '-' 1886 normalized = re.sub(r"\s", "_", normalized) # convert all whitespace to '_' 1887 return normalized
1888
1889 1890 ################################# 1891 # sanitizeEnvironment() function 1892 ################################# 1893 1894 -def sanitizeEnvironment():
1895 """ 1896 Sanitizes the operating system environment. 1897 1898 The operating system environment is contained in C{os.environ}. This method 1899 sanitizes the contents of that dictionary. 1900 1901 Currently, all it does is reset the locale (removing C{$LC_*}) and set the 1902 default language (C{$LANG}) to L{DEFAULT_LANGUAGE}. This way, we can count 1903 on consistent localization regardless of what the end-user has configured. 1904 This is important for code that needs to parse program output. 1905 1906 The C{os.environ} dictionary is modifed in-place. If C{$LANG} is already 1907 set to the proper value, it is not re-set, so we can avoid the memory leaks 1908 that are documented to occur on BSD-based systems. 1909 1910 @return: Copy of the sanitized environment. 1911 """ 1912 for var in LOCALE_VARS: 1913 if os.environ.has_key(var): 1914 del os.environ[var] 1915 if os.environ.has_key(LANG_VAR): 1916 if os.environ[LANG_VAR] != DEFAULT_LANGUAGE: # no need to reset if it exists (avoid leaks on BSD systems) 1917 os.environ[LANG_VAR] = DEFAULT_LANGUAGE 1918 return os.environ.copy()
1919 1938
1939 1940 ######################### 1941 # checkUnique() function 1942 ######################### 1943 1944 -def checkUnique(prefix, values):
1945 """ 1946 Checks that all values are unique. 1947 1948 The values list is checked for duplicate values. If there are 1949 duplicates, an exception is thrown. All duplicate values are listed in 1950 the exception. 1951 1952 @param prefix: Prefix to use in the thrown exception 1953 @param values: List of values to check 1954 1955 @raise ValueError: If there are duplicates in the list 1956 """ 1957 values.sort() 1958 duplicates = [] 1959 for i in range(1, len(values)): 1960 if values[i-1] == values[i]: 1961 duplicates.append(values[i]) 1962 if duplicates: 1963 raise ValueError("%s %s" % (prefix, duplicates))
1964
1965 1966 ####################################### 1967 # parseCommaSeparatedString() function 1968 ####################################### 1969 1970 -def parseCommaSeparatedString(commaString):
1971 """ 1972 Parses a list of values out of a comma-separated string. 1973 1974 The items in the list are split by comma, and then have whitespace 1975 stripped. As a special case, if C{commaString} is C{None}, then C{None} 1976 will be returned. 1977 1978 @param commaString: List of values in comma-separated string format. 1979 @return: Values from commaString split into a list, or C{None}. 1980 """ 1981 if commaString is None: 1982 return None 1983 else: 1984 pass1 = commaString.split(",") 1985 pass2 = [] 1986 for item in pass1: 1987 item = item.strip() 1988 if len(item) > 0: 1989 pass2.append(item) 1990 return pass2
1991