Package dbf :: Module tables
[hide private]

Source Code for Module dbf.tables

   1  "table definitions" 
   2  import os 
   3  import sys 
   4  import csv 
   5  import codecs 
   6  import locale 
   7  import unicodedata 
   8  import weakref 
   9  from array import array 
  10  from bisect import bisect_left, bisect_right 
  11  from decimal import Decimal 
  12  from shutil import copyfileobj 
  13  import dbf 
  14  from dbf import _io as io 
  15  from dbf.dates import Date, DateTime, Time 
  16  from dbf.exceptions import Bof, Eof, DbfError, DataOverflow, FieldMissing, NonUnicode, DoNotIndex 
  17   
  18  input_decoding = locale.getdefaultlocale()[1]    # treat non-unicode data as ... 
  19  default_codepage = 'cp1252'  # if no codepage specified on dbf creation, use this 
  20  return_ascii = False         # if True -- convert back to icky ascii, losing chars if no mapping 
  21   
  22  version_map = { 
  23          '\x02' : 'FoxBASE', 
  24          '\x03' : 'dBase III Plus', 
  25          '\x04' : 'dBase IV', 
  26          '\x05' : 'dBase V', 
  27          '\x30' : 'Visual FoxPro', 
  28          '\x31' : 'Visual FoxPro (auto increment field)', 
  29          '\x43' : 'dBase IV SQL', 
  30          '\x7b' : 'dBase IV w/memos', 
  31          '\x83' : 'dBase III Plus w/memos', 
  32          '\x8b' : 'dBase IV w/memos', 
  33          '\x8e' : 'dBase IV w/SQL table', 
  34          '\xf5' : 'FoxPro w/memos'} 
  35   
  36  code_pages = { 
  37          '\x00' : ('ascii', "plain ol' ascii"), 
  38          '\x01' : ('cp437', 'U.S. MS-DOS'), 
  39          '\x02' : ('cp850', 'International MS-DOS'), 
  40          '\x03' : ('cp1252', 'Windows ANSI'), 
  41          '\x04' : ('mac_roman', 'Standard Macintosh'), 
  42          '\x08' : ('cp865', 'Danish OEM'), 
  43          '\x09' : ('cp437', 'Dutch OEM'), 
  44          '\x0A' : ('cp850', 'Dutch OEM (secondary)'), 
  45          '\x0B' : ('cp437', 'Finnish OEM'), 
  46          '\x0D' : ('cp437', 'French OEM'), 
  47          '\x0E' : ('cp850', 'French OEM (secondary)'), 
  48          '\x0F' : ('cp437', 'German OEM'), 
  49          '\x10' : ('cp850', 'German OEM (secondary)'), 
  50          '\x11' : ('cp437', 'Italian OEM'), 
  51          '\x12' : ('cp850', 'Italian OEM (secondary)'), 
  52          '\x13' : ('cp932', 'Japanese Shift-JIS'), 
  53          '\x14' : ('cp850', 'Spanish OEM (secondary)'), 
  54          '\x15' : ('cp437', 'Swedish OEM'), 
  55          '\x16' : ('cp850', 'Swedish OEM (secondary)'), 
  56          '\x17' : ('cp865', 'Norwegian OEM'), 
  57          '\x18' : ('cp437', 'Spanish OEM'), 
  58          '\x19' : ('cp437', 'English OEM (Britain)'), 
  59          '\x1A' : ('cp850', 'English OEM (Britain) (secondary)'), 
  60          '\x1B' : ('cp437', 'English OEM (U.S.)'), 
  61          '\x1C' : ('cp863', 'French OEM (Canada)'), 
  62          '\x1D' : ('cp850', 'French OEM (secondary)'), 
  63          '\x1F' : ('cp852', 'Czech OEM'), 
  64          '\x22' : ('cp852', 'Hungarian OEM'), 
  65          '\x23' : ('cp852', 'Polish OEM'), 
  66          '\x24' : ('cp860', 'Portugese OEM'), 
  67          '\x25' : ('cp850', 'Potugese OEM (secondary)'), 
  68          '\x26' : ('cp866', 'Russian OEM'), 
  69          '\x37' : ('cp850', 'English OEM (U.S.) (secondary)'), 
  70          '\x40' : ('cp852', 'Romanian OEM'), 
  71          '\x4D' : ('cp936', 'Chinese GBK (PRC)'), 
  72          '\x4E' : ('cp949', 'Korean (ANSI/OEM)'), 
  73          '\x4F' : ('cp950', 'Chinese Big 5 (Taiwan)'), 
  74          '\x50' : ('cp874', 'Thai (ANSI/OEM)'), 
  75          '\x57' : ('cp1252', 'ANSI'), 
  76          '\x58' : ('cp1252', 'Western European ANSI'), 
  77          '\x59' : ('cp1252', 'Spanish ANSI'), 
  78          '\x64' : ('cp852', 'Eastern European MS-DOS'), 
  79          '\x65' : ('cp866', 'Russian MS-DOS'), 
  80          '\x66' : ('cp865', 'Nordic MS-DOS'), 
  81          '\x67' : ('cp861', 'Icelandic MS-DOS'), 
  82          '\x68' : (None, 'Kamenicky (Czech) MS-DOS'), 
  83          '\x69' : (None, 'Mazovia (Polish) MS-DOS'), 
  84          '\x6a' : ('cp737', 'Greek MS-DOS (437G)'), 
  85          '\x6b' : ('cp857', 'Turkish MS-DOS'), 
  86          '\x78' : ('cp950', 'Traditional Chinese (Hong Kong SAR, Taiwan) Windows'), 
  87          '\x79' : ('cp949', 'Korean Windows'), 
  88          '\x7a' : ('cp936', 'Chinese Simplified (PRC, Singapore) Windows'), 
  89          '\x7b' : ('cp932', 'Japanese Windows'), 
  90          '\x7c' : ('cp874', 'Thai Windows'), 
  91          '\x7d' : ('cp1255', 'Hebrew Windows'), 
  92          '\x7e' : ('cp1256', 'Arabic Windows'), 
  93          '\xc8' : ('cp1250', 'Eastern European Windows'), 
  94          '\xc9' : ('cp1251', 'Russian Windows'), 
  95          '\xca' : ('cp1254', 'Turkish Windows'), 
  96          '\xcb' : ('cp1253', 'Greek Windows'), 
  97          '\x96' : ('mac_cyrillic', 'Russian Macintosh'), 
  98          '\x97' : ('mac_latin2', 'Macintosh EE'), 
  99          '\x98' : ('mac_greek', 'Greek Macintosh') } 
 100   
 101  if sys.version_info[:2] < (2, 6): 
102 # define our own property type 103 - class property(object):
104 "Emulate PyProperty_Type() in Objects/descrobject.c" 105
106 - def __init__(self, fget=None, fset=None, fdel=None, doc=None):
107 self.fget = fget 108 self.fset = fset 109 self.fdel = fdel 110 self.__doc__ = doc or fget.__doc__
111 - def __call__(self, func):
112 self.fget = func 113 if not self.__doc__: 114 self.__doc__ = fget.__doc__
115 - def __get__(self, obj, objtype=None):
116 if obj is None: 117 return self 118 if self.fget is None: 119 raise AttributeError, "unreadable attribute" 120 return self.fget(obj)
121 - def __set__(self, obj, value):
122 if self.fset is None: 123 raise AttributeError, "can't set attribute" 124 self.fset(obj, value)
125 - def __delete__(self, obj):
126 if self.fdel is None: 127 raise AttributeError, "can't delete attribute" 128 self.fdel(obj)
129 - def setter(self, func):
130 self.fset = func 131 return self
132 - def deleter(self, func):
133 self.fdel = func 134 return self
135 # Internal classes
136 -class _DbfRecord(object):
137 """Provides routines to extract and save data within the fields of a dbf record.""" 138 __slots__ = ['_recnum', '_layout', '_data', '_dirty', '__weakref__']
139 - def _retrieveFieldValue(yo, record_data, fielddef):
140 """calls appropriate routine to fetch value stored in field from array 141 @param record_data: the data portion of the record 142 @type record_data: array of characters 143 @param fielddef: description of the field definition 144 @type fielddef: dictionary with keys 'type', 'start', 'length', 'end', 'decimals', and 'flags' 145 @returns: python data stored in field""" 146 147 field_type = fielddef['type'] 148 retrieve = yo._layout.fieldtypes[field_type]['Retrieve'] 149 datum = retrieve(record_data, fielddef, yo._layout.memo) 150 if field_type in yo._layout.character_fields: 151 datum = yo._layout.decoder(datum)[0] 152 if yo._layout.return_ascii: 153 try: 154 datum = yo._layout.output_encoder(datum)[0] 155 except UnicodeEncodeError: 156 datum = unicodedata.normalize('NFD', datum).encode('ascii','ignore') 157 return datum
158 - def _updateFieldValue(yo, fielddef, value):
159 "calls appropriate routine to convert value to ascii bytes, and save it in record" 160 field_type = fielddef['type'] 161 update = yo._layout.fieldtypes[field_type]['Update'] 162 if field_type in yo._layout.character_fields: 163 if not isinstance(value, unicode): 164 if yo._layout.input_decoder is None: 165 raise NonUnicode("String not in unicode format, no default encoding specified") 166 value = yo._layout.input_decoder(value)[0] # input ascii => unicode 167 value = yo._layout.encoder(value)[0] # unicode => table ascii 168 bytes = array('c', update(value, fielddef, yo._layout.memo)) 169 size = fielddef['length'] 170 if len(bytes) > size: 171 raise DataOverflow("tried to store %d bytes in %d byte field" % (len(bytes), size)) 172 blank = array('c', ' ' * size) 173 start = fielddef['start'] 174 end = start + size 175 blank[:len(bytes)] = bytes[:] 176 yo._data[start:end] = blank[:] 177 yo._dirty = True
178 - def _update_disk(yo, location='', data=None):
179 if not yo._layout.inmemory: 180 if yo._recnum < 0: 181 raise DbfError("Attempted to update record that has been packed") 182 if location == '': 183 location = yo._recnum * yo._layout.header.record_length + yo._layout.header.start 184 if data is None: 185 data = yo._data 186 yo._layout.dfd.seek(location) 187 yo._layout.dfd.write(data) 188 yo._dirty = False 189 for index in yo.record_table._indexen: 190 index(yo)
191 - def __call__(yo, *specs):
192 results = [] 193 if not specs: 194 specs = yo._layout.index 195 specs = _normalize_tuples(tuples=specs, length=2, filler=[_nop]) 196 for field, func in specs: 197 results.append(func(yo[field])) 198 return tuple(results)
199
200 - def __contains__(yo, key):
201 return key in yo._layout.fields or key in ['record_number','delete_flag']
202 - def __iter__(yo):
203 return (yo[field] for field in yo._layout.fields)
204 - def __getattr__(yo, name):
205 if name[0:2] == '__' and name[-2:] == '__': 206 raise AttributeError, 'Method %s is not implemented.' % name 207 elif name == 'record_number': 208 return yo._recnum 209 elif name == 'delete_flag': 210 return yo._data[0] != ' ' 211 elif not name in yo._layout.fields: 212 raise FieldMissing(name) 213 try: 214 fielddef = yo._layout[name] 215 value = yo._retrieveFieldValue(yo._data[fielddef['start']:fielddef['end']], fielddef) 216 return value 217 except DbfError, error: 218 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 219 raise
220 - def __getitem__(yo, item):
221 if type(item) in (int, long): 222 if not -yo._layout.header.field_count <= item < yo._layout.header.field_count: 223 raise IndexError("Field offset %d is not in record" % item) 224 return yo[yo._layout.fields[item]] 225 elif type(item) == slice: 226 sequence = [] 227 for index in yo._layout.fields[item]: 228 sequence.append(yo[index]) 229 return sequence 230 elif type(item) == str: 231 return yo.__getattr__(item) 232 else: 233 raise TypeError("%s is not a field name" % item)
234 - def __len__(yo):
235 return yo._layout.header.field_count
236 - def __new__(cls, recnum, layout, kamikaze='', _fromdisk=False):
237 """record = ascii array of entire record; layout=record specification; memo = memo object for table""" 238 record = object.__new__(cls) 239 record._dirty = False 240 record._recnum = recnum 241 record._layout = layout 242 if layout.blankrecord is None and not _fromdisk: 243 record._createBlankRecord() 244 record._data = layout.blankrecord 245 if recnum == -1: # not a disk-backed record 246 return record 247 elif type(kamikaze) == array: 248 record._data = kamikaze[:] 249 elif type(kamikaze) == str: 250 record._data = array('c', kamikaze) 251 else: 252 record._data = kamikaze._data[:] 253 datalen = len(record._data) 254 if datalen < layout.header.record_length: 255 record._data.extend(layout.blankrecord[datalen:]) 256 elif datalen > layout.header.record_length: 257 record._data = record._data[:layout.header.record_length] 258 if not _fromdisk and not layout.inmemory: 259 record._update_disk() 260 return record
261 - def __setattr__(yo, name, value):
262 if name in yo.__slots__: 263 object.__setattr__(yo, name, value) 264 return 265 elif not name in yo._layout.fields: 266 raise FieldMissing(name) 267 fielddef = yo._layout[name] 268 try: 269 yo._updateFieldValue(fielddef, value) 270 except DbfError, error: 271 error.message = "field --%s-- is %s -> %s" % (name, yo._layout.fieldtypes[fielddef['type']]['Type'], error.message) 272 error.data = name 273 raise
274 - def __setitem__(yo, name, value):
275 if type(name) == str: 276 yo.__setattr__(name, value) 277 elif type(name) in (int, long): 278 yo.__setattr__(yo._layout.fields[name], value) 279 elif type(name) == slice: 280 sequence = [] 281 for field in yo._layout.fields[name]: 282 sequence.append(field) 283 if len(sequence) != len(value): 284 raise DbfError("length of slices not equal") 285 for field, val in zip(sequence, value): 286 yo[field] = val 287 else: 288 raise TypeError("%s is not a field name" % name)
289 - def __str__(yo):
290 result = [] 291 for seq, field in enumerate(yo.field_names): 292 result.append("%3d - %-10s: %s" % (seq, field, yo[field])) 293 return '\n'.join(result)
294 - def __repr__(yo):
295 return yo._data.tostring()
296 - def _createBlankRecord(yo):
297 "creates a blank record data chunk" 298 layout = yo._layout 299 ondisk = layout.ondisk 300 layout.ondisk = False 301 yo._data = array('c', ' ' * layout.header.record_length) 302 layout.memofields = [] 303 for field in layout.fields: 304 yo._updateFieldValue(layout[field], layout.fieldtypes[layout[field]['type']]['Blank']()) 305 if layout[field]['type'] in layout.memotypes: 306 layout.memofields.append(field) 307 layout.blankrecord = yo._data[:] 308 layout.ondisk = ondisk
309 - def delete_record(yo):
310 "marks record as deleted" 311 yo._data[0] = '*' 312 yo._dirty = True 313 return yo
314 @property
315 - def field_names(yo):
316 "fields in table/record" 317 return yo._layout.fields[:]
318 - def gather_fields(yo, dictionary, drop=False): # dict, drop_missing=False):
319 "saves a dictionary into a record's fields\nkeys with no matching field will raise a FieldMissing exception unless drop_missing = True" 320 old_data = yo._data[:] 321 try: 322 for key in dictionary: 323 if not key in yo.field_names: 324 if drop: 325 continue 326 raise FieldMissing(key) 327 yo.__setattr__(key, dictionary[key]) 328 except: 329 yo._data[:] = old_data 330 raise 331 return yo
332 @property
333 - def has_been_deleted(yo):
334 "marked for deletion?" 335 return yo._data[0] == '*'
336 - def read_record(yo):
337 "refresh record data from disk" 338 size = yo._layout.header.record_length 339 location = yo._recnum * size + yo._layout.header.start 340 yo._layout.dfd.seek(location) 341 yo._data[:] = yo._meta.dfd.read(size) 342 yo._dirty = False 343 return yo
344 @property
345 - def record_number(yo):
346 "physical record number" 347 return yo._recnum
348 @property
349 - def record_table(yo):
350 table = yo._layout.table() 351 if table is None: 352 raise DbfError("table is no longer available") 353 return table
354 - def check_index(yo):
355 for dbfindex in yo._layout.table()._indexen: 356 dbfindex(yo)
357 - def reset_record(yo, keep_fields=None):
358 "blanks record" 359 if keep_fields is None: 360 keep_fields = [] 361 keep = {} 362 for field in keep_fields: 363 keep[field] = yo[field] 364 if yo._layout.blankrecord == None: 365 yo._createBlankRecord() 366 yo._data[:] = yo._layout.blankrecord[:] 367 for field in keep_fields: 368 yo[field] = keep[field] 369 yo._dirty = True 370 return yo
371 - def scatter_fields(yo, blank=False):
372 "returns a dictionary of fieldnames and values which can be used with gather_fields(). if blank is True, values are empty." 373 keys = yo._layout.fields 374 if blank: 375 values = [yo._layout.fieldtypes[yo._layout[key]['type']]['Blank']() for key in keys] 376 else: 377 values = [yo[field] for field in keys] 378 return dict(zip(keys, values))
379 - def undelete_record(yo):
380 "marks record as active" 381 yo._data[0] = ' ' 382 yo._dirty = True 383 return yo
384 - def write_record(yo, **kwargs):
385 "write record data to disk" 386 if kwargs: 387 yo.gather_fields(kwargs) 388 if yo._dirty: 389 yo._update_disk() 390 return 1 391 return 0
392 -class _DbfMemo(object):
393 """Provides access to memo fields as dictionaries 394 must override _init, _get_memo, and _put_memo to 395 store memo contents to disk"""
396 - def _init(yo):
397 "initialize disk file usage"
398 - def _get_memo(yo, block):
399 "retrieve memo contents from disk"
400 - def _put_memo(yo, data):
401 "store memo contents to disk"
402 - def __init__(yo, meta):
403 "" 404 yo.meta = meta 405 yo.memory = {} 406 yo.nextmemo = 1 407 yo._init() 408 yo.meta.newmemofile = False
409 - def get_memo(yo, block, field):
410 "gets the memo in block" 411 if yo.meta.ignorememos or not block: 412 return '' 413 if yo.meta.ondisk: 414 return yo._get_memo(block) 415 else: 416 return yo.memory[block]
417 - def put_memo(yo, data):
418 "stores data in memo file, returns block number" 419 if yo.meta.ignorememos or data == '': 420 return 0 421 if yo.meta.inmemory: 422 thismemo = yo.nextmemo 423 yo.nextmemo += 1 424 yo.memory[thismemo] = data 425 else: 426 thismemo = yo._put_memo(data) 427 return thismemo
428 -class _Db3Memo(_DbfMemo):
429 - def _init(yo):
430 "dBase III specific" 431 yo.meta.memo_size= 512 432 yo.record_header_length = 2 433 if yo.meta.ondisk and not yo.meta.ignorememos: 434 if yo.meta.newmemofile: 435 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 436 yo.meta.mfd.write(io.packLongInt(1) + '\x00' * 508) 437 else: 438 try: 439 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 440 yo.meta.mfd.seek(0) 441 yo.nextmemo = io.unpackLongInt(yo.meta.mfd.read(4)) 442 except: 443 raise DbfError("memo file appears to be corrupt")
444 - def _get_memo(yo, block):
445 block = int(block) 446 yo.meta.mfd.seek(block * yo.meta.memo_size) 447 eom = -1 448 data = '' 449 while eom == -1: 450 newdata = yo.meta.mfd.read(yo.meta.memo_size) 451 if not newdata: 452 return data 453 data += newdata 454 eom = data.find('\x1a\x1a') 455 return data[:eom].rstrip()
456 - def _put_memo(yo, data):
457 data = data.rstrip() 458 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 459 blocks = length // yo.meta.memo_size 460 if length % yo.meta.memo_size: 461 blocks += 1 462 thismemo = yo.nextmemo 463 yo.nextmemo = thismemo + blocks 464 yo.meta.mfd.seek(0) 465 yo.meta.mfd.write(io.packLongInt(yo.nextmemo)) 466 yo.meta.mfd.seek(thismemo * yo.meta.memo_size) 467 yo.meta.mfd.write(data) 468 yo.meta.mfd.write('\x1a\x1a') 469 double_check = yo._get_memo(thismemo) 470 if len(double_check) != len(data): 471 uhoh = open('dbf_memo_dump.err','wb') 472 uhoh.write('thismemo: %d' % thismemo) 473 uhoh.write('nextmemo: %d' % yo.nextmemo) 474 uhoh.write('saved: %d bytes' % len(data)) 475 uhoh.write(data) 476 uhoh.write('retrieved: %d bytes' % len(double_check)) 477 uhoh.write(double_check) 478 uhoh.close() 479 raise DbfError("unknown error: memo not saved") 480 return thismemo
481 -class _VfpMemo(_DbfMemo):
482 - def _init(yo):
483 "Visual Foxpro 6 specific" 484 if yo.meta.ondisk and not yo.meta.ignorememos: 485 yo.record_header_length = 8 486 if yo.meta.newmemofile: 487 if yo.meta.memo_size == 0: 488 yo.meta.memo_size = 1 489 elif 1 < yo.meta.memo_size < 33: 490 yo.meta.memo_size *= 512 491 yo.meta.mfd = open(yo.meta.memoname, 'w+b') 492 nextmemo = 512 // yo.meta.memo_size 493 if nextmemo * yo.meta.memo_size < 512: 494 nextmemo += 1 495 yo.nextmemo = nextmemo 496 yo.meta.mfd.write(io.packLongInt(nextmemo, bigendian=True) + '\x00\x00' + \ 497 io.packShortInt(yo.meta.memo_size, bigendian=True) + '\x00' * 504) 498 else: 499 try: 500 yo.meta.mfd = open(yo.meta.memoname, 'r+b') 501 yo.meta.mfd.seek(0) 502 header = yo.meta.mfd.read(512) 503 yo.nextmemo = io.unpackLongInt(header[:4], bigendian=True) 504 yo.meta.memo_size = io.unpackShortInt(header[6:8], bigendian=True) 505 except: 506 raise DbfError("memo file appears to be corrupt")
507 - def _get_memo(yo, block):
508 yo.meta.mfd.seek(block * yo.meta.memo_size) 509 header = yo.meta.mfd.read(8) 510 length = io.unpackLongInt(header[4:], bigendian=True) 511 return yo.meta.mfd.read(length)
512 - def _put_memo(yo, data):
513 data = data.rstrip() # no trailing whitespace 514 yo.meta.mfd.seek(0) 515 thismemo = io.unpackLongInt(yo.meta.mfd.read(4), bigendian=True) 516 yo.meta.mfd.seek(0) 517 length = len(data) + yo.record_header_length # room for two ^Z at end of memo 518 blocks = length // yo.meta.memo_size 519 if length % yo.meta.memo_size: 520 blocks += 1 521 yo.meta.mfd.write(io.packLongInt(thismemo+blocks, bigendian=True)) 522 yo.meta.mfd.seek(thismemo*yo.meta.memo_size) 523 yo.meta.mfd.write('\x00\x00\x00\x01' + io.packLongInt(len(data), bigendian=True) + data) 524 return thismemo
525 # Public classes
526 -class DbfTable(object):
527 """Provides a framework for dbf style tables.""" 528 _version = 'basic memory table' 529 _versionabbv = 'dbf' 530 _fieldtypes = { 531 'D' : { 'Type':'Date', 'Init':io.addDate, 'Blank':Date.today, 'Retrieve':io.retrieveDate, 'Update':io.updateDate, }, 532 'L' : { 'Type':'Logical', 'Init':io.addLogical, 'Blank':bool, 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, }, 533 'M' : { 'Type':'Memo', 'Init':io.addMemo, 'Blank':str, 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, } } 534 _memoext = '' 535 _memotypes = tuple('M', ) 536 _memoClass = _DbfMemo 537 _yesMemoMask = '' 538 _noMemoMask = '' 539 _fixed_fields = ('M','D','L') # always same length in table 540 _variable_fields = tuple() # variable length in table 541 _character_fields = tuple('M', ) # field representing character data 542 _decimal_fields = tuple() # text-based numeric fields 543 _numeric_fields = tuple() # fields representing a number 544 _dbfTableHeader = array('c', '\x00' * 32) 545 _dbfTableHeader[0] = '\x00' # table type - none 546 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 547 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 548 _dbfTableHeader[29] = '\x00' # code page -- none, using plain ascii 549 _dbfTableHeader = _dbfTableHeader.tostring() 550 _dbfTableHeaderExtra = '' 551 _supported_tables = [] 552 _read_only = False 553 _meta_only = False 554 _use_deleted = True 555 backup = False
556 - class _DbfLists(object):
557 "implements the weakref structure for DbfLists"
558 - def __init__(yo):
559 yo._lists = set()
560 - def __iter__(yo):
561 yo._lists = set([s for s in yo._lists if s() is not None]) 562 return (s() for s in yo._lists if s() is not None)
563 - def __len__(yo):
564 yo._lists = set([s for s in yo._lists if s() is not None]) 565 return len(yo._lists)
566 - def add(yo, new_list):
567 yo._lists.add(weakref.ref(new_list)) 568 yo._lists = set([s for s in yo._lists if s() is not None])
569 - class _Indexen(object):
570 "implements the weakref structure for seperate indexes"
571 - def __init__(yo):
572 yo._indexen = set()
573 - def __iter__(yo):
574 yo._indexen = set([s for s in yo._indexen if s() is not None]) 575 return (s() for s in yo._indexen if s() is not None)
576 - def __len__(yo):
577 yo._indexen = set([s for s in yo._indexen if s() is not None]) 578 return len(yo._indexen)
579 - def add(yo, new_list):
580 yo._indexen.add(weakref.ref(new_list)) 581 yo._indexen = set([s for s in yo._indexen if s() is not None])
582 - class _MetaData(dict):
583 blankrecord = None 584 fields = None 585 filename = None 586 dfd = None 587 memoname = None 588 newmemofile = False 589 memo = None 590 mfd = None 591 ignorememos = False 592 memofields = None 593 current = -1
594 - class _TableHeader(object):
595 - def __init__(yo, data):
596 if len(data) != 32: 597 raise DbfError('table header should be 32 bytes, but is %d bytes' % len(data)) 598 yo._data = array('c', data + '\x0d')
599 - def codepage(yo, cp=None):
600 "get/set code page of table" 601 if cp is None: 602 return yo._data[29] 603 else: 604 cp, sd, ld = _codepage_lookup(cp) 605 yo._data[29] = cp 606 return cp
607 @property
608 - def data(yo):
609 "main data structure" 610 date = io.packDate(Date.today()) 611 yo._data[1:4] = array('c', date) 612 return yo._data.tostring()
613 @data.setter
614 - def data(yo, bytes):
615 if len(bytes) < 32: 616 raise DbfError("length for data of %d is less than 32" % len(bytes)) 617 yo._data[:] = array('c', bytes)
618 @property
619 - def extra(yo):
620 "extra dbf info (located after headers, before data records)" 621 fieldblock = yo._data[32:] 622 for i in range(len(fieldblock)//32+1): 623 cr = i * 32 624 if fieldblock[cr] == '\x0d': 625 break 626 else: 627 raise DbfError("corrupt field structure") 628 cr += 33 # skip past CR 629 return yo._data[cr:].tostring()
630 @extra.setter
631 - def extra(yo, data):
632 fieldblock = yo._data[32:] 633 for i in range(len(fieldblock)//32+1): 634 cr = i * 32 635 if fieldblock[cr] == '\x0d': 636 break 637 else: 638 raise DbfError("corrupt field structure") 639 cr += 33 # skip past CR 640 yo._data[cr:] = array('c', data) # extra 641 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start
642 @property
643 - def field_count(yo):
644 "number of fields (read-only)" 645 fieldblock = yo._data[32:] 646 for i in range(len(fieldblock)//32+1): 647 cr = i * 32 648 if fieldblock[cr] == '\x0d': 649 break 650 else: 651 raise DbfError("corrupt field structure") 652 return len(fieldblock[:cr]) // 32
653 @property
654 - def fields(yo):
655 "field block structure" 656 fieldblock = yo._data[32:] 657 for i in range(len(fieldblock)//32+1): 658 cr = i * 32 659 if fieldblock[cr] == '\x0d': 660 break 661 else: 662 raise DbfError("corrupt field structure") 663 return fieldblock[:cr].tostring()
664 @fields.setter
665 - def fields(yo, block):
666 fieldblock = yo._data[32:] 667 for i in range(len(fieldblock)//32+1): 668 cr = i * 32 669 if fieldblock[cr] == '\x0d': 670 break 671 else: 672 raise DbfError("corrupt field structure") 673 cr += 32 # convert to indexing main structure 674 fieldlen = len(block) 675 if fieldlen % 32 != 0: 676 raise DbfError("fields structure corrupt: %d is not a multiple of 32" % fieldlen) 677 yo._data[32:cr] = array('c', block) # fields 678 yo._data[8:10] = array('c', io.packShortInt(len(yo._data))) # start 679 fieldlen = fieldlen // 32 680 recordlen = 1 # deleted flag 681 for i in range(fieldlen): 682 recordlen += ord(block[i*32+16]) 683 yo._data[10:12] = array('c', io.packShortInt(recordlen))
684 @property
685 - def record_count(yo):
686 "number of records (maximum 16,777,215)" 687 return io.unpackLongInt(yo._data[4:8].tostring())
688 @record_count.setter
689 - def record_count(yo, count):
690 yo._data[4:8] = array('c', io.packLongInt(count))
691 @property
692 - def record_length(yo):
693 "length of a record (read_only) (max of 65,535)" 694 return io.unpackShortInt(yo._data[10:12].tostring())
695 @property
696 - def start(yo):
697 "starting position of first record in file (must be within first 64K)" 698 return io.unpackShortInt(yo._data[8:10].tostring())
699 @start.setter
700 - def start(yo, pos):
701 yo._data[8:10] = array('c', io.packShortInt(pos))
702 @property
703 - def update(yo):
704 "date of last table modification (read-only)" 705 return io.unpackDate(yo._data[1:4].tostring())
706 @property
707 - def version(yo):
708 "dbf version" 709 return yo._data[0]
710 @version.setter
711 - def version(yo, ver):
712 yo._data[0] = ver
713 - class _Table(object):
714 "implements the weakref table for records"
715 - def __init__(yo, count, meta):
716 yo._meta = meta 717 yo._weakref_list = [weakref.ref(lambda x: None)] * count
718 - def __getitem__(yo, index):
719 maybe = yo._weakref_list[index]() 720 if maybe is None: 721 if index < 0: 722 index += yo._meta.header.record_count 723 size = yo._meta.header.record_length 724 location = index * size + yo._meta.header.start 725 yo._meta.dfd.seek(location) 726 if yo._meta.dfd.tell() != location: 727 raise ValueError("unable to seek to offset %d in file" % location) 728 bytes = yo._meta.dfd.read(size) 729 if not bytes: 730 raise ValueError("unable to read record data from %s at location %d" % (yo._meta.filename, location)) 731 maybe = _DbfRecord(recnum=index, layout=yo._meta, kamikaze=bytes, _fromdisk=True) 732 yo._weakref_list[index] = weakref.ref(maybe) 733 return maybe
734 - def append(yo, record):
735 yo._weakref_list.append(weakref.ref(record))
736 - def clear(yo):
737 yo._weakref_list[:] = []
738 - class DbfIterator(object):
739 "returns records using current index"
740 - def __init__(yo, table):
741 yo._table = table 742 yo._index = -1 743 yo._more_records = True
744 - def __iter__(yo):
745 return yo
746 - def next(yo):
747 while yo._more_records: 748 yo._index += 1 749 if yo._index >= len(yo._table): 750 yo._more_records = False 751 continue 752 record = yo._table[yo._index] 753 if not yo._table.use_deleted and record.has_been_deleted: 754 continue 755 return record 756 else: 757 raise StopIteration
758 - def _buildHeaderFields(yo):
759 "constructs fieldblock for disk table" 760 fieldblock = array('c', '') 761 memo = False 762 yo._meta.header.version = chr(ord(yo._meta.header.version) & ord(yo._noMemoMask)) 763 for field in yo._meta.fields: 764 if yo._meta.fields.count(field) > 1: 765 raise DbfError("corrupted field structure (noticed in _buildHeaderFields)") 766 fielddef = array('c', '\x00' * 32) 767 fielddef[:11] = array('c', io.packStr(field)) 768 fielddef[11] = yo._meta[field]['type'] 769 fielddef[12:16] = array('c', io.packLongInt(yo._meta[field]['start'])) 770 fielddef[16] = chr(yo._meta[field]['length']) 771 fielddef[17] = chr(yo._meta[field]['decimals']) 772 fielddef[18] = chr(yo._meta[field]['flags']) 773 fieldblock.extend(fielddef) 774 if yo._meta[field]['type'] in yo._meta.memotypes: 775 memo = True 776 yo._meta.header.fields = fieldblock.tostring() 777 if memo: 778 yo._meta.header.version = chr(ord(yo._meta.header.version) | ord(yo._yesMemoMask)) 779 if yo._meta.memo is None: 780 yo._meta.memo = yo._memoClass(yo._meta)
781 - def _checkMemoIntegrity(yo):
782 "dBase III specific" 783 if yo._meta.header.version == '\x83': 784 try: 785 yo._meta.memo = yo._memoClass(yo._meta) 786 except: 787 yo._meta.dfd.close() 788 yo._meta.dfd = None 789 raise 790 if not yo._meta.ignorememos: 791 for field in yo._meta.fields: 792 if yo._meta[field]['type'] in yo._memotypes: 793 if yo._meta.header.version != '\x83': 794 yo._meta.dfd.close() 795 yo._meta.dfd = None 796 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 797 elif not os.path.exists(yo._meta.memoname): 798 yo._meta.dfd.close() 799 yo._meta.dfd = None 800 raise DbfError("Table structure corrupt: memo fields exist without memo file") 801 break
802 - def _initializeFields(yo):
803 "builds the FieldList of names, types, and descriptions from the disk file" 804 yo._meta.fields[:] = [] 805 offset = 1 806 fieldsdef = yo._meta.header.fields 807 if len(fieldsdef) % 32 != 0: 808 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 809 if len(fieldsdef) // 32 != yo.field_count: 810 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 811 for i in range(yo.field_count): 812 fieldblock = fieldsdef[i*32:(i+1)*32] 813 name = io.unpackStr(fieldblock[:11]) 814 type = fieldblock[11] 815 if not type in yo._meta.fieldtypes: 816 raise DbfError("Unknown field type: %s" % type) 817 start = offset 818 length = ord(fieldblock[16]) 819 offset += length 820 end = start + length 821 decimals = ord(fieldblock[17]) 822 flags = ord(fieldblock[18]) 823 if name in yo._meta.fields: 824 raise DbfError('Duplicate field name found: %s' % name) 825 yo._meta.fields.append(name) 826 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
827 - def _fieldLayout(yo, i):
828 "Returns field information Name Type(Length[,Decimals])" 829 name = yo._meta.fields[i] 830 type = yo._meta[name]['type'] 831 length = yo._meta[name]['length'] 832 decimals = yo._meta[name]['decimals'] 833 if type in yo._decimal_fields: 834 description = "%s %s(%d,%d)" % (name, type, length, decimals) 835 elif type in yo._fixed_fields: 836 description = "%s %s" % (name, type) 837 else: 838 description = "%s %s(%d)" % (name, type, length) 839 return description
840 - def _loadtable(yo):
841 "loads the records from disk to memory" 842 if yo._meta_only: 843 raise DbfError("%s has been closed, records are unavailable" % yo.filename) 844 dfd = yo._meta.dfd 845 header = yo._meta.header 846 dfd.seek(header.start) 847 allrecords = dfd.read() # kludge to get around mysterious errno 0 problems 848 dfd.seek(0) 849 length = header.record_length 850 for i in range(header.record_count): 851 record_data = allrecords[length*i:length*i+length] 852 yo._table.append(_DbfRecord(i, yo._meta, allrecords[length*i:length*i+length], _fromdisk=True)) 853 dfd.seek(0)
854 - def _list_fields(yo, specs, sep=','):
855 if specs is None: 856 specs = yo.field_names 857 elif isinstance(specs, str): 858 specs = specs.split(sep) 859 else: 860 specs = list(specs) 861 specs = [s.strip() for s in specs] 862 return specs
863 - def _update_disk(yo, headeronly=False):
864 "synchronizes the disk file with current data" 865 if yo._meta.inmemory: 866 return 867 fd = yo._meta.dfd 868 fd.seek(0) 869 fd.write(yo._meta.header.data) 870 if not headeronly: 871 for record in yo._table: 872 record._update_disk() 873 fd.flush() 874 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length) 875 if 'db3' in yo._versionabbv: 876 fd.seek(0, os.SEEK_END) 877 fd.write('\x1a') # required for dBase III 878 fd.flush() 879 fd.truncate(yo._meta.header.start + yo._meta.header.record_count * yo._meta.header.record_length + 1)
880
881 - def __contains__(yo, key):
882 return key in yo.field_names
883 - def __enter__(yo):
884 return yo
885 - def __exit__(yo, *exc_info):
886 yo.close()
887 - def __getattr__(yo, name):
888 if name in ('_table'): 889 if yo._meta.ondisk: 890 yo._table = yo._Table(len(yo), yo._meta) 891 else: 892 yo._table = [] 893 yo._loadtable() 894 return object.__getattribute__(yo, name)
895 - def __getitem__(yo, value):
896 if type(value) == int: 897 if not -yo._meta.header.record_count <= value < yo._meta.header.record_count: 898 raise IndexError("Record %d is not in table." % value) 899 return yo._table[value] 900 elif type(value) == slice: 901 sequence = List(desc='%s --> %s' % (yo.filename, value), field_names=yo.field_names) 902 yo._dbflists.add(sequence) 903 for index in range(len(yo))[value]: 904 record = yo._table[index] 905 if yo.use_deleted is True or not record.has_been_deleted: 906 sequence.append(record) 907 return sequence 908 else: 909 raise TypeError('type <%s> not valid for indexing' % type(value))
910 - def __init__(yo, filename=':memory:', field_specs=None, memo_size=128, ignore_memos=False, 911 read_only=False, keep_memos=False, meta_only=False, codepage=None):
912 """open/create dbf file 913 filename should include path if needed 914 field_specs can be either a ;-delimited string or a list of strings 915 memo_size is always 512 for db3 memos 916 ignore_memos is useful if the memo file is missing or corrupt 917 read_only will load records into memory, then close the disk file 918 keep_memos will also load any memo fields into memory 919 meta_only will ignore all records, keeping only basic table information 920 codepage will override whatever is set in the table itself""" 921 if filename[0] == filename[-1] == ':': 922 if field_specs is None: 923 raise DbfError("field list must be specified for memory tables") 924 elif type(yo) is DbfTable: 925 raise DbfError("only memory tables supported") 926 yo._dbflists = yo._DbfLists() 927 yo._indexen = yo._Indexen() 928 yo._meta = meta = yo._MetaData() 929 meta.table = weakref.ref(yo) 930 meta.filename = filename 931 meta.fields = [] 932 meta.fieldtypes = yo._fieldtypes 933 meta.fixed_fields = yo._fixed_fields 934 meta.variable_fields = yo._variable_fields 935 meta.character_fields = yo._character_fields 936 meta.decimal_fields = yo._decimal_fields 937 meta.numeric_fields = yo._numeric_fields 938 meta.memotypes = yo._memotypes 939 meta.ignorememos = ignore_memos 940 meta.memo_size = memo_size 941 meta.input_decoder = codecs.getdecoder(input_decoding) # from ascii to unicode 942 meta.output_encoder = codecs.getencoder(input_decoding) # and back to ascii 943 meta.return_ascii = return_ascii 944 meta.header = header = yo._TableHeader(yo._dbfTableHeader) 945 header.extra = yo._dbfTableHeaderExtra 946 header.data #force update of date 947 if filename[0] == filename[-1] == ':': 948 yo._table = [] 949 meta.ondisk = False 950 meta.inmemory = True 951 meta.memoname = filename 952 else: 953 base, ext = os.path.splitext(filename) 954 if ext == '': 955 meta.filename = base + '.dbf' 956 meta.memoname = base + yo._memoext 957 meta.ondisk = True 958 meta.inmemory = False 959 if field_specs: 960 if meta.ondisk: 961 meta.dfd = open(meta.filename, 'w+b') 962 meta.newmemofile = True 963 yo.add_fields(field_specs) 964 header.codepage(codepage or default_codepage) 965 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 966 meta.decoder = codecs.getdecoder(sd) 967 meta.encoder = codecs.getencoder(sd) 968 return 969 try: 970 dfd = meta.dfd = open(meta.filename, 'r+b') 971 except IOError, e: 972 raise DbfError(str(e)) 973 dfd.seek(0) 974 meta.header = header = yo._TableHeader(dfd.read(32)) 975 if not header.version in yo._supported_tables: 976 dfd.close() 977 dfd = None 978 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version))) 979 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 980 yo._meta.decoder = codecs.getdecoder(sd) 981 yo._meta.encoder = codecs.getencoder(sd) 982 fieldblock = dfd.read(header.start - 32) 983 for i in range(len(fieldblock)//32+1): 984 fieldend = i * 32 985 if fieldblock[fieldend] == '\x0d': 986 break 987 else: 988 raise DbfError("corrupt field structure in header") 989 if len(fieldblock[:fieldend]) % 32 != 0: 990 raise DbfError("corrupt field structure in header") 991 header.fields = fieldblock[:fieldend] 992 header.extra = fieldblock[fieldend+1:] # skip trailing \r 993 yo._initializeFields() 994 yo._checkMemoIntegrity() 995 meta.current = -1 996 if len(yo) > 0: 997 meta.current = 0 998 dfd.seek(0) 999 if meta_only: 1000 yo.close(keep_table=False, keep_memos=False) 1001 elif read_only: 1002 yo.close(keep_table=True, keep_memos=keep_memos) 1003 if codepage is not None: 1004 cp, sd, ld = _codepage_lookup(codepage) 1005 yo._meta.decoder = codecs.getdecoder(sd) 1006 yo._meta.encoder = codecs.getencoder(sd)
1007
1008 - def __iter__(yo):
1009 return yo.DbfIterator(yo)
1010 - def __len__(yo):
1011 return yo._meta.header.record_count
1012 - def __nonzero__(yo):
1013 return yo._meta.header.record_count != 0
1014 - def __repr__(yo):
1015 if yo._read_only: 1016 return __name__ + ".Table('%s', read_only=True)" % yo._meta.filename 1017 elif yo._meta_only: 1018 return __name__ + ".Table('%s', meta_only=True)" % yo._meta.filename 1019 else: 1020 return __name__ + ".Table('%s')" % yo._meta.filename
1021 - def __str__(yo):
1022 if yo._read_only: 1023 status = "read-only" 1024 elif yo._meta_only: 1025 status = "meta-only" 1026 else: 1027 status = "read/write" 1028 str = """ 1029 Table: %s 1030 Type: %s 1031 Codepage: %s 1032 Status: %s 1033 Last updated: %s 1034 Record count: %d 1035 Field count: %d 1036 Record length: %d """ % (yo.filename, version_map.get(yo._meta.header.version, 1037 'unknown - ' + hex(ord(yo._meta.header.version))), yo.codepage, status, 1038 yo.last_update, len(yo), yo.field_count, yo.record_length) 1039 str += "\n --Fields--\n" 1040 for i in range(len(yo._meta.fields)): 1041 str += "%11d) %s\n" % (i, yo._fieldLayout(i)) 1042 return str
1043 @property
1044 - def codepage(yo):
1045 return "%s (%s)" % code_pages[yo._meta.header.codepage()]
1046 @codepage.setter
1047 - def codepage(yo, cp):
1048 cp = code_pages[yo._meta.header.codepage(cp)][0] 1049 yo._meta.decoder = codecs.getdecoder(cp) 1050 yo._meta.encoder = codecs.getencoder(cp) 1051 yo._update_disk(headeronly=True)
1052 @property
1053 - def field_count(yo):
1054 "the number of fields in the table" 1055 return yo._meta.header.field_count
1056 @property
1057 - def field_names(yo):
1058 "a list of the fields in the table" 1059 return yo._meta.fields[:]
1060 @property
1061 - def filename(yo):
1062 "table's file name, including path (if specified on open)" 1063 return yo._meta.filename
1064 @property
1065 - def last_update(yo):
1066 "date of last update" 1067 return yo._meta.header.update
1068 @property
1069 - def memoname(yo):
1070 "table's memo name (if path included in filename on open)" 1071 return yo._meta.memoname
1072 @property
1073 - def record_length(yo):
1074 "number of bytes in a record" 1075 return yo._meta.header.record_length
1076 @property
1077 - def record_number(yo):
1078 "index number of the current record" 1079 return yo._meta.current
1080 @property
1081 - def supported_tables(yo):
1082 "allowable table types" 1083 return yo._supported_tables
1084 @property
1085 - def use_deleted(yo):
1086 "process or ignore deleted records" 1087 return yo._use_deleted
1088 @use_deleted.setter
1089 - def use_deleted(yo, new_setting):
1090 yo._use_deleted = new_setting
1091 @property
1092 - def version(yo):
1093 "returns the dbf type of the table" 1094 return yo._version
1095 - def add_fields(yo, field_specs):
1096 """adds field(s) to the table layout; format is Name Type(Length,Decimals)[; Name Type(Length,Decimals)[...]] 1097 backup table is created with _backup appended to name 1098 then modifies current structure""" 1099 all_records = [record for record in yo] 1100 if yo: 1101 yo.create_backup() 1102 yo._meta.blankrecord = None 1103 meta = yo._meta 1104 offset = meta.header.record_length 1105 fields = yo._list_fields(field_specs, sep=';') 1106 for field in fields: 1107 try: 1108 name, format = field.split() 1109 if name[0] == '_' or name[0].isdigit() or not name.replace('_','').isalnum(): 1110 raise DbfError("%s invalid: field names must start with a letter, and can only contain letters, digits, and _" % name) 1111 name = name.lower() 1112 if name in meta.fields: 1113 raise DbfError("Field '%s' already exists" % name) 1114 field_type = format[0].upper() 1115 if len(name) > 10: 1116 raise DbfError("Maximum field name length is 10. '%s' is %d characters long." % (name, len(name))) 1117 if not field_type in meta.fieldtypes.keys(): 1118 raise DbfError("Unknown field type: %s" % field_type) 1119 length, decimals = yo._meta.fieldtypes[field_type]['Init'](format) 1120 except ValueError: 1121 raise DbfError("invalid field specifier: %s" % field) 1122 start = offset 1123 end = offset + length 1124 offset = end 1125 meta.fields.append(name) 1126 meta[name] = {'type':field_type, 'start':start, 'length':length, 'end':end, 'decimals':decimals, 'flags':0} 1127 if meta[name]['type'] in yo._memotypes and meta.memo is None: 1128 meta.memo = yo._memoClass(meta) 1129 for record in yo: 1130 record[name] = meta.fieldtypes[field_type]['Blank']() 1131 yo._buildHeaderFields() 1132 yo._update_disk()
1133 - def append(yo, kamikaze='', drop=False, multiple=1):
1134 "adds <multiple> blank records, and fills fields with dict/tuple values if present" 1135 if not yo.field_count: 1136 raise DbfError("No fields defined, cannot append") 1137 empty_table = len(yo) == 0 1138 dictdata = False 1139 tupledata = False 1140 if not isinstance(kamikaze, _DbfRecord): 1141 if isinstance(kamikaze, dict): 1142 dictdata = kamikaze 1143 kamikaze = '' 1144 elif isinstance(kamikaze, tuple): 1145 tupledata = kamikaze 1146 kamikaze = '' 1147 newrecord = _DbfRecord(recnum=yo._meta.header.record_count, layout=yo._meta, kamikaze=kamikaze) 1148 yo._table.append(newrecord) 1149 yo._meta.header.record_count += 1 1150 if dictdata: 1151 newrecord.gather_fields(dictdata, drop=drop) 1152 elif tupledata: 1153 for index, item in enumerate(tupledata): 1154 newrecord[index] = item 1155 elif kamikaze == str: 1156 for field in yo._meta.memofields: 1157 newrecord[field] = '' 1158 elif kamikaze: 1159 for field in yo._meta.memofields: 1160 newrecord[field] = kamikaze[field] 1161 newrecord.write_record() 1162 multiple -= 1 1163 if multiple: 1164 data = newrecord._data 1165 single = yo._meta.header.record_count 1166 total = single + multiple 1167 while single < total: 1168 multi_record = _DbfRecord(single, yo._meta, kamikaze=data) 1169 yo._table.append(multi_record) 1170 for field in yo._meta.memofields: 1171 multi_record[field] = newrecord[field] 1172 single += 1 1173 multi_record.write_record() 1174 yo._meta.header.record_count = total # += multiple 1175 yo._meta.current = yo._meta.header.record_count - 1 1176 newrecord = multi_record 1177 yo._update_disk(headeronly=True) 1178 if empty_table: 1179 yo._meta.current = 0 1180 return newrecord
1181 - def bof(yo, _move=False):
1182 "moves record pointer to previous usable record; returns True if no more usable records" 1183 current = yo._meta.current 1184 try: 1185 while yo._meta.current > 0: 1186 yo._meta.current -= 1 1187 if yo.use_deleted or not yo.current().has_been_deleted: 1188 break 1189 else: 1190 yo._meta.current = -1 1191 return True 1192 return False 1193 finally: 1194 if not _move: 1195 yo._meta.current = current
1196 - def bottom(yo, get_record=False):
1197 """sets record pointer to bottom of table 1198 if get_record, seeks to and returns last (non-deleted) record 1199 DbfError if table is empty 1200 Bof if all records deleted and use_deleted is False""" 1201 yo._meta.current = yo._meta.header.record_count 1202 if get_record: 1203 try: 1204 return yo.prev() 1205 except Bof: 1206 yo._meta.current = yo._meta.header.record_count 1207 raise Eof()
1208 - def close(yo, keep_table=False, keep_memos=False):
1209 """closes disk files 1210 ensures table data is available if keep_table 1211 ensures memo data is available if keep_memos""" 1212 yo._meta.inmemory = True 1213 if keep_table: 1214 replacement_table = [] 1215 for record in yo._table: 1216 replacement_table.append(record) 1217 yo._table = replacement_table 1218 else: 1219 if yo._meta.ondisk: 1220 yo._meta_only = True 1221 if yo._meta.mfd is not None: 1222 if not keep_memos: 1223 yo._meta.ignorememos = True 1224 else: 1225 memo_fields = [] 1226 for field in yo.field_names: 1227 if yo.is_memotype(field): 1228 memo_fields.append(field) 1229 for record in yo: 1230 for field in memo_fields: 1231 record[field] = record[field] 1232 yo._meta.mfd.close() 1233 yo._meta.mfd = None 1234 if yo._meta.ondisk: 1235 yo._meta.dfd.close() 1236 yo._meta.dfd = None 1237 if keep_table: 1238 yo._read_only = True 1239 yo._meta.ondisk = False
1240 - def create_backup(yo, new_name=None, overwrite=False):
1241 "creates a backup table -- ignored if memory table" 1242 if yo.filename[0] == yo.filename[-1] == ':': 1243 return 1244 if new_name is None: 1245 new_name = os.path.splitext(yo.filename)[0] + '_backup.dbf' 1246 else: 1247 overwrite = True 1248 if overwrite or not yo.backup: 1249 bkup = open(new_name, 'wb') 1250 try: 1251 yo._meta.dfd.seek(0) 1252 copyfileobj(yo._meta.dfd, bkup) 1253 yo.backup = new_name 1254 finally: 1255 bkup.close()
1256 - def create_index(yo, key):
1257 return Index(yo, key)
1258 - def current(yo, index=False):
1259 "returns current logical record, or its index" 1260 if yo._meta.current < 0: 1261 raise Bof() 1262 elif yo._meta.current >= yo._meta.header.record_count: 1263 raise Eof() 1264 if index: 1265 return yo._meta.current 1266 return yo._table[yo._meta.current]
1267 - def delete_fields(yo, doomed):
1268 """removes field(s) from the table 1269 creates backup files with _backup appended to the file name, 1270 then modifies current structure""" 1271 doomed = yo._list_fields(doomed) 1272 for victim in doomed: 1273 if victim not in yo._meta.fields: 1274 raise DbfError("field %s not in table -- delete aborted" % victim) 1275 all_records = [record for record in yo] 1276 yo.create_backup() 1277 for victim in doomed: 1278 yo._meta.fields.pop(yo._meta.fields.index(victim)) 1279 start = yo._meta[victim]['start'] 1280 end = yo._meta[victim]['end'] 1281 for record in yo: 1282 record._data = record._data[:start] + record._data[end:] 1283 for field in yo._meta.fields: 1284 if yo._meta[field]['start'] == end: 1285 end = yo._meta[field]['end'] 1286 yo._meta[field]['start'] = start 1287 yo._meta[field]['end'] = start + yo._meta[field]['length'] 1288 start = yo._meta[field]['end'] 1289 yo._buildHeaderFields() 1290 yo._update_disk()
1291 - def eof(yo, _move=False):
1292 "moves record pointer to next usable record; returns True if no more usable records" 1293 current = yo._meta.current 1294 try: 1295 while yo._meta.current < yo._meta.header.record_count - 1: 1296 yo._meta.current += 1 1297 if yo.use_deleted or not yo.current().has_been_deleted: 1298 break 1299 else: 1300 yo._meta.current = yo._meta.header.record_count 1301 return True 1302 return False 1303 finally: 1304 if not _move: 1305 yo._meta.current = current
1306 - def export(yo, records=None, filename=None, field_specs=None, format='csv', header=True):
1307 """writes the table using CSV or tab-delimited format, using the filename 1308 given if specified, otherwise the table name""" 1309 if filename is not None: 1310 path, filename = os.path.split(filename) 1311 else: 1312 path, filename = os.path.split(yo.filename) 1313 filename = os.path.join(path, filename) 1314 field_specs = yo._list_fields(field_specs) 1315 if records is None: 1316 records = yo 1317 format = format.lower() 1318 if format not in ('csv', 'tab', 'fixed'): 1319 raise DbfError("export format: csv, tab, or fixed -- not %s" % format) 1320 if format == 'fixed': 1321 format = 'txt' 1322 base, ext = os.path.splitext(filename) 1323 if ext.lower() in ('', '.dbf'): 1324 filename = base + "." + format[:3] 1325 fd = open(filename, 'w') 1326 try: 1327 if format == 'csv': 1328 csvfile = csv.writer(fd, dialect='dbf') 1329 if header: 1330 csvfile.writerow(field_specs) 1331 for record in records: 1332 fields = [] 1333 for fieldname in field_specs: 1334 fields.append(record[fieldname]) 1335 csvfile.writerow(fields) 1336 elif format == 'tab': 1337 if header: 1338 fd.write('\t'.join(field_specs) + '\n') 1339 for record in records: 1340 fields = [] 1341 for fieldname in field_specs: 1342 fields.append(str(record[fieldname])) 1343 fd.write('\t'.join(fields) + '\n') 1344 else: # format == 'fixed' 1345 header = open("%s_layout.txt" % os.path.splitext(filename)[0], 'w') 1346 header.write("%-15s Size\n" % "Field Name") 1347 header.write("%-15s ----\n" % ("-" * 15)) 1348 sizes = [] 1349 for field in field_specs: 1350 size = yo.size(field)[0] 1351 sizes.append(size) 1352 header.write("%-15s %3d\n" % (field, size)) 1353 header.write('\nTotal Records in file: %d\n' % len(records)) 1354 header.close() 1355 for record in records: 1356 fields = [] 1357 for i, field_name in enumerate(field_specs): 1358 fields.append("%-*s" % (sizes[i], record[field_name])) 1359 fd.write(''.join(fields) + '\n') 1360 finally: 1361 fd.close() 1362 fd = None 1363 return len(records)
1364 - def get_record(yo, recno):
1365 "returns record at physical_index[recno]" 1366 return yo._table[recno]
1367 - def goto(yo, criteria):
1368 """changes the record pointer to the first matching (non-deleted) record 1369 criteria should be either a tuple of tuple(value, field, func) triples, 1370 or an integer to go to""" 1371 if isinstance(criteria, int): 1372 if not -yo._meta.header.record_count <= criteria < yo._meta.header.record_count: 1373 raise IndexError("Record %d does not exist" % criteria) 1374 if criteria < 0: 1375 criteria += yo._meta.header.record_count 1376 yo._meta.current = criteria 1377 return yo.current() 1378 criteria = _normalize_tuples(tuples=criteria, length=3, filler=[_nop]) 1379 specs = tuple([(field, func) for value, field, func in criteria]) 1380 match = tuple([value for value, field, func in criteria]) 1381 current = yo.current(index=True) 1382 matchlen = len(match) 1383 while not yo.Eof(): 1384 record = yo.current() 1385 results = record(*specs) 1386 if results == match: 1387 return record 1388 return yo.goto(current)
1389 - def is_decimal(yo, name):
1390 "returns True if name is a variable-length field type" 1391 return yo._meta[name]['type'] in yo._decimal_fields
1392 - def is_memotype(yo, name):
1393 "returns True if name is a memo type field" 1394 return yo._meta[name]['type'] in yo._memotypes
1395 - def new(yo, filename, field_specs=None, codepage=None):
1396 "returns a new table of the same type" 1397 if field_specs is None: 1398 field_specs = yo.structure() 1399 if not (filename[0] == filename[-1] == ':'): 1400 path, name = os.path.split(filename) 1401 if path == "": 1402 filename = os.path.join(os.path.split(yo.filename)[0], filename) 1403 elif name == "": 1404 filename = os.path.join(path, os.path.split(yo.filename)[1]) 1405 if codepage is None: 1406 codepage = yo._meta.header.codepage()[0] 1407 return yo.__class__(filename, field_specs, codepage=codepage)
1408 - def next(yo):
1409 "set record pointer to next (non-deleted) record, and return it" 1410 if yo.eof(_move=True): 1411 raise Eof() 1412 return yo.current()
1413 - def open(yo):
1414 meta = yo._meta 1415 meta.inmemory = False 1416 meta.ondisk = True 1417 yo._read_only = False 1418 yo._meta_only = False 1419 if '_table' in dir(yo): 1420 del yo._table 1421 dfd = meta.dfd = open(meta.filename, 'r+b') 1422 dfd.seek(0) 1423 meta.header = header = yo._TableHeader(dfd.read(32)) 1424 if not header.version in yo._supported_tables: 1425 dfd.close() 1426 dfd = None 1427 raise DbfError("Unsupported dbf type: %s [%x]" % (version_map.get(meta.header.version, 'Unknown: %s' % meta.header.version), ord(meta.header.version))) 1428 cp, sd, ld = _codepage_lookup(meta.header.codepage()) 1429 meta.decoder = codecs.getdecoder(sd) 1430 meta.encoder = codecs.getencoder(sd) 1431 fieldblock = dfd.read(header.start - 32) 1432 for i in range(len(fieldblock)//32+1): 1433 fieldend = i * 32 1434 if fieldblock[fieldend] == '\x0d': 1435 break 1436 else: 1437 raise DbfError("corrupt field structure in header") 1438 if len(fieldblock[:fieldend]) % 32 != 0: 1439 raise DbfError("corrupt field structure in header") 1440 header.fields = fieldblock[:fieldend] 1441 header.extra = fieldblock[fieldend+1:] # skip trailing \r 1442 yo._initializeFields() 1443 yo._checkMemoIntegrity() 1444 meta.current = -1 1445 if len(yo) > 0: 1446 meta.current = 0 1447 dfd.seek(0)
1448
1449 - def pack(yo, _pack=True):
1450 "physically removes all deleted records" 1451 for dbfindex in yo._indexen: 1452 dbfindex.clear() 1453 newtable = [] 1454 index = 0 1455 offset = 0 # +1 for each purged record 1456 for record in yo._table: 1457 found = False 1458 if record.has_been_deleted and _pack: 1459 for dbflist in yo._dbflists: 1460 if dbflist._purge(record, record.record_number - offset, 1): 1461 found = True 1462 record._recnum = -1 1463 else: 1464 record._recnum = index 1465 newtable.append(record) 1466 index += 1 1467 if found: 1468 offset += 1 1469 found = False 1470 yo._table.clear() 1471 for record in newtable: 1472 yo._table.append(record) 1473 yo._meta.header.record_count = index 1474 yo._current = -1 1475 yo._update_disk() 1476 yo.reindex()
1477 - def prev(yo):
1478 "set record pointer to previous (non-deleted) record, and return it" 1479 if yo.bof(_move=True): 1480 raise Bof 1481 return yo.current()
1482 - def query(yo, sql_command=None, python=None):
1483 "uses exec to perform queries on the table" 1484 if sql_command: 1485 return sql(yo, sql_command) 1486 elif python is None: 1487 raise DbfError("query: python parameter must be specified") 1488 possible = List(desc="%s --> %s" % (yo.filename, python), field_names=yo.field_names) 1489 yo._dbflists.add(possible) 1490 query_result = {} 1491 select = 'query_result["keep"] = %s' % python 1492 g = {} 1493 use_deleted = yo.use_deleted 1494 for record in yo: 1495 query_result['keep'] = False 1496 g['query_result'] = query_result 1497 exec select in g, record 1498 if query_result['keep']: 1499 possible.append(record) 1500 record.write_record() 1501 return possible
1502 - def reindex(yo):
1503 for dbfindex in yo._indexen: 1504 dbfindex.reindex()
1505 - def rename_field(yo, oldname, newname):
1506 "renames an existing field" 1507 if yo: 1508 yo.create_backup() 1509 if not oldname in yo._meta.fields: 1510 raise DbfError("field --%s-- does not exist -- cannot rename it." % oldname) 1511 if newname[0] == '_' or newname[0].isdigit() or not newname.replace('_','').isalnum(): 1512 raise DbfError("field names cannot start with _ or digits, and can only contain the _, letters, and digits") 1513 newname = newname.lower() 1514 if newname in yo._meta.fields: 1515 raise DbfError("field --%s-- already exists" % newname) 1516 if len(newname) > 10: 1517 raise DbfError("maximum field name length is 10. '%s' is %d characters long." % (newname, len(newname))) 1518 yo._meta[newname] = yo._meta[oldname] 1519 yo._meta.fields[yo._meta.fields.index(oldname)] = newname 1520 yo._buildHeaderFields() 1521 yo._update_disk(headeronly=True)
1522 - def size(yo, field):
1523 "returns size of field as a tuple of (length, decimals)" 1524 if field in yo: 1525 return (yo._meta[field]['length'], yo._meta[field]['decimals']) 1526 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1527 - def structure(yo, fields=None):
1528 """return list of fields suitable for creating same table layout 1529 @param fields: list of fields or None for all fields""" 1530 field_specs = [] 1531 fields = yo._list_fields(fields) 1532 try: 1533 for name in fields: 1534 field_specs.append(yo._fieldLayout(yo.field_names.index(name))) 1535 except ValueError: 1536 raise DbfError("field --%s-- does not exist" % name) 1537 return field_specs
1538 - def top(yo, get_record=False):
1539 """sets record pointer to top of table; if get_record, seeks to and returns first (non-deleted) record 1540 DbfError if table is empty 1541 Eof if all records are deleted and use_deleted is False""" 1542 yo._meta.current = -1 1543 if get_record: 1544 try: 1545 return yo.next() 1546 except Eof: 1547 yo._meta.current = -1 1548 raise Bof()
1549 - def type(yo, field):
1550 "returns type of field" 1551 if field in yo: 1552 return yo._meta[field]['type'] 1553 raise DbfError("%s is not a field in %s" % (field, yo.filename))
1554 - def zap(yo, areyousure=False):
1555 """removes all records from table -- this cannot be undone! 1556 areyousure must be True, else error is raised""" 1557 if areyousure: 1558 if yo._meta.inmemory: 1559 yo._table = [] 1560 else: 1561 yo._table.clear() 1562 yo._meta.header.record_count = 0 1563 yo._current = -1 1564 yo._update_disk() 1565 else: 1566 raise DbfError("You must say you are sure to wipe the table")
1567 -class Db3Table(DbfTable):
1568 """Provides an interface for working with dBase III tables.""" 1569 _version = 'dBase III Plus' 1570 _versionabbv = 'db3' 1571 _fieldtypes = { 1572 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 1573 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 1574 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 1575 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 1576 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addNumeric} } 1577 _memoext = '.dbt' 1578 _memotypes = ('M',) 1579 _memoClass = _Db3Memo 1580 _yesMemoMask = '\x80' 1581 _noMemoMask = '\x7f' 1582 _fixed_fields = ('D','L','M') 1583 _variable_fields = ('C','N') 1584 _character_fields = ('C','M') 1585 _decimal_fields = ('N',) 1586 _numeric_fields = ('N',) 1587 _dbfTableHeader = array('c', '\x00' * 32) 1588 _dbfTableHeader[0] = '\x03' # version - dBase III w/o memo's 1589 _dbfTableHeader[8:10] = array('c', io.packShortInt(33)) 1590 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1591 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1592 _dbfTableHeader = _dbfTableHeader.tostring() 1593 _dbfTableHeaderExtra = '' 1594 _supported_tables = ['\x03', '\x83'] 1595 _read_only = False 1596 _meta_only = False 1597 _use_deleted = True
1598 - def _checkMemoIntegrity(yo):
1599 "dBase III specific" 1600 if yo._meta.header.version == '\x83': 1601 try: 1602 yo._meta.memo = yo._memoClass(yo._meta) 1603 except: 1604 yo._meta.dfd.close() 1605 yo._meta.dfd = None 1606 raise 1607 if not yo._meta.ignorememos: 1608 for field in yo._meta.fields: 1609 if yo._meta[field]['type'] in yo._memotypes: 1610 if yo._meta.header.version != '\x83': 1611 yo._meta.dfd.close() 1612 yo._meta.dfd = None 1613 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 1614 elif not os.path.exists(yo._meta.memoname): 1615 yo._meta.dfd.close() 1616 yo._meta.dfd = None 1617 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1618 break
1619 - def _initializeFields(yo):
1620 "builds the FieldList of names, types, and descriptions" 1621 yo._meta.fields[:] = [] 1622 offset = 1 1623 fieldsdef = yo._meta.header.fields 1624 if len(fieldsdef) % 32 != 0: 1625 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 1626 if len(fieldsdef) // 32 != yo.field_count: 1627 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 1628 for i in range(yo.field_count): 1629 fieldblock = fieldsdef[i*32:(i+1)*32] 1630 name = io.unpackStr(fieldblock[:11]) 1631 type = fieldblock[11] 1632 if not type in yo._meta.fieldtypes: 1633 raise DbfError("Unknown field type: %s" % type) 1634 start = offset 1635 length = ord(fieldblock[16]) 1636 offset += length 1637 end = start + length 1638 decimals = ord(fieldblock[17]) 1639 flags = ord(fieldblock[18]) 1640 yo._meta.fields.append(name) 1641 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1642 -class FpTable(DbfTable):
1643 'Provides an interface for working with FoxPro 2 tables' 1644 _version = 'Foxpro' 1645 _versionabbv = 'fp' 1646 _fieldtypes = { 1647 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 1648 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric}, 1649 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric}, 1650 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 1651 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 1652 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addVfpMemo}, 1653 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 1654 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 1655 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} } 1656 _memoext = '.fpt' 1657 _memotypes = ('G','M','P') 1658 _memoClass = _VfpMemo 1659 _yesMemoMask = '\xf5' # 1111 0101 1660 _noMemoMask = '\x03' # 0000 0011 1661 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 1662 _variable_fields = ('C','F','N') 1663 _character_fields = ('C','M') # field representing character data 1664 _decimal_fields = ('F','N') 1665 _numeric_fields = ('B','F','I','N','Y') 1666 _supported_tables = ('\x03', '\xf5') 1667 _dbfTableHeader = array('c', '\x00' * 32) 1668 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 1669 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263)) 1670 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1671 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1672 _dbfTableHeader = _dbfTableHeader.tostring() 1673 _dbfTableHeaderExtra = '\x00' * 263 1674 _use_deleted = True
1675 - def _checkMemoIntegrity(yo):
1676 if os.path.exists(yo._meta.memoname): 1677 try: 1678 yo._meta.memo = yo._memoClass(yo._meta) 1679 except: 1680 yo._meta.dfd.close() 1681 yo._meta.dfd = None 1682 raise 1683 if not yo._meta.ignorememos: 1684 for field in yo._meta.fields: 1685 if yo._meta[field]['type'] in yo._memotypes: 1686 if not os.path.exists(yo._meta.memoname): 1687 yo._meta.dfd.close() 1688 yo._meta.dfd = None 1689 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1690 break
1691 - def _initializeFields(yo):
1692 "builds the FieldList of names, types, and descriptions" 1693 yo._meta.fields[:] = [] 1694 offset = 1 1695 fieldsdef = yo._meta.header.fields 1696 if len(fieldsdef) % 32 != 0: 1697 raise DbfError("field definition block corrupt: %d bytes in size" % len(fieldsdef)) 1698 if len(fieldsdef) // 32 != yo.field_count: 1699 raise DbfError("Header shows %d fields, but field definition block has %d fields" % (yo.field_count, len(fieldsdef)//32)) 1700 for i in range(yo.field_count): 1701 fieldblock = fieldsdef[i*32:(i+1)*32] 1702 name = io.unpackStr(fieldblock[:11]) 1703 type = fieldblock[11] 1704 if not type in yo._meta.fieldtypes: 1705 raise DbfError("Unknown field type: %s" % type) 1706 elif type == '0': 1707 return # ignore nullflags 1708 start = offset 1709 length = ord(fieldblock[16]) 1710 offset += length 1711 end = start + length 1712 decimals = ord(fieldblock[17]) 1713 flags = ord(fieldblock[18]) 1714 yo._meta.fields.append(name) 1715 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1716
1717 -class VfpTable(DbfTable):
1718 'Provides an interface for working with Visual FoxPro 6 tables' 1719 _version = 'Visual Foxpro v6' 1720 _versionabbv = 'vfp' 1721 _fieldtypes = { 1722 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 1723 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency}, 1724 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble}, 1725 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric}, 1726 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric}, 1727 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger}, 1728 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 1729 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 1730 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime}, 1731 'M' : {'Type':'Memo', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo}, 1732 'G' : {'Type':'General', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo}, 1733 'P' : {'Type':'Picture', 'Retrieve':io.retrieveVfpMemo, 'Update':io.updateVfpMemo, 'Blank':str, 'Init':io.addVfpMemo}, 1734 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} } 1735 _memoext = '.fpt' 1736 _memotypes = ('G','M','P') 1737 _memoClass = _VfpMemo 1738 _yesMemoMask = '\x30' # 0011 0000 1739 _noMemoMask = '\x30' # 0011 0000 1740 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 1741 _variable_fields = ('C','F','N') 1742 _character_fields = ('C','M') # field representing character data 1743 _decimal_fields = ('F','N') 1744 _numeric_fields = ('B','F','I','N','Y') 1745 _supported_tables = ('\x30',) 1746 _dbfTableHeader = array('c', '\x00' * 32) 1747 _dbfTableHeader[0] = '\x30' # version - Foxpro 6 0011 0000 1748 _dbfTableHeader[8:10] = array('c', io.packShortInt(33+263)) 1749 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 1750 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 1751 _dbfTableHeader = _dbfTableHeader.tostring() 1752 _dbfTableHeaderExtra = '\x00' * 263 1753 _use_deleted = True
1754 - def _checkMemoIntegrity(yo):
1755 if os.path.exists(yo._meta.memoname): 1756 try: 1757 yo._meta.memo = yo._memoClass(yo._meta) 1758 except: 1759 yo._meta.dfd.close() 1760 yo._meta.dfd = None 1761 raise 1762 if not yo._meta.ignorememos: 1763 for field in yo._meta.fields: 1764 if yo._meta[field]['type'] in yo._memotypes: 1765 if not os.path.exists(yo._meta.memoname): 1766 yo._meta.dfd.close() 1767 yo._meta.dfd = None 1768 raise DbfError("Table structure corrupt: memo fields exist without memo file") 1769 break
1770 - def _initializeFields(yo):
1771 "builds the FieldList of names, types, and descriptions" 1772 yo._meta.fields[:] = [] 1773 offset = 1 1774 fieldsdef = yo._meta.header.fields 1775 for i in range(yo.field_count): 1776 fieldblock = fieldsdef[i*32:(i+1)*32] 1777 name = io.unpackStr(fieldblock[:11]) 1778 type = fieldblock[11] 1779 if not type in yo._meta.fieldtypes: 1780 raise DbfError("Unknown field type: %s" % type) 1781 elif type == '0': 1782 return # ignore nullflags 1783 start = io.unpackLongInt(fieldblock[12:16]) 1784 length = ord(fieldblock[16]) 1785 offset += length 1786 end = start + length 1787 decimals = ord(fieldblock[17]) 1788 flags = ord(fieldblock[18]) 1789 yo._meta.fields.append(name) 1790 yo._meta[name] = {'type':type,'start':start,'length':length,'end':end,'decimals':decimals,'flags':flags}
1791 -class List(object):
1792 "list of Dbf records, with set-like behavior" 1793 _desc = ''
1794 - def __init__(yo, new_records=None, desc=None, key=None, field_names=None):
1795 yo.field_names = field_names 1796 yo._list = [] 1797 yo._set = set() 1798 if key is not None: 1799 yo.key = key 1800 if key.__doc__ is None: 1801 key.__doc__ = 'unknown' 1802 key = yo.key 1803 yo._current = -1 1804 if isinstance(new_records, yo.__class__) and key is new_records.key: 1805 yo._list = new_records._list[:] 1806 yo._set = new_records._set.copy() 1807 yo._current = 0 1808 elif new_records is not None: 1809 for record in new_records: 1810 value = key(record) 1811 item = (record.record_table, record.record_number, value) 1812 if value not in yo._set: 1813 yo._set.add(value) 1814 yo._list.append(item) 1815 yo._current = 0 1816 if desc is not None: 1817 yo._desc = desc
1818 - def __add__(yo, other):
1819 key = yo.key 1820 if isinstance(other, (DbfTable, list)): 1821 other = yo.__class__(other, key=key) 1822 if isinstance(other, yo.__class__): 1823 result = yo.__class__() 1824 result._set = yo._set.copy() 1825 result._list[:] = yo._list[:] 1826 result.key = yo.key 1827 if key is other.key: # same key? just compare key values 1828 for item in other._list: 1829 if item[2] not in result._set: 1830 result._set.add(item[2]) 1831 result._list.append(item) 1832 else: # different keys, use this list's key on other's records 1833 for rec in other: 1834 value = key(rec) 1835 if value not in result._set: 1836 result._set.add(value) 1837 result._list.append((rec.record_table, rec.record_number, value)) 1838 result._current = 0 if result else -1 1839 return result 1840 return NotImplemented
1841 - def __contains__(yo, record):
1842 if isinstance(record, tuple): 1843 item = record 1844 else: 1845 item = yo.key(record) 1846 return item in yo._set
1847 - def __delitem__(yo, key):
1848 if isinstance(key, int): 1849 item = yo._list.pop[key] 1850 yo._set.remove(item[2]) 1851 elif isinstance(key, slice): 1852 yo._set.difference_update([item[2] for item in yo._list[key]]) 1853 yo._list.__delitem__(key) 1854 else: 1855 raise TypeError
1856 - def __getitem__(yo, key):
1857 if isinstance(key, int): 1858 count = len(yo._list) 1859 if not -count <= key < count: 1860 raise IndexError("Record %d is not in list." % key) 1861 return yo._get_record(*yo._list[key]) 1862 elif isinstance(key, slice): 1863 result = yo.__class__() 1864 result._list[:] = yo._list[key] 1865 result._set = set(result._list) 1866 result.key = yo.key 1867 result._current = 0 if result else -1 1868 return result 1869 else: 1870 raise TypeError('indices must be integers')
1871 - def __iter__(yo):
1872 return (table.get_record(recno) for table, recno, value in yo._list)
1873 - def __len__(yo):
1874 return len(yo._list)
1875 - def __nonzero__(yo):
1876 return len(yo) > 0
1877 - def __radd__(yo, other):
1878 return yo.__add__(other)
1879 - def __repr__(yo):
1880 if yo._desc: 1881 return "%s(key=%s - %s - %d records)" % (yo.__class__, yo.key.__doc__, yo._desc, len(yo._list)) 1882 else: 1883 return "%s(key=%s - %d records)" % (yo.__class__, yo.key.__doc__, len(yo._list))
1884 - def __rsub__(yo, other):
1885 key = yo.key 1886 if isinstance(other, (DbfTable, list)): 1887 other = yo.__class__(other, key=key) 1888 if isinstance(other, yo.__class__): 1889 result = yo.__class__() 1890 result._list[:] = other._list[:] 1891 result._set = other._set.copy() 1892 result.key = key 1893 lost = set() 1894 if key is other.key: 1895 for item in yo._list: 1896 if item[2] in result._list: 1897 result._set.remove(item[2]) 1898 lost.add(item) 1899 else: 1900 for rec in other: 1901 value = key(rec) 1902 if value in result._set: 1903 result._set.remove(value) 1904 lost.add((rec.record_table, rec.record_number, value)) 1905 result._list = [item for item in result._list if item not in lost] 1906 result._current = 0 if result else -1 1907 return result 1908 return NotImplemented
1909 - def __sub__(yo, other):
1910 key = yo.key 1911 if isinstance(other, (DbfTable, list)): 1912 other = yo.__class__(other, key=key) 1913 if isinstance(other, yo.__class__): 1914 result = yo.__class__() 1915 result._list[:] = yo._list[:] 1916 result._set = yo._set.copy() 1917 result.key = key 1918 lost = set() 1919 if key is other.key: 1920 for item in other._list: 1921 if item[2] in result._set: 1922 result._set.remove(item[2]) 1923 lost.add(item[2]) 1924 else: 1925 for rec in other: 1926 value = key(rec) 1927 if value in result._set: 1928 result._set.remove(value) 1929 lost.add(value) 1930 result._list = [item for item in result._list if item[2] not in lost] 1931 result._current = 0 if result else -1 1932 return result 1933 return NotImplemented
1934 - def _maybe_add(yo, item):
1935 if item[2] not in yo._set: 1936 yo._set.add(item[2]) 1937 yo._list.append(item)
1938 - def _get_record(yo, table=None, rec_no=None, value=None):
1939 if table is rec_no is None: 1940 table, rec_no, value = yo._list[yo._current] 1941 return table.get_record(rec_no)
1942 - def _purge(yo, record, old_record_number, offset):
1943 partial = record.record_table, old_record_number 1944 records = sorted(yo._list, key=lambda item: (item[0], item[1])) 1945 for item in records: 1946 if partial == item[:2]: 1947 found = True 1948 break 1949 elif partial[0] is item[0] and partial[1] < item[1]: 1950 found = False 1951 break 1952 else: 1953 found = False 1954 if found: 1955 yo._list.pop(yo._list.index(item)) 1956 yo._set.remove(item[2]) 1957 start = records.index(item) + found 1958 for item in records[start:]: 1959 if item[0] is not partial[0]: # into other table's records 1960 break 1961 i = yo._list.index(item) 1962 yo._set.remove(item[2]) 1963 item = item[0], (item[1] - offset), item[2] 1964 yo._list[i] = item 1965 yo._set.add(item[2]) 1966 return found
1967 - def append(yo, new_record):
1968 yo._maybe_add((new_record.record_table, new_record.record_number, yo.key(new_record))) 1969 if yo._current == -1 and yo._list: 1970 yo._current = 0 1971 return new_record
1972 - def bottom(yo):
1973 if yo._list: 1974 yo._current = len(yo._list) - 1 1975 return yo._get_record() 1976 raise DbfError("dbf.List is empty")
1977 - def clear(yo):
1978 yo._list = [] 1979 yo._set = set() 1980 yo._current = -1
1981 - def current(yo):
1982 if yo._current < 0: 1983 raise Bof() 1984 elif yo._current == len(yo._list): 1985 raise Eof() 1986 return yo._get_record()
1987 - def extend(yo, new_records):
1988 key = yo.key 1989 if isinstance(new_records, yo.__class__): 1990 if key is new_records.key: # same key? just compare key values 1991 for item in new_records._list: 1992 yo._maybe_add(item) 1993 else: # different keys, use this list's key on other's records 1994 for rec in new_records: 1995 value = key(rec) 1996 yo._maybe_add((rec.record_table, rec.record_number, value)) 1997 else: 1998 for record in new_records: 1999 value = key(rec) 2000 yo._maybe_add((rec.record_table, rec.record_number, value)) 2001 if yo._current == -1 and yo._list: 2002 yo._current = 0
2003 - def goto(yo, index_number):
2004 if yo._list: 2005 if 0 <= index_number <= len(yo._list): 2006 yo._current = index_number 2007 return yo._get_record() 2008 raise DbfError("index %d not in dbf.List of %d records" % (index_number, len(yo._list))) 2009 raise DbfError("dbf.List is empty")
2010 - def index(yo, sort=None, reverse=False):
2011 "sort= ((field_name, func), (field_name, func),) | 'ORIGINAL'" 2012 if sort is None: 2013 results = [] 2014 for field, func in yo._meta.index: 2015 results.append("%s(%s)" % (func.__name__, field)) 2016 return ', '.join(results + ['reverse=%s' % yo._meta.index_reversed]) 2017 yo._meta.index_reversed = reverse 2018 if sort == 'ORIGINAL': 2019 yo._index = range(yo._meta.header.record_count) 2020 yo._meta.index = 'ORIGINAL' 2021 if reverse: 2022 yo._index.reverse() 2023 return 2024 new_sort = _normalize_tuples(tuples=sort, length=2, filler=[_nop]) 2025 yo._meta.index = tuple(new_sort) 2026 yo._meta.orderresults = [''] * len(yo) 2027 for record in yo: 2028 yo._meta.orderresults[record.record_number] = record() 2029 yo._index.sort(key=lambda i: yo._meta.orderresults[i], reverse=reverse)
2030 - def index(yo, record, start=None, stop=None):
2031 item = record.record_table, record.record_number, yo.key(record) 2032 if start is None: 2033 start = 0 2034 if stop is None: 2035 stop = len(yo._list) 2036 return yo._list.index(item, start, stop)
2037 - def insert(yo, i, record):
2038 item = record.record_table, record.record_number, yo.key(record) 2039 if item not in yo._set: 2040 yo._set.add(item[2]) 2041 yo._list.insert(i, item)
2042 - def key(yo, record):
2043 "table_name, record_number" 2044 return record.record_table, record.record_number
2045 - def next(yo):
2046 if yo._current < len(yo._list): 2047 yo._current += 1 2048 if yo._current < len(yo._list): 2049 return yo._get_record() 2050 raise Eof()
2051 - def pop(yo, index=None):
2052 if index is None: 2053 table, recno, value = yo._list.pop() 2054 else: 2055 table, recno, value = yo._list.pop(index) 2056 yo._set.remove(value) 2057 return yo._get_record(table, recno, value)
2058 - def prev(yo):
2059 if yo._current >= 0: 2060 yo._current -= 1 2061 if yo._current > -1: 2062 return yo._get_record() 2063 raise Bof()
2064 - def remove(yo, record):
2065 item = record.record_table, record.record_number, yo.key(record) 2066 yo._list.remove(item) 2067 yo._set.remove(item[2])
2068 - def reverse(yo):
2069 return yo._list.reverse()
2070 - def top(yo):
2071 if yo._list: 2072 yo._current = 0 2073 return yo._get_record() 2074 raise DbfError("dbf.List is empty")
2075 - def sort(yo, key=None, reverse=False):
2076 if key is None: 2077 return yo._list.sort(reverse=reverse) 2078 return yo._list.sort(key=lambda item: key(item[0].get_record(item[1])), reverse=reverse)
2079
2080 -class DbfCsv(csv.Dialect):
2081 "csv format for exporting tables" 2082 delimiter = ',' 2083 doublequote = True 2084 escapechar = None 2085 lineterminator = '\n' 2086 quotechar = '"' 2087 skipinitialspace = True 2088 quoting = csv.QUOTE_NONNUMERIC
2089 -class Index(object):
2090 - class IndexIterator(object):
2091 "returns records using this index"
2092 - def __init__(yo, table, records):
2093 yo.table = table 2094 yo.records = records 2095 yo.index = 0
2096 - def __iter__(yo):
2097 return yo
2098 - def next(yo):
2099 while yo.index < len(yo.records): 2100 record = yo.table.get_record(yo.records[yo.index]) 2101 yo.index += 1 2102 if not yo.table.use_deleted and record.has_been_deleted: 2103 continue 2104 return record 2105 else: 2106 raise StopIteration
2107 - def __init__(yo, table, key, field_names=None):
2108 yo._table = table 2109 yo._values = [] # ordered list of values 2110 yo._rec_by_val = [] # matching record numbers 2111 yo._records = {} # record numbers:values 2112 yo.__doc__ = key.__doc__ or 'unknown' 2113 yo.key = key 2114 yo.field_names = field_names or table.field_names 2115 for record in table: 2116 value = key(record) 2117 if value is DoNotIndex: 2118 continue 2119 rec_num = record.record_number 2120 if not isinstance(value, tuple): 2121 value = (value, ) 2122 vindex = bisect_right(yo._values, value) 2123 yo._values.insert(vindex, value) 2124 yo._rec_by_val.insert(vindex, rec_num) 2125 yo._records[rec_num] = value 2126 table._indexen.add(yo)
2127 - def __call__(yo, record):
2128 rec_num = record.record_number 2129 if rec_num in yo._records: 2130 value = yo._records[rec_num] 2131 vindex = bisect_left(yo._values, value) 2132 yo._values.pop(vindex) 2133 yo._rec_by_val.pop(vindex) 2134 value = yo.key(record) 2135 if value is DoNotIndex: 2136 return 2137 if not isinstance(value, tuple): 2138 value = (value, ) 2139 vindex = bisect_right(yo._values, value) 2140 yo._values.insert(vindex, value) 2141 yo._rec_by_val.insert(vindex, rec_num) 2142 yo._records[rec_num] = value
2143 - def __contains__(yo, match):
2144 if isinstance(match, _DbfRecord): 2145 if match.record_table is yo._table: 2146 return match.record_number in yo._records 2147 match = yo.key(match) 2148 elif not isinstance(match, tuple): 2149 match = (match, ) 2150 return yo.find(match) != -1
2151 - def __getitem__(yo, key):
2152 if isinstance(key, int): 2153 count = len(yo._values) 2154 if not -count <= key < count: 2155 raise IndexError("Record %d is not in list." % key) 2156 rec_num = yo._rec_by_val[key] 2157 return yo._table.get_record(rec_num) 2158 elif isinstance(key, slice): 2159 result = List(field_names=yo._table.field_names) 2160 yo._table._dbflists.add(result) 2161 start, stop, step = key.start, key.stop, key.step 2162 if start is None: start = 0 2163 if stop is None: stop = len(yo._rec_by_val) 2164 if step is None: step = 1 2165 for loc in range(start, stop, step): 2166 record = yo._table.get_record(yo._rec_by_val[loc]) 2167 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2168 result._current = 0 if result else -1 2169 return result 2170 elif isinstance (key, (str, unicode, tuple, _DbfRecord)): 2171 if isinstance(key, _DbfRecord): 2172 key = yo.key(key) 2173 elif not isinstance(key, tuple): 2174 key = (key, ) 2175 loc = yo.find(key) 2176 if loc == -1: 2177 raise KeyError(key) 2178 return yo._table.get_record(yo._rec_by_val[loc]) 2179 else: 2180 raise TypeError('indices must be integers, match objects must by strings or tuples')
2181 - def __enter__(yo):
2182 return yo
2183 - def __exit__(yo, *exc_info):
2184 yo._table.close() 2185 yo._values[:] = [] 2186 yo._rec_by_val[:] = [] 2187 yo._records.clear() 2188 return False
2189 - def __iter__(yo):
2190 return yo.IndexIterator(yo._table, yo._rec_by_val)
2191 - def __len__(yo):
2192 return len(yo._records)
2193 - def _partial_match(yo, target, match):
2194 target = target[:len(match)] 2195 if isinstance(match[-1], (str, unicode)): 2196 target = list(target) 2197 target[-1] = target[-1][:len(match[-1])] 2198 target = tuple(target) 2199 return target == match
2200 - def _purge(yo, rec_num):
2201 value = yo._records.get(rec_num) 2202 if value is not None: 2203 vindex = bisect_left(yo._values, value) 2204 del yo._records[rec_num] 2205 yo._values.pop(vindex) 2206 yo._rec_by_val.pop(vindex)
2207 - def _search(yo, match, lo=0, hi=None):
2208 if hi is None: 2209 hi = len(yo._values) 2210 return bisect_left(yo._values, match, lo, hi)
2211 - def clear(yo):
2212 "removes all entries from index" 2213 yo._values[:] = [] 2214 yo._rec_by_val[:] = [] 2215 yo._records.clear()
2216 - def close(yo):
2217 yo._table.close()
2218 - def find(yo, match, partial=False):
2219 "returns numeric index of (partial) match, or -1" 2220 if isinstance(match, _DbfRecord): 2221 if match.record_number in yo._records: 2222 return yo._values.index(yo._records[match.record_number]) 2223 else: 2224 return -1 2225 if not isinstance(match, tuple): 2226 match = (match, ) 2227 loc = yo._search(match) 2228 while loc < len(yo._values) and yo._values[loc] == match: 2229 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted: 2230 loc += 1 2231 continue 2232 return loc 2233 if partial: 2234 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match): 2235 if not yo._table.use_deleted and yo._table.get_record(yo._rec_by_val[loc]).has_been_deleted: 2236 loc += 1 2237 continue 2238 return loc 2239 return -1
2240 - def find_index(yo, match):
2241 "returns numeric index of either (partial) match, or position of where match would be" 2242 if isinstance(match, _DbfRecord): 2243 if match.record_number in yo._records: 2244 return yo._values.index(yo._records[match.record_number]) 2245 else: 2246 match = yo.key(match) 2247 if not isinstance(match, tuple): 2248 match = (match, ) 2249 loc = yo._search(match) 2250 return loc
2251 @classmethod
2252 - def from_file(cls, table, index_file):
2253 2254 def get_idx_records(data, length, howmany): 2255 ptr = 0 2256 current = 0 2257 while current < howmany: 2258 key = data[ptr:ptr+length].replace('\x00','') 2259 rec = io.unpackLongInt(data[ptr+length:ptr+length+4], bigendian=True) 2260 yield key, recnum 2261 ptr += length + 4 2262 current += 1
2263 2264 def next_item(idx_file, node_loc, keylen): 2265 idx_file.seek(node_loc) 2266 data_chunk = idx_file.read(512) 2267 attributes = io.unpackShortInt(data_chunk[:2]) 2268 howmany = io.unpackShortInt(data_chunk[2:4]) 2269 if attributes in (2, 3): 2270 for key, recnum in get_idx_records(data_chunk[12:512], keylen, howmany): 2271 yield key, recnum 2272 else: 2273 for ignore, next_node in get_idx_records(data_chunk[12:512], keylen, howmany): 2274 print ignore, next_node 2275 for key, recnum in next_item(idx_file, next_node, keylen): 2276 yield key, recnum
2277 2278 2279 idx = object.__new__(cls) 2280 #- idx.key = lambda rec: DoNotIndex 2281 data = open(index_file, 'rb') 2282 header = data.read(512) 2283 rootnode = io.unpackLongInt(header[:4]) 2284 keylen = io.unpackShortInt(header[12:14]) 2285 idx.__doc__ = header[16:236].replace('\x00','') 2286 for_expr = header[236:456].replace('\x00','') 2287 if for_expr: 2288 idx.__doc__ += ' for ' + for_expr.replace('=','==') 2289 for rec in next_item(data, rootnode, keylen): 2290 print rec 2291
2292 - def index(yo, match, partial=False):
2293 "returns numeric index of (partial) match, or raises ValueError" 2294 loc = yo.find(match, partial) 2295 if loc == -1: 2296 if isinstance(match, _DbfRecord): 2297 raise ValueError("table <%s> record [%d] not in index <%s>" % (yo._table.filename, match.record_number, yo.__doc__)) 2298 else: 2299 raise ValueError("match criteria <%s> not in index" % (match, )) 2300 return loc
2301 - def reindex(yo):
2302 "reindexes all records" 2303 for record in yo._table: 2304 yo(record)
2305 - def query(yo, sql_command=None, python=None):
2306 """recognized sql commands are SELECT, UPDATE, REPLACE, INSERT, DELETE, and RECALL""" 2307 if sql_command: 2308 return sql(yo, sql_command) 2309 elif python is None: 2310 raise DbfError("query: python parameter must be specified") 2311 possible = List(desc="%s --> %s" % (yo._table.filename, python), field_names=yo._table.field_names) 2312 yo._table._dbflists.add(possible) 2313 query_result = {} 2314 select = 'query_result["keep"] = %s' % python 2315 g = {} 2316 for record in yo: 2317 query_result['keep'] = False 2318 g['query_result'] = query_result 2319 exec select in g, record 2320 if query_result['keep']: 2321 possible.append(record) 2322 record.write_record() 2323 return possible
2324 - def search(yo, match, partial=False):
2325 "returns dbf.List of all (partially) matching records" 2326 result = List(field_names=yo._table.field_names) 2327 yo._table._dbflists.add(result) 2328 if not isinstance(match, tuple): 2329 match = (match, ) 2330 loc = yo._search(match) 2331 if loc == len(yo._values): 2332 return result 2333 while loc < len(yo._values) and yo._values[loc] == match: 2334 record = yo._table.get_record(yo._rec_by_val[loc]) 2335 if not yo._table.use_deleted and record.has_been_deleted: 2336 loc += 1 2337 continue 2338 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2339 loc += 1 2340 if partial: 2341 while loc < len(yo._values) and yo._partial_match(yo._values[loc], match): 2342 record = yo._table.get_record(yo._rec_by_val[loc]) 2343 if not yo._table.use_deleted and record.has_been_deleted: 2344 loc += 1 2345 continue 2346 result._maybe_add(item=(yo._table, yo._rec_by_val[loc], result.key(record))) 2347 loc += 1 2348 return result
2349 2350 csv.register_dialect('dbf', DbfCsv) 2351 2352 sql_functions = { 2353 'select':None, 2354 'update':None, 2355 'insert':None, 2356 'delete':None, 2357 'count': None}
2358 -def sql_criteria(records, criteria):
2359 "creates a function matching the sql criteria" 2360 function = """def func(records): 2361 \"\"\"%s\"\"\" 2362 matched = List(field_names=records[0].field_names) 2363 for rec in records: 2364 %s 2365 2366 if %s: 2367 matched.append(rec) 2368 return matched""" 2369 fields = [] 2370 for field in records[0].field_names: 2371 if field in criteria: 2372 fields.append(field) 2373 fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields]) 2374 g = {'List':List} 2375 function %= (criteria, fields, criteria) 2376 #- print function 2377 exec function in g 2378 return g['func']
2379
2380 -def sql_cmd(records, command):
2381 "creates a function matching to apply command to each record in records" 2382 function = """def func(records): 2383 \"\"\"%s\"\"\" 2384 changed = 0 2385 for rec in records: 2386 %s 2387 2388 %s 2389 2390 %s 2391 changed += rec.write_record() 2392 return changed""" 2393 fields = [] 2394 for field in records[0].field_names: 2395 if field in command: 2396 fields.append(field) 2397 pre_fields = '\n '.join(['%s = rec.%s' % (field, field) for field in fields]) 2398 post_fields = '\n '.join(['rec.%s = %s' % (field, field) for field in fields]) 2399 g = dbf.sql_user_functions.copy() 2400 if '=' not in command and ' with ' in command.lower(): 2401 offset = command.lower().index(' with ') 2402 command = command[:offset] + ' = ' + command[offset+6:] 2403 function %= (command, pre_fields, command, post_fields) 2404 #- print function 2405 exec function in g 2406 return g['func']
2407
2408 -def sql(records, command):
2409 """recognized sql commands are SELECT, UPDATE, INSERT, DELETE, and RECALL""" 2410 table = records[0].record_table 2411 sql_command = command 2412 no_condition = False 2413 if ' for ' in command: 2414 command, condition = command.split(' for ') 2415 condition = sql_criteria(records, condition) 2416 else: 2417 def condition(records): 2418 return records[:]
2419 no_condition = True 2420 name, command = command.split(' ', 1) 2421 name = name.lower() 2422 field_names = table.field_names 2423 if name == 'select': 2424 if command.strip() != '*': 2425 field_names = command.replace(' ','').split(',') 2426 def command(records): 2427 return 2428 else: 2429 command = sql_cmd(records, command) 2430 if name not in ('delete','insert','recall','select','update','replace'): 2431 raise DbfError("unrecognized sql command: %s" % name.upper()) 2432 if name == 'insert' and not no_condition: 2433 raise DbfError("FOR clause not allowed with INSERT") 2434 possible = List(desc=sql_command, field_names=field_names) 2435 tables = set() 2436 if name == 'insert': 2437 raise DbfError("INSERT not currently implemented") 2438 record = table.append() 2439 command(record) 2440 record.write_record() 2441 record.check_index() 2442 possible.append(record) 2443 changed = 0 2444 else: 2445 possible = condition(records) 2446 possible.field_names = field_names 2447 changed = command(possible) 2448 for record in possible: 2449 tables.add(record.record_table) 2450 if name == 'delete': 2451 record.delete_record() 2452 elif name == 'recall': 2453 record.undelete_record() 2454 elif name == 'select': 2455 pass 2456 elif name == 'update' or name == 'replace': 2457 pass 2458 #command(record) 2459 else: 2460 raise DbfError("unrecognized sql command: %s" % sql.upper) 2461 record.write_record() 2462 for list_table in tables: 2463 list_table._dbflists.add(possible) 2464 possible.modified = changed 2465 return possible
2466 -def _nop(value):
2467 "returns parameter unchanged" 2468 return value
2469 -def _normalize_tuples(tuples, length, filler):
2470 "ensures each tuple is the same length, using filler[-missing] for the gaps" 2471 final = [] 2472 for t in tuples: 2473 if len(t) < length: 2474 final.append( tuple([item for item in t] + filler[len(t)-length:]) ) 2475 else: 2476 final.append(t) 2477 return tuple(final)
2478 -def _codepage_lookup(cp):
2479 if cp not in code_pages: 2480 for code_page in sorted(code_pages.keys()): 2481 sd, ld = code_pages[code_page] 2482 if cp == sd or cp == ld: 2483 if sd is None: 2484 raise DbfError("Unsupported codepage: %s" % ld) 2485 cp = code_page 2486 break 2487 else: 2488 raise DbfError("Unsupported codepage: %s" % cp) 2489 sd, ld = code_pages[cp] 2490 return cp, sd, ld
2491 -def ascii(new_setting=None):
2492 "get/set return_ascii setting" 2493 global return_ascii 2494 if new_setting is None: 2495 return return_ascii 2496 else: 2497 return_ascii = new_setting
2498 -def codepage(cp=None):
2499 "get/set default codepage for any new tables" 2500 global default_codepage 2501 cp, sd, ld = _codepage_lookup(cp or default_codepage) 2502 default_codepage = sd 2503 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2504 -def encoding(cp=None):
2505 "get/set default encoding for non-unicode strings passed into a table" 2506 global input_decoding 2507 cp, sd, ld = _codepage_lookup(cp or input_decoding) 2508 default_codepage = sd 2509 return "%s (LDID: 0x%02x - %s)" % (sd, ord(cp), ld)
2510 -class _Db4Table(DbfTable):
2511 version = 'dBase IV w/memos (non-functional)' 2512 _versionabbv = 'db4' 2513 _fieldtypes = { 2514 'C' : {'Type':'Character', 'Retrieve':io.retrieveCharacter, 'Update':io.updateCharacter, 'Blank':str, 'Init':io.addCharacter}, 2515 'Y' : {'Type':'Currency', 'Retrieve':io.retrieveCurrency, 'Update':io.updateCurrency, 'Blank':Decimal(), 'Init':io.addVfpCurrency}, 2516 'B' : {'Type':'Double', 'Retrieve':io.retrieveDouble, 'Update':io.updateDouble, 'Blank':float, 'Init':io.addVfpDouble}, 2517 'F' : {'Type':'Float', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':float, 'Init':io.addVfpNumeric}, 2518 'N' : {'Type':'Numeric', 'Retrieve':io.retrieveNumeric, 'Update':io.updateNumeric, 'Blank':int, 'Init':io.addVfpNumeric}, 2519 'I' : {'Type':'Integer', 'Retrieve':io.retrieveInteger, 'Update':io.updateInteger, 'Blank':int, 'Init':io.addVfpInteger}, 2520 'L' : {'Type':'Logical', 'Retrieve':io.retrieveLogical, 'Update':io.updateLogical, 'Blank':bool, 'Init':io.addLogical}, 2521 'D' : {'Type':'Date', 'Retrieve':io.retrieveDate, 'Update':io.updateDate, 'Blank':Date.today, 'Init':io.addDate}, 2522 'T' : {'Type':'DateTime', 'Retrieve':io.retrieveVfpDateTime, 'Update':io.updateVfpDateTime, 'Blank':DateTime.now, 'Init':io.addVfpDateTime}, 2523 'M' : {'Type':'Memo', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2524 'G' : {'Type':'General', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2525 'P' : {'Type':'Picture', 'Retrieve':io.retrieveMemo, 'Update':io.updateMemo, 'Blank':str, 'Init':io.addMemo}, 2526 '0' : {'Type':'_NullFlags', 'Retrieve':io.unsupportedType, 'Update':io.unsupportedType, 'Blank':int, 'Init':None} } 2527 _memoext = '.dbt' 2528 _memotypes = ('G','M','P') 2529 _memoClass = _VfpMemo 2530 _yesMemoMask = '\x8b' # 0011 0000 2531 _noMemoMask = '\x04' # 0011 0000 2532 _fixed_fields = ('B','D','G','I','L','M','P','T','Y') 2533 _variable_fields = ('C','F','N') 2534 _character_fields = ('C','M') # field representing character data 2535 _decimal_fields = ('F','N') 2536 _numeric_fields = ('B','F','I','N','Y') 2537 _supported_tables = ('\x04', '\x8b') 2538 _dbfTableHeader = ['\x00'] * 32 2539 _dbfTableHeader[0] = '\x8b' # version - Foxpro 6 0011 0000 2540 _dbfTableHeader[10] = '\x01' # record length -- one for delete flag 2541 _dbfTableHeader[29] = '\x03' # code page -- 437 US-MS DOS 2542 _dbfTableHeader = ''.join(_dbfTableHeader) 2543 _dbfTableHeaderExtra = '' 2544 _use_deleted = True
2545 - def _checkMemoIntegrity(yo):
2546 "dBase III specific" 2547 if yo._meta.header.version == '\x8b': 2548 try: 2549 yo._meta.memo = yo._memoClass(yo._meta) 2550 except: 2551 yo._meta.dfd.close() 2552 yo._meta.dfd = None 2553 raise 2554 if not yo._meta.ignorememos: 2555 for field in yo._meta.fields: 2556 if yo._meta[field]['type'] in yo._memotypes: 2557 if yo._meta.header.version != '\x8b': 2558 yo._meta.dfd.close() 2559 yo._meta.dfd = None 2560 raise DbfError("Table structure corrupt: memo fields exist, header declares no memos") 2561 elif not os.path.exists(yo._meta.memoname): 2562 yo._meta.dfd.close() 2563 yo._meta.dfd = None 2564 raise DbfError("Table structure corrupt: memo fields exist without memo file") 2565 break
2566