Package mvpa :: Package tests :: Module tests_warehouse
[hide private]
[frames] | no frames]

Source Code for Module mvpa.tests.tests_warehouse

  1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  # vi: set ft=python sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Miscelaneous functions/datasets to be used in the unit tests""" 
 10   
 11  __docformat__ = 'restructuredtext' 
 12   
 13  from os import environ 
 14   
 15  import unittest, traceback, sys 
 16  import numpy as N 
 17   
 18  from mvpa import cfg 
 19  from mvpa.datasets import Dataset 
 20  from mvpa.datasets.splitters import OddEvenSplitter 
 21  from mvpa.datasets.masked import MaskedDataset 
 22  from mvpa.clfs.base import Classifier 
 23  from mvpa.misc.state import ClassWithCollections 
 24  from mvpa.misc.data_generators import * 
 25   
 26  __all__ = [ 'datasets', 'sweepargs', 'N', 'unittest' ] 
 27   
 28  if __debug__: 
 29      from mvpa.base import debug 
 30      __all__.append('debug') 
 31   
 32   
33 -def sweepargs(**kwargs):
34 """Decorator function to sweep over a given set of classifiers 35 36 :Parameters: 37 clfs : list of `Classifier` 38 List of classifiers to run method on 39 40 Often some unittest method can be ran on multiple classifiers. 41 So this decorator aims to do that 42 """ 43 def unittest_method(method): 44 def do_sweep(*args_, **kwargs_): 45 def untrain_clf(argvalue): 46 """Little helper""" 47 if isinstance(argvalue, Classifier): 48 # clear classifier after its use -- just to be sure ;-) 49 argvalue.retrainable = False 50 argvalue.untrain()
51 52 failed_tests = {} 53 for argname in kwargs.keys(): 54 for argvalue in kwargs[argname]: 55 if isinstance(argvalue, Classifier): 56 # clear classifier before its use 57 argvalue.untrain() 58 if isinstance(argvalue, ClassWithCollections): 59 argvalue.states.reset() 60 # update kwargs_ 61 kwargs_[argname] = argvalue 62 # do actual call 63 try: 64 if __debug__: 65 debug('TEST', 'Running %s on args=%s and kwargs=%s' % 66 (method.__name__, `args_`, `kwargs_`)) 67 method(*args_, **kwargs_) 68 except AssertionError, e: 69 estr = str(e) 70 etype, value, tb = sys.exc_info() 71 # literal representation of exception tb, so 72 # we could group them later on 73 eidstr = ' '.join( 74 [l for l in traceback.format_exception(etype, value, tb) 75 if not ('do_sweep' in l or 'unittest.py' in l 76 or 'AssertionError' in l or 'Traceback (most' in l)]) 77 78 # Store exception information for later on groupping 79 if not eidstr in failed_tests: 80 failed_tests[eidstr] = [] 81 82 failed_tests[eidstr].append( 83 # skip top-most tb in sweep_args 84 (argname, `argvalue`, tb.tb_next, estr)) 85 86 if __debug__: 87 msg = "%s on %s=%s" % (estr, argname, `argvalue`) 88 debug('TEST', 'Failed unittest: %s\n%s' % (eidstr, msg)) 89 untrain_clf(argvalue) 90 # TODO: handle different levels of unittests properly 91 if cfg.getboolean('tests', 'quick', False): 92 # on TESTQUICK just run test for 1st entry in the list, 93 # the rest are omitted 94 # TODO: proper partitioning of unittests 95 break 96 97 if len(failed_tests): 98 # Lets now create a single AssertionError exception which would nicely 99 # incorporate all failed exceptions 100 multiple = len(failed_tests) != 1 # is it unique? 101 # if so, we don't need to reinclude traceback since it 102 # would be spitted out anyways below 103 estr = "" 104 cestr = "lead to failures of unittest %s" % method.__name__ 105 if multiple: 106 estr += "\n Different scenarios %s (specific tracebacks are below):" % cestr 107 else: 108 estr += "\n Single scenario %s:" % cestr 109 for ek, els in failed_tests.iteritems(): 110 estr += '\n' 111 if multiple: estr += ek 112 estr += " on\n %s" % (" ".join( 113 ["%s=%s%s\n" % (ea, eav, 114 # Why didn't I just do regular for loop? ;) 115 ":\n ".join([x for x in [' ', es] if x != ''])) 116 for ea, eav, etb, es in els])) 117 etb = els[0][2] # take first one... they all should be identical 118 raise AssertionError(estr), None, etb 119 120 do_sweep.func_name = method.func_name 121 return do_sweep 122 123 if len(kwargs) > 1: 124 raise NotImplementedError 125 return unittest_method 126 127 # Define datasets to be used all over. Split-half later on is used to 128 # split into training/testing 129 # 130 snr_scale = cfg.getAsDType('tests', 'snr scale', float, default=1.0) 131 132 specs = {'large' : { 'perlabel': 99, 'nchunks': 11, 'nfeatures': 20, 'snr': 8 * snr_scale}, 133 'medium' :{ 'perlabel': 24, 'nchunks': 6, 'nfeatures': 14, 'snr': 8 * snr_scale}, 134 'small' : { 'perlabel': 12, 'nchunks': 4, 'nfeatures': 6, 'snr' : 14 * snr_scale} } 135 nonbogus_pool = [0, 1, 3, 5] 136 137 datasets = {} 138 139 for kind, spec in specs.iteritems(): 140 # set of univariate datasets 141 for nlabels in [ 2, 3, 4 ]: 142 basename = 'uni%d%s' % (nlabels, kind) 143 nonbogus_features=nonbogus_pool[:nlabels] 144 bogus_features = filter(lambda x:not x in nonbogus_features, 145 range(spec['nfeatures'])) 146 147 dataset = normalFeatureDataset( 148 nlabels=nlabels, 149 nonbogus_features=nonbogus_features, 150 **spec) 151 dataset.nonbogus_features = nonbogus_features 152 dataset.bogus_features = bogus_features 153 oes = OddEvenSplitter() 154 splits = [(train, test) for (train, test) in oes(dataset)] 155 for i, replication in enumerate( ['test', 'train'] ): 156 dataset_ = splits[0][i] 157 dataset_.nonbogus_features = nonbogus_features 158 dataset_.bogus_features = bogus_features 159 datasets["%s_%s" % (basename, replication)] = dataset_ 160 161 # full dataset 162 datasets[basename] = dataset 163 164 # sample 3D 165 total = 2*spec['perlabel'] 166 nchunks = spec['nchunks'] 167 data = N.random.standard_normal(( total, 3, 6, 6 )) 168 labels = N.concatenate( ( N.repeat( 0, spec['perlabel'] ), 169 N.repeat( 1, spec['perlabel'] ) ) ) 170 chunks = N.asarray(range(nchunks)*(total/nchunks)) 171 mask = N.ones( (3, 6, 6) ) 172 mask[0,0,0] = 0 173 mask[1,3,2] = 0 174 datasets['3d%s' % kind] = MaskedDataset(samples=data, labels=labels, 175 chunks=chunks, mask=mask) 176 177 # some additional datasets 178 datasets['dumb2'] = dumbFeatureBinaryDataset() 179 datasets['dumb'] = dumbFeatureDataset() 180 # dataset with few invariant features 181 _dsinv = dumbFeatureDataset() 182 _dsinv.samples = N.hstack((_dsinv.samples, 183 N.zeros((_dsinv.nsamples, 1)), 184 N.ones((_dsinv.nsamples, 1)))) 185 datasets['dumbinv'] = _dsinv 186 187 # Datasets for regressions testing 188 datasets['sin_modulated'] = multipleChunks(sinModulated, 4, 30, 1) 189 datasets['sin_modulated_test'] = sinModulated(30, 1, flat=True) 190 191 # simple signal for linear regressors 192 datasets['chirp_linear'] = multipleChunks(chirpLinear, 6, 50, 10, 2, 0.3, 0.1) 193 datasets['chirp_linear_test'] = chirpLinear(20, 5, 2, 0.4, 0.1) 194 195 datasets['wr1996'] = multipleChunks(wr1996, 4, 50) 196 datasets['wr1996_test'] = wr1996(50) 197