Package mvpa :: Package tests :: Module test_svm
[hide private]
[frames] | no frames]

Source Code for Module mvpa.tests.test_svm

  1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*- 
  2  # vi: set ft=python sts=4 ts=4 sw=4 et: 
  3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  4  # 
  5  #   See COPYING file distributed along with the PyMVPA package for the 
  6  #   copyright and license terms. 
  7  # 
  8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ## 
  9  """Unit tests for SVM classifier""" 
 10   
 11  from sets import Set 
 12   
 13  from mvpa.datasets.splitters import NFoldSplitter 
 14  from mvpa.clfs.meta import ProxyClassifier 
 15  from mvpa.clfs.transerror import TransferError 
 16  from mvpa.algorithms.cvtranserror import CrossValidatedTransferError 
 17   
 18  from tests_warehouse import pureMultivariateSignal 
 19  from tests_warehouse import * 
 20  from tests_warehouse_clfs import * 
21 22 -class SVMTests(unittest.TestCase):
23 24 # @sweepargs(nl_clf=clfswh['non-linear', 'svm'] ) 25 # @sweepargs(nl_clf=clfswh['non-linear', 'svm'] )
26 - def testMultivariate(self):
27 mv_perf = [] 28 mv_lin_perf = [] 29 uv_perf = [] 30 31 l_clf = clfswh['linear', 'svm'][0] 32 nl_clf = clfswh['non-linear', 'svm'][0] 33 34 #orig_keys = nl_clf.param._params.keys() 35 #nl_param_orig = nl_clf.param._params.copy() 36 37 # l_clf = LinearNuSVMC() 38 39 # XXX ??? not sure what below meant and it is obsolete if 40 # using SG... commenting out for now 41 # for some reason order is not preserved thus dictionaries are not 42 # the same any longer -- lets compare values 43 #self.failUnlessEqual([nl_clf.param._params[k] for k in orig_keys], 44 # [nl_param_orig[k] for k in orig_keys], 45 # msg="New instance mustn't override values in previously created") 46 ## and keys separately 47 #self.failUnlessEqual(Set(nl_clf.param._params.keys()), 48 # Set(orig_keys), 49 # msg="New instance doesn't change set of parameters in original") 50 51 # We must be able to deepcopy not yet trained SVMs now 52 import mvpa.support.copy as copy 53 try: 54 nl_clf.untrain() 55 nl_clf_copy = copy.deepcopy(nl_clf) 56 except: 57 self.fail(msg="Failed to deepcopy not-yet trained SVM %s" % nl_clf) 58 59 for i in xrange(20): 60 train = pureMultivariateSignal( 20, 3 ) 61 test = pureMultivariateSignal( 20, 3 ) 62 63 # use non-linear CLF on 2d data 64 nl_clf.train(train) 65 p_mv = nl_clf.predict(test.samples) 66 mv_perf.append(N.mean(p_mv==test.labels)) 67 68 # use linear CLF on 2d data 69 l_clf.train(train) 70 p_lin_mv = l_clf.predict(test.samples) 71 mv_lin_perf.append(N.mean(p_lin_mv==test.labels)) 72 73 # use non-linear CLF on 1d data 74 nl_clf.train(train.selectFeatures([0])) 75 p_uv = nl_clf.predict(test.selectFeatures([0]).samples) 76 uv_perf.append(N.mean(p_uv==test.labels)) 77 78 mean_mv_perf = N.mean(mv_perf) 79 mean_mv_lin_perf = N.mean(mv_lin_perf) 80 mean_uv_perf = N.mean(uv_perf) 81 82 # non-linear CLF has to be close to perfect 83 self.failUnless( mean_mv_perf > 0.9 ) 84 # linear CLF cannot learn this problem! 85 self.failUnless( mean_mv_perf > mean_mv_lin_perf ) 86 # univariate has insufficient information 87 self.failUnless( mean_uv_perf < mean_mv_perf )
88 89 90 # XXX for now works only with linear... think it through -- should 91 # work non-linear, shouldn't it? 92 # now all non-linear have C>0 thus skipped anyways 93 94 # TODO: For some reason libsvm's weight assignment has no effect 95 # as well -- need to be fixed :-/ 96 @sweepargs(clf=clfswh['svm', 'sg', '!regression', '!gnpp', '!meta'])
97 - def testCperClass(self, clf):
98 try: 99 if clf.C > 0: 100 # skip those with fixed C 101 return 102 except: 103 # classifier has no C 104 return 105 106 if clf.C < -5: 107 # too soft margin helps to fight disbalance, thus skip 108 # it in testing 109 return 110 #print clf 111 ds = datasets['uni2small'].copy() 112 ds__ = datasets['uni2small'].copy() 113 # 114 # ballanced set 115 # Lets add a bit of noise to drive classifier nuts. same 116 # should be done for disballanced set 117 ds__.samples = ds__.samples + 0.5 * N.random.normal(size=(ds__.samples.shape)) 118 # 119 # disballanced set 120 # lets overpopulate label 0 121 times = 10 122 ds_ = ds.selectSamples(range(ds.nsamples) + range(ds.nsamples/2) * times) 123 ds_.samples = ds_.samples + 0.7 * N.random.normal(size=(ds_.samples.shape)) 124 spl = ds_.samplesperlabel 125 #print ds_.labels, ds_.chunks 126 127 cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(), 128 enable_states='confusion') 129 e = cve(ds__) 130 if cfg.getboolean('tests', 'labile', default='yes'): 131 # without disballance we should already have some hits 132 self.failUnless(cve.confusion.stats["P'"][1] > 0) 133 134 e = cve(ds_) 135 if cfg.getboolean('tests', 'labile', default='yes'): 136 self.failUnless(cve.confusion.stats["P'"][1] < 5, 137 msg="With disballance we should have almost no " 138 "hits. Got %f" % cve.confusion.stats["P'"][1]) 139 #print "D:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1] 140 141 # Set '1 C per label' 142 oldC = clf.C 143 ratio = N.sqrt(float(spl[0])/spl[1]) 144 clf.C = (-1/ratio, -1*ratio) 145 try: 146 e_ = cve(ds_) 147 # reassign C 148 clf.C = oldC 149 except: 150 clf.C = oldC 151 raise 152 #print "B:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1] 153 if cfg.getboolean('tests', 'labile', default='yes'): 154 # Finally test if we get any 'hit' for minor category. In the 155 # classifier, which has way to 'ballance' should be non-0 156 self.failUnless(cve.confusion.stats["P'"][1] > 0)
157 158
159 - def testSillyness(self):
160 """Test if we raise exceptions on incorrect specifications 161 """ 162 163 if externals.exists('libsvm') or externals.exists('shogun'): 164 self.failUnlessRaises(TypeError, SVM, C=1.0, nu=2.3) 165 166 if externals.exists('libsvm'): 167 self.failUnlessRaises(TypeError, libsvm.SVM, C=1.0, nu=2.3) 168 self.failUnlessRaises(TypeError, LinearNuSVMC, C=2.3) 169 self.failUnlessRaises(TypeError, LinearCSVMC, nu=2.3) 170 171 if externals.exists('shogun'): 172 self.failUnlessRaises(TypeError, sg.SVM, C=10, kernel_type='RBF', 173 coef0=3)
174
175 -def suite():
176 return unittest.makeSuite(SVMTests)
177 178 179 if __name__ == '__main__': 180 import runner 181