Package mvpa :: Package tests :: Module test_svm

[hide private]
[frames] | no frames]

# Source Code for Module mvpa.tests.test_svm

      1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
      2  # vi: set ft=python sts=4 ts=4 sw=4 et:
      3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      4  #
      5  #   See COPYING file distributed along with the PyMVPA package for the
      6  #   copyright and license terms.
      7  #
      8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      9  """Unit tests for SVM classifier"""
     10
     11  from mvpa.datasets.splitters import NFoldSplitter
     12  from mvpa.clfs.meta import ProxyClassifier
     13  from mvpa.clfs.transerror import TransferError
     14  from mvpa.algorithms.cvtranserror import CrossValidatedTransferError
     15
     16  from tests_warehouse import pureMultivariateSignal
     17  from tests_warehouse import *
     18  from tests_warehouse_clfs import *
     19
     20 -class SVMTests(unittest.TestCase):
     21
     22  #    @sweepargs(nl_clf=clfswh['non-linear', 'svm'] )
     23  #    @sweepargs(nl_clf=clfswh['non-linear', 'svm'] )
     24 -    def testMultivariate(self):
     25          mv_perf = []
     26          mv_lin_perf = []
     27          uv_perf = []
     28
     29          l_clf = clfswh['linear', 'svm'][0]
     30          nl_clf = clfswh['non-linear', 'svm'][0]
     31
     32          #orig_keys = nl_clf.param._params.keys()
     33          #nl_param_orig = nl_clf.param._params.copy()
     34
     35          # l_clf = LinearNuSVMC()
     36
     37          # XXX ??? not sure what below meant and it is obsolete if
     38          # using SG... commenting out for now
     39          # for some reason order is not preserved thus dictionaries are not
     40          # the same any longer -- lets compare values
     41          #self.failUnlessEqual([nl_clf.param._params[k] for k in orig_keys],
     42          #                     [nl_param_orig[k] for k in orig_keys],
     43          #   msg="New instance mustn't override values in previously created")
     44          ## and keys separately
     45          #self.failUnlessEqual(set(nl_clf.param._params.keys()),
     46          #                     set(orig_keys),
     47          #   msg="New instance doesn't change set of parameters in original")
     48
     49          # We must be able to deepcopy not yet trained SVMs now
     50          import mvpa.support.copy as copy
     51          try:
     52              nl_clf.untrain()
     53              nl_clf_copy = copy.deepcopy(nl_clf)
     54          except:
     55              self.fail(msg="Failed to deepcopy not-yet trained SVM %s" % nl_clf)
     56
     57          for i in xrange(20):
     58              train = pureMultivariateSignal( 20, 3 )
     59              test = pureMultivariateSignal( 20, 3 )
     60
     61              # use non-linear CLF on 2d data
     62              nl_clf.train(train)
     63              p_mv = nl_clf.predict(test.samples)
     64              mv_perf.append(N.mean(p_mv==test.labels))
     65
     66              # use linear CLF on 2d data
     67              l_clf.train(train)
     68              p_lin_mv = l_clf.predict(test.samples)
     69              mv_lin_perf.append(N.mean(p_lin_mv==test.labels))
     70
     71              # use non-linear CLF on 1d data
     72              nl_clf.train(train.selectFeatures([0]))
     73              p_uv = nl_clf.predict(test.selectFeatures([0]).samples)
     74              uv_perf.append(N.mean(p_uv==test.labels))
     75
     76          mean_mv_perf = N.mean(mv_perf)
     77          mean_mv_lin_perf = N.mean(mv_lin_perf)
     78          mean_uv_perf = N.mean(uv_perf)
     79
     80          # non-linear CLF has to be close to perfect
     81          self.failUnless( mean_mv_perf > 0.9 )
     82          # linear CLF cannot learn this problem!
     83          self.failUnless( mean_mv_perf > mean_mv_lin_perf )
     84          # univariate has insufficient information
     85          self.failUnless( mean_uv_perf < mean_mv_perf )
     86
     87
     88      # XXX for now works only with linear... think it through -- should
     89      # work non-linear, shouldn't it?
     90      # now all non-linear have C>0 thus skipped anyways
     91
     92      # TODO: For some reason libsvm's weight assignment has no effect
     93      # as well -- need to be fixed :-/
     94      @sweepargs(clf=clfswh['svm', 'sg', '!regression', '!gnpp', '!meta'])
     95 -    def testCperClass(self, clf):
     96          try:
     97              if clf.C > 0:
     98                  # skip those with fixed C
     99                  return
    100          except:
    101              # classifier has no C
    102              return
    103
    104          if clf.C < -5:
    105              # too soft margin helps to fight disbalance, thus skip
    106              # it in testing
    107              return
    108          #print clf
    109          ds = datasets['uni2small'].copy()
    110          ds__ = datasets['uni2small'].copy()
    111          #
    112          # ballanced set
    113          # Lets add a bit of noise to drive classifier nuts. same
    114          # should be done for disballanced set
    115          ds__.samples = ds__.samples + 0.5 * N.random.normal(size=(ds__.samples.shape))
    116          #
    117          # disballanced set
    118          # lets overpopulate label 0
    119          times = 10
    120          ds_ = ds.selectSamples(range(ds.nsamples) + range(ds.nsamples/2) * times)
    121          ds_.samples = ds_.samples + 0.7 * N.random.normal(size=(ds_.samples.shape))
    122          spl = ds_.samplesperlabel
    123          #print ds_.labels, ds_.chunks
    124
    125          cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(),
    126                                            enable_states='confusion')
    127          e = cve(ds__)
    128          if cfg.getboolean('tests', 'labile', default='yes'):
    129              # without disballance we should already have some hits
    130              self.failUnless(cve.confusion.stats["P'"][1] > 0)
    131
    132          e = cve(ds_)
    133          if cfg.getboolean('tests', 'labile', default='yes'):
    134              self.failUnless(cve.confusion.stats["P'"][1] < 5,
    135                              msg="With disballance we should have almost no "
    136                              "hits. Got %f" % cve.confusion.stats["P'"][1])
    137              #print "D:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1]
    138
    139          # Set '1 C per label'
    140          # recreate cvte since previous might have operated on copies
    141          cve = CrossValidatedTransferError(TransferError(clf), NFoldSplitter(),
    142                                            enable_states='confusion')
    143          oldC = clf.C
    144          ratio = N.sqrt(float(spl[0])/spl[1])
    145          clf.C = (-1/ratio, -1*ratio)
    146          try:
    147              e_ = cve(ds_)
    148              # reassign C
    149              clf.C = oldC
    150          except:
    151              clf.C = oldC
    152              raise
    153          #print "B:", cve.confusion.stats["P'"][1], cve.confusion.stats['MCC'][1]
    154          if cfg.getboolean('tests', 'labile', default='yes'):
    155              # Finally test if we get any 'hit' for minor category. In the
    156              # classifier, which has way to 'ballance' should be non-0
    157              self.failUnless(cve.confusion.stats["P'"][1] > 0)
    158
    159
    160 -    def testSillyness(self):
    161          """Test if we raise exceptions on incorrect specifications
    162          """
    163
    164          if externals.exists('libsvm') or externals.exists('shogun'):
    165              self.failUnlessRaises(TypeError, SVM,  C=1.0, nu=2.3)
    166
    167          if externals.exists('libsvm'):
    168              self.failUnlessRaises(TypeError, libsvm.SVM,  C=1.0, nu=2.3)
    169              self.failUnlessRaises(TypeError, LinearNuSVMC, C=2.3)
    170              self.failUnlessRaises(TypeError, LinearCSVMC, nu=2.3)
    171
    172          if externals.exists('shogun'):
    173              self.failUnlessRaises(TypeError, sg.SVM, C=10, kernel_type='RBF',
    174                                    coef0=3)
    175
    176 -def suite():
    177      return unittest.makeSuite(SVMTests)
    178
    179
    180  if __name__ == '__main__':
    181      import runner
    182