Package mvpa :: Package misc :: Package io :: Module base

[hide private]
[frames] | no frames]

# Source Code for Module mvpa.misc.io.base

      1  # emacs: -*- mode: python; py-indent-offset: 4; indent-tabs-mode: nil -*-
      2  # vi: set ft=python sts=4 ts=4 sw=4 et:
      3  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      4  #
      5  #   See COPYING file distributed along with the PyMVPA package for the
      6  #   copyright and license terms.
      7  #
      8  ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ### ##
      9  """Some little helper for reading (and writing) common formats from and to
     10  disk."""
     11
     12  __docformat__ = 'restructuredtext'
     13
     14  import numpy as N
     15  import mvpa.support.copy as copy
     16  from mvpa.base.dochelpers import enhancedDocString
     17  from re import sub as re_sub
     18  from mvpa.base import warning
     19
     20  from mvpa.misc.support import Event
     21
     22  if __debug__:
     23      from mvpa.base import debug
     24
     25
     26 -class DataReader(object):
     27      """Base class for data readers.
     28
     29      Every subclass has to put all information into to variable:
     30
     31      `self._data`: ndarray
     32          The data array has to have the samples separating dimension along the
     33          first axis.
     34      `self._props`: dict
     35          All other meaningful information has to be stored in a dictionary.
     36
     37      This class provides two methods (and associated properties) to retrieve
     38      this information.
     39      """
     40 -    def __init__(self):
     41          """Cheap init.
     42          """
     43          self._props = {}
     44          self._data = None
     45
     46
     47 -    def getPropsAsDict(self):
     48          """Return the dictionary with the data properties.
     49          """
     50          return self._props
     51
     52
     53 -    def getData(self):
     54          """Return the data array.
     55          """
     56          return self._data
     57
     58
     59      data  = property(fget=getData, doc="Data array")
     60      props = property(fget=getPropsAsDict, doc="Property dict")
     61
     62
     63
     64 -class ColumnData(dict):
     65      """Read data that is stored in columns of text files.
     66
     67      All read data is available via a dictionary-like interface. If
     68      column headers are available, the column names serve as dictionary keys.
     69      If no header exists an articfical key is generated: str(number_of_column).
     70
     71      Splitting of text file lines is performed by the standard split() function
     72      (which gets passed the `sep` argument as separator string) and each
     73      element is converted into the desired datatype.
     74
     75      Because data is read into a dictionary no two columns can have the same
     76      name in the header! Each column is stored as a list in the dictionary.
     77      """
     78 -    def __init__(self, source, header=True, sep=None, headersep=None,
     79                   dtype=float, skiplines=0):
     80          """Read data from file into a dictionary.
     81
     82          :Parameters:
     83            source : basestring or dict
     84              If values is given as a string all data is read from the
     85              file and additonal keyword arguments can be sued to
     86              customize the read procedure. If a dictionary is passed
     87              a deepcopy is performed.
     88            header : bool or list of basestring
     89              Indicates whether the column names should be read from the
     90              first line (`header=True`). If `header=False` unique
     91              column names will be generated (see class docs). If
     92              `header` is a python list, it's content is used as column
     93              header names and its length has to match the number of
     94              columns in the file.
     95            sep : basestring or None
     96              Separator string. The actual meaning depends on the output
     97              format (see class docs).
     98            headersep : basestring or None
     99              Separator string used in the header. The actual meaning
    100              depends on the output format (see class docs).
    101            dtype : type or list(types)
    102              Desired datatype(s). Datatype per column get be specified by
    103              passing a list of types.
    104            skiplines : int
    105              Number of lines to skip at the beginning of the file.
    106          """
    107          # init base class
    108          dict.__init__(self)
    109
    110          # intialize with default
    111          self._header_order = None
    112
    113          if isinstance(source, str):
    114              self._fromFile(source, header=header, sep=sep, headersep=headersep,
    115                             dtype=dtype, skiplines=skiplines)
    116
    117          elif isinstance(source, dict):
    118              for k, v in source.iteritems():
    119                  self[k] = v
    120              # check data integrity
    121              self._check()
    122
    123          else:
    124              raise ValueError, 'Unkown source for ColumnData [%s]' \
    125                                % `type(source)`
    126
    127          # generate missing properties for each item in the header
    128          classdict = self.__class__.__dict__
    129          for k in self.keys():
    130              if not classdict.has_key(k):
    131                  getter = "lambda self: self._getAttrib('%s')" % (k)
    132                  # Sanitarize the key, substitute ' []' with '_'
    133                  k_ = re_sub('[[\] ]', '_', k)
    134                  # replace multipe _s
    135                  k_ = re_sub('__+', '_', k_)
    136                  # remove quotes
    137                  k_ = re_sub('["\']', '', k_)
    138                  if __debug__:
    139                      debug("IOH", "Registering property %s for ColumnData key %s"
    140                            % (k_, k))
    141                  # make sure to import class directly into local namespace
    142                  # otherwise following does not work for classes defined
    143                  # elsewhere
    144                  exec 'from %s import %s' % (self.__module__,
    145                                              self.__class__.__name__)
    146                  exec "%s.%s = property(fget=%s)"  % \
    147                       (self.__class__.__name__, k_, getter)
    148                  # TODO!!! Check if it is safe actually here to rely on value of
    149                  #         k in lambda. May be it is treated as continuation and
    150                  #         some local space would override it????
    151                  #setattr(self.__class__,
    152                  #        k,
    153                  #        property(fget=lambda x: x._getAttrib("%s" % k)))
    154                  # it seems to be error-prone due to continuation...
    155
    156
    157      __doc__ = enhancedDocString('ColumnData', locals())
    158
    159
    160 -    def _getAttrib(self, key):
    161          """Return corresponding value if given key is known to current instance
    162
    163          Is used for automatically added properties to the class.
    164
    165          :Raises:
    166            ValueError:
    167              If `key` is not known to given instance
    168
    169          :Returns:
    170            Value if `key` is known
    171          """
    172          if self.has_key(key):
    173              return self[key]
    174          else:
    175              raise ValueError, "Instance %s has no data about %s" \
    176                  % (`self`, `key`)
    177
    178
    179 -    def __str__(self):
    180          s = self.__class__.__name__
    181          if len(self.keys())>0:
    182              s += " %d rows, %d columns [" % \
    183                   (self.getNRows(), self.getNColumns())
    184              s += reduce(lambda x, y: x+" %s" % y, self.keys())
    185              s += "]"
    186          return s
    187
    188 -    def _check(self):
    189          """Performs some checks for data integrity.
    190          """
    191          length = None
    192          for k in self.keys():
    193              if length == None:
    194                  length = len(self[k])
    195              else:
    196                  if not len(self[k]) == length:
    197                      raise ValueError, "Data integrity lost. Columns do not " \
    198                                        "have equal length."
    199
    200
    201 -    def _fromFile(self, filename, header, sep, headersep,
    202                    dtype, skiplines):
    203          """Loads column data from file -- clears object first.
    204          """
    205          # make a clean table
    206          self.clear()
    207
    208          file_ = open(filename, 'r')
    209
    210          self._header_order = None
    211
    212          [ file_.readline() for x in range(skiplines) ]
    213          """Simply skip some lines"""
    214          # make column names, either take header or generate
    215          if header == True:
    216              # read first line and split by 'sep'
    217              hdr = file_.readline().split(headersep)
    218              # remove bogus empty header titles
    219              hdr = filter(lambda x:len(x.strip()), hdr)
    220              self._header_order = hdr
    221          elif isinstance(header, list):
    222              hdr = header
    223          else:
    224              hdr = [ str(i) for i in xrange(len(file_.readline().split(sep))) ]
    225              # reset file to not miss the first line
    226              file_.seek(0)
    227              [ file_.readline() for x in range(skiplines) ]
    228
    229
    230          # string in lists: one per column
    231          tbl = [ [] for i in xrange(len(hdr)) ]
    232
    233          # do per column dtypes
    234          if not isinstance(dtype, list):
    235              dtype = [dtype] * len(hdr)
    236
    237          # parse line by line and feed into the lists
    238          for line in file_:
    239              # get rid of leading and trailing whitespace
    240              line = line.strip()
    241              # ignore empty lines and comment lines
    242              if not line or line.startswith('#'):
    243                  continue
    244              l = line.split(sep)
    245
    246              if not len(l) == len(hdr):
    247                  raise RuntimeError, \
    248                        "Number of entries in line [%i] does not match number " \
    249                        "of columns in header [%i]." % (len(l), len(hdr))
    250
    251              for i, v in enumerate(l):
    252                  if not dtype[i] is None:
    253                      try:
    254                          v = dtype[i](v)
    255                      except ValueError:
    256                          warning("Can't convert %s to desired datatype %s." %
    257                                  (`v`, `dtype`) + " Leaving original type")
    258                  tbl[i].append(v)
    259
    260          # check
    261          if not len(tbl) == len(hdr):
    262              raise RuntimeError, "Number of columns read from file does not " \
    263                                  "match the number of header entries."
    264
    265          # fill dict
    266          for i, v in enumerate(hdr):
    267              self[v] = tbl[i]
    268
    269
    270 -    def __iadd__(self, other):
    271          """Merge column data.
    272          """
    273          # for all columns in the other object
    274          for k, v in other.iteritems():
    275              if not self.has_key(k):
    276                  raise ValueError, 'Unknown key [%s].' % `k`
    277              if not isinstance(v, list):
    278                  raise ValueError, 'Can only merge list data, but got [%s].' \
    279                                    % `type(v)`
    280              # now it seems to be ok
    281              # XXX check for datatype?
    282              self[k] += v
    283
    284          # look for problems, like columns present in self, but not in other
    285          self._check()
    286
    287          return self
    288
    289
    290 -    def selectSamples(self, selection):
    291          """Return new ColumnData with selected samples"""
    292
    293          data = copy.deepcopy(self)
    294          for k, v in data.iteritems():
    295              data[k] = [v[x] for x in selection]
    296
    297          data._check()
    298          return data
    299
    300
    301 -    def getNColumns(self):
    302          """Returns the number of columns.
    303          """
    304          return len(self.keys())
    305
    306
    307 -    def tofile(self, filename, header=True, header_order=None, sep=' '):
    308          """Write column data to a text file.
    309
    310          :Parameters:
    311            filename : basestring
    312              Target filename
    313            header : bool
    314              If `True` a column header is written, using the column
    315              keys. If `False` no header is written.
    316            header_order : None or list of basestring
    317              If it is a list of strings, they will be used instead
    318              of simply asking for the dictionary keys. However
    319              these strings must match the dictionary keys in number
    320              and identity. This argument type can be used to
    321              determine the order of the columns in the output file.
    322              The default value is `None`. In this case the columns
    323              will be in an arbitrary order.
    324            sep : basestring
    325              String that is written as a separator between to data columns.
    326          """
    327          # XXX do the try: except: dance
    328          file_ = open(filename, 'w')
    329
    330          # write header
    331          if header_order == None:
    332              if self._header_order is None:
    333                  col_hdr = self.keys()
    334              else:
    335                  # use stored order + newly added keys at the last columns
    336                  col_hdr = self._header_order + \
    337                            list(set(self.keys()).difference(
    338                                                  set(self._header_order)))
    339          else:
    340              if not len(header_order) == self.getNColumns():
    341                  raise ValueError, 'Header list does not match number of ' \
    342                                    'columns.'
    343              for k in header_order:
    344                  if not self.has_key(k):
    345                      raise ValueError, 'Unknown key [%s]' % `k`
    346              col_hdr = header_order
    347
    348          if header == True:
    349              file_.write(sep.join(col_hdr) + '\n')
    350
    351          # for all rows
    352          for r in xrange(self.getNRows()):
    353              # get attributes for all keys
    354              l = [str(self[k][r]) for k in col_hdr]
    355              # write to file with proper separator
    356              file_.write(sep.join(l) + '\n')
    357
    358          file_.close()
    359
    360
    361 -    def getNRows(self):
    362          """Returns the number of rows.
    363          """
    364          # no data no rows (after Bob Marley)
    365          if not len(self.keys()):
    366              return 0
    367          # otherwise first key is as good as any other
    368          else:
    369              return len(self[self.keys()[0]])
    370
    371      ncolumns = property(fget=getNColumns)
    372      nrows = property(fget=getNRows)
    373
    374
    375
    376 -class SampleAttributes(ColumnData):
    377      """Read and write PyMVPA sample attribute definitions from and to text
    378      files.
    379      """
    380 -    def __init__(self, source, literallabels=False, header=None):
    381          """Read PyMVPA sample attributes from disk.
    382
    383          :Parameters:
    384            source: basestring
    385              Filename of an atrribute file
    386            literallabels: bool
    387              Either labels are given as literal strings
    388            header: None or bool or list of str
    389              If None, ['labels', 'chunks'] is assumed. Otherwise the same
    390              behavior as of `ColumnData`
    391          """
    392          if literallabels:
    393              dtypes = [str, float]
    394          else:
    395              dtypes = float
    396
    397          if header is None:
    398              header = ['labels', 'chunks']
    399          ColumnData.__init__(self, source,
    400                              header=header,
    401                              sep=None, dtype=dtypes)
    402
    403
    404 -    def tofile(self, filename):
    405          """Write sample attributes to a text file.
    406          """
    407          ColumnData.tofile(self, filename,
    408                            header=False,
    409                            header_order=['labels', 'chunks'],
    410                            sep=' ')
    411
    412
    413 -    def getNSamples(self):
    414          """Returns the number of samples in the file.
    415          """
    416          return self.getNRows()
    417
    418
    419 -    def toEvents(self, **kwargs):
    420          """Convert into a list of `Event` instances.
    421
    422          Each change in the label or chunks value is taken as a new event onset.
    423          The length of an event is determined by the number of identical
    424          consecutive label-chunk combinations. Since the attributes list has no
    425          sense of absolute timing, both `onset` and `duration` are determined and
    426          stored in #samples units.
    427
    428          :Parameters:
    429            kwargs
    430              Any keyword arugment provided would be replicated, through all
    431              the entries.
    432          """
    433          events = []
    434          prev_onset = 0
    435          old_comb = None
    436          duration = 1
    437          # over all samples
    438          for r in xrange(self.nrows):
    439              # the label-chunk combination
    440              comb = (self.labels[r], self.chunks[r])
    441
    442              # check if things changed
    443              if not comb == old_comb:
    444                  # did we ever had an event
    445                  if not old_comb is None:
    446                      events.append(
    447                          Event(onset=prev_onset, duration=duration,
    448                                label=old_comb[0], chunk=old_comb[1], **kwargs))
    449                      # reset duration for next event
    450                      duration = 1
    451                      # store the current samples as onset for the next event
    452                      prev_onset = r
    453
    454                  # update the reference combination
    455                  old_comb = comb
    456              else:
    457                  # current event is lasting
    458                  duration += 1
    459
    460          # push the last event in the pipeline
    461          if not old_comb is None:
    462              events.append(
    463                  Event(onset=prev_onset, duration=duration,
    464                        label=old_comb[0], chunk=old_comb[1], **kwargs))
    465
    466          return events
    467
    468
    469      nsamples = property(fget=getNSamples)
    470
    471
    472 -class SensorLocations(ColumnData):
    473      """Base class for sensor location readers.
    474
    475      Each subclass should provide x, y, z coordinates via the `pos_x`, `pos_y`,
    476      and `pos_z` attrbibutes.
    477
    478      Axes should follow the following convention:
    479
    480        x-axis: left -> right
    481        y-axis: anterior -> posterior
    482        z-axis: superior -> inferior
    483      """
    484 -    def __init__(self, *args, **kwargs):
    485          """Pass arguments to ColumnData.
    486          """
    487          ColumnData.__init__(self, *args, **kwargs)
    488
    489
    490 -    def locations(self):
    491          """Get the sensor locations as an array.
    492
    493          :Returns:
    494            (nchannels x 3) array with coordinates in (x, y, z)
    495          """
    496          return N.array((self.pos_x, self.pos_y, self.pos_z)).T
    497
    498
    499
    500 -class XAVRSensorLocations(SensorLocations):
    501      """Read sensor location definitions from a specific text file format.
    502
    503      File layout is assumed to be 5 columns:
    504
    505        1. sensor name
    506        2. some useless integer
    507        3. position on x-axis
    508        4. position on y-axis
    509        5. position on z-axis
    510      """
    511 -    def __init__(self, source):
    512          """Read sensor locations from file.
    513
    514          :Parameter:
    515            source : filename of an attribute file
    516          """
    517          SensorLocations.__init__(
    518              self, source,
    519              header=['names', 'some_number', 'pos_x', 'pos_y', 'pos_z'],
    520              sep=None, dtype=[str, int, float, float, float])
    521
    522
    523 -class TuebingenMEGSensorLocations(SensorLocations):
    524      """Read sensor location definitions from a specific text file format.
    525
    526      File layout is assumed to be 7 columns:
    527
    528        1:   sensor name
    529        2:   position on y-axis
    530        3:   position on x-axis
    531        4:   position on z-axis
    532        5-7: same as 2-4, but for some outer surface thingie.
    533
    534      Note that x and y seem to be swapped, ie. y as defined by SensorLocations
    535      conventions seems to be first axis and followed by x.
    536
    537      Only inner surface coordinates are reported by `locations()`.
    538      """
    539 -    def __init__(self, source):
    540          """Read sensor locations from file.
    541
    542          :Parameter:
    543            source : filename of an attribute file
    544          """
    545          SensorLocations.__init__(
    546              self, source,
    547              header=['names', 'pos_y', 'pos_x', 'pos_z',
    548                      'pos_y2', 'pos_x2', 'pos_z2'],
    549              sep=None, dtype=[str, float, float, float, float, float, float])
    550
    551
    552 -def design2labels(columndata, baseline_label=0,
    553                    func=lambda x: x > 0.0):
    554      """Helper to convert design matrix into a list of labels
    555
    556      Given a design, assign a single label to any given sample
    557
    558      TODO: fix description/naming
    559
    560      :Parameters:
    561        columndata : ColumnData
    562          Attributes where each known will be considered as a separate
    563          explanatory variable (EV) in the design.
    564        baseline_label
    565          What label to assign for samples where none of EVs was given a value
    566        func : functor
    567          Function which decides either a value should be considered
    568
    569      :Output:
    570        list of labels which are taken from column names in
    571        ColumnData and baseline_label
    572
    573      """
    574      # doing it simple naive way but it should be of better control if
    575      # we decide to process columndata with non-numeric entries etc
    576      keys = columndata.keys()
    577      labels = []
    578      for row in xrange(columndata.nrows):
    579          entries = [ columndata[key][row] for key in keys ]
    580          # which entries get selected
    581          selected = filter(lambda x: func(x[1]), zip(keys, entries))
    582          nselected = len(selected)
    583
    584          if nselected > 1:
    585              # if there is more than a single one -- we are in problem
    586              raise ValueError, "Row #%i with items %s has multiple entries " \
    587                    "meeting the criterion. Cannot decide on the label" % \
    588                    (row, entries)
    589          elif nselected == 1:
    590              label = selected[0][0]
    591          else:
    592              label = baseline_label
    593          labels.append(label)
    594      return labels
    595
    596
    597  __known_chunking_methods = {
    598      'alllabels': 'Each chunk must contain instances of all labels'
    599      }
    600
    601 -def labels2chunks(labels, method="alllabels", ignore_labels=None):
    602      """Automagically decide on chunks based on labels
    603
    604      :Parameters:
    605        labels
    606          labels to base chunking on
    607        method : basestring
    608          codename for method to use. Known are %s
    609        ignore_labels : list of basestring
    610          depends on the method. If method ``alllabels``, then don't
    611          seek for such labels in chunks. E.g. some 'reject' samples
    612
    613      :rtype: list
    614      """ % __known_chunking_methods.keys()
    615
    616      chunks = []
    617      if ignore_labels is None:
    618          ignore_labels = []
    619      alllabels = set(labels).difference(set(ignore_labels))
    620      if method == 'alllabels':
    621          seenlabels = set()
    622          lastlabel = None
    623          chunk = 0
    624          for label in labels:
    625              if label != lastlabel:
    626                  if seenlabels == alllabels:
    627                      chunk += 1
    628                      seenlabels = set()
    629                  lastlabel = label
    630                  if not label in ignore_labels:
    631                      seenlabels.update([label])
    632              chunks.append(chunk)
    633          chunks = N.array(chunks)
    634          # fix up a bit the trailer
    635          if seenlabels != alllabels:
    636              chunks[chunks == chunk] = chunk-1
    637          chunks = list(chunks)
    638      else:
    639          errmsg = "Unknown method to derive chunks is requested. Known are:\n"
    640          for method, descr in __known_chunking_methods.iteritems():
    641              errmsg += "  %s : %s\n" % (method, descr)
    642          raise ValueError, errmsg
    643      return chunks
    644