Package logilab :: Package common :: Module proc

[frames] | no frames]

# Source Code for Module logilab.common.proc

      1  # copyright 2003-2011 LOGILAB S.A. (Paris, FRANCE), all rights reserved.
      2  # contact http://www.logilab.fr/ -- mailto:contact@logilab.fr
      3  #
      4  # This file is part of logilab-common.
      5  #
      6  # logilab-common is free software: you can redistribute it and/or modify it under
      7  # the terms of the GNU Lesser General Public License as published by the Free
      8  # Software Foundation, either version 2.1 of the License, or (at your option) any
      9  # later version.
     10  #
     11  # logilab-common is distributed in the hope that it will be useful, but WITHOUT
     12  # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS
     13  # FOR A PARTICULAR PURPOSE.  See the GNU Lesser General Public License for more
     14  # details.
     15  #
     16  # You should have received a copy of the GNU Lesser General Public License along
     17  # with logilab-common.  If not, see <http://www.gnu.org/licenses/>.
     18  """module providing:
     19  * process information (linux specific: rely on /proc)
     20  * a class for resource control (memory / time / cpu time)
     21
     22  This module doesn't work on windows platforms (only tested on linux)
     23
     24  :organization: Logilab
     25
     26
     27
     28  """
     29  __docformat__ = "restructuredtext en"
     30
     31  import os
     32  import stat
     33  from resource import getrlimit, setrlimit, RLIMIT_CPU, RLIMIT_AS
     34  from signal import signal, SIGXCPU, SIGKILL, SIGUSR2, SIGUSR1
     35  from threading import Timer, currentThread, Thread, Event
     36  from time import time
     37
     38  from logilab.common.tree import Node
     39
     40 -class NoSuchProcess(Exception): pass
     41
     42 -def proc_exists(pid):
     43      """check the a pid is registered in /proc
     44      raise NoSuchProcess exception if not
     45      """
     46      if not os.path.exists('/proc/%s' % pid):
     47          raise NoSuchProcess()
     48
     49  PPID = 3
     50  UTIME = 13
     51  STIME = 14
     52  CUTIME = 15
     53  CSTIME = 16
     54  VSIZE = 22
     55
     56 -class ProcInfo(Node):
     57      """provide access to process information found in /proc"""
     58
     59 -    def __init__(self, pid):
     60          self.pid = int(pid)
     61          Node.__init__(self, self.pid)
     62          proc_exists(self.pid)
     63          self.file = '/proc/%s/stat' % self.pid
     64          self.ppid = int(self.status()[PPID])
     65
     66 -    def memory_usage(self):
     67          """return the memory usage of the process in Ko"""
     68          try :
     69              return int(self.status()[VSIZE])
     70          except IOError:
     71              return 0
     72
     73 -    def lineage_memory_usage(self):
     74          return self.memory_usage() + sum([child.lineage_memory_usage()
     75                                            for child in self.children])
     76
     77 -    def time(self, children=0):
     78          """return the number of jiffies that this process has been scheduled
     79          in user and kernel mode"""
     80          status = self.status()
     81          time = int(status[UTIME]) + int(status[STIME])
     82          if children:
     83              time += int(status[CUTIME]) + int(status[CSTIME])
     84          return time
     85
     86 -    def status(self):
     87          """return the list of fields found in /proc/<pid>/stat"""
     88          return open(self.file).read().split()
     89
     90 -    def name(self):
     91          """return the process name found in /proc/<pid>/stat
     92          """
     93          return self.status()[1].strip('()')
     94
     95 -    def age(self):
     96          """return the age of the process
     97          """
     98          return os.stat(self.file)[stat.ST_MTIME]
     99
    100 -class ProcInfoLoader:
    101      """manage process information"""
    102
    103 -    def __init__(self):
    104          self._loaded = {}
    105
    106 -    def list_pids(self):
    107          """return a list of existent process ids"""
    108          for subdir in os.listdir('/proc'):
    109              if subdir.isdigit():
    110                  yield int(subdir)
    111
    112 -    def load(self, pid):
    113          """get a ProcInfo object for a given pid"""
    114          pid = int(pid)
    115          try:
    116              return self._loaded[pid]
    117          except KeyError:
    118              procinfo = ProcInfo(pid)
    119              procinfo.manager = self
    120              self._loaded[pid] = procinfo
    121              return procinfo
    122
    123
    124 -    def load_all(self):
    125          """load all processes information"""
    126          for pid in self.list_pids():
    127              try:
    128                  procinfo = self.load(pid)
    129                  if procinfo.parent is None and procinfo.ppid:
    130                      pprocinfo = self.load(procinfo.ppid)
    131                      pprocinfo.append(procinfo)
    132              except NoSuchProcess:
    133                  pass
    134
    135
    136  try:
    137 -    class ResourceError(BaseException):
    138          """Error raise when resource limit is reached"""
    139          limit = "Unknown Resource Limit"
    140  except NameError:
    141 -    class ResourceError(Exception):
    142          """Error raise when resource limit is reached"""
    143          limit = "Unknown Resource Limit"
    144
    145
    146 -class XCPUError(ResourceError):
    147      """Error raised when CPU Time limit is reached"""
    148      limit = "CPU Time"
    149
    150 -class LineageMemoryError(ResourceError):
    151      """Error raised when the total amount of memory used by a process and
    152      it's child is reached"""
    153      limit = "Lineage total Memory"
    154
    155 -class TimeoutError(ResourceError):
    156      """Error raised when the process is running for to much time"""
    157      limit = "Real Time"
    158
    159  # Can't use subclass because the StandardError MemoryError raised
    160  RESOURCE_LIMIT_EXCEPTION = (ResourceError, MemoryError)
    161
    162
    163 -class MemorySentinel(Thread):
    164      """A class checking a process don't use too much memory in a separated
    165      daemonic thread
    166      """
    167 -    def __init__(self, interval, memory_limit, gpid=os.getpid()):
    168          Thread.__init__(self, target=self._run, name="Test.Sentinel")
    169          self.memory_limit = memory_limit
    170          self._stop = Event()
    171          self.interval = interval
    172          self.setDaemon(True)
    173          self.gpid = gpid
    174
    175 -    def stop(self):
    176          """stop ap"""
    177          self._stop.set()
    178
    179 -    def _run(self):
    180          pil = ProcInfoLoader()
    181          while not self._stop.isSet():
    182              if self.memory_limit <= pil.load(self.gpid).lineage_memory_usage():
    183                  os.killpg(self.gpid, SIGUSR1)
    184              self._stop.wait(self.interval)
    185
    186
    187 -class ResourceController:
    188
    189 -    def __init__(self, max_cpu_time=None, max_time=None, max_memory=None,
    190                   max_reprieve=60):
    191          if SIGXCPU == -1:
    192              raise RuntimeError("Unsupported platform")
    193          self.max_time = max_time
    194          self.max_memory = max_memory
    195          self.max_cpu_time = max_cpu_time
    196          self._reprieve = max_reprieve
    197          self._timer = None
    198          self._msentinel = None
    199          self._old_max_memory = None
    200          self._old_usr1_hdlr = None
    201          self._old_max_cpu_time = None
    202          self._old_usr2_hdlr = None
    203          self._old_sigxcpu_hdlr = None
    204          self._limit_set = 0
    205          self._abort_try = 0
    206          self._start_time = None
    207          self._elapse_time = 0
    208
    209 -    def _hangle_sig_timeout(self, sig, frame):
    210          raise TimeoutError()
    211
    212 -    def _hangle_sig_memory(self, sig, frame):
    213          if self._abort_try < self._reprieve:
    214              self._abort_try += 1
    215              raise LineageMemoryError("Memory limit reached")
    216          else:
    217              os.killpg(os.getpid(), SIGKILL)
    218
    219 -    def _handle_sigxcpu(self, sig, frame):
    220          if self._abort_try < self._reprieve:
    221              self._abort_try += 1
    222              raise XCPUError("Soft CPU time limit reached")
    223          else:
    224              os.killpg(os.getpid(), SIGKILL)
    225
    226 -    def _time_out(self):
    227          if self._abort_try < self._reprieve:
    228              self._abort_try += 1
    229              os.killpg(os.getpid(), SIGUSR2)
    230              if self._limit_set > 0:
    231                  self._timer = Timer(1, self._time_out)
    232                  self._timer.start()
    233          else:
    234              os.killpg(os.getpid(), SIGKILL)
    235
    236 -    def setup_limit(self):
    237          """set up the process limit"""
    238          assert currentThread().getName() == 'MainThread'
    239          os.setpgrp()
    240          if self._limit_set <= 0:
    241              if self.max_time is not None:
    242                  self._old_usr2_hdlr = signal(SIGUSR2, self._hangle_sig_timeout)
    243                  self._timer = Timer(max(1, int(self.max_time) - self._elapse_time),
    244                                      self._time_out)
    245                  self._start_time = int(time())
    246                  self._timer.start()
    247              if self.max_cpu_time is not None:
    248                  self._old_max_cpu_time = getrlimit(RLIMIT_CPU)
    249                  cpu_limit = (int(self.max_cpu_time), self._old_max_cpu_time[1])
    250                  self._old_sigxcpu_hdlr = signal(SIGXCPU, self._handle_sigxcpu)
    251                  setrlimit(RLIMIT_CPU, cpu_limit)
    252              if self.max_memory is not None:
    253                  self._msentinel = MemorySentinel(1, int(self.max_memory) )
    254                  self._old_max_memory = getrlimit(RLIMIT_AS)
    255                  self._old_usr1_hdlr = signal(SIGUSR1, self._hangle_sig_memory)
    256                  as_limit = (int(self.max_memory), self._old_max_memory[1])
    257                  setrlimit(RLIMIT_AS, as_limit)
    258                  self._msentinel.start()
    259          self._limit_set += 1
    260
    261 -    def clean_limit(self):
    262          """reinstall the old process limit"""
    263          if self._limit_set > 0:
    264              if self.max_time is not None:
    265                  self._timer.cancel()
    266                  self._elapse_time += int(time())-self._start_time
    267                  self._timer = None
    268                  signal(SIGUSR2, self._old_usr2_hdlr)
    269              if self.max_cpu_time is not None:
    270                  setrlimit(RLIMIT_CPU, self._old_max_cpu_time)
    271                  signal(SIGXCPU, self._old_sigxcpu_hdlr)
    272              if self.max_memory is not None:
    273                  self._msentinel.stop()
    274                  self._msentinel = None
    275                  setrlimit(RLIMIT_AS, self._old_max_memory)
    276                  signal(SIGUSR1, self._old_usr1_hdlr)
    277          self._limit_set -= 1
    278