# pathos.helpers module documentation

## mp_helper module

map helper functions

random_seed(s=None)

  sets the seed for calls to ‘random()’

random_state(module='random', new=False, seed='!')

  return a (optionally manually seeded) random generator

  For a given module, return an object that has random number generation (RNG) methods available. If new=False, use the global copy of the RNG object. If seed=’!’, do not reseed the RNG (using seed=None ‘removes’ any seeding). If seed=’*’, use a seed that depends on the process id (PID); this is useful for building RNGs that are different across multiple threads or processes.

starargs(f)

  decorator to convert a many-arg function to a single-arg function

## pp_helper module

class ApplyResult(task)

  Bases: `_Task`

  result object for an ‘apply’ method in parallelpython

  enables a pp._Task to mimic the multiprocessing.pool.ApplyResult interface

  Initializes the task

  __annotate_func__ = None
  __annotations_cache__ = {}
  __call__(raw_result=False)

    Retrieves result of the task

  __firstlineno__ = 25
  __static_attributes__ = ('_success', '_task', 'finished', 'result', 'unpickled')
  __unpickle()

    Unpickles the result of the task

  property callback
  property callbackargs
  finalize(sresult)

    Finalizes the task *internal use only*

  property finished
  get(timeout=None)

    Retrieves result of the task

  property group
  property lock
  ready()

    Checks if the result is ready

  property server
  successful()

    Measures whether result is ready and loaded w/o printing

  property tid
  wait(timeout=None)

    Waits for the task

class MapResult(size, callback=None, callbackargs=(), group='default')

  Bases: `object`

  __annotate_func__ = None
  __annotations_cache__ = {}
  __call__()

    Retrieve the results of the tasks

  __firstlineno__ = 124
  __static_attributes__ = ('__queue', '__tasks', '_chunksize', '_number_left', '_success', '_value', 'callback', 'callbackargs', 'finished', 'group', 'unpickled')
  __unpickle()

    Unpickles the results of the tasks

  _set(i, task)
  finalize(*results)

    finalize the tasks *internal use only*

  get(timeout=None)

    Retrieves results of the tasks

  queue(*tasks)

    Fill the MapResult with ApplyResult objects

  ready()

    Checks if the result is ready

  successful()

    Measures whether result is ready and loaded w/o printing

  wait(timeout=None)

    Wait for the tasks

class Server(ncpus='autodetect', ppservers=(), secret=None, restart=False, proto=2, socket_timeout=3600)

  Bases: `object`

  Parallel Python SMP execution server class

  Creates Server instance

  ncpus - the number of worker processes to start on the local computer, if parameter is omitted it will be set to the number of processors in the system ppservers - list of active parallel python execution servers to connect with secret - passphrase for network connections, if omitted a default passphrase will be used. It’s highly recommended to use a custom passphrase for all network connections. restart - restart the worker process after each task completion proto - protocol number for pickle module socket_timeout - socket timeout in seconds, which is the maximum time a remote job could be executed. Increase this value if you have long running jobs or decrease if connectivity to remote ppservers is often lost.

  With ncpus = 1 all tasks are executed consequently. For the best performance either use the default “autodetect” value or set ncpus to the total number of processors in the system.

  __add_to_active_tasks(num)

    Updates the number of active tasks

  __annotate_func__ = None
  __annotations_cache__ = {}
  __connect()

    Connects to all remote ppservers

  __del__()
  __detect_ncpus()

    Detects the number of effective CPUs in the system

  __dumpsfunc(funcs, modules)

    Serializes functions and modules

  __find_modules(prefix, dict)

    recursively finds all the modules in dict

  __firstlineno__ = 282
  __gentid()

    Generates a unique job ID number

  __get_source(func)

    Fetches source of the function

  __scheduler()

    Schedules jobs for execution

  __stat_add_job(node)

    Increments job count on the node

  __stat_add_time(node, time_add)

    Updates total runtime on the node

  __static_attributes__ = ('__accurate_stats', '__active_rworkers_list_lock', '__active_tasks', '__active_tasks_lock', '__connect_locks', '__creation_time', '__ncpus', '__pickle_proto', '__queue', '__queue_lock', '__restart_on_free', '__rworkers', '__rworkers_reserved', '__sfuncHM', '__sourcesHM', '__stats', '__stats_lock', '__tid', '__waittasks', '__waittasks_lock', '__workers', '_exiting', 'auto_ppservers', 'autopp_list', 'discover', 'logger', 'ppservers', 'secret', 'socket_timeout')
  __update_active_rworkers(id, count)

    Updates list of active rworkers

  _run_local(job, sfunc, sargs, worker)

    Runs a job locally

  _run_remote(job, sfunc, sargs, rworker)

    Runs a job remotelly

  connect1(host, port, persistent=True)

    Conects to a remote ppserver specified by host and port

  default_port = 60000
  default_secret = 'epo20pdosl;dksldkmm'
  destroy()

    Kills ppworkers and closes open files

  get_active_nodes()

    Returns active nodes as a dictionary [keys - nodes, values - ncpus]

  get_ncpus()

    Returns the number of local worker processes (ppworkers)

  get_stats()

    Returns job execution statistics as a dictionary

  insert(sfunc, sargs, task=None)

    Inserts function into the execution queue. It’s intended for internal use only (in ppserver).

  print_stats()

    Prints job execution statistics. Useful for benchmarking on clusters

  set_ncpus(ncpus='autodetect')

    Sets the number of local worker processes (ppworkers)

    ncpus - the number of worker processes, if parammeter is omitted

      it will be set to the number of processors in the system

  submit(func, args=(), depfuncs=(), modules=(), callback=None, callbackargs=(), group='default', globals=None)

    Submits function to the execution queue

    func - function to be executed args - tuple with arguments of the ‘func’ depfuncs - tuple with functions which might be called from ‘func’ modules - tuple with module names to import callback - function which will be called with argument list equal to callbackargs+(result,) as soon as calculation is done callbackargs - additional arguments for callback function group - job group, is used when wait(group) is called to wait for jobs in a given group to finish globals - dictionary from which all modules, functions and classes will be imported, for instance: globals=globals()

  wait(group=None)

    Waits for all jobs in a given group to finish. If group is omitted waits for all jobs to finish

exception TimeoutError

  Bases: `ProcessError`

  __firstlineno__ = 20
  __static_attributes__ = ()

_ApplyResult

  alias of `ApplyResult`

_MapResult

  alias of `MapResult`

class _Task(server, tid, callback=None, callbackargs=(), group='default')

  Bases: `object`

  Class describing single task (job)

  Initializes the task

  _Task__unpickle()

    Unpickles the result of the task

  __annotate_func__ = None
  __annotations_cache__ = {}
  __call__(raw_result=False)

    Retrieves result of the task

  __firstlineno__ = 73
  __static_attributes__ = ('callback', 'callbackargs', 'finished', 'group', 'lock', 'result', 'server', 'sresult', 'tid', 'unpickled')
  finalize(sresult)

    Finalizes the task.

    For internal use only

  wait()

    Waits for the task