DAX3 (version 3.3) | index /build/pegasus-wms-aFKmji/pegasus-wms-4.4.0+DAX3.py

`API for generating Pegasus DAXes
The classes in this module can be used to generate DAXes that can be
read by Pegasus.
The official DAX schema is here: http://pegasus.isi.edu/schema/dax-3.2.xsd
Here is an example showing how to create the diamond DAX using this API:
# Create ADAG object
diamond = ADAG("diamond")
# Add input file to the DAX-level replica catalog
a = File("f.a")
a.addPFN(PFN("gsiftp://site.com/inputs/f.a","site"))
diamond.addFile(a)
# Add executables to the DAX-level replica catalog
e_preprocess = Executable(namespace="diamond", name="preprocess", version="4.0", os="linux", arch="x86_64")
e_preprocess.addPFN(PFN("gsiftp://site.com/bin/preprocess","site"))
diamond.addExecutable(e_preprocess)
e_findrange = Executable(namespace="diamond", name="findrange", version="4.0", os="linux", arch="x86_64")
e_findrange.addPFN(PFN("gsiftp://site.com/bin/findrange","site"))
diamond.addExecutable(e_findrange)
e_analyze = Executable(namespace="diamond", name="analyze", version="4.0", os="linux", arch="x86_64")
e_analyze.addPFN(PFN("gsiftp://site.com/bin/analyze","site"))
diamond.addExecutable(e_analyze)
# Add a preprocess job
preprocess = Job(e_preprocess)
b1 = File("f.b1")
b2 = File("f.b2")
preprocess.addArguments("-a preprocess","-T60","-i",a,"-o",b1,b2)
preprocess.uses(a, link=Link.INPUT)
preprocess.uses(b1, link=Link.OUTPUT, transfer=True)
preprocess.uses(b2, link=Link.OUTPUT, transfer=True)
diamond.addJob(preprocess)
# Add left Findrange job
frl = Job(e_findrange)
c1 = File("f.c1")
frl.addArguments("-a findrange","-T60","-i",b1,"-o",c1)
frl.uses(b1, link=Link.INPUT)
frl.uses(c1, link=Link.OUTPUT, transfer=True)
diamond.addJob(frl)
# Add right Findrange job
frr = Job(e_findrange)
c2 = File("f.c2")
frr.addArguments("-a findrange","-T60","-i",b2,"-o",c2)
frr.uses(b2, link=Link.INPUT)
frr.uses(c2, link=Link.OUTPUT, transfer=True)
diamond.addJob(frr)
# Add Analyze job
analyze = Job(e_analyze)
d = File("f.d")
analyze.addArguments("-a analyze","-T60","-i",c1,c2,"-o",d)
analyze.uses(c1, link=Link.INPUT)
analyze.uses(c2, link=Link.INPUT)
analyze.uses(d, link=Link.OUTPUT, transfer=True, register=True)
diamond.addJob(analyze)
# Add control-flow dependencies
diamond.depends(parent=preprocess, child=frl)
diamond.depends(parent=preprocess, child=frr)
diamond.depends(parent=frl, child=analyze)
diamond.depends(parent=frr, child=analyze)
# Write the DAX to stdout
import sys
diamond.writeXML(sys.stdout)
# Write the DAX to a file
f = open("diamond.dax","w")
diamond.writeXML(f)
f.close()`

Modules

``

codecs datetime | os shlex | sys |

Classes

``

Arch
Dependency
Element
Invoke
InvokeMixin
  ADAG
Link
Metadata
MetadataMixin
Namespace
OS
PFNMixin
Profile
ProfileMixin
  AbstractJob(ProfileMixin, UseMixin, InvokeMixin)
    DAG
    DAX
    Job
  CatalogType(ProfileMixin, MetadataMixin, PFNMixin)
    Executable(CatalogType, InvokeMixin)
    File
  PFN
Transfer
Use
UseMixin
  Transformation(UseMixin, InvokeMixin)
When
exceptions.Exception(exceptions.BaseException)
  DAX3Error
    DuplicateError
    FormatError
    NotFoundError
    ParseError

class ADAG(InvokeMixin)

``

`ADAG(name[,count][,index])
Representation of a directed acyclic graph in XML (DAX).
Examples:
dax = ADAG('diamond')
or, if you want to use the old style count/index partitioning stuff:
part5 = ADAG('partition_5',count=10,index=5)
Adding jobs:
a = Job(...)
dax.addJob(a)
Adding parent-child control-flow dependency:
dax.addDependency(Dependency(parent=a,child=b))
dax.addDependency(Dependency(parent=a,child=c))
dax.addDependency(Dependency(parent=b,child=d))
dax.addDependency(Dependency(parent=c,child=d))
or:
dax.depends(child=b, parent=a)
Adding Files (not required if you have a replica catalog):
input = File(...)
dax.addFile(input)
Adding Executables (not required if you have a transformation catalog):
exe = Executable(...)
dax.addExecutable(exe)
Adding Transformations (not required if you have a transformation catalog):
xform = Transformation(...)
dax.addTransformation(xform)
Writing a DAX out to a file:
f = open('diamond.dax','w')
dax.writeXML(f)
f.close()
`

Methods defined here:

__init__(self, name, count=None, index=None)
  `Arguments:
  name: The name of the workflow
  count: Total number of DAXes that will be created
  index: Zero-based index of this DAX`

__str__(self)

__unicode__(self)

addDAG(self, dag)
  `Add a sub-DAG (synonym for addJob)`

addDAX(self, dax)
  `Add a sub-DAX (synonym for addJob)`

addDependency(self, dep)
  `Add a dependency to the workflow
  The old way to call this method is no longer valid. Please change:
  adag.addDependency(parent="ID01", child="ID02", edge_label="E01")
  to be:
  adag.addDependency(Dependency(parent="ID01", child="ID02", edge_label="E01"))
  or:
  adag.depends(parent="ID01", child="ID02", edge_label="E01")`

addExecutable(self, executable)
  `Add an executable to this ADAG`

addFile(self, file)
  `Add a file to the DAX`

addJob(self, job)
  `Add a job to this ADAG`

addTransformation(self, transformation)
  `Add a transformation to this ADAG`

clearDependencies(self)
  `Remove all dependencies`

clearExecutables(self)
  `Remove all executables`

clearFiles(self)
  `Remove all files`

clearJobs(self)
  `Remove all jobs`

clearTransformations(self)
  `Remove all transformations`

depends(self, child, parent, edge_label=None)
  `Add a dependency to the workflow
  Arguments:
  child: The child job/dax/dag or id
  parent: The parent job/dax/dag or id
  edge_label: A label for the edge (optional)`

getJob(self, jobid)
  `Get a Job/DAG/DAX`

hasDependency(self, dep)
  `Check to see if dependency exists`

hasExecutable(self, executable)
  `Check if executable is in this ADAG`

hasFile(self, file)
  `Check to see if file is in this ADAG`

hasJob(self, job)
  `Test to see if job is in this ADAG
  The job parameter can be an object or a job ID`

hasTransformation(self, transformation)
  `Check to see if transformation is in this ADAG`

nextJobID(self)
  `Get an autogenerated ID for the next job`

removeDependency(self, dep)
  `Remove dependency from workflow`

removeExecutable(self, executable)
  `Remove executable from this ADAG`

removeFile(self, file)
  `Remove file from this ADAG`

removeJob(self, job)
  `Remove job from this ADAG`

removeTransformation(self, transformation)
  `Remove transformation from this ADAG`

toXML(self)
  `Get the XML string for this ADAG
  This is primarily intended for testing. If you have a large ADAG
  you should use writeXML instead.`

writeXML(self, out)
  `Write the ADAG as XML to a stream`

writeXMLFile(self, filename)
  `Write the ADAG to an XML file`

Methods inherited from InvokeMixin:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class AbstractJob(ProfileMixin, UseMixin, InvokeMixin)
`` | `The base class for Job, DAX, and DAG`

Method resolution order:
  AbstractJob
  ProfileMixin
  UseMixin
  InvokeMixin

Methods defined here:

__init__(self, id=None, node_label=None)

addArguments(self, *arguments)
  `Add one or more arguments to the job (this will add whitespace)`

addRawArguments(self, *arguments)
  `Add one or more arguments to the job (whitespace will NOT be added)`

clearArguments(self)
  `Remove all arguments from this job`

clearStderr(self)
  `Remove stderr file`

clearStdin(self)
  `Remove stdin file`

clearStdout(self)
  `Remove stdout file`

getArguments(self)
  `Get the arguments of this job`

innerXML(self, element)
  `Return an XML representation of this job`

setStderr(self, filename)
  `Redirect stderr to a file`

setStdin(self, filename)
  `Redirect stdin from a file`

setStdout(self, filename)
  `Redirect stdout to a file`

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

Methods inherited from UseMixin:

addUse(self, use)
  `Add Use to this object`

clearUses(self)
  `Remove all uses from this object`

hasUse(self, use)
  `Test to see if this object has use`

removeUse(self, use)
  `Remove use from this object`

uses(self, arg, link=None, register=None, transfer=None, optional=None, namespace=None, version=None, executable=None, size=None)

Methods inherited from InvokeMixin:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class Arch
`` | `Architecture types. See Executable.`

Data and other attributes defined here:

AMD64 = 'amd64'

IA64 = 'ia64'

PPC = 'ppc'

PPC_64 = 'ppc_64'

SPARCV7 = 'sparcv7'

SPARCV9 = 'sparcv9'

X86 = 'x86'

X86_64 = 'x86_64'

class CatalogType(ProfileMixin, MetadataMixin, PFNMixin)
`` | `Base class for File and Executable`

Method resolution order:
  CatalogType
  ProfileMixin
  MetadataMixin
  PFNMixin

Methods defined here:

__init__(self, name)
  `All arguments specify the workflow-level behavior of this File. Job-level
  behavior can be defined when adding the File to a Job's uses. If the
  properties are not overridden at the job-level, then the workflow-level
  values are used as defaults.
  If this LFN is to be used as a job's stdin/stdout/stderr then the value
  of link is ignored when generating the <std*> tags.
  Arguments:
  name: The name of the file (required)`

innerXML(self, parent)

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

Methods inherited from MetadataMixin:

addMetadata(self, metadata)
  `Add metadata to this object`

clearMetadata(self)
  `Remove all metadata from this object`

hasMetadata(self, metadata)
  `Does this object have metadata?`

metadata(self, key, type, value)
  `Declarative metadata addition`

removeMetadata(self, metadata)
  `Remove meta from this object`

Methods inherited from PFNMixin:

PFN(self, url, site=None)
  `Declarative PFN addition`

addPFN(self, pfn)
  `Add a PFN to this object`

clearPFNs(self)
  `Remove all PFNs from this object`

hasPFN(self, pfn)
  `Does this object have pfn?`

removePFN(self, pfn)
  `Remove PFN from this object`

class DAG(AbstractJob)
`` | `DAG(file[,id][,node_label]) This job represents a sub-DAG that will be executed by this workflow. Examples: dagjob1 = DAG(file="foo.dag") dagfile = File("foo.dag") dagjob2 = DAG(dagfile)`

Method resolution order:
  DAG
  AbstractJob
  ProfileMixin
  UseMixin
  InvokeMixin

Methods defined here:

__init__(self, file, id=None, node_label=None)
  `The name argument can be either a string, or a File object. If
  it is a File object, then this job will inherit its name from the
  File and the File will be added in a <uses> with transfer=True,
  register=False, and link=input.
  Arguments:
  file: The logical name of the DAG file, or the DAG File object
  id: The ID of the DAG job [default: autogenerated]
  node_label: The label for this job to use in graphing`

__str__(self)

__unicode__(self)

toXML(self)
  `Return an XML representation of this DAG`

Methods inherited from AbstractJob:

addArguments(self, *arguments)
  `Add one or more arguments to the job (this will add whitespace)`

addRawArguments(self, *arguments)
  `Add one or more arguments to the job (whitespace will NOT be added)`

clearArguments(self)
  `Remove all arguments from this job`

clearStderr(self)
  `Remove stderr file`

clearStdin(self)
  `Remove stdin file`

clearStdout(self)
  `Remove stdout file`

getArguments(self)
  `Get the arguments of this job`

innerXML(self, element)
  `Return an XML representation of this job`

setStderr(self, filename)
  `Redirect stderr to a file`

setStdin(self, filename)
  `Redirect stdin from a file`

setStdout(self, filename)
  `Redirect stdout to a file`

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

Methods inherited from UseMixin:

addUse(self, use)
  `Add Use to this object`

clearUses(self)
  `Remove all uses from this object`

hasUse(self, use)
  `Test to see if this object has use`

removeUse(self, use)
  `Remove use from this object`

uses(self, arg, link=None, register=None, transfer=None, optional=None, namespace=None, version=None, executable=None, size=None)

Methods inherited from InvokeMixin:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class DAX(AbstractJob)
`` | `DAX(file[,id][,node_label]) This job represents a sub-DAX that will be planned and executed by the workflow. Examples: daxjob1 = DAX("foo.dax") daxfile = File("foo.dax") daxjob2 = DAX(daxfile)`

Method resolution order:
  DAX
  AbstractJob
  ProfileMixin
  UseMixin
  InvokeMixin

Methods defined here:

__init__(self, file, id=None, node_label=None)
  `The name argument can be either a string, or a File object. If
  it is a File object, then this job will inherit its name from the
  File and the File will be added in a <uses> with transfer=True,
  register=False, and link=input.
  Arguments:
  file: The logical name of the DAX file or the DAX File object
  id: The id of the DAX job [default: autogenerated]
  node_label: The label for this job to use in graphing`

__str__(self)

__unicode__(self)

toXML(self)
  `Return an XML representation of this job`

Methods inherited from AbstractJob:

addArguments(self, *arguments)
  `Add one or more arguments to the job (this will add whitespace)`

addRawArguments(self, *arguments)
  `Add one or more arguments to the job (whitespace will NOT be added)`

clearArguments(self)
  `Remove all arguments from this job`

clearStderr(self)
  `Remove stderr file`

clearStdin(self)
  `Remove stdin file`

clearStdout(self)
  `Remove stdout file`

getArguments(self)
  `Get the arguments of this job`

innerXML(self, element)
  `Return an XML representation of this job`

setStderr(self, filename)
  `Redirect stderr to a file`

setStdin(self, filename)
  `Redirect stdin from a file`

setStdout(self, filename)
  `Redirect stdout to a file`

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

Methods inherited from UseMixin:

addUse(self, use)
  `Add Use to this object`

clearUses(self)
  `Remove all uses from this object`

hasUse(self, use)
  `Test to see if this object has use`

removeUse(self, use)
  `Remove use from this object`

uses(self, arg, link=None, register=None, transfer=None, optional=None, namespace=None, version=None, executable=None, size=None)

Methods inherited from InvokeMixin:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class DAX3Error(exceptions.Exception)

``

Method resolution order:
  DAX3Error
  exceptions.Exception
  exceptions.BaseException
  __builtin__.object

Data descriptors defined here:

__weakref__
  `list of weak references to the object (if defined)`

Methods inherited from exceptions.Exception:

__init__(...)
  `x.__init__(...) initializes x; see help(type(x)) for signature`

Data and other attributes inherited from exceptions.Exception:

__new__ = <built-in method __new__ of type object>
  `T.__new__(S, ...) -> a new object with type S, a subtype of T`

Methods inherited from exceptions.BaseException:

__delattr__(...)
  `x.__delattr__('name') <==> del x.name`

__getattribute__(...)
  `x.__getattribute__('name') <==> x.name`

__getitem__(...)
  `x.__getitem__(y) <==> x[y]`

__getslice__(...)
  `x.__getslice__(i, j) <==> x[i:j]
  Use of negative indices is not supported.`

__reduce__(...)

__repr__(...)
  `x.__repr__() <==> repr(x)`

__setattr__(...)
  `x.__setattr__('name', value) <==> x.name = value`

__setstate__(...)

__str__(...)
  `x.__str__() <==> str(x)`

__unicode__(...)

Data descriptors inherited from exceptions.BaseException:

__dict__

args

message

class Dependency
`` | `A dependency between two nodes in the ADAG`

Methods defined here:

__eq__(self, other)
  `Equal dependencies have the same parent and child`

__hash__(self)

__init__(self, parent, child, edge_label=None)

__str__(self)

__unicode__(self)

class DuplicateError(DAX3Error)

``

Method resolution order:
  DuplicateError
  DAX3Error
  exceptions.Exception
  exceptions.BaseException
  __builtin__.object

Data descriptors inherited from DAX3Error:

__weakref__
  `list of weak references to the object (if defined)`

Methods inherited from exceptions.Exception:

__init__(...)
  `x.__init__(...) initializes x; see help(type(x)) for signature`

Data and other attributes inherited from exceptions.Exception:

__new__ = <built-in method __new__ of type object>
  `T.__new__(S, ...) -> a new object with type S, a subtype of T`

Methods inherited from exceptions.BaseException:

__delattr__(...)
  `x.__delattr__('name') <==> del x.name`

__getattribute__(...)
  `x.__getattribute__('name') <==> x.name`

__getitem__(...)
  `x.__getitem__(y) <==> x[y]`

__getslice__(...)
  `x.__getslice__(i, j) <==> x[i:j]
  Use of negative indices is not supported.`

__reduce__(...)

__repr__(...)
  `x.__repr__() <==> repr(x)`

__setattr__(...)
  `x.__setattr__('name', value) <==> x.name = value`

__setstate__(...)

__str__(...)
  `x.__str__() <==> str(x)`

__unicode__(...)

Data descriptors inherited from exceptions.BaseException:

__dict__

args

message

class Element
`` | `Representation of an XML element for formatting output`

Methods defined here:

__init__(self, name, attrs=[])

__str__(self)

__unicode__(self)

comment(self, message)

element(self, element)

flatten(self)

text(self, value)

write(self, stream=<open file '<stdout>', mode 'w'>, level=0, flatten=False)

class Executable(CatalogType, InvokeMixin)

``

`Executable(name[,namespace][,version][,arch][,os][,osrelease][,osversion][,glibc][,installed])
An entry for an executable in the DAX-level replica catalog.
Examples:
grep = Executable("grep")
grep = Executable(namespace="os",name="grep",version="2.3")
grep = Executable(namespace="os",name="grep",version="2.3",arch=Arch.X86)
grep = Executable(namespace="os",name="grep",version="2.3",arch=Arch.X86,os=OS.LINUX)
`

Method resolution order:
  Executable
  CatalogType
  ProfileMixin
  MetadataMixin
  PFNMixin
  InvokeMixin

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, name, namespace=None, version=None, arch=None, os=None, osrelease=None, osversion=None, glibc=None, installed=None)
  `Arguments:
  name: Logical name of executable
  namespace: Executable namespace
  version: Executable version
  arch: Architecture that this exe was compiled for
  os: Name of os that this exe was compiled for
  osrelease: Release of os that this exe was compiled for
  osversion: Version of os that this exe was compiled for
  glibc: Version of glibc this exe was compiled against
  installed: Is the executable installed (true), or stageable (false)`

__str__(self)

__unicode__(self)

toXML(self)
  `Returns an XML representation of this file as a filename tag`

Methods inherited from CatalogType:

innerXML(self, parent)

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

Methods inherited from MetadataMixin:

addMetadata(self, metadata)
  `Add metadata to this object`

clearMetadata(self)
  `Remove all metadata from this object`

hasMetadata(self, metadata)
  `Does this object have metadata?`

metadata(self, key, type, value)
  `Declarative metadata addition`

removeMetadata(self, metadata)
  `Remove meta from this object`

Methods inherited from PFNMixin:

PFN(self, url, site=None)
  `Declarative PFN addition`

addPFN(self, pfn)
  `Add a PFN to this object`

clearPFNs(self)
  `Remove all PFNs from this object`

hasPFN(self, pfn)
  `Does this object have pfn?`

removePFN(self, pfn)
  `Remove PFN from this object`

Methods inherited from InvokeMixin:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class File(CatalogType)
`` | `File(name) A file entry for the DAX-level replica catalog, or a reference to a logical file used by the workflow. Examples: input = File('input.txt') Example use in job: input = File('input.txt') output = File('output.txt') job = Job(name="compute") job.uses(input, link=Link.INPUT, transfer=True) job.uses(output, link=Link.OUTPUT, transfer=True, register=True)`

Method resolution order:
  File
  CatalogType
  ProfileMixin
  MetadataMixin
  PFNMixin

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, name)
  `All arguments specify the workflow-level behavior of this File. Job-level
  behavior can be defined when adding the File to a Job's uses. If the
  properties are not overridden at the job-level, then the workflow-level
  values are used as defaults.
  If this LFN is to be used as a job's stdin/stdout/stderr then the value
  of link is ignored when generating the <std*> tags.
  Arguments:
  name: The name of the file (required)`

__str__(self)

__unicode__(self)

toArgumentXML(self)
  `Returns an XML representation of this File with no inner elements`

toStdioXML(self, tag)
  `Returns an XML representation of this file as a stdin/out/err tag`

toXML(self)
  `Return the XML representation of this File with inner elements`

Methods inherited from CatalogType:

innerXML(self, parent)

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

Methods inherited from MetadataMixin:

addMetadata(self, metadata)
  `Add metadata to this object`

clearMetadata(self)
  `Remove all metadata from this object`

hasMetadata(self, metadata)
  `Does this object have metadata?`

metadata(self, key, type, value)
  `Declarative metadata addition`

removeMetadata(self, metadata)
  `Remove meta from this object`

Methods inherited from PFNMixin:

PFN(self, url, site=None)
  `Declarative PFN addition`

addPFN(self, pfn)
  `Add a PFN to this object`

clearPFNs(self)
  `Remove all PFNs from this object`

hasPFN(self, pfn)
  `Does this object have pfn?`

removePFN(self, pfn)
  `Remove PFN from this object`

class FormatError(DAX3Error)

``

Method resolution order:
  FormatError
  DAX3Error
  exceptions.Exception
  exceptions.BaseException
  __builtin__.object

Data descriptors inherited from DAX3Error:

__weakref__
  `list of weak references to the object (if defined)`

Methods inherited from exceptions.Exception:

__init__(...)
  `x.__init__(...) initializes x; see help(type(x)) for signature`

Data and other attributes inherited from exceptions.Exception:

__new__ = <built-in method __new__ of type object>
  `T.__new__(S, ...) -> a new object with type S, a subtype of T`

Methods inherited from exceptions.BaseException:

__delattr__(...)
  `x.__delattr__('name') <==> del x.name`

__getattribute__(...)
  `x.__getattribute__('name') <==> x.name`

__getitem__(...)
  `x.__getitem__(y) <==> x[y]`

__getslice__(...)
  `x.__getslice__(i, j) <==> x[i:j]
  Use of negative indices is not supported.`

__reduce__(...)

__repr__(...)
  `x.__repr__() <==> repr(x)`

__setattr__(...)
  `x.__setattr__('name', value) <==> x.name = value`

__setstate__(...)

__str__(...)
  `x.__str__() <==> str(x)`

__unicode__(...)

Data descriptors inherited from exceptions.BaseException:

__dict__

args

message

class Invoke

``

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, when, what)

__str__(self)

__unicode__(self)

toXML(self)

class InvokeMixin

``

Methods defined here:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class Job(AbstractJob)

``

`Job((name|Executable|Transformation)[,id][,namespace][,version][,node_label])
This class defines the specifics of a job to run in an abstract manner.
All filename references still refer to logical files. All references
transformations also refer to logical transformations, though
physical location hints can be passed through profiles.
Examples:
sleep = Job(id="ID0001",name="sleep")
jbsim = Job(id="ID0002",name="jbsim",namespace="cybershake",version="2.1")
merge = Job("jbsim")
You can create a Job based on a Transformation:
mDiff_xform = Transformation("mDiff", ...)
mDiff_job = Job(mDiff_xform)
Or an Executable:
mDiff_exe = Executable("mDiff", ...)
mDiff_job = Job(mDiff_exe)
Several arguments can be added at the same time:
input = File(...)
output = File(...)
job.addArguments("-i",input,"-o",output)
Profiles are added similarly:
job.addProfile(Profile(Namespace.ENV, key='PATH', value='/bin'))
job.profile(Namespace.ENV, "PATH", "/bin")
Adding file uses is simple, and you can override global File attributes:
job.uses(input, Link.INPUT)
job.uses(output, Link.OUTPUT, transfer=True, register=True)
`

Method resolution order:
  Job
  AbstractJob
  ProfileMixin
  UseMixin
  InvokeMixin

Methods defined here:

__init__(self, name, id=None, namespace=None, version=None, node_label=None)
  `The ID for each job should be unique in the DAX. If it is None, then
  it will be automatically generated when the job is added to the DAX.
  The name, namespace, and version should match what you have in your
  transformation catalog. For example, if namespace="foo" name="bar"
  and version="1.0", then the transformation catalog should have an
  entry for "foo::bar:1.0".
  The name argument can be either a string, or a Transformation object. If
  it is a Transformation object, then the job will inherit the name, namespace,
  and version from the Transformation.
  Arguments:
  name: The transformation name or Transformation object (required)
  id: A unique identifier for the job (optional)
  namespace: The namespace of the transformation (optional)
  version: The transformation version (optional)
  node_label: The label for this job to use in graphing (optional)`

__str__(self)

__unicode__(self)

toXML(self)

Methods inherited from AbstractJob:

addArguments(self, *arguments)
  `Add one or more arguments to the job (this will add whitespace)`

addRawArguments(self, *arguments)
  `Add one or more arguments to the job (whitespace will NOT be added)`

clearArguments(self)
  `Remove all arguments from this job`

clearStderr(self)
  `Remove stderr file`

clearStdin(self)
  `Remove stdin file`

clearStdout(self)
  `Remove stdout file`

getArguments(self)
  `Get the arguments of this job`

innerXML(self, element)
  `Return an XML representation of this job`

setStderr(self, filename)
  `Redirect stderr to a file`

setStdin(self, filename)
  `Redirect stdin from a file`

setStdout(self, filename)
  `Redirect stdout to a file`

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

Methods inherited from UseMixin:

addUse(self, use)
  `Add Use to this object`

clearUses(self)
  `Remove all uses from this object`

hasUse(self, use)
  `Test to see if this object has use`

removeUse(self, use)
  `Remove use from this object`

uses(self, arg, link=None, register=None, transfer=None, optional=None, namespace=None, version=None, executable=None, size=None)

Methods inherited from InvokeMixin:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class Link
`` | `Linkage attributes. See File, Executable and uses().`

Data and other attributes defined here:

INOUT = 'inout'

INPUT = 'input'

NONE = 'none'

OUTPUT = 'output'

class Metadata
`` | `Metadata(key,type,value) A way to add metadata to File and Executable objects. This is useful if you want to annotate the DAX with things like file sizes, application-specific attributes, etc. There is currently no restriction on the type. Examples: s = Metadata('size','int','12') a = Metadata('algorithm','string','plav')`

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, key, type, value)
  `Arguments:
  key: The key name of the item
  type: The type of the value (e.g. string, int, float)
  value: The value of the item`

__str__(self)

__unicode__(self)

toXML(self)

class MetadataMixin

``

Methods defined here:

addMetadata(self, metadata)
  `Add metadata to this object`

clearMetadata(self)
  `Remove all metadata from this object`

hasMetadata(self, metadata)
  `Does this object have metadata?`

metadata(self, key, type, value)
  `Declarative metadata addition`

removeMetadata(self, metadata)
  `Remove meta from this object`

class Namespace
`` | `Namespace values recognized by Pegasus. See Executable, Transformation, and Job.`

Data and other attributes defined here:

CONDOR = 'condor'

DAGMAN = 'dagman'

ENV = 'env'

GLOBUS = 'globus'

HINTS = 'hints'

PEGASUS = 'pegasus'

SELECTOR = 'selector'

STAT = 'stat'

class NotFoundError(DAX3Error)

``

Method resolution order:
  NotFoundError
  DAX3Error
  exceptions.Exception
  exceptions.BaseException
  __builtin__.object

Data descriptors inherited from DAX3Error:

__weakref__
  `list of weak references to the object (if defined)`

Methods inherited from exceptions.Exception:

__init__(...)
  `x.__init__(...) initializes x; see help(type(x)) for signature`

Data and other attributes inherited from exceptions.Exception:

__new__ = <built-in method __new__ of type object>
  `T.__new__(S, ...) -> a new object with type S, a subtype of T`

Methods inherited from exceptions.BaseException:

__delattr__(...)
  `x.__delattr__('name') <==> del x.name`

__getattribute__(...)
  `x.__getattribute__('name') <==> x.name`

__getitem__(...)
  `x.__getitem__(y) <==> x[y]`

__getslice__(...)
  `x.__getslice__(i, j) <==> x[i:j]
  Use of negative indices is not supported.`

__reduce__(...)

__repr__(...)
  `x.__repr__() <==> repr(x)`

__setattr__(...)
  `x.__setattr__('name', value) <==> x.name = value`

__setstate__(...)

__str__(...)
  `x.__str__() <==> str(x)`

__unicode__(...)

Data descriptors inherited from exceptions.BaseException:

__dict__

args

message

class OS
`` | `OS types. See Executable.`

Data and other attributes defined here:

AIX = 'aix'

LINUX = 'linux'

MACOS = 'macos'

SUNOS = 'sunos'

WINDOWS = 'windows'

class PFN(ProfileMixin)
`` | `PFN(url[,site]) A physical file name. Used to provide URLs for files and executables in the DAX-level replica catalog. PFNs can be added to File and Executable. Examples: PFN('http://site.com/path/to/file.txt','site') PFN('http://site.com/path/to/file.txt',site='site') PFN('http://site.com/path/to/file.txt')`

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, url, site=None)
  `Arguments:
  url: The url of the file.
  site: The name of the site. [default: local]`

__str__(self)

__unicode__(self)

toXML(self)

Methods inherited from ProfileMixin:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

class PFNMixin

``

Methods defined here:

PFN(self, url, site=None)
  `Declarative PFN addition`

addPFN(self, pfn)
  `Add a PFN to this object`

clearPFNs(self)
  `Remove all PFNs from this object`

hasPFN(self, pfn)
  `Does this object have pfn?`

removePFN(self, pfn)
  `Remove PFN from this object`

class ParseError(DAX3Error)

``

Method resolution order:
  ParseError
  DAX3Error
  exceptions.Exception
  exceptions.BaseException
  __builtin__.object

Data descriptors inherited from DAX3Error:

__weakref__
  `list of weak references to the object (if defined)`

Methods inherited from exceptions.Exception:

__init__(...)
  `x.__init__(...) initializes x; see help(type(x)) for signature`

Data and other attributes inherited from exceptions.Exception:

__new__ = <built-in method __new__ of type object>
  `T.__new__(S, ...) -> a new object with type S, a subtype of T`

Methods inherited from exceptions.BaseException:

__delattr__(...)
  `x.__delattr__('name') <==> del x.name`

__getattribute__(...)
  `x.__getattribute__('name') <==> x.name`

__getitem__(...)
  `x.__getitem__(y) <==> x[y]`

__getslice__(...)
  `x.__getslice__(i, j) <==> x[i:j]
  Use of negative indices is not supported.`

__reduce__(...)

__repr__(...)
  `x.__repr__() <==> repr(x)`

__setattr__(...)
  `x.__setattr__('name', value) <==> x.name = value`

__setstate__(...)

__str__(...)
  `x.__str__() <==> str(x)`

__unicode__(...)

Data descriptors inherited from exceptions.BaseException:

__dict__

args

message

class Profile

``

`Profile(namespace,key,value)
A Profile captures scheduler-, system-, and environment-specific
parameters in a uniform fashion. Each profile declaration assigns a value
to a key within a namespace.
Profiles can be added to Job, DAX, DAG, File, Executable, and PFN.
Examples:
path = Profile(Namespace.ENV,'PATH','/bin')
vanilla = Profile(Namespace.CONDOR,'universe','vanilla')
path = Profile(namespace='env',key='PATH',value='/bin')
path = Profile('env','PATH','/bin')
`

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, namespace, key, value)
  `Arguments:
  namespace: The namespace of the profile (see Namespace)
  key: The key name. Can be anything that responds to str().
  value: The value for the profile. Can be anything that responds to str().`

__str__(self)

__unicode__(self)

toXML(self)
  `Return an XML element for this profile`

class ProfileMixin

``

Methods defined here:

addProfile(self, profile)
  `Add a profile to this object`

clearProfiles(self)
  `Remove all profiles from this object`

hasProfile(self, profile)
  `Does this object have profile?`

profile(self, namespace, key, value)
  `Declarative profile addition`

removeProfile(self, profile)
  `Remove profile from this object`

class Transfer
`` | `Transfer types for uses. See Executable, File.`

Data and other attributes defined here:

FALSE = 'false'

OPTIONAL = 'optional'

TRUE = 'true'

class Transformation(UseMixin, InvokeMixin)

``

`Transformation((name|executable)[,namespace][,version])
A logical transformation. This is basically defining one or more
entries in the transformation catalog. You can think of it like a macro
for adding <uses> to your jobs. You can define a transformation that
uses several files and/or executables, and refer to it when creating
a job. If you do, then all of the uses defined for that transformation
will be copied to the job during planning.
This code:
in = File("input.txt")
exe = Executable("exe")
t = Transformation(namespace="foo", name="bar", version="baz")
t.uses(in)
t.uses(exe)
j = Job(t)
is equivalent to:
in = File("input.txt")
exe = Executable("exe")
j = Job(namespace="foo", name="bar", version="baz")
j.uses(in)
j.uses(exe)
Examples:
Transformation(name='mDiff')
Transformation(namespace='montage',name='mDiff')
Transformation(namespace='montage',name='mDiff',version='3.0')
Using one executable:
mProjectPP = Executable(namespace="montage",name="mProjectPP",version="3.0")
x_mProjectPP = Transformation(mProjectPP)
Using several executables:
mDiff = Executable(namespace="montage",name="mProjectPP",version="3.0")
mFitplane = Executable(namespace="montage",name="mFitplane",version="3.0")
mDiffFit = Executable(namespace="montage",name="mDiffFit",version="3.0")
x_mDiffFit = Transformation(mDiffFit)
x_mDiffFit.uses(mDiff)
x_mDiffFit.uses(mFitplane)
Config files too:
conf = File("jbsim.conf")
jbsim = Executable(namespace="scec",name="jbsim")
x_jbsim = Transformation(jbsim)
x_jbsim.uses(conf)
`

Method resolution order:
  Transformation
  UseMixin
  InvokeMixin

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, name, namespace=None, version=None)
  `The name argument can be either a string or an Executable object.
  If it is an Executable object, then the Transformation inherits
  its name, namespace and version from the Executable, and the
  Transformation is set to use the Executable with link=input,
  transfer=true, and register=False.
  Arguments:
  name: The name of the transformation
  namespace: The namespace of the xform (optional)
  version: The version of the xform (optional)`

__str__(self)

__unicode__(self)

toXML(self)
  `Return an XML representation of this transformation`

Methods inherited from UseMixin:

addUse(self, use)
  `Add Use to this object`

clearUses(self)
  `Remove all uses from this object`

hasUse(self, use)
  `Test to see if this object has use`

removeUse(self, use)
  `Remove use from this object`

uses(self, arg, link=None, register=None, transfer=None, optional=None, namespace=None, version=None, executable=None, size=None)

Methods inherited from InvokeMixin:

addInvoke(self, invoke)
  `Add invoke to this object`

clearInvokes(self)
  `Remove all Invoke objects`

hasInvoke(self, invoke)
  `Test to see if this object has invoke`

invoke(self, when, what)
  `Invoke executable 'what' when job reaches status 'when'. The value of
  'what' should be a command that can be executed on the submit host.
  The list of valid values for 'when' is:
  WHEN MEANING
  ========== =======================================================
  never never invoke
  start invoke just before job gets submitted.
  on_error invoke after job finishes with failure (exitcode != 0).
  on_success invoke after job finishes with success (exitcode == 0).
  at_end invoke after job finishes, regardless of exit status.
  all like start and at_end combined.
  Examples:
  obj.invoke('at_end','/usr/bin/mail -s "job done" juve@usc.edu')
  obj.invoke('on_error','/usr/bin/update_db -failure')`

removeInvoke(self, invoke)
  `Remove invoke from this object`

class Use

``

`Use(file[,link][,register][,transfer][,optional]
[,namespace][,version][,executable][,size])
Use of a logical file name. Used for referencing files in the DAX.
Attributes:
file: A string, File or Executable representing the logical file
link: Is this file a job input, output or both (See LFN) (optional)
register: Should this file be registered in RLS? (True/False) (optional)
transfer: Should this file be transferred? (True/False or See LFN) (optional)
optional: Is this file optional, or should its absence be an error? (optional)
namespace: Namespace of executable (optional)
version: version of executable (optional)
executable: Is file an executable? (True/False) (optional)
size: The size of the file (optional)
For Use objects that are added to Transformations, the attributes 'link', 'register',
'transfer', 'optional' and 'size' are ignored.
If a File object is passed in as 'file', then the default value for executable
is 'false'. Similarly, if an Executable object is passed in, then the default
value for executable is 'true'.
`

Methods defined here:

__eq__(self, other)

__hash__(self)

__init__(self, name, link=None, register=None, transfer=None, optional=None, namespace=None, version=None, executable=None, size=None)

__str__(self)

__unicode__(self)

toJobXML(self)

toTransformationXML(self)

class UseMixin

``

Methods defined here:

addUse(self, use)
  `Add Use to this object`

clearUses(self)
  `Remove all uses from this object`

hasUse(self, use)
  `Test to see if this object has use`

removeUse(self, use)
  `Remove use from this object`

uses(self, arg, link=None, register=None, transfer=None, optional=None, namespace=None, version=None, executable=None, size=None)

class When
`` | `Job states for notifications. See Job/DAX/DAG.invoke().`

Data and other attributes defined here:

ALL = 'all'

AT_END = 'at_end'

NEVER = 'never'

ON_ERROR = 'on_error'

ON_SUCCESS = 'on_success'

START = 'start'

Functions

``

main()
  `Simple smoke test`

parse(infile)

parseString(string)

Data
`` |  | SCHEMA_LOCATION = 'http://pegasus.isi.edu/schema/dax-3.4.xsd' SCHEMA_NAMESPACE = 'http://pegasus.isi.edu/schema/DAX' SCHEMA_VERSION = '3.4' __author__ = 'Gideon Juve <juve@usc.edu>' __version__ = '3.3'

Author
`` |  | Gideon Juve <juve@usc.edu>