# $Id: ScriptObject.py,v 1.4.50.1 2004/03/17 21:25:09 ggraham Exp $ # # The ScriptObject module implements a data structure which implements # as attributes all of the things needed to encapsulate completely a # fully specified job in the application sense, ie- except for location # or batch manager. This data structure should contain all of the # information needed by the ShahKar grid/batch manager to create a # job on the specified resources. # # The ScriptObject needs to be persistent in the sense that a ShahKar # Configurator must be able to save it and load it again later for # job submission of resubmission. Therefore, the attributes must also # have this property. Attributes can be given in three initial types: # (1) Ordinary: An attribute of this type is a key/value pair. # (2) SmallFile: A parameter of this type represents a small control # file or helper script as often are required by our applications. # The ScriptObject contains directly the contents of objects of # type SmallFile. # (3) ExternalFile: A parameter of this type represents a large data # file or any other file whose existence is guaranteed by an # external reference to an RLS, for example. These are currently # represented by only metadata, but later could also be interfaced # with the FileMetaBroker class. # The types are extensible through an interface that requires specification # of the attribute schema, a function that can save data for the attribute # into the persistent form, and a function that can load data for the # attribute from the persistem form. The persistence consists of a unique # directory per scriptObject that contains a job.info file plus any ancillary # files that need to be transported along with the scriptObject. The # job.info file contains the metadata needed to reconstruct all of the # saved attributes. # # # GEG 15-March-2004 # import sys,os,string import UserDict # ### Ordinary Parameter Type # # "Ordinary" type just has a single value. Schema_OrdinaryType=tuple(['Value']) # # The Save function should take a back reference to the scriptObject and # the name of the attribute being saved. It should return a formatted line # that will be saved in the "job.info" file in the persistent path. def SaveOrdinaryType(scrObj,mdName) : mdLine=mdName+' MDType=Ordinary Value='+scrObj[mdName]+'\n' return mdLine # # The Load function should take a back reference to the scriptObject and # the name of the attribute being loaded. Optionally an unparsed line from # the job.info file corresponding to the attribute is returned. If present, # this line should be used to populate the attribute in the scriptObject. If # not, the job.info file will be opened to get the line. def LoadOrdinaryType(scrObj,mdName,mdLine=None) : if mdLine==None : mdLine=scrObj._getMetadata(mdName) mdDictionary=scrObj._parseMetadata(mdLine)[1] scrObj[mdName]=mdDictionary['Value'] # # "SmallFile" type has a FileName and FileContents. Schema_SmallFileType=tuple(['FileName','FileContents']) # # This Save function saves the filename into the job.info file, and saves the # FileContents into a new file in the persistent path. def SaveSmallFileType(scrObj,mdName) : fname=scrObj.GetAttributeValue(mdName,'FileName') mdLine=mdName+' MDType=SmallFile FileName='+fname+'\n' file=open(scrObj._persistentPath+'/'+fname,'w') for line in scrObj.GetAttributeValue(mdName,'FileContents') : file.write(line+'\n') file.close() return mdLine # # This Load function loads the filename from the job.info file, and loads the # FileContents from the referenced file in the persistentPath. def LoadSmallFileType(scrObj,mdName,mdLine=None) : if mdLine==None : mdLine=scrObj._getMetadata(mdName) mdDictionary=scrObj._parseMetadata(mdLine)[1] fname=mdDictionary['FileName'] scrObj.SetAttributeValue(mdName,FileName=fname) file=open(scrObj._persistentPath+'/'+fname,'r') scrObj.SetAttributeValue(mdName,FileContents=file.readlines()) file.close() # # Exception class - will derive from ShahKarException later class ScriptObjectException(Exception) : pass # # ScriptObject class has a UserDict interface for backwards compatibility. # API: # ScriptObject() # C'tor takes no args # AddItem(mdName,mdType,values=None,**kw) # # mdName is a new attribute name # # mdType is a registered type # # values is a vector of initials for the attribute # # (if not given, a vector of 'null' is used) # # initial values can be given by keyword also. # class ScriptObject(UserDict.UserDict) : # def __init__(self) : # UserDict.UserDict.__init__(self) # self._jobInfoFile='job.info' self._persistentPath=None # unique directory where scriptObject # will do Save and Load # self._attributes=[] # Ordered list of attributes self._attributeTypes=[] # Ordered list of registered attribute types self._systemAttributes=[] # List of attibutes that cannot be deleted # No interface to this; must set in c'tor # self._attributeTypes_ByName={} # Map of attr. to its type self._attributeValues_ByName={} # Map of attr. to its value struct self._attributeSchemas_ByType={} # Map of type to its schema self._attributeSaveMethods_ByType={} # Map of type to its save method self._attributeLoadMethods_ByType={} # Map of type to its load method # # Register the predefined types Ordinary and SmallFile self.RegisterAttributeType("Ordinary",Schema_OrdinaryType,\ SaveOrdinaryType,LoadOrdinaryType) self.RegisterAttributeType("SmallFile",Schema_SmallFileType,\ SaveSmallFileType,LoadSmallFileType) # # Add the system tags # ScriptObjectName is a user filled tag that identifies the scriptObject self.AddItem('ScriptObjectName','Ordinary') self._systemAttributes.append('ScriptObjectName') # ExecutableName is a reference to another metadata element, probably # a SmallFile, which is the executable assoc. with the scriptObject. self.AddItem('ExecutableName','Ordinary') self._systemAttributes.append('ExecutableName') # UniqueKey is a comma separated list of other metadata elements that # the UniqueString() method can use to construct a unique identifier self.AddItem('UniqueKey','Ordinary') self._systemAttributes.append('UniqueKey') # # # _getMetadata will load the job.info file and return the metadata line # in it corresponding to mdName def _getMetadata(self,mdName) : jifName=self._persistentPath+'/'+self._jobInfoFile if not os.path.exists(jifName) : # Probably get here if this was called before "Save()" was ever called raise ScriptObjectException,'_getMetadata: File not found: '+jifName jif=open(jifName,'r') jifContents=jib.readlines() jif.close() mdLine=None for line in jifContents : if string.split(line)[0] == mdName : mdLine=line break if mdLine == None : raise ScritpObjectException, 'Metadata Line not found: '+mdName return mdLine # # # _parseMetadata will take a metadata line from the job.info file and # return the a tuple of the attribute name and its metadata struct. def _parseMetadata(self,line) : tokens=string.split(line) if len(tokens) == 0 : raise ScriptObjectException, "_parseMetadata: Emtpy input." mdName=tokens[0] tmpDict={} for item in tokens[1:] : key=string.split(item,'=')[0] value=string.split(item,'=')[1] tmpDict[key]=value return (mdName,tmpDict) # # # AddItem will add a new attribute with name mdName of type mdType. # Optionally, if a list or tuple of values is given, the attribute struct # will be initialized with these values. Otherwise, a vector of 'null' # will be used. Elements in the attribute struct can also be initialized # by Keyword argument. If mdType is "Ordinary" then this also updates the # UserDict. def AddItem(self,mdName,mdType,value=None,**kwargs) : if mdName in self._attributes : raise ScriptObjectException, "AddItem: Tried to add duplicate attribute: "\ +mdName self._attributeTypes_ByName[mdName]=mdType self._attributeValues_ByName[mdName]={} self._attributes.append(mdName) method=ScriptObject.SetAttributeValue useValue=value if useValue == None : useValue=tuple(map(lambda x : 'null', self._attributeSchemas_ByType[mdType])) apply(method,(self,mdName,useValue,),kwargs) if mdType == 'Ordinary' : self.data[mdName]=self._attributeValues_ByName[mdName]['Value'] # # # Native method to set attribute values. The attribute struct will be # set with the given vector of values. Otherwise, a vector of 'null' # will be used. Elements in the attribute struct can also be set # by Keyword argument. If the type is "Ordinary" then this also updates the # UserDict. def SetAttributeValue(self,mdName,value=None,**kwargs) : if mdName not in self._attributes: raise ScriptObjectException,"SetAttributeValue: Non-existent attribute named: "+mdName mdType=self._attributeTypes_ByName[mdName] # First do the vector if value != None : if len(value) == len(self._attributeSchemas_ByType[mdType]) : i=0 for item in self._attributeSchemas_ByType[mdType] : self._attributeValues_ByName[mdName][item]=value[i] i=i+1 else: raise ScriptObjectException,'SetAttributeValue: Wrong number of value vector elements.' # Then do the keywords for key in kwargs.keys() : if key in self.GetAttributeSchema(mdName) : self._attributeValues_ByName[mdName][key]=kwargs[key] else : pass # Do not throw exception here; other clients may need kw. # # # UserDict setitem interface will set/create "Ordinary" attributes only. def __setitem__(self,key,value) : if key not in self._attributes : self.AddItem(key,'Ordinary') mdType=self._attributeTypes_ByName[key] if not mdType == 'Ordinary' : raise ScriptObjectException, "__setitem__: Tried to set non-Ordinary attribute." self.SetAttributeValue(key,Value=value) # # # UserDict delitem attribute has to work consistently. Only works on # "Ordinary" attributes. def __delitem__(self,key) : # We will not delete "system" attributes if key in self._systemAttributes : raise ScriptObjectException, "__delitem__: Tried to delete system attribute: "+key if key not in self._attributes: raise ScriptObjectException,"__delitem__: Non-existent attribute named: "+key mdType=self.GetAttributeType(key) if not mdType == 'Ordinary' : raise ScriptObjectException, "__delitem__: Tried to delete non-Ordinary attribute: "+key # OK- delete stuff... UserDict.UserDict.__delitem__(self,key) del self._attributeTypes_ByName[key] del self._attributeValues_ByName[key] del self._attributes[self._attributes.index(key)] # Check if this attribute happened to be part of UniqueKey. # If so, remove it. This should emit a warning to ShahKar Logger. uk=string.split(self['UniqueKey'],',') if key in uk : del uk[uk.index(key)] if len(uk) > 0 : self['UniqueKey']=string.join(uk,',') else: self['UniqueKey']='null' # # # Return the type associated with attribute mdName. def GetAttributeType(self,mdName) : if mdName not in self._attributes: raise ScriptObjectException,"GetAttributeType: Non-existent attribute named: "+mdName mdType=self._attributeTypes_ByName[mdName] return mdType # # # Return the schema associated with attribute mdName. def GetAttributeSchema(self,mdName) : if mdName not in self._attributes: raise ScriptObjectException,"GetAttributeSchema: Non-existent attribute named: "+mdName mdType=self._attributeTypes_ByName[mdName] return self._attributeSchemas_ByType[mdType] # # # Gets an attribute value. If a subSelect on the attribute schema # is not given, returns the whole attribute struct as a tuple. def GetAttributeValue(self,mdName,subSelect=None) : if mdName not in self._attributes: raise ScriptObjectException,"GetAttributeValue: Non-existent attribute named: "+mdName if subSelect==None: valList=[] for key in self.GetAttributeSchema(mdName) : valList.append(self._attributeValues_ByName[mdName][key]) return tuple(valList) else : if subSelect not in self.GetAttributeSchema(mdName) : raise ScriptObjectException,"GetAttributeValue: In attribute "+\ mdName+", non-existent attribute schema element named: "+\ subSelect return self._attributeValues_ByName[mdName][subSelect] # # # UserDict __getitem__ interface. Works for "Ordinary" attributes only. def __getitem__(self,key) : mdType=self._attributeTypes_ByName[key] if not mdType == 'Ordinary' : raise ScriptObjectException, "__getitem__: Tried to get non-Ordinary attribute." return self.GetAttributeValue(key,'Value') # # # Accessor for attribute list def Attributes(self) : return self._attributes # # # Adds an existing metadata element to the list of what makes up a # "UniqueKey" def AddUniqueKey(self,mdName) : if mdName not in self._attributes: raise ScriptObjectException,"AddUniqueKey: Non-existent attribute named: "+mdName currentUK=self['UniqueKey'] if currentUK=='null' : newUK=mdName else: newUK=currentUK+','+mdName self.SetAttributeValue('UniqueKey',Value=newUK) # # # Based on contents of UniqueKey, generates a UniqueString identifying # this ScriptObject def UniqueString(self) : if self['UniqueKey']=='null' : return '' currentUK=self['UniqueKey'] keys=string.split(currentUK,',') uString='' for key in keys : uString=uString+key+'='+self[key]+',' if len(uString)>0 : uString=uString[:-1] return uString # # # Sets the Executable tag. (Could also use setitem interface for this, # but it would bypass the consistency check.) def SetAsExecutable(self,mdName) : if mdName not in self._attributes: raise ScriptObjectException,"SetAsExecutable: Non-existent attribute named: "+mdName self.SetAttributeValue('ExecutableName',Value=mdName) # Gets the value of the Executable tag. (Could also use getitem interface # for this...) def WhichIsExecutable(self) : return self['ExecutableName'] # # # Accessors for the persistent path def SetPersistentPath(self,path) : self._persistentPath=path def GetPersistentPath(self) : return self._persistentPath # # # Save method. Marshalls the information provided by the save functions # provided with each type to create a job info file that can reconstruct # this scriptObject later. def Save(self) : if self._persistentPath == None : raise ScriptObjectException, "Save: SetPersistentPath() not called yet." if not os.path.exists(self._persistentPath) : os.makedirs(self._persistentPath) jif=open(self._persistentPath+'/'+self._jobInfoFile,'w') for item in self._attributes: mdType=self._attributeTypes_ByName[item] func=self._attributeSaveMethods_ByType[mdType] mdline=apply(func,(self,item,)) jif.write(mdline) jif.close() # # # Load method. Marshalls the information saved by the save functions # provided with each type to populate the scriptObject attributes from # information in the job info file def Load(self,force=0) : if self._persistentPath == None : raise ScriptObjectException, "Load: SetPersistentPath() not called yet." if not os.path.exists(self._persistentPath) : raise ScriptObjectException, "Load: PersistentPath does not exist: "+\ self._persistentPath jifName=self._persistentPath+'/'+self._jobInfoFile if not os.path.exists(jifName) : raise ScriptObjectException, "Load: Job info does not exist: "+\ jifName jif=open(jifName,'r') jifContents=jif.readlines() jif.close() for line in jifContents : item,mdDict=self._parseMetadata(line) mdType=mdDict['MDType'] if not item in self._attributes : self.AddItem(item,mdType) elif self.GetAttributeType(item) != mdType : raise ScriptObjectException, "Load: Attribute type found in Job Info"+\ " file does not match declared type: "+item func=self._attributeLoadMethods_ByType[mdType] apply(func,(self,item,line)) # # # Attribute Type Registration interface def RegisterAttributeType(self,mdType,attrSchema,funcSave,funcLoad) : if not mdType in self._attributeTypes : self._attributeTypes.append(mdType) self._attributeSchemas_ByType[mdType]=attrSchema self._attributeSaveMethods_ByType[mdType]=funcSave self._attributeLoadMethods_ByType[mdType]=funcLoad # if __name__ == '__main__' : # # Test 1: Compilation and Constructor # scrObj=ScriptObject() print "Compiled and Constructed OK." print # # Test 2: Adding/Setting Ordinary Parameters # import commands print "Setting persistent path to $PWD" scrObj.SetPersistentPath(os.environ['PWD']) print "Result from ScritpObject:", scrObj.GetPersistentPath() print # print "Setting ScriptObjectName parameter to 'DateScriptObject'" print "Using list interface:" scrObj.SetAttributeValue('ScriptObjectName',['DateScriptObject']) print "Result from ScriptObject: ",\ scrObj['ScriptObjectName'] print "Result in tuple form: ",scrObj.GetAttributeValue('ScriptObjectName') print print "Using keyword interface (and '_KW' added):" scrObj.SetAttributeValue('ScriptObjectName',Value='DateScriptObject_KW') print "Result from ScriptObject: ",\ scrObj['ScriptObjectName'] print # dt=string.join(string.split(commands.getoutput('date')),'-') print "Adding an ordinary parameter 'DateFormatString' with value '+%H:%m:%S':" print " (using __setitem__ interface)" scrObj['DateFormatString']='+%H:%m:%S' print "Result from ScriptObject", scrObj.GetAttributeValue('DateFormatString') print "Adding an ordinary parameter 'CreationTime' with value "+dt+" by keyword:" print " (using AddItem interface)" scrObj.AddItem('CreationTime','Ordinary',Value=dt) print "Result from ScriptObject", scrObj.GetAttributeValue('CreationTime') print # # Test 3: Adding/Setting SmallFile Parameters # print "Adding an SmallFile parameter 'WrapperScript' with name 'date.sh'" print "with contents: #!/bin/sh; date $1; echo I was created at $2" scrObj.AddItem('WrapperScript','SmallFile',FileName='date.sh') scrObj.SetAttributeValue('WrapperScript',FileContents=['#!/bin/sh','date $1',\ 'echo I was created at $2']) print "Result from ScriptObject", scrObj.GetAttributeValue('WrapperScript') print "Set as Executable:" scrObj.SetAsExecutable('WrapperScript') print "Result from ScriptObject:",scrObj.WhichIsExecutable() print # # Test 3.5 UniqueString # print "Setting UniqueString to ScriptObjectName,CreationTime" scrObj.AddUniqueKey('ScriptObjectName') scrObj.AddUniqueKey('CreationTime') print "Result from ScriptObject: ",scrObj.UniqueString() print # # Test 4: Save and Load # print "Saving ScriptObject" if not os.path.exists('testScriptObject') : os.mkdir('testScriptObject') scrObj.SetPersistentPath('testScriptObject') scrObj.Save() print print "Contents of persistent path:" ppath=scrObj.GetPersistentPath() dirContents=os.listdir(ppath) for file in dirContents : print " Contents of "+file+':' fContents=open(ppath+'/'+file,'r').readlines() for line in fContents : print " "+line print print "Moving ScriptObject" os.rename('testScriptObject','TestScriptObject') print print "Loading New ScriptObject - Soft Load" scrObj_2=ScriptObject() scrObj_2.SetPersistentPath('TestScriptObject') scrObj_2.AddItem('CreationTime','Ordinary') scrObj_2.AddItem('DateFormatString','Ordinary') scrObj_2.AddItem('WrapperScript','SmallFile') scrObj_2.Load() print "Results:" print " ",scrObj_2.GetAttributeValue('ScriptObjectName') print " ",scrObj_2.GetAttributeValue('ExecutableName') print " ",scrObj_2.GetAttributeValue('CreationTime') print " ",scrObj_2.GetAttributeValue('DateFormatString') print " ",scrObj_2.GetAttributeValue('WrapperScript') print print "Loading New ScriptObject - Hard Load" scrObj_3=ScriptObject() scrObj_3.SetPersistentPath('TestScriptObject') scrObj_3.Load() print "Results:" print " ",scrObj_3.GetAttributeValue('ScriptObjectName') print " ",scrObj_3.GetAttributeValue('ExecutableName') print " ",scrObj_3.GetAttributeValue('CreationTime') print " ",scrObj_3.GetAttributeValue('DateFormatString') print " ",scrObj_3.GetAttributeValue('WrapperScript') print print "Running Script:" exeAttr=scrObj_3['ExecutableName'] exeName=scrObj_3.GetAttributeValue(exeAttr,'FileName') fname=scrObj_3.GetPersistentPath()+'/'+exeName os.system('chmod +x '+fname) arg1=scrObj_3['DateFormatString'] arg2=scrObj_3['CreationTime'] print commands.getoutput(fname+' '+arg1+' '+arg2) print # # Test 5: Register New Type # print "Registering Imaginary Type" ImaginaryNumberSchema=['X','Y'] def SaveImaginaryType(scrObj,mdName) : x=scrObj.GetAttributeValue(mdName,'X') y=scrObj.GetAttributeValue(mdName,'Y') mdLine=mdName+" MDType=Imaginary X="+x+" Y="+y+'\n' xyfn=scrObj.GetPersistentPath()+'/xy.txt' xy=string.atof(x) * string.atof(y) xyf=open(xyfn,'w') xyf.write("%f\n"%xy) xyf.close() return mdLine def LoadImaginaryType(scrObj,mdName,mdLine=None) : if mdLine==None : mdLine=scrObj._getMetadata(mdName) mdDictionary=scrObj._parseMetadata(mdLine)[1] x=mdDictionary['X'] y=mdDictionary['Y'] scrObj.SetAttributeValue(mdName,X=x,Y=y) contents=open(scrObj_3.GetPersistentPath()+'/xy.txt','r').readlines() print "Contents of xy.txt from last Save(): ",contents scrObj_3.RegisterAttributeType('Imaginary',\ ImaginaryNumberSchema,\ SaveImaginaryType,\ LoadImaginaryType) print "Adding Item of Type Imaginary: x=3.5, y=4.4" scrObj_3.AddItem('Z','Imaginary',X='3.5',Y='4.4') print "Result from ScriptObject: ",scrObj_3.GetAttributeValue('Z') print "Saving ..." scrObj_3.Save() print "Loading ..." scrObj_3.Load() # # Test 6: Deletions # print "Current Attributes: ",scrObj_3.Attributes() print "Try to delete ordinary parameter CreationTime: " del scrObj_3['CreationTime'] print "Current Attributes: ",scrObj_3.Attributes() print # # Test 7: Exceptions # print "Testing some exceptions:" print "Adding duplicate attribute: " try : scrObj_3.AddItem('DateFormatString','OrdinaryType') except Exception, e: print " ",e print "Using __setitem__ to set a SmallFile attribute: " try : scrObj_3['WrapperScript'] = 'null' except Exception, e: print " ",e print "Using SetAttributeValue to set non-existent attribute: " try : scrObj_3.SetAttributeValue('IDoNotExist',('null')) except Exception, e: print " ",e print "Using SetAttributeValue with wrong number of values in vector: " try : scrObj_3.SetAttributeValue('DateFormatString',('null','null')) except Exception, e: print " ",e print "Deleting System Attribute: " try : del scrObj_3['UniqueKey'] except Exception, e: print " ",e print "Deleting non-existent attribute: " try : del scrObj_3['IDoNotExist'] except Exception, e: print " ",e print "Deleting non-Ordinary attribute: " try : del scrObj_3['WrapperScript'] except Exception, e: print " ",e print "Getting non-existent attribute type: " try : scrObj_3.GetAttributeType('IDoNotExist') except Exception, e: print " ",e print "Getting non-existent attribute schema: " try : scrObj_3.GetAttributeSchema('IDoNotExist') except Exception, e: print " ",e print "Getting non-existent attribute value: " try : scrObj_3.GetAttributeValue('IDoNotExist') except Exception, e: print " ",e print "Getting non-existent attribute member: " try : scrObj_3.GetAttributeValue('DateFormatString','IDoNotExist') except Exception, e: print " ",e print "Using __getitem__ interface for non-Ordinary attributes: " try : x=scrObj_3['WrapperScript'] except Exception, e: print " ",e print "Adding non-existent element to UniqueKey" try : scrObj_3.AddUniqueKey('IDoNotExist') except Exception, e: print " ",e print "Setting non-existent element as executable" try : scrObj_3.SetAsExecutable('IDoNotExist') except Exception, e: print " ",e scrObj_4 = ScriptObject() print "Saving scriptObject before persistent path is set" try : scrObj_4.Save() except Exception, e: print " ",e print "Loading scriptObject before persistent path is set" try : scrObj_4.Load() except Exception, e: print " ",e scrObj_4.SetPersistentPath('IDoNotExist') print "Loading scriptObject with non-existent persistent path" try : scrObj_4.Load() except Exception, e: print " ",e print "Load from existing path with inconsistent schema" scrObj_4.SetPersistentPath('TestScriptObject') scrObj_4.AddItem('DateFormatString','SmallFile') try : scrObj_4.Load() except Exception, e: print " ",e # # Clean Up # print "Cleaning Up" os.system('rm -rf TestScriptObject')