""" qtgui/JobControl.py: CCP4MG Molecular Graphics Program Copyright (C) 2001-2008 University of York, CCLRC Copyright (C) 2009-2010 University of York Copyright (C) 2012 STFC This library is free software: you can redistribute it and/or modify it under the terms of the GNU Lesser General Public License version 3, modified in accordance with the provisions of the license to address the requirements of UK law. You should have received a copy of the modified GNU Lesser General Public License along with this library. If not, copies may be downloaded from http://www.ccp4.ac.uk/ccp4license.php This program is distributed in the hope that it will be useful, but WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU Lesser General Public License for more details. """ from global_definitions import * from PyQt4 import QtCore,QtGui class JobControl(QtCore.QObject): def __init__(self,parent=None): QtCore.QObject.__init__(self,parent) import os,utils self.log_dir = os.path.join(utils.get_CCP4MG(),'logs') self.pending_jobs = {} self.pending_jobs_list = [] self.last_job_id = 0 self.reviewWindow = None MAINWINDOW().addMenuDefinition('job_control', { 'text' : 'Review jobs', 'tip' : 'Show details of external program runs', 'slot' : self.openReview }, 'Tools') #------------------------------------------------------------------------------ def get_jobs_list(self): #------------------------------------------------------------------------------ job_list = [] for root in self.pending_jobs_list: job_list.append(self.formatted_job(root)) #print 'JobControl.get_jobs_list',job_list return job_list #------------------------------------------------------------------------------ def formatted_job(self,root): #------------------------------------------------------------------------------ import time number = (str(root)+' ')[0:4] name = (self.pending_jobs[root]['name'] + ' ')[0:20] text = number + ' '+ name + ' ' if self.pending_jobs[root]['finish_time']: if self.pending_jobs[root]['status']: text = text + ' FAILED ' + time.asctime(time.localtime(self.pending_jobs[root]['finish_time'])) else: text = text + ' FINISHED ' + time.asctime(time.localtime(self.pending_jobs[root]['finish_time'])) else: text = text + ' STARTED ' + time.asctime(time.localtime(self.pending_jobs[root]['start_time'])) return text #------------------------------------------------------------------------------ def getparams(self): #------------------------------------------------------------------------------ pars = {} for item in ['pending_jobs','last_job_id']: pars[item]=getattr(self,item) return pars #-------------------------------------------------------------------------------- def setparams(self,params = {}): #-------------------------------------------------------------------------------- #print "JobControl.setparams",params for key in params.keys(): setattr(self,key,params[key]) #---------------------------------------------------------------------- def read_log_file(self,root='',file=''): #---------------------------------------------------------------------- import os if file: root = os.path.splitext(os.path.split(file)[-1])[0] elif root and self.pending_jobs.has_key(root) and self.pending_jobs[root].has_key('logfile'): file = self.pending_jobs[root]['logfile'] #print "jobcontrol.read_log_file",file pars = { 'status': '', 'report' : '', 'task' : '', 'job_id' : '' } if os.path.exists(file): f = open(file,'r') else: f = None if f: content = f.read() #print "jobcontrol.read_log_file",content f.close() import string for line in string.split(content,'\n'): words = string.split(line) if words: if words[0] == "#TERMINATION": pars['status']=words[2] if words[0] == "#REPORT": import re pars['report']=re.sub('#REPORT','',line) elif words[0] == "#SCRIPT": pars['task']=words[2] elif words[0] == "#JOB_ID": pars['job_id']=words[1] return pars #------------------------------------------------------------------------ def start_job(self,name='',interpreter='',command='',args=[], \ registry_info=[],handler=[],progress={},log=''): #------------------------------------------------------------------------ #### DO NOT USE #### The Qt based start_process replaces this method import os import sys import types import utils import time self.last_job_id = self.last_job_id + 1 logfile = os.path.join(utils.get_CCP4MG(),'logs',name+'_'+str(self.last_job_id)+'.log') #print "start_job",logfile if interpreter: if interpreter == 'python': if sys.platform == 'win32': #cmd = os.path.join(os.environ['CCP4MG'],'pythondist','bin','python.exe') cmd = os.path.join(sys.prefix,'python.exe') else: # BEWARE !sys.prefix does not have the 'bin' on Fedora2/frodo print "prefix",sys.prefix cmd = '/usr/bin/python' argstring = utils.msvcescape(command) else: cmd = command argstring = '' if isinstance(args,types.ListType): for item in args: argstring = argstring + ' ' + item else: argstring = argstring + ' ' + args argstring = argstring + ' -job_id '+ str(self.last_job_id) + ' -task ' + name if sys.platform == 'win32': print "Starting process",cmd,"argstring",argstring try: os.spawnl(os.P_NOWAIT,cmd,"--",argstring) #os.spawnv(os.P_NOWAIT,cmd,args) except: print "ERROR spawning:",cmd,'arguments:',argstring return [1,"ERROR spawning: "+cmd+' arguments: '+argstring] else: try: os.system(cmd+' '+argstring+' &') except: print "ERROR running:",cmd,'arguments:',argstring return [1,"ERROR running: "+cmd+' arguments: '+argstring] root = name+'_'+str(self.last_job_id) self.pending_jobs[root] = {} self.pending_jobs[root]['start_time'] = int(time.time()) self.pending_jobs[root]['handler'] = handler #self.pending_jobs[root]['progress'] = progress # ??? explicit log file name -- not used at the moment #self.pending_jobs[root]['log'] = log if progress: # Put up GUI window with progress bar #print "JobControl.start_job progress",progress self.pending_jobs[root]['gui'] = mgProgressDialog(labelText=progress['LABEL'],cancelButtonText='Cancel',title=progress['TITLE'],help=progress['HELP']) return [0,self.last_job_id] #------------------------------------------------------------------------------ def handle_progress_gui(self,**keywords): #------------------------------------------------------------------------------ #print "in handle_progress_gui" pass #------------------------------------------------------------------------ def start_process(self,name='',interpreter='',command='',args=[], \ handler=[],progress={},log='',add_job_id=0,process_modifier=[],**kw): #------------------------------------------------------------------------ ''' Use Qt QProcess to run external job ''' import os,sys,types,time import utils from PyQt4 import QtCore import guiUtils self.last_job_id = self.last_job_id + 1 logfile = os.path.join(utils.get_CCP4MG(),'logs',name+'_'+str(self.last_job_id)+'.log') #print "start_process",logfile if interpreter: if interpreter == 'python': if sys.platform == 'win32': #cmd = os.path.join(os.environ['CCP4MG'],'pythondist','bin','python.exe') cmd = os.path.join(sys.prefix,'python.exe') elif sys.platform == 'darwin': cmd = os.path.join(os.environ['CCP4MG'],'bin','QtMG') else: print "Running",command,"using python from",sys.prefix cmd = os.path.join(sys.prefix , 'bin', 'python') arglist = [command] else: cmd = command arglist = [] # if isinstance(args,types.ListType): for item in args: arglist.append(item) else: import string arglist.extend(string.split(args)) l = [] for item in arglist: l.append(QtCore.QString(unicode(item,'utf-8'))) # external ccp4mg processes can handle job_id and task arguments if add_job_id: l.extend([QtCore.QString('-job_id'), QtCore.QString(str(self.last_job_id)), QtCore.QString('-task'), QtCore.QString(name)]) root = self.last_job_id process = QtCore.QProcess(self) process.setObjectName(name) if process_modifier: apply(process_modifier[0],[process],(process_modifier[1])) self.connect(process,QtCore.SIGNAL('finished(int)'),guiUtils.partial(self.end_process,root)) if DEVELOPER(): print "JobControl process.start",root,cmd,arglist if interpreter == 'python' and sys.platform == 'win32': self.set_process_environment(process) #for item in process.systemEnvironment(): print item process.start(cmd,QtCore.QStringList(l)) self.pending_jobs_list.append(root) self.pending_jobs[root] = {'name': name, 'command' : cmd, 'arglist' : arglist, 'error' : '', 'output': '', 'finish_time': 0, 'start_time' : int(time.time()), 'handler' : handler, 'process' : process, 'logfile' : log } if progress: # Put up GUI window with progress bar self.pending_jobs[root]['gui'] = mgProgress(progress['LABEL'],title=progress['TITLE'],help=progress['HELP']) if self.reviewWindow: self.reviewWindow.updateSelectJob(root,self.formatted_job(root)) return [0,self.last_job_id] #------------------------------------------------------------------------ def end_process(self,root,**kw): #------------------------------------------------------------------------ import time #print "JOBCONTROL.end_process",self.pending_jobs,root,kw if self.pending_jobs.has_key(root): self.pending_jobs[root]['status'] = self.pending_jobs[root]['process'].exitStatus() self.pending_jobs[root]['finish_time'] = int(time.time()) self.pending_jobs[root]['error'] = str(self.pending_jobs[root]['process'].readAllStandardError()) self.pending_jobs[root]['output'] = str(self.pending_jobs[root]['process'].readAllStandardOutput()) #if DEVELOPER(): # print 'JobControl',root,'output:',self.pending_jobs[root]['output'] # print 'JobControl',root,'error:',self.pending_jobs[root]['error'] self.pending_jobs[root]['finish_time'] = int(time.time()) if self.pending_jobs[root].has_key('gui'): self.pending_jobs[root]['gui'].close() handler = self.pending_jobs[root].get('handler',[]) if handler: if DEVELOPER(): apply(handler[0],[root],(handler[1])) else: try: apply(handler[0],[root],(handler[1])) except: print "ERROR in JobControl.end_process - failed to run handler for ",root else: print "ERROR in JobControl.end_process - process id unrecognised",root if self.reviewWindow: self.reviewWindow.updateSelectJob(root,self.formatted_job(root)) def setJobData(self,id,attribute='',value=''): if self.pending_jobs.has_key(id): self.pending_jobs[id][attribute] = value def getJobData(self,id,attribute='error'): return self.pending_jobs.get(id,{}).get(attribute,'') def deleteJob(self,id): if self.pending_jobs.has_key(id): del self.pending_jobs[id] #---------------------------------------------------------------------- def set_process_environment(self,process,**kw): #---------------------------------------------------------------------- # Modify the QProcess process environment import os,sys env = process.systemEnvironment() #for ii in range(env.count()): print str(env[ii]) pypath_list = sys.path pypath = pypath_list[0] for item in pypath_list[1:]:pypath= pypath+';'+item #print 'pypath',pypath env.append('PYTHONPATH='+pypath) process.setEnvironment(env) return 0 #-------------------------------------------------------------------- def openReview(self): #-------------------------------------------------------------------- import MGSimpleDialog if not self.reviewWindow: self.reviewWindow = jobReviewGui(MAINWINDOW()) self.connect(self.reviewWindow,QtCore.SIGNAL('show'),self.showStatus) self.reviewWindow.show() self.reviewWindow.raise_() #-------------------------------------------------------------------- def showStatus(self): #-------------------------------------------------------------------- root = self.reviewWindow.selectedJob() if not self.pending_jobs.has_key(root): return cmd = self.pending_jobs[root]['command'] + u' ' for item in self.pending_jobs[root]['arglist']: if type(item) == str: cmd = cmd + u' ' + unicode(item,'utf-8') else: cmd = cmd + u' ' + item output = 'Run command: ' + cmd +'\n\n' name = '' logfile = '' if hasattr(self,"pending_jobs") and self.pending_jobs.has_key('root'): if self.pending_jobs[root].has_key('name'): name = self.pending_jobs[root]['name'] if self.pending_jobs[root].has_key('logfile'): name = self.pending_jobs[root]['logfile'] if self.pending_jobs[root].has_key('status'): output = output + 'Finish status: '+str(self.pending_jobs[root]['status'])+'\n\n' if self.pending_jobs[root].has_key('error'): output = output + 'Standard error:\n'+self.pending_jobs[root]['error']+'\n\n' if self.pending_jobs[root].has_key('output'): output = output + 'Standard out:\n'+self.pending_jobs[root]['output'] self.reviewWindow.loadText( root, name, output, logfile) #-------------------------------------------------------------------- #-------------------------------------------------------------------- #-------------------------------------------------------------------- class mgProgress( QtGui.QDialog ): #-------------------------------------------------------------------- #-------------------------------------------------------------------- #-------------------------------------------------------------------- #-------------------------------------------------------------------- def __init__(self,text,title='Progress',help=''): #-------------------------------------------------------------------- QtGui.QDialog.__init__(self,MAINWINDOW()) self.cancelCall = '' self.setWindowTitle(title) self.setAttribute(QtCore.Qt.WA_DeleteOnClose) layout = QtGui.QVBoxLayout() import string for line in string.split(text,'\n'): widget = QtGui.QLabel(line,self) layout.addWidget(widget) widget = QtGui.QProgressBar(self) widget.setRange(0,0) widget.setObjectName('progress') layout.addWidget(widget) widget = QtGui.QPushButton('Cancel job',self) self.connect(widget,QtCore.SIGNAL('clicked()'),self.cancelProgress) layout.addWidget(widget) self.setLayout(layout) self.show() #-------------------------------------------------------------------- def setCancel(self,cancelCall): #-------------------------------------------------------------------- self.cancelCall = cancelCall #-------------------------------------------------------------------- def setProgress(self,value): #-------------------------------------------------------------------- widget = self.findChild(QtGui.QProgressBar,'progress') if widget: widget.setValue(value) #-------------------------------------------------------------------- def cancelProgress(self): #-------------------------------------------------------------------- if self.cancelCall: self.cancelCall() self.close() #-------------------------------------------------------------------- #-------------------------------------------------------------------- #-------------------------------------------------------------------- class mgProgressDialog(QtGui.QProgressDialog): #-------------------------------------------------------------------- #-------------------------------------------------------------------- #-------------------------------------------------------------------- def __init__(self,labelText='',cancelButtonText='',minimum=1,maximum=100,parent=None,title='',help=''): if not parent: parent = MAINWINDOW() QtGui.QProgressDialog.__init__(self,labelText,cancelButtonText,minimum,maximum,parent) self.setAttribute(QtCore.Qt.WA_DeleteOnClose) self.setWindowTitle(title) #-------------------------------------------------------------------- #-------------------------------------------------------------------- #-------------------------------------------------------------------- class jobReviewGui(QtGui.QDialog): #-------------------------------------------------------------------- #-------------------------------------------------------------------- #-------------------------------------------------------------------- def __init__(self,parent): QtGui.QDialog.__init__(self,parent) self.setWindowTitle('CCP4mg: Review jobs') layout = QtGui.QVBoxLayout() frame_margin = 1 self.select_frame = QtGui.QGroupBox('Select job to review',self) frame_layout = QtGui.QVBoxLayout() frame_layout.setContentsMargins(frame_margin,frame_margin,frame_margin,frame_margin) frame_layout.setSpacing(frame_margin) self.selectJob= QtGui.QListWidget(self) self.selectJob.setSelectionMode(QtGui.QAbstractItemView.SingleSelection) self.selectJob.setMinimumHeight(30) #self.selectJob.setDefaultFont(QtGui.QFont('Courier',12)) font = self.selectJob.viewOptions().font #font.setFamily('Courier') import guiUtils self.connect(self.selectJob,QtCore.SIGNAL('currentRowChanged(int)'),guiUtils.partial(self.emit,QtCore.SIGNAL('show'))) frame_layout.addWidget(self.selectJob) self.select_frame.setLayout(frame_layout) self.error_frame = QtGui.QGroupBox('Program output',self) frame_layout = QtGui.QVBoxLayout() frame_layout.setContentsMargins(frame_margin,frame_margin,frame_margin,frame_margin) frame_layout.setSpacing(frame_margin) self.error = QtGui.QTextEdit(self) self.error.setReadOnly(1) self.error.setMinimumWidth(600) self.error.setMinimumHeight(100) self.error.setFontFamily('Courier') frame_layout.addWidget(self.error) self.error_frame.setLayout(frame_layout) self.log_frame = QtGui.QGroupBox('Log file',self) frame_layout = QtGui.QVBoxLayout() frame_layout.setContentsMargins(frame_margin,frame_margin,frame_margin,frame_margin) frame_layout.setSpacing(frame_margin) self.logfile = QtGui.QTextEdit(self) self.logfile.setReadOnly(1) self.logfile.setFontFamily('Courier') frame_layout.addWidget(self.logfile) self.log_frame.setLayout(frame_layout) self.splitter = QtGui.QSplitter(QtCore.Qt.Vertical,self) layout.addWidget(self.splitter) self.splitter.addWidget(self.select_frame) self.splitter.addWidget(self.error_frame) self.splitter.addWidget(self.log_frame) self.splitter.setStretchFactor(2,3) self.splitter.setSizes([2,2,3]) dialog_buttons = QtGui.QDialogButtonBox(self) cancel_button = dialog_buttons.addButton('Close',QtGui.QDialogButtonBox.RejectRole) #self.showButton = dialog_buttons.addButton('Show job details',QtGui.QDialogButtonBox.ApplyRole) self.connect(cancel_button,QtCore.SIGNAL('clicked()'),self.close) #import guiUtils #self.connect(self.showButton,QtCore.SIGNAL('clicked()'),guiUtils.partial(self.emit,QtCore.SIGNAL('show'))) layout.addWidget(dialog_buttons) self.loadSelectJob() self.selectJob.show() self.setLayout(layout) def loadSelectJob(self): self.selectJob.blockSignals(True) self.selectJob.clear() for item in JOBCONTROL().get_jobs_list(): widget = self.selectJob.addItem(item) self.selectJob.blockSignals(False) def addJob(self,item): self.selectJob.blockSignals(True) self.selectJob.addItem(item) self.selectJob.blockSignals(False) def selectedJob(self): irow = self.selectJob.currentRow() if irow<0: return '' text = str(self.selectJob.item(irow).text()) return int(text.split()[0]) def loadText(self,root=-1,name='',error='',logfile=''): self.error_frame.setTitle('Output from job number '+str(root)+' '+name) self.error.setText(error) log_text = '' import os if logfile: self.log_frame.setTitle('Log file: '+logfile) if not os.path.exists(logfile): log_text = log_text + 'Error - log file does not exists' else: try: f = open(logfile,'r') text = f.read() f.close() log_text = log_text + '\n' + text except: log_text = log_text + 'Error reading log file' else: self.log_frame.setTitle('No log file') self.logfile.setText(log_text) sizes = self.splitter.sizes() if sizes[0]>0.9: l_sel = self.selectJob.count() l_log = len(log_text.split('\n')) l_err = len(error.split('\n')) def updateSelectJob(self,root,item_text): row = root-1 #print 'updateSelectJob',row,item_text,'count', self.selectJob.count() self.selectJob.blockSignals(True) if row< self.selectJob.count(): item = self.selectJob.item(row) if item: item.setText(item_text) else: self.selectJob.addItem(item_text) self.selectJob.blockSignals(False)