Package obitools :: Package parallel :: Module jobqueue
[hide private]
[frames] | no frames]

Source Code for Module obitools.parallel.jobqueue

  1  import threading 
  2  from logging import warning,info 
  3  from time import sleep,time 
  4   
  5  from obitools.parallel import TaskPool 
  6   
  7   
8 -class JobPool(dict):
9 ''' 10 JobPool is dedicated to manage a job queue. These jobs 11 will run in a limited number of thread. 12 ''' 13
14 - def __init__(self,count,precision=0.01):
15 ''' 16 17 @param count: number of thread dedicated to this JobPool 18 @type count: int 19 @param precision: delay between two check for new job (in second) 20 @type precision: float 21 ''' 22 self._iterator = JobIterator(self) 23 self._taskPool = TaskPool(self._iterator, 24 self._runJob, 25 count) 26 self._precision=precision 27 self._toRun=set() 28 self._runnerThread = threading.Thread(target=self._runner) 29 self._runnerThread.start() 30 self._finalyzed=False
31
32 - def _runner(self):
33 for rep in self._taskPool: 34 info('Job %d finnished' % id(rep)) 35 info('All jobs in %d JobPool finished' % id(self))
36
37 - def _jobIterator(self):
38 return self._iterator
39
40 - def _runJob(self,job):
41 job.started= time() 42 info('Job %d started' % id(job)) 43 job.result = job() 44 job.ended = time() 45 job.finished=True 46 return job
47
48 - def submit(self,job,priority=1.0,userid=None):
49 ''' 50 Submit a new job to the JobPool. 51 52 @param job: the new submited job 53 @type job: Job instance 54 @param priority: priority level of this job (higher is better) 55 @type priority: float 56 @param userid: a user identifier (Default is None) 57 58 @return: job identifier 59 @rtype: int 60 ''' 61 62 assert not self._finalyzed,\ 63 "This jobPool does not accept new job" 64 if job.submitted is not None: 65 warning('Job %d was already submitted' % id(job)) 66 return id(job) 67 68 job.submitted = time() 69 job.priority = priority 70 job.userid = userid 71 i=id(job) 72 job.id=id 73 self[i]=job 74 self._toRun.add(job) 75 76 info('Job %d submitted' % i) 77 78 return i
79
80 - def finalyze(self):
81 ''' 82 Indicate to the JobPool, that no new jobs will 83 be submitted. 84 ''' 85 self._iterator.finalyze() 86 self._finalyzed=True
87
88 - def __del__(self):
89 self.finalyze()
90 91
92 -class JobIterator(object):
93 - def __init__(self,pool):
94 self._pool = pool 95 self._finalyze=False 96 self._nextLock=threading.Lock()
97 98
99 - def __iter__(self):
100 return self
101
102 - def finalyze(self):
103 ''' 104 Indicate to the JobIterator, that no new jobs will 105 be submitted. 106 ''' 107 self._finalyze=True
108 109
110 - def next(self):
111 ''' 112 113 @return: the next job to run 114 @rtype: Job instance 115 ''' 116 self._nextLock.acquire() 117 while self._pool._toRun or not self._finalyze: 118 rep = None 119 maxScore=0 120 for k in self._pool._toRun: 121 s = k.runScore() 122 if s > maxScore: 123 maxScore=s 124 rep=k 125 if rep is not None: 126 self._pool._toRun.remove(rep) 127 self._nextLock.release() 128 return (rep,) 129 sleep(self._pool._precision) 130 self._nextLock.release() 131 info('No more jobs in %d JobPool' % id(self._pool)) 132 raise StopIteration
133 134 135
136 -class Job(object):
137
138 - def __init__(self,pool=None,function=None,*args,**kwargs):
139 ''' 140 Create a new job 141 142 @param pool: the jobpool used to run job. Can be None to not 143 execute the job immediately. 144 @type pool: JobPool instance 145 146 @param function: the function to run for the job 147 @type function: callable object 148 149 @param args: parametters for function call 150 @param kwargs: named parametters for function call 151 152 @precondition: function cannot be None 153 ''' 154 assert function is not None 155 self._args=args 156 self._kwargs = kwargs 157 self._function = function 158 self.running = False 159 self.finished= False 160 self.submitted = None 161 self.priority = None 162 self.userid = None 163 164 if pool is not None: 165 pool.submit(self)
166
167 - def runScore(self):
168 ''' 169 @return: the score used to ordonnance job in the queue 170 @rtype: C{float} 171 ''' 172 173 return (time() - self.submitted) * self.priority
174
175 - def __call__(self):
176 return self._function(*self._args,**self._kwargs)
177