1 import threading
2 from logging import warning,info
3 from time import sleep,time
4
5 from obitools.parallel import TaskPool
6
7
9 '''
10 JobPool is dedicated to manage a job queue. These jobs
11 will run in a limited number of thread.
12 '''
13
14 - def __init__(self,count,precision=0.01):
15 '''
16
17 @param count: number of thread dedicated to this JobPool
18 @type count: int
19 @param precision: delay between two check for new job (in second)
20 @type precision: float
21 '''
22 self._iterator = JobIterator(self)
23 self._taskPool = TaskPool(self._iterator,
24 self._runJob,
25 count)
26 self._precision=precision
27 self._toRun=set()
28 self._runnerThread = threading.Thread(target=self._runner)
29 self._runnerThread.start()
30 self._finalyzed=False
31
33 for rep in self._taskPool:
34 info('Job %d finnished' % id(rep))
35 info('All jobs in %d JobPool finished' % id(self))
36
39
41 job.started= time()
42 info('Job %d started' % id(job))
43 job.result = job()
44 job.ended = time()
45 job.finished=True
46 return job
47
48 - def submit(self,job,priority=1.0,userid=None):
49 '''
50 Submit a new job to the JobPool.
51
52 @param job: the new submited job
53 @type job: Job instance
54 @param priority: priority level of this job (higher is better)
55 @type priority: float
56 @param userid: a user identifier (Default is None)
57
58 @return: job identifier
59 @rtype: int
60 '''
61
62 assert not self._finalyzed,\
63 "This jobPool does not accept new job"
64 if job.submitted is not None:
65 warning('Job %d was already submitted' % id(job))
66 return id(job)
67
68 job.submitted = time()
69 job.priority = priority
70 job.userid = userid
71 i=id(job)
72 job.id=id
73 self[i]=job
74 self._toRun.add(job)
75
76 info('Job %d submitted' % i)
77
78 return i
79
81 '''
82 Indicate to the JobPool, that no new jobs will
83 be submitted.
84 '''
85 self._iterator.finalyze()
86 self._finalyzed=True
87
90
91
94 self._pool = pool
95 self._finalyze=False
96 self._nextLock=threading.Lock()
97
98
101
103 '''
104 Indicate to the JobIterator, that no new jobs will
105 be submitted.
106 '''
107 self._finalyze=True
108
109
111 '''
112
113 @return: the next job to run
114 @rtype: Job instance
115 '''
116 self._nextLock.acquire()
117 while self._pool._toRun or not self._finalyze:
118 rep = None
119 maxScore=0
120 for k in self._pool._toRun:
121 s = k.runScore()
122 if s > maxScore:
123 maxScore=s
124 rep=k
125 if rep is not None:
126 self._pool._toRun.remove(rep)
127 self._nextLock.release()
128 return (rep,)
129 sleep(self._pool._precision)
130 self._nextLock.release()
131 info('No more jobs in %d JobPool' % id(self._pool))
132 raise StopIteration
133
134
135
137
138 - def __init__(self,pool=None,function=None,*args,**kwargs):
139 '''
140 Create a new job
141
142 @param pool: the jobpool used to run job. Can be None to not
143 execute the job immediately.
144 @type pool: JobPool instance
145
146 @param function: the function to run for the job
147 @type function: callable object
148
149 @param args: parametters for function call
150 @param kwargs: named parametters for function call
151
152 @precondition: function cannot be None
153 '''
154 assert function is not None
155 self._args=args
156 self._kwargs = kwargs
157 self._function = function
158 self.running = False
159 self.finished= False
160 self.submitted = None
161 self.priority = None
162 self.userid = None
163
164 if pool is not None:
165 pool.submit(self)
166
168 '''
169 @return: the score used to ordonnance job in the queue
170 @rtype: C{float}
171 '''
172
173 return (time() - self.submitted) * self.priority
174
176 return self._function(*self._args,**self._kwargs)
177