Package obitools :: Package parallel
[hide private]
[frames] | no frames]

Source Code for Package obitools.parallel

 1  import threading 
 2   
3 -class TaskPool(object):
4
5 - def __init__(self,iterable,function,count=2):
6 self.pool = [] 7 self.queue= [] 8 self.plock= threading.Lock() 9 self.qlock= threading.Lock() 10 self.function=function 11 self.event=threading.Event() 12 self.iterable=iterable 13 for i in xrange(count): 14 Task(self)
15
16 - def register(self,task):
17 self.plock.acquire() 18 self.pool.append(task) 19 self.plock.release() 20 self.ready(task)
21
22 - def unregister(self,task):
23 task.thread.join() 24 self.plock.acquire() 25 self.pool.remove(task) 26 self.plock.release()
27 28
29 - def ready(self,task):
30 self.qlock.acquire() 31 self.queue.append(task) 32 self.qlock.release() 33 self.event.set()
34
35 - def __iter__(self):
36 for data in self.iterable: 37 while not self.queue: 38 self.event.wait() 39 self.event.clear() 40 self.qlock.acquire() 41 task=self.queue.pop(0) 42 self.qlock.release() 43 if hasattr(task, 'rep'): 44 yield task.rep 45 #print "send ",data 46 if isinstance(data,dict): 47 task.submit(**data) 48 else: 49 task.submit(*data) 50 51 while self.pool: 52 self.pool[0].finish() 53 while self.queue: 54 self.event.clear() 55 self.qlock.acquire() 56 task=self.queue.pop(0) 57 self.qlock.release() 58 if hasattr(task, 'rep'): 59 yield task.rep
60 61 62 63 64
65 -class Task(object):
66 - def __init__(self,pool):
67 self.pool = pool 68 self.lock = threading.Lock() 69 self.dataOk = threading.Event() 70 self.repOk = threading.Event() 71 self.args = None 72 self.kwargs=None 73 self.stop=False 74 self.thread = threading.Thread(target=self) 75 self.thread.start() 76 self.pool.register(self)
77
78 - def __call__(self):
79 self.dataOk.wait() 80 while(not self.stop): 81 self.lock.acquire() 82 self.dataOk.clear() 83 self.rep=self.pool.function(*self.args,**self.kwargs) 84 self.pool.ready(self) 85 self.lock.release() 86 self.dataOk.wait()
87
88 - def submit(self,*args,**kwargs):
89 self.args=args 90 self.kwargs=kwargs 91 self.dataOk.set()
92
93 - def finish(self):
94 self.lock.acquire() 95 self.stop=True 96 self.dataOk.set() 97 self.pool.unregister(self)
98