使用隊列下載扭曲的文件

Question

我想使用扭曲和（例如）20個客戶端線程從隊列中下載多個文件。任何示例？

Answer 1

使用select模塊通過輪詢或threading模塊對此進行線程處理。

Answer 2

from twisted.internet.defer import inlineCallbacks, DeferredQueue 

@inlineCallbacks 
def worker(queue): 
    while 1: 
     url = yield queue.get() # wait for a url from the queue 

     if url is None: # insert None into the queue to kill workers 
      queue.put(None) 
      return # done 

     data = yield download(url) # download the file 
     process(data) # do stuff with it 


queue = DeferredQueue() # your queue 

# make workers 
MAX = 20 
workers = [worker(queue) for _ in range(MAX)] 

Answer 3

以下是https://github.com/caolan/async到Python的翻譯。

from twisted.internet import defer 
class Queue: 
    workers = 0 
    tasks = [] 
    def __init__(self, worker, concurrency): 
     self.worker = worker 
     self.concurrency = concurrency 
     self.saturated = None 
     self.empty = None 
     self.drain = None 
    def push(self, data): 
     deferred = defer.Deferred() 
     self.tasks.append({'data': data, 'callback': deferred}) 
     if self.saturated and len(tasks) == concurrency: 
      self.saturated() 
     self.process() 
     return deferred 
    def task_finished(self, *args): 
     self.workers = self.workers - 1 
     if self.drain and len(self.tasks) + self.workers == 0: 
      self.drain() 
     self.process() 
    def process(self): 
     if self.workers >= self.concurrency or len(self.tasks) == 0: 
      return 
     task = self.tasks.pop(0) 
     if self.empty and len(self.tasks) == 0: 
      self.empty() 
     self.workers = self.workers + 1 
     d = self.worker(task['data']) 
     d.addCallback(self.task_finished) 
     d.addCallback(task['callback'].callback) 

from twisted.web import client 
from twisted.internet import reactor 
def dl_worker(data): 
    url = data[0] 
    fname = data[1] 
    print "Download file:", fname 
    d = client.downloadPage(url, fname) 
    return d # very important! 

q = Queue(dl_worker, 2) 
q.drain = reactor.stop 
for i in range(0,3): 
    q.push(["http://download.thinkbroadband.com/5MB.zip", "file"+str(i)]) 
reactor.run() 

我希望通過雕文的QC：D乾杯！