0
from scrapy import project, signals
from scrapy.crawler import Settings
from scrapy.crawler import CrawlerProcess
from scrapy.xlib.pydispatch import dispatcher
from multiprocessing.queues import Queue
import multiprocessing
class CrawlerWorker(multiprocessing.Process):
def __init__(self, spider, result_queue):
multiprocessing.Process.__init__(self)
self.result_queue = result_queue
self.crawler = Crawler(Settings())
if not hasattr(project, 'crawler'):
self.crawler.install()
self.crawler.configure()
self.items = []
self.spider = spider
dispatcher.connect(self._item_passed, signals.item_passed)
def _item_passed(self, item):
self.items.append(item)
def run(self):
self.crawler.crawl(self.spider)
self.crawler.start()
self.crawler.stop()
self.result_queue.put(self.items)
嘗試從scrapy.conf.settings只使用CrawlerProcess(設置)我得到一個錯誤,它看起來像有之間的scrapy醫生說這裏什麼http://doc.scrapy.org/en/latest/topics/practices.htmlScrapy:導入錯誤:無法導入名稱設置
的差異我正在遵循一個較舊的scrapy版本,我正在試圖使它與0.16的scrapy一起工作。
這是運行python腳本後的錯誤。
Traceback (most recent call last):
File "server.py", line 5, in <module>
from scraper import Scraper
File "/home/me/spider/spider/scraper.py", line 6, in <module>
from crawlerworker import CrawlerWorker
File "/home/me/spider/spider/crawlerworker.py", line 2, in <module>
from scrapy.crawler import Settings
ImportError: cannot import name Settings
你能錯誤添加到問題嗎? – Talvalin 2013-04-09 10:10:33
更新了有錯誤的問題 – KJW 2013-04-09 15:25:21