Scrapy越來越表

Question

我試圖從HTML表中抽取數據內的鏈接數據，Texas Death Row

我可以使用下面的蜘蛛腳本從表中拉現有數據：

from scrapy.spider import BaseSpider 
from scrapy.selector import HtmlXPathSelector 

from texasdeath.items import DeathItem 

class DeathSpider(BaseSpider): 
    name = "death" 
    allowed_domains = ["tdcj.state.tx.us"] 
    start_urls = [ 
     "https://www.tdcj.state.tx.us/death_row/dr_executed_offenders.html" 
    ] 



    def parse(self, response): 
     hxs = HtmlXPathSelector(response) 
     sites = hxs.select('//table/tbody/tr') 
     for site in sites: 
      item = DeathItem() 
      item['firstName'] = site.select('td[5]/text()').extract() 
      item['lastName'] = site.select('td[4]/text()').extract() 
      item['Age'] = site.select('td[7]/text()').extract() 
      item['Date'] = site.select('td[8]/text()').extract() 
      item['Race'] = site.select('td[9]/text()').extract() 
      item['County'] = site.select('td[10]/text()').extract() 
      yield item 

問題是否還有表格中的鏈接，我試圖調用並從鏈接中獲取數據以將其附加到我的項目中。

Scrapy教程在這裏，Scrapy Tutorial似乎有關於如何從目錄中提取數據的指南。但是我很難搞清楚如何從主頁面獲取數據，以及如何從表格中的鏈接返回數據。

Answer 1

代替產生物品yield a Request並通過meta內的item。這包括在文檔here中。

蜘蛛的示例實現，如果它導致違法者「詳細信息」頁面，它將遵循「違規者信息」鏈接（有時會導致圖像 - 在這種情況下蜘蛛會輸出它目前的內容） ：

from urlparse import urljoin 

import scrapy 


class DeathItem(scrapy.Item): 
    firstName = scrapy.Field() 
    lastName = scrapy.Field() 
    Age = scrapy.Field() 
    Date = scrapy.Field() 
    Race = scrapy.Field() 
    County = scrapy.Field() 
    Gender = scrapy.Field() 


class DeathSpider(scrapy.Spider): 
    name = "death" 
    allowed_domains = ["tdcj.state.tx.us"] 
    start_urls = [ 
     "https://www.tdcj.state.tx.us/death_row/dr_executed_offenders.html" 
    ] 

    def parse(self, response): 
     sites = response.xpath('//table/tbody/tr') 
     for site in sites: 
      item = DeathItem() 

      item['firstName'] = site.xpath('td[5]/text()').extract() 
      item['lastName'] = site.xpath('td[4]/text()').extract() 
      item['Age'] = site.xpath('td[7]/text()').extract() 
      item['Date'] = site.xpath('td[8]/text()').extract() 
      item['Race'] = site.xpath('td[9]/text()').extract() 
      item['County'] = site.xpath('td[10]/text()').extract() 

      url = urljoin(response.url, site.xpath("td[2]/a/@href").extract_first()) 
      if url.endswith("html"): 
       yield scrapy.Request(url, meta={"item": item}, callback=self.parse_details) 
      else: 
       yield item 

    def parse_details(self, response): 
     item = response.meta["item"] 
     item["Gender"] = response.xpath("//td[. = 'Gender']/following-sibling::td[1]/text()").extract() 
     yield item