此代碼爲什麼只下載一個頁面的數據？

Question

我已經嘗試了很多次，但它不工作：

import requests 
from lxml import html, etree 
from selenium import webdriver 
import time, json 

#how many page do you want to scan 
page_numnotint = input("how many page do you want to scan") 
page_num = int(page_numnotint) 
file_name = 'jd_goods_data.json' 


url = 'https://list.jd.com/list.html?cat=1713,3264,3414&page=1&delivery=1&sort=sort_totalsales15_desc&trans=1&JL=4_10_0#J_main' 
driver = webdriver.Chrome() 
driver.get(url) 
base_html = driver.page_source 
selctor = etree.HTML(base_html) 
date_info = [] 
name_data, price_data = [], [] 
jd_goods_data = {} 
for q in range(page_num): 
    i = int(1) 
    while True: 
     name_string = '//*[@id="plist"]/ul/li[%d]/div/div[3]/a/em/text()' %(i) 
     price_string = '//*[@id="plist"]/ul/li[%d]/div/div[2]/strong[1]/i/text()' %(i) 
     if i == 60: 
      break 
     else: 
      i += 1 
     name = selctor.xpath(name_string)[0] 
     name_data.append(name) 
     price = selctor.xpath(price_string)[0] 
     price_data.append(price) 
     jd_goods_data[name] = price 

     print(name_data) 
     with open(file_name, 'w') as f: 
      json.dump(jd_goods_data, f) 
    time.sleep(2) 
    driver.find_element_by_xpath('//*[@id="J_bottomPage"]/span[1]/a[10]').click() 
    time.sleep(2) 

    # for k, v in jd_goods_data.items(): 
    #  print(k,v) 

我想下載一些細節，但它不工作。如果您輸入2進行掃描，它只會下載一個頁面的詳細信息，但會下載兩次！

Answer 1

好吧，你定義q，但你並沒有真正使用它。在這種情況下，慣例是將這個未使用的變量命名爲_。我的意思是，與其做

for q in range(page_num): 

你應該做的

for _ in range(page_num): 

因此，其他編程人員將直接知道你不使用q，只希望重複您的操作。

這意味着（由於某些原因）行driver.find_element_by_xpath('//*[@id="J_bottomPage"]/span[1]/a[10]').click()不能正確執行。肯定有辦法讓它工作。但在你的情況下，我啓發式地看到你的url包含一個名爲page的參數。我建議你改用它。這因此導致實際使用的變量q因爲如此，如下：

import requests 
from lxml import html,etree 
from selenium import webdriver 
import time, json 

#how many page do you want to scan 
page_numnotint = input("how many page do you want to scan") 
page_num = int(page_numnotint) 
file_name = 'jd_goods_data.json' 

driver = webdriver.Chrome() 
date_info = [] 
name_data, price_data = [], [] 
jd_goods_data = {} 
for q in range(page_num): 
    url = 'https://list.jd.com/list.html?cat=1713,3264,3414&page={page}&delivery=1&sort=sort_totalsales15_desc&trans=1&JL=4_10_0#J_main'.format(page=q) 
    driver.get(url) 
    base_html = driver.page_source 
    selctor = etree.HTML(base_html) 
    i = 1 
    while True: 
     name_string = '//*[@id="plist"]/ul/li[%d]/div/div[3]/a/em/text()' %(i) 
     price_string = '//*[@id="plist"]/ul/li[%d]/div/div[2]/strong[1]/i/text()' %(i) 
     if i == 60: 
      break 
     else: 
      i += 1 
     name = selctor.xpath(name_string)[0] 
     name_data.append(name) 
     price = selctor.xpath(price_string)[0] 
     price_data.append(price) 
     jd_goods_data[name] = price 

     print(name_data) 

with open(file_name, 'w') as f: 
    json.dump(jd_goods_data, f) 

driver.quit()