使用PyQt4和美麗的湯來瀏覽網頁

Question

我試圖從網站中的多個頁面中提取一些數據，並使用Javascript生成內容。 所以我使用PyQt4和美麗的湯來循環頁面並提取一些數據字段。

import sys 
from bs4 import BeautifulSoup 
from PyQt4.QtGui import QApplication 
from PyQt4.QtCore import QUrl 
from PyQt4.QtWebKit import QWebPage 


class Client(QWebPage): 

    def __init__(self, url): 
     self.app = QApplication(sys.argv) 
     QWebPage.__init__(self) 
     self.loadFinished.connect(self.on_page_load) 
     self.mainFrame().load(QUrl(url)) 
     self.app.exec_() 

    def on_page_load(self): 
     self.app.quit() 

products_titles = [] 
urls= ['url1', 'url2', 'url3'] 

for url in urls: 
    print "Parsing URL: " + url + '\n' 
    client_response = Client(url) 
    source = client_response.mainFrame().toHtml() 
    soup = BeautifulSoup(source, "html.parser") 
    print get_product_category(soup) 

但是當我運行它擊碎並給出了此錯誤：

QObject::connect: Cannot connect (null)::configurationAdded(QNetworkConfiguration) to QNetworkConfigurationManager::configurationAdded(QNetworkConfiguration) 
QObject::connect: Cannot connect (null)::configurationRemoved(QNetworkConfiguration) to QNetworkConfigurationManager::configurationRemoved(QNetworkConfiguration) 
QObject::connect: Cannot connect (null)::configurationChanged(QNetworkConfiguration) to QNetworkConfigurationManager::configurationChanged(QNetworkConfiguration) 
QObject::connect: Cannot connect (null)::onlineStateChanged(bool) to QNetworkConfigurationManager::onlineStateChanged(bool) 
QObject::connect: Cannot connect (null)::configurationUpdateComplete() to QNetworkConfigurationManager::updateCompleted() 
[1] 14809 segmentation fault python products.py 

我不知道我是我做錯了，請你知道什麼事情幫助。

Answer 1

我會發送一個URL列表，並讓QApplication的一個實例按順序加載它們，而不是實例化和銷燬一堆QApplications。

換句話說嘗試更多的東西像這樣...

import sys 
from bs4 import BeautifulSoup 
from PyQt4.QtGui import QApplication 
from PyQt4.QtCore import QUrl, pyqtSignal 
from PyQt4.QtWebKit import QWebPage 

class Client(QWebPage): 

    new_url = pyqtSignal(['QString'], name='new_url') 

    def __init__(self, urls): 
     self.app = QApplication(sys.argv) 
     self.urls = urls 
     self.pages = dict() 
     QWebPage.__init__(self) 
     self.new_url.connect(self.load_url) 
     self.loadFinished.connect(self.on_page_load) 
     if len(self.urls): 
      self.new_url.emit(urls.pop()) 
     self.app.exec_() 

    def load_url(self, url): 
     self.current_url = url 
     print "Loading: {0}".format(url) 
     self.mainFrame().load(QUrl(url)) 

    def on_page_load(self): 
     print "Retrieved: {0}".format(self.current_url) 
     self.pages[self.current_url] = unicode(self.mainFrame().toHtml()) 
     if len(self.urls): 
      self.new_url.emit(self.urls.pop()) 
     else: 
      self.app.quit() 

urls= ['http://www.google.com', 'http://www.yahoo.com', 'http://www.bing.com'] 

client = Client(urls) 
for (url, page) in client.pages.items(): 
    soup = BeautifulSoup(page, "html.parser") 
    print "{0}\t{1}".format(url, soup.title.text) 

重新實例一堆QApplications的似乎是一個非常糟糕的主意，我可以理解這種情況下分割故障。但是，在分段錯誤之前的網絡錯誤對我來說似乎有點奇怪。試一試，看看你是否有更好的運氣。它對我來說工作得很好。