如何用幻影鏈接

Question

PhantomJS可用於BeautifulSoup的替代方案嗎？

我試圖在Etsy上搜索並訪問所有鏈接的術語。在Python中，我知道如何做到這一點（使用BeautifulSoup），但今天我想看看我能否用PhantomJS做同樣的事情。我不是很遠。

此腳本應在Etsy上搜索「hello kitty」，並將所有產品返回 <a class="listing-thumb" href=...></a>並在控制檯中打印它們。理想情況下，我會在稍後訪問它們並獲取我需要的信息。現在它只是凍結。有任何想法嗎？

var page = require('webpage').create(); 
var url = 'http://www.etsy.com/search?q=hello%20kitty'; 

page.open(url, function(status){ 
    // list all the a.href links in the hello kitty etsy page 
    var link = page.evaluate(function() { 
     return document.querySelectorAll('a.listing-thumb'); 
    }); 
    for(var i = 0; i < link.length; i++){ console.log(link[i].href); } 
    phantom.exit(); 
}); 

我玩弄了使用CasperJS，這可能是更好的設計。

Answer 1

您的代碼唯一的問題是您不瞭解phantomjs範圍。你有幻像和頁面範圍。您試圖從頁面範圍（page.evaluate在頁面範圍內運行）返回JavaScript DOM對象引用（那些不能被序列化）到虛幻主範圍。我認爲這是不可能的。這裏如下代碼工作：

var page = require('webpage').create(); 
var url = 'http://www.etsy.com/search?q=hello%20kitty'; 

// for debug (to see if page returns status code 200) 
page.onResourceReceived = function(response) { 
    if (response.url === url) { 
     console.log('Resorce: "' + response.url + '" status: ' + response.status); 

     if (response.status === 200) { 
      console.log(response.url); 
      for (var i = 0; i < response.headers.length; i++) { 
       console.log(response.headers[i].name + ': ' + response.headers[i].value); 
      } 
     } 
    } 
}; 

page.onLoadFinished = function(status){ 
    console.log('Status: ' + status); 

    console.log('Starting evaluate...'); 
    var links = page.evaluate(function() { 
     var nodes = [], 
      matches = document.querySelectorAll("a.listing-thumb"); 

      for(var i = 0; i < matches.length; ++i) { 
       nodes.push(matches[i].href); 
      } 

      return nodes; 
    }); 
    console.log('Done evaluate... count: ' + links.length); 

    if (links && links.length > 0) { 
     for(var i = 0; i < links.length; ++i) { 
      console.log('(' + i + ') ' + links[i]); 
     } 
    } else { 
     console.log("No match found!"); 
    } 

    phantom.exit(0); 
}; 

page.open(url); 

Answer 2

PhantomJS evaluate()不能序列和返回複雜的對象，如HTML元素或的NodeLists，所以你必須之前將它們映射到序列化的東西：

var page = require('webpage').create(); 
var url = 'http://www.etsy.com/search?q=hello%20kitty'; 

page.open(url, function(status) { 
    // list all the a.href links in the hello kitty etsy page 
    var links = page.evaluate(function() { 
     return [].map.call(document.querySelectorAll('a.listing-thumb'), function(link) { 
      return link.getAttribute('href'); 
     }); 
    }); 
    console.log(links.join('\n')); 
    phantom.exit(); 
}); 

注：這裏我們用[].map.call()以便將NodeList作爲標準Array來處理。

Answer 3

這是我最近寫的一些代碼，使用PhantomJs來刮取網址，如果你只提供一個URL，它會在網頁上顯示所有的URLs，如果你提供一個參數class|id後跟一個「class/id name」僅顯示類/標識的URL。

////////////////////////////////////////////////////////// 
///// PhantomJS URL Scraper v.1.3 ///// 
// 
// Copyrighted by +A.M.Danischewski 2016+ (c) 
// This program may be reutilized without limits, provided this 
// notice remain intact. 
// 
// Usage: phantomjs phantom_urls.js <URL> [["class"|"id"] [<query id/class name>]] 
// 
// Argument 1: URL -- "https://www.youtube.com/watch?v=8TniRMwL2Vg" 
// Argument 2: "class" or "id" 
// Argument 3: If Argument 2 was provided, "class name" or "id name" 
// 
// By default this program will display ALL urls from a user supplied URL. 
// If a class name or id name is provided then only URL's from the class 
// or id are displayed. 
// 
/////////////////////////////////// 

var page = require('webpage').create(), 
    system = require('system'), 
    address; 

if (system.args.length === 1) { 
    console.log(' Usage: phantomjs phantom_urls.js <URL> [["class"|"id"] [<query id/class name>]]'); 
    phantom.exit(); 
} 

address = system.args[1]; 
querytype= system.args[2]; 
queryclass = system.args[3]; 
page.open(address, function(status) { 
    if (status !== 'success') { 
    console.log('Error loading address: '+address); 
    } else { 
    //console.log('Success! In loading address: '+address); 
    } 
}); 

page.onConsoleMessage = function(msg) { 
    console.log(msg); 
} 

page.onLoadFinished = function(status) { 
    var dynclass="function() { window.class_urls = new Array(); window.class_urls_next=0; var listings = document.getElementsByClassName('"+queryclass+"'); for (var i=0; i < listings.length; i++) { var el = listings[i]; var ellnks=[].map.call(el.querySelectorAll('a'),function(link) {return link.getAttribute('href');}); var elhtml=el.innerHTML; window.class_urls.push(ellnks.join('\\n')); }; return window.class_urls;}"; 
    var dynid="function() { window.id_urls = new Array(); window.id_urls_next=0; var listings = document.getElementById('"+queryclass+"'); var ellnks=[].map.call(listings.querySelectorAll('a'),function(link) {return link.getAttribute('href');}); var elhtml=listings.innerHTML; window.id_urls.push(ellnks.join('\\n')); return window.id_urls;}"; 
    var allurls="function() { var links = page.evaluate(function() { return [].map.call(document.querySelectorAll('a'), function(link) { return link.getAttribute('href'); };); };); console.log(links.join('\\n')); }"; 
    var page_eval_function=""; 
    if (querytype === "class") { 
    console.log(page.evaluate(dynclass).toString().replace(/,/g, "\n")); 
    } else if (querytype === "id") { 
    console.log(page.evaluate(dynid).toString().replace(/,/g, "\n")); 
    } else { 
    var links = page.evaluate(function() { 
     return [].map.call(document.querySelectorAll('a'), function(link) { 
      return link.getAttribute('href'); 
     }); 
    });  
     console.log(links.join('\n')); 
    }    
    phantom.exit(); 
};