使用Cookie刮掉動態頁面

Question

我正試圖抓取一組郵編。 https://www.chase.com/mortgage/loan-officer/search-results.html#action-search;zipcode-11747;lastname-;language-

如果你把它放到瀏覽器中，你會得到結果，但是在代碼中嘗試這樣做會失敗。

首先我想

HttpWebRequest request = (HttpWebRequest)System.Net.WebRequest.Create(URI); 
var sr = new System.IO.StreamReader(resp.GetResponseStream()); 
string page= sr.ReadToEnd().Trim(); 

而是通過插件產生的小提琴手沒有工作，這個代碼以及任。沒有結果返回。那麼我究竟在想什麼？

private void MakeRequests() 
{ 
    HttpWebResponse response; 
    string responseText; 

    if (Request_www_chase_com(out response)) 
    { 
     responseText = ReadResponse(response); 

     response.Close(); 
    } 
} 

private static string ReadResponse(HttpWebResponse response) 
{ 
    using (Stream responseStream = response.GetResponseStream()) 
    { 
     Stream streamToRead = responseStream; 
     if (response.ContentEncoding.ToLower().Contains("gzip")) 
     { 
      streamToRead = new GZipStream(streamToRead, CompressionMode.Decompress); 
     } 
     else if (response.ContentEncoding.ToLower().Contains("deflate")) 
     { 
      streamToRead = new DeflateStream(streamToRead, CompressionMode.Decompress); 
     } 

     using (StreamReader streamReader = new StreamReader(streamToRead, Encoding.UTF8)) 
     { 
      return streamReader.ReadToEnd(); 
     } 
    } 
} 

private bool Request_www_chase_com(out HttpWebResponse response) 
{ 
    response = null; 

    try 
    { 
     HttpWebRequest request = (HttpWebRequest)WebRequest.Create("https://www.chase.com/mortgage/loan-officer/search-results.html"); 

     request.KeepAlive = true; 
     request.Headers.Set(HttpRequestHeader.CacheControl, "max-age=0"); 
     request.Accept = "text/html,application/xhtml+xml,application/xml;q=0.9,image/webp,*/*;q=0.8"; 
     request.UserAgent = "Mozilla/5.0 (Windows NT 6.3; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/32.0.1700.76 Safari/537.36"; 
     request.Headers.Add("DNT", @"1"); 
     request.Referer = "https://mail.google.com/mail/u/0/?shva=1"; 
     request.Headers.Set(HttpRequestHeader.AcceptEncoding, "gzip,deflate,sdch"); 
     request.Headers.Set(HttpRequestHeader.AcceptLanguage, "en-US,en;q=0.8"); 
     request.Headers.Set(HttpRequestHeader.Cookie, @"v1st=3B46E5CCD302C2DE; marketlist=68|90|152|170|198; chasezip=zipcode=11577&county=Nassau&state=NY; ASP.NET_SessionId=kwybehscfioasswbl20wb14f; PC_1_0=n%3Dundefined|u%3Dundefined|l%3Dundefined|zip%3D11577|lastUpdate%3D2014-01-24|lastSent%3D2014-01-24|home%3Dpersonal|; SessionPersistence=CLICKSTREAMCLOUD%3A%3DvisitorId%3D%7CPROFILEDATA%3A%3D%7CSURFERINFO%3A%3Dbrowser%3DChrome%2COS%3DWindows%2Cresolution%3D1366x768%7C; fsr.s=%7B%22v2%22%3A-2%2C%22v1%22%3A1%2C%22rid%22%3A%22d464cf6-82273859-c860-572f-2944b%22%2C%22to%22%3A5%2C%22c%22%3A%22https%3A%2F%2Fwww.chase.com%2Fmortgage%2Floan-officer%2Fsearch-results.html%23action-search%3Bzipcode-11747%3Blastname-%3Blanguage-%22%2C%22pv%22%3A12%2C%22lc%22%3A%7B%22d18%22%3A%7B%22v%22%3A12%2C%22s%22%3Atrue%7D%7D%2C%22cd%22%3A18%2C%22sd%22%3A18%2C%22f%22%3A1390649574789%7D"); 
     request.IfModifiedSince = DateTime.Parse("Fri, 24 Jan 2014 20:18:51 GMT"); 

     response = (HttpWebResponse)request.GetResponse(); 
    } 
    catch (WebException e) 
    { 
     if (e.Status == WebExceptionStatus.ProtocolError) response = (HttpWebResponse)e.Response; 
     else return false; 
    } 
    catch (Exception) 
    { 
     if (response != null) response.Close(); 
     return false; 
    } 

    return true; 
} 

Answer 1

爲了使這項工作，你需要解析HTML，然後下載並運行JavaScript的。不要編寫自己的瀏覽器，而應使用Web瀏覽器控件加載頁面，然後擦除其內部的HTML。

Answer 2

該頁面使用AJAX來創建結果，這樣你會在你的迴應看是最初的HTML