C＃控制檯/服務器訪問網站

Question

我正在從事一個C＃項目，我需要從沒有API或Web服務的安全網站獲取數據。我的計劃是登錄，進入我需要的頁面，並解析出HTML以獲取我需要登錄到數據庫的數據位。現在我正在使用控制檯應用程序進行測試，但最終這將轉換爲Azure服務總線應用程序。

爲了得到任何東西，您必須登錄到他們的login.cfm頁面，這意味着我需要在頁面上加載用戶名和密碼輸入控件，然後單擊提交按鈕。然後導航到我需要解析的頁面。因爲我沒有一個'瀏覽器'來解析控件，我試圖使用各種C＃.NET類來到頁面，設置用戶名和密碼，然後單擊提交，但似乎沒有任何工作。

我可以看看或.NET類的任何示例我應該審查是否爲這類項目設計的？

謝謝！

Answer 1

使用WebClient類中System.Net

對於會話cookie的持久性，你必須做出一個自定義的WebClient類。

#region webclient with cookies 
public class WebClientX : WebClient 
{ 
    public CookieContainer cookies = new CookieContainer(); 
    protected override WebRequest GetWebRequest(Uri location) 
    { 
     WebRequest req = base.GetWebRequest(location); 
     if (req is HttpWebRequest) 
      (req as HttpWebRequest).CookieContainer = cookies; 
     return req; 
    } 
    protected override WebResponse GetWebResponse(WebRequest request) 
    { 
     WebResponse res = base.GetWebResponse(request); 
     if (res is HttpWebResponse) 
      cookies.Add((res as HttpWebResponse).Cookies); 
     return res; 
    } 
} 
#endregion 

使用FireBug之類的瀏覽器插件或Chrome中內置的開發工具來獲取提交表單時發送的HTTP POST數據。使用WebClientX類發送這些POST並分析響應HTML。

當你已經知道格式正在使用一個簡單的Regex.Match時，解析HTML的最快方法是。因此，您應該使用開發工具瀏覽您的瀏覽器中的操作，記錄您的POST，URL和HTML內容，然後使用WebClientX執行相同的任務。

Answer 2

好的，這裏是完整的代碼登錄到一個頁面，然後登錄後從第二頁讀取。

class Program 
     { 
      static void Main(string[] args) 
      { 

       string uriString = "http://www.remotesite.com/login.cfm"; 

       // Create a new WebClient instance. 
       WebClientX myWebClient = new WebClientX(); 

       // Create a new NameValueCollection instance to hold some custom parameters to be posted to the URL. 
       NameValueCollection myNameValueCollection = new NameValueCollection(); 

       // Add necessary parameter/value pairs to the name/value container. 
       myNameValueCollection.Add("userid", "myname"); 
       myNameValueCollection.Add("mypassword", "mypassword"); 

       Console.WriteLine("\nUploading to {0} ...", uriString); 
       // 'The Upload(String,NameValueCollection)' implicitly method sets HTTP POST as the request method.    
       byte[] responseArray = myWebClient.UploadValues(uriString, myNameValueCollection); 

       // Decode and display the response. 
       Console.WriteLine("\nResponse received was :\n{0}", Encoding.ASCII.GetString(responseArray)); 

       Console.WriteLine("\n\n\n pausing..."); 
       Console.ReadKey(); 

       // Go to 2nd page on the site to get additional data 
       Stream myStream = myWebClient.OpenRead("https://www.remotesite.com/status_results.cfm?t=8&prog=d"); 

       Console.WriteLine("\nDisplaying Data :\n"); 
       StreamReader sr = new StreamReader(myStream); 
       StringBuilder sb = new StringBuilder(); 

       using (StreamReader reader = new StreamReader(myStream, System.Text.Encoding.UTF8)) 
       { 
        string line; 
        while ((line = reader.ReadLine()) != null) 
        { 
         sb.Append(line + "\r\n"); 
        } 
       } 

       using (StreamWriter outfile = new StreamWriter(@"Logfile1.txt")) 
       { 
        outfile.Write(sb.ToString()); 
       } 

       Console.WriteLine(sb.ToString()); 

       Console.WriteLine("\n\n\n pausing..."); 
       Console.ReadKey(); 

      } 

     } 

     public class WebClientX : WebClient 
     { 
      public CookieContainer cookies = new CookieContainer(); 
      protected override WebRequest GetWebRequest(Uri location) 
      // public override WebRequest GetWebRequest(Uri location) 
      { 
       WebRequest req = base.GetWebRequest(location); 
       if (req is HttpWebRequest) 
        (req as HttpWebRequest).CookieContainer = cookies; 
       return req; 
      } 
      protected override WebResponse GetWebResponse(WebRequest request) 
      { 
       WebResponse res = base.GetWebResponse(request); 
       if (res is HttpWebResponse) 
        cookies.Add((res as HttpWebResponse).Cookies); 
       return res; 
      } 
     }