字符字符串從互聯網

Question

下載HTML使用下面的代碼改變後，我可以從互聯網上下載的文件的HTML：

WebClient wc = new WebClient(); 

// .... 

string downloadedFile = wc.DownloadString("http://www.myurl.com/"); 

但是，有時文件包含像é以「有趣」的字符é,←至â†和フシギダネ至フシギダãƒ。

我認爲這可能是是與不同的Unicode類型或東西，因爲每個字符被改爲2分新的，也許每個字符被劈成兩半，但我在這方面知之甚少。你認爲什麼是錯的？

Answer 1

下面是支持gzip和檢查編碼標題和meta標籤，以便將其正確地解碼包的下載類。

實例化類，並調用GetPage()。

public class HttpDownloader 
{ 
    private readonly string _referer; 
    private readonly string _userAgent; 

    public Encoding Encoding { get; set; } 
    public WebHeaderCollection Headers { get; set; } 
    public Uri Url { get; set; } 

    public HttpDownloader(string url, string referer, string userAgent) 
    { 
     Encoding = Encoding.GetEncoding("ISO-8859-1"); 
     Url = new Uri(url); // verify the uri 
     _userAgent = userAgent; 
     _referer = referer; 
    } 

    public string GetPage() 
    { 
     HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url); 
     if (!string.IsNullOrEmpty(_referer)) 
      request.Referer = _referer; 
     if (!string.IsNullOrEmpty(_userAgent)) 
      request.UserAgent = _userAgent; 

     request.Headers.Add(HttpRequestHeader.AcceptEncoding, "gzip,deflate"); 

     using (HttpWebResponse response = (HttpWebResponse)request.GetResponse()) 
     { 
      Headers = response.Headers; 
      Url = response.ResponseUri; 
      return ProcessContent(response); 
     } 

    } 

    private string ProcessContent(HttpWebResponse response) 
    { 
     SetEncodingFromHeader(response); 

     Stream s = response.GetResponseStream(); 
     if (response.ContentEncoding.ToLower().Contains("gzip")) 
      s = new GZipStream(s, CompressionMode.Decompress); 
     else if (response.ContentEncoding.ToLower().Contains("deflate")) 
      s = new DeflateStream(s, CompressionMode.Decompress); 

     MemoryStream memStream = new MemoryStream(); 
     int bytesRead; 
     byte[] buffer = new byte[0x1000]; 
     for (bytesRead = s.Read(buffer, 0, buffer.Length); bytesRead > 0; bytesRead = s.Read(buffer, 0, buffer.Length)) 
     { 
      memStream.Write(buffer, 0, bytesRead); 
     } 
     s.Close(); 
     string html; 
     memStream.Position = 0; 
     using (StreamReader r = new StreamReader(memStream, Encoding)) 
     { 
      html = r.ReadToEnd().Trim(); 
      html = CheckMetaCharSetAndReEncode(memStream, html); 
     }    

     return html; 
    } 

    private void SetEncodingFromHeader(HttpWebResponse response) 
    { 
     string charset = null; 
     if (string.IsNullOrEmpty(response.CharacterSet)) 
     { 
      Match m = Regex.Match(response.ContentType, @";\s*charset\s*=\s*(?<charset>.*)", RegexOptions.IgnoreCase); 
      if (m.Success) 
      { 
       charset = m.Groups["charset"].Value.Trim(new[] { '\'', '"' }); 
      } 
     } 
     else 
     { 
      charset = response.CharacterSet; 
     } 
     if (!string.IsNullOrEmpty(charset)) 
     { 
      try 
      { 
       Encoding = Encoding.GetEncoding(charset); 
      } 
      catch (ArgumentException) 
      { 
      } 
     } 
    } 

    private string CheckMetaCharSetAndReEncode(Stream memStream, string html) 
    { 
     Match m = new Regex(@"<meta\s+.*?charset\s*=\s*""?(?<charset>[A-Za-z0-9_-]+)""?", RegexOptions.Singleline | RegexOptions.IgnoreCase).Match(html);    
     if (m.Success) 
     { 
      string charset = m.Groups["charset"].Value.ToLower() ?? "iso-8859-1"; 
      if ((charset == "unicode") || (charset == "utf-16")) 
      { 
       charset = "utf-8"; 
      } 

      try 
      { 
       Encoding metaEncoding = Encoding.GetEncoding(charset); 
       if (Encoding != metaEncoding) 
       { 
        memStream.Position = 0L; 
        StreamReader recodeReader = new StreamReader(memStream, metaEncoding); 
        html = recodeReader.ReadToEnd().Trim(); 
        recodeReader.Close(); 
       } 
      } 
      catch (ArgumentException) 
      { 
      } 
     } 

     return html; 
    } 
} 

Answer 2

試試這個

string downloadedFile = wc.DownloadString("http://www.myurl.com"); 

我永諾刪除最後一個「溼地」，它現在的工作，直到就像一個魅力。但我可能是也危害

Answer 3

因爲我不能發表評論（口碑不足），我會發佈一個附加的應答。我常規使用Mikael的優秀課程，但是我遇到了一個嘗試查找字符集元信息的正則表達式的實際問題。這

Match m = new Regex(@"<meta\s+.*?charset\s*=\s*(?<charset>[A-Za-z0-9_-]+)", RegexOptions.Singleline | RegexOptions.IgnoreCase).Match(html); 

失敗這個

<meta charset="UTF-8"/> 

，而這

Match m = new Regex(@"<meta\s+.*?charset\s*=\s*""?(?<charset>[A-Za-z0-9_-]+)""?", RegexOptions.Singleline | RegexOptions.IgnoreCase).Match(html); 

沒有。

謝謝Mikael。