2012-07-26 107 views
0

我有安排在以下虛擬目錄S3存儲多種數據文件(CSV):獲取所有桶的內容對象

s3://mybucketname/data/Type1/ 

桶名稱:mybucketname
關鍵字前綴:/數據/類型1/

,我想將這些csv文件的內容與此存儲桶+鍵 合併,並以字符串形式返回輸出。代碼 -

public string GetBucketContents(string bucketName) 
{ 
    string bucketContent = ""; 
    ListObjectsRequest request = new ListObjectsRequest 
    { 
     BucketName = bucketName, 
     Prefix = "data/Type1/" 
    }; 

    using (ListObjectsResponse response = amazonS3Client.ListObjects(request)) 
    { 
     //Loop all objects in S3 bucket 
     foreach (S3Object s3Object in response.S3Objects) 
     { 
      //Get S3 object @ s3Object.Key 
      GetObjectRequest objectRequest = new GetObjectRequest().WithBucketName(bucketName).WithKey(s3Object.Key);     
      using (GetObjectResponse objectResponse = amazonS3Client.GetObject(objectRequest)) 
      { 
       using (StreamReader reader = new StreamReader(objectResponse.ResponseStream)) 
       { 
        bucketContent += reader.ReadToEnd(); 
       } 
      } 
     } 
    } 

    return bucketContent; 
} 

我收到預期:-)
輸出但是還是想確認是否有任何更好的(性能明智)的方式來實現這一目標?

謝謝!

回答

0

如果您使用的是.NET的一個較新版本的支持aync並等待ajaxy魔法,那麼你可以立刻解僱所有的讀取請求關閉,然後等待他們都返回來構建你的字符串。在大數據讀取時,這可以提高性能並減少大量時間。

下面是一些代碼:(你需要修改字符串以滿足您的目的,擺脫秒錶,但已經包含,所以你可以看到的時間差)

public string GetBucketContents(string bucketname) 

    string prefix = "S3/PathToCSVs/"; 
    StringBuilder sb = new StringBuilder(); 

    List<Task<string>> tasks = new List<Task<string>>(); 

    Stopwatch sw = new Stopwatch(); 

    // Begin timing 
    sw.Start(); 

    using (client = new AmazonS3Client(AWSAccessKey, AWSSecretKey, Amazon.RegionEndpoint.USWest2)) 
    { 
     try 
     { 
      ListObjectsRequest request = new ListObjectsRequest 
      { 
       BucketName = bucketname, 
       Prefix = prefix 
      }; 

      do 
      { 
       ListObjectsResponse response = client.ListObjects(request); 


       // Process response. 
       foreach (S3Object entry in response.S3Objects) 
       { 
        GetObjectRequest gor = new GetObjectRequest 
        { 
         BucketName = bucketname, 
         Key = entry.Key 
        }; 


        tasks.Add(Task<string>.Factory.StartNew(() => 
        { 
         return ProcessResults(gor, sw); 
        })); 

       } 

       // If response is truncated, set the marker to get the next 
       // set of keys. 
       if (response.IsTruncated) 
       { 
        request.Marker = response.NextMarker; 
       } 
       else 
       { 
        request = null; 
       } 
      } while (request != null); 


      Task.WaitAll(tasks.ToArray()); 

      foreach (var s in tasks) 
      { 
       sb.AppendLine(sw.Elapsed.ToString() + "<br/>"); 
       sb.AppendLine(s.Result + "<br/>"); 
      } 


      // Stop timing 
      sw.Stop(); 
      sb.AppendLine(sw.Elapsed.ToString() + "<br/>"); 

      return sb.ToString(); 

     } 
     catch (AmazonS3Exception amazonS3Exception) 
     { 
      if (amazonS3Exception.ErrorCode != null && (amazonS3Exception.ErrorCode.Equals("InvalidAccessKeyId") || amazonS3Exception.ErrorCode.Equals("InvalidSecurity"))) 
      { 
       return "Check the provided AWS Credentials. To sign up for service, go to http://aws.amazon.com/s3"; 
      } 
      else 
      { 
       return string.Format("Error occurred. Message:'{0}' when listing objects", amazonS3Exception.Message); 
      } 
     } 
    } 
} 

public static string ProcessResults(GetObjectRequest gor, Stopwatch sw) 
{ 
    string result = ""; 

    using (GetObjectResponse goresp = client.GetObject(gor)) 
    { 
     using (StreamReader reader = new StreamReader(goresp.ResponseStream)) 
     { 
      result = sw.Elapsed.ToString() + "<br/>" + reader.ReadToEnd() + "<br/>"; 
     } 
    } 

    return result; 
}