服務器崩潰時，使用NodeJS從url下載大文件

Question

我試圖從使用來自nodejs的請求模塊的url（需要登錄）下載文件（+ 200mb），但是當它完成下載時服務器開始減速直到崩潰或變得非常緩慢。

這裏是我當前的代碼（它下載整個文件，但最終我的服務器崩潰）：

//Required modules 
var http = require('http'), 
url = require("url"), 
fs = require('fs'), 
request = require('request'), 
path = require("path"), 
events = require("events"), 
j = request.jar(), 
request = request.defaults({ jar : j }); 

// make te request login in with cookies 
console.log("downloading file :)"); 
request({ 
    url:"http://example.com/", 
    method:"POST", 
    form:{u: "username",p: "password"} 
    }, 
    function(error,response,body){ 
     setTimeout(function(){ 
      request 
       .get('http://example.com/test.ashx?file=15') 
       .on('error', function(err) { 
       console.log(err); 
       }) 
       .pipe(fs.createWriteStream("/var/www/filesDir/CustomName.zip")); 
       console.log(body); 
     },1000) 
    } 
); 

我試着從this answer應用另一種解決方案，但由於某些原因，文件沒有正確下載，它總是隻顯示「下載進度：0字節」，可能與登錄訪問有關。

這裏我把其他的代碼我試圖從最後一句來實現：

var http = require('http'); 
var fs = require('fs'); 
var url = require("url"); 
var request = require('request'); 
var path = require("path"); 
var events = require("events"); 
var j = request.jar(); 
var request = request.defaults({ jar : j }); 


request({ 
    url:"http://example.com/", 
    method:"POST", 
    form:{u:"username",p:"password"} 
    }, function(error,response,body){ 
       var downloadfile = "http://example.com/test.ashx?file=15"; 
       var host = url.parse(downloadfile).hostname; 
       var filename = "1977.zip"; 
       var req = http.request({port: 80, host: host, method: 'GET'}); 
       console.log("Downloading file: " + filename); 
       console.log("Before download request"); 
       req.end(); 

       dlprogress = 0; 

       setInterval(function() { 
        console.log("Download progress: " + dlprogress + " bytes"); 
       }, 1000); 

       req.addListener('response', function (response) { 
        var downloadfile = fs.createWriteStream(filename, {'flags': 'a'}); 
        console.log("File size " + filename + ": " + response.headers['content-length'] + " bytes."); 
        response.addListener('data', function (chunk) { 
         dlprogress += chunk.length; 
         downloadfile.write(chunk, encoding='binary'); 
        }); 
        response.addListener("end", function() { 
         downloadfile.end(); 
         console.log("Finished downloading " + filename); 
        }); 

       }); 
     } 
); 

不要緊，你決定要幫我用哪種方式。

Answer 1

我最後只是這樣，我測試的代碼多次，服務器也不再崩潰：

var request = require('request'); 
var filed = require('filed'); 
var j = request.jar(); 
var request = request.defaults({ jar : j }); 

// make the request and login 
request({ 
    url: "http://example.com/login", 
    method:"POST", 
    // 'u' and 'p' are the field names on the form 
    form:{u:"username",p:"password"} 
    }, function(error,response,body){ 
     setTimeout(function(){ 
      var downloadURL = 'http://example.com/download/file.zip'; 
      var downloadPath = "/path/to/download/localNameForFile.zip"; 

      var downloadFile = filed(downloadPath); 
      var r = request(downloadURL).pipe(downloadFile); 

      r.on('data', function(data) { 
       console.log('binary data received'); 
      }); 
      downloadFile.on('end', function() { 
       console.log(downloadPath, 'file downloaded to path'); 
      }); 

      downloadFile.on('error', function (err) { 
       console.log(err, 'error downloading file'); 
      }); 
     },3000) 
    } 
);