將3mill的數據遷移到MongoDB

Question

我試圖將mysql數據移動到MongoDB。該記錄超過300萬行。並且每次修改我的節點啓動腳本後，內存用完了

"scripts": { 
    "debug": "node debug app.js", 
    "start": "node --max_old_space_size=5120 --optimize_for_size --max_executable_size=5120 --stack_size=5120 app.js", 
    "test": "node ./node_modules/mocha/bin/mocha test/bootstrap.test.js " 
} 

我試過使用普通的回調函數並承諾，但仍然沒有任何結果。

整體功能

var citiesInit = function(_setup) { 
    var connection = mysql.createConnection({ 
    host  : 'localhost', 
    user  : 'root', 
    password : 'password', 
    database : 'WorldCities' 
    }); 
    connection.connect(); 
    connection.query('SELECT * FROM thecities', function(err, rows, fields){ 
    if (err) { 
     console.log("The error: ", err); 
    } 
    var cityData = []; 

    rows.forEach(function(theCity){ 
     var data = { 
     name: theCity.city, 
     state_code: theCity.region, 
     country_code: theCity.country, 
     population: theCity.population, 
     location: { 
      type: "Point", 
      coordinates: [parseFloat(parseFloat(theCity.longitude).toFixed(4)), parseFloat(parseFloat(theCity.latitude).toFixed(4))] 
     }, 
     max_latitude: parseFloat(parseFloat(theCity.latitude).toFixed(4)), 
     max_longitude: parseFloat(parseFloat(theCity.longitude).toFixed(4)), 
     min_latitude: parseFloat(parseFloat(theCity.latitude).toFixed(4)), 
     min_longitude: parseFloat(parseFloat(theCity.longitude).toFixed(4)) 
     } 

     var cityCreate = City.create(data); 
     cityData.push(cityCreate); 
    }) 

    console.log('The Saved row is: ', cityData.length); 

    //With Promise 
    Promise.all([cityData]).spread(function (cityData){ 
     if (cityData) { 
     console.log(cityData.length); 
     Setup.findOrCreate({slug: 'city', setup: true}, {name: 'City', slug: 'city', setup: true}).exec(function cb(err, setup){ 
      console.log("Cities initialized successfully"); 
     }); 
     } 
    }) 
    .catch(function (err){ 
     console.log(err); 
    }) 

    //Without Promise 
    City.create(cityData).exec(function cityCB(err, cities){ 
     if (err) { 
     console.log(err); 
     } 

     if (cities.length) { 
     console.log(cities.length); 
     Setup.findOrCreate({slug: 'city', setup: true}, {name: 'City', slug: 'city', setup: true}).exec(function cb(err, setup){ 
      console.log("Cities initialized successfully"); 
     }); 
     } 
    }) 
    }); 
}; 

我使用Streams太

connection.query('SELECT * FROM thecities') 
    .stream() 
    .pipe(stream.Transform({ 
     objectMode: true, 
     transform:function(data, encoding, callback){ 
     var data = { 
      name: data.city, 
      state_code: data.region, 
      country_code: data.country, 
      population: data.population, 
      location: { 
       type: "Point", 
       coordinates: [parseFloat(parseFloat(data.longitude).toFixed(4)), parseFloat(parseFloat(data.latitude).toFixed(4))] 
      }, 
      max_latitude: parseFloat(parseFloat(data.latitude).toFixed(4)), 
      max_longitude: parseFloat(parseFloat(data.longitude).toFixed(4)), 
      min_latitude: parseFloat(parseFloat(data.latitude).toFixed(4)), 
      min_longitude: parseFloat(parseFloat(data.longitude).toFixed(4)) 
     } 

     City.create(data).exec(function cityCB(err, city){ 
      if (err) { 
      console.log(err); 
      } 

      //if (city.state) { console.log(city.state) } 
      callback(); 
     }) 
     } 
    })) 
    .on('finish', function(){ 
     connection.end(); 
     Setup.findOrCreate({slug: 'city', setup: true}, {name: 'City', slug: 'city', setup: true}).exec(function cb(err, setup){ 
      console.log("Cities initialized successfully"); 
      organizationInit(); 
     }); 
    }) 

上如何去任何幫助試過嗎？謝謝。

Answer 1

MySQL支持光標分批取少量的查詢，以節省內存：

• https://dev.mysql.com/doc/refman/5.7/en/declare-cursor.html
• https://dev.mysql.com/doc/refman/5.7/en/fetch.html

請參閱How to loop through a table using a cursor in MySQL?