超過1個lac數據的內存不足異常

Question

您好我正面臨着在兩列中插入1lac數據的問題（Id & Id）。任何人都可以優化代碼。

public class edgeConnection { 
static ArrayList al3 = new ArrayList(); 
static HashSet set=null; 
//static HashMap hm = null; 
//static int val ; 
//Database connection 
public static DataSource getMySQLDataSource() throws Exception { 
    Properties props = new Properties(); 
    FileInputStream fis = null; 
    MysqlDataSource mysqlDS = null; 

    try { 
     fis = new FileInputStream("D:/Assignments/Sequence/db.properties"); 
    } catch (FileNotFoundException e) { 
     // TODO Auto-generated catch block 
     e.printStackTrace(); 
    } 
    props.load(fis); 
    mysqlDS = new MysqlDataSource(); 
    mysqlDS.setURL(props.getProperty("MYSQL_DB_URL")); 
    mysqlDS.setUser(props.getProperty("MYSQL_DB_USERNAME")); 
    mysqlDS.setPassword(props.getProperty("MYSQL_DB_PASSWORD")); 
    return mysqlDS; 
} 

//Adding values to Hashset 
private static int addNode(){ 
    set = new HashSet(); 
    for(int i=1;i<=10000;i++){/*Change 10000 to 30000*/ 
     set.add(i); 
    } 
    return 0; 
} 
private static int keyNode(int i){ 
    int counter = 1; 
    Iterator it = set.iterator(); 
    while(it.hasNext()) 
    { 
     int value = (int) it.next(); 
     if(i==counter) 
     { 
      //System.out.println("key value returned ::"+value); 
      return value; 
     } 
     counter++; 
    } 
    return 0; 
} 
private static String pairGenerator(){ 
    ArrayList numbers1 = new ArrayList<Integer>(); 
    Random randomGenerator1 = new Random(); 
    while (numbers1.size() < 1) 
    { 
     int random = randomGenerator1 .nextInt(15); 
     if (!numbers1.contains(random)) { 
      numbers1.add(random); 
     } 
    } 
    Iterator it1 = numbers1.iterator(); 
    while(it1.hasNext()){ 
     return(String.valueOf(it1.next())); 
    } 
    return null; 
} 

private static List valueNodes(){ 
    //Generate no randomly. 
    ArrayList<Integer> numbers = new ArrayList<Integer>(); 
    Random randomGenerator = new Random(); 
    String size = pairGenerator(); 
    int size1= Integer.parseInt(size)+1; 
    //System.out.println("the size1 is ::"+size1); 
    while (numbers.size() < size1) 
    { 
     int random = randomGenerator .nextInt(10000);/*Change 10000 to 50000*/ 
     if (!numbers.contains(random)) { 
      numbers.add(random); 
     } 
    } 
    Iterator it = numbers.iterator(); 
    al3.clear(); 

    while(it.hasNext()){ 
     int listvalue = (int) it.next(); 
     al3.add(listvalue); 

     //System.out.println(it.next()); 
    } 
    //System.out.println(al3); 
    return al3; 
} 
public static void main(String[] args) throws Exception { 
    Connection con = null; 
    PreparedStatement pst = null; 
    ResultSet rs = null; 
    HashMap<Integer, List<String>> hm = new HashMap<Integer, List<String>>(); 
    addNode(); 
    //System.out.println("size of set is:"+set.size()); 
    try { 
     con = getMySQLDataSource().getConnection(); 
     List<Integer> valueList = new ArrayList<Integer>(); 
     int nodeId; 
     for(int i=1;i<=set.size();i++) 
     { 
      hm.put(keyNode(i), valueNodes()); 
      Iterator iter = hm.entrySet().iterator(); 
      while(iter.hasNext()) 
      { 
       Map.Entry entry = (Map.Entry) iter.next(); 
       System.out.println(entry.getKey()+"<-->"+" "+entry.getValue()); 
       nodeId = (int) entry.getKey(); 
       valueList = (List<Integer>) entry.getValue(); 
       //System.out.println("size of value list : "+valueList.size()); 
       for(int j = 0;j<valueList.size();j++) 
       { 
        pst = con.prepareStatement("insert into nodes_connection values (?,?)"); 
        pst.setInt(1, nodeId); 
        if(valueList.get(j)!=0) 
        { 
         pst.setInt(2,valueList.get(j)); 
        } 
        else{ 
         int updatedValue = valueList.get(j)+10000;/*Change 10000 to 30000*/ 
         pst.setInt(2,updatedValue); 
        } 
        pst.executeUpdate(); 
        //System.out.println(j+"record updated.."); 
       } 
       iter.remove(); 
      } 
     } 
     System.out.println("Record successfully added"); 
    } catch (SQLException e) { 
     e.printStackTrace(); 
    }finally{ 
     try { 
      if(rs != null) rs.close(); 
      if(pst != null) pst.close(); 
      if(con != null) con.close(); 
     } catch (SQLException e) { 
      e.printStackTrace(); 
     } 
    } 


}` 

我需要刪除arraylist和hashmap。我怎麼能優化代碼，以便我不會得到任何Java堆內存異常。

Answer 1

由於您插入的行很多，因此您應該使用批量更新，而不是一次插入一行。

PreparedStatement pst = con.prepareStatement("insert into nodes_connection values (?,?)"); 
for(int i=1;i<=set.size();i++) 
    { 
     hm.put(keyNode(i), valueNodes()); 
     Iterator iter = hm.entrySet().iterator(); 
     while(iter.hasNext()) 
     { 
      Map.Entry entry = (Map.Entry) iter.next(); 
      System.out.println(entry.getKey()+"<-->"+" "+entry.getValue()); 
      nodeId = (int) entry.getKey(); 
      valueList = (List<Integer>) entry.getValue(); 
      //System.out.println("size of value list : "+valueList.size()); 
      for(int j = 0;j<valueList.size();j++) 
      { 

       pst.setInt(1, nodeId); 
       if(valueList.get(j)!=0) 
       { 
        pst.setInt(2,valueList.get(j)); 
       } 
       else{ 
        int updatedValue = valueList.get(j)+10000;/*Change 10000 to 30000*/ 
        pst.setInt(2,updatedValue); 
       } 
       pst.addBatch() 
       //System.out.println(j+"record updated.."); 
      } 
      iter.remove(); 
     } 
    } 


    pst.executeBatch() 

你可以去here瞭解批量插入的更多信息。

Answer 2

無論你做什麼最簡單的事情就是increase the heap size。

如果這是你只會運行一次，或者四處玩耍，基本上任何不是生產關鍵或處理大量數據的東西，那麼增加堆將會給你你想要的。

如果您確實需要保留內存佔用，您將需要流式傳輸數據，而不是將其全部讀取到內存中。尋找你的代碼似乎每行數據都沒有任何關係的文件中的數據的其餘部分，因此在僞代碼的方式，像下面將工作：

For each line in file 
    Calculate data to be inserted into database 
    Update database 

您可以提高工作效率更通過代替每行更新數據庫，批量更新，這將改變你的僞代碼：

For each line in file 
    Calculate data to be inserted into database 
    Add update to a JDBC batch 
    If batch size > :somelimit 
     execute batch 
Execute final batch