0
我正在解析xml技術，並決定在DOM解析器上使用SAX。這些數據，每個數據庫都有近6KB的數百萬個xml文件。我正在使用SAXparser。解析數以百萬計的XML文件--Java
我遍歷所有的文件調用parser.parse（文件句柄）一個接一個，但10萬後，我得到一個堆內存不足的錯誤。當我試圖轉儲堆並讀取它時，我看到很多char數組和字符串被存儲。
的問題是，我該如何在數百萬的小文件解析堆沒有錯誤。
import javax.xml.parsers.*;
import org.xml.sax.*;
import org.xml.sax.helpers.*;
import java.util.*;
import java.io.*;
import java.util.logging.Level;
import java.util.logging.Logger;
/**
*
* @author Ajinkya Jumbad
*/
public class dataset {
static List<String> cols;
public HashMap<String, HashMap> hm = new HashMap<>();
static int i =0;
dataset() {
String coln[] = {
"UID",
"Name",
"NationID",
"Born",
"Age",
"IntCaps",
"IntGoals",
"U21Caps",
"U21Goals",
"Height",
"Weight",
"AerialAbility",
"CommandOfArea",
"Communication",
"Eccentricity",
"Handling",
"Kicking",
"OneOnOnes",
"Reflexes",
"RushingOut",
"TendencyToPunch",
"Throwing",
"Corners",
"Crossing",
"Dribbling",
"Finishing",
"FirstTouch",
"Freekicks",
"Heading",
"LongShots",
"Longthrows",
"Marking",
"Passing",
"PenaltyTaking",
"Tackling",
"Technique",
"Aggression",
"Anticipation",
"Bravery",
"Composure",
"Concentration",
"Vision",
"Decisions",
"Determination",
"Flair",
"Leadership",
"OffTheBall",
"Positioning",
"Teamwork",
"Workrate",
"Acceleration",
"Agility",
"Balance",
"Jumping",
"LeftFoot",
"NaturalFitness",
"Pace",
"RightFoot",
"Stamina",
"Strength",
"Consistency",
"Dirtiness",
"ImportantMatches",
"InjuryProness",
"Versatility",
"Adaptability",
"Ambition",
"Loyalty",
"Pressure",
"Professional",
"Sportsmanship",
"Temperament",
"Controversy",
"PositionsDesc",
"Goalkeeper",
"Sweeper",
"Striker",
"AttackingMidCentral",
"AttackingMidLeft",
"AttackingMidRight",
"DefenderCentral",
"DefenderLeft",
"DefenderRight",
"DefensiveMidfielder",
"MidfielderCentral",
"MidfielderLeft",
"MidfielderRight",
"WingBackLeft",
"WingBackRight"};
cols = Arrays.asList(coln);
try {
File f = new File("C:\\Users\\Ajinkya Jumbad\\Desktop\\fmdata");
//File files[] = f.listFiles();
for (File file : f.listFiles()) {
//System.out.println(file.getAbsolutePath());
if (file.isFile()) {
parse p = new parse(file);
}
}
//savefile();
} catch (Exception ex) {
Logger.getLogger(dataset.class.getName()).log(Level.SEVERE, null, ex);
}
}
private void savefile() {
try {
String file_name = "dataset.csv";
FileWriter w = new FileWriter(file_name);
writecsv ws = new writecsv();
boolean first = true;
StringBuilder sb = new StringBuilder();
for (String key : cols) {
if (!first) {
sb.append(",");
}
sb.append(key);
first = false;
}
sb.append("\n");
w.append(sb.toString());
for (String uid : hm.keySet()) {
ws.writeLine(w, hm.get(uid));
}
w.close();
} catch (Exception e) {
System.out.println(e.getMessage());
}
}
public class parse{
parse(File file){
try {
SAXParserFactory parserfac = SAXParserFactory.newInstance();
parserfac.setNamespaceAware(true);
SAXParser parser = parserfac.newSAXParser();
DefaultHandler handler = new DefaultHandler(){
HashMap<String, String> ht;
@Override
public void startDocument() {
ht = new HashMap<>();
}
@Override
public void startElement(String namespaceURI,
String localName,
String qName,
Attributes atts) {
if (atts.getValue("Value") != null && cols.contains(localName)) {
//System.out.println(localName);
String key = localName;
ht.put(key, atts.getValue("Value"));
}
}
@Override
public void endDocument() {
String uid = ht.get("UID");
hm.put(uid, ht);
dataset.i += 1;
if(dataset.i%100 == 0){
System.out.println(dataset.i);
}
}
@Override
public void characters(char ch[], int start, int length) throws SAXException {
}
};
parser.parse(file, handler);
} catch (Exception ex) {
Logger.getLogger(dataset.class.getName()).log(Level.SEVERE, null, ex);
}
}
}
public static void main(String[] args) {
dataset ds = new dataset();
}
}
那麼你的問題是什麼？ – Tavo
沒有看你的代碼，很難幫助你。一個猜測就是你繼續打開這些文件，但你永遠不會關閉它們並釋放資源。 – alfasin
您是否曾嘗試使用DOM解析器，因爲您說您是按文件解析文件，每個文件只有6KB？嘗試關閉XML連接以釋放內存資源 – harshavmb