Loading large text files into java vectors and outof memmory
Hi there,
need your help for the following:
i'm trying to load large ammoubnts of data into a Vector in order to concatenate several text files and treat them, but i'm getting outofmemory error. I even tried using xml structure and saving to database but the error is still the same. Can you help?
thanks
here's the code:
publicvoid Concatenate(){
try{
//for(int i=0;i<1;i++) {
vEntries =new Vector();
for(int i=0;i<BopFiles.length;i++){
MainPanel.WriteLog("reading file " + BopFiles[i] +"...");
FileInputStream fis =new FileInputStream(BopFiles[i]);
BufferedInputStream bis =new BufferedInputStream(fis);
DataInputStream in =new DataInputStream(bis);
String line = in.readLine();
Database db =new Database();
Connection conn = db.open();
while(line !=null){
DivideLine(BopFiles[i], line);
line = in.readLine();
}
FreeMemory(db, conn);
}
MainPanel.WriteLog("Num of elements: " + root.getChildNodes().getLength());
MainPanel.WriteLog("Done!");
}catch (Exception e){
e.printStackTrace();
}
}
publicvoid DivideLine(String file, String line){
if (line.toLowerCase().startsWith("00694")){
Header hd =new Header();
hd.headerFile = file;
hd.headerLine = line;
vHeaders.add(hd);
}elseif (line.toLowerCase().startsWith("10694")){
Line entry =new Line();
Vector vString =new Vector();
Vector vType =new Vector();
Vector vValue =new Vector();
entry.name = line.substring(45, 150).trim();
entry.number = line.substring(30, 45).trim();
entry.nif = line.substring(213, 222).trim();
entry.index=BopIndex;
entry.message=line;
entry.file=file;
String series = line.substring(252);
StringTokenizer st =new StringTokenizer(series,"A");
while (st.hasMoreTokens()){
String token=st.nextToken();
if(!token.startsWith(" ")){
vString.add(token);
vType.add(token.substring(2,4));
vValue.add(token.substring(4));
}
token=null;
}
entry.strings=new String[vString.size()];
vString.copyInto(entry.strings);
entry.types=new String[vType.size()];
vType.copyInto(entry.types);
entry.values=new String[vType.size()];
vValue.copyInto(entry.values);
vEntries.add(entry);
entry=null;
vString=null;
vType=null;
vValue=null;
st=null;
series=null;
line=null;
file=null;
MainPanel.SetCount(BopIndex);
BopIndex ++;
}
}
publicvoid FreeMemory(Database db, Connection conn){
try{
//db.update("CREATE TABLE entries (message VARCHAR(1000))");
db.update("DELETE FROM entries;");
PreparedStatement ps =null;
for(int i=0; i><vEntries.size(); i++ ){
Line entry = (Line) vEntries.get(i);
String value ="" + entry.message;
if(!value.equals("")){
try{
ps = conn.prepareStatement("INSERT INTO entries (message) VALUES('" + Tools.RemoveSingleQuote(value) +"');");
ps.execute();
}catch(Exception e){
e.printStackTrace();
System.out.println("error in number->" + i);
}
}
}
MainPanel.WriteLog("Releasing memory...");
vEntries =null;
vEntries =new Vector();
System.gc();
}catch (Exception e1){
e1.printStackTrace();
}
}
