мы выходим из памяти (процесс go запускается и потребляет% 100 оперативной памяти), пытаясь выбрать данные и вставить их в базу данных SQLite, программу, написанную на java, и данные настолько велики, что мы даже сделал для него разбиение на страницы, он выбирает 100000 строк и вставляет их в базу данных SQLite, чтобы выяснить проблему, мы закомментировали всю строку из кода, который вставляет данные, и мы увидели, что потребление останова оперативной памяти составляет% 6
package dbex;
import java.sql.*;
import java.util.ArrayList;
public class DBEX {
String url = "jdbc:oracle:thin:@192.168.120.46:1521:db";
String user = "dbuser";
String password = "dbpass";
ArrayList<Table> Tales = new ArrayList<>();
public static void main(String[] args) {
DBEX db = new DBEX();
db.CreateDB();
}
public int CountRecords(String table) {
int result = 0;
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getCountCon = DriverManager.getConnection(url, user, password);
Statement CountSt = getCountCon.createStatement();
ResultSet countRs = CountSt.executeQuery("SELECT COUNT(*) as num FROM " + table);
while (countRs.next()) {
result = countRs.getInt("num");
}
} catch (Exception ex) {
}
return result;
}
public void dumpData() {
try {
int incrementRecord = 100000;
Class.forName("org.sqlite.JDBC");
Class.forName("oracle.jdbc.driver.OracleDriver");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Table table = null;
ArrayList<String> columns = new ArrayList<>();
Connection getDataCon = DriverManager.getConnection(url, user, password);
getDataCon.setAutoCommit(false);
Statement dataSt = getDataCon.createStatement();
dataSt.setFetchSize(incrementRecord);
ResultSet dataRs = null;
Connection LiteInsertCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
LiteInsertCon.setAutoCommit(false);
PreparedStatement insertStatement = null;
int numRecord = 0;
for (int i = 0; i < Tales.size(); i++) {
table = Tales.get(i);
String tableName = table.name;
System.out.println("table::" + tableName);
System.out.println("Dumping data from " + tableName);
String InsertSQL = "INSERT INTO " + tableName + " ( ";
String vals = "VALUES (";
String selectSQL = "SELECT * FROM ( SELECT b.*, ROWNUM RN FROM ( SELECT ";
System.out.println("geting column name...");
columns = table.Columns;
for (int j = 0; j < columns.size(); j++) {
String columnName = columns.get(j);
if (j == 0) {
vals += " ? ";
InsertSQL += columnName + " ";
selectSQL += columnName + " ";
}else {
InsertSQL += " , " + columnName + " ";
vals += ", ? ";
selectSQL += " , " + columnName + " ";
}
}
System.out.println("Number of column of table " + tableName + " is " + columns.size());
vals += ")";
InsertSQL += ")" + vals;
selectSQL += "FROM " + tableName + " ORDER BY "+table.primary_key+" ASC ) b WHERE ROWNUM <= :TO ) WHERE RN > :FROM ";
System.out.println(selectSQL);
System.out.println(InsertSQL);
int parIndex = 0;
String colName = null;
String data = null;
numRecord = CountRecords(tableName);
int from = 0;
int to = incrementRecord;
if (to > numRecord) {
System.out.println("Table data is less than "+incrementRecord+" geting all data "+numRecord+" " );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", numRecord + "").replace(":FROM", "0"));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
} else {
while (numRecord >= to) {
System.out.println("page "+(to/incrementRecord)+" of "+(numRecord/incrementRecord) );
dataRs = dataSt.executeQuery(selectSQL.replace(":TO", to + "").replace(":FROM", from+""));
while (dataRs.next()) {
insertStatement = LiteInsertCon.prepareStatement(InsertSQL);
insertStatement.setFetchSize(incrementRecord);
for (; parIndex < columns.size(); parIndex++) {
colName = columns.get(parIndex);
data = dataRs.getString(colName);
if (data != null) {
insertStatement.setString(parIndex + 1, data);
} else {
insertStatement.setString(parIndex + 1, " ");
}
parIndex++;
}
parIndex = 0;
insertStatement.executeUpdate();
insertStatement = null;
}
dataRs=null;
from = to;
to += incrementRecord;
System.gc();
}
}
}
} catch (Exception ex) {
System.out.println(ex.toString());
}
}
class Table {
public String name;
public String primary_key;
ArrayList< String> Columns = new ArrayList<>();
public Table(String name, String primary_key, ArrayList< String> Columns) {
this.name = name;
this.primary_key = primary_key;
this.Columns = Columns;
}
}
public void CreateDB() {
try {
Class.forName("oracle.jdbc.driver.OracleDriver");
System.out.println("Oracle JDBC driver found");
DriverManager.registerDriver(new oracle.jdbc.driver.OracleDriver());
Connection getTableCon = DriverManager.getConnection(url, user, password);
Connection getColumnCon = DriverManager.getConnection(url, user, password);
Connection getKeyCon = DriverManager.getConnection(url, user, password);
System.out.println("connected to database");
Statement getTableSt = getTableCon.createStatement();
PreparedStatement getColumnSt = getColumnCon.prepareStatement("SELECT DISTINCT column_name FROM all_tab_cols WHERE table_name = ? ");
PreparedStatement getKeySt = getKeyCon.prepareStatement("SELECT cols.table_name, cols.column_name, cols.position, cons.status, cons.owner FROM all_constraints cons, all_cons_columns cols WHERE cols.table_name = ? AND cons.constraint_type = 'P' AND cons.constraint_name = cols.constraint_name AND cons.owner = cols.owner ORDER BY cols.table_name, cols.position");
ResultSet getTableRs = getTableSt.executeQuery("SELECT * FROM all_tables WHERE OWNER not in('SYS','OUTLN','SYSTEM')");
ResultSet getColumnRs;
ResultSet getKeyRs;
Class.forName("org.sqlite.JDBC");
System.out.println("Sqlite JDBC driver found");
Connection LiteCreateTableCon = null;
LiteCreateTableCon = DriverManager.getConnection("jdbc:sqlite:" + "DBEX" + ".db");
Statement statement = LiteCreateTableCon.createStatement();
String keyName = "";
while (getTableRs.next()) {
String tableName = getTableRs.getString("table_name");
System.out.println("table: " + tableName);
getColumnSt.setString(1, tableName);
System.out.println("SELECT * FROM all_tab_cols WHERE table_name = " + tableName + "");
getColumnRs = getColumnSt.executeQuery();
String createTableSQL = "create table if not exists " + tableName + " ( ";
int index = 0;
ArrayList<String> Columns = new ArrayList<>();
while (getColumnRs.next()){
String columnName = getColumnRs.getString("column_name");
Columns.add(columnName);
System.out.println("column: " + columnName);
if (index == 0) {
createTableSQL += columnName + " string";
} else {
createTableSQL += " , " + columnName + " string";
}
index++;
}
createTableSQL += ")";
System.out.println(createTableSQL);
statement.executeUpdate(createTableSQL);
getKeySt.setString(1, tableName);
getKeyRs = getKeySt.executeQuery();
keyName = "1";
while (getKeyRs.next()) {
keyName = getKeyRs.getString("column_name");
}
Tales.add(new Table(tableName, keyName, Columns));
}
getTableCon.close();
getColumnCon.close();
getKeyCon.close();
System.out.println("All table created...");
}catch (Exception ex) {
System.out.println(ex.toString());
}
dumpData();
}
}