|
@@ -6,8 +6,6 @@ import com.sun.xml.xsom.parser.XSOMParser;
|
|
import org.apache.commons.io.ByteOrderMark;
|
|
import org.apache.commons.io.ByteOrderMark;
|
|
import org.apache.commons.io.input.BOMInputStream;
|
|
import org.apache.commons.io.input.BOMInputStream;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
import org.apache.commons.lang3.StringUtils;
|
|
-import org.apache.commons.lang3.tuple.ImmutablePair;
|
|
|
|
-import org.apache.commons.lang3.tuple.Pair;
|
|
|
|
import org.apache.log4j.Logger;
|
|
import org.apache.log4j.Logger;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Autowired;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
import org.springframework.beans.factory.annotation.Value;
|
|
@@ -42,70 +40,6 @@ public class FiasSyncService {
|
|
@Autowired
|
|
@Autowired
|
|
private DataSource dataSource;
|
|
private DataSource dataSource;
|
|
|
|
|
|
- private class MyHandler extends DefaultHandler {
|
|
|
|
- private MyHandler(){}
|
|
|
|
-
|
|
|
|
- private long limit = 1999;
|
|
|
|
- private long count = 0;
|
|
|
|
- private long total = 0;
|
|
|
|
-
|
|
|
|
- private String tableName;
|
|
|
|
- private Map<String, String> cols;
|
|
|
|
- private Map<String, List<Object[]>> cache = new HashMap<>();
|
|
|
|
-
|
|
|
|
- MyHandler(String tableName, Map<String, String> cols){
|
|
|
|
- this.tableName = tableName;
|
|
|
|
- this.cols = cols;
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- private void flush(boolean end){
|
|
|
|
- if ((count > limit) || end) {
|
|
|
|
- for(String sql : cache.keySet())
|
|
|
|
- new JdbcTemplate(dataSource).batchUpdate(sql, cache.get(sql));
|
|
|
|
-
|
|
|
|
- log.info("flush "+ end + ", total "+ total);
|
|
|
|
- count = 0;
|
|
|
|
- cache.clear();
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- private void insert(Attributes attributes){
|
|
|
|
- String[] names = new String[attributes.getLength()];
|
|
|
|
- String[] values = new String[attributes.getLength()];
|
|
|
|
- String[] wildcards = new String[attributes.getLength()];
|
|
|
|
- for(int i = 0; i < attributes.getLength(); i++){
|
|
|
|
- names[i] = attributes.getLocalName(i);
|
|
|
|
- values[i] = attributes.getValue(i);
|
|
|
|
- wildcards[i] = "?";
|
|
|
|
- }
|
|
|
|
- String sql = "INSERT INTO fias." + tableName + "("+StringUtils.join(names, ',')+")" + "VALUES ("+StringUtils.join(wildcards, ',')+")";
|
|
|
|
- try {
|
|
|
|
- //new JdbcTemplate(dataSource).update(sql, values);
|
|
|
|
- if(!cache.containsKey(sql)) cache.put(sql, new ArrayList<>());
|
|
|
|
- cache.get(sql).add(values);
|
|
|
|
- count++;
|
|
|
|
- total++;
|
|
|
|
- flush(false);
|
|
|
|
- } catch (Exception e){
|
|
|
|
- e.printStackTrace();
|
|
|
|
- throw new RuntimeException(e);
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- @Override
|
|
|
|
- public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
|
|
|
|
- if (attributes.getLength() > 0){
|
|
|
|
- if (cols.containsKey(attributes.getLocalName(0)))
|
|
|
|
- insert(attributes);
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
- @Override
|
|
|
|
- public void endDocument() throws SAXException {
|
|
|
|
- flush(true);
|
|
|
|
- }
|
|
|
|
- }
|
|
|
|
-
|
|
|
|
private void fillTable(List<String> name, Map<String, String> cols){
|
|
private void fillTable(List<String> name, Map<String, String> cols){
|
|
try {
|
|
try {
|
|
Files.walk(Paths.get(fiasPath))
|
|
Files.walk(Paths.get(fiasPath))
|
|
@@ -138,12 +72,8 @@ public class FiasSyncService {
|
|
}
|
|
}
|
|
|
|
|
|
private void createTable(String name, Map<String, String> cols){
|
|
private void createTable(String name, Map<String, String> cols){
|
|
- try {
|
|
|
|
- new JdbcTemplate(dataSource).execute("DROP TABLE " + name);
|
|
|
|
- } catch (Exception e){
|
|
|
|
- e.printStackTrace();
|
|
|
|
- }
|
|
|
|
- String sql = "CREATE TABLE IF NOT EXISTS fias." + name;
|
|
|
|
|
|
+ new JdbcTemplate(dataSource).execute("DROP TABLE IF EXISTS FIAS." + name);
|
|
|
|
+ String sql = "CREATE TABLE IF NOT EXISTS FIAS." + name;
|
|
List<String> colsList = new ArrayList<>();
|
|
List<String> colsList = new ArrayList<>();
|
|
cols.forEach((n, t) -> colsList.add(n + " " + t));
|
|
cols.forEach((n, t) -> colsList.add(n + " " + t));
|
|
sql = sql + "("+ StringUtils.join(colsList, ',') +")";
|
|
sql = sql + "("+ StringUtils.join(colsList, ',') +")";
|
|
@@ -161,7 +91,7 @@ public class FiasSyncService {
|
|
type = "LONG";
|
|
type = "LONG";
|
|
break;
|
|
break;
|
|
case "string":
|
|
case "string":
|
|
- type = "VARCHAR(512)";
|
|
|
|
|
|
+ type = "VARCHAR(1024)";
|
|
break;
|
|
break;
|
|
case "date":
|
|
case "date":
|
|
type = "DATE";
|
|
type = "DATE";
|
|
@@ -218,4 +148,74 @@ public class FiasSyncService {
|
|
e.printStackTrace();
|
|
e.printStackTrace();
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
+
|
|
|
|
+ private class MyHandler extends DefaultHandler {
|
|
|
|
+ private long limit = 9999;
|
|
|
|
+ private long count = 0;
|
|
|
|
+ private long total = 0;
|
|
|
|
+ private String tableName;
|
|
|
|
+ private Map<String, String> cols;
|
|
|
|
+ private Map<String, List<Object[]>> cache = new HashMap<>();
|
|
|
|
+
|
|
|
|
+ private MyHandler() {
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ MyHandler(String tableName, Map<String, String> cols) {
|
|
|
|
+ this.tableName = tableName;
|
|
|
|
+ this.cols = cols;
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ private void flush(boolean end) {
|
|
|
|
+ if ((count > limit) || end) {
|
|
|
|
+ for (String sql : cache.keySet())
|
|
|
|
+ new JdbcTemplate(dataSource).batchUpdate(sql, cache.get(sql));
|
|
|
|
+
|
|
|
|
+ log.info("flush " + end + ", total " + total);
|
|
|
|
+ count = 0;
|
|
|
|
+ cache.clear();
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ private void insert(Attributes attributes) {
|
|
|
|
+ String[] names = new String[attributes.getLength()];
|
|
|
|
+ String[] values = new String[attributes.getLength()];
|
|
|
|
+ String[] wildcards = new String[attributes.getLength()];
|
|
|
|
+ for (int i = 0; i < attributes.getLength(); i++) {
|
|
|
|
+ names[i] = attributes.getLocalName(i);
|
|
|
|
+ values[i] = attributes.getValue(i);
|
|
|
|
+ wildcards[i] = "?";
|
|
|
|
+ }
|
|
|
|
+ String sql = "INSERT INTO FIAS." + tableName + "(" + StringUtils.join(names, ',') + ")" + "VALUES (" + StringUtils.join(wildcards, ',') + ")";
|
|
|
|
+ try {
|
|
|
|
+ //new JdbcTemplate(dataSource).update(sql, values);
|
|
|
|
+ if (!cache.containsKey(sql)) cache.put(sql, new ArrayList<>());
|
|
|
|
+ cache.get(sql).add(values);
|
|
|
|
+ count++;
|
|
|
|
+ total++;
|
|
|
|
+ flush(false);
|
|
|
|
+ } catch (Exception e) {
|
|
|
|
+ e.printStackTrace();
|
|
|
|
+ throw new RuntimeException(e);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ @Override
|
|
|
|
+ public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
|
|
|
|
+ if (attributes.getLength() > 0) {
|
|
|
|
+ if (cols.containsKey(attributes.getLocalName(0)))
|
|
|
|
+ insert(attributes);
|
|
|
|
+ }
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ @Override
|
|
|
|
+ public void startDocument() throws SAXException {
|
|
|
|
+ new JdbcTemplate(dataSource).execute("START TRANSACTION");
|
|
|
|
+ }
|
|
|
|
+
|
|
|
|
+ @Override
|
|
|
|
+ public void endDocument() throws SAXException {
|
|
|
|
+ flush(true);
|
|
|
|
+ new JdbcTemplate(dataSource).execute("COMMIT");
|
|
|
|
+ }
|
|
|
|
+ }
|
|
}
|
|
}
|