initial with fias

.gitignore

@@ -0,0 +1,6 @@
+/.ideaDataSources
+/dataSources
+/target
+/*.iml
+/*.ipr
+/*.iws

pom.xml

@@ -0,0 +1,86 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+         xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+    <modelVersion>4.0.0</modelVersion>
+
+    <groupId>in.ocsf</groupId>
+    <artifactId>report-api</artifactId>
+    <version>0.0.1</version>
+    <packaging>war</packaging>
+
+    <properties>
+        <java.version>1.8</java.version>
+    </properties>
+
+    <parent>
+        <groupId>org.springframework.boot</groupId>
+        <artifactId>spring-boot-starter-parent</artifactId>
+        <version>1.5.2.RELEASE</version>
+    </parent>
+
+    <dependencies>
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-web</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-data-jpa</artifactId>
+        </dependency>
+
+        <dependency>
+            <groupId>org.springframework.boot</groupId>
+            <artifactId>spring-boot-starter-tomcat</artifactId>
+            <scope>provided</scope>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/mysql/mysql-connector-java -->
+        <dependency>
+            <groupId>mysql</groupId>
+            <artifactId>mysql-connector-java</artifactId>
+            <version>5.1.40</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/com.sun.xsom/xsom -->
+        <dependency>
+            <groupId>com.sun.xsom</groupId>
+            <artifactId>xsom</artifactId>
+            <version>20140925</version>
+        </dependency>
+
+        <!-- https://mvnrepository.com/artifact/org.apache.commons/commons-lang3 -->
+        <dependency>
+            <groupId>org.apache.commons</groupId>
+            <artifactId>commons-lang3</artifactId>
+            <version>3.5</version>
+        </dependency>
+
+        <dependency>
+            <groupId>commons-io</groupId>
+            <artifactId>commons-io</artifactId>
+            <version>2.5</version>
+        </dependency>
+
+    </dependencies>
+
+    <build>
+        <plugins>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-war-plugin</artifactId>
+                <version>3.0.0</version>
+            </plugin>
+            <plugin>
+                <groupId>org.apache.maven.plugins</groupId>
+                <artifactId>maven-compiler-plugin</artifactId>
+                <configuration>
+                    <source>1.8</source>
+                    <target>1.8</target>
+                    <compilerVersion>1.8</compilerVersion>
+                </configuration>
+            </plugin>
+        </plugins>
+    </build>
+</project>

src/main/java/in/ocsf/app/App.java

@@ -0,0 +1,23 @@
+package in.ocsf.app;/* kpmy 05.03.2017 */
+
+import org.springframework.boot.SpringApplication;
+import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.boot.builder.SpringApplicationBuilder;
+import org.springframework.boot.web.support.SpringBootServletInitializer;
+import org.springframework.scheduling.annotation.EnableAsync;
+
+@SpringBootApplication
+@EnableAsync
+public class App extends SpringBootServletInitializer {
+
+    private static Class<App> applicationClass = App.class;
+
+    public static void main(String[] args) {
+        SpringApplication.run(applicationClass, args);
+    }
+
+    @Override
+    protected SpringApplicationBuilder configure(SpringApplicationBuilder application) {
+        return application.sources(applicationClass);
+    }
+}

src/main/java/in/ocsf/app/FiasSyncService.java

@@ -0,0 +1,221 @@
+package in.ocsf.app;/* kpmy 05.03.2017 */
+
+import com.sun.xml.xsom.XSElementDecl;
+import com.sun.xml.xsom.XSSchemaSet;
+import com.sun.xml.xsom.parser.XSOMParser;
+import org.apache.commons.io.ByteOrderMark;
+import org.apache.commons.io.input.BOMInputStream;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.tuple.ImmutablePair;
+import org.apache.commons.lang3.tuple.Pair;
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Value;
+import org.springframework.jdbc.core.JdbcTemplate;
+import org.springframework.scheduling.annotation.Async;
+import org.springframework.stereotype.Service;
+import org.xml.sax.Attributes;
+import org.xml.sax.InputSource;
+import org.xml.sax.SAXException;
+import org.xml.sax.helpers.DefaultHandler;
+
+import javax.sql.DataSource;
+import javax.xml.parsers.SAXParser;
+import javax.xml.parsers.SAXParserFactory;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.io.Reader;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.util.*;
+
+@Service
+public class FiasSyncService {
+
+    private final Logger log = Logger.getLogger(getClass());
+
+    @Value("${fias.path}")
+    private String fiasPath;
+
+    @Autowired
+    private DataSource dataSource;
+
+    private class MyHandler extends DefaultHandler {
+        private MyHandler(){}
+
+        private long limit = 1999;
+        private long count = 0;
+        private long total = 0;
+
+        private String tableName;
+        private Map<String, String> cols;
+        private Map<String, List<Object[]>> cache = new HashMap<>();
+
+        MyHandler(String tableName, Map<String, String> cols){
+            this.tableName = tableName;
+            this.cols = cols;
+        }
+
+        private void flush(boolean end){
+            if ((count > limit) || end) {
+                for(String sql : cache.keySet())
+                    new JdbcTemplate(dataSource).batchUpdate(sql, cache.get(sql));
+
+                log.info("flush "+ end + ", total "+ total);
+                count = 0;
+                cache.clear();
+            }
+        }
+
+        private void insert(Attributes attributes){
+            String[] names = new String[attributes.getLength()];
+            String[] values = new String[attributes.getLength()];
+            String[] wildcards = new String[attributes.getLength()];
+            for(int i = 0; i < attributes.getLength(); i++){
+                names[i] = attributes.getLocalName(i);
+                values[i] = attributes.getValue(i);
+                wildcards[i] = "?";
+            }
+            String sql = "INSERT INTO fias." + tableName + "("+StringUtils.join(names, ',')+")" + "VALUES ("+StringUtils.join(wildcards, ',')+")";
+            try {
+                //new JdbcTemplate(dataSource).update(sql, values);
+                if(!cache.containsKey(sql)) cache.put(sql, new ArrayList<>());
+                cache.get(sql).add(values);
+                count++;
+                total++;
+                flush(false);
+            } catch (Exception e){
+                e.printStackTrace();
+                throw new RuntimeException(e);
+            }
+        }
+
+        @Override
+        public void startElement(String uri, String localName, String qName, Attributes attributes) throws SAXException {
+            if (attributes.getLength() > 0){
+                if (cols.containsKey(attributes.getLocalName(0)))
+                    insert(attributes);
+            }
+        }
+
+        @Override
+        public void endDocument() throws SAXException {
+            flush(true);
+        }
+    }
+
+    private void fillTable(List<String> name, Map<String, String> cols){
+        try {
+            Files.walk(Paths.get(fiasPath))
+                .filter(Files::isRegularFile)
+                .filter(p -> p.toString().toLowerCase().endsWith(".xml"))
+                .filter(p -> {
+                    List<String> nameList = new ArrayList<>();
+                    for (String part : Arrays.asList(p.getFileName().toString().split("_", -1))) {
+                        if (StringUtils.isNumeric(part)) break;
+                        nameList.add(part);
+                    }
+                    return StringUtils.join(name, '_').equals(StringUtils.join(nameList, '_'));
+                }).findAny().ifPresent(p -> {
+                    try {
+                        SAXParserFactory factory = SAXParserFactory.newInstance();
+                        SAXParser saxParser = factory.newSAXParser();
+                        InputStream inputStream= new FileInputStream(p.toFile());
+                        inputStream = new BOMInputStream(inputStream, ByteOrderMark.UTF_8);
+                        Reader reader = new InputStreamReader(inputStream,"UTF-8");
+                        InputSource is = new InputSource(reader);
+                        is.setEncoding("UTF-8");
+                        saxParser.parse(is, new MyHandler(StringUtils.join(name, '_'), cols));
+                    } catch (Exception e){
+                        e.printStackTrace();
+                    }
+                });
+        } catch (Exception e) {
+
+        }
+    }
+
+    private void createTable(String name, Map<String, String> cols){
+        try {
+            new JdbcTemplate(dataSource).execute("DROP TABLE " + name);
+        } catch (Exception e){
+            e.printStackTrace();
+        }
+        String sql = "CREATE TABLE IF NOT EXISTS fias." + name;
+        List<String> colsList = new ArrayList<>();
+        cols.forEach((n, t) -> colsList.add(n + " " + t));
+        sql = sql + "("+ StringUtils.join(colsList, ',') +")";
+        new JdbcTemplate(dataSource).execute(sql);
+    }
+
+    private void useBase(List<String> name, XSElementDecl base){
+        Map<String, String> cols = new LinkedHashMap<>();
+        base.getType().asComplexType().iterateAttributeUses().forEachRemaining(a -> {
+            String type;
+            String typeName = a.getDecl().getType().getPrimitiveType().getName();
+            switch (typeName){
+                case "integer":
+                case "decimal":
+                    type = "LONG";
+                    break;
+                case "string":
+                    type = "VARCHAR(512)";
+                    break;
+                case "date":
+                    type = "DATE";
+                    break;
+                default:
+                    throw new RuntimeException("unknown type " + typeName);
+            }
+            if(type == null) throw new RuntimeException("unknown type");
+            cols.put(a.getDecl().getName(), type);
+        });
+        createTable(StringUtils.join(name, '_'), cols);
+        fillTable(name, cols);
+    }
+
+    private void useSchema(Path path) throws Exception{
+        log.info(path.toString() + " start");
+        SAXParserFactory parserFactory = SAXParserFactory.newInstance();
+        parserFactory.setNamespaceAware(true);
+        XSOMParser xsomParser = new XSOMParser(parserFactory);
+        xsomParser.parse(path.toFile());
+        XSSchemaSet xsd = xsomParser.getResult();
+        xsd.iterateSchema().forEachRemaining(s -> {
+            s.iterateElementDecls().forEachRemaining(elem -> {
+                try {
+                    XSElementDecl child = elem.getType().asComplexType().getContentType().asParticle().getTerm().asModelGroup().getChildren()[0].getTerm().asElementDecl();
+                    List<String> nameList = new ArrayList<>();
+                    for(String part : Arrays.asList(path.getFileName().toString().split("_", -1))){
+                        if(StringUtils.isNumeric(part)) break;
+                        nameList.add(part);
+                    }
+                    useBase(nameList, child);
+                } catch (Exception e){
+                    e.printStackTrace();
+                }
+            });
+        });
+        log.info(path.toString() + " done");
+    }
+
+    @Async
+    public void sync(){
+        try {
+            Files.walk(Paths.get(fiasPath))
+                    .filter(Files::isRegularFile)
+                    .filter(p -> p.toString().toLowerCase().endsWith(".xsd"))
+                    .forEach(p -> {
+                        try {
+                            useSchema(p);
+                        } catch (Exception e) {
+                            e.printStackTrace();
+                        }
+                    });
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+    }
+}

src/main/java/in/ocsf/app/TestController.java

@@ -0,0 +1,34 @@
+package in.ocsf.app;/* kpmy 05.03.2017 */
+
+import in.ocsf.report.ReportPrototype;
+import in.ocsf.report.StagePrototype;
+import in.ocsf.report.TablePrototype;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.web.bind.annotation.RequestMapping;
+import org.springframework.web.bind.annotation.RequestParam;
+import org.springframework.web.bind.annotation.RestController;
+
+@RestController
+public class TestController {
+
+    @Autowired
+    private FiasSyncService fiasSyncService;
+
+    @RequestMapping("test/fias")
+    public Object fias(){
+        fiasSyncService.sync();
+        return null;
+    }
+
+    @RequestMapping("test/init")
+    public Object init(@RequestParam("name") String name){
+        ReportPrototype report = new ReportPrototype();
+
+        TablePrototype table0 = new TablePrototype();
+
+        StagePrototype stage0 = new StagePrototype();
+
+
+        return null;
+    }
+}

src/main/java/in/ocsf/report/ReportPrototype.java

@@ -0,0 +1,16 @@
+package in.ocsf.report;/* kpmy 05.03.2017 */
+
+import java.util.ArrayList;
+import java.util.List;
+
+public class ReportPrototype {
+    private List<TablePrototype> tables = new ArrayList<TablePrototype>();
+
+    public List<TablePrototype> getTables() {
+        return tables;
+    }
+
+    public void setTables(List<TablePrototype> tables) {
+        this.tables = tables;
+    }
+}

src/main/java/in/ocsf/report/StagePrototype.java

@@ -0,0 +1,4 @@
+package in.ocsf.report;/* kpmy 05.03.2017 */
+
+public class StagePrototype {
+}

src/main/java/in/ocsf/report/TablePrototype.java

@@ -0,0 +1,4 @@
+package in.ocsf.report;/* kpmy 05.03.2017 */
+
+public class TablePrototype {
+}

src/main/resources/application-dev.properties

@@ -0,0 +1,5 @@
+spring.datasource.url=jdbc:mysql://127.0.0.1:3306/fias?verifyServerCertificate=false&useSSL=false&requireSSL=false&useUnicode=yes&characterEncoding=UTF-8
+spring.datasource.username=root
+spring.datasource.password=00018152
+
+fias.path=d:\\tmp\\fias

src/main/resources/static/index.html

@@ -0,0 +1,11 @@
+<!DOCTYPE html>
+<html lang="en">
+<head>
+    <meta charset="UTF-8">
+    <title>Отчёты</title>
+</head>
+<body>
+    <a href="test/init?name=Test0">test 0</a>
+    <a href="test/fias">fias</a>
+</body>
+</html>

src/main/webapp/WEB-INF/web.xml

@@ -0,0 +1,14 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<web-app xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+         xmlns="http://xmlns.jcp.org/xml/ns/javaee"
+         xsi:schemaLocation="http://xmlns.jcp.org/xml/ns/javaee
+                      http://xmlns.jcp.org/xml/ns/javaee/web-app_4_0.xsd"
+         version="4.0">
+
+    <servlet>
+        <servlet-name>report-api</servlet-name>
+        <load-on-startup>1</load-on-startup>
+    </servlet>
+
+
+</web-app>