Revize 0d303b8f

Added two external catalog items sources

re #9752

backend/src/main/java/cz/zcu/kiv/backendapi/external/ExternalCatalogItemItemServiceImpl.java

@@ -9 +9 @@
 import org.apache.tomcat.util.http.fileupload.FileUtils;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.web.multipart.MultipartFile;
 
 import javax.persistence.Table;
 import java.io.*;
@@ -40 +41 @@
      * Buffer size
      */
     public static final int BUFFER_SIZE = 1024;
+
     /**
      * Directory where files for external directory will be stored
      */
     private static final String DIRECTORY_FOR_EXTERNAL_FILES = "sources";
+
     /**
      * URL for Pleiades file
      */
     private static final String PLEIADES_FILE_URL = "https://atlantides.org/downloads/pleiades/dumps/pleiades-names-latest.csv.gz";
+
     /**
      * Name of Pleiades file
      */
     private static final String PLEIADES_FILE_NAME = "pleiades-names-latest.csv";
+
     /**
      * URL for Pleiades file – needs to be formatted (more sources)
      */
     private static final String GEONAMES_FILE_URL = "https://download.geonames.org/export/dump/%s.zip";
+
     /**
      * Name of GeoNames file – needs to be formatted (more sources)
      */
     private static final String GEONAMES_FILE_NAME = "%s.txt";
+
     /**
-     * URL for CIGS file
+     * Name of CIGS file
      */
-    private static final String CIGS_FILE_URL = "https://zenodo.org/record/5642899/files/CIGS_v1_4_20211101.csv";
+    private static final String CIGS_FILE_NAME = "CIGS.csv";
+
     /**
-     * Name of CIGS file
+     * Name of ANE file
      */
-    private static final String CIGS_FILE_NAME = "CIGS_v1_4_20211101.csv";
+    private static final String ANE_FILE = "ANE.csv";
+
     /**
      * Batch size for saving items
      */
@@ -110 +119 @@
 
 
     @Override
-    public void updateCatalog() {
+    public void updateCatalog(MultipartFile file) {
         log.info("Updating external catalog");
+
         try {
             Files.createDirectories(Paths.get(DIRECTORY_FOR_EXTERNAL_FILES)); // creates directory if not exists
             FileUtils.cleanDirectory(new File(DIRECTORY_FOR_EXTERNAL_FILES)); // cleans the directory
-            externalCatalogItemRepository.deleteAll(); // clears database – updated list will be stored later
-            addPleiadesSource();
-            addGeonamesSources();
-            addCigsSources();
         } catch (Exception e) {
             e.printStackTrace();
         }
+
+        externalCatalogItemRepository.deleteAll(); // clears database – updated list will be stored later
+
+        try {
+            addPleiadesRecords();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        try {
+            addGeonamesRecords();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        try {
+            if (file.getOriginalFilename() != null) {
+                File cigsFile = new File(new File(DIRECTORY_FOR_EXTERNAL_FILES), file.getOriginalFilename());
+                Files.copy(file.getInputStream(), cigsFile.toPath());
+                addCigsRecords(cigsFile);
+            }
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
+        try {
+            addAneRecords();
+        } catch (Exception e) {
+            e.printStackTrace();
+        }
+
         log.info("External catalog updated");
     }
 
@@ -134 +171 @@
     }
 
     /**
-     * Downloads, extracts and reads Pleiades sources and saves them to database
+     * Downloads, extracts and reads records from Pleiades and saves them to database
      */
-    private void addPleiadesSource() {
+    private void addPleiadesRecords() {
         List<ExternalCatalogItem> externalCatalogItems = new ArrayList<>();
         byte[] buffer = new byte[BUFFER_SIZE];
         File pleiadesFile = new File(new File(DIRECTORY_FOR_EXTERNAL_FILES), PLEIADES_FILE_NAME);
@@ -170 +207 @@
             ex.printStackTrace();
         }
         saveAllWithThreads(externalCatalogItems);
+
+        log.info("Records from Pleiades added");
     }
 
     /**
-     * Downloads, extracts and reads GeoNames sources and saves them to database
+     * Downloads, extracts and reads records from GeoNames and saves them to database
      */
-    private void addGeonamesSources() {
+    private void addGeonamesRecords() {
         byte[] buffer = new byte[BUFFER_SIZE];
         for (String countryCode : ExternalCatalogItem.COUNTRY_CODES.keySet()) {
             List<ExternalCatalogItem> externalCatalogItems = new ArrayList<>();
@@ -211 +250 @@
             }
             saveAllWithThreads(externalCatalogItems);
         }
+
+        log.info("Records from GeoNames added");
     }
 
     /**
-     * Downloads and reads CIGS sources and saves them to database
+     * Saves records from ANE to database
      */
-    private void addCigsSources() {
+    private void addAneRecords() {
         List<ExternalCatalogItem> externalCatalogItems = new ArrayList<>();
-        byte[] buffer = new byte[BUFFER_SIZE];
-        File cigsFile = new File(new File(DIRECTORY_FOR_EXTERNAL_FILES), CIGS_FILE_NAME);
-
-        try (InputStream inputStream = new URL(CIGS_FILE_URL).openStream();
-             FileOutputStream fileOutputStream = new FileOutputStream(cigsFile)) {
-            int bytes_read;
-
-            while ((bytes_read = inputStream.read(buffer)) > 0) {
-
-                fileOutputStream.write(buffer, 0, bytes_read);
+        // Reads file and adds catalog items to list
+        try (InputStream csvData = getClass().getClassLoader().getResourceAsStream(ANE_FILE)) {
+            if (csvData == null) {
+                return;
+            }
+            CSVParser parser = CSVParser.parse(csvData, StandardCharsets.UTF_8, CSVFormat.Builder.create(CSVFormat.DEFAULT)
+                    .setHeader()
+                    .setSkipHeaderRecord(true)
+                    .build());
+            for (CSVRecord csvRecord : parser) {
+                ExternalCatalogItem e = new ExternalCatalogItem(csvRecord.toList(), ExternalSource.ANE);
+                externalCatalogItems.add(e);
             }
         } catch (IOException e) {
             e.printStackTrace();
         }
+        saveAllWithThreads(externalCatalogItems);
+
+        log.info("Records from ANE added");
+    }
 
-        log.info("The CIGS file was downloaded successfully");
+
+    /**
+     * Reads records from CIGS and saves them to database
+     *
+     * @param cigsFile CIGS file
+     */
+    private void addCigsRecords(File cigsFile) {
+        List<ExternalCatalogItem> externalCatalogItems = new ArrayList<>();
 
         try (InputStream csvData = new FileInputStream(cigsFile)) {
             CSVParser parser = CSVParser.parse(csvData, StandardCharsets.UTF_8, CSVFormat.Builder.create(CSVFormat.DEFAULT)
@@ -248 +302 @@
             e.printStackTrace();
         }
         saveAllWithThreads(externalCatalogItems);
+
+        log.info("Records from CIGS added");
     }
 
     /**