Revize 7eddfeab

Re #7918: Zrychlení nahrávání dokumentů
- ověření před vytvářením cores v Solru, jestli už neexistují
- úprava readme

README.md

@@ -51 +51 @@
 # Database - Solr
 ## Solr - spuštění lokálně
 - stáhnout si Solr [zde](https://lucene.apache.org/solr/downloads.html)
-- přesunout se do složky ...\solr-8.4.1\bin 
+- do složky ...\solr-8.4.1\server\solr přesunout celý obsah projektové složky aswi2020anonymous\solr\core-configs
+- přesunout se do složky ...\solr-8.4.1\bin
 - nastartovat Solr příkazem `solr start`
-- vytvořit core "documents" příkazem `solr create -c documents`
 ## Solr - spuštění v dockeru
 - nainstalovat docker viz Návody
 - nastartovat docker 

be/fulltextsearch/src/main/java/cz/zcu/kiv/aswi/fulltextsearch/SolrService.java

@@ -52 +52 @@
 
     private static final int SEARCH_QUERY_ROWS = 100;
 
+    private static final int DEFAULT_LINE_FIELD_COUNT = 150;
+    private static final int DEFAULT_WORD_FIELD_COUNT = 30;
+
     private HttpSolrClient solrClientDocuments;
     private HttpSolrClient solrClientFiles;
     private List<String> fieldNamesDocumentCore = new ArrayList<>();
@@ -61 +64 @@
     }
 
     private void connect() {
-        LOG.info("Connecting to Solr");
+        LOG.info("Connecting to Solr...");
         String urlString = URL + ":8983/solr/";
 
         try {
-            LOG.info("Before HttpSolrClient.Builder");
+            // documents
             solrClientDocuments = new HttpSolrClient.Builder(urlString).build();
-            LOG.info("After HttpSolrClient.Builder");
-
-            LOG.info("Before Create Core");
-            CoreAdminRequest.createCore(CORE_NAME_DOCUMENTS, CORE_NAME_DOCUMENTS, solrClientDocuments, "conf/solrconfig.xml", "documents-schema.xml");
-            LOG.info("After Create Core");
-
-            LOG.info("Before setBase URL");
+            if (CoreAdminRequest.getStatus(CORE_NAME_DOCUMENTS, solrClientDocuments).getCoreStatus(CORE_NAME_DOCUMENTS).size() < 1) {
+                CoreAdminRequest.createCore(CORE_NAME_DOCUMENTS, CORE_NAME_DOCUMENTS, solrClientDocuments, "conf/solrconfig.xml", "documents-schema.xml");
+                LOG.info("Core " + CORE_NAME_DOCUMENTS + " has been created.");
+            } else {
+                LOG.info("Core " + CORE_NAME_DOCUMENTS + " already exists.");
+            }
             solrClientDocuments.setBaseURL(solrClientDocuments.getBaseURL() + "/" +  CORE_NAME_DOCUMENTS);
-            LOG.info("After setBase URL");
-
 
+            // files
             solrClientFiles = new HttpSolrClient.Builder(urlString).build();
-            CoreAdminRequest.createCore(CORE_NAME_FILES, CORE_NAME_FILES, solrClientFiles, "conf/solrconfig.xml", "files-schema.xml");
+            if (CoreAdminRequest.getStatus(CORE_NAME_FILES, solrClientFiles).getCoreStatus(CORE_NAME_FILES).size() < 1) {
+                CoreAdminRequest.createCore(CORE_NAME_FILES, CORE_NAME_FILES, solrClientFiles, "conf/solrconfig.xml", "files-schema.xml");
+                LOG.info("Core " + CORE_NAME_FILES + " has been created.");
+            } else {
+                LOG.info("Core " + CORE_NAME_FILES + " already exists.");
+            }
             solrClientFiles.setBaseURL(solrClientFiles.getBaseURL() + "/"  + CORE_NAME_FILES);
-            LOG.info("After try");
-        } catch (SolrServerException e) {
-            LOG.error("SolrServerException: " + e);
-            e.printStackTrace();
-        } catch (IOException e) {
-            LOG.error("IOException: " + e);
-            e.printStackTrace();
+
+            LOG.info("Connected to Solr.");
+        } catch (SolrServerException | IOException e ) {
+            LOG.error("Couldn't connect to Solr! " + e);
         }
-        LOG.error("Connect finished");
     }
 
     public int uploadFiles(List<MultipartFile> files) throws IOException, SolrServerException, JAXBException {
@@ -139 +141 @@
         LOG.trace("SolrService method addFiles called");
         LOG.debug("Uploading files to Solr");
         getFieldNamesFromDocumentCore();
-        addBasicFieldsToDocumentsCore();
-
-        addAllFieldsToFilesCore();
 
         int count = 0;
         Set<String> keys = pairs.keySet();
@@ -184 +183 @@
         addToDocumentsCore(doc, imageFile.getOriginalFilename());
     }
 
-    private void addAllFieldsToFilesCore() throws IOException, SolrServerException {
-        LOG.trace("SolrService method addAllFieldsToFilesCore called");
-        List<String> fieldNamesFilesCore = new ArrayList<>();
-
-        SchemaRequest.Fields fieldRequest = new SchemaRequest.Fields();
-        SchemaResponse.FieldsResponse fieldsResponse = fieldRequest.process(solrClientFiles);
-        List<Map<String, Object>> fields = fieldsResponse.getFields();
-
-        for (Map<String, Object> field: fields) {
-            fieldNamesFilesCore.add(field.get("name").toString());
-        }
-
-        String[] filenames = new String[]{ FIELD_DOC_FILENAME, FIELD_IMG_FILENAME };
-        String[] contents = new String[]{ FIELD_DOC_CONTENT, FIELD_IMG_CONTENT };
-
-        LOG.debug("Adding all fields to File core");
-        for (String name: filenames) {
-            if (fieldNamesFilesCore.contains(name)) continue;
-
-            try {
-                SchemaRequest.Update request = getAddTextFieldRequest(name);
-                request.process(solrClientFiles);
-            } catch (Exception e) {
-                LOG.error("Couldn't add field = " + name);
-                LOG.error(e.toString());
-            }
-        }
-
-        for (String name: contents) {
-            if (fieldNamesFilesCore.contains(name)) continue;
-
-            try {
-                Map<String, Object> fieldAttributes = new LinkedHashMap<>();
-                fieldAttributes.put("name", name);
-                fieldAttributes.put("type", "binary");
-                fieldAttributes.put("indexed", false);
-                fieldAttributes.put("stored", true);
-                fieldAttributes.put("multiValued", false);
-                SchemaRequest.Update request = new SchemaRequest.AddField(fieldAttributes);
-                request.process(solrClientFiles);
-            } catch (Exception e) {
-                LOG.error("Couldn't add field = " + name);
-                LOG.error(e.toString());
-            }
-        }
-    }
-
-    private void addBasicFieldsToDocumentsCore() {
-        LOG.trace("SolrService method addBasicFieldsToDocumentsCore called");
-        LOG.debug("Adding basic fields to Documents core");
-        addFieldToDocumentCore(FIELD_DOC_FILENAME, getAddTextFieldRequest(FIELD_DOC_FILENAME));
-        addFieldToDocumentCore(FIELD_IMG_FILENAME, getAddTextFieldRequest(FIELD_IMG_FILENAME));
-        addFieldToDocumentCore(FIELD_TEXT_REGION, getAddTextFieldRequest(FIELD_TEXT_REGION));
-        addFieldToDocumentCore(FIELD_TEXT_REGION_COORDS, getNotIndexedAddTextFieldRequest(FIELD_TEXT_REGION_COORDS));
-    }
-
     private void getFieldNamesFromDocumentCore() throws IOException, SolrServerException {
         LOG.trace("SolrService method getFieldNamesFromDocumentCore called");
         LOG.debug("Getting field names from Documents core");
@@ -474 +417 @@
                 TextLine textLine = textLines[j];
                 String prefix = PREFIX_TEXT_LINE + j;
 
-                addFieldToDocumentCore(prefix, getAddTextFieldRequest(prefix));
-                solrInputDocument.addField(prefix, textLine.getTextEquiv().getUnicode());
+                if (j >= DEFAULT_LINE_FIELD_COUNT) {
+                    addFieldToDocumentCore(prefix, getAddTextFieldRequest(prefix));
+                    addFieldToDocumentCore(prefix + SUFFIX_COORDS, getNotIndexedAddTextFieldRequest(prefix + SUFFIX_COORDS));
+                }
 
-                addFieldToDocumentCore(prefix + SUFFIX_COORDS, getNotIndexedAddTextFieldRequest(prefix + SUFFIX_COORDS));
+                solrInputDocument.addField(prefix, textLine.getTextEquiv().getUnicode());
                 solrInputDocument.addField(prefix + SUFFIX_COORDS, textLine.getLineCoords().getPointsString());
 
                 TextWord[] textWords = textLine.getTextWords();
@@ -486 +431 @@
                     TextWord textWord = textWords[k];
                     String wordPrefix = PREFIX_TEXT_WORD + k + "_" + prefix;
 
-                    addFieldToDocumentCore(wordPrefix, getAddTextFieldRequest(wordPrefix));
-                    solrInputDocument.addField(wordPrefix, textWord.getTextEquiv().getUnicode());
+                    if (k >= DEFAULT_WORD_FIELD_COUNT) {
+                        addFieldToDocumentCore(wordPrefix, getAddTextFieldRequest(wordPrefix));
+                        addFieldToDocumentCore(wordPrefix + SUFFIX_COORDS, getNotIndexedAddTextFieldRequest(wordPrefix + SUFFIX_COORDS));
+                    }
 
-                    addFieldToDocumentCore(wordPrefix + SUFFIX_COORDS, getNotIndexedAddTextFieldRequest(wordPrefix + SUFFIX_COORDS));
+                    solrInputDocument.addField(wordPrefix, textWord.getTextEquiv().getUnicode());
                     solrInputDocument.addField(wordPrefix + SUFFIX_COORDS, textWord.getWordCoords().getPointsString());
                 }
             }

be/fulltextsearch/src/main/resources/application.properties

@@ -1 +1 @@
 spring.servlet.multipart.max-file-size = -1
-spring.servlet.multipart.max-request-size = -1
+spring.servlet.multipart.max-request-size = -1
+
+logging.level.cz.zcu.kiv.aswi.fulltextsearch=INFO