Revize 0a2832fb

Re #8089
- pridana nova proměna do konfig souboru
- predalana tabulka v databasi s nazvem DATASETS
- nove obsahuje "key-name" jako klíč a "display-name" pro zobrazení

modules/crawler/Pipeline.py

@@ -152 +152 @@
     # get all unprocessed files from dataset
     not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
 
+    database_connection = DatabaseLoader.create_database_connection()
+
+    DatabaseLoader.check_or_update_datasets_collection(database_connection,config)
+
+    DatabaseLoader.update_devices_collection(database_connection,config)
+
+
+    # load every file
+    for not_loaded_file in not_loaded_files:
+        #check if file is not in database already if it is skip
+        test = DatabaseLoader.check_if_database_doesnt_contain_file(database_connection,dataset_name,not_loaded_file)
+        if test == False:
+            logging.info(dataset_name + " could not load " + not_loaded_file + " to database because he is already there.")
+            FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
+            continue
+        # load processed data
+        processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
+        # load processed data to database
+        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
+        FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
+
+    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
+
+
+def load_data_to_database_crone(config):
+    """
+    Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
+    loads data appends coordination from configurations
+    and exports it into the database
+    After successful exporting updates ignore.txt
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    dataset_path = dataset_name + '/'
+
+    # get all unprocessed files from dataset
+    not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
+
+    database_connection = DatabaseLoader.create_database_connection()
+
     # load every file
     for not_loaded_file in not_loaded_files:
         # load processed data
         processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
         # load processed data to database
-        DatabaseLoader.load_data_to_database(dataset_name, processed_data)
+        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
         FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
 
-    logging.info(dataset_name + " has loaded to databse " + str(len(not_loaded_files)) + " newly processed files.")
+    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
 
 
 def run_full_pipeline(dataset_name):
@@ -185 +227 @@
         load_data_to_database(config)
 
 
-
 def run_full_pipeline_crone(dataset_name):
     """
     Loads config file and starts full pipeline
@@ -207 +248 @@
         validation_test = validate_process_data(config)
 
         if validation_test:
-            load_data_to_database(config)
+            load_data_to_database_crone(config)