Revize 70e660a8

Re #7927
implementovana funkce pro pridani neznamych zarizeni
do konfiguracnich souboru
predelana struktura konfig souboru devices jsou ted nakonci
-vygenerovany konfig soubory pro vsechny datasety

python-module/Pipeline.py

@@ -1 @@
-from Utilities import FolderProcessor, ConfigureLoader
+from Utilities import FolderProcessor, ConfigureFunctions
 from Utilities.Database import DatabaseLoader
 
 
@@ -35 +35 @@
         FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
 
 
+def validate_process_data(config):
+    processed_devices_set = FolderProcessor.get_devices_set(PROCESSED_DATA_PATH + config["dataset-name"] + '/')
+    unknown_devices_set = FolderProcessor.get_unknown_devices_set(config,processed_devices_set)
+    unknown_devices_size = len(unknown_devices_set)
+
+    if unknown_devices_size != 0:
+        print("There is " + str(unknown_devices_size) + " unknown devies")
+        ConfigureFunctions.update_configuration(CONFIG_FILES_PATH + config["dataset-name"] + ".yaml", unknown_devices_set)
+        return False
+
+
 def load_data_to_database(config):
 
     dataset_name = config["dataset-name"]
@@ -54 +65 @@
 
 
 def run_full_pipeline(dataset_name):
-    config = ConfigureLoader.load_configuration(CONFIG_FILES_PATH + dataset_name)
+    config = ConfigureFunctions.load_configuration(CONFIG_FILES_PATH + dataset_name)
     crawl_data(config)
     process_data(config["dataset-name"])
-    #load_data_to_database(config)
+
+    validation_test = validate_process_data(config)
+
+    if validation_test:
+        load_data_to_database(config)
+