Revize 753d424e

Predelany komentáře upravena kontrola datasetu

modules/crawler/DatasetConfigs/KOLOBEZKY.yaml

@@ -33 +33 @@
   - stojan-borska:
       x: 49.734518
       y: 13.359475
-

modules/crawler/Utilities/Database/database_loader.py

@@ -241 +241 @@
     for name in collections:
         if pattern.match(name):
             mydb[name].drop()
+            print("Dropping: " + name)
 
-    database_record_logs.reset_ignore_set_processed(dataset_name)
     database_record_logs.reset_ignore_set_loaded(dataset_name)

modules/crawler/fully_clean_database.py

@@ +1 @@
+from Utilities.Database import database_loader
+
+def clean_database():
+    """
+    Drops every collection in database
+    """
+    # Creating connection
+    mydb = database_loader.create_database_connection()
+
+    # Retrieve list of all collections
+    collections = mydb.list_collection_names()
+
+    # Drop of all collections
+    for name in collections:
+        print(name)
+        mydb[name].drop()
+
+
+print('Data z databáze budou smazána:')
+clean_database()

modules/crawler/pipeline.py

@@ -86 +86 @@
     dataset_name = config["dataset-name"]
     dataset_path = dataset_name + '/'
 
-    changes_in_devices = database_loader.update_devices_collection(config)
-
-    if changes_in_devices == True:
-        logging.info(dataset_name + " contains changes in devices configuration. Deleteing old data and preparing new")
-        database_loader.reset_dataset_database(dataset_name)
-        folder_processor.clean_folder(PROCESSED_DATA_PATH + dataset_path)
-
-
     process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "_processor", globals(), locals(),
                                    ['process_file']).process_file
 
@@ -194 +186 @@
 
     database_loader.check_or_update_datasets_collection(database_connection,config)
 
+    changes_in_devices = database_loader.update_devices_collection(config)
+
+    if changes_in_devices == True:
+        logging.info(dataset_name + " contains changes in devices configuration. Deleting old data and preparing new")
+        database_loader.reset_dataset_database(dataset_name)
+
     # get all unprocessed files from dataset
     ignore_set = database_record_logs.load_ignore_set_loaded(dataset_name)
     not_loaded_files = folder_processor.list_of_all_new_files(ignore_set,PROCESSED_DATA_PATH + dataset_path)
@@ -261 +259 @@
         dataset_name: name of dataset that has existing configuration file
     """
     logging.info("Starting pipeline for dataset " + dataset_name)
-    print("Processing dataset " + dataset_name + " you can watch the progress in log contained in CrawlerLogs folder")
+    print("Zpracovávám dataset " + dataset_name + " průběh lze sledovat v logu umístěném v in CrawlerLogs folder")
     
     config = configure_functions.load_configuration(dataset_name)
     crawl_data(config)

modules/crawler/reset_datasets.py

@@ -13 +13 @@
 CONFIG_FILES_PATH = "DatasetConfigs"
 
 
-def reset_dataset(dataset_name):
+def hard_reset_dataset(dataset_name):
     """
     Resets all saved data in dataset except config and implementation
     Args:
@@ -29 +29 @@
     database_loader.remove_dataset_database(dataset_name)
 
 
-def reset_all_datasets():
+def soft_reset_dataset(dataset_name):
+    """
+    Resets all saved data in dataset except config and implementation
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    path = PROCESSED_DATA_PATH + dataset_name + "/"
+    folder_processor.clean_folder(path)
+
+    database_loader.remove_dataset_database(dataset_name)
+
+
+
+def soft_reset_all_datasets():
     """
     Resets all saved data in all datasets with config file except configs and implementation
     """
     datasets = os.listdir(CONFIG_FILES_PATH)
 
     for dataset in datasets:
-        reset_dataset(dataset.split('.')[0])
+        soft_reset_dataset(dataset.split('.')[0])
 
 
+def hard_reset_all_datasets():
+    """
+    Resets all saved data in all datasets with config file except configs and implementation
+    """
+    datasets = os.listdir(CONFIG_FILES_PATH)
+
+    for dataset in datasets:
+        hard_reset_dataset(dataset.split('.')[0])
+
 
 print("Zadejte jméno Datasetu který chcete resetovat (pokud všechny zadejte '-ALL'):\n")
 
 dataset_name = input().upper()
 
+print("Chcete smazat i stažené stažená data ? (ANO/NE) \n")
+
+vstup = input().upper()
+
 if dataset_name == '-ALL':
-    reset_all_datasets()
+    if vstup == 'ANO':
+        hard_reset_all_datasets()
+    elif vstup == 'NE':
+        soft_reset_all_datasets()
+    else:
+        print('Neplatný vstup (ANO/NE)')
 else:
     test = configure_functions.check_if_there_is_a_config_file(dataset_name)
     if test == True:
-        reset_dataset(dataset_name)
+        if vstup == 'ANO':
+            hard_reset_dataset(dataset_name)
+        elif vstup == 'NE':
+            soft_reset_dataset(dataset_name)
+        else:
+            print('Neplatný vstup (ANO/NE)')
     else:
         print("Tento dataset v architektuře neexistuje")
+