Revize 753d424e

Predelany komentáře upravena kontrola datasetu

modules/crawler/pipeline.py

@@ -86 +86 @@
     dataset_name = config["dataset-name"]
     dataset_path = dataset_name + '/'
 
-    changes_in_devices = database_loader.update_devices_collection(config)
-
-    if changes_in_devices == True:
-        logging.info(dataset_name + " contains changes in devices configuration. Deleteing old data and preparing new")
-        database_loader.reset_dataset_database(dataset_name)
-        folder_processor.clean_folder(PROCESSED_DATA_PATH + dataset_path)
-
-
     process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "_processor", globals(), locals(),
                                    ['process_file']).process_file
 
@@ -194 +186 @@
 
     database_loader.check_or_update_datasets_collection(database_connection,config)
 
+    changes_in_devices = database_loader.update_devices_collection(config)
+
+    if changes_in_devices == True:
+        logging.info(dataset_name + " contains changes in devices configuration. Deleting old data and preparing new")
+        database_loader.reset_dataset_database(dataset_name)
+
     # get all unprocessed files from dataset
     ignore_set = database_record_logs.load_ignore_set_loaded(dataset_name)
     not_loaded_files = folder_processor.list_of_all_new_files(ignore_set,PROCESSED_DATA_PATH + dataset_path)
@@ -261 +259 @@
         dataset_name: name of dataset that has existing configuration file
     """
     logging.info("Starting pipeline for dataset " + dataset_name)
-    print("Processing dataset " + dataset_name + " you can watch the progress in log contained in CrawlerLogs folder")
+    print("Zpracovávám dataset " + dataset_name + " průběh lze sledovat v logu umístěném v in CrawlerLogs folder")
     
     config = configure_functions.load_configuration(dataset_name)
     crawl_data(config)