Revize 27eee939

fix for bad configuration of dataset

modules/crawler/pipeline.py

@@ -298 +298 @@
         dataset_name: name of dataset that has existing configuration file
     """
     logging.info("Starting pipeline for dataset " + dataset_name)
-    print("Zpracovávám dataset " + dataset_name +
-          ", průběh lze sledovat v logu umístěném v adresáři CrawlerLogs")
+    try:
+        print("Zpracovávám dataset " + dataset_name +
+              ", průběh lze sledovat v logu umístěném v adresáři CrawlerLogs")
 
-    config = configure_functions.load_configuration(dataset_name)
-    crawl_data(config)
-    process_data(config)
+        config = configure_functions.load_configuration(dataset_name)
+        crawl_data(config)
+        process_data(config)
 
-    validation_test = validate_process_data(config)
+        validation_test = validate_process_data(config)
 
-    if validation_test:
-        load_data_to_database(config)
+        if validation_test:
+            load_data_to_database(config)
+    except Exception as e:
+        logging.error("Pipeline for " + dataset_name + " failed")
+        logging.error(e)
 
 
 def run_full_pipeline_crone(dataset_name: str) -> None:
@@ -323 +327 @@
     """
     logging.info("Starting pipeline for dataset " + dataset_name)
 
-    config = configure_functions.load_configuration(dataset_name)
-    update_test = check_last_update(config)
-    if update_test:
-        crawl_data(config)
-        process_data_crone(config["dataset-name"])
+    try:
+        config = configure_functions.load_configuration(dataset_name)
+        update_test = check_last_update(config)
+        if update_test:
+            crawl_data(config)
+            process_data_crone(config["dataset-name"])
 
-        validation_test = validate_process_data(config)
+            validation_test = validate_process_data(config)
 
-        if validation_test:
-            load_data_to_database_crone(config)
+            if validation_test:
+                load_data_to_database_crone(config)
+    except:
+        print("Pipeline for dataset " + dataset_name + " failed")