Revize 2d129043

Re #7939
- upravena struktura processorů v pipeline
- pridani kontroly validity dat

modules/crawler/Pipeline.py

@@ -1 +1 @@
 from Utilities import FolderProcessor, ConfigureFunctions
 from Utilities.Database import DatabaseLoader
+from Utilities.CSV import CSVutils
 
 import logging
 from datetime import date
@@ -96 +97 @@
     logging.info(dataset_name + " has downloaded " + str(len(not_processed_files)) + " new files")
 
     for not_processed_file in not_processed_files:
-        process_file_func(CRAWLED_DATA_PATH + dataset_path + not_processed_file)
+        path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
+        date_dic = process_file_func(path)
+        CSVutils.export_data_to_csv(path, date_dic)
         FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
 
     logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")