Revize af7609b5

Re #8193 - refactoring crawler

modules/crawler/pipeline.py

@@ -1 +1 @@
 from Utilities import folder_processor, configure_functions
 from Utilities.Database import database_loader, database_record_logs
 from Utilities.CSV import csv_utils
+from shared_types import ConfigType
 import os
 import pymongo
 
@@ -20 +21 @@
 
 #logger
 logging.basicConfig(filename=CRAWLER_LOGS_PATH + 'Applicationlog-' +
-                               date.today().strftime("%b-%Y") + '.log',
-                               level=logging.INFO,
-                               format='%(asctime)s %(message)s')
+                    date.today().strftime("%b-%Y") + '.log',
+                    level=logging.INFO,
+                    format='%(asctime)s %(message)s')
 
 
-def check_last_update(config):
+def check_last_update(config: ConfigType) -> bool:
     """
     Loads integer from updated.txt in CrawlerLogs/"dataset_name"
     representing number of days from last update if number equals
@@ -56 +57 @@
         return False
 
 
-def crawl_data(config):
+def crawl_data(config: ConfigType) -> None:
     """
       Imports dataset crawler in DatasetCrawler/"dataset_name"_crawler.py
       runs crawler.
@@ -73 +74 @@
     dataset_name += '/'
 
 
-def process_data(config):
+def process_data(config: ConfigType) -> None:
     """
     Goes trough every not processed file(list of processed files is saved in databse)
     Imports dataset processor in DatasetProcessing/"dataset_name"_processor.py
@@ -100 +101 @@
         path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
         date_dic = process_file_func(path)
         csv_utils.export_data_to_csv(path, date_dic)
+        print("Vytvářím: " + not_processed_file)
         database_record_logs.update_ignore_set_processed(
             dataset_name, not_processed_file)
 
@@ -107 +109 @@
                  str(len(not_processed_files)) + " newly crawled files")
 
 
-def process_data_crone(config):
+def process_data_crone(config: ConfigType) -> None:
     """
     Goes trough every not processed file(list of processed files is saved in database)
     Imports dataset processor in DatasetProcessing/"dataset_name"_processor.py
@@ -146 +148 @@
                  str(len(not_processed_files)) + " newly crawled files")
 
 
-def validate_process_data(config):
+def validate_process_data(config: ConfigType) -> bool:
     """
     Function goes through newly processed data and checks theirs status
 
@@ -186 +188 @@
     return True
 
 
-def load_data_to_database(config):
+def load_data_to_database(config: ConfigType) -> None:
     """
     Goes trough every not loaded file(list of loaded files is saved in database)
     loads data appends coordination from configurations
@@ -207 +209 @@
     changes_in_devices = database_loader.update_devices_collection(config)
 
     if changes_in_devices == True:
-        logging.info(
-            dataset_name +
-            " contains changes in devices configuration. Deleting old data and preparing new"
-        )
+        logg_string = dataset_name + " contains changes in devices configuration. Deleting old data and preparing new"
+        logg_string_cs = dataset_name + " obsahuje změny v konfiguračním souboru. Probíha odstraňování starých dat a připravení nových."
+        logging.info(logg_string)
+        print(logg_string_cs)
         database_loader.reset_dataset_database(dataset_name)
 
     # get all unprocessed files from dataset
@@ -230 +232 @@
         database_record_logs.update_ignore_set_loaded(dataset_name,
                                                       not_loaded_file)
 
-    logging.info(dataset_name + " has loaded to database " +
-                 str(len(not_loaded_files)) + " newly processed files.")
+    logg_string = dataset_name + " has loaded to database " + str(
+        len(not_loaded_files)) + " newly processed files."
+    logg_string_cs = dataset_name + " načetl " + str(
+        len(not_loaded_files)) + " nových zpracovaných souborů \n"
+
+    logging.info(logg_string)
+    print(logg_string_cs)
 
     client = pymongo.MongoClient()
     client.close()
 
 
-def load_data_to_database_crone(config):
+def load_data_to_database_crone(config: ConfigType) -> None:
     """
     Goes trough every not loaded file(list of loaded files is saved in database)
     loads data appends coordination from configurations
@@ -280 +287 @@
     client.close()
 
 
-def run_full_pipeline(dataset_name):
+def run_full_pipeline(dataset_name: str) -> None:
     """
     Loads config file and starts full pipeline
     -crawl data
@@ -292 +299 @@
     """
     logging.info("Starting pipeline for dataset " + dataset_name)
     print("Zpracovávám dataset " + dataset_name +
-          " průběh lze sledovat v logu umístěném v in CrawlerLogs folder")
+          ", průběh lze sledovat v logu umístěném v adresáři CrawlerLogs")
 
     config = configure_functions.load_configuration(dataset_name)
     crawl_data(config)
@@ -304 +311 @@
         load_data_to_database(config)
 
 
-def run_full_pipeline_crone(dataset_name):
+def run_full_pipeline_crone(dataset_name: str) -> None:
     """
     Loads config file and starts full pipeline
     -crawl data