commit 43697fec59f7d0cc9af2604af6351ff75f0ed089
Author: petrh <petrh@zcu.cz>
Date:   Tue May 5 15:20:33 2020 +0200

    Re #7937
    Pridano logovani do slozky CrawlerLogs/CommonRecords

diff --git a/modules/crawler/CrawlerLogs/JISupdated.txt b/modules/crawler/CrawlerLogs/JISupdated.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/modules/crawler/CrawlerLogs/updated.txt b/modules/crawler/CrawlerLogs/updated.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/modules/crawler/Pipeline.py b/modules/crawler/Pipeline.py
index 5fd916b..39b3c1b 100644
--- a/modules/crawler/Pipeline.py
+++ b/modules/crawler/Pipeline.py
@@ -1,6 +1,10 @@
 from Utilities import FolderProcessor, ConfigureFunctions
 from Utilities.Database import DatabaseLoader
 
+import logging
+from datetime import date
+
+
 # Path to crawled data
 CRAWLED_DATA_PATH = "CrawledData/"
 # Path to processed data
@@ -13,6 +17,13 @@ CRAWLER_LIB_PATH = "DatasetCrawler."
 PROCESSOR_LIB_PATH = "DatasetProcessing."
 
 
+#logger
+logging.basicConfig(filename=CRAWLER_LOGS_PATH + "CommonRecords/" + 'Applicationlog-' + date.today().strftime("%b-%Y") + '.log',
+                    level=logging.INFO,
+                    format='%(asctime)s %(message)s'
+                    )
+
+
 def check_last_update(config):
     """
     Loads integer from updated.txt in CrawlerLogs/"dataset_name"
@@ -37,13 +48,13 @@ def check_last_update(config):
         confing_update_period = int(config["update-period"])
 
         if config["update-period"] <= last_update:
-            print("Dataset " + dataset_name + " is being updated")
+            logging.info("Dataset " + dataset_name + " is being updated today")
             file.write("0")
             file.truncate()
             return True
         else:
             last_update_days = last_update + 1
-            print("Dataset " + dataset_name + " will be updated in " + str(confing_update_period - last_update_days) + "days")
+            logging.info("Dataset " + dataset_name + " will be updated in " + str(confing_update_period - last_update_days) + "days")
             file.write(str(last_update_days))
             file.truncate()
             return False
@@ -60,8 +71,8 @@ def crawl_data(config):
     """
     dataset_name = config["dataset-name"]
 
-    my_function = __import__(CRAWLER_LIB_PATH + dataset_name + "Crawler", globals(), locals(), ['crawl']).crawl
-    my_function(config)
+    crawl_func = __import__(CRAWLER_LIB_PATH + dataset_name + "Crawler", globals(), locals(), ['crawl']).crawl
+    crawl_func(config)
 
     dataset_name += '/'
 
@@ -82,11 +93,14 @@ def process_data(dataset_name):
                                    ['process_file']).process_file
 
     not_processed_files = FolderProcessor.list_of_all_files(CRAWLED_DATA_PATH + dataset_path)
+    logging.info(dataset_name + " has downloaded " + str(len(not_processed_files)) + " new files")
 
     for not_processed_file in not_processed_files:
         process_file_func(CRAWLED_DATA_PATH + dataset_path + not_processed_file)
         FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
 
+    logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")
+
 
 def validate_process_data(config):
     """
@@ -105,15 +119,15 @@ def validate_process_data(config):
     unknown_devices_size = len(unknown_devices_set)
 
     if unknown_devices_size != 0:
-        print("There is " + str(unknown_devices_size) + " unknown devices\n")
-        print("Adding devices to " + config["dataset-name"] + " config file\n")
+        logging.info("There is " + str(unknown_devices_size) + " unknown devices")
+        logging.info("Adding devices to " + config["dataset-name"] + " config file")
         ConfigureFunctions.update_configuration(config["dataset-name"], unknown_devices_set)
         return False
 
     for device in config["devices"]:
         device = config["devices"][device]
         if device["x"] == "UNKNOWN!" or device["y"] == "UNKNOWN!":
-            print(config["dataset-name"] + " Config file contains devices with UNKOWN! values please update them\n")
+            logging.info(config["dataset-name"] + " config file contains devices with UNKOWN! values please update them!!")
             return False
 
     return True
@@ -143,6 +157,8 @@ def load_data_to_database(config):
         DatabaseLoader.load_data_to_database(dataset_name, processed_data)
         FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
 
+    logging.info(dataset_name + " has loaded to databse " + str(len(not_loaded_files)) + " newly processed files.")
+
 
 def run_full_pipeline(dataset_name):
     """
@@ -154,6 +170,8 @@ def run_full_pipeline(dataset_name):
     Args:
         dataset_name: name of dataset that has existing configuration file
     """
+    logging.info("Starting pipeline for dataset " + dataset_name)
+    
     config = ConfigureFunctions.load_configuration(dataset_name)
     crawl_data(config)
     process_data(config["dataset-name"])
@@ -162,7 +180,6 @@ def run_full_pipeline(dataset_name):
 
     if validation_test:
         load_data_to_database(config)
-        print("Dataset " + dataset_name + " has been sucessfully updated\n")
 
 
 
@@ -176,6 +193,8 @@ def run_full_pipeline_crone(dataset_name):
     Args:
         dataset_name: name of dataset that has existing configuration file
     """
+    logging.info("Starting pipeline for dataset " + dataset_name)
+
     config = ConfigureFunctions.load_configuration(dataset_name)
     update_test = check_last_update(config)
     if update_test:
@@ -185,4 +204,5 @@ def run_full_pipeline_crone(dataset_name):
         validation_test = validate_process_data(config)
 
         if validation_test:
-            load_data_to_database(config)
\ No newline at end of file
+            load_data_to_database(config)
+            
\ No newline at end of file
