commit 2d1290438085a4a6fbf4bde52aa869a47af92f87
Author: petrh <petrh@zcu.cz>
Date:   Wed May 6 12:07:28 2020 +0200

    Re #7939
    - upravena struktura processorů v pipeline
    - pridani kontroly validity dat

diff --git a/modules/crawler/.gitignore b/modules/crawler/.gitignore
index bccd59e..d856eb2 100644
--- a/modules/crawler/.gitignore
+++ b/modules/crawler/.gitignore
@@ -1,3 +1,3 @@
 *__pycache__*
 *.CSV
-python-module/venv
+CrawlerLogs/CommonRecords/*
\ No newline at end of file
diff --git a/modules/crawler/DatasetProcessing/JISProcessor.py b/modules/crawler/DatasetProcessing/JISProcessor.py
index 5cbde73..29138d8 100644
--- a/modules/crawler/DatasetProcessing/JISProcessor.py
+++ b/modules/crawler/DatasetProcessing/JISProcessor.py
@@ -17,9 +17,9 @@ def process_file(filename):
         False if not implemented
         True when implemented
     """
-    with open(filename, "r", encoding="utf-8") as file:
+    date_dict = dict()
 
-        date_dict = dict()
+    with open(filename, "r", encoding="utf-8") as file:
 
         for line in file:
 
@@ -35,8 +35,7 @@ def process_file(filename):
             if name in date_dict[date]:
                 date_dict[date][name].occurrence += int(occurrence)
             else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, int(occurrence))
+                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, occurrence)
 
-    CSVutils.export_data_to_csv(filename, date_dict)
-    return True
+    return date_dict
 
diff --git a/modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py b/modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py
index eba863a..48d9c07 100644
--- a/modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py
+++ b/modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py
@@ -17,9 +17,9 @@ def process_file(filename):
         False if not implemented
         True when implemented
     """
-    with open(filename, "r") as file:
+    date_dict = dict()
 
-        date_dict = dict()
+    with open(filename, "r") as file:
 
         for line in file:
 
@@ -36,6 +36,5 @@ def process_file(filename):
             else:
                 date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, 1)
 
-    CSVutils.export_data_to_csv(filename, date_dict)
-    return True
+    return date_dict
 
diff --git a/modules/crawler/DatasetProcessing/WIFIProcessor.py b/modules/crawler/DatasetProcessing/WIFIProcessor.py
index e059c68..270a935 100644
--- a/modules/crawler/DatasetProcessing/WIFIProcessor.py
+++ b/modules/crawler/DatasetProcessing/WIFIProcessor.py
@@ -17,10 +17,10 @@ def process_file(filename):
         False if not implemented
         True when implemented
     """
+    date_dict = dict()
+    
     with open(filename, "r", encoding="utf-8") as file:
 
-        date_dict = dict()
-
         for line in file:
 
             array = line.split(";")
@@ -37,6 +37,5 @@ def process_file(filename):
             else:
                 date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, int(occurrence))
 
-    CSVutils.export_data_to_csv(filename, date_dict)
-    return True
+    return date_dict
 
diff --git a/modules/crawler/Pipeline.py b/modules/crawler/Pipeline.py
index 39b3c1b..21df4fa 100644
--- a/modules/crawler/Pipeline.py
+++ b/modules/crawler/Pipeline.py
@@ -1,5 +1,6 @@
 from Utilities import FolderProcessor, ConfigureFunctions
 from Utilities.Database import DatabaseLoader
+from Utilities.CSV import CSVutils
 
 import logging
 from datetime import date
@@ -96,7 +97,9 @@ def process_data(dataset_name):
     logging.info(dataset_name + " has downloaded " + str(len(not_processed_files)) + " new files")
 
     for not_processed_file in not_processed_files:
-        process_file_func(CRAWLED_DATA_PATH + dataset_path + not_processed_file)
+        path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
+        date_dic = process_file_func(path)
+        CSVutils.export_data_to_csv(path, date_dic)
         FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
 
     logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")
diff --git a/modules/crawler/PrepareNewDataset.py b/modules/crawler/PrepareNewDataset.py
index fa5a3df..af1cf77 100644
--- a/modules/crawler/PrepareNewDataset.py
+++ b/modules/crawler/PrepareNewDataset.py
@@ -48,8 +48,7 @@ def create_default_processor(dataset_name):
         file.write("\n")
         file.write("def process_file(filename):\n")
         file.write("    \"\"\"\n")
-        file.write("    Method that take path to crawled file and outputs date dictionary using method:\n")
-        file.write("    CSVutils.export_data_to_csv(filename, date_dict)\n")
+        file.write("    Method that take path to crawled file and outputs date dictionary:\n")
         file.write("    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)\n")
         file.write("    and value is dictionary where keys devices (specified in configuration file)\n")
         file.write("    and value is CSVDataLine.CSVDataLine with device,date and occurrence\n")
@@ -58,13 +57,14 @@ def create_default_processor(dataset_name):
         file.write("    filename: name of processed file\n")
         file.write("\n")
         file.write("    Returns:\n")
-        file.write("    False if not implemented\n")
-        file.write("    True when implemented\n")
+        file.write("    None if not implemented\n")
+        file.write("    date_dict when implemented\n")
         file.write("    \"\"\"\n")
+        file.write("    date_dict = dict()\n")
+        file.write("\n")
         file.write("    #with open(filename, \"r\") as file:\n")
         file.write("    print(\"You must implements process_file method first!\")\n")
-        file.write("    #CSVutils.export_data_to_csv(filename, date_dict)\n")
-        file.write("    return False\n")
+        file.write("    return None\n")
 
 
 def create_default_crawler(dataset_name):
diff --git a/modules/crawler/Utilities/CSV/CSVDataLine.py b/modules/crawler/Utilities/CSV/CSVDataLine.py
index 577af78..fa25580 100644
--- a/modules/crawler/Utilities/CSV/CSVDataLine.py
+++ b/modules/crawler/Utilities/CSV/CSVDataLine.py
@@ -3,10 +3,19 @@ class CSVDataLine:
     Class that specifies the look of data line in processed csv file
     prepared for database
     """
+
     def __init__(self, name, date, occurrence):
+        try:
+            test_val = int(occurrence)
+        except ValueError:
+            print("Occurence should be and integer value!")
+
+        if len(date) != 10:
+            raise ValueError("Invalid date format ddmmYYYYhh expected!")    
+
         self.name = name
         self.date = date
-        self.occurrence = occurrence
+        self.occurrence = test_val
 
     def to_csv(self):
         return self.name + ";" + str(self.occurrence) + ";" + self.date
diff --git a/modules/crawler/Utilities/CSV/CSVutils.py b/modules/crawler/Utilities/CSV/CSVutils.py
index c68e2f6..1111ab3 100644
--- a/modules/crawler/Utilities/CSV/CSVutils.py
+++ b/modules/crawler/Utilities/CSV/CSVutils.py
@@ -1,3 +1,6 @@
+import inspect
+from Utilities.CSV import CSVDataLine
+
 # Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
 
@@ -36,5 +39,10 @@ def export_data_to_csv(filename, data_dict):
     with open(PROCESSED_DATA_PATH + filename[12:], "w+") as file:
 
         for date in data_dict:
+            if len(date) != 10:
+                raise ValueError("Invalid date format for key value --> ddmmYYYYhh expected!")   
             for data in data_dict[date]:
-                file.write(data_dict[date][data].to_csv() + '\n')
\ No newline at end of file
+                csv_line = data_dict[date][data]
+                if not isinstance(csv_line,CSVDataLine.CSVDataLine):
+                    raise ValueError("data_dict is expected to have CSVDataLine as values")
+                file.write(csv_line.to_csv() + '\n')
\ No newline at end of file
diff --git a/modules/crawler/Utilities/Database/DatabaseLoader.py b/modules/crawler/Utilities/Database/DatabaseLoader.py
index 9409c09..d52664d 100644
--- a/modules/crawler/Utilities/Database/DatabaseLoader.py
+++ b/modules/crawler/Utilities/Database/DatabaseLoader.py
@@ -62,9 +62,12 @@ def get_data_from_file(filename, config):
         csv_column = line.split(";")
 
         name = csv_column[0]
+
         occurrence = csv_column[1]
         date = csv_column[2]
 
+
+
         database_data_line = DatabaseDataLine.DatabaseDataLine(name, devices[name]["x"]
                                                                , devices[name]["y"], date, occurrence)
 
