Revize 2d129043

Re #7939
- upravena struktura processorů v pipeline
- pridani kontroly validity dat

modules/crawler/.gitignore

@@ -1 +1 @@
 *__pycache__*
 *.CSV
-python-module/venv
+CrawlerLogs/CommonRecords/*

modules/crawler/DatasetProcessing/JISProcessor.py

@@ -17 +17 @@
         False if not implemented
         True when implemented
     """
-    with open(filename, "r", encoding="utf-8") as file:
+    date_dict = dict()
 
-        date_dict = dict()
+    with open(filename, "r", encoding="utf-8") as file:
 
         for line in file:
 
@@ -35 +35 @@
             if name in date_dict[date]:
                 date_dict[date][name].occurrence += int(occurrence)
             else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, int(occurrence))
+                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, occurrence)
 
-    CSVutils.export_data_to_csv(filename, date_dict)
-    return True
+    return date_dict
 

modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py

@@ -17 +17 @@
         False if not implemented
         True when implemented
     """
-    with open(filename, "r") as file:
+    date_dict = dict()
 
-        date_dict = dict()
+    with open(filename, "r") as file:
 
         for line in file:
 
@@ -36 +36 @@
             else:
                 date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, 1)
 
-    CSVutils.export_data_to_csv(filename, date_dict)
-    return True
+    return date_dict
 

modules/crawler/DatasetProcessing/WIFIProcessor.py

@@ -17 +17 @@
         False if not implemented
         True when implemented
     """
+    date_dict = dict()
+    
     with open(filename, "r", encoding="utf-8") as file:
 
-        date_dict = dict()
-
         for line in file:
 
             array = line.split(";")
@@ -37 +37 @@
             else:
                 date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, int(occurrence))
 
-    CSVutils.export_data_to_csv(filename, date_dict)
-    return True
+    return date_dict
 

modules/crawler/Pipeline.py

@@ -1 +1 @@
 from Utilities import FolderProcessor, ConfigureFunctions
 from Utilities.Database import DatabaseLoader
+from Utilities.CSV import CSVutils
 
 import logging
 from datetime import date
@@ -96 +97 @@
     logging.info(dataset_name + " has downloaded " + str(len(not_processed_files)) + " new files")
 
     for not_processed_file in not_processed_files:
-        process_file_func(CRAWLED_DATA_PATH + dataset_path + not_processed_file)
+        path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
+        date_dic = process_file_func(path)
+        CSVutils.export_data_to_csv(path, date_dic)
         FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
 
     logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")

modules/crawler/PrepareNewDataset.py

@@ -48 +48 @@
         file.write("\n")
         file.write("def process_file(filename):\n")
         file.write("    \"\"\"\n")
-        file.write("    Method that take path to crawled file and outputs date dictionary using method:\n")
-        file.write("    CSVutils.export_data_to_csv(filename, date_dict)\n")
+        file.write("    Method that take path to crawled file and outputs date dictionary:\n")
         file.write("    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)\n")
         file.write("    and value is dictionary where keys devices (specified in configuration file)\n")
         file.write("    and value is CSVDataLine.CSVDataLine with device,date and occurrence\n")
@@ -58 +57 @@
         file.write("    filename: name of processed file\n")
         file.write("\n")
         file.write("    Returns:\n")
-        file.write("    False if not implemented\n")
-        file.write("    True when implemented\n")
+        file.write("    None if not implemented\n")
+        file.write("    date_dict when implemented\n")
         file.write("    \"\"\"\n")
+        file.write("    date_dict = dict()\n")
+        file.write("\n")
         file.write("    #with open(filename, \"r\") as file:\n")
         file.write("    print(\"You must implements process_file method first!\")\n")
-        file.write("    #CSVutils.export_data_to_csv(filename, date_dict)\n")
-        file.write("    return False\n")
+        file.write("    return None\n")
 
 
 def create_default_crawler(dataset_name):

modules/crawler/Utilities/CSV/CSVDataLine.py

@@ -3 +3 @@
     Class that specifies the look of data line in processed csv file
     prepared for database
     """
+
     def __init__(self, name, date, occurrence):
+        try:
+            test_val = int(occurrence)
+        except ValueError:
+            print("Occurence should be and integer value!")
+
+        if len(date) != 10:
+            raise ValueError("Invalid date format ddmmYYYYhh expected!")    
+
         self.name = name
         self.date = date
-        self.occurrence = occurrence
+        self.occurrence = test_val
 
     def to_csv(self):
         return self.name + ";" + str(self.occurrence) + ";" + self.date

modules/crawler/Utilities/CSV/CSVutils.py

@@ +1 @@
+import inspect
+from Utilities.CSV import CSVDataLine
+
 # Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
 
@@ -36 +39 @@
     with open(PROCESSED_DATA_PATH + filename[12:], "w+") as file:
 
         for date in data_dict:
+            if len(date) != 10:
+                raise ValueError("Invalid date format for key value --> ddmmYYYYhh expected!")   
             for data in data_dict[date]:
-                file.write(data_dict[date][data].to_csv() + '\n')
+                csv_line = data_dict[date][data]
+                if not isinstance(csv_line,CSVDataLine.CSVDataLine):
+                    raise ValueError("data_dict is expected to have CSVDataLine as values")
+                file.write(csv_line.to_csv() + '\n')

modules/crawler/Utilities/Database/DatabaseLoader.py

@@ -62 +62 @@
         csv_column = line.split(";")
 
         name = csv_column[0]
+
         occurrence = csv_column[1]
         date = csv_column[2]
 
+
+
         database_data_line = DatabaseDataLine.DatabaseDataLine(name, devices[name]["x"]
                                                                , devices[name]["y"], date, occurrence)