Revize 81980e82

Re #8160 new dataset

modules/crawler/Utilities/Database/database_loader.py

@@ -1 +1 @@
 from Utilities.Database import database_data_line, database_record_logs
 from Utilities import configure_functions
+from Utilities.helpers import should_skip
 import pymongo
 import re
 
@@ -67 +68 @@
 
         name = csv_column[0]
 
-        if devices[name]["x"] == "SKIP" or devices[name]["y"] == "SKIP":
+        if should_skip(devices[name]):
             continue
 
         occurrence = csv_column[1]
@@ -80 +81 @@
         if date_without_hours not in date_dict:
             date_dict[date_without_hours] = list()
 
-        date_dict[date_without_hours].append(
-            data_line.to_dictionary())
+        date_dict[date_without_hours].append(data_line.to_dictionary())
 
     return date_dict
 
 
-def load_data_to_database(database_connection,dataset_name, data_dic, file_name):
+def load_data_to_database(database_connection, dataset_name, data_dic,
+                          file_name):
     """
     Takes data_dic created in method get_data_from_file
     and loads into into database where collection name is dataset_name + data_dic key
@@ -106 +107 @@
         date_dataset.insert_many(data_dic[date])
 
 
-
-def check_or_update_datasets_collection(database_connection,config):
+def check_or_update_datasets_collection(database_connection, config):
     """
     Checks if DATASETS collection contains dataset and if display name was not updated
 
@@ -127 +127 @@
     dataset_present = collection_datasets.find_one(query)
 
     if dataset_present is None:
-        collection_datasets.insert_one({'key-name': dataset_name, 'display-name': display_name,'updated': 0})
+        collection_datasets.insert_one({
+            'key-name': dataset_name,
+            'display-name': display_name,
+            'updated': 0
+        })
     elif dataset_present['display-name'] != display_name:
-        newvalues = { "$set": { 'display-name': display_name } }
+        newvalues = {"$set": {'display-name': display_name}}
         collection_datasets.update_one(query, newvalues)
 
 
@@ -153 +157 @@
 
     change_in_devices = False
 
-    collection_devices = database_connection[dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
+    collection_devices = database_connection[
+        dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
 
     devices_cursor = collection_devices.find()
 
@@ -161 +166 @@
 
     for device in devices_cursor:
         name = device['name']
-        db_device_dict[name] = {'name': name, 'x': device['x'] , 'y': device['y']}
-
+        db_device_dict[name] = {
+            'name': name,
+            'x': device['x'],
+            'y': device['y']
+        }
 
-    valid_devices = configure_functions.return_dictionary_of_valid_devices(devices)
+    valid_devices = configure_functions.return_dictionary_of_valid_devices(
+        devices)
 
     if len(valid_devices.keys()) != len(db_device_dict.keys()):
         change_in_devices = True
-    
+
     if change_in_devices == False:
         for device in valid_devices.keys():
             if device in db_device_dict:
@@ -180 +189 @@
                     change_in_devices = True
                     break
 
-
     if change_in_devices == True:
         collection_devices.delete_many({})
         devices_list = list()
 
         for device in devices.keys():
-            x = devices[device]['x']
-            y = devices[device]['y']
-            if not (x == "SKIP" or x == "UNKNOWN!" or y == "SKIP" or y == "UNKNOWN!"):
-                devices_list.append({'name': device , 'x': x , 'y': y })
+            if not (should_skip(devices[device])):
+                devices_list.append({
+                    'name': device,
+                    'x': devices[device]['x'],
+                    'y': devices[device]['y']
+                })
 
         collection_devices.insert_many(devices_list)
 
     return change_in_devices
 
-    
+
 def remove_dataset_database(dataset_name):
     """
     Removes dataset entries from database
@@ -211 +221 @@
     collection_datasets.delete_one({"key-name": dataset_name})
     print("Removing record from DATASETS collection")
 
-
     # Retrieve list of all collections
     collections = mydb.list_collection_names()
 
@@ -232 +241 @@
     # Creating connection
     mydb = create_database_connection()
 
-    pattern = re.compile(dataset_name+'[0-9]+-[0-9]+-+[0-9]+')
+    pattern = re.compile(dataset_name + '[0-9]+-[0-9]+-+[0-9]+')
 
     # Retrieve list of all collections
     collections = mydb.list_collection_names()