Revize 0a2832fb

Re #8089
- pridana nova proměna do konfig souboru
- predalana tabulka v databasi s nazvem DATASETS
- nove obsahuje "key-name" jako klíč a "display-name" pro zobrazení

modules/crawler/Utilities/Database/DatabaseLoader.py

@@ -11 +11 @@
 MONGODB_DATA_DATABASE = "open-data-db"
 # mongodb collection with aviable datasets
 MONGODB_DATASET_COLLECTION = "DATASETS"
+# mongodb collection with aviable diveces of datasets
+MONGODB_DATASET_DEVICES_COLLECTION = "DEVICES"
+# mongodb collection with aviable diveces of datasets
+MONGODB_DATASET_LOADED_FILES_COLLECTION = "FILES"
 
 # Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
@@ -63 +67 @@
 
         name = csv_column[0]
 
+        if devices[name]["x"] == "SKIP" or devices[name]["y"] == "SKIP":
+            continue
+
         occurrence = csv_column[1]
         date = csv_column[2]
         database_data_line = DatabaseDataLine.DatabaseDataLine(
@@ -79 +86 @@
     return date_dict
 
 
-def load_data_to_database(dataset_name, data_dic):
+def load_data_to_database(database_connection,dataset_name, data_dic, file_name):
     """
     Takes data_dic created in method get_data_from_file
     and loads into into database where collection name is dataset_name + data_dic key
     and data lines are line in collection
 
     Args:
-        dataset_name: name of dataset that has existing configuration file
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
         data_dic: dictionary of data lines created in get_data_from_file
+        file_name: name of file containing data
+    """
+
+    for date in data_dic:
+        dataset_collections = database_connection[dataset_name]
+        dataset_collections.insert_one({'name': dataset_name + date})
+        date_dataset = database_connection[dataset_name + date]
+        date_dataset.insert_many(data_dic[date])
+
+    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]
+    collection_loaded_files.insert_one({'file': file_name})
+
+
+
+def check_or_update_datasets_collection(database_connection,config):
     """
-    database = create_database_connection()
+    Checks if DATASETS collection contains dataset and if display name was not updated
 
+    Args:
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
+    """
     # collection where are specified aviable datasets
-    collection_datasets = database[MONGODB_DATASET_COLLECTION]
+    collection_datasets = database_connection[MONGODB_DATASET_COLLECTION]
+
+    dataset_name = config['dataset-name']
+    display_name = config['display-name']
+
+    query = {'key-name': dataset_name}
 
     # check if newly added data already have a dataset specified in collection
-    dataset_present = collection_datasets.find_one({'name': dataset_name})
+    dataset_present = collection_datasets.find_one(query)
 
     if dataset_present is None:
-        collection_datasets.insert_one({'name': dataset_name})
+        collection_datasets.insert_one({'key-name': dataset_name, 'display-name': display_name})
+    elif dataset_present['display-name'] != display_name:
+        newvalues = { "$set": { 'display-name': display_name } }
+        collection_datasets.update_one(query, newvalues)
 
-    for date in data_dic:
-        dataset_collections = database[dataset_name]
-        dataset_collections.insert_one({'name': dataset_name + date})
-        date_dataset = database[dataset_name + date]
-        date_dataset.insert_many(data_dic[date])
+
+def update_devices_collection(database_connection,config):
+    """
+    Checks if dataset_name collection contains every device with current set up
+
+    Args:
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config['dataset-name']
+
+    collection_devices = database_connection[dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
+
+    collection_devices.delete_many({})
+
+    devices = config['devices']
+
+    devices_list = list()
+
+    for device in devices.keys():
+        if devices[device]['x'] != "SKIP" or devices[device]['y'] != "SKIP":
+            devices_list.append({'name': device , 'x': devices[device]['x'] , 'y': devices[device]['y'] })
+
+    collection_devices.insert_many(devices_list)
+
+
+def check_if_database_doesnt_contain_file(database_connection,dataset_name,file_name):
+    """
+    Checks if dataset_name collection contains every device with current set up
+
+    Args:
+        database_connection: created connection to a MONGODB
+        filename: checked file name
+    """
+
+    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]
+
+    query = {'file': file_name}
+
+    # check if newly added data already have a dataset specified in collection
+    dataset_present = collection_loaded_files.find_one(query)
+
+    if dataset_present is None:
+        return True
+    else:
+        return False