Revize 0a2832fb

Re #8089
- pridana nova proměna do konfig souboru
- predalana tabulka v databasi s nazvem DATASETS
- nove obsahuje "key-name" jako klíč a "display-name" pro zobrazení

modules/crawler/DatasetConfigs/JIS.yaml

@@ -1 +1 @@
 # jmeno datasetu, pod kterym bude zobrazen v aplikaci
+display-name: Snímače JIS
+# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře
 dataset-name: JIS
 # root slozka, ktera obsahuje odkazy na dataset
 url: https://openstore.zcu.cz/

modules/crawler/DatasetConfigs/KOLOBEZKY.yaml

@@ -1 +1 @@
 # jmeno datasetu, pod kterym bude zobrazen v aplikaci
+display-name: Půjčování koloběžek
+# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře
 dataset-name: KOLOBEZKY
 # root slozka, ktera obsahuje odkazy na dataset
 url: https://openstore.zcu.cz/

modules/crawler/DatasetConfigs/WIFI.yaml

@@ -1 +1 @@
 # jmeno datasetu, pod kterym bude zobrazen v aplikaci
+display-name: Wi-Fi síť ZČU
+# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře
 dataset-name: WIFI
 # root slozka, ktera obsahuje odkazy na dataset
 url: https://openstore.zcu.cz/

modules/crawler/DatasetProcessing/WIFIProcessor.py

@@ -32 +32 @@
                 date_dict[date] = dict()
 
             if name in date_dict[date]:
-                date_dict[date][name].occurrence += int(occurrence)
+                date_dict[date][name].occurrence += max(date_dict[date][name].occurrence,int(occurrence))
             else:
                 date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, int(occurrence))
 

modules/crawler/Pipeline.py

@@ -152 +152 @@
     # get all unprocessed files from dataset
     not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
 
+    database_connection = DatabaseLoader.create_database_connection()
+
+    DatabaseLoader.check_or_update_datasets_collection(database_connection,config)
+
+    DatabaseLoader.update_devices_collection(database_connection,config)
+
+
+    # load every file
+    for not_loaded_file in not_loaded_files:
+        #check if file is not in database already if it is skip
+        test = DatabaseLoader.check_if_database_doesnt_contain_file(database_connection,dataset_name,not_loaded_file)
+        if test == False:
+            logging.info(dataset_name + " could not load " + not_loaded_file + " to database because he is already there.")
+            FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
+            continue
+        # load processed data
+        processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
+        # load processed data to database
+        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
+        FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
+
+    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
+
+
+def load_data_to_database_crone(config):
+    """
+    Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
+    loads data appends coordination from configurations
+    and exports it into the database
+    After successful exporting updates ignore.txt
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    dataset_path = dataset_name + '/'
+
+    # get all unprocessed files from dataset
+    not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
+
+    database_connection = DatabaseLoader.create_database_connection()
+
     # load every file
     for not_loaded_file in not_loaded_files:
         # load processed data
         processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
         # load processed data to database
-        DatabaseLoader.load_data_to_database(dataset_name, processed_data)
+        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
         FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
 
-    logging.info(dataset_name + " has loaded to databse " + str(len(not_loaded_files)) + " newly processed files.")
+    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
 
 
 def run_full_pipeline(dataset_name):
@@ -185 +227 @@
         load_data_to_database(config)
 
 
-
 def run_full_pipeline_crone(dataset_name):
     """
     Loads config file and starts full pipeline
@@ -207 +248 @@
         validation_test = validate_process_data(config)
 
         if validation_test:
-            load_data_to_database(config)
+            load_data_to_database_crone(config)
             

modules/crawler/PrepareNewDataset.py

@@ -24 +24 @@
     with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
         file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
         file.write("dataset-name: " + dataset_name + "\n")
+        file.write("# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře\n")
+        file.write("dataset-name: " + dataset_name + "\n")
         file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
         file.write("url: ZDE VLOZTE URL\n")
         file.write("# volitelny parameter, ktery specifikuje vzor jmrna datasetu, ktera se budou stahovat\n")

modules/crawler/Utilities/Database/DatabaseLoader.py

@@ -11 +11 @@
 MONGODB_DATA_DATABASE = "open-data-db"
 # mongodb collection with aviable datasets
 MONGODB_DATASET_COLLECTION = "DATASETS"
+# mongodb collection with aviable diveces of datasets
+MONGODB_DATASET_DEVICES_COLLECTION = "DEVICES"
+# mongodb collection with aviable diveces of datasets
+MONGODB_DATASET_LOADED_FILES_COLLECTION = "FILES"
 
 # Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
@@ -63 +67 @@
 
         name = csv_column[0]
 
+        if devices[name]["x"] == "SKIP" or devices[name]["y"] == "SKIP":
+            continue
+
         occurrence = csv_column[1]
         date = csv_column[2]
         database_data_line = DatabaseDataLine.DatabaseDataLine(
@@ -79 +86 @@
     return date_dict
 
 
-def load_data_to_database(dataset_name, data_dic):
+def load_data_to_database(database_connection,dataset_name, data_dic, file_name):
     """
     Takes data_dic created in method get_data_from_file
     and loads into into database where collection name is dataset_name + data_dic key
     and data lines are line in collection
 
     Args:
-        dataset_name: name of dataset that has existing configuration file
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
         data_dic: dictionary of data lines created in get_data_from_file
+        file_name: name of file containing data
+    """
+
+    for date in data_dic:
+        dataset_collections = database_connection[dataset_name]
+        dataset_collections.insert_one({'name': dataset_name + date})
+        date_dataset = database_connection[dataset_name + date]
+        date_dataset.insert_many(data_dic[date])
+
+    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]
+    collection_loaded_files.insert_one({'file': file_name})
+
+
+
+def check_or_update_datasets_collection(database_connection,config):
     """
-    database = create_database_connection()
+    Checks if DATASETS collection contains dataset and if display name was not updated
 
+    Args:
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
+    """
     # collection where are specified aviable datasets
-    collection_datasets = database[MONGODB_DATASET_COLLECTION]
+    collection_datasets = database_connection[MONGODB_DATASET_COLLECTION]
+
+    dataset_name = config['dataset-name']
+    display_name = config['display-name']
+
+    query = {'key-name': dataset_name}
 
     # check if newly added data already have a dataset specified in collection
-    dataset_present = collection_datasets.find_one({'name': dataset_name})
+    dataset_present = collection_datasets.find_one(query)
 
     if dataset_present is None:
-        collection_datasets.insert_one({'name': dataset_name})
+        collection_datasets.insert_one({'key-name': dataset_name, 'display-name': display_name})
+    elif dataset_present['display-name'] != display_name:
+        newvalues = { "$set": { 'display-name': display_name } }
+        collection_datasets.update_one(query, newvalues)
 
-    for date in data_dic:
-        dataset_collections = database[dataset_name]
-        dataset_collections.insert_one({'name': dataset_name + date})
-        date_dataset = database[dataset_name + date]
-        date_dataset.insert_many(data_dic[date])
+
+def update_devices_collection(database_connection,config):
+    """
+    Checks if dataset_name collection contains every device with current set up
+
+    Args:
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config['dataset-name']
+
+    collection_devices = database_connection[dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
+
+    collection_devices.delete_many({})
+
+    devices = config['devices']
+
+    devices_list = list()
+
+    for device in devices.keys():
+        if devices[device]['x'] != "SKIP" or devices[device]['y'] != "SKIP":
+            devices_list.append({'name': device , 'x': devices[device]['x'] , 'y': devices[device]['y'] })
+
+    collection_devices.insert_many(devices_list)
+
+
+def check_if_database_doesnt_contain_file(database_connection,dataset_name,file_name):
+    """
+    Checks if dataset_name collection contains every device with current set up
+
+    Args:
+        database_connection: created connection to a MONGODB
+        filename: checked file name
+    """
+
+    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]
+
+    query = {'file': file_name}
+
+    # check if newly added data already have a dataset specified in collection
+    dataset_present = collection_loaded_files.find_one(query)
+
+    if dataset_present is None:
+        return True
+    else:
+        return False