Revize af7609b5

Re #8193 - refactoring crawler

modules/crawler/Utilities/Database/database_loader.py

@@ -1 +1 @@
 from Utilities.Database import database_data_line, database_record_logs
 from Utilities import configure_functions
 from Utilities.helpers import should_skip, detect_change
+from shared_types import ConfigType
+from typing import Dict
 import pymongo
 import re
 
@@ -20 +22 @@
 # Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
 
+DatabaseConnectionType = Dict[str, any]
 
-def create_database_connection():
+
+def create_database_connection() -> pymongo.database.Database:
     """
     Creates connection to mongoDB
 
@@ -38 +42 @@
     return database
 
 
-def get_data_from_file(filename, config):
+def get_data_from_file(filename: str, config: ConfigType) -> Dict[str, any]:
     """
         Opens processed file, reads it line by line
         name, ocurrence, date
@@ -59 +63 @@
     f = open(dataset_path + filename, "r")
 
     devices = config["devices"]
-    date_dict = dict()
+    date_dict = {}
 
     for line in f:
         line = line[:-1]
@@ -86 +90 @@
     return date_dict
 
 
-def load_data_to_database(database_connection, dataset_name, data_dic,
-                          file_name):
+def load_data_to_database(database_connection: DatabaseConnectionType,
+                          dataset_name: str, data_dic: Dict[str, any],
+                          file_name: str) -> None:
     """
     Takes data_dic created in method get_data_from_file
     and loads into into database where collection name is dataset_name + data_dic key
@@ -107 +112 @@
         date_dataset.insert_many(data_dic[date])
 
 
-def check_or_update_datasets_collection(database_connection, config):
+def check_or_update_datasets_collection(
+        database_connection: DatabaseConnectionType, config: ConfigType):
     """
     Checks if DATASETS collection contains dataset and if display name was not updated
 
@@ -116 +122 @@
         config: loaded configuration file of dataset
     """
     # collection where are specified aviable datasets
-    compareKeys = ['display-name',
-                   'display-color']
+    compareKeys = ['display-name', 'display-color']
     collection_datasets = database_connection[MONGODB_DATASET_COLLECTION]
 
     query = {'key-name': config['dataset-name']}
@@ -139 +144 @@
         collection_datasets.update_one(query, {"$set": newVal})
 
 
-def update_devices_collection(config):
+def update_devices_collection(config: ConfigType):
     """
     Checks if there are any changes in devices specified in config file against 
     devices processed and loaded into the database
@@ -164 +169 @@
 
     devices_cursor = collection_devices.find()
 
-    db_device_dict = dict()
+    db_device_dict = {}
 
     for device in devices_cursor:
         name = device['name']
@@ -208 +213 @@
     return change_in_devices
 
 
-def remove_dataset_database(dataset_name):
+def remove_dataset_database(dataset_name: str):
     """
     Removes dataset entries from database
     Args:
@@ -221 +226 @@
     collection_datasets = mydb[MONGODB_DATASET_COLLECTION]
 
     collection_datasets.delete_one({"key-name": dataset_name})
-    print("Removing record from DATASETS collection")
+    print("Odstraňování záznamu z DATASETS kolekce")
 
     # Retrieve list of all collections
     collections = mydb.list_collection_names()
@@ -230 +235 @@
     for name in collections:
         if name.startswith(dataset_name):
             mydb[name].drop()
-            print("Dropping: " + name)
+            print("Odstraňuji: " + name)
 
 
-def reset_dataset_database(dataset_name):
+def reset_dataset_database(dataset_name: str):
     """
     Reset dataset in database 
      - delete everything from except crawled links and mention in DATASETS collection
@@ -252 +257 @@
     for name in collections:
         if pattern.match(name):
             mydb[name].drop()
-            print("Dropping: " + name)
+            print("Odstraňuji: " + name)
 
     database_record_logs.reset_ignore_set_loaded(dataset_name)