Revize af7609b5

Re #8193 - refactoring crawler

modules/crawler/Utilities/configure_functions.py

@@ -1 +1 @@
 import yaml
 import os
+from typing import Dict, Set
+from shared_types import StringSetType
 from Utilities.Database import database_record_logs
 from Utilities.helpers import should_skip
 
@@ -9 +11 @@
 CONFIG_FILE_TYPE = ".yaml"
 
 
-def load_configuration(dataset_name):
+def load_configuration(dataset_name: str) -> Dict[str, any]:
     """
     Loads yaml configuration file into memory
 
@@ -22 +24 @@
     with open(CONFIG_FILES_PATH + dataset_name + CONFIG_FILE_TYPE, "r") as f:
         data = yaml.load(f, Loader=yaml.FullLoader)
 
-    devices_dic = dict()
+    devices_dic = {}
 
     if data["devices"] is not None:
         for item in data["devices"]:
@@ -33 +35 @@
     return data
 
 
-def update_configuration(dataset_name, new_devices):
+def update_configuration(dataset_name: str,
+                         new_devices: StringSetType) -> None:
     """
     Open dataset and appends new_devices to the end
 
@@ -53 +56 @@
             file.write("\n")
 
 
-def check_if_there_is_a_config_file(dataset_name):
+def check_if_there_is_a_config_file(dataset_name: str) -> bool:
     """
     Goes trough all config files (represeting valid dataset in database)
     and checks if dataset_name is there
@@ -75 +78 @@
     return False
 
 
-def return_dictionary_of_valid_devices(devices):
+def return_dictionary_of_valid_devices(
+        devices: Dict[str, any]) -> Dict[str, Dict[str, any]]:
     """
     Iterates over all devices specified in config file
 
@@ -87 +91 @@
     Returns:   
         Dictonary containing only valid devices
     """
-    valid_devices = dict()
+    valid_devices = {}
 
     for device in devices.keys():
         if not should_skip(devices[device]):