Revize af7609b5

Re #8193 - refactoring crawler

modules/crawler/Utilities/folder_processor.py

@@ -1 +1 @@
 import os
 import zipfile
+from shared_types import ConfigType, StringSetType
 from Utilities.CSV import csv_utils
 from Utilities.Database import database_record_logs
 
 
-def list_of_all_new_files(ignore_set,path):
+def list_of_all_new_files(ignore_set: StringSetType,
+                          path: str) -> StringSetType:
     """
     Get all files from directory and all files written in ignore.txt
     and return the difference
@@ -16 +18 @@
     """
     files_in_dir = os.listdir(path)
 
-
     return set(files_in_dir).difference(ignore_set)
 
 
-
-def get_devices_set(dataset_name,path):
+def get_devices_set(dataset_name: str, path: str) -> StringSetType:
     """
      Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
      Extracts names from not loaded file which should be in first column
@@ -34 +34 @@
         set of unique names contained in not loaded files
     """
     ignore_set = database_record_logs.load_ignore_set_loaded(dataset_name)
-    files_in_dir = list_of_all_new_files(ignore_set,path)
+    files_in_dir = list_of_all_new_files(ignore_set, path)
 
     unique_names = set()
 
     for file_path in files_in_dir:
-        unique_names.update(csv_utils.get_unique_names_from_file(path+file_path, 0))
+        unique_names.update(
+            csv_utils.get_unique_names_from_file(path + file_path, 0))
 
     return unique_names
 
 
-def get_unknown_devices_set(config, devices):
+def get_unknown_devices_set(config: ConfigType,
+                            devices: StringSetType) -> StringSetType:
     """
     Compares config and devices a return difference
 
@@ -61 +63 @@
     return unknown_devices_set
 
 
-def unzip_all_csv_zip_files_in_folder(path):
+def unzip_all_csv_zip_files_in_folder(path: str) -> None:
     """
     Load all files from directory and unzip those which end by .zip
     After unziping deletes the zip file
@@ -83 +85 @@
         os.remove(zip_file)
 
 
-def clean_folder(path):
+def clean_folder(path: str) -> None:
     """
     Deletes all files in folder
 
@@ -93 +95 @@
     files = os.listdir(path)
 
     for file in files:
-        os.remove(path+file)
+        os.remove(path + file)