Revize af7609b5

Re #8193 - refactoring crawler

modules/crawler/Utilities/Database/database_record_logs.py

@@ -1 +1 @@
 from Utilities.Database import database_loader
-
+from shared_types import StringSetType
 # mongodb collection with with already downloaded links
 MONGODB_DATASET_LINK_COLLECTION = "LINKS"
 # mongodb collection with with already processed files
@@ -10 +10 @@
 MONGODB_DATASET_COLLECTION = "DATASETS"
 
 
-def load_ignore_set_links(dataset_name):
+def load_ignore_set_links(dataset_name: str) -> StringSetType:
     """
     Loades from database links of already downloaded files by crawler
     
@@ -32 +32 @@
     return ignore_set
 
 
-def update_ignore_set_links(dataset_name,link):
+def update_ignore_set_links(dataset_name: str, link: str) -> None:
     """
     Adds links of newly crawled files to the database
     
@@ -44 +44 @@
 
     my_col = connection[dataset_name + MONGODB_DATASET_LINK_COLLECTION]
 
-    my_col.insert({ "name": link})
+    my_col.insert({"name": link})
 
 
-def reset_ignore_set_links(dataset_name):
+def reset_ignore_set_links(dataset_name: str) -> None:
     """
     Drops collection of already downloaded links
     
@@ -62 +62 @@
     my_col.drop()
 
 
-
-def load_ignore_set_processed(dataset_name):
+def load_ignore_set_processed(dataset_name: str) -> StringSetType:
     """
     Loads from database set of already processed files
     
@@ -85 +84 @@
     return ignore_set
 
 
-def update_ignore_set_processed(dataset_name,filename):
+def update_ignore_set_processed(dataset_name: str, filename: str) -> None:
     """
     Adds files of newly processed files to the database
     
@@ -97 +96 @@
 
     my_col = connection[dataset_name + MONGODB_DATASET_PROCESSED_COLLECTION]
 
-    my_col.insert({ "name": filename})
-
+    my_col.insert({"name": filename})
 
 
-def reset_ignore_set_processed(dataset_name):
+def reset_ignore_set_processed(dataset_name: str) -> None:
     """
     Drops collection of already processed files
     
@@ -116 +114 @@
     my_col.drop()
 
 
-
-def load_ignore_set_loaded(dataset_name):
+def load_ignore_set_loaded(dataset_name: str) -> StringSetType:
     """
     Loads from database set of already loaded files in database
     
@@ -139 +136 @@
     return ignore_set
 
 
-
-def update_ignore_set_loaded(dataset_name,filename):
+def update_ignore_set_loaded(dataset_name: str, filename: str) -> None:
     """
     Adds files of newly loaded files to the database
     
@@ -152 +148 @@
 
     my_col = connection[dataset_name + MONGODB_DATASET_LOADED_COLLECTION]
 
-    my_col.insert({ "name": filename})
+    my_col.insert({"name": filename})
 
 
-def reset_ignore_set_loaded(dataset_name):
+def reset_ignore_set_loaded(dataset_name: str) -> None:
     """
     Drops collection of already loaded files
     
@@ -170 +166 @@
     my_col.drop()
 
 
-def load_updated(dataset_name):
+def load_updated(dataset_name: str) -> int:
     """
     Loads value of (days from last update) from db
     
@@ -184 +180 @@
 
     my_col = connection[MONGODB_DATASET_COLLECTION]
 
-    data = my_col.find_one({'key-name': dataset_name},{'updated'})
+    data = my_col.find_one({'key-name': dataset_name}, {'updated'})
 
     updated = int(data['updated'])
 
     return updated
 
 
-def update_updated(dataset_name,value):
+def update_updated(dataset_name: str, value: int):
     """
     Updates value of (days from last update) in db
     
@@ -203 +199 @@
 
     my_col = connection[MONGODB_DATASET_COLLECTION]
 
-    myquery = { 'key-name': dataset_name }
-    new_values = { "$set": { "updated": value } }
+    myquery = {'key-name': dataset_name}
+    new_values = {"$set": {"updated": value}}
 
-    my_col.update_one(myquery,new_values)
+    my_col.update_one(myquery, new_values)