Revize af7609b5

Re #8193 - refactoring crawler

modules/crawler/Utilities/Crawler/basic_crawler_functions.py

@@ -3 +3 @@
 from Utilities import folder_processor
 from Utilities.Database import database_record_logs
 from bs4 import BeautifulSoup
+from typing import List
 
 # Path to crawler logs
 CRAWLER_LOGS_PATH = "CrawlerLogs/"
 # Path to crawled data
 CRAWLED_DATA_PATH = "CrawledData/"
+LinksType = List[str]
 
 
-def get_all_links(url):
+def get_all_links(url: str) -> LinksType:
     """
     Sends http request to url, downloads all data,
     extract links
@@ -34 +36 @@
     return links
 
 
-def filter_links(links, regex):
+def filter_links(links: LinksType, regex: str) -> LinksType:
     """
     Filters list of links using regex
 
@@ -54 +56 @@
     return filtered_links
 
 
-def create_absolute_links(links, archive):
+def create_absolute_links(links: LinksType, archive: str) -> LinksType:
     """
         Appends archive path to every link in links
     Args:
@@ -72 +74 @@
     return absolute_links
 
 
-def remove_downloaded_links(links, dataset_name):
+def remove_downloaded_links(links: LinksType, dataset_name: str) -> LinksType:
     """
     Loads already downloaded links from CRAWLER_LOGS_PATH ignore.txt
     Args:
@@ -88 +90 @@
     return final_links
 
 
-def download_file_from_url(url, dataset_name):
+def download_file_from_url(url: str, dataset_name: str) -> None:
     """
     Downloads file on provided url and saves it to path
     Args: