Revize 34cf65cd

Fixed issues found in code documentation

python-module/DatasetCrawler/KOLOBEZKYCrawler.py

@@ -1 +1 @@
 from Utilities import FolderProcessor
-from Utilities.Crawler import BasicCrawler
+from Utilities.Crawler import BasicCrawlerFunctions
 
 # Path to crawled data
 CRAWLED_DATA_PATH = "CrawledData/"
@@ -20 +20 @@
     regex = config['regex']
     path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
 
-    first_level_links = BasicCrawler.get_all_links(url)
-    filtered_first_level_links = BasicCrawler.filter_links(first_level_links, "^OD_ZCU")
-    absolute_first_level_links = BasicCrawler.create_absolute_links(filtered_first_level_links, url)
+    first_level_links = BasicCrawlerFunctions.get_all_links(url)
+    filtered_first_level_links = BasicCrawlerFunctions.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_first_level_links, url)
 
     files = []
 
     for link in absolute_first_level_links:
-        second_level_links = BasicCrawler.get_all_links(link)
-        filtered_second_level_links = BasicCrawler.filter_links(second_level_links, regex)
-        absolute_second_level_links = BasicCrawler.create_absolute_links(filtered_second_level_links, link)
-        final_links = BasicCrawler.remove_downloaded_links(absolute_second_level_links, dataset_name)
+        second_level_links = BasicCrawlerFunctions.get_all_links(link)
+        filtered_second_level_links = BasicCrawlerFunctions.filter_links(second_level_links, regex)
+        absolute_second_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_second_level_links, link)
+        final_links = BasicCrawlerFunctions.remove_downloaded_links(absolute_second_level_links, dataset_name)
 
         for file_link in final_links:
             files.append(file_link)
 
     for file in files:
-        BasicCrawler.download_file_from_url(file, "CrawledData/" + dataset_name + "/", dataset_name)
+        BasicCrawlerFunctions.download_file_from_url(file, CRAWLED_DATA_PATH + dataset_name + "/", dataset_name)
 
-    FolderProcessor.unzip_all_csv_zip_files_in_folder("CrawledData/" + dataset_name + "/")
+    FolderProcessor.unzip_all_csv_zip_files_in_folder(CRAWLED_DATA_PATH + dataset_name + "/")