Revize 70217608

Re #7930 Implementovan crawler pro JIS

python-module/CrawledData/JIS/ignore.txt

@@ +1 @@
+ignore.txt

python-module/DatasetConfigs/JIS.yaml

@@ +1 @@
+# jmeno datasetu, pod kterym bude zobrazen v aplikaci
+dataset-name: JIS
+# pozice jednotlivych zarizeni, ktera jsou v datasetu
+devices:
+  - example1:
+      x: 12.3
+      y: 32.1
+
+  - example2:
+      x: 32.1
+      y: 12.3
+
+# root slozka, ktera obsahuje odkazy na dataset
+url: ZDE VLOZTE URL/
+# voliteln? parameter, kter? specifikuje vzor jm?na dataset?, kter? se budou stahovat
+regex: ZDE VLOZTE REGEX
+# volitelny parametr, ktery udava jak casto se budou hledat nove datasety, pokud prazdne, tak defaultni hodnota (dny)
+update-period: ZDE VLOZTE HODNOTU

python-module/DatasetCrawler/JISCrawler.py

@@ +1 @@
+from Utilities import FolderProcessor
+from Utilities.Crawler import BasicCrawler
+
+
+def crawl(config):
+
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+
+    first_level_links = BasicCrawler.get_all_links(url)
+    filtered_first_level_links = BasicCrawler.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = BasicCrawler.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = BasicCrawler.get_all_links(link)
+        filtered_second_level_links = BasicCrawler.filter_links(second_level_links, regex)
+        absolute_second_level_links = BasicCrawler.create_absolute_links(filtered_second_level_links, link)
+        final_links = BasicCrawler.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        BasicCrawler.download_file_from_url(file, "CrawledData/" + dataset_name + "/", dataset_name)
+
+    FolderProcessor.unzip_all_csv_zip_files_in_folder("CrawledData/" + dataset_name + "/")

python-module/DatasetProcessing/JISProcessor.py

@@ +1 @@
+def process_file(filename):
+    print("You must implements process_file method first!")

python-module/ProcessedData/JIS/ignore.txt

@@ +1 @@
+ignore.txt