Revize 3692d853

Úprava komentářů

modules/crawler/DatasetCrawler/JIS_crawler.py

@@ -30 +30 @@
         second_level_links = basic_crawler_functions.get_all_links(link)
         filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
         absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
-        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
 
-        for file_link in final_links:
+        for file_link in absolute_second_level_links:
             files.append(file_link)
 
+    files = basic_crawler_functions.remove_downloaded_links(files, dataset_name)
+
     for file in files:
         basic_crawler_functions.download_file_from_url(file, dataset_name)
 

modules/crawler/DatasetCrawler/KOLOBEZKY_crawler.py

@@ -30 +30 @@
         second_level_links = basic_crawler_functions.get_all_links(link)
         filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
         absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
-        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
 
-        for file_link in final_links:
+        for file_link in absolute_second_level_links:
             files.append(file_link)
 
+    files = basic_crawler_functions.remove_downloaded_links(files, dataset_name)
+
     for file in files:
         basic_crawler_functions.download_file_from_url(file, dataset_name)
 

modules/crawler/DatasetCrawler/WIFI_crawler.py

@@ -30 +30 @@
         second_level_links = basic_crawler_functions.get_all_links(link)
         filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
         absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
-        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
 
-        for file_link in final_links:
+        for file_link in absolute_second_level_links:
             files.append(file_link)
 
+    files = basic_crawler_functions.remove_downloaded_links(files, dataset_name)
+
     for file in files:
         basic_crawler_functions.download_file_from_url(file, dataset_name)
 

modules/crawler/DatasetProcessing/JIS_processor.py

@@ -5 +5 @@
 def process_file(filename):
     """
     Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    Date dictionary is a dictionary where keys are dates in format YYYY-mm-dd-hh (2018-04-08-15)
     and value is dictionary where keys are devices (specified in configuration file)
     and value is CSVDataLine.csv_data_line with device,date and occurrence
 

modules/crawler/DatasetProcessing/KOLOBEZKY_processor.py

@@ -5 +5 @@
 def process_file(filename):
     """
     Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    Date dictionary is a dictionary where keys are dates in format YYYY-mm-dd-hh (2018-04-08-15)
     and value is dictionary where keys are devices (specified in configuration file)
     and value is CSVDataLine.csv_data_line with device,date and occurrence
 

modules/crawler/DatasetProcessing/WIFI_processor.py

@@ -5 +5 @@
 def process_file(filename):
     """
     Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    Date dictionary is a dictionary where keys are dates in format YYYY-mm-dd-hh (2018-04-08-15)
     and value is dictionary where keys are devices (specified in configuration file)
     and value is CSVDataLine.csv_data_line with device,date and occurrence
 

modules/crawler/prepare_new_dataset.py

@@ -21 +21 @@
     """
     with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
         file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
-        file.write("dataset-name: " + dataset_name + "\n")
+        file.write("display-name: " + dataset_name + "\n")
         file.write("# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře\n")
         file.write("dataset-name: " + dataset_name + "\n")
         file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
@@ -49 +49 @@
         file.write("def process_file(filename):\n")
         file.write("    \"\"\"\n")
         file.write("    Method that take path to crawled file and outputs date dictionary:\n")
-        file.write("    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)\n")
+        file.write("    Date dictionary is a dictionary where keys are dates in format YYYY-mm-dd-hh (2018-04-08-15)\n")
         file.write("    and value is dictionary where keys are devices (specified in configuration file)\n")
         file.write("    and value is CSVDataLine.csv_data_line with device,date and occurrence\n")
         file.write("\n")
@@ -124 +124 @@
     create_default_config_file(dataset_name)
 
 print("Zadejte jméno nového datasetu:\n")
-prepare_dataset_structure(input().upper())
+
+dataset_name = input().upper()
+
+if dataset_name.isalpha():
+    prepare_dataset_structure(dataset_name)
+    print("Architektura vytvořena \n")
+else:
+    print("Jméno musí obsahovat pouze písmena z abecedy (bez mezer)\n")