Revize 2494ea3a

Upravenen system spousteni scriptu

modules/crawler/PrepareNewDataset.py

@@ -1 +1 @@
 import os
 
 # Path to crawled data
-CRAWLED_DATA_PATH = "../CrawledData/"
+CRAWLED_DATA_PATH = "CrawledData/"
 # Path to processed data
-PROCESSED_DATA_PATH = "../ProcessedData/"
+PROCESSED_DATA_PATH = "ProcessedData/"
 # Path to crawler logs
-CRAWLER_LOGS_PATH = "../CrawlerLogs/"
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
 # Path for DatasetCrawlers implementations
-CRAWLER_PROGRAM_PATH = "../DatasetCrawler"
+CRAWLER_PROGRAM_PATH = "DatasetCrawler"
 # Path for DatasetProcessors implementations
-PROCESSOR_PROGRAM_PATH = "../DatasetProcessing"
+PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
 # Path to dataset configuration files
-CONFIG_FILES_PATH = "../DatasetConfigs"
+CONFIG_FILES_PATH = "DatasetConfigs"
 
 
 def create_default_config_file(dataset_name):
@@ -61 +61 @@
         file.write("    False if not implemented\n")
         file.write("    True when implemented\n")
         file.write("    \"\"\"\n")
+        file.write("    #with open(filename, \"r\") as file:\n")
         file.write("    print(\"You must implements process_file method first!\")\n")
         file.write("    #CSVutils.export_data_to_csv(filename, date_dict)\n")
         file.write("    return False\n")
@@ -154 +155 @@
     create_default_processor(dataset_name)
     create_default_config_file(dataset_name)
 
-
-prepare_dataset_structure("TEST")
+print("Zadejte jméno nového datasetu:\n")
+prepare_dataset_structure(input())

modules/crawler/RemoveDataset.py

@@ -29 +29 @@
     os.remove(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py")
     os.remove(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py")
 
-    print("Dataset: " + dataset_name + " removed")
+    print("Dataset " + dataset_name + " removed")
 
-remove_dataset("TEST")
+print("Zadejte jméno Datasetu který chcete odstranit:\n")
+remove_dataset(input())

modules/crawler/RemoveDatasetDatabase.py

@@ -28 +28 @@
 
     print("Database Cleaned")
 
-
-remove_dataset_database("KOLOBEZKY")
+print("Zadejte jméno Datasetu který chcete odstranit z databáze:\n")
+remove_dataset_database(input())

modules/crawler/ResetDataset.py

@@ -52 +52 @@
     create_ignore_file(path, None)
     create_updated_file(path)
 
-
-def reset_all_datasets():
-    """
-    Resets all saved data in all datasets with config file except configs and implementation
-    """
-    datasets = os.listdir(CONFIG_FILES_PATH)
-
-    for dataset in datasets:
-        reset_dataset(dataset.split('.')[0])
-
-
-reset_all_datasets()
+print("Zadejte jméno Datasetu který chcete resetovat:\n")
+reset_dataset(input())

modules/crawler/ResetDatasets.py

@@ +1 @@
+import os
+from Utilities import FolderProcessor
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
+# Path to dataset configuration files
+CONFIG_FILES_PATH = "DatasetConfigs"
+
+
+def create_ignore_file(path, text):
+    """
+    Creates ignore file
+    Args:
+        path: path to directory for creating ignore.txt
+        text: text that will be on first line of ignore.txt can be None
+    """
+    with open(path + "/ignore.txt", "w") as file:
+        if text is not None:
+            file.write(text + "\n")
+
+
+def create_updated_file(path):
+    """
+    Creates updated file
+    Args:
+        path: path to directory for creating updated.txt
+    """
+    with open(path + "/updated.txt", "w") as file:
+            file.write(str(0) + "\n")
+
+
+def reset_dataset(dataset_name):
+    """
+    Resets all saved data in dataset except config and implementation
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    path = CRAWLED_DATA_PATH + dataset_name + "/"
+    FolderProcessor.clean_folder(path)
+    create_ignore_file(path, "ignore.txt")
+
+    path = PROCESSED_DATA_PATH + dataset_name + "/"
+    FolderProcessor.clean_folder(path)
+    create_ignore_file(path, "ignore.txt")
+
+    path = CRAWLER_LOGS_PATH + dataset_name + "/"
+    FolderProcessor.clean_folder(path)
+    create_ignore_file(path, None)
+    create_updated_file(path)
+
+
+def reset_all_datasets():
+    """
+    Resets all saved data in all datasets with config file except configs and implementation
+    """
+    datasets = os.listdir(CONFIG_FILES_PATH)
+
+    for dataset in datasets:
+        reset_dataset(dataset.split('.')[0])
+
+
+reset_all_datasets()