aswi2020sebela-gitlab/modules/crawler/reset_datasets.py @ d6ca840d

import os
from Utilities import folder_processor
from Utilities.Database import database_loader
from Utilities import configure_functions
 
# Path to crawled data
CRAWLED_DATA_PATH = "CrawledData/"
# Path to processed data
PROCESSED_DATA_PATH = "ProcessedData/"
# Path to crawler logs
CRAWLER_LOGS_PATH = "CrawlerLogs/"
# Path to dataset configuration files
CONFIG_FILES_PATH = "DatasetConfigs"


def reset_dataset(dataset_name):
    """
    Resets all saved data in dataset except config and implementation
    Args:
        dataset_name: name of dataset that has existing configuration file
    """

    path = CRAWLED_DATA_PATH + dataset_name + "/"
    folder_processor.clean_folder(path)

    path = PROCESSED_DATA_PATH + dataset_name + "/"
    folder_processor.clean_folder(path)

    database_loader.remove_dataset_database(dataset_name)


def reset_all_datasets():
    """
    Resets all saved data in all datasets with config file except configs and implementation
    """
    datasets = os.listdir(CONFIG_FILES_PATH)

    for dataset in datasets:
        reset_dataset(dataset.split('.')[0])



print("Zadejte jméno Datasetu který chcete resetovat (pokud všechny zadejte '-ALL'):\n")

dataset_name = input().upper()

if dataset_name == '-ALL':
    reset_all_datasets()
else:
    test = configure_functions.check_if_there_is_a_config_file(dataset_name)
    if test == True:
        reset_dataset(dataset_name)
    else:
        print("Tento dataset v architektuře neexistuje")