aswi2020sebela-gitlab/modules/crawler/reset_datasets.py @ 753d424e

import os
from Utilities import folder_processor
from Utilities.Database import database_loader
from Utilities import configure_functions
 
# Path to crawled data
CRAWLED_DATA_PATH = "CrawledData/"
# Path to processed data
PROCESSED_DATA_PATH = "ProcessedData/"
# Path to crawler logs
CRAWLER_LOGS_PATH = "CrawlerLogs/"
# Path to dataset configuration files
CONFIG_FILES_PATH = "DatasetConfigs"


def hard_reset_dataset(dataset_name):
    """
    Resets all saved data in dataset except config and implementation
    Args:
        dataset_name: name of dataset that has existing configuration file
    """

    path = CRAWLED_DATA_PATH + dataset_name + "/"
    folder_processor.clean_folder(path)

    path = PROCESSED_DATA_PATH + dataset_name + "/"
    folder_processor.clean_folder(path)

    database_loader.remove_dataset_database(dataset_name)


def soft_reset_dataset(dataset_name):
    """
    Resets all saved data in dataset except config and implementation
    Args:
        dataset_name: name of dataset that has existing configuration file
    """
    path = PROCESSED_DATA_PATH + dataset_name + "/"
    folder_processor.clean_folder(path)

    database_loader.remove_dataset_database(dataset_name)



def soft_reset_all_datasets():
    """
    Resets all saved data in all datasets with config file except configs and implementation
    """
    datasets = os.listdir(CONFIG_FILES_PATH)

    for dataset in datasets:
        soft_reset_dataset(dataset.split('.')[0])


def hard_reset_all_datasets():
    """
    Resets all saved data in all datasets with config file except configs and implementation
    """
    datasets = os.listdir(CONFIG_FILES_PATH)

    for dataset in datasets:
        hard_reset_dataset(dataset.split('.')[0])


print("Zadejte jméno Datasetu který chcete resetovat (pokud všechny zadejte '-ALL'):\n")

dataset_name = input().upper()

print("Chcete smazat i stažené stažená data ? (ANO/NE) \n")

vstup = input().upper()

if dataset_name == '-ALL':
    if vstup == 'ANO':
        hard_reset_all_datasets()
    elif vstup == 'NE':
        soft_reset_all_datasets()
    else:
        print('Neplatný vstup (ANO/NE)')
else:
    test = configure_functions.check_if_there_is_a_config_file(dataset_name)
    if test == True:
        if vstup == 'ANO':
            hard_reset_dataset(dataset_name)
        elif vstup == 'NE':
            soft_reset_dataset(dataset_name)
        else:
            print('Neplatný vstup (ANO/NE)')
    else:
        print("Tento dataset v architektuře neexistuje")