aswi2020sebela-gitlab/modules/crawler/ResetDatasets.py @ 0a2832fb

import os
from Utilities import FolderProcessor

# Path to crawled data
CRAWLED_DATA_PATH = "CrawledData/"
# Path to processed data
PROCESSED_DATA_PATH = "ProcessedData/"
# Path to crawler logs
CRAWLER_LOGS_PATH = "CrawlerLogs/"
# Path to dataset configuration files
CONFIG_FILES_PATH = "DatasetConfigs"


def create_ignore_file(path, text):
    """
    Creates ignore file
    Args:
        path: path to directory for creating ignore.txt
        text: text that will be on first line of ignore.txt can be None
    """
    with open(path + "/ignore.txt", "w") as file:
        if text is not None:
            file.write(text + "\n")


def create_updated_file(path):
    """
    Creates updated file
    Args:
        path: path to directory for creating updated.txt
    """
    with open(path + "/updated.txt", "w") as file:
            file.write(str(0) + "\n")


def reset_dataset(dataset_name):
    """
    Resets all saved data in dataset except config and implementation
    Args:
        dataset_name: name of dataset that has existing configuration file
    """
    path = CRAWLED_DATA_PATH + dataset_name + "/"
    FolderProcessor.clean_folder(path)
    create_ignore_file(path, "ignore.txt")

    path = PROCESSED_DATA_PATH + dataset_name + "/"
    FolderProcessor.clean_folder(path)
    create_ignore_file(path, "ignore.txt")

    path = CRAWLER_LOGS_PATH + dataset_name + "/"
    FolderProcessor.clean_folder(path)
    create_ignore_file(path, None)
    create_updated_file(path)


def reset_all_datasets():
    """
    Resets all saved data in all datasets with config file except configs and implementation
    """
    datasets = os.listdir(CONFIG_FILES_PATH)

    for dataset in datasets:
        reset_dataset(dataset.split('.')[0])


reset_all_datasets()