aswi2020sebela-gitlab/modules/crawler/force_update_datasets.py @ bc0a7bfd

import pipeline
import os
from Utilities import configure_functions

# Path to configuration files
CONFIG_FILES_PATH = "DatasetConfigs/"


def run_pipeline_for_one_datasets(dataset_name):
    print("Probíhá update datasetu " + dataset_name)
    pipeline.run_full_pipeline(dataset_name)


def run_pipeline_for_all_datasets():
    """
    Runs whole DataScript pipeline for every dataset that has existing configuration file
    """
    files_in_dir = os.listdir(CONFIG_FILES_PATH)

    for file in files_in_dir:
        name = file.split('.')[0]
        print("Probíhá update datasetu " + name)
        pipeline.run_full_pipeline(name)


print("Zadejte jméno Datasetu který chcete updatovat (pokud všechny zadejte '-ALL'):\n")

dataset_name = input().upper()

if dataset_name == '-ALL':
    run_pipeline_for_all_datasets()
else:
    test = configure_functions.check_if_there_is_a_config_file(dataset_name)
    if test == True:
        run_pipeline_for_one_datasets(dataset_name)
    else:
        print("Tento dataset v architektuře neexistuje")