aswi2020sebela-gitlab/modules/crawler/crone_update_script.py @ 6b7c4d1c

import pipeline
import os

# Path to configuration files
CONFIG_FILES_PATH = "DatasetConfigs/"


def run_pipeline_for_all_datasets():
    """
    Runs whole DataScript pipeline for every dataset that has existing configuration file
    """
    files_in_dir = os.listdir(CONFIG_FILES_PATH)

    for file in files_in_dir:
        name = file.split('.')[0]
        pipeline.run_full_pipeline_crone(name)


run_pipeline_for_all_datasets()