aswi2020sebela-gitlab/modules/crawler/CroneUpdateScript.py @ 587b1c57

import Pipeline
import os

# Path to configuration files
CONFIG_FILES_PATH = "DatasetConfigs/"


def run_pipeline_for_all_datasets():
    """
    Runs whole DataScript pipeline for every dataset that has existing configuration file
    """
    files_in_dir = os.listdir(CONFIG_FILES_PATH)

    for file in files_in_dir:
        name = file.split('.')
        Pipeline.run_full_pipeline_crone(name[0])


run_pipeline_for_all_datasets()