aswi2020sebela-gitlab/modules/crawler/main.py @ 9540d301

import Pipeline
import os

# Path to configuration files
CONFIG_FILES_PATH = "DatasetConfigs/"


def run_pipeline_for_all_datasets():
    """
    Runs whole DataScript pipeline for every dataset that has existing configuration file
    """
    files_in_dir = os.listdir(CONFIG_FILES_PATH)

    for file in files_in_dir:
        name = file.split('.')
        Pipeline.run_full_pipeline(name[0])


def run_pipeline_for_one_dataset(dataset_name):
    """
    Runs whole DataScript pipeline for only one dataset

    Args:
        dataset_name: name of dataset that has existing configuration file
    """
    Pipeline.run_full_pipeline(dataset_name)


run_pipeline_for_all_datasets()