1
|
import Pipeline
|
2
|
import os
|
3
|
|
4
|
# Path to configuration files
|
5
|
CONFIG_FILES_PATH = "DatasetConfigs/"
|
6
|
|
7
|
|
8
|
def run_pipeline_for_all_datasets():
|
9
|
"""
|
10
|
Runs whole DataScript pipeline for every dataset that has existing configuration file
|
11
|
"""
|
12
|
files_in_dir = os.listdir(CONFIG_FILES_PATH)
|
13
|
|
14
|
for file in files_in_dir:
|
15
|
name = file.split('.')
|
16
|
Pipeline.run_full_pipeline(name[0])
|
17
|
|
18
|
|
19
|
def run_pipeline_for_one_dataset(dataset_name):
|
20
|
"""
|
21
|
Runs whole DataScript pipeline for only one dataset
|
22
|
|
23
|
Args:
|
24
|
dataset_name: name of dataset that has existing configuration file
|
25
|
"""
|
26
|
Pipeline.run_full_pipeline(dataset_name)
|
27
|
|
28
|
|
29
|
run_pipeline_for_all_datasets()
|