| 1 |
c8f3051b
|
petrh
|
import Pipeline
|
| 2 |
|
|
import os
|
| 3 |
|
|
|
| 4 |
04a2b5a4
|
petrh
|
# Path to configuration files
|
| 5 |
c8f3051b
|
petrh
|
CONFIG_FILES_PATH = "DatasetConfigs/"
|
| 6 |
|
|
|
| 7 |
|
|
|
| 8 |
|
|
def run_pipeline_for_all_datasets():
|
| 9 |
04a2b5a4
|
petrh
|
"""
|
| 10 |
|
|
Runs whole DataScript pipeline for every dataset that has existing configuration file
|
| 11 |
|
|
"""
|
| 12 |
c8f3051b
|
petrh
|
files_in_dir = os.listdir(CONFIG_FILES_PATH)
|
| 13 |
|
|
|
| 14 |
|
|
for file in files_in_dir:
|
| 15 |
04a2b5a4
|
petrh
|
name = file.split('.')
|
| 16 |
|
|
Pipeline.run_full_pipeline(name[0])
|
| 17 |
c8f3051b
|
petrh
|
|
| 18 |
|
|
|
| 19 |
|
|
run_pipeline_for_all_datasets()
|