|
1
|
import pipeline
|
|
2
|
import os
|
|
3
|
|
|
4
|
# Path to configuration files
|
|
5
|
CONFIG_FILES_PATH = "DatasetConfigs/"
|
|
6
|
|
|
7
|
|
|
8
|
def run_pipeline_for_all_datasets() -> None:
|
|
9
|
"""
|
|
10
|
Runs whole DataScript pipeline for every dataset that has existing configuration file
|
|
11
|
"""
|
|
12
|
files_in_dir = os.listdir(CONFIG_FILES_PATH)
|
|
13
|
|
|
14
|
for file in files_in_dir:
|
|
15
|
name = file.split('.')[0]
|
|
16
|
pipeline.run_full_pipeline_crone(name)
|
|
17
|
|
|
18
|
|
|
19
|
def main() -> None:
|
|
20
|
run_pipeline_for_all_datasets()
|
|
21
|
|
|
22
|
|
|
23
|
if __name__ == "__main__":
|
|
24
|
main()
|