1 |
d6ca840d
|
petrh
|
import pipeline
|
2 |
34baf808
|
petrh
|
import os
|
3 |
|
|
|
4 |
|
|
# Path to configuration files
|
5 |
|
|
CONFIG_FILES_PATH = "DatasetConfigs/"
|
6 |
|
|
|
7 |
|
|
|
8 |
af7609b5
|
Tomáš Ballák
|
def run_pipeline_for_all_datasets() -> None:
|
9 |
34baf808
|
petrh
|
"""
|
10 |
|
|
Runs whole DataScript pipeline for every dataset that has existing configuration file
|
11 |
|
|
"""
|
12 |
|
|
files_in_dir = os.listdir(CONFIG_FILES_PATH)
|
13 |
|
|
|
14 |
|
|
for file in files_in_dir:
|
15 |
d6ca840d
|
petrh
|
name = file.split('.')[0]
|
16 |
|
|
pipeline.run_full_pipeline_crone(name)
|
17 |
34baf808
|
petrh
|
|
18 |
|
|
|
19 |
af7609b5
|
Tomáš Ballák
|
def main() -> None:
|
20 |
|
|
run_pipeline_for_all_datasets()
|
21 |
|
|
|
22 |
|
|
|
23 |
|
|
if __name__ == "__main__":
|
24 |
|
|
main()
|