aswi2020sebela-gitlab/modules/crawler/crone_update_script.py @ 94c6eb49
| 1 | d6ca840d | petrh | import pipeline |
|---|---|---|---|
| 2 | 34baf808 | petrh | import os |
| 3 | |||
| 4 | # Path to configuration files
|
||
| 5 | CONFIG_FILES_PATH = "DatasetConfigs/" |
||
| 6 | |||
| 7 | |||
| 8 | def run_pipeline_for_all_datasets(): |
||
| 9 | """
|
||
| 10 | Runs whole DataScript pipeline for every dataset that has existing configuration file
|
||
| 11 | """
|
||
| 12 | files_in_dir = os.listdir(CONFIG_FILES_PATH) |
||
| 13 | |||
| 14 | for file in files_in_dir: |
||
| 15 | d6ca840d | petrh | name = file.split('.')[0] |
| 16 | pipeline.run_full_pipeline_crone(name) |
||
| 17 | 34baf808 | petrh | |
| 18 | |||
| 19 | run_pipeline_for_all_datasets() |