|
1
|
import pipeline
|
|
2
|
import os
|
|
3
|
from Utilities import configure_functions
|
|
4
|
|
|
5
|
# Path to configuration files
|
|
6
|
CONFIG_FILES_PATH = "DatasetConfigs/"
|
|
7
|
|
|
8
|
|
|
9
|
def run_pipeline_for_one_datasets(dataset_name):
|
|
10
|
print("Probíhá update datasetu " + dataset_name)
|
|
11
|
pipeline.run_full_pipeline(dataset_name)
|
|
12
|
|
|
13
|
|
|
14
|
def run_pipeline_for_all_datasets():
|
|
15
|
"""
|
|
16
|
Runs whole DataScript pipeline for every dataset that has existing configuration file
|
|
17
|
"""
|
|
18
|
files_in_dir = os.listdir(CONFIG_FILES_PATH)
|
|
19
|
|
|
20
|
for file in files_in_dir:
|
|
21
|
name = file.split('.')[0]
|
|
22
|
print("Probíhá update datasetu " + name)
|
|
23
|
pipeline.run_full_pipeline(name)
|
|
24
|
|
|
25
|
|
|
26
|
print("Zadejte jméno Datasetu který chcete updatovat (pokud všechny zadejte '-ALL'):\n")
|
|
27
|
|
|
28
|
dataset_name = input().upper()
|
|
29
|
|
|
30
|
if dataset_name == '-ALL':
|
|
31
|
run_pipeline_for_all_datasets()
|
|
32
|
else:
|
|
33
|
test = configure_functions.check_if_there_is_a_config_file(dataset_name)
|
|
34
|
if test == True:
|
|
35
|
run_pipeline_for_one_datasets(dataset_name)
|
|
36
|
else:
|
|
37
|
print("Tento dataset v architektuře neexistuje")
|