Revize c3c67472
Přidáno uživatelem Petr Hlaváč před asi 4 roky(ů)
python-module/Pipeline.py | ||
---|---|---|
1 |
from Utilities import FolderProcessor, ConfigureLoader
|
|
1 |
from Utilities import FolderProcessor, ConfigureFunctions
|
|
2 | 2 |
from Utilities.Database import DatabaseLoader |
3 | 3 |
|
4 | 4 |
|
... | ... | |
35 | 35 |
FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file) |
36 | 36 |
|
37 | 37 |
|
38 |
def validate_process_data(config): |
|
39 |
processed_devices_set = FolderProcessor.get_devices_set(PROCESSED_DATA_PATH + config["dataset-name"] + '/') |
|
40 |
unknown_devices_set = FolderProcessor.get_unknown_devices_set(config,processed_devices_set) |
|
41 |
unknown_devices_size = len(unknown_devices_set) |
|
42 |
|
|
43 |
if unknown_devices_size != 0: |
|
44 |
print("There is " + str(unknown_devices_size) + " unknown devies") |
|
45 |
ConfigureFunctions.update_configuration(CONFIG_FILES_PATH + config["dataset-name"] + ".yaml", unknown_devices_set) |
|
46 |
return False |
|
47 |
|
|
48 |
|
|
38 | 49 |
def load_data_to_database(config): |
39 | 50 |
|
40 | 51 |
dataset_name = config["dataset-name"] |
... | ... | |
54 | 65 |
|
55 | 66 |
|
56 | 67 |
def run_full_pipeline(dataset_name): |
57 |
config = ConfigureLoader.load_configuration(CONFIG_FILES_PATH + dataset_name)
|
|
68 |
config = ConfigureFunctions.load_configuration(CONFIG_FILES_PATH + dataset_name)
|
|
58 | 69 |
crawl_data(config) |
59 | 70 |
process_data(config["dataset-name"]) |
60 |
#load_data_to_database(config) |
|
71 |
|
|
72 |
validation_test = validate_process_data(config) |
|
73 |
|
|
74 |
if validation_test: |
|
75 |
load_data_to_database(config) |
|
76 |
|
Také k dispozici: Unified diff
Re #7927
implementovana funkce pro pridani neznamych zarizeni
predelana struktura konfig souboru devices jsou ted nakoncido konfiguracnich souboru
-vygenerovany konfig soubory pro vsechny datasety