1 |
d6ca840d
|
petrh
|
import os
|
2 |
|
|
from Utilities import folder_processor
|
3 |
|
|
from Utilities.Database import database_loader
|
4 |
|
|
from Utilities import configure_functions
|
5 |
af7609b5
|
Tomáš Ballák
|
|
6 |
d6ca840d
|
petrh
|
# Path to crawled data
|
7 |
|
|
CRAWLED_DATA_PATH = "CrawledData/"
|
8 |
|
|
# Path to processed data
|
9 |
|
|
PROCESSED_DATA_PATH = "ProcessedData/"
|
10 |
|
|
# Path to crawler logs
|
11 |
|
|
CRAWLER_LOGS_PATH = "CrawlerLogs/"
|
12 |
|
|
# Path to dataset configuration files
|
13 |
|
|
CONFIG_FILES_PATH = "DatasetConfigs"
|
14 |
|
|
|
15 |
|
|
|
16 |
af7609b5
|
Tomáš Ballák
|
def hard_reset_dataset(dataset_name: str) -> None:
|
17 |
d6ca840d
|
petrh
|
"""
|
18 |
|
|
Resets all saved data in dataset except config and implementation
|
19 |
|
|
Args:
|
20 |
|
|
dataset_name: name of dataset that has existing configuration file
|
21 |
|
|
"""
|
22 |
|
|
|
23 |
|
|
path = CRAWLED_DATA_PATH + dataset_name + "/"
|
24 |
|
|
folder_processor.clean_folder(path)
|
25 |
|
|
|
26 |
|
|
path = PROCESSED_DATA_PATH + dataset_name + "/"
|
27 |
|
|
folder_processor.clean_folder(path)
|
28 |
|
|
|
29 |
|
|
database_loader.remove_dataset_database(dataset_name)
|
30 |
|
|
|
31 |
|
|
|
32 |
af7609b5
|
Tomáš Ballák
|
def soft_reset_dataset(dataset_name: str) -> None:
|
33 |
753d424e
|
petrh
|
"""
|
34 |
|
|
Resets all saved data in dataset except config and implementation
|
35 |
|
|
Args:
|
36 |
|
|
dataset_name: name of dataset that has existing configuration file
|
37 |
|
|
"""
|
38 |
|
|
path = PROCESSED_DATA_PATH + dataset_name + "/"
|
39 |
|
|
folder_processor.clean_folder(path)
|
40 |
|
|
|
41 |
|
|
database_loader.remove_dataset_database(dataset_name)
|
42 |
|
|
|
43 |
|
|
|
44 |
af7609b5
|
Tomáš Ballák
|
def soft_reset_all_datasets() -> None:
|
45 |
d6ca840d
|
petrh
|
"""
|
46 |
|
|
Resets all saved data in all datasets with config file except configs and implementation
|
47 |
|
|
"""
|
48 |
|
|
datasets = os.listdir(CONFIG_FILES_PATH)
|
49 |
|
|
|
50 |
|
|
for dataset in datasets:
|
51 |
753d424e
|
petrh
|
soft_reset_dataset(dataset.split('.')[0])
|
52 |
d6ca840d
|
petrh
|
|
53 |
|
|
|
54 |
af7609b5
|
Tomáš Ballák
|
def hard_reset_all_datasets() -> None:
|
55 |
753d424e
|
petrh
|
"""
|
56 |
|
|
Resets all saved data in all datasets with config file except configs and implementation
|
57 |
|
|
"""
|
58 |
|
|
datasets = os.listdir(CONFIG_FILES_PATH)
|
59 |
|
|
|
60 |
|
|
for dataset in datasets:
|
61 |
|
|
hard_reset_dataset(dataset.split('.')[0])
|
62 |
|
|
|
63 |
d6ca840d
|
petrh
|
|
64 |
af7609b5
|
Tomáš Ballák
|
def main() -> None:
|
65 |
|
|
print(
|
66 |
|
|
"Zadejte jméno Datasetu který chcete resetovat (pokud všechny zadejte '-ALL'):\n"
|
67 |
|
|
)
|
68 |
d6ca840d
|
petrh
|
|
69 |
af7609b5
|
Tomáš Ballák
|
dataset_name = input().upper()
|
70 |
d6ca840d
|
petrh
|
|
71 |
af7609b5
|
Tomáš Ballák
|
print("Chcete smazat i stažená data ? (ANO/NE) \n")
|
72 |
753d424e
|
petrh
|
|
73 |
af7609b5
|
Tomáš Ballák
|
input_decision = input().upper()
|
74 |
753d424e
|
petrh
|
|
75 |
af7609b5
|
Tomáš Ballák
|
if dataset_name == '-ALL':
|
76 |
528cc257
|
petrh
|
if input_decision == 'ANO':
|
77 |
af7609b5
|
Tomáš Ballák
|
hard_reset_all_datasets()
|
78 |
528cc257
|
petrh
|
elif input_decision == 'NE':
|
79 |
af7609b5
|
Tomáš Ballák
|
soft_reset_all_datasets()
|
80 |
753d424e
|
petrh
|
else:
|
81 |
|
|
print('Neplatný vstup (ANO/NE)')
|
82 |
d6ca840d
|
petrh
|
else:
|
83 |
af7609b5
|
Tomáš Ballák
|
test = configure_functions.check_if_there_is_a_config_file(
|
84 |
|
|
dataset_name)
|
85 |
|
|
if test == True:
|
86 |
|
|
if input_decision == 'ANO':
|
87 |
|
|
hard_reset_dataset(dataset_name)
|
88 |
|
|
elif input_decision == 'NE':
|
89 |
|
|
soft_reset_dataset(dataset_name)
|
90 |
|
|
else:
|
91 |
|
|
print('Neplatný vstup (ANO/NE)')
|
92 |
|
|
else:
|
93 |
|
|
print("Tento dataset v architektuře neexistuje")
|
94 |
|
|
|
95 |
753d424e
|
petrh
|
|
96 |
af7609b5
|
Tomáš Ballák
|
if __name__ == "__main__":
|
97 |
|
|
main()
|