1
|
import os
|
2
|
# Path to crawled data
|
3
|
CRAWLED_DATA_PATH = "CrawledData/"
|
4
|
# Path to processed data
|
5
|
PROCESSED_DATA_PATH = "ProcessedData/"
|
6
|
# Path for DatasetCrawlers implementations
|
7
|
CRAWLER_PROGRAM_PATH = "DatasetCrawler"
|
8
|
# Path for DatasetProcessors implementations
|
9
|
PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
|
10
|
# Path to dataset configuration files
|
11
|
CONFIG_FILES_PATH = "DatasetConfigs"
|
12
|
# Default color for visualization of dataset (buble info in map)
|
13
|
DEFAULT_COLOR = "#000000"
|
14
|
|
15
|
|
16
|
def create_default_config_file(dataset_name: str) -> None:
|
17
|
"""
|
18
|
Creates default config file
|
19
|
|
20
|
Args:
|
21
|
dataset_name: Name of newly created dataset
|
22
|
"""
|
23
|
with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
|
24
|
file.write("# Name of the dataset inside the application\n")
|
25
|
file.write("display-name: " + dataset_name + "\n")
|
26
|
file.write(
|
27
|
"# Color for the dataset in a hex value (default value #000000)\n")
|
28
|
file.write(f'display-color: \'{DEFAULT_COLOR}\' \n')
|
29
|
file.write(
|
30
|
"# One word dataset name (structure of all modules will be affected by this)\n"
|
31
|
)
|
32
|
file.write("dataset-name: " + dataset_name + "\n")
|
33
|
file.write("# Url for the source of this dataset\n")
|
34
|
file.write("url: ENTER URL HERE\n")
|
35
|
file.write(
|
36
|
"# Optional parameter which specifies a pattern of the datasets name\n"
|
37
|
)
|
38
|
file.write(
|
39
|
"# Example: DATASET_NAME_[0-9][0-9]_[0-9][0-9][0-9][0-9].zip\n")
|
40
|
file.write(
|
41
|
"# - DATASET_NAME_01_2020.zip where '01_2020' specifies date in this dataset\n"
|
42
|
)
|
43
|
file.write("regex: ENTER REGEX HERE\n")
|
44
|
file.write(
|
45
|
"# Optional parameter which specifies the way of searching new datasets (if empty the period is set to every day)\n"
|
46
|
)
|
47
|
file.write("update-period: ENTER UPDATE PERIOD HERE\n")
|
48
|
file.write("# Coordinates of every datasets device (entinty)\n")
|
49
|
file.write("devices:\n")
|
50
|
|
51
|
|
52
|
def create_default_processor(dataset_name: str) -> None:
|
53
|
"""
|
54
|
Creates default processor for dataset
|
55
|
|
56
|
Args:
|
57
|
dataset_name: Name of newly created dataset
|
58
|
"""
|
59
|
with open(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "_processor.py",
|
60
|
"w") as file:
|
61
|
file.write("from Utilities.CSV import csv_data_line\n")
|
62
|
file.write("from shared_types import DateDict")
|
63
|
file.write("\n")
|
64
|
file.write("\n")
|
65
|
file.write("def process_file(filename: str) -> DateDict:\n")
|
66
|
file.write(" \"\"\"\n")
|
67
|
file.write(
|
68
|
" Method that takes the path to crawled file and outputs date dictionary:\n"
|
69
|
)
|
70
|
file.write(
|
71
|
" Date dictionary is a dictionary where keys are dates in format YYYY-mm-dd-hh (2018-04-08-15)\n"
|
72
|
)
|
73
|
file.write(
|
74
|
" and value is dictionary where keys are devices (specified in configuration file)\n"
|
75
|
)
|
76
|
file.write(
|
77
|
" and value is CSVDataLine.csv_data_line with device,date and occurrence\n"
|
78
|
)
|
79
|
file.write("\n")
|
80
|
file.write(" Args:\n")
|
81
|
file.write(" filename: name of the processed file\n")
|
82
|
file.write("\n")
|
83
|
file.write(" Returns:\n")
|
84
|
file.write(" None if not implemented\n")
|
85
|
file.write(" date_dict when implemented\n")
|
86
|
file.write(" \"\"\"\n")
|
87
|
file.write(" date_dict: DateDict = {}\n")
|
88
|
file.write("\n")
|
89
|
file.write(" #with open(filename, \"r\") as file:\n")
|
90
|
file.write(
|
91
|
" print(\"You must implement the process_file method first!\")\n"
|
92
|
)
|
93
|
file.write(" return date_dict\n")
|
94
|
|
95
|
|
96
|
def create_default_crawler(dataset_name: str) -> None:
|
97
|
"""
|
98
|
Creates default crawler for dataset
|
99
|
|
100
|
Args:
|
101
|
dataset_name: Name of newly created dataset
|
102
|
"""
|
103
|
|
104
|
with open(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "_crawler.py",
|
105
|
"w") as file:
|
106
|
file.write("from shared_types import ConfigType\n")
|
107
|
file.write("# Path to crawled data\n")
|
108
|
file.write(f'CRAWLED_DATA_PATH = "{CRAWLED_DATA_PATH}" \n')
|
109
|
file.write("\n")
|
110
|
file.write("\n")
|
111
|
file.write("def crawl(config: ConfigType):\n")
|
112
|
file.write(" \"\"\"\n")
|
113
|
file.write(
|
114
|
" Implementation the crawl method which downloads new data to the path_for_files\n"
|
115
|
)
|
116
|
file.write(" For keeping the project structure\n")
|
117
|
file.write(" url , regex, and dataset_name from config\n")
|
118
|
file.write(
|
119
|
" You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py\n"
|
120
|
)
|
121
|
file.write("\n")
|
122
|
file.write(" Args:\n")
|
123
|
file.write(" config: loaded configuration file of dataset\n")
|
124
|
file.write(" \"\"\"\n")
|
125
|
file.write(" dataset_name = config[\"dataset-name\"]\n")
|
126
|
file.write(" url = config['url']\n")
|
127
|
file.write(" regex = config['regex']\n")
|
128
|
file.write(
|
129
|
" path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'\n")
|
130
|
file.write(
|
131
|
" print(\"Není implementován crawler pro získávání dat!\")\n")
|
132
|
|
133
|
|
134
|
def prepare_dataset_structure(dataset_name: str) -> None:
|
135
|
"""
|
136
|
Prepares folders for new dataset
|
137
|
Args:
|
138
|
dataset_name: Name of newly created dataset
|
139
|
"""
|
140
|
|
141
|
# create folder for crawled data
|
142
|
path = CRAWLED_DATA_PATH + dataset_name
|
143
|
try:
|
144
|
os.mkdir(path)
|
145
|
except os.error as e:
|
146
|
print(e)
|
147
|
print("Creation of the directory %s failed" % path)
|
148
|
|
149
|
# create folder for processed data
|
150
|
path = PROCESSED_DATA_PATH + dataset_name
|
151
|
try:
|
152
|
os.mkdir(path)
|
153
|
except OSError:
|
154
|
print("Nelze vytvořit adresář %s" % path)
|
155
|
|
156
|
create_default_crawler(dataset_name)
|
157
|
create_default_processor(dataset_name)
|
158
|
create_default_config_file(dataset_name)
|
159
|
|
160
|
|
161
|
def main() -> None:
|
162
|
print("Zadejte jméno nového datasetu:\n")
|
163
|
dataset_name = input().upper()
|
164
|
|
165
|
if dataset_name.isalpha():
|
166
|
prepare_dataset_structure(dataset_name)
|
167
|
print("Architektura vytvořena \n")
|
168
|
else:
|
169
|
print("Jméno musí obsahovat pouze písmena z abecedy (bez mezer)\n")
|
170
|
|
171
|
|
172
|
if __name__ == "__main__":
|
173
|
main()
|