Revize 81980e82

Re #8160 new dataset

modules/crawler/.devcontainer/devcontainer.json

@@ -1 +1 @@
 {
     //TODO: dodat extensionu pro yaml soubory!!!
     "forwardPorts": [
-        8000
+        8000,
+        443,
+        80
     ],
     "extensions": [
         "vscode-icons-team.vscode-icons",
@@ -16 +18 @@
         "../../../docker-compose.yml",
         "../../../docker-compose-dev.yml"
     ],
+    "settings": {
+        "terminal.integrated.shell.linux": "/bin/sh"
+    },
     "mounts": [
         "source=vscode-extensions-python,target=/src/.vscode-server/extensions,type=volume"
     ]

modules/crawler/DatasetConfigs/JIS.yaml

@@ -131 +131 @@
       y: 13.3586728
 
   - STUD_UB211:
-      x: 49.7251661
-      y: 13.3530436
+      x: 49.7256172
+      y: 13.3532294
 
   - KL-Posilovna:
       x: 49.7368708
@@ -179 +179 @@
       y: 13.37270
 
   - STUD_UB113:
-      x: 49.7253256
-      y: 13.3531456
+      x: 49.7256406
+      y: 13.3531944
 
   - Menza1-kasa-p:
       x: 49.7473178

modules/crawler/DatasetConfigs/OBSAZENIMISTNOSTI.yaml

@@ +1 @@
+# jmeno datasetu, pod kterym bude zobrazen v aplikaci
+display-name: Obsazení místností
+# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře
+dataset-name: OBSAZENIMISTNOSTI
+# root slozka, ktera obsahuje odkazy na dataset
+url: https://openstore.zcu.cz/
+# volitelny parameter, ktery specifikuje vzor jmena datasetu, ktera se budou stahovat
+regex: OD_ZCU_OBSAZENI_[0-9][0-9]_[0-9][0-9][0-9][0-9]_CSV.zip
+# volitelny parametr, ktery udava jednou za kolik dní se budou nová data vyhledávat
+update-period: 24
+# pozice jednotlivych zarizeni, ktera jsou v datasetu
+devices:
+  - EP-110:
+      x: 49.7240758
+      y: 13.3503422
+  - EP-120:
+      x: 49.7242717
+      y: 13.3502872
+  - EP-130:
+      x: 49.7242311
+      y: 13.3506561
+  - EP-206:
+      x: 49.7240550
+      y: 13.3506817
+  - EP-208:
+      x: 49.7239544
+      y: 13.3507031
+  - UF-124:
+      x: 49.7253672
+      y: 13.3522111
+  - UP-101:
+      x: 49.7247431
+      y: 13.3504881
+  - UP-104:
+      x: 49.7247206
+      y: 13.3502172
+  - UP-108:
+      x: 49.7249114
+      y: 13.3501636
+  - UP-112:
+      x: 49.7249600
+      y: 13.3504022
+  - UP-115:
+      x: 49.7249669
+      y: 13.3504667
+  - US-207:
+      x: 49.7268203
+      y: 13.3517944
+  - US-217:
+      x: 49.7268950
+      y: 13.3515047
+  - UU-104:
+      x: 49.7248192
+      y: 13.3521414
+  - UU-106:
+      x: 49.7248089
+      y: 13.3519550
+  - UU-206:
+      x: 49.7248089
+      y: 13.3519550
+  - UU-305b:
+      x: 49.7248072
+      y: 13.3520450
+  - UU-305:
+      x: 49.7248089
+      y: 13.3519550
+  - UU-306:
+      x: 49.7249978
+      y: 13.3518961
+  - UU-307:
+      x: 49.7246900
+      y: 13.3518664
+  - UU-308:
+      x: 49.7248919
+      y: 13.3518222
+  - UU-309:
+      x: 49.7248764
+      y: 13.3516278
+
+  - SP-107:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-302:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-333:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-303:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-310:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-215:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-608:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - RJ-208:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-311:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-118:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-234:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-108:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-207:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-452:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-214:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-105:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-334:
+      x: 49.7268169
+      y: 13.3525456
+
+  - KL-310:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - PC-124:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CD-121:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UB-106:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-117:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-109:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-411:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-610:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-112:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-222:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-439:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-304:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-202:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-137:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-308:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-410:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UX-236:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-018:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-321:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-308:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-203:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-126:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-210:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-210:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-331:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-213:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EK-715:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-328:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-312:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - TS-330:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-210:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EL-209:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-326:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-305:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-120:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SD-200:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-607:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-303:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-305:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-307:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-211:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-024:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-106:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-233:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-110:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UK-311:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-108:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-117:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-140:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-211:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-416:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-303:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-207:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-409:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-207:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-202:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-410:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SD-325:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-305:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-105:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-112:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-104:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KO-106:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-309:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-422:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-208:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-103:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-105:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-211:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-219:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - ST-210:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-311:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-109:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-309:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-104:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-503:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-212:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-142:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UU-208:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-220:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-026:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CD-201:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-329:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-210:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-107:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EL-311:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-319:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-103:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UU-105:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-200:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-236:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - ST-304:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-221:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-111:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SO-118:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UU-108:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-106:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-236:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-311:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-303:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-114:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-602:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SD-206a:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - ST-306:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-205:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-330:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-203:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-207:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-506:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-208:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-309:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-228:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-302:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-224:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-307:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-108:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-104:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UK-221:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-223:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-504:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UX-127:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-208:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-453:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-209:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CD-154:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-302:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-505:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-241:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-301:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-122:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-208:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UK-226:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-432:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-102:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - PC-116:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-308:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CD-254:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-302:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - RJ-209:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-306:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-100:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - RJ-317:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - RJ-315:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-206:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-111:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-203:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - ST-201:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-323:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-206:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-210:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-110:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-300:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-209:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-325:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-332:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-302:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-309:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-107:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EL-510:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-310:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-221:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-223:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - RJ-205:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-201:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-304:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-320:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-302:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SD-203:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CD-304:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-417:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SP-420:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UL-409:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UU-209:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - PC-423:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UX-234a:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-213:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-147:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-311:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-237:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-202:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-324:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-107:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-235:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - VC-301:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UF-138:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-305:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-008:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-303:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-242:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - JJ-331:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-311:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-304:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - ST-309:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-332:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - RJ-211:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - SD-204:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-333:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-317:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-310:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - EU-503:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-329:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - PC-408:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-326:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-102:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-107:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UC-327:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-305:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - CH-204:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - ST-408:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-315:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - KL-303:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - UN-657:
+      x: UNKNOWN!
+      y: UNKNOWN!
+
+  - HJ-218:
+      x: UNKNOWN!
+      y: UNKNOWN!

modules/crawler/DatasetCrawler/OBSAZENIMISTNOSTI_crawler.py

@@ +1 @@
+from Utilities import folder_processor
+from Utilities.Crawler import basic_crawler_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def crawl(config):
+    """
+    Implement crawl method that downloads new data to path_for_files
+    For keeping the project structure
+    url , regex, and dataset_name from config
+    You can use already implemented functions from Utilities/Crawler/basic_crawler_functions.py
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    first_level_links = basic_crawler_functions.get_all_links(url)
+
+    filtered_first_level_links = basic_crawler_functions.filter_links(
+        first_level_links, "^OD_ZCU")
+
+    OFFSET_YEAR_START = -5
+    OFFSET_YEAR_END = -1
+    MONTH_SIZE = 2
+
+    #Seperate links by year
+    links_by_year = {}
+    for item in filtered_first_level_links:
+        if item[OFFSET_YEAR_START:OFFSET_YEAR_END] not in links_by_year:
+            links_by_year[item[OFFSET_YEAR_START:OFFSET_YEAR_END]] = []
+        else:
+            links_by_year[item[OFFSET_YEAR_START:OFFSET_YEAR_END]].append(item)
+
+    #Latest links of years to array
+    links = []
+    for _key, value in links_by_year.items():
+        links.append(
+            max(value,
+                key=lambda x: int(x[OFFSET_YEAR_START - MONTH_SIZE - 1:
+                                    OFFSET_YEAR_START - 1])))
+
+    absolute_first_level_links = basic_crawler_functions.create_absolute_links(
+        links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = basic_crawler_functions.get_all_links(link)
+        filtered_second_level_links = basic_crawler_functions.filter_links(
+            second_level_links, regex)
+        absolute_second_level_links = basic_crawler_functions.create_absolute_links(
+            filtered_second_level_links, link)
+
+        for file_link in absolute_second_level_links:
+            files.append(file_link)
+
+    files = basic_crawler_functions.remove_downloaded_links(
+        files, dataset_name)
+
+    for file in files:
+        basic_crawler_functions.download_file_from_url(file, dataset_name)
+
+    folder_processor.unzip_all_csv_zip_files_in_folder(path_for_files)

modules/crawler/DatasetProcessing/OBSAZENIMISTNOSTI_processor.py

@@ +1 @@
+from Utilities.CSV import csv_data_line
+from Utilities import date_formating
+import logging
+from datetime import date
+import time
+import datetime
+
+logging.basicConfig(filename='../../CrawlerLogs' + 'Crawlerlog-' +
+                    date.today().strftime("%b-%Y") + '.log',
+                    level=logging.INFO,
+                    format='%(asctime)s %(message)s')
+
+
+def process_file(filename):
+    """
+    Method that take path to crawled file and outputs date dictionary:
+    Date dictionary is a dictionary where keys are dates in format YYYY-mm-dd-hh (2018-04-08-15)
+    and value is dictionary where keys are devices (specified in configuration file)
+    and value is CSVDataLine.csv_data_line with device,date and occurrence
+
+    Args:
+    filename: name of processed file
+
+    Returns:
+    None if not implemented
+    date_dict when implemented
+    """
+    date_dict = dict()
+
+    with open(filename, "r") as file:
+
+        YEAR_START = 1
+        YEAR_END = 11
+        for line in file:
+
+            array = line.split(";")
+
+            #pick later time
+            time_ = max(
+                array[2][1:-1],
+                array[3][1:-1],
+                key=lambda x: time.mktime(
+                    datetime.datetime.strptime(x, "%H:%M").timetuple()))
+
+            date = date_formating.date_time_formatter(
+                array[14][YEAR_START:YEAR_END] + " " + time_)
+
+            name = array[10][1:-1]
+            if name == "":
+                continue
+
+            if date not in date_dict:
+                date_dict[date] = {}
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurrence = int(array[12])
+            else:
+                date_dict[date][name] = csv_data_line.CSVDataLine(
+                    name, date, int(array[12]))
+
+    return date_dict

modules/crawler/Utilities/Crawler/basic_crawler_functions.py

@@ -99 +99 @@
 
     # splits url and extract last part that contains filename
     url_parts = url.split("/")
-    file_name = url_parts[len(url_parts)-1]
+    file_name = url_parts[len(url_parts) - 1]
 
     data_path = CRAWLED_DATA_PATH + dataset_name + '/'
 

modules/crawler/Utilities/Database/database_loader.py

@@ -1 +1 @@
 from Utilities.Database import database_data_line, database_record_logs
 from Utilities import configure_functions
+from Utilities.helpers import should_skip
 import pymongo
 import re
 
@@ -67 +68 @@
 
         name = csv_column[0]
 
-        if devices[name]["x"] == "SKIP" or devices[name]["y"] == "SKIP":
+        if should_skip(devices[name]):
             continue
 
         occurrence = csv_column[1]
@@ -80 +81 @@
         if date_without_hours not in date_dict:
             date_dict[date_without_hours] = list()
 
-        date_dict[date_without_hours].append(
-            data_line.to_dictionary())
+        date_dict[date_without_hours].append(data_line.to_dictionary())
 
     return date_dict
 
 
-def load_data_to_database(database_connection,dataset_name, data_dic, file_name):
+def load_data_to_database(database_connection, dataset_name, data_dic,
+                          file_name):
     """
     Takes data_dic created in method get_data_from_file
     and loads into into database where collection name is dataset_name + data_dic key
@@ -106 +107 @@
         date_dataset.insert_many(data_dic[date])
 
 
-
-def check_or_update_datasets_collection(database_connection,config):
+def check_or_update_datasets_collection(database_connection, config):
     """
     Checks if DATASETS collection contains dataset and if display name was not updated
 
@@ -127 +127 @@
     dataset_present = collection_datasets.find_one(query)
 
     if dataset_present is None:
-        collection_datasets.insert_one({'key-name': dataset_name, 'display-name': display_name,'updated': 0})
+        collection_datasets.insert_one({
+            'key-name': dataset_name,
+            'display-name': display_name,
+            'updated': 0
+        })
     elif dataset_present['display-name'] != display_name:
-        newvalues = { "$set": { 'display-name': display_name } }
+        newvalues = {"$set": {'display-name': display_name}}
         collection_datasets.update_one(query, newvalues)
 
 
@@ -153 +157 @@
 
     change_in_devices = False
 
-    collection_devices = database_connection[dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
+    collection_devices = database_connection[
+        dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
 
     devices_cursor = collection_devices.find()
 
@@ -161 +166 @@
 
     for device in devices_cursor:
         name = device['name']
-        db_device_dict[name] = {'name': name, 'x': device['x'] , 'y': device['y']}
-
+        db_device_dict[name] = {
+            'name': name,
+            'x': device['x'],
+            'y': device['y']
+        }
 
-    valid_devices = configure_functions.return_dictionary_of_valid_devices(devices)
+    valid_devices = configure_functions.return_dictionary_of_valid_devices(
+        devices)
 
     if len(valid_devices.keys()) != len(db_device_dict.keys()):
         change_in_devices = True
-    
+
     if change_in_devices == False:
         for device in valid_devices.keys():
             if device in db_device_dict:
@@ -180 +189 @@
                     change_in_devices = True
                     break
 
-
     if change_in_devices == True:
         collection_devices.delete_many({})
         devices_list = list()
 
         for device in devices.keys():
-            x = devices[device]['x']
-            y = devices[device]['y']
-            if not (x == "SKIP" or x == "UNKNOWN!" or y == "SKIP" or y == "UNKNOWN!"):
-                devices_list.append({'name': device , 'x': x , 'y': y })
+            if not (should_skip(devices[device])):
+                devices_list.append({
+                    'name': device,
+                    'x': devices[device]['x'],
+                    'y': devices[device]['y']
+                })
 
         collection_devices.insert_many(devices_list)
 
     return change_in_devices
 
-    
+
 def remove_dataset_database(dataset_name):
     """
     Removes dataset entries from database
@@ -211 +221 @@
     collection_datasets.delete_one({"key-name": dataset_name})
     print("Removing record from DATASETS collection")
 
-
     # Retrieve list of all collections
     collections = mydb.list_collection_names()
 
@@ -232 +241 @@
     # Creating connection
     mydb = create_database_connection()
 
-    pattern = re.compile(dataset_name+'[0-9]+-[0-9]+-+[0-9]+')
+    pattern = re.compile(dataset_name + '[0-9]+-[0-9]+-+[0-9]+')
 
     # Retrieve list of all collections
     collections = mydb.list_collection_names()

modules/crawler/Utilities/configure_functions.py

@@ -1 +1 @@
 import yaml
 import os
 from Utilities.Database import database_record_logs
+from Utilities.helpers import should_skip
 
 # Path to dataset configuration files
 CONFIG_FILES_PATH = "DatasetConfigs/"
@@ -41 +42 @@
         new_devices: list or set of new devices for dataset
     """
 
-    with open(CONFIG_FILES_PATH + dataset_name + CONFIG_FILE_TYPE, "a") as file:
+    with open(CONFIG_FILES_PATH + dataset_name + CONFIG_FILE_TYPE,
+              "a") as file:
         for device in new_devices:
-            file.write("  - "+device+":\n")
+            if device == "":
+                continue
+            file.write("  - " + device + ":\n")
             file.write("      x: UNKNOWN!\n")
             file.write("      y: UNKNOWN!\n")
             file.write("\n")
@@ -65 +69 @@
 
     for dataset in datasets:
         name = dataset.split('.')
-        if name[0] == dataset_name :
+        if name[0] == dataset_name:
             return True
 
     return False
 
+
 def return_dictionary_of_valid_devices(devices):
     """
     Iterates over all devices specified in config file
@@ -85 +90 @@
     valid_devices = dict()
 
     for device in devices.keys():
-        x = devices[device]['x']
-        y = devices[device]['y']
-        if not (x == "SKIP" or x == "UNKNOWN!" or y == "SKIP" or y == "UNKNOWN!"):
-            valid_devices[device] = {'name': device, 'x': x , 'y': y}
+        if not should_skip(devices[device]):
+            valid_devices[device] = {
+                'name': device,
+                'x': devices[device]['x'],
+                'y': devices[device]['y']
+            }
 
     return valid_devices

modules/crawler/Utilities/helpers.py

@@ +1 @@
+SKIP = "SKIP"
+UNKNOWN = "UNKNOWN!"
+
+
+def should_skip(device) -> bool:
+    return device['x'] == SKIP or device['y'] == SKIP or device[
+        'x'] == UNKNOWN or device['y'] == UNKNOWN

modules/crawler/fully_clean_database.py

@@ -1 +1 @@
 from Utilities.Database import database_loader
 
+
+#TODO: smazat vsechny pomocny soubory po cisteni databaze + prejmenovat
 def clean_database():
     """
     Drops every collection in database

modules/crawler/pipeline.py

@@ -7 +7 @@
 import logging
 from datetime import date
 
-
 # Path to crawled data
 CRAWLED_DATA_PATH = "CrawledData/"
 # Path to processed data
@@ -19 +18 @@
 # Path to dataset processor implementations
 PROCESSOR_LIB_PATH = "DatasetProcessing."
 
-
 #logger
-logging.basicConfig(filename=CRAWLER_LOGS_PATH + 'Applicationlog-' + date.today().strftime("%b-%Y") + '.log',
-                    level=logging.INFO,
-                    format='%(asctime)s %(message)s'
-                    )
+logging.basicConfig(filename=CRAWLER_LOGS_PATH + 'Applicationlog-' +
+                               date.today().strftime("%b-%Y") + '.log',
+                               level=logging.INFO,
+                               format='%(asctime)s %(message)s')
 
 
 def check_last_update(config):
@@ -47 +45 @@
 
     if config["update-period"] <= last_update:
         logging.info("Dataset " + dataset_name + " is being updated today")
-        database_record_logs.update_updated(dataset_name,0)
+        database_record_logs.update_updated(dataset_name, 0)
         return True
     else:
         last_update_days = last_update + 1
-        logging.info("Dataset " + dataset_name + " will be updated in " + str(int(config["update-period"]) - last_update_days) + "days")
-        database_record_logs.update_updated(dataset_name,last_update + 1)
+        logging.info("Dataset " + dataset_name + " will be updated in " +
+                     str(int(config["update-period"]) - last_update_days) +
+                     "days")
+        database_record_logs.update_updated(dataset_name, last_update + 1)
         return False
 
 
-
 def crawl_data(config):
     """
       Imports dataset crawler in DatasetCrawler/"dataset_name"_crawler.py
@@ -67 +66 @@
     """
     dataset_name = config["dataset-name"]
 
-    crawl_func = __import__(CRAWLER_LIB_PATH + dataset_name + "_crawler", globals(), locals(), ['crawl']).crawl
+    crawl_func = __import__(CRAWLER_LIB_PATH + dataset_name + "_crawler",
+                            globals(), locals(), ['crawl']).crawl