commit d6ca840d5183689d868c91fce876b499be962603
Author: petrh <petrh@zcu.cz>
Date:   Sat May 23 22:55:52 2020 +0000

    Re #8116
    predelano ukladani
    prejmenovany soubory podle python konvenci

diff --git a/.gitignore b/.gitignore
index 264aefc..1013987 100644
--- a/.gitignore
+++ b/.gitignore
@@ -1,5 +1,4 @@
 logs
-database
 website/vendor/
 /website/var
 cache
diff --git a/docker-compose-prod.yml b/docker-compose-prod.yml
index 32d2b89..10827a7 100644
--- a/docker-compose-prod.yml
+++ b/docker-compose-prod.yml
@@ -9,4 +9,8 @@ services:
               volumes: 
                      - /etc/certificate:/certificate
                      - ./docker/nginx/sites-dev:/etc/nginx/sites-available
-                     - /acme-challenge:/var/www/symfony/public/.well-known/acme-challenge/
\ No newline at end of file
+                     - /acme-challenge:/var/www/symfony/public/.well-known/acme-challenge/
+       crawler:
+              volumes: 
+                     - /logs/crawler:/src/CrawlerLogs
+                     - /data/crawler:/src/CrawledData
\ No newline at end of file
diff --git a/docker-compose.yml b/docker-compose.yml
index ebc6807..5866c8a 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -11,7 +11,7 @@ services:
                         - backend
                 volumes:
                         - ./modules/crawler/:/src
-                        - ./logs/crawler:/log/
+                        - ./logs/crawler:/src/CrawlerLogs
                 container_name: "heatmap_crawler"
                 environment:
                         - TZ=Europe/Prague
diff --git a/modules/crawler/.gitignore b/modules/crawler/.gitignore
index 2517278..e68ff3b 100644
--- a/modules/crawler/.gitignore
+++ b/modules/crawler/.gitignore
@@ -1,2 +1,5 @@
 *__pycache__*
-*.CSV
\ No newline at end of file
+*.CSV
+CrawlerLogs
+CrawledData
+ProcessedData
\ No newline at end of file
diff --git a/modules/crawler/CrawledData/JIS/ignore.txt b/modules/crawler/CrawledData/JIS/ignore.txt
deleted file mode 100644
index 8ea087a..0000000
--- a/modules/crawler/CrawledData/JIS/ignore.txt
+++ /dev/null
@@ -1 +0,0 @@
-ignore.txt
diff --git a/modules/crawler/CrawledData/KOLOBEZKY/ignore.txt b/modules/crawler/CrawledData/KOLOBEZKY/ignore.txt
deleted file mode 100644
index 8ea087a..0000000
--- a/modules/crawler/CrawledData/KOLOBEZKY/ignore.txt
+++ /dev/null
@@ -1 +0,0 @@
-ignore.txt
diff --git a/modules/crawler/CrawledData/WIFI/ignore.txt b/modules/crawler/CrawledData/WIFI/ignore.txt
deleted file mode 100644
index 8ea087a..0000000
--- a/modules/crawler/CrawledData/WIFI/ignore.txt
+++ /dev/null
@@ -1 +0,0 @@
-ignore.txt
diff --git a/modules/crawler/CrawlerLogs/CommonRecords/.gitignore b/modules/crawler/CrawlerLogs/CommonRecords/.gitignore
deleted file mode 100644
index 44c5ea8..0000000
--- a/modules/crawler/CrawlerLogs/CommonRecords/.gitignore
+++ /dev/null
@@ -1,4 +0,0 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore
\ No newline at end of file
diff --git a/modules/crawler/CrawlerLogs/JIS/ignore.txt b/modules/crawler/CrawlerLogs/JIS/ignore.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/modules/crawler/CrawlerLogs/JIS/updated.txt b/modules/crawler/CrawlerLogs/JIS/updated.txt
deleted file mode 100644
index 573541a..0000000
--- a/modules/crawler/CrawlerLogs/JIS/updated.txt
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/modules/crawler/CrawlerLogs/KOLOBEZKY/ignore.txt b/modules/crawler/CrawlerLogs/KOLOBEZKY/ignore.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/modules/crawler/CrawlerLogs/KOLOBEZKY/updated.txt b/modules/crawler/CrawlerLogs/KOLOBEZKY/updated.txt
deleted file mode 100644
index 573541a..0000000
--- a/modules/crawler/CrawlerLogs/KOLOBEZKY/updated.txt
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/modules/crawler/CrawlerLogs/WIFI/ignore.txt b/modules/crawler/CrawlerLogs/WIFI/ignore.txt
deleted file mode 100644
index e69de29..0000000
diff --git a/modules/crawler/CrawlerLogs/WIFI/updated.txt b/modules/crawler/CrawlerLogs/WIFI/updated.txt
deleted file mode 100644
index 573541a..0000000
--- a/modules/crawler/CrawlerLogs/WIFI/updated.txt
+++ /dev/null
@@ -1 +0,0 @@
-0
diff --git a/modules/crawler/CroneUpdateScript.py b/modules/crawler/CroneUpdateScript.py
deleted file mode 100644
index 68e31e5..0000000
--- a/modules/crawler/CroneUpdateScript.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import Pipeline
-import os
-
-# Path to configuration files
-CONFIG_FILES_PATH = "DatasetConfigs/"
-
-
-def run_pipeline_for_all_datasets():
-    """
-    Runs whole DataScript pipeline for every dataset that has existing configuration file
-    """
-    files_in_dir = os.listdir(CONFIG_FILES_PATH)
-
-    for file in files_in_dir:
-        name = file.split('.')
-        Pipeline.run_full_pipeline_crone(name[0])
-
-
-run_pipeline_for_all_datasets()
diff --git a/modules/crawler/DatasetConfigs/KOLOBEZKY.yaml b/modules/crawler/DatasetConfigs/KOLOBEZKY.yaml
index 234ae29..6d0d721 100644
--- a/modules/crawler/DatasetConfigs/KOLOBEZKY.yaml
+++ b/modules/crawler/DatasetConfigs/KOLOBEZKY.yaml
@@ -33,4 +33,4 @@ devices:
   - stojan-borska:
       x: 49.734518
       y: 13.359475
-      
\ No newline at end of file
+
diff --git a/modules/crawler/DatasetCrawler/JISCrawler.py b/modules/crawler/DatasetCrawler/JISCrawler.py
deleted file mode 100644
index adbb507..0000000
--- a/modules/crawler/DatasetCrawler/JISCrawler.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from Utilities import FolderProcessor
-from Utilities.Crawler import BasicCrawlerFunctions
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-
-
-def crawl(config):
-    """
-    Implement crawl method that downloads new data to path_for_files
-    For keeping the project structure
-    url , regex, and dataset_name from config
-    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    url = config['url']
-    regex = config['regex']
-    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
-
-    first_level_links = BasicCrawlerFunctions.get_all_links(url)
-    filtered_first_level_links = BasicCrawlerFunctions.filter_links(first_level_links, "^OD_ZCU")
-    absolute_first_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_first_level_links, url)
-
-    files = []
-
-    for link in absolute_first_level_links:
-        second_level_links = BasicCrawlerFunctions.get_all_links(link)
-        filtered_second_level_links = BasicCrawlerFunctions.filter_links(second_level_links, regex)
-        absolute_second_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_second_level_links, link)
-        final_links = BasicCrawlerFunctions.remove_downloaded_links(absolute_second_level_links, dataset_name)
-
-        for file_link in final_links:
-            files.append(file_link)
-
-    for file in files:
-        BasicCrawlerFunctions.download_file_from_url(file, dataset_name)
-
-    FolderProcessor.unzip_all_csv_zip_files_in_folder(path_for_files)
diff --git a/modules/crawler/DatasetCrawler/JIS_crawler.py b/modules/crawler/DatasetCrawler/JIS_crawler.py
new file mode 100644
index 0000000..101a410
--- /dev/null
+++ b/modules/crawler/DatasetCrawler/JIS_crawler.py
@@ -0,0 +1,41 @@
+from Utilities import folder_processor
+from Utilities.Crawler import basic_crawler_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def crawl(config):
+    """
+    Implement crawl method that downloads new data to path_for_files
+    For keeping the project structure
+    url , regex, and dataset_name from config
+    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    first_level_links = basic_crawler_functions.get_all_links(url)
+    filtered_first_level_links = basic_crawler_functions.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = basic_crawler_functions.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = basic_crawler_functions.get_all_links(link)
+        filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
+        absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
+        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        basic_crawler_functions.download_file_from_url(file, dataset_name)
+
+    folder_processor.unzip_all_csv_zip_files_in_folder(path_for_files)
diff --git a/modules/crawler/DatasetCrawler/KOLOBEZKYCrawler.py b/modules/crawler/DatasetCrawler/KOLOBEZKYCrawler.py
deleted file mode 100644
index adbb507..0000000
--- a/modules/crawler/DatasetCrawler/KOLOBEZKYCrawler.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from Utilities import FolderProcessor
-from Utilities.Crawler import BasicCrawlerFunctions
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-
-
-def crawl(config):
-    """
-    Implement crawl method that downloads new data to path_for_files
-    For keeping the project structure
-    url , regex, and dataset_name from config
-    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    url = config['url']
-    regex = config['regex']
-    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
-
-    first_level_links = BasicCrawlerFunctions.get_all_links(url)
-    filtered_first_level_links = BasicCrawlerFunctions.filter_links(first_level_links, "^OD_ZCU")
-    absolute_first_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_first_level_links, url)
-
-    files = []
-
-    for link in absolute_first_level_links:
-        second_level_links = BasicCrawlerFunctions.get_all_links(link)
-        filtered_second_level_links = BasicCrawlerFunctions.filter_links(second_level_links, regex)
-        absolute_second_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_second_level_links, link)
-        final_links = BasicCrawlerFunctions.remove_downloaded_links(absolute_second_level_links, dataset_name)
-
-        for file_link in final_links:
-            files.append(file_link)
-
-    for file in files:
-        BasicCrawlerFunctions.download_file_from_url(file, dataset_name)
-
-    FolderProcessor.unzip_all_csv_zip_files_in_folder(path_for_files)
diff --git a/modules/crawler/DatasetCrawler/KOLOBEZKY_crawler.py b/modules/crawler/DatasetCrawler/KOLOBEZKY_crawler.py
new file mode 100644
index 0000000..b26d3f6
--- /dev/null
+++ b/modules/crawler/DatasetCrawler/KOLOBEZKY_crawler.py
@@ -0,0 +1,41 @@
+from Utilities import folder_processor
+from Utilities.Crawler import basic_crawler_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def crawl(config):
+    """
+    Implement crawl method that downloads new data to path_for_files
+    For keeping the project structure
+    url , regex, and dataset_name from config
+    You can use already implemented functions from Utilities/Crawler/basic_crawler_functions.py
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    first_level_links = basic_crawler_functions.get_all_links(url)
+    filtered_first_level_links = basic_crawler_functions.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = basic_crawler_functions.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = basic_crawler_functions.get_all_links(link)
+        filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
+        absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
+        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        basic_crawler_functions.download_file_from_url(file, dataset_name)
+
+    folder_processor.unzip_all_csv_zip_files_in_folder(path_for_files)
diff --git a/modules/crawler/DatasetCrawler/WIFICrawler.py b/modules/crawler/DatasetCrawler/WIFICrawler.py
deleted file mode 100644
index adbb507..0000000
--- a/modules/crawler/DatasetCrawler/WIFICrawler.py
+++ /dev/null
@@ -1,41 +0,0 @@
-from Utilities import FolderProcessor
-from Utilities.Crawler import BasicCrawlerFunctions
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-
-
-def crawl(config):
-    """
-    Implement crawl method that downloads new data to path_for_files
-    For keeping the project structure
-    url , regex, and dataset_name from config
-    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    url = config['url']
-    regex = config['regex']
-    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
-
-    first_level_links = BasicCrawlerFunctions.get_all_links(url)
-    filtered_first_level_links = BasicCrawlerFunctions.filter_links(first_level_links, "^OD_ZCU")
-    absolute_first_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_first_level_links, url)
-
-    files = []
-
-    for link in absolute_first_level_links:
-        second_level_links = BasicCrawlerFunctions.get_all_links(link)
-        filtered_second_level_links = BasicCrawlerFunctions.filter_links(second_level_links, regex)
-        absolute_second_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_second_level_links, link)
-        final_links = BasicCrawlerFunctions.remove_downloaded_links(absolute_second_level_links, dataset_name)
-
-        for file_link in final_links:
-            files.append(file_link)
-
-    for file in files:
-        BasicCrawlerFunctions.download_file_from_url(file, dataset_name)
-
-    FolderProcessor.unzip_all_csv_zip_files_in_folder(path_for_files)
diff --git a/modules/crawler/DatasetCrawler/WIFI_crawler.py b/modules/crawler/DatasetCrawler/WIFI_crawler.py
new file mode 100644
index 0000000..b26d3f6
--- /dev/null
+++ b/modules/crawler/DatasetCrawler/WIFI_crawler.py
@@ -0,0 +1,41 @@
+from Utilities import folder_processor
+from Utilities.Crawler import basic_crawler_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def crawl(config):
+    """
+    Implement crawl method that downloads new data to path_for_files
+    For keeping the project structure
+    url , regex, and dataset_name from config
+    You can use already implemented functions from Utilities/Crawler/basic_crawler_functions.py
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    first_level_links = basic_crawler_functions.get_all_links(url)
+    filtered_first_level_links = basic_crawler_functions.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = basic_crawler_functions.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = basic_crawler_functions.get_all_links(link)
+        filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
+        absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
+        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        basic_crawler_functions.download_file_from_url(file, dataset_name)
+
+    folder_processor.unzip_all_csv_zip_files_in_folder(path_for_files)
diff --git a/modules/crawler/DatasetProcessing/JISProcessor.py b/modules/crawler/DatasetProcessing/JISProcessor.py
deleted file mode 100644
index d139bd2..0000000
--- a/modules/crawler/DatasetProcessing/JISProcessor.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from Utilities.CSV import CSVDataLine
-from Utilities import DateFormating
-
-
-def process_file(filename):
-    """
-    Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
-    and value is dictionary where keys devices (specified in configuration file)
-    and value is CSVDataLine.CSVDataLine with device,date and occurrence
-
-    Args:
-    filename: name of processed file
-
-    Returns:
-    None if not implemented
-    date_dict when implemented
-    """
-    date_dict = dict()
-
-    with open(filename, "r", encoding="utf-8") as file:
-
-        for line in file:
-
-            array = line.split(";")
-
-            date = DateFormating.date_time_formatter(array[1][1:-1])
-            name = array[0][1:-1]
-            occurrence = array[2][:-1]
-
-            if date not in date_dict:
-                date_dict[date] = dict()
-
-            if name in date_dict[date]:
-                date_dict[date][name].occurrence += int(occurrence)
-            else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, occurrence)
-
-    return date_dict
-
diff --git a/modules/crawler/DatasetProcessing/JIS_processor.py b/modules/crawler/DatasetProcessing/JIS_processor.py
new file mode 100644
index 0000000..6474f21
--- /dev/null
+++ b/modules/crawler/DatasetProcessing/JIS_processor.py
@@ -0,0 +1,40 @@
+from Utilities.CSV import csv_data_line
+from Utilities import date_formating
+
+
+def process_file(filename):
+    """
+    Method that take path to crawled file and outputs date dictionary:
+    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    and value is dictionary where keys are devices (specified in configuration file)
+    and value is CSVDataLine.csv_data_line with device,date and occurrence
+
+    Args:
+    filename: name of processed file
+
+    Returns:
+    None if not implemented
+    date_dict when implemented
+    """
+    date_dict = dict()
+
+    with open(filename, "r", encoding="utf-8") as file:
+
+        for line in file:
+
+            array = line.split(";")
+
+            date = date_formating.date_time_formatter(array[1][1:-1])
+            name = array[0][1:-1]
+            occurrence = array[2][:-1]
+
+            if date not in date_dict:
+                date_dict[date] = dict()
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurrence += int(occurrence)
+            else:
+                date_dict[date][name] = csv_data_line.CSVDataLine(name, date, occurrence)
+
+    return date_dict
+
diff --git a/modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py b/modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py
deleted file mode 100644
index 85d65e2..0000000
--- a/modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py
+++ /dev/null
@@ -1,39 +0,0 @@
-from Utilities.CSV import CSVDataLine
-from Utilities import DateFormating
-
-
-def process_file(filename):
-    """
-    Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
-    and value is dictionary where keys devices (specified in configuration file)
-    and value is CSVDataLine.CSVDataLine with device,date and occurrence
-
-    Args:
-    filename: name of processed file
-
-    Returns:
-    None if not implemented
-    date_dict when implemented
-    """
-    date_dict = dict()
-
-    with open(filename, "r") as file:
-
-        for line in file:
-
-            array = line.split(";")
-
-            date = DateFormating.date_time_formatter(array[0][1:-1])
-            name = array[1][1:-1]
-
-            if date not in date_dict:
-                date_dict[date] = dict()
-
-            if name in date_dict[date]:
-                date_dict[date][name].occurrence += 1
-            else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, 1)
-
-    return date_dict
-
diff --git a/modules/crawler/DatasetProcessing/KOLOBEZKY_processor.py b/modules/crawler/DatasetProcessing/KOLOBEZKY_processor.py
new file mode 100644
index 0000000..819d770
--- /dev/null
+++ b/modules/crawler/DatasetProcessing/KOLOBEZKY_processor.py
@@ -0,0 +1,39 @@
+from Utilities.CSV import csv_data_line
+from Utilities import date_formating
+
+
+def process_file(filename):
+    """
+    Method that take path to crawled file and outputs date dictionary:
+    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    and value is dictionary where keys are devices (specified in configuration file)
+    and value is CSVDataLine.csv_data_line with device,date and occurrence
+
+    Args:
+    filename: name of processed file
+
+    Returns:
+    None if not implemented
+    date_dict when implemented
+    """
+    date_dict = dict()
+
+    with open(filename, "r") as file:
+
+        for line in file:
+
+            array = line.split(";")
+
+            date = date_formating.date_time_formatter(array[0][1:-1])
+            name = array[1][1:-1]
+
+            if date not in date_dict:
+                date_dict[date] = dict()
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurrence += 1
+            else:
+                date_dict[date][name] = csv_data_line.CSVDataLine(name, date, 1)
+
+    return date_dict
+
diff --git a/modules/crawler/DatasetProcessing/WIFIProcessor.py b/modules/crawler/DatasetProcessing/WIFIProcessor.py
deleted file mode 100644
index c071f85..0000000
--- a/modules/crawler/DatasetProcessing/WIFIProcessor.py
+++ /dev/null
@@ -1,40 +0,0 @@
-from Utilities.CSV import CSVDataLine
-from Utilities import DateFormating
-
-
-def process_file(filename):
-    """
-    Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
-    and value is dictionary where keys devices (specified in configuration file)
-    and value is CSVDataLine.CSVDataLine with device,date and occurrence
-
-    Args:
-    filename: name of processed file
-
-    Returns:
-    None if not implemented
-    date_dict when implemented
-    """
-    date_dict = dict()
-
-    with open(filename, "r", encoding="utf-8") as file:
-
-        for line in file:
-
-            array = line.split(";")
-
-            date = DateFormating.date_time_formatter(array[4][1:-2])
-            name = array[1][1:-1]
-            occurrence = array[0]
-
-            if date not in date_dict:
-                date_dict[date] = dict()
-
-            if name in date_dict[date]:
-                date_dict[date][name].occurrence += max(date_dict[date][name].occurrence,int(occurrence))
-            else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, int(occurrence))
-
-    return date_dict
-
diff --git a/modules/crawler/DatasetProcessing/WIFI_processor.py b/modules/crawler/DatasetProcessing/WIFI_processor.py
new file mode 100644
index 0000000..a5572d1
--- /dev/null
+++ b/modules/crawler/DatasetProcessing/WIFI_processor.py
@@ -0,0 +1,40 @@
+from Utilities.CSV import csv_data_line
+from Utilities import date_formating
+
+
+def process_file(filename):
+    """
+    Method that take path to crawled file and outputs date dictionary:
+    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    and value is dictionary where keys are devices (specified in configuration file)
+    and value is CSVDataLine.csv_data_line with device,date and occurrence
+
+    Args:
+    filename: name of processed file
+
+    Returns:
+    None if not implemented
+    date_dict when implemented
+    """
+    date_dict = dict()
+
+    with open(filename, "r", encoding="utf-8") as file:
+
+        for line in file:
+
+            array = line.split(";")
+
+            date = date_formating.date_time_formatter(array[4][1:-2])
+            name = array[1][1:-1]
+            occurrence = array[0]
+
+            if date not in date_dict:
+                date_dict[date] = dict()
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurrence += max(date_dict[date][name].occurrence,int(occurrence))
+            else:
+                date_dict[date][name] = csv_data_line.CSVDataLine(name, date, int(occurrence))
+
+    return date_dict
+
diff --git a/modules/crawler/ForceUpdateDataset.py b/modules/crawler/ForceUpdateDataset.py
deleted file mode 100644
index 53e275b..0000000
--- a/modules/crawler/ForceUpdateDataset.py
+++ /dev/null
@@ -1,5 +0,0 @@
-import Pipeline
-import os
-
-print("Zadejte jméno Datasetu který chcete upadtovat:\n")
-Pipeline.run_full_pipeline(input())
diff --git a/modules/crawler/ForceUpdateDatasets.py b/modules/crawler/ForceUpdateDatasets.py
deleted file mode 100644
index 820f9f8..0000000
--- a/modules/crawler/ForceUpdateDatasets.py
+++ /dev/null
@@ -1,19 +0,0 @@
-import Pipeline
-import os
-
-# Path to configuration files
-CONFIG_FILES_PATH = "DatasetConfigs/"
-
-
-def run_pipeline_for_all_datasets():
-    """
-    Runs whole DataScript pipeline for every dataset that has existing configuration file
-    """
-    files_in_dir = os.listdir(CONFIG_FILES_PATH)
-
-    for file in files_in_dir:
-        name = file.split('.')
-        Pipeline.run_full_pipeline(name[0])
-
-
-run_pipeline_for_all_datasets()
diff --git a/modules/crawler/Pipeline.py b/modules/crawler/Pipeline.py
deleted file mode 100644
index 054d5c6..0000000
--- a/modules/crawler/Pipeline.py
+++ /dev/null
@@ -1,252 +0,0 @@
-from Utilities import FolderProcessor, ConfigureFunctions
-from Utilities.Database import DatabaseLoader
-from Utilities.CSV import CSVutils
-
-import logging
-from datetime import date
-
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset crawler implementations
-CRAWLER_LIB_PATH = "DatasetCrawler."
-# Path to dataset processor implementations
-PROCESSOR_LIB_PATH = "DatasetProcessing."
-
-
-#logger
-logging.basicConfig(filename=CRAWLER_LOGS_PATH + "CommonRecords/" + 'Applicationlog-' + date.today().strftime("%b-%Y") + '.log',
-                    level=logging.INFO,
-                    format='%(asctime)s %(message)s'
-                    )
-
-
-def check_last_update(config):
-    """
-    Loads integer from updated.txt in CrawlerLogs/"dataset_name"
-    representing number of days from last update if number equals
-    number in confing update period updates it and reset number of
-    days to zero else increment the number
-
-    Arguments:
-        config loaded configuration file of dataset
-
-    Returns:
-       True if updating
-       Else if incementing days from last update
-    """
-    dataset_name = config["dataset-name"]
-
-    with open(CRAWLER_LOGS_PATH + dataset_name + "/" + "updated.txt", "r+") as file:
-        last_update = file.read()
-        last_update = int(last_update)
-        file.seek(0)
-
-        confing_update_period = int(config["update-period"])
-
-        if config["update-period"] <= last_update:
-            logging.info("Dataset " + dataset_name + " is being updated today")
-            file.write("0")
-            file.truncate()
-            return True
-        else:
-            last_update_days = last_update + 1
-            logging.info("Dataset " + dataset_name + " will be updated in " + str(confing_update_period - last_update_days) + "days")
-            file.write(str(last_update_days))
-            file.truncate()
-            return False
-
-
-
-def crawl_data(config):
-    """
-      Imports dataset crawler in DatasetCrawler/"dataset_name"Crawler.py
-      runs crawler.
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-
-    crawl_func = __import__(CRAWLER_LIB_PATH + dataset_name + "Crawler", globals(), locals(), ['crawl']).crawl
-    crawl_func(config)
-
-    dataset_name += '/'
-
-
-def process_data(dataset_name):
-    """
-    Goes trough every not processed file(not contained in CrawledData/"dataset_name"/ignore.txt)
-    Imports dataset processor in DatasetProcessing/"dataset_name"Processor.py
-    Runs processor on every file
-    After successful processing updates ignore.txt
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    dataset_path = dataset_name + '/'
-
-    process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "Processor", globals(), locals(),
-                                   ['process_file']).process_file
-
-    not_processed_files = FolderProcessor.list_of_all_files(CRAWLED_DATA_PATH + dataset_path)
-    logging.info(dataset_name + " has downloaded " + str(len(not_processed_files)) + " new files")
-
-    for not_processed_file in not_processed_files:
-        path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
-        date_dic = process_file_func(path)
-        CSVutils.export_data_to_csv(path, date_dic)
-        FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
-
-    logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")
-
-
-def validate_process_data(config):
-    """
-    Function goes through newly processed data and checks theirs status
-
-    Args:
-        config: loaded configuration file of dataset
-
-    Returns:
-        boolean variable TRUE/FALSE.
-        Data processed correctly - TRUE
-        Wrong format or NEW unknown devices - FALSE
-    """
-    processed_devices_set = FolderProcessor.get_devices_set(PROCESSED_DATA_PATH + config["dataset-name"] + '/')
-    unknown_devices_set = FolderProcessor.get_unknown_devices_set(config, processed_devices_set)
-    unknown_devices_size = len(unknown_devices_set)
-
-    if unknown_devices_size != 0:
-        logging.info("There is " + str(unknown_devices_size) + " unknown devices")
-        logging.info("Adding devices to " + config["dataset-name"] + " config file")
-        ConfigureFunctions.update_configuration(config["dataset-name"], unknown_devices_set)
-        return False
-
-    for device in config["devices"]:
-        device = config["devices"][device]
-        if device["x"] == "UNKNOWN!" or device["y"] == "UNKNOWN!":
-            logging.info(config["dataset-name"] + " config file contains devices with UNKOWN! values please update them!!")
-            return False
-
-    return True
-
-
-def load_data_to_database(config):
-    """
-    Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
-    loads data appends coordination from configurations
-    and exports it into the database
-    After successful exporting updates ignore.txt
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    dataset_path = dataset_name + '/'
-
-    # get all unprocessed files from dataset
-    not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
-
-    database_connection = DatabaseLoader.create_database_connection()
-
-    DatabaseLoader.check_or_update_datasets_collection(database_connection,config)
-
-    DatabaseLoader.update_devices_collection(database_connection,config)
-
-
-    # load every file
-    for not_loaded_file in not_loaded_files:
-        #check if file is not in database already if it is skip
-        test = DatabaseLoader.check_if_database_doesnt_contain_file(database_connection,dataset_name,not_loaded_file)
-        if test == False:
-            logging.info(dataset_name + " could not load " + not_loaded_file + " to database because he is already there.")
-            FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
-            continue
-        # load processed data
-        processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
-        # load processed data to database
-        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
-        FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
-
-    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
-
-
-def load_data_to_database_crone(config):
-    """
-    Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
-    loads data appends coordination from configurations
-    and exports it into the database
-    After successful exporting updates ignore.txt
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    dataset_path = dataset_name + '/'
-
-    # get all unprocessed files from dataset
-    not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
-
-    database_connection = DatabaseLoader.create_database_connection()
-
-    # load every file
-    for not_loaded_file in not_loaded_files:
-        # load processed data
-        processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
-        # load processed data to database
-        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
-        FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
-
-    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
-
-
-def run_full_pipeline(dataset_name):
-    """
-    Loads config file and starts full pipeline
-    -crawl data
-    -process data
-    -load data to database
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    logging.info("Starting pipeline for dataset " + dataset_name)
-    
-    config = ConfigureFunctions.load_configuration(dataset_name)
-    crawl_data(config)
-    process_data(config["dataset-name"])
-
-    validation_test = validate_process_data(config)
-
-    if validation_test:
-        load_data_to_database(config)
-
-
-def run_full_pipeline_crone(dataset_name):
-    """
-    Loads config file and starts full pipeline
-    -crawl data
-    -process data
-    -load data to database
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    logging.info("Starting pipeline for dataset " + dataset_name)
-
-    config = ConfigureFunctions.load_configuration(dataset_name)
-    update_test = check_last_update(config)
-    if update_test:
-        crawl_data(config)
-        process_data(config["dataset-name"])
-
-        validation_test = validate_process_data(config)
-
-        if validation_test:
-            load_data_to_database_crone(config)
-            
\ No newline at end of file
diff --git a/modules/crawler/PrepareNewDataset.py b/modules/crawler/PrepareNewDataset.py
deleted file mode 100644
index 2417796..0000000
--- a/modules/crawler/PrepareNewDataset.py
+++ /dev/null
@@ -1,161 +0,0 @@
-import os
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path for DatasetCrawlers implementations
-CRAWLER_PROGRAM_PATH = "DatasetCrawler"
-# Path for DatasetProcessors implementations
-PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-
-
-def create_default_config_file(dataset_name):
-    """
-    Creates default config file
-
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-    with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
-        file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
-        file.write("dataset-name: " + dataset_name + "\n")
-        file.write("# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře\n")
-        file.write("dataset-name: " + dataset_name + "\n")
-        file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
-        file.write("url: ZDE VLOZTE URL\n")
-        file.write("# volitelny parameter, ktery specifikuje vzor jmrna datasetu, ktera se budou stahovat\n")
-        file.write("regex: ZDE VLOZTE REGEX\n")
-        file.write("# volitelny parametr, ktery udava jak casto se budou hledat nove datasety, pokud prazdne, "
-                   "tak defaultni hodnota (dny)\n")
-        file.write("update-period: ZDE VLOZTE HODNOTU\n")
-        file.write("# pozice jednotlivych zarizeni, ktera jsou v datasetu\n")
-        file.write("devices:\n")
-
-
-def create_default_processor(dataset_name):
-    """
-    Creates default processor for dataset
-
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-    with open(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py", "w") as file:
-        file.write("from Utilities.CSV import CSVDataLine")
-        file.write("\n")
-        file.write("\n")
-        file.write("def process_file(filename):\n")
-        file.write("    \"\"\"\n")
-        file.write("    Method that take path to crawled file and outputs date dictionary:\n")
-        file.write("    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)\n")
-        file.write("    and value is dictionary where keys devices (specified in configuration file)\n")
-        file.write("    and value is CSVDataLine.CSVDataLine with device,date and occurrence\n")
-        file.write("\n")
-        file.write("    Args:\n")
-        file.write("    filename: name of processed file\n")
-        file.write("\n")
-        file.write("    Returns:\n")
-        file.write("    None if not implemented\n")
-        file.write("    date_dict when implemented\n")
-        file.write("    \"\"\"\n")
-        file.write("    date_dict = dict()\n")
-        file.write("\n")
-        file.write("    #with open(filename, \"r\") as file:\n")
-        file.write("    print(\"You must implements process_file method first!\")\n")
-        file.write("    return None\n")
-
-
-def create_default_crawler(dataset_name):
-    """
-    Creates default crawler for dataset
-
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-
-    with open(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py", "w") as file:
-        file.write("# Path to crawled data\n")
-        file.write("CRAWLED_DATA_PATH = \"CrawledData/\"\n")
-        file.write("\n")
-        file.write("\n")
-        file.write("def crawl(config):\n")
-        file.write("    \"\"\"\n")
-        file.write("    Implement crawl method that downloads new data to path_for_files\n")
-        file.write("    For keeping the project structure\n")
-        file.write("    url , regex, and dataset_name from config\n")
-        file.write("    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py\n")
-        file.write("\n")
-        file.write("    Args:\n")
-        file.write("        config: loaded configuration file of dataset\n")
-        file.write("    \"\"\"\n")
-        file.write("    dataset_name = config[\"dataset-name\"]\n")
-        file.write("    url = config['url']\n")
-        file.write("    regex = config['regex']\n")
-        file.write("    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'\n")
-        file.write("    print(\"You must implements Crawl method first!\")\n")
-
-
-def create_ignore_file(path, text):
-    """
-    Creates ignore file
-    Args:
-        path: path to directory for creating ignore.txt
-        text: text that will be on first line of ignore.txt can be None
-    """
-    with open(path + "/ignore.txt", "w") as file:
-        if text is not None:
-            file.write(text + "\n")
-
-def create_updated_file(path):
-    """
-    Creates updated file
-    Args:
-        path: path to directory for creating updated.txt
-    """
-    with open(path + "/updated.txt", "w") as file:
-            file.write(str(0) + "\n")
-
-
-def prepare_dataset_structure(dataset_name):
-    """
-    Prepares folders for new dataset
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-
-    # create folder for crawled data
-    try:
-        path = CRAWLED_DATA_PATH+dataset_name
-        os.mkdir(path)
-        create_ignore_file(path, "ignore.txt")
-    except os.error as e:
-        print(e)
-        print("Creation of the directory %s failed" % path)
-
-    # create folder for processed data
-    try:
-        path = PROCESSED_DATA_PATH + dataset_name
-        os.mkdir(path)
-        create_ignore_file(path, "ignore.txt")
-    except OSError:
-        print("Creation of the directory %s failed" % path)
-
-    # create folder for crawler logs
-    try:
-        path = CRAWLER_LOGS_PATH + dataset_name
-        os.mkdir(path)
-        create_ignore_file(path, None)
-        create_updated_file(path)
-    except OSError:
-        print("Creation of the directory %s failed" % path)
-
-    create_default_crawler(dataset_name)
-    create_default_processor(dataset_name)
-    create_default_config_file(dataset_name)
-
-print("Zadejte jméno nového datasetu:\n")
-prepare_dataset_structure(input())
diff --git a/modules/crawler/ProcessedData/JIS/ignore.txt b/modules/crawler/ProcessedData/JIS/ignore.txt
deleted file mode 100644
index 8ea087a..0000000
--- a/modules/crawler/ProcessedData/JIS/ignore.txt
+++ /dev/null
@@ -1 +0,0 @@
-ignore.txt
diff --git a/modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt b/modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt
deleted file mode 100644
index 8ea087a..0000000
--- a/modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt
+++ /dev/null
@@ -1 +0,0 @@
-ignore.txt
diff --git a/modules/crawler/ProcessedData/WIFI/ignore.txt b/modules/crawler/ProcessedData/WIFI/ignore.txt
deleted file mode 100644
index 8ea087a..0000000
--- a/modules/crawler/ProcessedData/WIFI/ignore.txt
+++ /dev/null
@@ -1 +0,0 @@
-ignore.txt
diff --git a/modules/crawler/RemoveDataset.py b/modules/crawler/RemoveDataset.py
deleted file mode 100644
index eed55e2..0000000
--- a/modules/crawler/RemoveDataset.py
+++ /dev/null
@@ -1,35 +0,0 @@
-import os
-import shutil
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-# Path for DatasetCrawlers implementations
-CRAWLER_PROGRAM_PATH = "DatasetCrawler"
-# Path for DatasetProcessors implementations
-PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
-
-
-def remove_dataset(dataset_name):
-    """
-    Remove dataset
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    shutil.rmtree(CRAWLED_DATA_PATH + dataset_name + "/")
-    shutil.rmtree(PROCESSED_DATA_PATH + dataset_name + "/")
-    shutil.rmtree(CRAWLER_LOGS_PATH + dataset_name + "/")
-
-    os.remove(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml")
-    os.remove(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py")
-    os.remove(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py")
-
-    print("Dataset " + dataset_name + " removed")
-
-print("Zadejte jméno Datasetu který chcete odstranit:\n")
-remove_dataset(input())
diff --git a/modules/crawler/RemoveDatasetDatabase.py b/modules/crawler/RemoveDatasetDatabase.py
deleted file mode 100644
index b7998bc..0000000
--- a/modules/crawler/RemoveDatasetDatabase.py
+++ /dev/null
@@ -1,31 +0,0 @@
-from Utilities.Database import DatabaseLoader
-
-
-def remove_dataset_database(dataset_name):
-    """
-    Removes dataset entries from database
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    # Creating connection
-    mydb = DatabaseLoader.create_database_connection()
-
-    # collection where are specified aviable datasets
-    collection_datasets = mydb[DatabaseLoader.MONGODB_DATASET_COLLECTION]
-
-    collection_datasets.delete_one({"name": dataset_name})
-    print("Removing record from DATASETS collection")
-
-
-    # Retrieve list of all collections
-    collections = mydb.list_collection_names()
-
-    # Drop of all collections
-    for name in collections:
-        if name.startswith(dataset_name):
-            mydb[name].drop()
-            print("Dropping: " + name)
-
-
-print("Zadejte jméno Datasetu který chcete odstranit z databáze:\n")
-remove_dataset_database(input())
\ No newline at end of file
diff --git a/modules/crawler/ResetDatabaseData.py b/modules/crawler/ResetDatabaseData.py
deleted file mode 100644
index 6295ba6..0000000
--- a/modules/crawler/ResetDatabaseData.py
+++ /dev/null
@@ -1,21 +0,0 @@
-from Utilities.Database import DatabaseLoader
-
-
-def clean_database():
-    """
-    Deletes all collections from database
-    """
-    # Create connection
-    mydb = DatabaseLoader.create_database_connection()
-
-    # Retrieve list of all collections
-    collections = mydb.list_collection_names()
-
-    # Drop of all collections
-    for name in collections:
-        mydb[name].drop()
-
-    print("Database Cleaned")
-
-
-clean_database()
\ No newline at end of file
diff --git a/modules/crawler/ResetDataset.py b/modules/crawler/ResetDataset.py
deleted file mode 100644
index 812f824..0000000
--- a/modules/crawler/ResetDataset.py
+++ /dev/null
@@ -1,56 +0,0 @@
-import os
-from Utilities import FolderProcessor
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-
-
-def create_ignore_file(path, text):
-    """
-    Creates ignore file
-    Args:
-        path: path to directory for creating ignore.txt
-        text: text that will be on first line of ignore.txt can be None
-    """
-    with open(path + "/ignore.txt", "w") as file:
-        if text is not None:
-            file.write(text + "\n")
-
-
-def create_updated_file(path):
-    """
-    Creates updated file
-    Args:
-        path: path to directory for creating updated.txt
-    """
-    with open(path + "/updated.txt", "w") as file:
-            file.write(str(0) + "\n")
-
-
-def reset_dataset(dataset_name):
-    """
-    Resets all saved data in dataset except config and implementation
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    path = CRAWLED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = PROCESSED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = CRAWLER_LOGS_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, None)
-    create_updated_file(path)
-
-print("Zadejte jméno Datasetu který chcete resetovat:\n")
-reset_dataset(input())
diff --git a/modules/crawler/ResetDatasets.py b/modules/crawler/ResetDatasets.py
deleted file mode 100644
index e2f8497..0000000
--- a/modules/crawler/ResetDatasets.py
+++ /dev/null
@@ -1,66 +0,0 @@
-import os
-from Utilities import FolderProcessor
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-
-
-def create_ignore_file(path, text):
-    """
-    Creates ignore file
-    Args:
-        path: path to directory for creating ignore.txt
-        text: text that will be on first line of ignore.txt can be None
-    """
-    with open(path + "/ignore.txt", "w") as file:
-        if text is not None:
-            file.write(text + "\n")
-
-
-def create_updated_file(path):
-    """
-    Creates updated file
-    Args:
-        path: path to directory for creating updated.txt
-    """
-    with open(path + "/updated.txt", "w") as file:
-            file.write(str(0) + "\n")
-
-
-def reset_dataset(dataset_name):
-    """
-    Resets all saved data in dataset except config and implementation
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    path = CRAWLED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = PROCESSED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = CRAWLER_LOGS_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, None)
-    create_updated_file(path)
-
-
-def reset_all_datasets():
-    """
-    Resets all saved data in all datasets with config file except configs and implementation
-    """
-    datasets = os.listdir(CONFIG_FILES_PATH)
-
-    for dataset in datasets:
-        reset_dataset(dataset.split('.')[0])
-
-
-reset_all_datasets()
diff --git a/modules/crawler/Utilities/CSV/CSVDataLine.py b/modules/crawler/Utilities/CSV/CSVDataLine.py
deleted file mode 100644
index 90f15e7..0000000
--- a/modules/crawler/Utilities/CSV/CSVDataLine.py
+++ /dev/null
@@ -1,22 +0,0 @@
-class CSVDataLine:
-    """
-    Class that specifies the look of data line in processed csv file
-    prepared for database
-    """
-
-    def __init__(self, name, date, occurrence):
-        try:
-            test_val = int(occurrence)
-        except ValueError:
-            print("Occurence should be and integer value!")
-
-        if len(date) != 13:
-            raise ValueError("Invalid date format YYYY-dd-mm-hh expected!")    
-
-        self.name = name
-        self.date = date
-        self.occurrence = test_val
-
-    def to_csv(self):
-        return self.name + ";" + str(self.occurrence) + ";" + self.date
-
diff --git a/modules/crawler/Utilities/CSV/CSVutils.py b/modules/crawler/Utilities/CSV/CSVutils.py
deleted file mode 100644
index 81ab498..0000000
--- a/modules/crawler/Utilities/CSV/CSVutils.py
+++ /dev/null
@@ -1,48 +0,0 @@
-import inspect
-from Utilities.CSV import CSVDataLine
-
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-
-
-def get_unique_names_from_file(filename, column_number):
-    """
-        Extract set of unique names from file
-    Args:
-        filename: path to processed file
-        column_number: unique names are expected in csv file on column_number
-
-    Returns:
-        set of unique names
-    """
-    # create set of unique names
-    name_set = set()
-
-    with open(filename, "r") as file:
-        # go through every line of line
-        for x in file:
-            # split by csv splitter ;
-            array = x.split(";")
-            # add string from chosen column to set
-            name_set.add(array[column_number])
-
-    return name_set
-
-
-def export_data_to_csv(filename, data_dict):
-    """
-        Takes data_dict and export it into a csv file
-    Args:
-        filename: name of exported file
-        data_dict: dictionary containing data from DatasetProcessor
-    """
-    with open(PROCESSED_DATA_PATH + filename[12:], "w+") as file:
-
-        for date in data_dict:
-            if len(date) != 13:
-                raise ValueError("Invalid date format for key value --> YYYY-mm-dd-hh expected!")   
-            for data in data_dict[date]:
-                csv_line = data_dict[date][data]
-                if not isinstance(csv_line,CSVDataLine.CSVDataLine):
-                    raise ValueError("data_dict is expected to have CSVDataLine as values")
-                file.write(csv_line.to_csv() + '\n')
\ No newline at end of file
diff --git a/modules/crawler/Utilities/CSV/csv_data_line.py b/modules/crawler/Utilities/CSV/csv_data_line.py
new file mode 100644
index 0000000..90f15e7
--- /dev/null
+++ b/modules/crawler/Utilities/CSV/csv_data_line.py
@@ -0,0 +1,22 @@
+class CSVDataLine:
+    """
+    Class that specifies the look of data line in processed csv file
+    prepared for database
+    """
+
+    def __init__(self, name, date, occurrence):
+        try:
+            test_val = int(occurrence)
+        except ValueError:
+            print("Occurence should be and integer value!")
+
+        if len(date) != 13:
+            raise ValueError("Invalid date format YYYY-dd-mm-hh expected!")    
+
+        self.name = name
+        self.date = date
+        self.occurrence = test_val
+
+    def to_csv(self):
+        return self.name + ";" + str(self.occurrence) + ";" + self.date
+
diff --git a/modules/crawler/Utilities/CSV/csv_utils.py b/modules/crawler/Utilities/CSV/csv_utils.py
new file mode 100644
index 0000000..2865e38
--- /dev/null
+++ b/modules/crawler/Utilities/CSV/csv_utils.py
@@ -0,0 +1,48 @@
+import inspect
+from Utilities.CSV import csv_data_line
+
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+
+
+def get_unique_names_from_file(filename, column_number):
+    """
+        Extract set of unique names from file
+    Args:
+        filename: path to processed file
+        column_number: unique names are expected in csv file on column_number
+
+    Returns:
+        set of unique names
+    """
+    # create set of unique names
+    name_set = set()
+
+    with open(filename, "r") as file:
+        # go through every line of line
+        for x in file:
+            # split by csv splitter ;
+            array = x.split(";")
+            # add string from chosen column to set
+            name_set.add(array[column_number])
+
+    return name_set
+
+
+def export_data_to_csv(filename, data_dict):
+    """
+        Takes data_dict and export it into a csv file
+    Args:
+        filename: name of exported file
+        data_dict: dictionary containing data from DatasetProcessor
+    """
+    with open(PROCESSED_DATA_PATH + filename[12:], "w+") as file:
+
+        for date in data_dict:
+            if len(date) != 13:
+                raise ValueError("Invalid date format for key value --> YYYY-mm-dd-hh expected!")   
+            for data in data_dict[date]:
+                csv_line = data_dict[date][data]
+                if not isinstance(csv_line,csv_data_line.CSVDataLine):
+                    raise ValueError("data_dict is expected to have CSVDataLine as values")
+                file.write(csv_line.to_csv() + '\n')
\ No newline at end of file
diff --git a/modules/crawler/Utilities/ConfigureFunctions.py b/modules/crawler/Utilities/ConfigureFunctions.py
deleted file mode 100644
index 1864337..0000000
--- a/modules/crawler/Utilities/ConfigureFunctions.py
+++ /dev/null
@@ -1,46 +0,0 @@
-import yaml
-
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs/"
-# Config file type
-CONFIG_FILE_TYPE = ".yaml"
-
-
-def load_configuration(dataset_name):
-    """
-    Loads yaml configuration file into memory
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-
-    Returns:
-        yaml configuration file as dictionary
-    """
-    with open(CONFIG_FILES_PATH + dataset_name + CONFIG_FILE_TYPE, "r") as f:
-        data = yaml.load(f, Loader=yaml.FullLoader)
-
-    devices_dic = dict()
-
-    if data["devices"] is not None:
-        for item in data["devices"]:
-            devices_dic.update(item)
-
-    data["devices"] = devices_dic
-
-    return data
-
-
-def update_configuration(dataset_name, new_devices):
-    """
-    Open dataset and appends new_devices to the end
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-        new_devices: list or set of new devices for dataset
-    """
-    with open(CONFIG_FILES_PATH + dataset_name + CONFIG_FILE_TYPE, "a") as file:
-        for device in new_devices:
-            file.write("  - "+device+":\n")
-            file.write("      x: UNKNOWN!\n")
-            file.write("      y: UNKNOWN!\n")
-            file.write("\n")
diff --git a/modules/crawler/Utilities/Crawler/BasicCrawlerFunctions.py b/modules/crawler/Utilities/Crawler/BasicCrawlerFunctions.py
deleted file mode 100644
index c94a1b4..0000000
--- a/modules/crawler/Utilities/Crawler/BasicCrawlerFunctions.py
+++ /dev/null
@@ -1,115 +0,0 @@
-import requests
-import re
-from Utilities import FolderProcessor
-from bs4 import BeautifulSoup
-
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-
-
-def get_all_links(url):
-    """
-    Sends http request to url, downloads all data,
-    extract links
-
-    Args:
-        url: url of website we want to search
-
-    Returns:
-        list of all links
-    """
-    # create response object
-    r = requests.get(url)
-
-    # create beautiful-soup object
-    soup = BeautifulSoup(r.content, 'html5lib')
-    links = []
-
-    for link in soup.findAll('a'):
-        links.append(link.get('href'))
-
-    return links
-
-
-def filter_links(links, regex):
-    """
-    Filters list of links using regex
-
-    Args:
-        links: list of links
-        regex: regex used for filtering
-
-    Returns:
-        filtered list of links
-    """
-    filtered_links = []
-
-    for link in links:
-        if re.search(regex, link):
-            filtered_links.append(link)
-
-    return filtered_links
-
-
-def create_absolute_links(links, archive):
-    """
-        Appends archive path to every link in links
-    Args:
-        links: list of relative links
-        archive: archive url
-
-    Returns:
-        list of absolute links
-    """
-    absolute_links = []
-
-    for link in links:
-        absolute_links.append(archive + link)
-
-    return absolute_links
-
-
-def remove_downloaded_links(links, dataset_name):
-    """
-    Loads already downloaded links from CRAWLER_LOGS_PATH ignore.txt
-    Args:
-        links: list of links
-        dataset_name: name of dataset that has existing configuration file
-
-    Returns:
-        List of links without already downloaded links
-    """
-    downloaded_links = FolderProcessor.load_ignore_set(CRAWLER_LOGS_PATH + dataset_name + "/")
-    final_links = set(links) - downloaded_links
-
-    return final_links
-
-
-def download_file_from_url(url, dataset_name):
-    """
-    Downloads file on provided url and saves it to path
-    Args:
-        url: url file we want to download
-        dataset_name: name of dataset that has existing configuration file
-    """
-    r = requests.get(url, stream=True)
-
-    # splits url and extract last part that contains filename
-    url_parts = url.split("/")
-    file_name = url_parts[len(url_parts)-1]
-
-    log_path = CRAWLER_LOGS_PATH + dataset_name + '/'
-    data_path = CRAWLED_DATA_PATH + dataset_name + '/'
-
-    # download file chunk by chunk so we can download large files
-    with open(data_path + file_name, "wb") as file:
-        for chunk in r.iter_content(chunk_size=1024):
-
-            # writing one chunk at a time to file
-            if chunk:
-                file.write(chunk)
-
-    # after successful download update list of already downloaded files
-    FolderProcessor.update_ignore_set(CRAWLER_LOGS_PATH + dataset_name + "/", url)
diff --git a/modules/crawler/Utilities/Crawler/basic_crawler_functions.py b/modules/crawler/Utilities/Crawler/basic_crawler_functions.py
new file mode 100644
index 0000000..0cbaed4
--- /dev/null
+++ b/modules/crawler/Utilities/Crawler/basic_crawler_functions.py
@@ -0,0 +1,115 @@
+import requests
+import re
+from Utilities import folder_processor
+from Utilities.Database import database_record_logs
+from bs4 import BeautifulSoup
+
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def get_all_links(url):
+    """
+    Sends http request to url, downloads all data,
+    extract links
+
+    Args:
+        url: url of website we want to search
+
+    Returns:
+        list of all links
+    """
+    # create response object
+    r = requests.get(url)
+
+    # create beautiful-soup object
+    soup = BeautifulSoup(r.content, 'html5lib')
+    links = []
+
+    for link in soup.findAll('a'):
+        links.append(link.get('href'))
+
+    return links
+
+
+def filter_links(links, regex):
+    """
+    Filters list of links using regex
+
+    Args:
+        links: list of links
+        regex: regex used for filtering
+
+    Returns:
+        filtered list of links
+    """
+    filtered_links = []
+
+    for link in links:
+        if re.search(regex, link):
+            filtered_links.append(link)
+
+    return filtered_links
+
+
+def create_absolute_links(links, archive):
+    """
+        Appends archive path to every link in links
+    Args:
+        links: list of relative links
+        archive: archive url
+
+    Returns:
+        list of absolute links
+    """
+    absolute_links = []
+
+    for link in links:
+        absolute_links.append(archive + link)
+
+    return absolute_links
+
+
+def remove_downloaded_links(links, dataset_name):
+    """
+    Loads already downloaded links from CRAWLER_LOGS_PATH ignore.txt
+    Args:
+        links: list of links
+        dataset_name: name of dataset that has existing configuration file
+
+    Returns:
+        List of links without already downloaded links
+    """
+    downloaded_links = database_record_logs.load_ignore_set_links(dataset_name)
+    final_links = set(links) - downloaded_links
+
+    return final_links
+
+
+def download_file_from_url(url, dataset_name):
+    """
+    Downloads file on provided url and saves it to path
+    Args:
+        url: url file we want to download
+        dataset_name: name of dataset that has existing configuration file
+    """
+    r = requests.get(url, stream=True)
+
+    # splits url and extract last part that contains filename
+    url_parts = url.split("/")
+    file_name = url_parts[len(url_parts)-1]
+
+    data_path = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    # download file chunk by chunk so we can download large files
+    with open(data_path + file_name, "wb") as file:
+        for chunk in r.iter_content(chunk_size=1024):
+
+            # writing one chunk at a time to file
+            if chunk:
+                file.write(chunk)
+
+    # after successful download update list of already downloaded files
+    database_record_logs.update_ignore_set_links(dataset_name, url)
diff --git a/modules/crawler/Utilities/Database/DatabaseDataLine.py b/modules/crawler/Utilities/Database/DatabaseDataLine.py
deleted file mode 100644
index 47e53d8..0000000
--- a/modules/crawler/Utilities/Database/DatabaseDataLine.py
+++ /dev/null
@@ -1,14 +0,0 @@
-class DatabaseDataLine:
-    """
-    Class that specifies the look of data line in database
-    """
-    def __init__(self, name, longitude, latitude, date, occurrence):
-        self.name = name
-        self.latitude = latitude
-        self.longitude = longitude
-        self.date = date
-        self.occurrence = occurrence
-
-    def to_dictionary(self):
-        return {"place": self.name, "x": self.longitude, "y": self.latitude, "number": self.occurrence,
-                "date": self.date}
diff --git a/modules/crawler/Utilities/Database/DatabaseLoader.py b/modules/crawler/Utilities/Database/DatabaseLoader.py
deleted file mode 100644
index e56e088..0000000
--- a/modules/crawler/Utilities/Database/DatabaseLoader.py
+++ /dev/null
@@ -1,183 +0,0 @@
-from Utilities.Database import DatabaseDataLine
-import pymongo
-
-# specify mongodb connection
-MONGODB_CONNECTION = "mongodb://root:root@database"
-# mongodb account name
-MONGODB_ACC_NAME = "root"
-# mongodb account password
-MONGODB_ACC_PASSWORD = "root"
-# mongodb data database
-MONGODB_DATA_DATABASE = "open-data-db"
-# mongodb collection with aviable datasets
-MONGODB_DATASET_COLLECTION = "DATASETS"
-# mongodb collection with aviable diveces of datasets
-MONGODB_DATASET_DEVICES_COLLECTION = "DEVICES"
-# mongodb collection with aviable diveces of datasets
-MONGODB_DATASET_LOADED_FILES_COLLECTION = "FILES"
-
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-
-
-def create_database_connection():
-    """
-    Creates connection to mongoDB
-    
-    Returns:
-        Connection to mongoDB
-    """
-    client = pymongo.MongoClient(MONGODB_CONNECTION)
-
-    # Authenticating
-    client.admin.authenticate(MONGODB_ACC_NAME, MONGODB_ACC_PASSWORD)
-
-    database = client[MONGODB_DATA_DATABASE]
-
-    return database
-
-
-def get_data_from_file(filename, config):
-    """
-        Opens processed file, reads it line by line
-        name, ocurrence, date
-        searches name in config and adds device map coordinates
-        than creates a dictionary with date without hours as key
-        and list of data lines as value.
-    Args:
-        filename: name of processed file
-        config: loaded configuration file of dataset
-
-    Returns:
-        dictionary with date without hours as key
-        and list of Datalines as value
-    """
-    dataset_name = config["dataset-name"]
-    dataset_path = PROCESSED_DATA_PATH + dataset_name + '/'
-
-    f = open(dataset_path + filename, "r")
-
-    devices = config["devices"]
-    date_dict = dict()
-
-    for line in f:
-        line = line[:-1]
-
-        csv_column = line.split(";")
-
-        name = csv_column[0]
-
-        if devices[name]["x"] == "SKIP" or devices[name]["y"] == "SKIP":
-            continue
-
-        occurrence = csv_column[1]
-        date = csv_column[2]
-        database_data_line = DatabaseDataLine.DatabaseDataLine(
-            name, devices[name]["x"], devices[name]["y"], date, occurrence)
-
-        # if you want to change table split by hours or months change this YYYY-mm-hh-dd
-        date_without_hours = date[:-3]
-        if date_without_hours not in date_dict:
-            date_dict[date_without_hours] = list()
-
-        date_dict[date_without_hours].append(
-            database_data_line.to_dictionary())
-
-    return date_dict
-
-
-def load_data_to_database(database_connection,dataset_name, data_dic, file_name):
-    """
-    Takes data_dic created in method get_data_from_file
-    and loads into into database where collection name is dataset_name + data_dic key
-    and data lines are line in collection
-
-    Args:
-        database_connection: created connection to a MONGODB
-        config: loaded configuration file of dataset
-        data_dic: dictionary of data lines created in get_data_from_file
-        file_name: name of file containing data
-    """
-
-    for date in data_dic:
-        dataset_collections = database_connection[dataset_name]
-        dataset_collections.insert_one({'date': date})
-        date_dataset = database_connection[dataset_name + date]
-        date_dataset.insert_many(data_dic[date])
-
-    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]
-    collection_loaded_files.insert_one({'file': file_name})
-
-
-
-def check_or_update_datasets_collection(database_connection,config):
-    """
-    Checks if DATASETS collection contains dataset and if display name was not updated
-
-    Args:
-        database_connection: created connection to a MONGODB
-        config: loaded configuration file of dataset
-    """
-    # collection where are specified aviable datasets
-    collection_datasets = database_connection[MONGODB_DATASET_COLLECTION]
-
-    dataset_name = config['dataset-name']
-    display_name = config['display-name']
-
-    query = {'key-name': dataset_name}
-
-    # check if newly added data already have a dataset specified in collection
-    dataset_present = collection_datasets.find_one(query)
-
-    if dataset_present is None:
-        collection_datasets.insert_one({'key-name': dataset_name, 'display-name': display_name})
-    elif dataset_present['display-name'] != display_name:
-        newvalues = { "$set": { 'display-name': display_name } }
-        collection_datasets.update_one(query, newvalues)
-
-
-def update_devices_collection(database_connection,config):
-    """
-    Checks if dataset_name collection contains every device with current set up
-
-    Args:
-        database_connection: created connection to a MONGODB
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config['dataset-name']
-
-    collection_devices = database_connection[dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
-
-    collection_devices.delete_many({})
-
-    devices = config['devices']
-
-    devices_list = list()
-
-    for device in devices.keys():
-        if devices[device]['x'] != "SKIP" or devices[device]['y'] != "SKIP":
-            devices_list.append({'name': device , 'x': devices[device]['x'] , 'y': devices[device]['y'] })
-
-    collection_devices.insert_many(devices_list)
-
-
-def check_if_database_doesnt_contain_file(database_connection,dataset_name,file_name):
-    """
-    Checks if dataset_name collection contains every device with current set up
-
-    Args:
-        database_connection: created connection to a MONGODB
-        filename: checked file name
-    """
-
-    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]
-
-    query = {'file': file_name}
-
-    # check if newly added data already have a dataset specified in collection
-    dataset_present = collection_loaded_files.find_one(query)
-
-    if dataset_present is None:
-        return True
-    else:
-        return False
\ No newline at end of file
diff --git a/modules/crawler/Utilities/Database/database_data_line.py b/modules/crawler/Utilities/Database/database_data_line.py
new file mode 100644
index 0000000..47e53d8
--- /dev/null
+++ b/modules/crawler/Utilities/Database/database_data_line.py
@@ -0,0 +1,14 @@
+class DatabaseDataLine:
+    """
+    Class that specifies the look of data line in database
+    """
+    def __init__(self, name, longitude, latitude, date, occurrence):
+        self.name = name
+        self.latitude = latitude
+        self.longitude = longitude
+        self.date = date
+        self.occurrence = occurrence
+
+    def to_dictionary(self):
+        return {"place": self.name, "x": self.longitude, "y": self.latitude, "number": self.occurrence,
+                "date": self.date}
diff --git a/modules/crawler/Utilities/Database/database_loader.py b/modules/crawler/Utilities/Database/database_loader.py
new file mode 100644
index 0000000..bce6c91
--- /dev/null
+++ b/modules/crawler/Utilities/Database/database_loader.py
@@ -0,0 +1,246 @@
+from Utilities.Database import database_data_line, database_record_logs
+from Utilities import configure_functions
+import pymongo
+import re
+
+# specify mongodb connection
+MONGODB_CONNECTION = "mongodb://root:root@database"
+# mongodb account name
+MONGODB_ACC_NAME = "root"
+# mongodb account password
+MONGODB_ACC_PASSWORD = "root"
+# mongodb data database
+MONGODB_DATA_DATABASE = "open-data-db"
+# mongodb collection with aviable datasets
+MONGODB_DATASET_COLLECTION = "DATASETS"
+# mongodb collection with aviable diveces of datasets
+MONGODB_DATASET_DEVICES_COLLECTION = "DEVICES"
+
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+
+
+def create_database_connection():
+    """
+    Creates connection to mongoDB
+    
+    Returns:
+        Connection to mongoDB
+    """
+    client = pymongo.MongoClient(MONGODB_CONNECTION)
+
+    # Authenticating
+    client.admin.authenticate(MONGODB_ACC_NAME, MONGODB_ACC_PASSWORD)
+
+    database = client[MONGODB_DATA_DATABASE]
+
+    return database
+
+
+def get_data_from_file(filename, config):
+    """
+        Opens processed file, reads it line by line
+        name, ocurrence, date
+        searches name in config and adds device map coordinates
+        than creates a dictionary with date without hours as key
+        and list of data lines as value.
+    Args:
+        filename: name of processed file
+        config: loaded configuration file of dataset
+
+    Returns:
+        dictionary with date without hours as key
+        and list of Datalines as value
+    """
+    dataset_name = config["dataset-name"]
+    dataset_path = PROCESSED_DATA_PATH + dataset_name + '/'
+
+    f = open(dataset_path + filename, "r")
+
+    devices = config["devices"]
+    date_dict = dict()
+
+    for line in f:
+        line = line[:-1]
+
+        csv_column = line.split(";")
+
+        name = csv_column[0]
+
+        if devices[name]["x"] == "SKIP" or devices[name]["y"] == "SKIP":
+            continue
+
+        occurrence = csv_column[1]
+        date = csv_column[2]
+        data_line = database_data_line.DatabaseDataLine(
+            name, devices[name]["x"], devices[name]["y"], date, occurrence)
+
+        # if you want to change table split by hours or months change this YYYY-mm-hh-dd
+        date_without_hours = date[:-3]
+        if date_without_hours not in date_dict:
+            date_dict[date_without_hours] = list()
+
+        date_dict[date_without_hours].append(
+            data_line.to_dictionary())
+
+    return date_dict
+
+
+def load_data_to_database(database_connection,dataset_name, data_dic, file_name):
+    """
+    Takes data_dic created in method get_data_from_file
+    and loads into into database where collection name is dataset_name + data_dic key
+    and data lines are line in collection
+
+    Args:
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
+        data_dic: dictionary of data lines created in get_data_from_file
+        file_name: name of file containing data
+    """
+
+    for date in data_dic:
+        dataset_collections = database_connection[dataset_name]
+        dataset_collections.insert_one({'date': date})
+        date_dataset = database_connection[dataset_name + date]
+        date_dataset.insert_many(data_dic[date])
+
+
+
+def check_or_update_datasets_collection(database_connection,config):
+    """
+    Checks if DATASETS collection contains dataset and if display name was not updated
+
+    Args:
+        database_connection: created connection to a MONGODB
+        config: loaded configuration file of dataset
+    """
+    # collection where are specified aviable datasets
+    collection_datasets = database_connection[MONGODB_DATASET_COLLECTION]
+
+    dataset_name = config['dataset-name']
+    display_name = config['display-name']
+
+    query = {'key-name': dataset_name}
+
+    # check if newly added data already have a dataset specified in collection
+    dataset_present = collection_datasets.find_one(query)
+
+    if dataset_present is None:
+        collection_datasets.insert_one({'key-name': dataset_name, 'display-name': display_name,'updated': 0})
+    elif dataset_present['display-name'] != display_name:
+        newvalues = { "$set": { 'display-name': display_name } }
+        collection_datasets.update_one(query, newvalues)
+
+
+def update_devices_collection(config):
+    """
+    Checks if there are any changes in devices specified in config file against 
+    devices processed and loaded into the database
+
+    If there are new devices replaces old device in databse by new ones
+
+    Args:
+        config: loaded configuration file of dataset
+
+    Returns:
+        True - when changes are found and devices replaced
+        False - when there were no changes
+    """
+    database_connection = create_database_connection()
+    dataset_name = config['dataset-name']
+    devices = config['devices']
+
+    change_in_devices = False
+
+    collection_devices = database_connection[dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]
+
+    devices_cursor = collection_devices.find()
+
+    db_device_dict = dict()
+
+    for device in devices_cursor:
+        name = device['name']
+        db_device_dict[name] = {'name': name, 'x': device['x'] , 'y': device['y']}
+
+
+    valid_devices = configure_functions.return_dictionary_of_valid_devices(devices)
+
+    if len(valid_devices.keys()) != len(db_device_dict.keys()):
+        change_in_devices = True
+    
+    if change_in_devices == False:
+        for device in valid_devices.keys():
+            if device in db_device_dict:
+                config_x = valid_devices[device]['x']
+                config_y = valid_devices[device]['y']
+                db_x = db_device_dict[device]['x']
+                db_y = db_device_dict[device]['y']
+                if config_x != db_x or config_y != db_y:
+                    change_in_devices = True
+                    break
+
+
+    if change_in_devices == True:
+        collection_devices.delete_many({})
+        devices_list = list()
+
+        for device in devices.keys():
+            x = devices[device]['x']
+            y = devices[device]['y']
+            if not (x == "SKIP" or x == "UNKNOWN!" or y == "SKIP" or y == "UNKNOWN!"):
+                devices_list.append({'name': device , 'x': x , 'y': y })
+
+        collection_devices.insert_many(devices_list)
+
+    return change_in_devices
+
+    
+def remove_dataset_database(dataset_name):
+    """
+    Removes dataset entries from database
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    # Creating connection
+    mydb = create_database_connection()
+
+    # collection where are specified aviable datasets
+    collection_datasets = mydb[MONGODB_DATASET_COLLECTION]
+
+    collection_datasets.delete_one({"key-name": dataset_name})
+    print("Removing record from DATASETS collection")
+
+
+    # Retrieve list of all collections
+    collections = mydb.list_collection_names()
+
+    # Drop of all collections
+    for name in collections:
+        if name.startswith(dataset_name):
+            mydb[name].drop()
+            print("Dropping: " + name)
+
+
+def reset_dataset_database(dataset_name):
+    """
+    Reset dataset in database 
+     - delete everything from except crawled links and mention in DATASETS collection
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    # Creating connection
+    mydb = create_database_connection()
+
+    pattern = re.compile(dataset_name+'[0-9]+-[0-9]+-+[0-9]+')
+
+    # Retrieve list of all collections
+    collections = mydb.list_collection_names()
+
+    # Drop of all collections
+    for name in collections:
+        if pattern.match(name):
+            mydb[name].drop()
+
+    database_record_logs.reset_ignore_set_processed(dataset_name)
+    database_record_logs.reset_ignore_set_loaded(dataset_name)
\ No newline at end of file
diff --git a/modules/crawler/Utilities/Database/database_record_logs.py b/modules/crawler/Utilities/Database/database_record_logs.py
new file mode 100644
index 0000000..37503f8
--- /dev/null
+++ b/modules/crawler/Utilities/Database/database_record_logs.py
@@ -0,0 +1,209 @@
+from Utilities.Database import database_loader
+
+# mongodb collection with with already downloaded links
+MONGODB_DATASET_LINK_COLLECTION = "LINKS"
+# mongodb collection with with already processed files
+MONGODB_DATASET_PROCESSED_COLLECTION = "PROCESSED"
+# mongodb collection with with already loaded links
+MONGODB_DATASET_LOADED_COLLECTION = "LOADED"
+# mongodb collection with aviable datasets with number of days from last update
+MONGODB_DATASET_COLLECTION = "DATASETS"
+
+
+def load_ignore_set_links(dataset_name):
+    """
+    Loades from database links of already downloaded files by crawler
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    ignore_set = set()
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_LINK_COLLECTION]
+
+    data = my_col.find()
+
+    for part in data:
+        ignore_set.add(part['name'])
+
+    return ignore_set
+
+
+def update_ignore_set_links(dataset_name,link):
+    """
+    Adds links of newly crawled files to the database
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_LINK_COLLECTION]
+
+    my_col.insert({ "name": link})
+
+
+def reset_ignore_set_links(dataset_name):
+    """
+    Drops collection of already downloaded links
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_LINK_COLLECTION]
+
+    my_col.drop()
+
+
+
+def load_ignore_set_processed(dataset_name):
+    """
+    Loads from database set of already processed files
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    ignore_set = set()
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_PROCESSED_COLLECTION]
+
+    data = my_col.find()
+
+    for part in data:
+        ignore_set.add(part['name'])
+
+    return ignore_set
+
+
+def update_ignore_set_processed(dataset_name,filename):
+    """
+    Adds files of newly processed files to the database
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_PROCESSED_COLLECTION]
+
+    my_col.insert({ "name": filename})
+
+
+
+def reset_ignore_set_processed(dataset_name):
+    """
+    Drops collection of already processed files
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_PROCESSED_COLLECTION]
+
+    my_col.drop()
+
+
+
+def load_ignore_set_loaded(dataset_name):
+    """
+    Loads from database set of already loaded files in database
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    ignore_set = set()
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_LOADED_COLLECTION]
+
+    data = my_col.find()
+
+    for part in data:
+        ignore_set.add(part['name'])
+
+    return ignore_set
+
+
+
+def update_ignore_set_loaded(dataset_name,filename):
+    """
+    Adds files of newly loaded files to the database
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_LOADED_COLLECTION]
+
+    my_col.insert({ "name": filename})
+
+
+def reset_ignore_set_loaded(dataset_name):
+    """
+    Drops collection of already loaded files
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[dataset_name + MONGODB_DATASET_LOADED_COLLECTION]
+
+    my_col.drop()
+
+
+def load_updated(dataset_name):
+    """
+    Loads value of (days from last update) from db
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    updated = 0
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[MONGODB_DATASET_COLLECTION]
+
+    data = my_col.find_one({'key-name': dataset_name},{'updated'})
+
+    updated = int(data['updated'])
+
+    return updated
+
+
+def update_updated(dataset_name,value):
+    """
+    Updates value of (days from last update) in db
+    
+    Returns:
+        dataset_name name of dataset that has existing configuration file
+    """
+
+    connection = database_loader.create_database_connection()
+
+    my_col = connection[MONGODB_DATASET_COLLECTION]
+
+    myquery = { 'key-name': dataset_name }
+    new_values = { "$set": { "updated": value } }
+
+    my_col.update_one(myquery,new_values)
diff --git a/modules/crawler/Utilities/DateFormating.py b/modules/crawler/Utilities/DateFormating.py
deleted file mode 100644
index 1ec11cf..0000000
--- a/modules/crawler/Utilities/DateFormating.py
+++ /dev/null
@@ -1,54 +0,0 @@
-def date_formatter(string_date):
-    """
-
-    Args:
-        string_date: string containing date in format 22.08.2018 12:27:00
-
-    Returns:
-        string of date in format YYYY-mm-dd-hh
-    """
-    if string_date[11].isspace():
-        pos = 0
-        srr = ""
-        for i in string_date:
-
-            if pos == 10:
-                srr = srr + '0'
-            else:
-                srr = srr + i
-
-            pos = pos + 1
-
-        string_date = srr
-
-    return_date = string_date[6:10] + '-' + string_date[3:5] + '-' + string_date[:2]
-
-    return return_date
-
-
-def date_time_formatter(string_date):
-    """
-    Converts one type of date format "dd.mm.yyyy hh.mm.ss" to date format YYYY-mm-dd-hh
-    Args:
-        string_date: string containing date in format 22.08.2018 12:27:00
-
-    Returns:
-        string of date in format YYYY-mm-dd-hh
-    """
-    if string_date[11].isspace():
-        pos = 0
-        srr = ""
-        for i in string_date:
-
-            if pos == 10:
-                srr = srr + '0'
-            else:
-                srr = srr + i
-
-            pos = pos + 1
-
-        string_date = srr
-
-    return_date = string_date[6:10] + '-' + string_date[3:5] + '-' + string_date[:2] + '-' + string_date[11:13]
-
-    return return_date
diff --git a/modules/crawler/Utilities/FolderProcessor.py b/modules/crawler/Utilities/FolderProcessor.py
deleted file mode 100644
index 5a23171..0000000
--- a/modules/crawler/Utilities/FolderProcessor.py
+++ /dev/null
@@ -1,124 +0,0 @@
-import os
-import zipfile
-from Utilities.CSV import CSVutils
-
-
-def list_of_all_files(path):
-    """
-    Get all files from directory and all files written in ignore.txt
-    and return the difference
-    Args:
-        path: path to Directory
-
-    Returns:
-        list with names of all files in directory
-    """
-    files_in_dir = os.listdir(path)
-
-    ignore_set = load_ignore_set(path)
-
-    return set(files_in_dir).difference(ignore_set)
-
-
-def load_ignore_set(path):
-    """
-    Reads ignore.txt line by line and add it to a set
-    Args:
-        path: Path to directory containing ignore.txt file
-
-    Returns:
-        list of names contained in ignore.txt file
-    """
-    ignore_set = set()
-
-    with open(path + "ignore.txt", "r") as file:
-
-        for line in file:
-            ignore_set.add(line[:-1])
-
-    return ignore_set
-
-
-def update_ignore_set(path, file_name):
-    """
-    Adds file_name to the ignore file
-    Args:
-        path: Path to directory containing ignore.txt file
-        file_name: name of file you want to add to ignore file
-    """
-    with open(path + "ignore.txt", "a") as file:
-        file.write(file_name + '\n')
-
-
-def get_devices_set(path):
-    """
-     Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
-     Extracts names from not loaded file which should be in first column
-     Creates set of unique devices_names
-
-    Args:
-        path: Path to Processed directory containing ignore.txt file
-
-    Returns:
-        set of unique names contained in not loaded files
-    """
-    files_in_dir = list_of_all_files(path)
-
-    unique_names = set()
-
-    for file_path in files_in_dir:
-        unique_names.update(CSVutils.get_unique_names_from_file(path+file_path, 0))
-
-    return unique_names
-
-
-def get_unknown_devices_set(config, devices):
-    """
-    Compares config and devices a return difference
-
-    Args:
-        config:  loaded configuration file of dataset
-        devices: set of unique devices contained in dataset
-
-    Returns:
-
-    """
-    devices_set = set(config["devices"].keys())
-    unknown_devices_set = devices.difference(devices_set)
-
-    return unknown_devices_set
-
-
-def unzip_all_csv_zip_files_in_folder(path):
-    """
-    Load all files from directory and unzip those which end by .zip
-    After unziping deletes the zip file
-    Args:
-        path: Path to CrawledData directory containing ignore.txt file
-    """
-    files_in_dir = os.listdir(path)
-    zips = []
-
-    for file in files_in_dir:
-        if file.endswith(".zip"):
-            zips.append(path + file)
-
-    for zip_file in zips:
-
-        with zipfile.ZipFile(zip_file, "r") as unziped_file:
-            unziped_file.extractall(path)
-
-        os.remove(zip_file)
-
-
-def clean_folder(path):
-    """
-    Deletes all files in folder
-
-    Args:
-        path: path to folder
-    """
-    files = os.listdir(path)
-
-    for file in files:
-        os.remove(path+file)
diff --git a/modules/crawler/Utilities/configure_functions.py b/modules/crawler/Utilities/configure_functions.py
new file mode 100644
index 0000000..7c93d5a
--- /dev/null
+++ b/modules/crawler/Utilities/configure_functions.py
@@ -0,0 +1,93 @@
+import yaml
+import os
+from Utilities.Database import database_record_logs
+
+# Path to dataset configuration files
+CONFIG_FILES_PATH = "DatasetConfigs/"
+# Config file type
+CONFIG_FILE_TYPE = ".yaml"
+
+
+def load_configuration(dataset_name):
+    """
+    Loads yaml configuration file into memory
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+
+    Returns:
+        yaml configuration file as dictionary
+    """
+    with open(CONFIG_FILES_PATH + dataset_name + CONFIG_FILE_TYPE, "r") as f:
+        data = yaml.load(f, Loader=yaml.FullLoader)
+
+    devices_dic = dict()
+
+    if data["devices"] is not None:
+        for item in data["devices"]:
+            devices_dic.update(item)
+
+    data["devices"] = devices_dic
+
+    return data
+
+
+def update_configuration(dataset_name, new_devices):
+    """
+    Open dataset and appends new_devices to the end
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+        new_devices: list or set of new devices for dataset
+    """
+
+    with open(CONFIG_FILES_PATH + dataset_name + CONFIG_FILE_TYPE, "a") as file:
+        for device in new_devices:
+            file.write("  - "+device+":\n")
+            file.write("      x: UNKNOWN!\n")
+            file.write("      y: UNKNOWN!\n")
+            file.write("\n")
+
+
+def check_if_there_is_a_config_file(dataset_name):
+    """
+    Goes trough all config files (represeting valid dataset in database)
+    and checks if dataset_name is there
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+
+    Returns:   
+        True - if contains
+        False - if not
+    """
+    datasets = os.listdir(CONFIG_FILES_PATH)
+
+    for dataset in datasets:
+        name = dataset.split('.')
+        if name[0] == dataset_name :
+            return True
+
+    return False
+
+def return_dictionary_of_valid_devices(devices):
+    """
+    Iterates over all devices specified in config file
+
+    Extracts only valid one (have both specified coordinates no UNKOWN! OR SKIP)
+
+    Args:
+        devices: dictionary of devices contained in config file
+
+    Returns:   
+        Dictonary containing only valid devices
+    """
+    valid_devices = dict()
+
+    for device in devices.keys():
+        x = devices[device]['x']
+        y = devices[device]['y']
+        if not (x == "SKIP" or x == "UNKNOWN!" or y == "SKIP" or y == "UNKNOWN!"):
+            valid_devices[device] = {'name': device, 'x': x , 'y': y}
+
+    return valid_devices
\ No newline at end of file
diff --git a/modules/crawler/Utilities/date_formating.py b/modules/crawler/Utilities/date_formating.py
new file mode 100644
index 0000000..1ec11cf
--- /dev/null
+++ b/modules/crawler/Utilities/date_formating.py
@@ -0,0 +1,54 @@
+def date_formatter(string_date):
+    """
+
+    Args:
+        string_date: string containing date in format 22.08.2018 12:27:00
+
+    Returns:
+        string of date in format YYYY-mm-dd-hh
+    """
+    if string_date[11].isspace():
+        pos = 0
+        srr = ""
+        for i in string_date:
+
+            if pos == 10:
+                srr = srr + '0'
+            else:
+                srr = srr + i
+
+            pos = pos + 1
+
+        string_date = srr
+
+    return_date = string_date[6:10] + '-' + string_date[3:5] + '-' + string_date[:2]
+
+    return return_date
+
+
+def date_time_formatter(string_date):
+    """
+    Converts one type of date format "dd.mm.yyyy hh.mm.ss" to date format YYYY-mm-dd-hh
+    Args:
+        string_date: string containing date in format 22.08.2018 12:27:00
+
+    Returns:
+        string of date in format YYYY-mm-dd-hh
+    """
+    if string_date[11].isspace():
+        pos = 0
+        srr = ""
+        for i in string_date:
+
+            if pos == 10:
+                srr = srr + '0'
+            else:
+                srr = srr + i
+
+            pos = pos + 1
+
+        string_date = srr
+
+    return_date = string_date[6:10] + '-' + string_date[3:5] + '-' + string_date[:2] + '-' + string_date[11:13]
+
+    return return_date
diff --git a/modules/crawler/Utilities/folder_processor.py b/modules/crawler/Utilities/folder_processor.py
new file mode 100644
index 0000000..93997ac
--- /dev/null
+++ b/modules/crawler/Utilities/folder_processor.py
@@ -0,0 +1,96 @@
+import os
+import zipfile
+from Utilities.CSV import csv_utils
+from Utilities.Database import database_record_logs
+
+
+def list_of_all_new_files(ignore_set,path):
+    """
+    Get all files from directory and all files written in ignore.txt
+    and return the difference
+    Args:
+        path: path to Directory
+        ignore_set: path to Directory
+    Returns:
+        list with names of all files in directory
+    """
+    files_in_dir = os.listdir(path)
+
+
+    return set(files_in_dir).difference(ignore_set)
+
+
+
+def get_devices_set(dataset_name,path):
+    """
+     Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
+     Extracts names from not loaded file which should be in first column
+     Creates set of unique devices_names
+
+    Args:
+        path: Path to Processed directory
+
+    Returns:
+        set of unique names contained in not loaded files
+    """
+    ignore_set = database_record_logs.load_ignore_set_loaded(dataset_name)
+    files_in_dir = list_of_all_new_files(ignore_set,path)
+
+    unique_names = set()
+
+    for file_path in files_in_dir:
+        unique_names.update(csv_utils.get_unique_names_from_file(path+file_path, 0))
+
+    return unique_names
+
+
+def get_unknown_devices_set(config, devices):
+    """
+    Compares config and devices a return difference
+
+    Args:
+        config:  loaded configuration file of dataset
+        devices: set of unique devices contained in dataset
+
+    Returns:
+        diffrences between two sets (unkown devices)
+    """
+    devices_set = set(config["devices"].keys())
+    unknown_devices_set = devices.difference(devices_set)
+
+    return unknown_devices_set
+
+
+def unzip_all_csv_zip_files_in_folder(path):
+    """
+    Load all files from directory and unzip those which end by .zip
+    After unziping deletes the zip file
+    Args:
+        path: Path to CrawledData directory containing ignore.txt file
+    """
+    files_in_dir = os.listdir(path)
+    zips = []
+
+    for file in files_in_dir:
+        if file.endswith(".zip"):
+            zips.append(path + file)
+
+    for zip_file in zips:
+
+        with zipfile.ZipFile(zip_file, "r") as unziped_file:
+            unziped_file.extractall(path)
+
+        os.remove(zip_file)
+
+
+def clean_folder(path):
+    """
+    Deletes all files in folder
+
+    Args:
+        path: path to folder
+    """
+    files = os.listdir(path)
+
+    for file in files:
+        os.remove(path+file)
diff --git a/modules/crawler/crone_update_script.py b/modules/crawler/crone_update_script.py
new file mode 100644
index 0000000..84a96bc
--- /dev/null
+++ b/modules/crawler/crone_update_script.py
@@ -0,0 +1,19 @@
+import pipeline
+import os
+
+# Path to configuration files
+CONFIG_FILES_PATH = "DatasetConfigs/"
+
+
+def run_pipeline_for_all_datasets():
+    """
+    Runs whole DataScript pipeline for every dataset that has existing configuration file
+    """
+    files_in_dir = os.listdir(CONFIG_FILES_PATH)
+
+    for file in files_in_dir:
+        name = file.split('.')[0]
+        pipeline.run_full_pipeline_crone(name)
+
+
+run_pipeline_for_all_datasets()
diff --git a/modules/crawler/crontab b/modules/crawler/crontab
index 46b74d3..ed903dc 100644
--- a/modules/crawler/crontab
+++ b/modules/crawler/crontab
@@ -1,2 +1,2 @@
-00 00 * * * cd /src && /usr/local/bin/python CroneUpdateScript.py
+00 00 * * * cd /src && /usr/local/bin/python crone_update_script.py
 # Mandatory blank line
\ No newline at end of file
diff --git a/modules/crawler/docker_prepare_structure.py b/modules/crawler/docker_prepare_structure.py
new file mode 100644
index 0000000..5119c46
--- /dev/null
+++ b/modules/crawler/docker_prepare_structure.py
@@ -0,0 +1,62 @@
+import os
+
+# Path to configuration files
+CONFIG_FILES_PATH = "DatasetConfigs/"
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
+
+
+def prepare_strucure_for_all_datasets():
+    """
+    Prepares folders that are necessery but does not contain code so they are excluded from gitlab by gitignore
+    """
+
+    if  not os.path.isdir(CRAWLED_DATA_PATH) :
+        try:
+            os.mkdir(CRAWLED_DATA_PATH)
+        except os.error as e:
+            print(e)
+            print("Creation of the directory %s failed" % CRAWLED_DATA_PATH)
+
+    if  not os.path.isdir(PROCESSED_DATA_PATH) :
+        try:
+            os.mkdir(PROCESSED_DATA_PATH)
+        except os.error as e:
+            print(e)
+            print("Creation of the directory %s failed" % PROCESSED_DATA_PATH)
+    
+    if  not os.path.isdir(CRAWLER_LOGS_PATH) :
+        try:
+            os.mkdir(CRAWLER_LOGS_PATH)
+        except os.error as e:
+            print(e)
+            print("Creation of the directory %s failed" % PROCESSED_DATA_PATH)
+
+
+    files_in_dir = os.listdir(CONFIG_FILES_PATH)
+
+    for file in files_in_dir:
+        name = file.split('.')
+        prepare_structure(name[0])
+
+
+def prepare_structure(dataset_name):
+    """
+    Create folder for every dataset in newly created folder for processed and crawled data
+    """
+
+    path =  CRAWLED_DATA_PATH + dataset_name
+    if  not os.path.isdir(path) :
+        os.mkdir(path)
+
+    path =  PROCESSED_DATA_PATH + dataset_name
+    if not  os.path.isdir(path):
+        os.mkdir(PROCESSED_DATA_PATH + dataset_name)
+
+
+print("Inicializuji počáteční strukturu pro stažená a zpracovaná data")
+prepare_strucure_for_all_datasets()
diff --git a/modules/crawler/force_update_datasets.py b/modules/crawler/force_update_datasets.py
new file mode 100644
index 0000000..abd6504
--- /dev/null
+++ b/modules/crawler/force_update_datasets.py
@@ -0,0 +1,37 @@
+import pipeline
+import os
+from Utilities import configure_functions
+
+# Path to configuration files
+CONFIG_FILES_PATH = "DatasetConfigs/"
+
+
+def run_pipeline_for_one_datasets(dataset_name):
+    print("Probíhá update datasetu " + dataset_name)
+    pipeline.run_full_pipeline(dataset_name)
+
+
+def run_pipeline_for_all_datasets():
+    """
+    Runs whole DataScript pipeline for every dataset that has existing configuration file
+    """
+    files_in_dir = os.listdir(CONFIG_FILES_PATH)
+
+    for file in files_in_dir:
+        name = file.split('.')[0]
+        print("Probíhá update datasetu " + name)
+        pipeline.run_full_pipeline(name)
+
+
+print("Zadejte jméno Datasetu který chcete updatovat (pokud všechny zadejte '-ALL'):\n")
+
+dataset_name = input().upper()
+
+if dataset_name == '-ALL':
+    run_pipeline_for_all_datasets()
+else:
+    test = configure_functions.check_if_there_is_a_config_file(dataset_name)
+    if test == True:
+        run_pipeline_for_one_datasets(dataset_name)
+    else:
+        print("Tento dataset v architektuře neexistuje")
diff --git a/modules/crawler/pipeline.py b/modules/crawler/pipeline.py
new file mode 100644
index 0000000..f76075e
--- /dev/null
+++ b/modules/crawler/pipeline.py
@@ -0,0 +1,298 @@
+from Utilities import folder_processor, configure_functions
+from Utilities.Database import database_loader, database_record_logs
+from Utilities.CSV import csv_utils
+import os
+import pymongo
+
+import logging
+from datetime import date
+
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
+# Path to dataset crawler implementations
+CRAWLER_LIB_PATH = "DatasetCrawler."
+# Path to dataset processor implementations
+PROCESSOR_LIB_PATH = "DatasetProcessing."
+
+
+#logger
+logging.basicConfig(filename=CRAWLER_LOGS_PATH + 'Applicationlog-' + date.today().strftime("%b-%Y") + '.log',
+                    level=logging.INFO,
+                    format='%(asctime)s %(message)s'
+                    )
+
+
+def check_last_update(config):
+    """
+    Loads integer from updated.txt in CrawlerLogs/"dataset_name"
+    representing number of days from last update if number equals
+    number in confing update period updates it and reset number of
+    days to zero else increment the number
+
+    Arguments:
+        config loaded configuration file of dataset
+
+    Returns:
+       True if updating
+       Else if incementing days from last update
+    """
+    dataset_name = config["dataset-name"]
+
+    last_update = database_record_logs.load_updated(dataset_name)
+
+    if config["update-period"] <= last_update:
+        logging.info("Dataset " + dataset_name + " is being updated today")
+        database_record_logs.update_updated(dataset_name,0)
+        return True
+    else:
+        last_update_days = last_update + 1
+        logging.info("Dataset " + dataset_name + " will be updated in " + str(int(config["update-period"]) - last_update_days) + "days")
+        database_record_logs.update_updated(dataset_name,last_update + 1)
+        return False
+
+
+
+def crawl_data(config):
+    """
+      Imports dataset crawler in DatasetCrawler/"dataset_name"_crawler.py
+      runs crawler.
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+
+    crawl_func = __import__(CRAWLER_LIB_PATH + dataset_name + "_crawler", globals(), locals(), ['crawl']).crawl
+    crawl_func(config)
+
+    dataset_name += '/'
+
+
+def process_data(config):
+    """
+    Goes trough every not processed file(list of processed files is saved in databse)
+    Imports dataset processor in DatasetProcessing/"dataset_name"_processor.py
+    Runs processor on every file
+    After successful processing updates database list of processed files
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    dataset_name = config["dataset-name"]
+    dataset_path = dataset_name + '/'
+
+    changes_in_devices = database_loader.update_devices_collection(config)
+
+    if changes_in_devices == True:
+        logging.info(dataset_name + " contains changes in devices configuration. Deleteing old data and preparing new")
+        database_loader.reset_dataset_database(dataset_name)
+        folder_processor.clean_folder(PROCESSED_DATA_PATH + dataset_path)
+
+
+    process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "_processor", globals(), locals(),
+                                   ['process_file']).process_file
+
+    ignore_set = database_record_logs.load_ignore_set_processed(dataset_name)
+    not_processed_files = folder_processor.list_of_all_new_files(ignore_set,CRAWLED_DATA_PATH + dataset_path)
+    logging.info(dataset_name + " found " + str(len(not_processed_files)) + " not processed files")
+
+    for not_processed_file in not_processed_files:
+        path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
+        date_dic = process_file_func(path)
+        csv_utils.export_data_to_csv(path, date_dic)
+        database_record_logs.update_ignore_set_processed(dataset_name, not_processed_file)
+
+    logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")
+
+
+def process_data_crone(config):
+    """
+    Goes trough every not processed file(list of processed files is saved in database)
+    Imports dataset processor in DatasetProcessing/"dataset_name"_processor.py
+    Runs processor on every file
+    After successful processing updates database list of processed files
+
+    Lightweight version for crone and production only
+    - excludes checks for changes of config file and coordinates and names
+    after these changes force_update_datasets.py should be called
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+
+    dataset_name = config["dataset-name"]
+    dataset_path = dataset_name + '/'
+
+    process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "_processor", globals(), locals(),
+                                   ['process_file']).process_file
+
+    ignore_set = database_record_logs.load_ignore_set_processed(dataset_name)
+    not_processed_files = folder_processor.list_of_all_new_files(ignore_set,CRAWLED_DATA_PATH + dataset_path)
+    logging.info(dataset_name + " found " + str(len(not_processed_files)) + " not processed files")
+
+    for not_processed_file in not_processed_files:
+        path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
+        date_dic = process_file_func(path)
+        csv_utils.export_data_to_csv(path, date_dic)
+        database_record_logs.update_ignore_set_processed(dataset_name, not_processed_file)
+
+    logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")
+
+
+def validate_process_data(config):
+    """
+    Function goes through newly processed data and checks theirs status
+
+    Args:
+        config: loaded configuration file of dataset
+
+    Returns:
+        boolean variable TRUE/FALSE.
+        Data processed correctly - TRUE
+        Wrong format or NEW unknown devices - FALSE
+    """
+    dataset_name = config["dataset-name"]
+
+    processed_devices_set = folder_processor.get_devices_set(dataset_name,PROCESSED_DATA_PATH +dataset_name + '/')
+    unknown_devices_set = folder_processor.get_unknown_devices_set(config, processed_devices_set)
+    unknown_devices_size = len(unknown_devices_set)
+
+    if unknown_devices_size != 0:
+        logging.info("There is " + str(unknown_devices_size) + " unknown devices")
+        logging.info("Adding devices to " + dataset_name + " config file")
+        configure_functions.update_configuration(dataset_name, unknown_devices_set)
+        return False
+
+    for device in config["devices"]:
+        device = config["devices"][device]
+        if device["x"] == "UNKNOWN!" or device["y"] == "UNKNOWN!":
+            logging.info(dataset_name + " config file contains devices with UNKOWN! values please update them!!")
+            return False
+
+    return True
+
+
+def load_data_to_database(config):
+    """
+    Goes trough every not loaded file(list of loaded files is saved in database)
+    loads data appends coordination from configurations
+    and exports it into the database
+    After successful processing updates database list of loaded files
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    dataset_path = dataset_name + '/'
+
+    database_connection = database_loader.create_database_connection()
+
+    database_loader.check_or_update_datasets_collection(database_connection,config)
+
+    # get all unprocessed files from dataset
+    ignore_set = database_record_logs.load_ignore_set_loaded(dataset_name)
+    not_loaded_files = folder_processor.list_of_all_new_files(ignore_set,PROCESSED_DATA_PATH + dataset_path)
+
+    # load every file
+    for not_loaded_file in not_loaded_files:
+        # load processed data
+        processed_data = database_loader.get_data_from_file(not_loaded_file, config)
+        # load processed data to database
+        database_loader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
+        database_record_logs.update_ignore_set_loaded(dataset_name, not_loaded_file)
+
+    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
+
+    client = pymongo.MongoClient()
+    client.close()
+
+
+def load_data_to_database_crone(config):
+    """
+    Goes trough every not loaded file(list of loaded files is saved in database)
+    loads data appends coordination from configurations
+    and exports it into the database
+    After successful processing updates database list of loaded files
+    
+    Lightweight version for crone and production only
+    - excludes checks for changes of config file and coordinates and names
+    after these changes force_update_datasets.py should be called
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    dataset_path = dataset_name + '/'
+
+    database_connection = database_loader.create_database_connection()
+
+    # get all unprocessed files from dataset
+    ignore_set = database_record_logs.load_ignore_set_loaded(dataset_name)
+    not_loaded_files = folder_processor.list_of_all_new_files(ignore_set,PROCESSED_DATA_PATH + dataset_path)
+
+    # load every file
+    for not_loaded_file in not_loaded_files:
+        # load processed data
+        processed_data = database_loader.get_data_from_file(not_loaded_file, config)
+        # load processed data to database
+        database_loader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
+        database_record_logs.update_ignore_set_loaded(dataset_name, not_loaded_file)
+
+    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
+
+    client = pymongo.MongoClient()
+    client.close()
+
+
+
+def run_full_pipeline(dataset_name):
+    """
+    Loads config file and starts full pipeline
+    -crawl data
+    -process data
+    -load data to database
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    logging.info("Starting pipeline for dataset " + dataset_name)
+    print("Processing dataset " + dataset_name + " you can watch the progress in log contained in CrawlerLogs folder")
+    
+    config = configure_functions.load_configuration(dataset_name)
+    crawl_data(config)
+    process_data(config)
+
+    validation_test = validate_process_data(config)
+
+    if validation_test:
+        load_data_to_database(config)
+
+
+def run_full_pipeline_crone(dataset_name):
+    """
+    Loads config file and starts full pipeline
+    -crawl data
+    -process data
+    -load data to database
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    logging.info("Starting pipeline for dataset " + dataset_name)
+
+    config = configure_functions.load_configuration(dataset_name)
+    update_test = check_last_update(config)
+    if update_test:
+        crawl_data(config)
+        process_data_crone(config["dataset-name"])
+
+        validation_test = validate_process_data(config)
+
+        if validation_test:
+            load_data_to_database_crone(config)
+            
\ No newline at end of file
diff --git a/modules/crawler/prepare_new_dataset.py b/modules/crawler/prepare_new_dataset.py
new file mode 100644
index 0000000..c1ef18a
--- /dev/null
+++ b/modules/crawler/prepare_new_dataset.py
@@ -0,0 +1,127 @@
+import os
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+# Path for DatasetCrawlers implementations
+CRAWLER_PROGRAM_PATH = "DatasetCrawler"
+# Path for DatasetProcessors implementations
+PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
+# Path to dataset configuration files
+CONFIG_FILES_PATH = "DatasetConfigs"
+
+
+def create_default_config_file(dataset_name):
+    """
+    Creates default config file
+
+    Args:
+        dataset_name: Name of newly created dataset
+    """
+    with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
+        file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
+        file.write("dataset-name: " + dataset_name + "\n")
+        file.write("# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře\n")
+        file.write("dataset-name: " + dataset_name + "\n")
+        file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
+        file.write("url: ZDE VLOZTE URL\n")
+        file.write("# volitelny parameter, ktery specifikuje vzor jmrna datasetu, ktera se budou stahovat\n")
+        file.write("regex: ZDE VLOZTE REGEX\n")
+        file.write("# volitelny parametr, ktery udava jak casto se budou hledat nove datasety, pokud prazdne, "
+                   "tak defaultni hodnota (dny)\n")
+        file.write("update-period: ZDE VLOZTE HODNOTU\n")
+        file.write("# pozice jednotlivych zarizeni, ktera jsou v datasetu\n")
+        file.write("devices:\n")
+
+
+def create_default_processor(dataset_name):
+    """
+    Creates default processor for dataset
+
+    Args:
+        dataset_name: Name of newly created dataset
+    """
+    with open(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "_processor.py", "w") as file:
+        file.write("from Utilities.CSV import csv_data_line")
+        file.write("\n")
+        file.write("\n")
+        file.write("def process_file(filename):\n")
+        file.write("    \"\"\"\n")
+        file.write("    Method that take path to crawled file and outputs date dictionary:\n")
+        file.write("    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)\n")
+        file.write("    and value is dictionary where keys are devices (specified in configuration file)\n")
+        file.write("    and value is CSVDataLine.csv_data_line with device,date and occurrence\n")
+        file.write("\n")
+        file.write("    Args:\n")
+        file.write("    filename: name of processed file\n")
+        file.write("\n")
+        file.write("    Returns:\n")
+        file.write("    None if not implemented\n")
+        file.write("    date_dict when implemented\n")
+        file.write("    \"\"\"\n")
+        file.write("    date_dict = dict()\n")
+        file.write("\n")
+        file.write("    #with open(filename, \"r\") as file:\n")
+        file.write("    print(\"You must implements process_file method first!\")\n")
+        file.write("    return None\n")
+
+
+def create_default_crawler(dataset_name):
+    """
+    Creates default crawler for dataset
+
+    Args:
+        dataset_name: Name of newly created dataset
+    """
+
+    with open(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "_crawler.py", "w") as file:
+        file.write("# Path to crawled data\n")
+        file.write("CRAWLED_DATA_PATH = \"CrawledData/\"\n")
+        file.write("\n")
+        file.write("\n")
+        file.write("def crawl(config):\n")
+        file.write("    \"\"\"\n")
+        file.write("    Implement crawl method that downloads new data to path_for_files\n")
+        file.write("    For keeping the project structure\n")
+        file.write("    url , regex, and dataset_name from config\n")
+        file.write("    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py\n")
+        file.write("\n")
+        file.write("    Args:\n")
+        file.write("        config: loaded configuration file of dataset\n")
+        file.write("    \"\"\"\n")
+        file.write("    dataset_name = config[\"dataset-name\"]\n")
+        file.write("    url = config['url']\n")
+        file.write("    regex = config['regex']\n")
+        file.write("    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'\n")
+        file.write("    print(\"You must implements Crawl method first!\")\n")
+
+
+def prepare_dataset_structure(dataset_name):
+    """
+    Prepares folders for new dataset
+    Args:
+        dataset_name: Name of newly created dataset
+    """
+
+    # create folder for crawled data
+    path = CRAWLED_DATA_PATH+dataset_name
+    try:
+        os.mkdir(path)
+    except os.error as e:
+        print(e)
+        print("Creation of the directory %s failed" % path)
+
+    # create folder for processed data
+    path = PROCESSED_DATA_PATH + dataset_name
+    try:
+        os.mkdir(path)
+    except OSError:
+        print("Creation of the directory %s failed" % path)
+
+    create_default_crawler(dataset_name)
+    create_default_processor(dataset_name)
+    create_default_config_file(dataset_name)
+
+print("Zadejte jméno nového datasetu:\n")
+prepare_dataset_structure(input().upper())
diff --git a/modules/crawler/remove_dataset.py b/modules/crawler/remove_dataset.py
new file mode 100644
index 0000000..6268b31
--- /dev/null
+++ b/modules/crawler/remove_dataset.py
@@ -0,0 +1,49 @@
+import os
+import shutil
+from Utilities.Database import database_loader
+from Utilities import configure_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
+# Path to dataset configuration files
+CONFIG_FILES_PATH = "DatasetConfigs"
+# Path for DatasetCrawlers implementations
+CRAWLER_PROGRAM_PATH = "DatasetCrawler"
+# Path for DatasetProcessors implementations
+PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
+
+
+def remove_dataset(dataset_name):
+    """
+    Remove dataset
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    shutil.rmtree(CRAWLED_DATA_PATH + dataset_name + "/")
+    shutil.rmtree(PROCESSED_DATA_PATH + dataset_name + "/")
+
+    os.remove(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "_crawler.py")
+    os.remove(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "_processor.py")
+    os.remove(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml")
+
+    print("Dataset " + dataset_name + " odebrán z architektury")
+
+    database_loader.remove_dataset_database(dataset_name)
+
+    print("Dataset " + dataset_name + " odebrán z databáze")
+
+print("Zadejte jméno Datasetu který chcete odstranit:\n")
+
+dataset_name = input().upper()
+test = configure_functions.check_if_there_is_a_config_file(dataset_name)
+
+if test == True:
+    remove_dataset(dataset_name)
+else:
+    print("Tento dataset v architektuře neexistuje")
+
+
diff --git a/modules/crawler/reset_datasets.py b/modules/crawler/reset_datasets.py
new file mode 100644
index 0000000..f25e794
--- /dev/null
+++ b/modules/crawler/reset_datasets.py
@@ -0,0 +1,54 @@
+import os
+from Utilities import folder_processor
+from Utilities.Database import database_loader
+from Utilities import configure_functions
+ 
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+# Path to processed data
+PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
+# Path to dataset configuration files
+CONFIG_FILES_PATH = "DatasetConfigs"
+
+
+def reset_dataset(dataset_name):
+    """
+    Resets all saved data in dataset except config and implementation
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+
+    path = CRAWLED_DATA_PATH + dataset_name + "/"
+    folder_processor.clean_folder(path)
+
+    path = PROCESSED_DATA_PATH + dataset_name + "/"
+    folder_processor.clean_folder(path)
+
+    database_loader.remove_dataset_database(dataset_name)
+
+
+def reset_all_datasets():
+    """
+    Resets all saved data in all datasets with config file except configs and implementation
+    """
+    datasets = os.listdir(CONFIG_FILES_PATH)
+
+    for dataset in datasets:
+        reset_dataset(dataset.split('.')[0])
+
+
+
+print("Zadejte jméno Datasetu který chcete resetovat (pokud všechny zadejte '-ALL'):\n")
+
+dataset_name = input().upper()
+
+if dataset_name == '-ALL':
+    reset_all_datasets()
+else:
+    test = configure_functions.check_if_there_is_a_config_file(dataset_name)
+    if test == True:
+        reset_dataset(dataset_name)
+    else:
+        print("Tento dataset v architektuře neexistuje")
diff --git a/scripts/build.sh b/scripts/build.sh
index 5945201..3683154 100755
--- a/scripts/build.sh
+++ b/scripts/build.sh
@@ -1,4 +1,6 @@
 #!/bin/bash
 docker-compose down
 docker-compose -f docker-compose.yml -f docker-compose-prod.yml up --build -d
-docker-compose exec -T php-fpm composer install --no-interaction --working-dir=/var/www/symfony --no-dev
\ No newline at end of file
+docker-compose exec -T php-fpm composer install --no-interaction --working-dir=/var/www/symfony --no-dev
+#python module initiation
+docker-compose exec -T crawler /bin/sh -c "cd /src && python docker_prepare_structure.py && echo '-all' | python force_update_datasets.py"
\ No newline at end of file
diff --git a/scripts/dev/build.bat b/scripts/dev/build.bat
index 91c952b..f3e253c 100644
--- a/scripts/dev/build.bat
+++ b/scripts/dev/build.bat
@@ -2,4 +2,5 @@ docker-compose -f docker-compose.yml -f docker-compose-dev.yml up --build -d
 docker-compose exec php-fpm composer config extra.symfony.allow-contrib true --no-interaction --working-dir=/var/www/symfony
 docker-compose exec php-fpm composer install --no-interaction --working-dir=/var/www/symfony
 docker-compose exec php-fpm composer update --no-interaction --working-dir=/var/www/symfony
-docker-compose exec php-fpm /bin/sh -c 'cd /var/www/symfony && npm install' 
\ No newline at end of file
+docker-compose exec php-fpm /bin/sh -c 'cd /var/www/symfony && npm install'
+docker-compose exec crawler /bin/sh -c "cd /src && python docker_prepare_structure.py && echo '-all' | python force_update_datasets.py"
\ No newline at end of file
diff --git a/scripts/dev/build.sh b/scripts/dev/build.sh
index bd7ac00..3562bbb 100755
--- a/scripts/dev/build.sh
+++ b/scripts/dev/build.sh
@@ -23,4 +23,5 @@ cat << "EOF"
     \|__| \|__| \|__|      \|__|     \|__|        \|__| \|__| \|__||\_________\    \|__|   \|__|\|__| \|_______| \|_______|
                                                                    \|_________|
 EOF
-docker-compose exec php-fpm /bin/sh -c 'cd /var/www/symfony && npm install'
\ No newline at end of file
+docker-compose exec php-fpm /bin/sh -c 'cd /var/www/symfony && npm install'
+docker-compose exec -T crawler /bin/sh -c "cd /src && python docker_prepare_structure.py && echo '-all' | python force_update_datasets.py"
\ No newline at end of file
