Revize d6ca840d

Re #8116
predelano ukladani
prejmenovany soubory podle python konvenci

.gitignore

@@ -1 +1 @@
 logs
-database
 website/vendor/
 /website/var
 cache

docker-compose-prod.yml

@@ -9 +9 @@
               volumes: 
                      - /etc/certificate:/certificate
                      - ./docker/nginx/sites-dev:/etc/nginx/sites-available
-                     - /acme-challenge:/var/www/symfony/public/.well-known/acme-challenge/
+                     - /acme-challenge:/var/www/symfony/public/.well-known/acme-challenge/
+       crawler:
+              volumes: 
+                     - /logs/crawler:/src/CrawlerLogs
+                     - /data/crawler:/src/CrawledData

docker-compose.yml

@@ -11 +11 @@
                         - backend
                 volumes:
                         - ./modules/crawler/:/src
-                        - ./logs/crawler:/log/
+                        - ./logs/crawler:/src/CrawlerLogs
                 container_name: "heatmap_crawler"
                 environment:
                         - TZ=Europe/Prague

modules/crawler/.gitignore

@@ -1 +1 @@
 *__pycache__*
-*.CSV
+*.CSV
+CrawlerLogs
+CrawledData
+ProcessedData

modules/crawler/CrawledData/JIS/ignore.txt

@@ -1 @@
-ignore.txt

modules/crawler/CrawledData/KOLOBEZKY/ignore.txt

@@ -1 @@
-ignore.txt

modules/crawler/CrawledData/WIFI/ignore.txt

@@ -1 @@
-ignore.txt

modules/crawler/CrawlerLogs/CommonRecords/.gitignore

@@ -1 @@
-# Ignore everything in this directory
-*
-# Except this file
-!.gitignore

modules/crawler/CrawlerLogs/JIS/updated.txt

@@ -1 @@
-0

modules/crawler/CrawlerLogs/KOLOBEZKY/updated.txt

@@ -1 @@
-0

modules/crawler/CrawlerLogs/WIFI/updated.txt

@@ -1 @@
-0

modules/crawler/CroneUpdateScript.py

@@ -1 @@
-import Pipeline
-import os
-
-# Path to configuration files
-CONFIG_FILES_PATH = "DatasetConfigs/"
-
-
-def run_pipeline_for_all_datasets():
-    """
-    Runs whole DataScript pipeline for every dataset that has existing configuration file
-    """
-    files_in_dir = os.listdir(CONFIG_FILES_PATH)
-
-    for file in files_in_dir:
-        name = file.split('.')
-        Pipeline.run_full_pipeline_crone(name[0])
-
-
-run_pipeline_for_all_datasets()

modules/crawler/DatasetConfigs/KOLOBEZKY.yaml

@@ -33 +33 @@
   - stojan-borska:
       x: 49.734518
       y: 13.359475
-      
+

modules/crawler/DatasetCrawler/JISCrawler.py

@@ -1 @@
-from Utilities import FolderProcessor
-from Utilities.Crawler import BasicCrawlerFunctions
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-
-
-def crawl(config):
-    """
-    Implement crawl method that downloads new data to path_for_files
-    For keeping the project structure
-    url , regex, and dataset_name from config
-    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    url = config['url']
-    regex = config['regex']
-    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
-
-    first_level_links = BasicCrawlerFunctions.get_all_links(url)
-    filtered_first_level_links = BasicCrawlerFunctions.filter_links(first_level_links, "^OD_ZCU")
-    absolute_first_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_first_level_links, url)
-
-    files = []
-
-    for link in absolute_first_level_links:
-        second_level_links = BasicCrawlerFunctions.get_all_links(link)
-        filtered_second_level_links = BasicCrawlerFunctions.filter_links(second_level_links, regex)
-        absolute_second_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_second_level_links, link)
-        final_links = BasicCrawlerFunctions.remove_downloaded_links(absolute_second_level_links, dataset_name)
-
-        for file_link in final_links:
-            files.append(file_link)
-
-    for file in files:
-        BasicCrawlerFunctions.download_file_from_url(file, dataset_name)
-
-    FolderProcessor.unzip_all_csv_zip_files_in_folder(path_for_files)

modules/crawler/DatasetCrawler/JIS_crawler.py

@@ +1 @@
+from Utilities import folder_processor
+from Utilities.Crawler import basic_crawler_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def crawl(config):
+    """
+    Implement crawl method that downloads new data to path_for_files
+    For keeping the project structure
+    url , regex, and dataset_name from config
+    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    first_level_links = basic_crawler_functions.get_all_links(url)
+    filtered_first_level_links = basic_crawler_functions.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = basic_crawler_functions.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = basic_crawler_functions.get_all_links(link)
+        filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
+        absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
+        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        basic_crawler_functions.download_file_from_url(file, dataset_name)
+
+    folder_processor.unzip_all_csv_zip_files_in_folder(path_for_files)

modules/crawler/DatasetCrawler/KOLOBEZKYCrawler.py

@@ -1 @@
-from Utilities import FolderProcessor
-from Utilities.Crawler import BasicCrawlerFunctions
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-
-
-def crawl(config):
-    """
-    Implement crawl method that downloads new data to path_for_files
-    For keeping the project structure
-    url , regex, and dataset_name from config
-    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    url = config['url']
-    regex = config['regex']
-    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
-
-    first_level_links = BasicCrawlerFunctions.get_all_links(url)
-    filtered_first_level_links = BasicCrawlerFunctions.filter_links(first_level_links, "^OD_ZCU")
-    absolute_first_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_first_level_links, url)
-
-    files = []
-
-    for link in absolute_first_level_links:
-        second_level_links = BasicCrawlerFunctions.get_all_links(link)
-        filtered_second_level_links = BasicCrawlerFunctions.filter_links(second_level_links, regex)
-        absolute_second_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_second_level_links, link)
-        final_links = BasicCrawlerFunctions.remove_downloaded_links(absolute_second_level_links, dataset_name)
-
-        for file_link in final_links:
-            files.append(file_link)
-
-    for file in files:
-        BasicCrawlerFunctions.download_file_from_url(file, dataset_name)
-
-    FolderProcessor.unzip_all_csv_zip_files_in_folder(path_for_files)

modules/crawler/DatasetCrawler/KOLOBEZKY_crawler.py

@@ +1 @@
+from Utilities import folder_processor
+from Utilities.Crawler import basic_crawler_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def crawl(config):
+    """
+    Implement crawl method that downloads new data to path_for_files
+    For keeping the project structure
+    url , regex, and dataset_name from config
+    You can use already implemented functions from Utilities/Crawler/basic_crawler_functions.py
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    first_level_links = basic_crawler_functions.get_all_links(url)
+    filtered_first_level_links = basic_crawler_functions.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = basic_crawler_functions.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = basic_crawler_functions.get_all_links(link)
+        filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
+        absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
+        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        basic_crawler_functions.download_file_from_url(file, dataset_name)
+
+    folder_processor.unzip_all_csv_zip_files_in_folder(path_for_files)

modules/crawler/DatasetCrawler/WIFICrawler.py

@@ -1 @@
-from Utilities import FolderProcessor
-from Utilities.Crawler import BasicCrawlerFunctions
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-
-
-def crawl(config):
-    """
-    Implement crawl method that downloads new data to path_for_files
-    For keeping the project structure
-    url , regex, and dataset_name from config
-    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    url = config['url']
-    regex = config['regex']
-    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
-
-    first_level_links = BasicCrawlerFunctions.get_all_links(url)
-    filtered_first_level_links = BasicCrawlerFunctions.filter_links(first_level_links, "^OD_ZCU")
-    absolute_first_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_first_level_links, url)
-
-    files = []
-
-    for link in absolute_first_level_links:
-        second_level_links = BasicCrawlerFunctions.get_all_links(link)
-        filtered_second_level_links = BasicCrawlerFunctions.filter_links(second_level_links, regex)
-        absolute_second_level_links = BasicCrawlerFunctions.create_absolute_links(filtered_second_level_links, link)
-        final_links = BasicCrawlerFunctions.remove_downloaded_links(absolute_second_level_links, dataset_name)
-
-        for file_link in final_links:
-            files.append(file_link)
-
-    for file in files:
-        BasicCrawlerFunctions.download_file_from_url(file, dataset_name)
-
-    FolderProcessor.unzip_all_csv_zip_files_in_folder(path_for_files)

modules/crawler/DatasetCrawler/WIFI_crawler.py

@@ +1 @@
+from Utilities import folder_processor
+from Utilities.Crawler import basic_crawler_functions
+
+# Path to crawled data
+CRAWLED_DATA_PATH = "CrawledData/"
+
+
+def crawl(config):
+    """
+    Implement crawl method that downloads new data to path_for_files
+    For keeping the project structure
+    url , regex, and dataset_name from config
+    You can use already implemented functions from Utilities/Crawler/basic_crawler_functions.py
+
+    Args:
+        config: loaded configuration file of dataset
+    """
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'
+
+    first_level_links = basic_crawler_functions.get_all_links(url)
+    filtered_first_level_links = basic_crawler_functions.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = basic_crawler_functions.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = basic_crawler_functions.get_all_links(link)
+        filtered_second_level_links = basic_crawler_functions.filter_links(second_level_links, regex)
+        absolute_second_level_links = basic_crawler_functions.create_absolute_links(filtered_second_level_links, link)
+        final_links = basic_crawler_functions.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        basic_crawler_functions.download_file_from_url(file, dataset_name)
+
+    folder_processor.unzip_all_csv_zip_files_in_folder(path_for_files)

modules/crawler/DatasetProcessing/JISProcessor.py

@@ -1 @@
-from Utilities.CSV import CSVDataLine
-from Utilities import DateFormating
-
-
-def process_file(filename):
-    """
-    Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
-    and value is dictionary where keys devices (specified in configuration file)
-    and value is CSVDataLine.CSVDataLine with device,date and occurrence
-
-    Args:
-    filename: name of processed file
-
-    Returns:
-    None if not implemented
-    date_dict when implemented
-    """
-    date_dict = dict()
-
-    with open(filename, "r", encoding="utf-8") as file:
-
-        for line in file:
-
-            array = line.split(";")
-
-            date = DateFormating.date_time_formatter(array[1][1:-1])
-            name = array[0][1:-1]
-            occurrence = array[2][:-1]
-
-            if date not in date_dict:
-                date_dict[date] = dict()
-
-            if name in date_dict[date]:
-                date_dict[date][name].occurrence += int(occurrence)
-            else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, occurrence)
-
-    return date_dict
-

modules/crawler/DatasetProcessing/JIS_processor.py

@@ +1 @@
+from Utilities.CSV import csv_data_line
+from Utilities import date_formating
+
+
+def process_file(filename):
+    """
+    Method that take path to crawled file and outputs date dictionary:
+    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    and value is dictionary where keys are devices (specified in configuration file)
+    and value is CSVDataLine.csv_data_line with device,date and occurrence
+
+    Args:
+    filename: name of processed file
+
+    Returns:
+    None if not implemented
+    date_dict when implemented
+    """
+    date_dict = dict()
+
+    with open(filename, "r", encoding="utf-8") as file:
+
+        for line in file:
+
+            array = line.split(";")
+
+            date = date_formating.date_time_formatter(array[1][1:-1])
+            name = array[0][1:-1]
+            occurrence = array[2][:-1]
+
+            if date not in date_dict:
+                date_dict[date] = dict()
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurrence += int(occurrence)
+            else:
+                date_dict[date][name] = csv_data_line.CSVDataLine(name, date, occurrence)
+
+    return date_dict
+

modules/crawler/DatasetProcessing/KOLOBEZKYProcessor.py

@@ -1 @@
-from Utilities.CSV import CSVDataLine
-from Utilities import DateFormating
-
-
-def process_file(filename):
-    """
-    Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
-    and value is dictionary where keys devices (specified in configuration file)
-    and value is CSVDataLine.CSVDataLine with device,date and occurrence
-
-    Args:
-    filename: name of processed file
-
-    Returns:
-    None if not implemented
-    date_dict when implemented
-    """
-    date_dict = dict()
-
-    with open(filename, "r") as file:
-
-        for line in file:
-
-            array = line.split(";")
-
-            date = DateFormating.date_time_formatter(array[0][1:-1])
-            name = array[1][1:-1]
-
-            if date not in date_dict:
-                date_dict[date] = dict()
-
-            if name in date_dict[date]:
-                date_dict[date][name].occurrence += 1
-            else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, 1)
-
-    return date_dict
-

modules/crawler/DatasetProcessing/KOLOBEZKY_processor.py

@@ +1 @@
+from Utilities.CSV import csv_data_line
+from Utilities import date_formating
+
+
+def process_file(filename):
+    """
+    Method that take path to crawled file and outputs date dictionary:
+    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    and value is dictionary where keys are devices (specified in configuration file)
+    and value is CSVDataLine.csv_data_line with device,date and occurrence
+
+    Args:
+    filename: name of processed file
+
+    Returns:
+    None if not implemented
+    date_dict when implemented
+    """
+    date_dict = dict()
+
+    with open(filename, "r") as file:
+
+        for line in file:
+
+            array = line.split(";")
+
+            date = date_formating.date_time_formatter(array[0][1:-1])
+            name = array[1][1:-1]
+
+            if date not in date_dict:
+                date_dict[date] = dict()
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurrence += 1
+            else:
+                date_dict[date][name] = csv_data_line.CSVDataLine(name, date, 1)
+
+    return date_dict
+

modules/crawler/DatasetProcessing/WIFIProcessor.py

@@ -1 @@
-from Utilities.CSV import CSVDataLine
-from Utilities import DateFormating
-
-
-def process_file(filename):
-    """
-    Method that take path to crawled file and outputs date dictionary:
-    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
-    and value is dictionary where keys devices (specified in configuration file)
-    and value is CSVDataLine.CSVDataLine with device,date and occurrence
-
-    Args:
-    filename: name of processed file
-
-    Returns:
-    None if not implemented
-    date_dict when implemented
-    """
-    date_dict = dict()
-
-    with open(filename, "r", encoding="utf-8") as file:
-
-        for line in file:
-
-            array = line.split(";")
-
-            date = DateFormating.date_time_formatter(array[4][1:-2])
-            name = array[1][1:-1]
-            occurrence = array[0]
-
-            if date not in date_dict:
-                date_dict[date] = dict()
-
-            if name in date_dict[date]:
-                date_dict[date][name].occurrence += max(date_dict[date][name].occurrence,int(occurrence))
-            else:
-                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, int(occurrence))
-
-    return date_dict
-

modules/crawler/DatasetProcessing/WIFI_processor.py

@@ +1 @@
+from Utilities.CSV import csv_data_line
+from Utilities import date_formating
+
+
+def process_file(filename):
+    """
+    Method that take path to crawled file and outputs date dictionary:
+    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)
+    and value is dictionary where keys are devices (specified in configuration file)
+    and value is CSVDataLine.csv_data_line with device,date and occurrence
+
+    Args:
+    filename: name of processed file
+
+    Returns:
+    None if not implemented
+    date_dict when implemented
+    """
+    date_dict = dict()
+
+    with open(filename, "r", encoding="utf-8") as file:
+
+        for line in file:
+
+            array = line.split(";")
+
+            date = date_formating.date_time_formatter(array[4][1:-2])
+            name = array[1][1:-1]
+            occurrence = array[0]
+
+            if date not in date_dict:
+                date_dict[date] = dict()
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurrence += max(date_dict[date][name].occurrence,int(occurrence))
+            else:
+                date_dict[date][name] = csv_data_line.CSVDataLine(name, date, int(occurrence))
+
+    return date_dict
+

modules/crawler/ForceUpdateDataset.py

@@ -1 @@
-import Pipeline
-import os
-
-print("Zadejte jméno Datasetu který chcete upadtovat:\n")
-Pipeline.run_full_pipeline(input())

modules/crawler/ForceUpdateDatasets.py

@@ -1 @@
-import Pipeline
-import os
-
-# Path to configuration files
-CONFIG_FILES_PATH = "DatasetConfigs/"
-
-
-def run_pipeline_for_all_datasets():
-    """
-    Runs whole DataScript pipeline for every dataset that has existing configuration file
-    """
-    files_in_dir = os.listdir(CONFIG_FILES_PATH)
-
-    for file in files_in_dir:
-        name = file.split('.')
-        Pipeline.run_full_pipeline(name[0])
-
-
-run_pipeline_for_all_datasets()

modules/crawler/Pipeline.py

@@ -1 @@
-from Utilities import FolderProcessor, ConfigureFunctions
-from Utilities.Database import DatabaseLoader
-from Utilities.CSV import CSVutils
-
-import logging
-from datetime import date
-
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset crawler implementations
-CRAWLER_LIB_PATH = "DatasetCrawler."
-# Path to dataset processor implementations
-PROCESSOR_LIB_PATH = "DatasetProcessing."
-
-
-#logger
-logging.basicConfig(filename=CRAWLER_LOGS_PATH + "CommonRecords/" + 'Applicationlog-' + date.today().strftime("%b-%Y") + '.log',
-                    level=logging.INFO,
-                    format='%(asctime)s %(message)s'
-                    )
-
-
-def check_last_update(config):
-    """
-    Loads integer from updated.txt in CrawlerLogs/"dataset_name"
-    representing number of days from last update if number equals
-    number in confing update period updates it and reset number of
-    days to zero else increment the number
-
-    Arguments:
-        config loaded configuration file of dataset
-
-    Returns:
-       True if updating
-       Else if incementing days from last update
-    """
-    dataset_name = config["dataset-name"]
-
-    with open(CRAWLER_LOGS_PATH + dataset_name + "/" + "updated.txt", "r+") as file:
-        last_update = file.read()
-        last_update = int(last_update)
-        file.seek(0)
-
-        confing_update_period = int(config["update-period"])
-
-        if config["update-period"] <= last_update:
-            logging.info("Dataset " + dataset_name + " is being updated today")
-            file.write("0")
-            file.truncate()
-            return True
-        else:
-            last_update_days = last_update + 1
-            logging.info("Dataset " + dataset_name + " will be updated in " + str(confing_update_period - last_update_days) + "days")
-            file.write(str(last_update_days))
-            file.truncate()
-            return False
-
-
-
-def crawl_data(config):
-    """
-      Imports dataset crawler in DatasetCrawler/"dataset_name"Crawler.py
-      runs crawler.
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-
-    crawl_func = __import__(CRAWLER_LIB_PATH + dataset_name + "Crawler", globals(), locals(), ['crawl']).crawl
-    crawl_func(config)
-
-    dataset_name += '/'
-
-
-def process_data(dataset_name):
-    """
-    Goes trough every not processed file(not contained in CrawledData/"dataset_name"/ignore.txt)
-    Imports dataset processor in DatasetProcessing/"dataset_name"Processor.py
-    Runs processor on every file
-    After successful processing updates ignore.txt
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    dataset_path = dataset_name + '/'
-
-    process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "Processor", globals(), locals(),
-                                   ['process_file']).process_file
-
-    not_processed_files = FolderProcessor.list_of_all_files(CRAWLED_DATA_PATH + dataset_path)
-    logging.info(dataset_name + " has downloaded " + str(len(not_processed_files)) + " new files")
-
-    for not_processed_file in not_processed_files:
-        path = CRAWLED_DATA_PATH + dataset_path + not_processed_file
-        date_dic = process_file_func(path)
-        CSVutils.export_data_to_csv(path, date_dic)
-        FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
-
-    logging.info(dataset_name + " has processed " + str(len(not_processed_files)) + " newly crawled files")
-
-
-def validate_process_data(config):
-    """
-    Function goes through newly processed data and checks theirs status
-
-    Args:
-        config: loaded configuration file of dataset
-
-    Returns:
-        boolean variable TRUE/FALSE.
-        Data processed correctly - TRUE
-        Wrong format or NEW unknown devices - FALSE
-    """
-    processed_devices_set = FolderProcessor.get_devices_set(PROCESSED_DATA_PATH + config["dataset-name"] + '/')
-    unknown_devices_set = FolderProcessor.get_unknown_devices_set(config, processed_devices_set)
-    unknown_devices_size = len(unknown_devices_set)
-
-    if unknown_devices_size != 0:
-        logging.info("There is " + str(unknown_devices_size) + " unknown devices")
-        logging.info("Adding devices to " + config["dataset-name"] + " config file")
-        ConfigureFunctions.update_configuration(config["dataset-name"], unknown_devices_set)
-        return False
-
-    for device in config["devices"]:
-        device = config["devices"][device]
-        if device["x"] == "UNKNOWN!" or device["y"] == "UNKNOWN!":
-            logging.info(config["dataset-name"] + " config file contains devices with UNKOWN! values please update them!!")
-            return False
-
-    return True
-
-
-def load_data_to_database(config):
-    """
-    Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
-    loads data appends coordination from configurations
-    and exports it into the database
-    After successful exporting updates ignore.txt
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    dataset_path = dataset_name + '/'
-
-    # get all unprocessed files from dataset
-    not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
-
-    database_connection = DatabaseLoader.create_database_connection()
-
-    DatabaseLoader.check_or_update_datasets_collection(database_connection,config)
-
-    DatabaseLoader.update_devices_collection(database_connection,config)
-
-
-    # load every file
-    for not_loaded_file in not_loaded_files:
-        #check if file is not in database already if it is skip
-        test = DatabaseLoader.check_if_database_doesnt_contain_file(database_connection,dataset_name,not_loaded_file)
-        if test == False:
-            logging.info(dataset_name + " could not load " + not_loaded_file + " to database because he is already there.")
-            FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
-            continue
-        # load processed data
-        processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
-        # load processed data to database
-        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
-        FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
-
-    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
-
-
-def load_data_to_database_crone(config):
-    """
-    Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
-    loads data appends coordination from configurations
-    and exports it into the database
-    After successful exporting updates ignore.txt
-
-    Args:
-        config: loaded configuration file of dataset
-    """
-    dataset_name = config["dataset-name"]
-    dataset_path = dataset_name + '/'
-
-    # get all unprocessed files from dataset
-    not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
-
-    database_connection = DatabaseLoader.create_database_connection()
-
-    # load every file
-    for not_loaded_file in not_loaded_files:
-        # load processed data
-        processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
-        # load processed data to database
-        DatabaseLoader.load_data_to_database(database_connection, dataset_name, processed_data, not_loaded_file)
-        FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
-
-    logging.info(dataset_name + " has loaded to database " + str(len(not_loaded_files)) + " newly processed files.")
-
-
-def run_full_pipeline(dataset_name):
-    """
-    Loads config file and starts full pipeline
-    -crawl data
-    -process data
-    -load data to database
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    logging.info("Starting pipeline for dataset " + dataset_name)
-    
-    config = ConfigureFunctions.load_configuration(dataset_name)
-    crawl_data(config)
-    process_data(config["dataset-name"])
-
-    validation_test = validate_process_data(config)
-
-    if validation_test:
-        load_data_to_database(config)
-
-
-def run_full_pipeline_crone(dataset_name):
-    """
-    Loads config file and starts full pipeline
-    -crawl data
-    -process data
-    -load data to database
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    logging.info("Starting pipeline for dataset " + dataset_name)
-
-    config = ConfigureFunctions.load_configuration(dataset_name)
-    update_test = check_last_update(config)
-    if update_test:
-        crawl_data(config)
-        process_data(config["dataset-name"])
-
-        validation_test = validate_process_data(config)
-
-        if validation_test:
-            load_data_to_database_crone(config)
-            

modules/crawler/PrepareNewDataset.py

@@ -1 @@
-import os
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path for DatasetCrawlers implementations
-CRAWLER_PROGRAM_PATH = "DatasetCrawler"
-# Path for DatasetProcessors implementations
-PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-
-
-def create_default_config_file(dataset_name):
-    """
-    Creates default config file
-
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-    with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
-        file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
-        file.write("dataset-name: " + dataset_name + "\n")
-        file.write("# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře\n")
-        file.write("dataset-name: " + dataset_name + "\n")
-        file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
-        file.write("url: ZDE VLOZTE URL\n")
-        file.write("# volitelny parameter, ktery specifikuje vzor jmrna datasetu, ktera se budou stahovat\n")
-        file.write("regex: ZDE VLOZTE REGEX\n")
-        file.write("# volitelny parametr, ktery udava jak casto se budou hledat nove datasety, pokud prazdne, "
-                   "tak defaultni hodnota (dny)\n")
-        file.write("update-period: ZDE VLOZTE HODNOTU\n")
-        file.write("# pozice jednotlivych zarizeni, ktera jsou v datasetu\n")
-        file.write("devices:\n")
-
-
-def create_default_processor(dataset_name):
-    """
-    Creates default processor for dataset
-
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-    with open(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py", "w") as file:
-        file.write("from Utilities.CSV import CSVDataLine")
-        file.write("\n")
-        file.write("\n")
-        file.write("def process_file(filename):\n")
-        file.write("    \"\"\"\n")
-        file.write("    Method that take path to crawled file and outputs date dictionary:\n")
-        file.write("    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)\n")
-        file.write("    and value is dictionary where keys devices (specified in configuration file)\n")
-        file.write("    and value is CSVDataLine.CSVDataLine with device,date and occurrence\n")
-        file.write("\n")
-        file.write("    Args:\n")
-        file.write("    filename: name of processed file\n")
-        file.write("\n")
-        file.write("    Returns:\n")
-        file.write("    None if not implemented\n")
-        file.write("    date_dict when implemented\n")
-        file.write("    \"\"\"\n")
-        file.write("    date_dict = dict()\n")
-        file.write("\n")
-        file.write("    #with open(filename, \"r\") as file:\n")
-        file.write("    print(\"You must implements process_file method first!\")\n")
-        file.write("    return None\n")
-
-
-def create_default_crawler(dataset_name):
-    """
-    Creates default crawler for dataset
-
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-
-    with open(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py", "w") as file:
-        file.write("# Path to crawled data\n")
-        file.write("CRAWLED_DATA_PATH = \"CrawledData/\"\n")
-        file.write("\n")
-        file.write("\n")
-        file.write("def crawl(config):\n")
-        file.write("    \"\"\"\n")
-        file.write("    Implement crawl method that downloads new data to path_for_files\n")
-        file.write("    For keeping the project structure\n")
-        file.write("    url , regex, and dataset_name from config\n")
-        file.write("    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py\n")
-        file.write("\n")
-        file.write("    Args:\n")
-        file.write("        config: loaded configuration file of dataset\n")
-        file.write("    \"\"\"\n")
-        file.write("    dataset_name = config[\"dataset-name\"]\n")
-        file.write("    url = config['url']\n")
-        file.write("    regex = config['regex']\n")
-        file.write("    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'\n")
-        file.write("    print(\"You must implements Crawl method first!\")\n")
-
-
-def create_ignore_file(path, text):
-    """
-    Creates ignore file
-    Args:
-        path: path to directory for creating ignore.txt
-        text: text that will be on first line of ignore.txt can be None
-    """
-    with open(path + "/ignore.txt", "w") as file:
-        if text is not None:
-            file.write(text + "\n")
-
-def create_updated_file(path):
-    """
-    Creates updated file
-    Args:
-        path: path to directory for creating updated.txt
-    """
-    with open(path + "/updated.txt", "w") as file:
-            file.write(str(0) + "\n")
-
-
-def prepare_dataset_structure(dataset_name):
-    """
-    Prepares folders for new dataset
-    Args:
-        dataset_name: Name of newly created dataset
-    """
-
-    # create folder for crawled data
-    try:
-        path = CRAWLED_DATA_PATH+dataset_name
-        os.mkdir(path)
-        create_ignore_file(path, "ignore.txt")
-    except os.error as e:
-        print(e)
-        print("Creation of the directory %s failed" % path)
-
-    # create folder for processed data
-    try:
-        path = PROCESSED_DATA_PATH + dataset_name
-        os.mkdir(path)
-        create_ignore_file(path, "ignore.txt")
-    except OSError:
-        print("Creation of the directory %s failed" % path)
-
-    # create folder for crawler logs
-    try:
-        path = CRAWLER_LOGS_PATH + dataset_name
-        os.mkdir(path)
-        create_ignore_file(path, None)
-        create_updated_file(path)
-    except OSError:
-        print("Creation of the directory %s failed" % path)
-
-    create_default_crawler(dataset_name)
-    create_default_processor(dataset_name)
-    create_default_config_file(dataset_name)
-
-print("Zadejte jméno nového datasetu:\n")
-prepare_dataset_structure(input())

modules/crawler/ProcessedData/JIS/ignore.txt

@@ -1 @@
-ignore.txt

modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt

@@ -1 @@
-ignore.txt

modules/crawler/ProcessedData/WIFI/ignore.txt

@@ -1 @@
-ignore.txt

modules/crawler/RemoveDataset.py

@@ -1 @@
-import os
-import shutil
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-# Path for DatasetCrawlers implementations
-CRAWLER_PROGRAM_PATH = "DatasetCrawler"
-# Path for DatasetProcessors implementations
-PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
-
-
-def remove_dataset(dataset_name):
-    """
-    Remove dataset
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    shutil.rmtree(CRAWLED_DATA_PATH + dataset_name + "/")
-    shutil.rmtree(PROCESSED_DATA_PATH + dataset_name + "/")
-    shutil.rmtree(CRAWLER_LOGS_PATH + dataset_name + "/")
-
-    os.remove(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml")
-    os.remove(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py")
-    os.remove(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py")
-
-    print("Dataset " + dataset_name + " removed")
-
-print("Zadejte jméno Datasetu který chcete odstranit:\n")
-remove_dataset(input())

modules/crawler/RemoveDatasetDatabase.py

@@ -1 @@
-from Utilities.Database import DatabaseLoader
-
-
-def remove_dataset_database(dataset_name):
-    """
-    Removes dataset entries from database
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    # Creating connection
-    mydb = DatabaseLoader.create_database_connection()
-
-    # collection where are specified aviable datasets
-    collection_datasets = mydb[DatabaseLoader.MONGODB_DATASET_COLLECTION]
-
-    collection_datasets.delete_one({"name": dataset_name})
-    print("Removing record from DATASETS collection")
-
-
-    # Retrieve list of all collections
-    collections = mydb.list_collection_names()
-
-    # Drop of all collections
-    for name in collections:
-        if name.startswith(dataset_name):
-            mydb[name].drop()
-            print("Dropping: " + name)
-
-
-print("Zadejte jméno Datasetu který chcete odstranit z databáze:\n")
-remove_dataset_database(input())

modules/crawler/ResetDatabaseData.py

@@ -1 @@
-from Utilities.Database import DatabaseLoader
-
-
-def clean_database():
-    """
-    Deletes all collections from database
-    """
-    # Create connection
-    mydb = DatabaseLoader.create_database_connection()
-
-    # Retrieve list of all collections
-    collections = mydb.list_collection_names()
-
-    # Drop of all collections
-    for name in collections:
-        mydb[name].drop()
-
-    print("Database Cleaned")
-
-
-clean_database()

modules/crawler/ResetDataset.py

@@ -1 @@
-import os
-from Utilities import FolderProcessor
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-
-
-def create_ignore_file(path, text):
-    """
-    Creates ignore file
-    Args:
-        path: path to directory for creating ignore.txt
-        text: text that will be on first line of ignore.txt can be None
-    """
-    with open(path + "/ignore.txt", "w") as file:
-        if text is not None:
-            file.write(text + "\n")
-
-
-def create_updated_file(path):
-    """
-    Creates updated file
-    Args:
-        path: path to directory for creating updated.txt
-    """
-    with open(path + "/updated.txt", "w") as file:
-            file.write(str(0) + "\n")
-
-
-def reset_dataset(dataset_name):
-    """
-    Resets all saved data in dataset except config and implementation
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    path = CRAWLED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = PROCESSED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = CRAWLER_LOGS_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, None)
-    create_updated_file(path)
-
-print("Zadejte jméno Datasetu který chcete resetovat:\n")
-reset_dataset(input())

modules/crawler/ResetDatasets.py

@@ -1 @@
-import os
-from Utilities import FolderProcessor
-
-# Path to crawled data
-CRAWLED_DATA_PATH = "CrawledData/"
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-# Path to crawler logs
-CRAWLER_LOGS_PATH = "CrawlerLogs/"
-# Path to dataset configuration files
-CONFIG_FILES_PATH = "DatasetConfigs"
-
-
-def create_ignore_file(path, text):
-    """
-    Creates ignore file
-    Args:
-        path: path to directory for creating ignore.txt
-        text: text that will be on first line of ignore.txt can be None
-    """
-    with open(path + "/ignore.txt", "w") as file:
-        if text is not None:
-            file.write(text + "\n")
-
-
-def create_updated_file(path):
-    """
-    Creates updated file
-    Args:
-        path: path to directory for creating updated.txt
-    """
-    with open(path + "/updated.txt", "w") as file:
-            file.write(str(0) + "\n")
-
-
-def reset_dataset(dataset_name):
-    """
-    Resets all saved data in dataset except config and implementation
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    path = CRAWLED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = PROCESSED_DATA_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, "ignore.txt")
-
-    path = CRAWLER_LOGS_PATH + dataset_name + "/"
-    FolderProcessor.clean_folder(path)
-    create_ignore_file(path, None)
-    create_updated_file(path)
-
-
-def reset_all_datasets():
-    """
-    Resets all saved data in all datasets with config file except configs and implementation
-    """
-    datasets = os.listdir(CONFIG_FILES_PATH)
-
-    for dataset in datasets:
-        reset_dataset(dataset.split('.')[0])
-
-
-reset_all_datasets()

modules/crawler/Utilities/CSV/CSVDataLine.py

@@ -1 @@
-class CSVDataLine:
-    """
-    Class that specifies the look of data line in processed csv file
-    prepared for database
-    """
-
-    def __init__(self, name, date, occurrence):
-        try:
-            test_val = int(occurrence)
-        except ValueError:
-            print("Occurence should be and integer value!")
-
-        if len(date) != 13:
-            raise ValueError("Invalid date format YYYY-dd-mm-hh expected!")    
-
-        self.name = name
-        self.date = date
-        self.occurrence = test_val
-
-    def to_csv(self):
-        return self.name + ";" + str(self.occurrence) + ";" + self.date
-

modules/crawler/Utilities/CSV/CSVutils.py

@@ -1 @@
-import inspect
-from Utilities.CSV import CSVDataLine
-
-# Path to processed data
-PROCESSED_DATA_PATH = "ProcessedData/"
-
-
-def get_unique_names_from_file(filename, column_number):
-    """
-        Extract set of unique names from file
-    Args:
-        filename: path to processed file
-        column_number: unique names are expected in csv file on column_number
-
-    Returns:
-        set of unique names
-    """
-    # create set of unique names
-    name_set = set()
-
-    with open(filename, "r") as file:
-        # go through every line of line
-        for x in file:
-            # split by csv splitter ;
-            array = x.split(";")
-            # add string from chosen column to set
-            name_set.add(array[column_number])
-
-    return name_set