Revize c8f3051b

Re #7924 beta version of datascript

.gitignore

@@ -3 +3 @@
 .idea
 dev-dump
 *.lock
+<<<<<<< HEAD
 .vscode-server
+=======
+*__pycache__*
+*.CSV
+python-module/venv
+>>>>>>> Re #7924 beta version of datascript

python-module/CrawledData/KOLOBEZKY/ignore.txt

@@ +1 @@
+ignore.txt
+OD_ZCU_KOLOBEZKY_07_2019.CSV
+OD_ZCU_KOLOBEZKY_06_2019.CSV
+OD_ZCU_KOLOBEZKY_00_2019.CSV
+OD_ZCU_KOLOBEZKY_08_2019.CSV

python-module/CrawlerLogs/KOLOBEZKY/ignore.txt

@@ +1 @@
+https://openstore.zcu.cz/OD_ZCU_00_2019/OD_ZCU_KOLOBEZKY_00_2019_CSV.zip
+https://openstore.zcu.cz/OD_ZCU_06_2019/OD_ZCU_KOLOBEZKY_06_2019_CSV.zip
+https://openstore.zcu.cz/OD_ZCU_07_2019/OD_ZCU_KOLOBEZKY_07_2019_CSV.zip
+https://openstore.zcu.cz/OD_ZCU_08_2019/OD_ZCU_KOLOBEZKY_08_2019_CSV.zip

python-module/DatasetConfigs/KOLOBEZKY.yaml

@@ +1 @@
+# jméno datasetu, pod kterým bude zobrazen v aplikaci
+dataset-name: KOLOBEZKY
+# pozice jednotlivých zařízení, která jsou v datasetu
+devices:
+  - stojan-knihovna:
+      x: 13.353319
+      y: 49.725145
+
+  - stojan-Machovka:
+      x: 13.368016
+      y: 49.725197
+
+  -  stojan-FDU:
+      x: 13.347909
+      y: 49.725360
+
+  - stojan-bory:
+      x: 13.350861
+      y: 49.724576
+
+  - stojan-zcu:
+      x: 13.365958
+      y: 49.726033
+
+  - stojan-borska:
+      x: 13.359475
+      y: 49.734518
+
+# root složka, která obsahuje odkazy na dataset
+url: https://openstore.zcu.cz/
+# volitelný parameter, který specifikuje vzor jména datasetů, které se budou stahovat
+regex: OD_ZCU_KOLOBEZKY_[0-9][0-9]_[0-9][0-9][0-9][0-9]_CSV.zip
+# volitelný parametr, který udává jak často se budou hledat nové datasety, pokud prázdné, tak defaultní hodnota (dny)
+update-period: 24

python-module/DatasetCrawler/KOLOBEZKYCrawler.py

@@ +1 @@
+from Utilities import FolderProcessor
+from Utilities.Crawler import BasicCrawler
+
+
+def crawl(config):
+
+    dataset_name = config["dataset-name"]
+    url = config['url']
+    regex = config['regex']
+
+    first_level_links = BasicCrawler.get_all_links(url)
+    filtered_first_level_links = BasicCrawler.filter_links(first_level_links, "^OD_ZCU")
+    absolute_first_level_links = BasicCrawler.create_absolute_links(filtered_first_level_links, url)
+
+    files = []
+
+    for link in absolute_first_level_links:
+        second_level_links = BasicCrawler.get_all_links(link)
+        filtered_second_level_links = BasicCrawler.filter_links(second_level_links, regex)
+        absolute_second_level_links = BasicCrawler.create_absolute_links(filtered_second_level_links, link)
+        final_links = BasicCrawler.remove_downloaded_links(absolute_second_level_links, dataset_name)
+
+        for file_link in final_links:
+            files.append(file_link)
+
+    for file in files:
+        BasicCrawler.download_file_from_url(file, "CrawledData/" + dataset_name + "/", dataset_name)
+
+    FolderProcessor.unzip_all_csv_zip_files_in_folder("CrawledData/" + dataset_name + "/")

python-module/DatasetProcessing/KOLOBEZKYProcessor.py

@@ +1 @@
+from Utilities.CSV import CSVDataLine, CSVutils
+from Utilities import DateFormating
+
+
+def process_file(filename):
+
+    with open(filename, "r") as file:
+
+        date_dict = dict()
+
+        for line in file:
+
+            array = line.split(";")
+
+            date = DateFormating.date_time_formater(array[0][1:-1])
+            name = array[1][1:-1]
+
+            if date not in date_dict:
+                date_dict[date] = dict()
+
+            if name in date_dict[date]:
+                date_dict[date][name].occurence += 1
+            else:
+                date_dict[date][name] = CSVDataLine.CSVDataLine(name, date, 1)
+
+        CSVutils.export_data_to_csv(filename, date_dict)

python-module/Pipeline.py

@@ +1 @@
+from Utilities import FolderProcessor, ConfigureLoader
+from Utilities.Database import DatabaseLoader
+
+
+CONFIG_FILES_PATH = "DatasetConfigs/"
+CRAWLED_DATA_PATH = "CrawledData/"
+PROCESSED_DATA_PATH = "ProcessedData/"
+CRAWLER_LIB_PATH = "DatasetCrawler."
+PROCESSOR_LIB_PATH = "DatasetProcessing."
+
+
+def crawl_data(config):
+
+    dataset_name = config["dataset-name"]
+
+    my_function = __import__(CRAWLER_LIB_PATH + dataset_name + "Crawler",globals(),locals(),['crawl']).crawl
+    my_function(config)
+
+    dataset_name += '/'
+
+
+def process_data(dataset_name):
+    dataset_path = dataset_name + '/'
+
+    process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "Processor",globals(),locals(),
+                                   ['process_file']).process_file
+
+    # get all not processed files from dataset
+    not_processed_files = FolderProcessor.list_of_all_files(CRAWLED_DATA_PATH + dataset_path)
+
+    # process every file
+    for not_processed_file in not_processed_files:
+        # call processing for one file in dataset
+        process_file_func(CRAWLED_DATA_PATH + dataset_path + not_processed_file)
+        FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
+
+
+def load_data_to_database(config):
+
+    dataset_name = config["dataset-name"]
+    dataset_path = dataset_name + '/'
+
+    # get all unprocessed files from dataset
+    not_loaded_files = FolderProcessor.list_of_all_files(PROCESSED_DATA_PATH + dataset_path)
+
+    # load every file
+    for not_loaded_file in not_loaded_files:
+        # load processed data
+        processed_data = DatabaseLoader.get_data_from_file(PROCESSED_DATA_PATH + dataset_path + not_loaded_file,
+                                                           config["devices"])
+        # load processed data to database
+        DatabaseLoader.load_data_to_database(dataset_name, processed_data)
+        FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
+
+
+def run_full_pipeline(dataset_name):
+    config = ConfigureLoader.load_configuration(CONFIG_FILES_PATH + dataset_name)
+    crawl_data(config)
+    process_data(config["dataset-name"])
+    load_data_to_database(config)

python-module/ProcessedData/KOLOBEZKY/ignore.txt

@@ +1 @@
+ignore.txt
+OD_ZCU_KOLOBEZKY_07_2019.CSV
+OD_ZCU_KOLOBEZKY_06_2019.CSV
+OD_ZCU_KOLOBEZKY_00_2019.CSV
+OD_ZCU_KOLOBEZKY_08_2019.CSV

python-module/Scripts/PrepareNewDataset.py

@@ +1 @@
+import os
+
+CRAWLED_DATA_PATH = "../CrawledData/"
+PROCESSED_DATA_PATH = "../ProcessedData/"
+CRAWLER_LOGS_PATH = "../CrawlerLogs/"
+CRAWLER_PROGRAM_PATH = "../DatasetCrawler"
+PROCESSOR_PROGRAM_PATH = "../DatasetProcessing"
+CONFIG_FILES_PATH = "../DatasetConfigs"
+
+
+def create_default_config_file(dataset_name):
+
+    with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
+        file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
+        file.write("dataset-name: " + dataset_name + "\n")
+        file.write("# pozice jednotlivych zarizeni, ktera jsou v datasetu\n")
+        file.write("devices:\n")
+        file.write("  - example1:\n")
+        file.write("      x: 12.3\n")
+        file.write("      y: 32.1\n")
+        file.write("\n")
+        file.write("  - example2:\n")
+        file.write("      x: 32.1\n")
+        file.write("      y: 12.3\n")
+        file.write("\n")
+        file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
+        file.write("url: ZDE VLOZTE URL/\n")
+        file.write("# volitelný parameter, který specifikuje vzor jména datasetů, které se budou stahovat\n")
+        file.write("regex: ZDE VLOZTE REGEX\n")
+        file.write("# volitelny parametr, ktery udava jak casto se budou hledat nove datasety, pokud prazdne, "
+                   "tak defaultni hodnota (dny)\n")
+        file.write("update-period: ZDE VLOZTE HODNOTU\n")
+
+
+def create_default_processor(dataset_name):
+    with open(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py", "w") as file:
+        file.write("def process_file(filename):\n")
+        file.write("    print(\"You must implements process_file method first!\")\n")
+
+
+def create_default_crawler(dataset_name):
+
+    with open(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py", "w") as file:
+        file.write("def crawl(config):\n")
+        file.write("    print(\"You must implements Crawl method first!\")\n")
+
+
+def create_ignore_file(path,text):
+
+    with open(path + "/ignore.txt", "w") as file:
+        if text is not None:
+            file.write(text + "\n")
+
+
+def prepare_dataset_structure(dataset_name):
+    jump_folder = "../"
+
+    # create folder for crawled data
+    try:
+        path = CRAWLED_DATA_PATH+dataset_name
+        os.mkdir(path)
+        create_ignore_file(path,"ignore.txt")
+    except os.error as e:
+        print(e)
+        print("Creation of the directory %s failed" % path)
+
+    # create folder for processed data
+    try:
+        path = PROCESSED_DATA_PATH + dataset_name
+        os.mkdir(path)
+        create_ignore_file(path, "ignore.txt")
+    except OSError:
+        print("Creation of the directory %s failed" % path)
+
+    # create folder for crawler logs
+    try:
+        path = CRAWLER_LOGS_PATH + dataset_name
+        os.mkdir(path)
+        create_ignore_file(path, None)
+    except OSError:
+        print("Creation of the directory %s failed" % path)
+
+    create_default_crawler(dataset_name)
+    create_default_processor(dataset_name)
+    create_default_config_file(dataset_name)
+
+
+prepare_dataset_structure("JIS")

python-module/Utilities/CSV/CSVDataLine.py

@@ +1 @@
+class CSVDataLine:
+
+    def __init__(self, name, date, occurence):
+        self.name = name
+        self.date = date
+        self.occurence = occurence
+
+    def to_csv(self):
+        return self.name + ";" + str(self.occurence) + ";" + self.date
+

python-module/Utilities/CSV/CSVutils.py

@@ +1 @@
+PROCESSED_DATA_PATH = "ProcessedData/"
+
+def get_unique_names_from_file(filename, column_number):
+    f = open(filename, "r")
+
+    # create set of unique names
+    name_set = set()
+
+    # go through every line of line
+    for x in f:
+        # split by csv splitter ;
+        array = x.split(";")
+        # add string from chosen column to set
+        name_set.add(array[column_number])
+
+    f.close()
+
+    return name_set
+
+
+def export_data_to_csv(filename, data_dict):
+    with open(PROCESSED_DATA_PATH + filename[12:], "w+") as file:
+
+        for date in data_dict:
+            for data in data_dict[date]:
+                file.write(data_dict[date][data].to_csv() + '\n')

python-module/Utilities/ConfigureLoader.py

@@ +1 @@
+import yaml
+
+
+def load_configuration(configure_file_name):
+
+    with open(configure_file_name) as f:
+        data = yaml.load(f, Loader=yaml.FullLoader)
+
+    devices_dic = dict()
+
+    for item in data["devices"]:
+        devices_dic.update(item)
+
+    data["devices"] = devices_dic
+
+    return data

python-module/Utilities/Crawler/BasicCrawler.py

@@ +1 @@
+import requests
+import re
+from Utilities import FolderProcessor
+from bs4 import BeautifulSoup
+
+
+def get_all_links(url):
+    # create response object
+    r = requests.get(url)
+
+    # create beautiful-soup object
+    soup = BeautifulSoup(r.content, 'html5lib')
+    links = []
+
+    for link in soup.findAll('a'):
+        links.append(link.get('href'))
+
+    return links
+
+
+def filter_links(links, regex):
+    fitlered_links = []
+
+    for link in links:
+        if re.search(regex,link):
+            fitlered_links.append(link)
+
+    return fitlered_links
+
+
+def create_absolute_links(links, archive):
+    absolute_links = []
+
+    for link in links:
+        absolute_links.append(archive + link)
+
+    return absolute_links
+
+
+def remove_downloaded_links(links,dataset_name):
+
+    downloaded_links = FolderProcessor.load_ignore_set("CrawlerLogs/" + dataset_name + "/")
+    final_links = set(links) - downloaded_links
+
+    return final_links
+
+
+def download_file_from_url(url,path, dataset_name):
+    r = requests.get(url, stream=True)
+
+    url_parts = url.split("/")
+    file_name = url_parts[len(url_parts)-1]
+
+    with open(path + file_name, "wb") as file:
+        for chunk in r.iter_content(chunk_size=1024):
+
+            # writing one chunk at a time to pdf file
+            if chunk:
+                file.write(chunk)
+
+    FolderProcessor.update_ignore_set("CrawlerLogs/" + dataset_name + "/", url)

python-module/Utilities/DateFormating.py

@@ +1 @@
+def date_formater(string_date):
+    if string_date[11].isspace():
+        pos = 0
+        srr = ""
+        for i in string_date:
+
+            if pos == 10:
+                srr = srr + '0'
+            else:
+                srr = srr + i
+
+            pos = pos + 1
+
+        string_date = srr
+
+    return_date = string_date[:2] + string_date[3:5] + string_date[6:10]
+
+    return return_date
+
+
+def date_time_formater(string_date):
+    if string_date[11].isspace():
+        pos = 0
+        srr = ""
+        for i in string_date:
+
+            if pos == 10:
+                srr = srr + '0'
+            else:
+                srr = srr + i
+
+            pos = pos + 1
+
+        string_date = srr
+
+    return_date = string_date[:2] + string_date[3:5] + string_date[6:10] + string_date[11:13]
+
+    return return_date

python-module/Utilities/FolderProcessor.py

@@ +1 @@
+import os
+import zipfile
+from Utilities import ConfigureLoader
+
+
+def list_of_all_files(path):
+    files_in_dir = os.listdir(path)
+
+    ignore_set = load_ignore_set(path)
+
+    return set(files_in_dir).difference(ignore_set)
+
+
+def load_ignore_set(path):
+    ignore_set = set()
+
+    with open(path + "ignore.txt", "r") as file:
+
+        for line in file:
+            ignore_set.add(line[:-1])
+
+    return ignore_set
+
+
+def update_ignore_set(path,file_name):
+
+    with open(path + "ignore.txt", "a") as file:
+        file.write(file_name + '\n')
+
+
+def unzip_all_csv_zip_files_in_folder(folder):
+
+    files_in_dir = os.listdir(folder)
+    zips = []
+
+    for file in files_in_dir:
+        if file.endswith(".zip"):
+            zips.append(folder + file)
+
+    for zip_file in zips:
+
+        with zipfile.ZipFile(zip_file, "r") as unziped_file:
+            unziped_file.extractall(folder)
+
+        os.remove(zip_file)
+
+

python-module/main.py

@@ +1 @@
+import Pipeline
+import os
+
+CONFIG_FILES_PATH = "DatasetConfigs/"
+
+
+def run_pipeline_for_all_datasets():
+    files_in_dir = os.listdir(CONFIG_FILES_PATH)
+
+    for file in files_in_dir:
+        Pipeline.run_full_pipeline(file)
+
+
+def run_pipeline_for_one_dataset(dataset_name):
+    Pipeline.run_full_pipeline(dataset_name)
+
+
+run_pipeline_for_all_datasets()

python-module/requirements.txt

@@ +1 @@
+beautifulsoup4==4.9.0
+certifi==2020.4.5.1
+chardet==3.0.4
+html5lib==1.0.1
+idna==2.9
+pymongo==3.10.1
+PyYAML==5.3.1
+requests==2.23.0
+six==1.14.0
+soupsieve==2.0
+urllib3==1.25.9
+webencodings==0.5.1