commit 34baf808ab2d7ea66288c47441f259d53ac76c01
Author: petrh <petrh@zcu.cz>
Date:   Mon May 4 12:45:42 2020 +0200

    Re #7965
    implementovat skript pro crone který spouští upadte datasetu podle configu

diff --git a/modules/crawler/CrawledData/JIS/ignore.txt b/modules/crawler/CrawledData/JIS/ignore.txt
index 2d42091..8ea087a 100644
--- a/modules/crawler/CrawledData/JIS/ignore.txt
+++ b/modules/crawler/CrawledData/JIS/ignore.txt
@@ -1,12 +1 @@
-ignore.txt
-OD_ZCU_JIS_10_2019.CSV
-OD_ZCU_JIS_03_2020.CSV
-OD_ZCU_JIS_02_2020.CSV
-OD_ZCU_JIS_00_2019.CSV
-OD_ZCU_JIS_08_2019.CSV
-OD_ZCU_JIS_12_2019.CSV
-OD_ZCU_JIS_09_2019.CSV
-OD_ZCU_JIS_01_2020.CSV
-OD_ZCU_JIS_06_2019.CSV
-OD_ZCU_JIS_11_2019.CSV
-OD_ZCU_JIS_07_2019.CSV
+ignore.txt
diff --git a/modules/crawler/CrawledData/KOLOBEZKY/ignore.txt b/modules/crawler/CrawledData/KOLOBEZKY/ignore.txt
index 950dbdf..8ea087a 100644
--- a/modules/crawler/CrawledData/KOLOBEZKY/ignore.txt
+++ b/modules/crawler/CrawledData/KOLOBEZKY/ignore.txt
@@ -1,5 +1 @@
-ignore.txt
-OD_ZCU_KOLOBEZKY_00_2019.CSV
-OD_ZCU_KOLOBEZKY_06_2019.CSV
-OD_ZCU_KOLOBEZKY_07_2019.CSV
-OD_ZCU_KOLOBEZKY_08_2019.CSV
+ignore.txt
diff --git a/modules/crawler/CrawledData/WIFI/ignore.txt b/modules/crawler/CrawledData/WIFI/ignore.txt
index 20f6b7a..8ea087a 100644
--- a/modules/crawler/CrawledData/WIFI/ignore.txt
+++ b/modules/crawler/CrawledData/WIFI/ignore.txt
@@ -1,5 +1 @@
-ignore.txt
-OD_ZCU_WIFI_07_2019.CSV
-OD_ZCU_WIFI_00_2019.CSV
-OD_ZCU_WIFI_06_2019.CSV
-OD_ZCU_WIFI_08_2019.CSV
+ignore.txt
diff --git a/modules/crawler/CrawlerLogs/JIS/ignore.txt b/modules/crawler/CrawlerLogs/JIS/ignore.txt
index c69ea8b..e69de29 100644
--- a/modules/crawler/CrawlerLogs/JIS/ignore.txt
+++ b/modules/crawler/CrawlerLogs/JIS/ignore.txt
@@ -1,11 +0,0 @@
-https://openstore.zcu.cz/OD_ZCU_00_2019/OD_ZCU_JIS_00_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_01_2020/OD_ZCU_JIS_01_2020_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_02_2020/OD_ZCU_JIS_02_2020_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_03_2020/OD_ZCU_JIS_03_2020_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_06_2019/OD_ZCU_JIS_06_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_07_2019/OD_ZCU_JIS_07_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_08_2019/OD_ZCU_JIS_08_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_09_2019/OD_ZCU_JIS_09_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_10_2019/OD_ZCU_JIS_10_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_11_2019/OD_ZCU_JIS_11_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_12_2019/OD_ZCU_JIS_12_2019_CSV.zip
diff --git a/modules/crawler/CrawlerLogs/JIS/updated.txt b/modules/crawler/CrawlerLogs/JIS/updated.txt
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/modules/crawler/CrawlerLogs/JIS/updated.txt
@@ -0,0 +1 @@
+0
diff --git a/modules/crawler/CrawlerLogs/JISupdated.txt b/modules/crawler/CrawlerLogs/JISupdated.txt
new file mode 100644
index 0000000..e69de29
diff --git a/modules/crawler/CrawlerLogs/KOLOBEZKY/ignore.txt b/modules/crawler/CrawlerLogs/KOLOBEZKY/ignore.txt
index b1f85e1..e69de29 100644
--- a/modules/crawler/CrawlerLogs/KOLOBEZKY/ignore.txt
+++ b/modules/crawler/CrawlerLogs/KOLOBEZKY/ignore.txt
@@ -1,4 +0,0 @@
-https://openstore.zcu.cz/OD_ZCU_00_2019/OD_ZCU_KOLOBEZKY_00_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_06_2019/OD_ZCU_KOLOBEZKY_06_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_07_2019/OD_ZCU_KOLOBEZKY_07_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_08_2019/OD_ZCU_KOLOBEZKY_08_2019_CSV.zip
diff --git a/modules/crawler/CrawlerLogs/KOLOBEZKY/updated.txt b/modules/crawler/CrawlerLogs/KOLOBEZKY/updated.txt
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/modules/crawler/CrawlerLogs/KOLOBEZKY/updated.txt
@@ -0,0 +1 @@
+0
diff --git a/modules/crawler/CrawlerLogs/WIFI/ignore.txt b/modules/crawler/CrawlerLogs/WIFI/ignore.txt
index 7f65c57..e69de29 100644
--- a/modules/crawler/CrawlerLogs/WIFI/ignore.txt
+++ b/modules/crawler/CrawlerLogs/WIFI/ignore.txt
@@ -1,4 +0,0 @@
-https://openstore.zcu.cz/OD_ZCU_00_2019/OD_ZCU_WIFI_00_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_06_2019/OD_ZCU_WIFI_06_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_07_2019/OD_ZCU_WIFI_07_2019_CSV.zip
-https://openstore.zcu.cz/OD_ZCU_08_2019/OD_ZCU_WIFI_08_2019_CSV.zip
diff --git a/modules/crawler/CrawlerLogs/WIFI/updated.txt b/modules/crawler/CrawlerLogs/WIFI/updated.txt
new file mode 100644
index 0000000..573541a
--- /dev/null
+++ b/modules/crawler/CrawlerLogs/WIFI/updated.txt
@@ -0,0 +1 @@
+0
diff --git a/modules/crawler/CrawlerLogs/updated.txt b/modules/crawler/CrawlerLogs/updated.txt
new file mode 100644
index 0000000..e69de29
diff --git a/modules/crawler/CroneUpdateScript.py b/modules/crawler/CroneUpdateScript.py
new file mode 100644
index 0000000..68e31e5
--- /dev/null
+++ b/modules/crawler/CroneUpdateScript.py
@@ -0,0 +1,19 @@
+import Pipeline
+import os
+
+# Path to configuration files
+CONFIG_FILES_PATH = "DatasetConfigs/"
+
+
+def run_pipeline_for_all_datasets():
+    """
+    Runs whole DataScript pipeline for every dataset that has existing configuration file
+    """
+    files_in_dir = os.listdir(CONFIG_FILES_PATH)
+
+    for file in files_in_dir:
+        name = file.split('.')
+        Pipeline.run_full_pipeline_crone(name[0])
+
+
+run_pipeline_for_all_datasets()
diff --git a/modules/crawler/ForceUpdateDataset.py b/modules/crawler/ForceUpdateDataset.py
new file mode 100644
index 0000000..53e275b
--- /dev/null
+++ b/modules/crawler/ForceUpdateDataset.py
@@ -0,0 +1,5 @@
+import Pipeline
+import os
+
+print("Zadejte jméno Datasetu který chcete upadtovat:\n")
+Pipeline.run_full_pipeline(input())
diff --git a/modules/crawler/ForceUpdateDatasets.py b/modules/crawler/ForceUpdateDatasets.py
new file mode 100644
index 0000000..820f9f8
--- /dev/null
+++ b/modules/crawler/ForceUpdateDatasets.py
@@ -0,0 +1,19 @@
+import Pipeline
+import os
+
+# Path to configuration files
+CONFIG_FILES_PATH = "DatasetConfigs/"
+
+
+def run_pipeline_for_all_datasets():
+    """
+    Runs whole DataScript pipeline for every dataset that has existing configuration file
+    """
+    files_in_dir = os.listdir(CONFIG_FILES_PATH)
+
+    for file in files_in_dir:
+        name = file.split('.')
+        Pipeline.run_full_pipeline(name[0])
+
+
+run_pipeline_for_all_datasets()
diff --git a/modules/crawler/Pipeline.py b/modules/crawler/Pipeline.py
index 012e532..5fd916b 100644
--- a/modules/crawler/Pipeline.py
+++ b/modules/crawler/Pipeline.py
@@ -5,12 +5,51 @@ from Utilities.Database import DatabaseLoader
 CRAWLED_DATA_PATH = "CrawledData/"
 # Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
 # Path to dataset crawler implementations
 CRAWLER_LIB_PATH = "DatasetCrawler."
 # Path to dataset processor implementations
 PROCESSOR_LIB_PATH = "DatasetProcessing."
 
 
+def check_last_update(config):
+    """
+    Loads integer from updated.txt in CrawlerLogs/"dataset_name"
+    representing number of days from last update if number equals
+    number in confing update period updates it and reset number of
+    days to zero else increment the number
+
+    Arguments:
+        config loaded configuration file of dataset
+
+    Returns:
+       True if updating
+       Else if incementing days from last update
+    """
+    dataset_name = config["dataset-name"]
+
+    with open(CRAWLER_LOGS_PATH + dataset_name + "/" + "updated.txt", "r+") as file:
+        last_update = file.read()
+        last_update = int(last_update)
+        file.seek(0)
+
+        confing_update_period = int(config["update-period"])
+
+        if config["update-period"] <= last_update:
+            print("Dataset " + dataset_name + " is being updated")
+            file.write("0")
+            file.truncate()
+            return True
+        else:
+            last_update_days = last_update + 1
+            print("Dataset " + dataset_name + " will be updated in " + str(confing_update_period - last_update_days) + "days")
+            file.write(str(last_update_days))
+            file.truncate()
+            return False
+
+
+
 def crawl_data(config):
     """
       Imports dataset crawler in DatasetCrawler/"dataset_name"Crawler.py
@@ -123,4 +162,27 @@ def run_full_pipeline(dataset_name):
 
     if validation_test:
         load_data_to_database(config)
+        print("Dataset " + dataset_name + " has been sucessfully updated\n")
+
+
+
+def run_full_pipeline_crone(dataset_name):
+    """
+    Loads config file and starts full pipeline
+    -crawl data
+    -process data
+    -load data to database
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    config = ConfigureFunctions.load_configuration(dataset_name)
+    update_test = check_last_update(config)
+    if update_test:
+        crawl_data(config)
+        process_data(config["dataset-name"])
+
+        validation_test = validate_process_data(config)
 
+        if validation_test:
+            load_data_to_database(config)
\ No newline at end of file
diff --git a/modules/crawler/PrepareNewDataset.py b/modules/crawler/PrepareNewDataset.py
index 982b346..2f4a154 100644
--- a/modules/crawler/PrepareNewDataset.py
+++ b/modules/crawler/PrepareNewDataset.py
@@ -107,6 +107,15 @@ def create_ignore_file(path, text):
         if text is not None:
             file.write(text + "\n")
 
+def create_updated_file(path):
+    """
+    Creates updated file
+    Args:
+        path: path to directory for creating updated.txt
+    """
+    with open(path + "/updated.txt", "w") as file:
+            file.write(str(0) + "\n")
+
 
 def prepare_dataset_structure(dataset_name):
     """
@@ -114,7 +123,6 @@ def prepare_dataset_structure(dataset_name):
     Args:
         dataset_name: Name of newly created dataset
     """
-    jump_folder = "../"
 
     # create folder for crawled data
     try:
@@ -138,6 +146,7 @@ def prepare_dataset_structure(dataset_name):
         path = CRAWLER_LOGS_PATH + dataset_name
         os.mkdir(path)
         create_ignore_file(path, None)
+        create_updated_file(path)
     except OSError:
         print("Creation of the directory %s failed" % path)
 
diff --git a/modules/crawler/ProcessedData/JIS/ignore.txt b/modules/crawler/ProcessedData/JIS/ignore.txt
index c5df76e..8ea087a 100644
--- a/modules/crawler/ProcessedData/JIS/ignore.txt
+++ b/modules/crawler/ProcessedData/JIS/ignore.txt
@@ -1 +1 @@
-ignore.txt
+ignore.txt
diff --git a/modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt b/modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt
index a9059c3..8ea087a 100644
--- a/modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt
+++ b/modules/crawler/ProcessedData/KOLOBEZKY/ignore.txt
@@ -1,5 +1 @@
-ignore.txt
-OD_ZCU_KOLOBEZKY_08_2019.CSV
-OD_ZCU_KOLOBEZKY_00_2019.CSV
-OD_ZCU_KOLOBEZKY_07_2019.CSV
-OD_ZCU_KOLOBEZKY_06_2019.CSV
+ignore.txt
diff --git a/modules/crawler/ProcessedData/WIFI/ignore.txt b/modules/crawler/ProcessedData/WIFI/ignore.txt
index c5df76e..8ea087a 100644
--- a/modules/crawler/ProcessedData/WIFI/ignore.txt
+++ b/modules/crawler/ProcessedData/WIFI/ignore.txt
@@ -1 +1 @@
-ignore.txt
+ignore.txt
diff --git a/modules/crawler/ResetDataset.py b/modules/crawler/ResetDataset.py
index f27379f..2366211 100644
--- a/modules/crawler/ResetDataset.py
+++ b/modules/crawler/ResetDataset.py
@@ -2,13 +2,13 @@ import os
 from Utilities import FolderProcessor
 
 # Path to crawled data
-CRAWLED_DATA_PATH = "../CrawledData/"
+CRAWLED_DATA_PATH = "CrawledData/"
 # Path to processed data
-PROCESSED_DATA_PATH = "../ProcessedData/"
+PROCESSED_DATA_PATH = "ProcessedData/"
 # Path to crawler logs
-CRAWLER_LOGS_PATH = "../CrawlerLogs/"
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
 # Path to dataset configuration files
-CONFIG_FILES_PATH = "../DatasetConfigs"
+CONFIG_FILES_PATH = "DatasetConfigs"
 
 
 def create_ignore_file(path, text):
@@ -23,6 +23,16 @@ def create_ignore_file(path, text):
             file.write(text + "\n")
 
 
+def create_updated_file(path):
+    """
+    Creates updated file
+    Args:
+        path: path to directory for creating updated.txt
+    """
+    with open(path + "/updated.txt", "w") as file:
+            file.write(str(0) + "\n")
+
+
 def reset_dataset(dataset_name):
     """
     Resets all saved data in dataset except config and implementation
@@ -40,6 +50,7 @@ def reset_dataset(dataset_name):
     path = CRAWLER_LOGS_PATH + dataset_name + "/"
     FolderProcessor.clean_folder(path)
     create_ignore_file(path, None)
+    create_updated_file(path)
 
 
 def reset_all_datasets():
diff --git a/modules/crawler/main.py b/modules/crawler/main.py
deleted file mode 100644
index f2b5629..0000000
--- a/modules/crawler/main.py
+++ /dev/null
@@ -1,29 +0,0 @@
-import Pipeline
-import os
-
-# Path to configuration files
-CONFIG_FILES_PATH = "DatasetConfigs/"
-
-
-def run_pipeline_for_all_datasets():
-    """
-    Runs whole DataScript pipeline for every dataset that has existing configuration file
-    """
-    files_in_dir = os.listdir(CONFIG_FILES_PATH)
-
-    for file in files_in_dir:
-        name = file.split('.')
-        Pipeline.run_full_pipeline(name[0])
-
-
-def run_pipeline_for_one_dataset(dataset_name):
-    """
-    Runs whole DataScript pipeline for only one dataset
-
-    Args:
-        dataset_name: name of dataset that has existing configuration file
-    """
-    Pipeline.run_full_pipeline(dataset_name)
-
-
-run_pipeline_for_all_datasets()
