Revize 34baf808

Re #7965
implementovat skript pro crone který spouští upadte datasetu podle configu

modules/crawler/Pipeline.py

@@ -5 +5 @@
 CRAWLED_DATA_PATH = "CrawledData/"
 # Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to crawler logs
+CRAWLER_LOGS_PATH = "CrawlerLogs/"
 # Path to dataset crawler implementations
 CRAWLER_LIB_PATH = "DatasetCrawler."
 # Path to dataset processor implementations
 PROCESSOR_LIB_PATH = "DatasetProcessing."
 
 
+def check_last_update(config):
+    """
+    Loads integer from updated.txt in CrawlerLogs/"dataset_name"
+    representing number of days from last update if number equals
+    number in confing update period updates it and reset number of
+    days to zero else increment the number
+
+    Arguments:
+        config loaded configuration file of dataset
+
+    Returns:
+       True if updating
+       Else if incementing days from last update
+    """
+    dataset_name = config["dataset-name"]
+
+    with open(CRAWLER_LOGS_PATH + dataset_name + "/" + "updated.txt", "r+") as file:
+        last_update = file.read()
+        last_update = int(last_update)
+        file.seek(0)
+
+        confing_update_period = int(config["update-period"])
+
+        if config["update-period"] <= last_update:
+            print("Dataset " + dataset_name + " is being updated")
+            file.write("0")
+            file.truncate()
+            return True
+        else:
+            last_update_days = last_update + 1
+            print("Dataset " + dataset_name + " will be updated in " + str(confing_update_period - last_update_days) + "days")
+            file.write(str(last_update_days))
+            file.truncate()
+            return False
+
+
+
 def crawl_data(config):
     """
       Imports dataset crawler in DatasetCrawler/"dataset_name"Crawler.py
@@ -123 +162 @@
 
     if validation_test:
         load_data_to_database(config)
+        print("Dataset " + dataset_name + " has been sucessfully updated\n")
+
+
+
+def run_full_pipeline_crone(dataset_name):
+    """
+    Loads config file and starts full pipeline
+    -crawl data
+    -process data
+    -load data to database
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    config = ConfigureFunctions.load_configuration(dataset_name)
+    update_test = check_last_update(config)
+    if update_test:
+        crawl_data(config)
+        process_data(config["dataset-name"])
+
+        validation_test = validate_process_data(config)
 
+        if validation_test:
+            load_data_to_database(config)