Revize 04a2b5a4

Re #7939
- pridana dokumentace metod a trid
- korekce chyb v jmenech promenych
- pridani informaci pro vygenerovane skripty

python-module/Pipeline.py

@@ -1 +1 @@
 from Utilities import FolderProcessor, ConfigureFunctions
 from Utilities.Database import DatabaseLoader
 
-
-CONFIG_FILES_PATH = "DatasetConfigs/"
+# Path to crawled data
 CRAWLED_DATA_PATH = "CrawledData/"
+# Path to processed data
 PROCESSED_DATA_PATH = "ProcessedData/"
+# Path to dataset crawler implementations
 CRAWLER_LIB_PATH = "DatasetCrawler."
+# Path to dataset processor implementations
 PROCESSOR_LIB_PATH = "DatasetProcessing."
 
 
 def crawl_data(config):
+    """
+      Imports dataset crawler in DatasetCrawler/"dataset_name"Crawler.py
+      runs crawler.
 
+    Args:
+        config: loaded configuration file of dataset
+    """
     dataset_name = config["dataset-name"]
 
-    my_function = __import__(CRAWLER_LIB_PATH + dataset_name + "Crawler",globals(),locals(),['crawl']).crawl
+    my_function = __import__(CRAWLER_LIB_PATH + dataset_name + "Crawler", globals(), locals(), ['crawl']).crawl
     my_function(config)
 
     dataset_name += '/'
 
 
 def process_data(dataset_name):
+    """
+    Goes trough every not processed file(not contained in CrawledData/"dataset_name"/ignore.txt)
+    Imports dataset processor in DatasetProcessing/"dataset_name"Processor.py
+    Runs processor on every file
+    After successful processing updates ignore.txt
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
     dataset_path = dataset_name + '/'
 
-    process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "Processor",globals(),locals(),
+    process_file_func = __import__(PROCESSOR_LIB_PATH + dataset_name + "Processor", globals(), locals(),
                                    ['process_file']).process_file
 
-    # get all not processed files from dataset
     not_processed_files = FolderProcessor.list_of_all_files(CRAWLED_DATA_PATH + dataset_path)
 
-    # process every file
     for not_processed_file in not_processed_files:
-        # call processing for one file in dataset
         process_file_func(CRAWLED_DATA_PATH + dataset_path + not_processed_file)
         FolderProcessor.update_ignore_set(CRAWLED_DATA_PATH + dataset_path, not_processed_file)
 
 
 def validate_process_data(config):
+    """
+    Function goes through newly processed data and checks theirs status
+
+    Args:
+        config: loaded configuration file of dataset
+
+    Returns:
+        boolean variable TRUE/FALSE.
+        Data processed correctly - TRUE
+        Wrong format or NEW unknown devices - FALSE
+    """
     processed_devices_set = FolderProcessor.get_devices_set(PROCESSED_DATA_PATH + config["dataset-name"] + '/')
-    unknown_devices_set = FolderProcessor.get_unknown_devices_set(config,processed_devices_set)
+    unknown_devices_set = FolderProcessor.get_unknown_devices_set(config, processed_devices_set)
     unknown_devices_size = len(unknown_devices_set)
 
     if unknown_devices_size != 0:
         print("There is " + str(unknown_devices_size) + " unknown devies")
-        ConfigureFunctions.update_configuration(CONFIG_FILES_PATH + config["dataset-name"] + ".yaml", unknown_devices_set)
+        ConfigureFunctions.update_configuration(config["dataset-name"], unknown_devices_set)
         return False
 
 
 def load_data_to_database(config):
-
+    """
+    Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
+    loads data appends coordination from configurations
+    and exports it into the database
+    After successful exporting updates ignore.txt
+
+    Args:
+        config: loaded configuration file of dataset
+    """
     dataset_name = config["dataset-name"]
     dataset_path = dataset_name + '/'
 
@@ -57 +90 @@
     # load every file
     for not_loaded_file in not_loaded_files:
         # load processed data
-        processed_data = DatabaseLoader.get_data_from_file(PROCESSED_DATA_PATH + dataset_path + not_loaded_file,
-                                                           config["devices"])
+        processed_data = DatabaseLoader.get_data_from_file(not_loaded_file, config)
         # load processed data to database
         DatabaseLoader.load_data_to_database(dataset_name, processed_data)
         FolderProcessor.update_ignore_set(PROCESSED_DATA_PATH + dataset_path, not_loaded_file)
 
 
 def run_full_pipeline(dataset_name):
-    config = ConfigureFunctions.load_configuration(CONFIG_FILES_PATH + dataset_name)
+    """
+    Loads config file and starts full pipeline
+    -crawl data
+    -process data
+    -load data to database
+
+    Args:
+        dataset_name: name of dataset that has existing configuration file
+    """
+    config = ConfigureFunctions.load_configuration(dataset_name)
     crawl_data(config)
     process_data(config["dataset-name"])