aswi2020sebela-gitlab/modules/crawler/prepare_new_dataset.py @ d6ca840d

import os

# Path to crawled data
CRAWLED_DATA_PATH = "CrawledData/"
# Path to processed data
PROCESSED_DATA_PATH = "ProcessedData/"
# Path for DatasetCrawlers implementations
CRAWLER_PROGRAM_PATH = "DatasetCrawler"
# Path for DatasetProcessors implementations
PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
# Path to dataset configuration files
CONFIG_FILES_PATH = "DatasetConfigs"


def create_default_config_file(dataset_name):
    """
    Creates default config file

    Args:
        dataset_name: Name of newly created dataset
    """
    with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
        file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
        file.write("dataset-name: " + dataset_name + "\n")
        file.write("# jednoslovný název datasetu, pod kterym bude reprezentovana v architektuře\n")
        file.write("dataset-name: " + dataset_name + "\n")
        file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
        file.write("url: ZDE VLOZTE URL\n")
        file.write("# volitelny parameter, ktery specifikuje vzor jmrna datasetu, ktera se budou stahovat\n")
        file.write("regex: ZDE VLOZTE REGEX\n")
        file.write("# volitelny parametr, ktery udava jak casto se budou hledat nove datasety, pokud prazdne, "
                   "tak defaultni hodnota (dny)\n")
        file.write("update-period: ZDE VLOZTE HODNOTU\n")
        file.write("# pozice jednotlivych zarizeni, ktera jsou v datasetu\n")
        file.write("devices:\n")


def create_default_processor(dataset_name):
    """
    Creates default processor for dataset

    Args:
        dataset_name: Name of newly created dataset
    """
    with open(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "_processor.py", "w") as file:
        file.write("from Utilities.CSV import csv_data_line")
        file.write("\n")
        file.write("\n")
        file.write("def process_file(filename):\n")
        file.write("    \"\"\"\n")
        file.write("    Method that take path to crawled file and outputs date dictionary:\n")
        file.write("    Date dictionary is a dictionary where keys are dates in format ddmmYYYYhh (0804201815)\n")
        file.write("    and value is dictionary where keys are devices (specified in configuration file)\n")
        file.write("    and value is CSVDataLine.csv_data_line with device,date and occurrence\n")
        file.write("\n")
        file.write("    Args:\n")
        file.write("    filename: name of processed file\n")
        file.write("\n")
        file.write("    Returns:\n")
        file.write("    None if not implemented\n")
        file.write("    date_dict when implemented\n")
        file.write("    \"\"\"\n")
        file.write("    date_dict = dict()\n")
        file.write("\n")
        file.write("    #with open(filename, \"r\") as file:\n")
        file.write("    print(\"You must implements process_file method first!\")\n")
        file.write("    return None\n")


def create_default_crawler(dataset_name):
    """
    Creates default crawler for dataset

    Args:
        dataset_name: Name of newly created dataset
    """

    with open(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "_crawler.py", "w") as file:
        file.write("# Path to crawled data\n")
        file.write("CRAWLED_DATA_PATH = \"CrawledData/\"\n")
        file.write("\n")
        file.write("\n")
        file.write("def crawl(config):\n")
        file.write("    \"\"\"\n")
        file.write("    Implement crawl method that downloads new data to path_for_files\n")
        file.write("    For keeping the project structure\n")
        file.write("    url , regex, and dataset_name from config\n")
        file.write("    You can use already implemented functions from Utilities/Crawler/BasicCrawlerFunctions.py\n")
        file.write("\n")
        file.write("    Args:\n")
        file.write("        config: loaded configuration file of dataset\n")
        file.write("    \"\"\"\n")
        file.write("    dataset_name = config[\"dataset-name\"]\n")
        file.write("    url = config['url']\n")
        file.write("    regex = config['regex']\n")
        file.write("    path_for_files = CRAWLED_DATA_PATH + dataset_name + '/'\n")
        file.write("    print(\"You must implements Crawl method first!\")\n")


def prepare_dataset_structure(dataset_name):
    """
    Prepares folders for new dataset
    Args:
        dataset_name: Name of newly created dataset
    """

    # create folder for crawled data
    path = CRAWLED_DATA_PATH+dataset_name
    try:
        os.mkdir(path)
    except os.error as e:
        print(e)
        print("Creation of the directory %s failed" % path)

    # create folder for processed data
    path = PROCESSED_DATA_PATH + dataset_name
    try:
        os.mkdir(path)
    except OSError:
        print("Creation of the directory %s failed" % path)

    create_default_crawler(dataset_name)
    create_default_processor(dataset_name)
    create_default_config_file(dataset_name)

print("Zadejte jméno nového datasetu:\n")
prepare_dataset_structure(input().upper())