aswi2020sebela-gitlab/python-module/Scripts/PrepareNewDataset.py @ 714acf1c

import os

CRAWLED_DATA_PATH = "../CrawledData/"
PROCESSED_DATA_PATH = "../ProcessedData/"
CRAWLER_LOGS_PATH = "../CrawlerLogs/"
CRAWLER_PROGRAM_PATH = "../DatasetCrawler"
PROCESSOR_PROGRAM_PATH = "../DatasetProcessing"
CONFIG_FILES_PATH = "../DatasetConfigs"


def create_default_config_file(dataset_name):

    with open(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml", "w") as file:
        file.write("# jmeno datasetu, pod kterym bude zobrazen v aplikaci\n")
        file.write("dataset-name: " + dataset_name + "\n")
        file.write("# pozice jednotlivych zarizeni, ktera jsou v datasetu\n")
        file.write("devices:\n")
        file.write("  - example1:\n")
        file.write("      x: 12.3\n")
        file.write("      y: 32.1\n")
        file.write("\n")
        file.write("  - example2:\n")
        file.write("      x: 32.1\n")
        file.write("      y: 12.3\n")
        file.write("\n")
        file.write("# root slozka, ktera obsahuje odkazy na dataset\n")
        file.write("url: ZDE VLOZTE URL/\n")
        file.write("# volitelný parameter, který specifikuje vzor jména datasetů, které se budou stahovat\n")
        file.write("regex: ZDE VLOZTE REGEX\n")
        file.write("# volitelny parametr, ktery udava jak casto se budou hledat nove datasety, pokud prazdne, "
                   "tak defaultni hodnota (dny)\n")
        file.write("update-period: ZDE VLOZTE HODNOTU\n")


def create_default_processor(dataset_name):
    with open(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py", "w") as file:
        file.write("def process_file(filename):\n")
        file.write("    print(\"You must implements process_file method first!\")\n")


def create_default_crawler(dataset_name):

    with open(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py", "w") as file:
        file.write("def crawl(config):\n")
        file.write("    print(\"You must implements Crawl method first!\")\n")


def create_ignore_file(path,text):

    with open(path + "/ignore.txt", "w") as file:
        if text is not None:
            file.write(text + "\n")


def prepare_dataset_structure(dataset_name):
    jump_folder = "../"

    # create folder for crawled data
    try:
        path = CRAWLED_DATA_PATH+dataset_name
        os.mkdir(path)
        create_ignore_file(path,"ignore.txt")
    except os.error as e:
        print(e)
        print("Creation of the directory %s failed" % path)

    # create folder for processed data
    try:
        path = PROCESSED_DATA_PATH + dataset_name
        os.mkdir(path)
        create_ignore_file(path, "ignore.txt")
    except OSError:
        print("Creation of the directory %s failed" % path)

    # create folder for crawler logs
    try:
        path = CRAWLER_LOGS_PATH + dataset_name
        os.mkdir(path)
        create_ignore_file(path, None)
    except OSError:
        print("Creation of the directory %s failed" % path)

    create_default_crawler(dataset_name)
    create_default_processor(dataset_name)
    create_default_config_file(dataset_name)


prepare_dataset_structure("WIFI")