aswi2020sebela-gitlab/modules/crawler/Utilities/Database/DatabaseLoader.py @ d6d75a03

from Utilities.Database import DatabaseDataLine
import pymongo

# specify mongodb connection
MONGODB_CONNECTION = "mongodb://root:root@database"
# mongodb account name
MONGODB_ACC_NAME = "root"
# mongodb account password
MONGODB_ACC_PASSWORD = "root"
# mongodb data database
MONGODB_DATA_DATABASE = "open-data-db"
# mongodb collection with aviable datasets
MONGODB_DATASET_COLLECTION = "DATASETS"

# Path to processed data
PROCESSED_DATA_PATH = "ProcessedData/"


def create_database_connection():
    """
    Creates connection to mongoDB
    
    Returns:
        Connection to mongoDB
    """
    client = pymongo.MongoClient(MONGODB_CONNECTION)

    # Authenticating
    client.admin.authenticate(MONGODB_ACC_NAME, MONGODB_ACC_PASSWORD)

    database = client[MONGODB_DATA_DATABASE]

    return database


def get_data_from_file(filename, config):
    """
        Opens processed file, reads it line by line
        name, ocurrence, date
        searches name in config and adds device map coordinates
        than creates a dictionary with date without hours as key
        and list of data lines as value.
    Args:
        filename: name of processed file
        config: loaded configuration file of dataset

    Returns:
        dictionary with date without hours as key
        and list of Datalines as value
    """
    dataset_name = config["dataset-name"]
    dataset_path = PROCESSED_DATA_PATH + dataset_name + '/'

    f = open(dataset_path + filename, "r")

    devices = config["devices"]
    date_dict = dict()

    for line in f:
        line = line[:-1]

        csv_column = line.split(";")

        name = csv_column[0]

        occurrence = csv_column[1]
        date = csv_column[2]
        database_data_line = DatabaseDataLine.DatabaseDataLine(
            name, devices[name]["x"], devices[name]["y"], date, occurrence)

        # if you want to change table split by hours or months change this YYYY-mm-hh-dd
        date_without_hours = date[:-3]
        if date_without_hours not in date_dict:
            date_dict[date_without_hours] = list()

        date_dict[date_without_hours].append(
            database_data_line.to_dictionary())

    return date_dict


def load_data_to_database(dataset_name, data_dic):
    """
    Takes data_dic created in method get_data_from_file
    and loads into into database where collection name is dataset_name + data_dic key
    and data lines are line in collection

    Args:
        dataset_name: name of dataset that has existing configuration file
        data_dic: dictionary of data lines created in get_data_from_file
    """
    database = create_database_connection()

    # collection where are specified aviable datasets
    collection_datasets = database[MONGODB_DATASET_COLLECTION]

    # check if newly added data already have a dataset specified in collection
    dataset_present = collection_datasets.find_one({'name': dataset_name})

    if dataset_present is None:
        collection_datasets.insert_one({'name': dataset_name})

    for date in data_dic:
        dataset_collections = database[dataset_name]
        dataset_collections.insert_one({'name': dataset_name + date})
        date_dataset = database[dataset_name + date]
        date_dataset.insert_many(data_dic[date])