aswi2020sebela-gitlab/modules/crawler/Utilities/Database/DatabaseLoader.py @ ce378142

from Utilities.Database import DatabaseDataLine
import pymongo

# specify mongodb connection
MONGODB_CONNECTION = "mongodb://root:root@database"
# mongodb account name
MONGODB_ACC_NAME = "root"
# mongodb account password
MONGODB_ACC_PASSWORD = "root"
# mongodb data database
MONGODB_DATA_DATABASE = "open-data-db"
# mongodb collection with aviable datasets
MONGODB_DATASET_COLLECTION = "DATASETS"
# mongodb collection with aviable diveces of datasets
MONGODB_DATASET_DEVICES_COLLECTION = "DEVICES"
# mongodb collection with aviable diveces of datasets
MONGODB_DATASET_LOADED_FILES_COLLECTION = "FILES"

# Path to processed data
PROCESSED_DATA_PATH = "ProcessedData/"


def create_database_connection():
    """
    Creates connection to mongoDB
    
    Returns:
        Connection to mongoDB
    """
    client = pymongo.MongoClient(MONGODB_CONNECTION)

    # Authenticating
    client.admin.authenticate(MONGODB_ACC_NAME, MONGODB_ACC_PASSWORD)

    database = client[MONGODB_DATA_DATABASE]

    return database


def get_data_from_file(filename, config):
    """
        Opens processed file, reads it line by line
        name, ocurrence, date
        searches name in config and adds device map coordinates
        than creates a dictionary with date without hours as key
        and list of data lines as value.
    Args:
        filename: name of processed file
        config: loaded configuration file of dataset

    Returns:
        dictionary with date without hours as key
        and list of Datalines as value
    """
    dataset_name = config["dataset-name"]
    dataset_path = PROCESSED_DATA_PATH + dataset_name + '/'

    f = open(dataset_path + filename, "r")

    devices = config["devices"]
    date_dict = dict()

    for line in f:
        line = line[:-1]

        csv_column = line.split(";")

        name = csv_column[0]

        if devices[name]["x"] == "SKIP" or devices[name]["y"] == "SKIP":
            continue

        occurrence = csv_column[1]
        date = csv_column[2]
        database_data_line = DatabaseDataLine.DatabaseDataLine(
            name, devices[name]["x"], devices[name]["y"], date, occurrence)

        # if you want to change table split by hours or months change this YYYY-mm-hh-dd
        date_without_hours = date[:-3]
        if date_without_hours not in date_dict:
            date_dict[date_without_hours] = list()

        date_dict[date_without_hours].append(
            database_data_line.to_dictionary())

    return date_dict


def load_data_to_database(database_connection,dataset_name, data_dic, file_name):
    """
    Takes data_dic created in method get_data_from_file
    and loads into into database where collection name is dataset_name + data_dic key
    and data lines are line in collection

    Args:
        database_connection: created connection to a MONGODB
        config: loaded configuration file of dataset
        data_dic: dictionary of data lines created in get_data_from_file
        file_name: name of file containing data
    """

    for date in data_dic:
        dataset_collections = database_connection[dataset_name]
        dataset_collections.insert_one({'date': date})
        date_dataset = database_connection[dataset_name + date]
        date_dataset.insert_many(data_dic[date])

    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]
    collection_loaded_files.insert_one({'file': file_name})



def check_or_update_datasets_collection(database_connection,config):
    """
    Checks if DATASETS collection contains dataset and if display name was not updated

    Args:
        database_connection: created connection to a MONGODB
        config: loaded configuration file of dataset
    """
    # collection where are specified aviable datasets
    collection_datasets = database_connection[MONGODB_DATASET_COLLECTION]

    dataset_name = config['dataset-name']
    display_name = config['display-name']

    query = {'key-name': dataset_name}

    # check if newly added data already have a dataset specified in collection
    dataset_present = collection_datasets.find_one(query)

    if dataset_present is None:
        collection_datasets.insert_one({'key-name': dataset_name, 'display-name': display_name})
    elif dataset_present['display-name'] != display_name:
        newvalues = { "$set": { 'display-name': display_name } }
        collection_datasets.update_one(query, newvalues)


def update_devices_collection(database_connection,config):
    """
    Checks if dataset_name collection contains every device with current set up

    Args:
        database_connection: created connection to a MONGODB
        config: loaded configuration file of dataset
    """
    dataset_name = config['dataset-name']

    collection_devices = database_connection[dataset_name + MONGODB_DATASET_DEVICES_COLLECTION]

    collection_devices.delete_many({})

    devices = config['devices']

    devices_list = list()

    for device in devices.keys():
        if devices[device]['x'] != "SKIP" or devices[device]['y'] != "SKIP":
            devices_list.append({'name': device , 'x': devices[device]['x'] , 'y': devices[device]['y'] })

    collection_devices.insert_many(devices_list)


def check_if_database_doesnt_contain_file(database_connection,dataset_name,file_name):
    """
    Checks if dataset_name collection contains every device with current set up

    Args:
        database_connection: created connection to a MONGODB
        filename: checked file name
    """

    collection_loaded_files = database_connection[dataset_name + MONGODB_DATASET_LOADED_FILES_COLLECTION]

    query = {'file': file_name}

    # check if newly added data already have a dataset specified in collection
    dataset_present = collection_loaded_files.find_one(query)

    if dataset_present is None:
        return True
    else:
        return False