aswi2020sebela-gitlab/modules/crawler/Utilities/FolderProcessor.py @ 0a2832fb

import os
import zipfile
from Utilities.CSV import CSVutils


def list_of_all_files(path):
    """
    Get all files from directory and all files written in ignore.txt
    and return the difference
    Args:
        path: path to Directory

    Returns:
        list with names of all files in directory
    """
    files_in_dir = os.listdir(path)

    ignore_set = load_ignore_set(path)

    return set(files_in_dir).difference(ignore_set)


def load_ignore_set(path):
    """
    Reads ignore.txt line by line and add it to a set
    Args:
        path: Path to directory containing ignore.txt file

    Returns:
        list of names contained in ignore.txt file
    """
    ignore_set = set()

    with open(path + "ignore.txt", "r") as file:

        for line in file:
            ignore_set.add(line[:-1])

    return ignore_set


def update_ignore_set(path, file_name):
    """
    Adds file_name to the ignore file
    Args:
        path: Path to directory containing ignore.txt file
        file_name: name of file you want to add to ignore file
    """
    with open(path + "ignore.txt", "a") as file:
        file.write(file_name + '\n')


def get_devices_set(path):
    """
     Goes trough every not loaded file(not contained in ProcessedData/ignore.txt)
     Extracts names from not loaded file which should be in first column
     Creates set of unique devices_names

    Args:
        path: Path to Processed directory containing ignore.txt file

    Returns:
        set of unique names contained in not loaded files
    """
    files_in_dir = list_of_all_files(path)

    unique_names = set()

    for file_path in files_in_dir:
        unique_names.update(CSVutils.get_unique_names_from_file(path+file_path, 0))

    return unique_names


def get_unknown_devices_set(config, devices):
    """
    Compares config and devices a return difference

    Args:
        config:  loaded configuration file of dataset
        devices: set of unique devices contained in dataset

    Returns:

    """
    devices_set = set(config["devices"].keys())
    unknown_devices_set = devices.difference(devices_set)

    return unknown_devices_set


def unzip_all_csv_zip_files_in_folder(path):
    """
    Load all files from directory and unzip those which end by .zip
    After unziping deletes the zip file
    Args:
        path: Path to CrawledData directory containing ignore.txt file
    """
    files_in_dir = os.listdir(path)
    zips = []

    for file in files_in_dir:
        if file.endswith(".zip"):
            zips.append(path + file)

    for zip_file in zips:

        with zipfile.ZipFile(zip_file, "r") as unziped_file:
            unziped_file.extractall(path)

        os.remove(zip_file)


def clean_folder(path):
    """
    Deletes all files in folder

    Args:
        path: path to folder
    """
    files = os.listdir(path)

    for file in files:
        os.remove(path+file)