1 |
1187e871
|
petrh
|
import os
|
2 |
|
|
import shutil
|
3 |
|
|
|
4 |
|
|
# Path to crawled data
|
5 |
|
|
CRAWLED_DATA_PATH = "../CrawledData/"
|
6 |
|
|
# Path to processed data
|
7 |
|
|
PROCESSED_DATA_PATH = "../ProcessedData/"
|
8 |
|
|
# Path to crawler logs
|
9 |
|
|
CRAWLER_LOGS_PATH = "../CrawlerLogs/"
|
10 |
|
|
# Path to dataset configuration files
|
11 |
|
|
CONFIG_FILES_PATH = "../DatasetConfigs"
|
12 |
|
|
# Path for DatasetCrawlers implementations
|
13 |
|
|
CRAWLER_PROGRAM_PATH = "../DatasetCrawler"
|
14 |
|
|
# Path for DatasetProcessors implementations
|
15 |
|
|
PROCESSOR_PROGRAM_PATH = "../DatasetProcessing"
|
16 |
|
|
|
17 |
|
|
|
18 |
|
|
def remove_dataset(dataset_name):
|
19 |
|
|
"""
|
20 |
|
|
Remove dataset
|
21 |
|
|
Args:
|
22 |
|
|
dataset_name: name of dataset that has existing configuration file
|
23 |
|
|
"""
|
24 |
|
|
shutil.rmtree(CRAWLED_DATA_PATH + dataset_name + "/")
|
25 |
|
|
shutil.rmtree(PROCESSED_DATA_PATH + dataset_name + "/")
|
26 |
|
|
shutil.rmtree(CRAWLER_LOGS_PATH + dataset_name + "/")
|
27 |
|
|
|
28 |
|
|
os.remove(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml")
|
29 |
|
|
os.remove(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py")
|
30 |
|
|
os.remove(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py")
|
31 |
|
|
|
32 |
|
|
print("Dataset: " + dataset_name + " removed");
|
33 |
|
|
|
34 |
|
|
remove_dataset("TEST");
|