| 1 |
1187e871
|
petrh
|
import os
|
| 2 |
|
|
import shutil
|
| 3 |
|
|
|
| 4 |
|
|
# Path to crawled data
|
| 5 |
728f8c5d
|
petrh
|
CRAWLED_DATA_PATH = "CrawledData/"
|
| 6 |
1187e871
|
petrh
|
# Path to processed data
|
| 7 |
728f8c5d
|
petrh
|
PROCESSED_DATA_PATH = "ProcessedData/"
|
| 8 |
1187e871
|
petrh
|
# Path to crawler logs
|
| 9 |
728f8c5d
|
petrh
|
CRAWLER_LOGS_PATH = "CrawlerLogs/"
|
| 10 |
1187e871
|
petrh
|
# Path to dataset configuration files
|
| 11 |
728f8c5d
|
petrh
|
CONFIG_FILES_PATH = "DatasetConfigs"
|
| 12 |
1187e871
|
petrh
|
# Path for DatasetCrawlers implementations
|
| 13 |
728f8c5d
|
petrh
|
CRAWLER_PROGRAM_PATH = "DatasetCrawler"
|
| 14 |
1187e871
|
petrh
|
# Path for DatasetProcessors implementations
|
| 15 |
728f8c5d
|
petrh
|
PROCESSOR_PROGRAM_PATH = "DatasetProcessing"
|
| 16 |
1187e871
|
petrh
|
|
| 17 |
|
|
|
| 18 |
|
|
def remove_dataset(dataset_name):
|
| 19 |
|
|
"""
|
| 20 |
|
|
Remove dataset
|
| 21 |
|
|
Args:
|
| 22 |
|
|
dataset_name: name of dataset that has existing configuration file
|
| 23 |
|
|
"""
|
| 24 |
|
|
shutil.rmtree(CRAWLED_DATA_PATH + dataset_name + "/")
|
| 25 |
|
|
shutil.rmtree(PROCESSED_DATA_PATH + dataset_name + "/")
|
| 26 |
|
|
shutil.rmtree(CRAWLER_LOGS_PATH + dataset_name + "/")
|
| 27 |
|
|
|
| 28 |
|
|
os.remove(CONFIG_FILES_PATH + "/" + dataset_name + ".yaml")
|
| 29 |
|
|
os.remove(CRAWLER_PROGRAM_PATH + "/" + dataset_name + "Crawler.py")
|
| 30 |
|
|
os.remove(PROCESSOR_PROGRAM_PATH + "/" + dataset_name + "Processor.py")
|
| 31 |
|
|
|
| 32 |
728f8c5d
|
petrh
|
print("Dataset: " + dataset_name + " removed")
|
| 33 |
1187e871
|
petrh
|
|
| 34 |
728f8c5d
|
petrh
|
remove_dataset("TEST")
|