Revize 34baf808
Přidáno uživatelem Petr Hlaváč před asi 4 roky(ů)
modules/crawler/ResetDataset.py | ||
---|---|---|
2 | 2 |
from Utilities import FolderProcessor |
3 | 3 |
|
4 | 4 |
# Path to crawled data |
5 |
CRAWLED_DATA_PATH = "../CrawledData/"
|
|
5 |
CRAWLED_DATA_PATH = "CrawledData/" |
|
6 | 6 |
# Path to processed data |
7 |
PROCESSED_DATA_PATH = "../ProcessedData/"
|
|
7 |
PROCESSED_DATA_PATH = "ProcessedData/" |
|
8 | 8 |
# Path to crawler logs |
9 |
CRAWLER_LOGS_PATH = "../CrawlerLogs/"
|
|
9 |
CRAWLER_LOGS_PATH = "CrawlerLogs/" |
|
10 | 10 |
# Path to dataset configuration files |
11 |
CONFIG_FILES_PATH = "../DatasetConfigs"
|
|
11 |
CONFIG_FILES_PATH = "DatasetConfigs" |
|
12 | 12 |
|
13 | 13 |
|
14 | 14 |
def create_ignore_file(path, text): |
... | ... | |
23 | 23 |
file.write(text + "\n") |
24 | 24 |
|
25 | 25 |
|
26 |
def create_updated_file(path): |
|
27 |
""" |
|
28 |
Creates updated file |
|
29 |
Args: |
|
30 |
path: path to directory for creating updated.txt |
|
31 |
""" |
|
32 |
with open(path + "/updated.txt", "w") as file: |
|
33 |
file.write(str(0) + "\n") |
|
34 |
|
|
35 |
|
|
26 | 36 |
def reset_dataset(dataset_name): |
27 | 37 |
""" |
28 | 38 |
Resets all saved data in dataset except config and implementation |
... | ... | |
40 | 50 |
path = CRAWLER_LOGS_PATH + dataset_name + "/" |
41 | 51 |
FolderProcessor.clean_folder(path) |
42 | 52 |
create_ignore_file(path, None) |
53 |
create_updated_file(path) |
|
43 | 54 |
|
44 | 55 |
|
45 | 56 |
def reset_all_datasets(): |
Také k dispozici: Unified diff
Re #7965
implementovat skript pro crone který spouští upadte datasetu podle configu