1
|
from Utilities.CSV import csv_data_line
|
2
|
from Utilities import date_formating
|
3
|
import logging
|
4
|
from datetime import date
|
5
|
import time
|
6
|
import datetime
|
7
|
|
8
|
logging.basicConfig(filename='../../CrawlerLogs' + 'Crawlerlog-' +
|
9
|
date.today().strftime("%b-%Y") + '.log',
|
10
|
level=logging.INFO,
|
11
|
format='%(asctime)s %(message)s')
|
12
|
|
13
|
|
14
|
def process_file(filename):
|
15
|
"""
|
16
|
Method that take path to crawled file and outputs date dictionary:
|
17
|
Date dictionary is a dictionary where keys are dates in format YYYY-mm-dd-hh (2018-04-08-15)
|
18
|
and value is dictionary where keys are devices (specified in configuration file)
|
19
|
and value is CSVDataLine.csv_data_line with device,date and occurrence
|
20
|
|
21
|
Args:
|
22
|
filename: name of processed file
|
23
|
|
24
|
Returns:
|
25
|
None if not implemented
|
26
|
date_dict when implemented
|
27
|
"""
|
28
|
date_dict = dict()
|
29
|
|
30
|
with open(filename, "r") as file:
|
31
|
|
32
|
YEAR_START = 1
|
33
|
YEAR_END = 11
|
34
|
for line in file:
|
35
|
|
36
|
array = line.split(";")
|
37
|
|
38
|
#pick later time
|
39
|
time_ = max(
|
40
|
array[2][1:-1],
|
41
|
array[3][1:-1],
|
42
|
key=lambda x: time.mktime(
|
43
|
datetime.datetime.strptime(x, "%H:%M").timetuple()))
|
44
|
|
45
|
date = date_formating.date_time_formatter(
|
46
|
array[14][YEAR_START:YEAR_END] + " " + time_)
|
47
|
|
48
|
name = array[10][1:-1]
|
49
|
if name == "":
|
50
|
continue
|
51
|
|
52
|
if date not in date_dict:
|
53
|
date_dict[date] = {}
|
54
|
|
55
|
if name in date_dict[date]:
|
56
|
date_dict[date][name].occurrence = int(array[12])
|
57
|
else:
|
58
|
date_dict[date][name] = csv_data_line.CSVDataLine(
|
59
|
name, date, int(array[12]))
|
60
|
|
61
|
return date_dict
|