aswi2021tri-musketyri-gitlab/Server/ServerApp/Predictor/PredictionController.cs @ c45404fb

//
// Author: Roman Kalivoda
//

using System;
using System.Collections.Generic;
using ServerApp.Connection.XMLProtocolHandler;
using ServerApp.Parser.Parsers;
using Newtonsoft.Json;
using ServerApp.WeatherPredictionParser;
using ServerApp.Parser.OutputInfo;
using log4net;
using System.IO;
using System.Text.RegularExpressions;
using System.Linq;

namespace ServerApp.Predictor
{
    /// <summary>
    /// Implentation of the <c>IPredicitionController</c> interface.
    /// </summary>
    public class PredictionController : IPredictionController
    {
        private static readonly ILog _log = LogManager.GetLogger(typeof(PredictionController));

        /// <summary>
        /// ID of the current predictor instances.
        /// </summary>
        private string PredictorID;

        /// <summary>
        /// Configuration of the <c>Predictor</c>
        /// </summary>
        public PredictorConfiguration Configuration { get; set; }

        /// <summary>
        /// Names of the files used to train the current predictor instances.
        /// </summary>
        private IEnumerable<string> DataFilenames;

        /// <summary>
        /// Current predictor instances
        /// </summary>
        private IPredictor[] Predictors;

        /// <summary>
        /// A mutual exclusive lock to pro
        /// </summary>
        private readonly object predictorsLock = new Object();

        /// <summary>
        /// A reference to a data parser.
        /// </summary>
        private IDataParser DataParser;

        /// <summary>
        /// A feature extractor instance.
        /// </summary>
        private FeatureExtractor FeatureExtractor;

        /// <summary>
        /// A weather prediction parser service
        /// </summary>
        private IJsonParser weatherService;

        /// <summary>
        /// Instantiates new prediction controller.
        /// </summary>
        /// <param name="dataParser">A data parser used to get training data.</param>
        public PredictionController(IJsonParser weatherService, IDataParser dataParser, string pathToConfig = null)
        {
            _log.Info("Constructing a new PredictionController instance.");
            this.weatherService = weatherService;
            // load config or get the default one
            if (pathToConfig is null)
            {
                pathToConfig = PredictorConfiguration.DEFAULT_CONFIG_PATH;
            }
            try
            {
                string json = File.ReadAllText(pathToConfig);
                this.Configuration = JsonConvert.DeserializeObject<PredictorConfiguration>(json);
            }
            catch (System.IO.FileNotFoundException e)
            {
                Console.WriteLine("Warning: could not find a configuration file, creating a new one:");
                Console.WriteLine(e.Message.PadLeft(4));
                this.Configuration = PredictorConfiguration.GetDefaultConfig();
            }

            this.DataParser = dataParser;
            var predictors = new IPredictor[this.Configuration.PredictorCount];
            this.FeatureExtractor = new FeatureExtractor(this.DataParser, this.Configuration);

            DirectoryInfo di = new DirectoryInfo(Configuration.ModelDataPath);
            _log.Debug($"Looking for directory: {Configuration.ModelDataPath}");
            if (! di.Exists)
            {
                _log.Info($"Creating the model data directory: {Configuration.ModelDataPath}");
                di.Create();
            }
            FileInfo[] files = di.GetFiles();
            var orderedEnumerable = Array.FindAll(files, f => Regex.IsMatch(f.Name, @"[-]?\d+_\d+.zip")).GroupBy(f => f.Name.Split("_.".ToCharArray())[0]).OrderBy(f => DateTime.FromBinary(Convert.ToInt64(f.Key)));
            if (orderedEnumerable.Where(g => g.Count() == Configuration.PredictorCount).Any())
            {
                _log.Info("Found existing predictors, loading the newest.");
                var predictorID = orderedEnumerable.Last().Select(f => f.Name.Split("_".ToCharArray())[0]).First();
                this.Load(predictorID);
            }
            else
            {
                _log.Info("No predictors found, creating new ones");
                for (int i = 0; i < this.Configuration.PredictorCount; i++)
                {
                    predictors[i] = new SdcaMaximumEntropyClassifier();
                }
                lock (predictorsLock)
                {
                    this.Predictors = predictors;
                }
            }
            
            PredictorConfiguration.SaveConfig(PredictorConfiguration.DEFAULT_CONFIG_PATH, Configuration);
        }
        public List<string> GetPredictors()
        {
            return new List<string>(this.Configuration.BuildingsToAreas.Keys);
        }

        public int Load(string predictorID)
        {
            DirectoryInfo di = new DirectoryInfo(Configuration.ModelDataPath); 
            if (!di.Exists)
            {
                _log.Warn("The model data directory could not be found.");
                return 2;
            }
            FileInfo[] files = di.GetFiles($"{predictorID}_*.zip");
            if (Array.FindAll(files, f => Regex.IsMatch(f.Name, $@"{predictorID}_\d+.zip")).Any()){
                IPredictor[] newPredictors = new IPredictor[this.Configuration.PredictorCount];
                try
                {
                    for (int i = 0; i < this.Configuration.PredictorCount; i++)
                    {
                        newPredictors[i] = new SdcaMaximumEntropyClassifier(Array.Find(files, f => Regex.IsMatch(f.Name, $"{predictorID}_{i}.zip")).FullName);
                    }
                    files = di.GetFiles($"{predictorID}.txt");
                    var dataFilenames = File.ReadLines(files[0].FullName); 
                    lock (predictorsLock)
                    {
                        this.Predictors = newPredictors;
                        this.DataFilenames = dataFilenames;
                        this.PredictorID = predictorID;
                    }
                } catch (FileNotFoundException e)
                {
                    _log.Error(e.ToString());
                    return 2;
                }
            } else
            {
                _log.Debug("Could not find predictor with given predictorID");
                return 1;
            }
            return 0;
        }

        public void Save()
        {
            DirectoryInfo di = new DirectoryInfo(Configuration.ModelDataPath);
            if (!di.Exists)
            {
                _log.Warn("The model data directory could not be found.");
                return;
            }

            lock (predictorsLock)
            {
                for (int i = 0; i < this.Configuration.PredictorCount; i++)
                {
                    Predictors[i].Save(Path.Combine(di.FullName, $"{PredictorID}_{i}.zip"));
                }
                File.WriteAllLinesAsync(Path.Combine(di.FullName, $"{PredictorID}.txt"), this.DataFilenames);
            }
        }

        public int Rollback()
        {
            DirectoryInfo di = new DirectoryInfo(Configuration.ModelDataPath);
            if (!di.Exists)
            {
                _log.Warn("The model data directory could not be found.");
                return 2;
            }
            FileInfo[] files = di.GetFiles();
            var orderedEnumerable = Array.FindAll(files, f => Regex.IsMatch(f.Name, @"[-]?\d+_\d+.zip")).GroupBy(f => f.Name.Split("_.".ToCharArray())[0]).OrderBy(f => DateTime.FromBinary(Convert.ToInt64(f.Key))).Where(f => ! f.Key.Equals(this.PredictorID));
            if (orderedEnumerable.Where(g => g.Count() == Configuration.PredictorCount).Any())
            {
                string RollbackedPredictorID = orderedEnumerable.Last().Select(f => f.Name.Split("_".ToCharArray())[0]).First();
                this.Delete(this.PredictorID);
                return this.Load(RollbackedPredictorID);
            } else
            {
                // indicate that older model does not exist
                return 1;
            }
        }

        private void Delete(string predictorID)
        {
            DirectoryInfo di = new DirectoryInfo(Configuration.ModelDataPath);
            if (!di.Exists)
            {
                _log.Warn("The model data directory could not be found.");
                return;
            }

            for (int i = 0; i < this.Configuration.PredictorCount; i++)
            {
                File.Delete(Path.Combine(di.FullName, $"{PredictorID}_{i}.zip"));
            }
            File.Delete(Path.Combine(di.FullName, $"{PredictorID}.txt"));
        }

        public Response Predict(Request request)
        {
            _log.Info($"Received a prediction request: endDate={request.useEndDate}, weather={request.useWeather}");
            DateTime start = new DateTime(year: request.start.year, month: request.start.month, day: request.start.day, hour: request.start.hour, minute: 0, second: 0);
            List<Prediction> predictions = new List<Prediction>();
            if (request.useEndDate)
            {
                DateTime end = new DateTime(year: request.end.year, month: request.end.month, day: request.end.day, hour: request.end.hour, minute: 0, second: 0);
                DateTime current = start;
                while (current <= end)
                {
                    _log.Debug($"Predicting for date {current.Date.ToShortDateString()}");
                    while (current.Hour <= Date.MAX_HOUR)
                    {
                        if (current.Date == end.Date && current.Hour > end.Hour)
                        {
                            break;  // break if the end date was reached and hour is after the end time.
                        }
                        _log.Debug($"Predicting for time {current.TimeOfDay.ToString()}");
                        var prediction = PredictSingle(request, current);
                        predictions.Add(prediction);
                        current = current.AddHours(this.Configuration.TimeResolution);
                    }
                    current = current.AddHours(23 - current.Hour + Date.MIN_HOUR);
                }
            }
            else
            {
                _log.Debug("Predicting for single DateTime.");
                predictions.Add(PredictSingle(request, start));
            }
            var response = new Response();
            response.hoursPerSegment = Configuration.TimeResolution;
            response.predicitons = predictions.ToArray();
            _log.Debug($"Created a response.");
            return response;
        }

        private Prediction PredictSingle(Request request, DateTime predictionTime)
        {
            double[] predictedValues = new double[this.Configuration.BuildingsToAreas.Count];
            string[] predictedLabels = new string[this.Predictors.Length];
            for (int i = 0; i < this.Predictors.Length; i++)
            {
                if (request.useWeather)
                {
                    _log.Debug("Predicting for requested weather.");
                    lock (predictorsLock)
                    {
                        predictedLabels[i] = this.Predictors[i].Predict(new ModelInput
                        {
                            Rain = (float)request.rain,
                            Temp = (float)request.temperature,
                            Wind = (float)request.wind,
                            Hour = predictionTime.Hour,
                            Time = predictionTime
                        });
                    }
                }
                else
                {
                    _log.Debug("Retrieving weather info from the weather service.");
                    weatherService.ParsePrediction();
                    WeatherInfo weatherInfo = weatherService.Predictions.Find(info => info.startTime.Date.Equals(predictionTime.Date) && predictionTime.TimeOfDay.Subtract(info.startTime.TimeOfDay).Hours < info.intervalLength);
                    if (weatherInfo is null)
                    {
                        predictedLabels[i] = null;
                    }
                    else
                    {
                        lock (predictorsLock)
                        {
                            predictedLabels[i] = this.Predictors[i].Predict(new ModelInput
                            {
                                Rain = weatherInfo.rain,
                                Temp = (float)weatherInfo.temp,
                                Wind = (float)weatherInfo.wind,
                                Hour = predictionTime.Hour,
                                Time = predictionTime
                            });
                        }
                    }
                }
            }
            for (int i = 0; i < predictedValues.Length; i++)
            {
                predictedValues[i] = this.FeatureExtractor.LabelToRatio(predictedLabels[this.Configuration.BuildingsToAreas[TagInfo.buildings[i]]]);
            }

            Prediction prediction = new Prediction();
            prediction.dateTime = new Date
            {
                year = predictionTime.Year,
                month = predictionTime.Month,
                day = predictionTime.Day,
                hour = predictionTime.Hour
            };
            prediction.predictions = predictedValues;
            _log.Debug($"Created prediction for DateTime: {prediction.dateTime}");
            return prediction;
        }

        public void Train()
        {
            DataParser.Parse(DateTime.MinValue, DateTime.MaxValue, this.Configuration.TimeResolution, wholeDay: false);
            for (int i = 0; i < this.Configuration.PredictorCount; i++)
            {
                // train on all available data
                List<ModelInput> data = FeatureExtractor.PrepareTrainingInput(i);
                Console.WriteLine("Training predictor with {0} samples.", data.Count);
                lock (predictorsLock)
                {
                    this.Predictors[i].Fit(data);
                }
            }
            lock (predictorsLock)
            {
                this.DataFilenames = this.DataParser.WeatherDataUsed.Concat(this.DataParser.ActivityDataUsed);
                this.PredictorID = DateTime.Now.ToBinary().ToString();
            }
            this.Save();
        }

        public IEnumerable<string> GetDataFileNames()
        {
            return this.DataFilenames;
        }
    }
}