Revize 0e7b6b11

Re #9049 Implementation of GetDataFileNames and Load methods

Server/ServerApp/Predictor/FeatureExtractor.cs

@@ -17 +17 @@
     /// </summary>
     class FeatureExtractor
     {
+
+        private const double Confidence = 99.0;
+
         private static readonly ILog _log = LogManager.GetLogger(typeof(FeatureExtractor));
 
         /// <summary>
@@ -90 +93 @@
         {
             MLContext mlContext = new MLContext();
             IDataView input = mlContext.Data.LoadFromEnumerable(data);
-            var pipeline = mlContext.Transforms.Conversion.ConvertType(nameof(ActivityInfo.amount)).Append(mlContext.Transforms.DetectIidSpike(nameof(AnomalyDetectionResult.Prediction), nameof(ActivityInfo.amount), 99.0, data.Count / 4));
+            var pipeline = mlContext.Transforms.Conversion.ConvertType(nameof(ActivityInfo.amount)).Append(mlContext.Transforms.DetectIidSpike(nameof(AnomalyDetectionResult.Prediction), nameof(ActivityInfo.amount), Confidence, data.Count / 4));
             ITransformer transformer = pipeline.Fit(mlContext.Data.LoadFromEnumerable(new List<ActivityInfo>()));
             IDataView transformedData = transformer.Transform(input);
             List<AnomalyDetectionResult> predictions = mlContext.Data.CreateEnumerable<AnomalyDetectionResult>(transformedData, false).ToList();
@@ -98 +101 @@
 
             for (int i=0; i<predictions.Count; i++)
             {
-                if(predictions[i].Prediction[0] == 1)
+                if(predictions[i].Prediction[2] < (1 - Confidence))
                 {
                     _log.Debug($"Rejecting an outlier activity: {predictions[i].Prediction[1]}, p-value: {predictions[i].Prediction[2]}, from: {data[i].startTime}");
                 } else

Server/ServerApp/Predictor/IPredictionController.cs

@@ -20 +20 @@
         /// <summary>
         /// Trains all predictors. The training data are taken from given path location.
         /// </summary>
-        /// <param name="locationKey">A string identifier of the location for which to train a predictor.</param>
-        void Train(string locationKey = null);
+        void Train();
 
         /// <summary>
         /// Loads trained predictors from files in <c>path</c>.
         /// </summary>
-        /// <param name="locationKey">A string identifier of the location for which to load a predictor.</param>
         /// <param name="path">A path to folder with trained prediction model.</param>
-        void Load(string locationKey = null, string path = null);
+        void Load(string path);
+
+        /// <summary>
+        /// Rolls back the predictors to previous trained instances.
+        /// </summary>
+        void Rollback();
 
         /// <summary>
         /// Predicts turnout level at given time supposing given weather conditions.
@@ -36 +39 @@
         /// <param name="request">A request with time and weather information.</param>
         /// <returns>A server response filled with predictions.</returns>
         Response Predict(Request request);
+
+        /// <summary>
+        /// Returns the filenames of data files used to train current predictor instances
+        /// </summary>
+        /// <returns>Collection of filenames</returns>
+        IEnumerable<string> GetDataFileNames();
     }
 }

Server/ServerApp/Predictor/NaiveBayesClassifier.cs

@@ -39 +39 @@
             _mlContext = new MLContext();
         }
 
+        public NaiveBayesClassifier(string filename) : this()
+        {
+            DataViewSchema modelSchema;
+            this._trainedModel = _mlContext.Model.Load(filename, out modelSchema);
+            // TODO check if the loaded model has valid input and output schema
+            this._predictionEngine = _mlContext.Model.CreatePredictionEngine<ModelInput, ModelOutput>(this._trainedModel);
+        }
+
         public void Fit(IEnumerable<ModelInput> trainInput)
         {
             this._trainingDataView = _mlContext.Data.LoadFromEnumerable(trainInput);

Server/ServerApp/Predictor/PredictionController.cs

@@ -9 +9 @@
 using Newtonsoft.Json;
 using ServerApp.WeatherPredictionParser;
 using ServerApp.Parser.OutputInfo;
-using System.Reflection;
 using log4net;
+using System.IO;
+using System.Text.RegularExpressions;
+using System.Linq;
 
 namespace ServerApp.Predictor
 {
@@ -21 +23 @@
     {
         private static readonly ILog _log = LogManager.GetLogger(typeof(PredictionController));
 
+        /// <summary>
+        /// ID of the current predictor instances.
+        /// </summary>
+        private string PredictorID;
+
         /// <summary>
         /// Configuration of the <c>Predictor</c>
         /// </summary>
-        private PredictorConfiguration Configuration;
+        public PredictorConfiguration Configuration { get; set; }
+
+        /// <summary>
+        /// Names of the files used to train the current predictor instances.
+        /// </summary>
+        private IEnumerable<string> DataFilenames;
 
-        private List<IPredictor> Predictors;
+        /// <summary>
+        /// Current predictor instances
+        /// </summary>
+        private IPredictor[] Predictors;
 
         /// <summary>
         /// A reference to a data parser.
@@ -49 +64 @@
         /// <param name="dataParser">A data parser used to get training data.</param>
         public PredictionController(IJsonParser weatherService, IDataParser dataParser, string pathToConfig = null)
         {
+            // TODO look for existing predictors
             _log.Info("Constructing a new PredictionController instance.");
             this.weatherService = weatherService;
             // load config or get the default one
@@ -58 +74 @@
             }
             try
             {
-                string json = System.IO.File.ReadAllText(pathToConfig);
+                string json = File.ReadAllText(pathToConfig);
                 this.Configuration = JsonConvert.DeserializeObject<PredictorConfiguration>(json);
             }
             catch (System.IO.IOException e)
@@ -69 +85 @@
             }
 
             this.DataParser = dataParser;
-            this.Predictors = new List<IPredictor>();
+            this.Predictors = new IPredictor[this.Configuration.PredictorCount];
             this.FeatureExtractor = new FeatureExtractor(this.DataParser, this.Configuration);
+            this.DataFilenames = dataParser.WeatherDataUsed.Concat(dataParser.ActivityDataUsed);
+            this.PredictorID = DateTime.Now.ToString();
 
             for (int i = 0; i < this.Configuration.PredictorCount; i++)
             {
-                Predictors.Add(new NaiveBayesClassifier());
+                Predictors[i] = new NaiveBayesClassifier();
             }
             PredictorConfiguration.SaveConfig(PredictorConfiguration.DEFAULT_CONFIG_PATH, Configuration);
         }
@@ -83 +101 @@
             return new List<string>(this.Configuration.BuildingsToAreas.Keys);
         }
 
-        public void Load(string locationKey = null, string path = null)
+        public void Load(string predictorID)
         {
-            if (locationKey is null)
-            {
-                throw new NotImplementedException();
-            }
-            else
+            DirectoryInfo di = new DirectoryInfo(Configuration.ModelDataPath);
+            FileInfo[] files = di.GetFiles($"{predictorID}_*.zip");
+            IPredictor[] newPredictors = new IPredictor[this.Configuration.PredictorCount];
+            for (int i = 0; i < this.Configuration.PredictorCount; i++)
             {
-                throw new NotImplementedException();
+                newPredictors[i] = new NaiveBayesClassifier(Array.Find(files, f => Regex.IsMatch(f.Name, $"{predictorID}_{i}.zip")).FullName);
             }
+            this.Predictors = newPredictors;
+            files = di.GetFiles($"{predictorID}.txt");
+            this.DataFilenames = File.ReadLines(files[0].FullName);
+            this.PredictorID = predictorID;
+        }
+
+        public void Rollback()
+        {
+
         }
 
         public Response Predict(Request request)
@@ -132 +158 @@
         private Prediction PredictSingle(Request request, DateTime predictionTime)
         {
             double[] predictedValues = new double[this.Configuration.BuildingsToAreas.Count];
-            string[] predictedLabels = new string[this.Predictors.Count];
-            for (int i = 0; i < this.Predictors.Count; i++)
+            string[] predictedLabels = new string[this.Predictors.Length];
+            for (int i = 0; i < this.Predictors.Length; i++)
             {
                 if (request.useWeather)
                 {
@@ -187 +213 @@
             return prediction;
         }
 
-        public void Train(string locationKey = null)
+        public void Train()
         {
-            if (locationKey is null)
-            // train all predictors
-            {
-                DataParser.Parse(DateTime.MinValue, DateTime.MaxValue, this.Configuration.TimeResolution, wholeDay: false);
-                for (int i = 0; i < this.Predictors.Count; i++)
+            DataParser.Parse(DateTime.MinValue, DateTime.MaxValue, this.Configuration.TimeResolution, wholeDay: false);
+                for (int i = 0; i < this.Predictors.Length; i++)
                 {
                     // train on all available data
                     List<ModelInput> data = FeatureExtractor.PrepareTrainingInput(i);
                     Console.WriteLine("Training predictor with {0} samples.", data.Count);
                     this.Predictors[i].Fit(data);
                 }
-            }
-            else
-            // train specified predictor only
-            {
-                throw new NotImplementedException();
-            }
         }
 
-
+        public IEnumerable<string> GetDataFileNames()
+        {
+            return this.DataFilenames;
+        }
     }
 }

Server/ServerApp/Predictor/PredictorConfiguration.cs

@@ -9 +9 @@
 
 namespace ServerApp.Predictor
 {
-    class PredictorConfiguration
+    public class PredictorConfiguration
     {
-        public static readonly string DEFAULT_CONFIG_PATH = Path.GetFullPath(Path.GetDirectoryName(Environment.CurrentDirectory) + @"\Predictor.config");
+        public static readonly string DEFAULT_CONFIG_PATH = Path.Combine(Path.GetDirectoryName(Environment.CurrentDirectory), "Predictor.config");
+
+        public static readonly string DEFAULT_MODEL_DATA_PATH = Path.Combine(Path.GetDirectoryName(Environment.CurrentDirectory), "model_data");
 
         public int TimeResolution { get; set; }
 
@@ -19 +21 @@
 
         public int PredictorCount { get; set; }
 
+        public string ModelDataPath { get; set; }
+
         public static PredictorConfiguration LoadConfig(string filename)
         {
             string json = System.IO.File.ReadAllText(filename);
@@ -44 +48 @@
             return new PredictorConfiguration
             {
                 TimeResolution = 3,
+                ModelDataPath = DEFAULT_MODEL_DATA_PATH,
                 PredictorCount = 7,
                 BuildingsToAreas = new Dictionary<string, int>
                 {