Revize cdeee9f8

Re #8955 implemented multiple predictors support

Server/ServerApp/Predictor/FeatureExtractor.cs

@@ -13 +13 @@
     /// <summary>
     /// A class responsible for preparation of features for classifiers.
     /// </summary>
-    public class FeatureExtractor
+    class FeatureExtractor
     {
         /// <summary>
         /// A DataParser instance used to access info objects.
         /// </summary>
-        private readonly IDataParser dataParser;
+        private readonly IDataParser DataParser;
 
-        private Dictionary<string, int> buildingsToAreas;
+        /// <summary>
+        /// A configuration object of the <c>Predictor</c> package
+        /// </summary>
+        private PredictorConfiguration Configuration;
 
         /// <summary>
         /// Instantiates new FeatureExtractor class.
         /// </summary>
         /// <param name="dataParser">Data parser used to access training data.</param>
-        public FeatureExtractor(IDataParser dataParser, Dictionary<string, int> buildingsToAreas)
+        public FeatureExtractor(IDataParser dataParser, PredictorConfiguration configuration)
         {
-            this.dataParser = dataParser;
-            this.buildingsToAreas = buildingsToAreas;
+            this.DataParser = dataParser;
+            this.Configuration = configuration;
         }
 
         /// <summary>
@@ -41 +44 @@
         /// <param name="interval"></param>
         /// <param name="wholeDay"></param>
         /// <returns></returns>
-        public List<ModelInput> PrepareTrainingInput(int area, DateTime startDate, DateTime endDate, int interval = 1, bool wholeDay = true)
+        public List<ModelInput> PrepareTrainingInput(int area, DateTime startDate, DateTime endDate, int interval = 3, bool wholeDay = true)
         {
-            dataParser.Parse(startDate, endDate, interval, wholeDay);
+            DataParser.Parse(startDate, endDate, interval, wholeDay);
             List<string> buildings = new List<string>();
 
             // find all buildings in area
-            foreach (KeyValuePair<string, int> kvp in buildingsToAreas)
+            foreach (KeyValuePair<string, int> kvp in Configuration.BuildingsToAreas)
             {
                 if (kvp.Value == area)
                 {
@@ -56 +59 @@
             }
 
             var res = new List<ModelInput>();
-            foreach (WeatherInfo val in dataParser.WeatherList)
+            foreach (WeatherInfo val in DataParser.WeatherList)
             {
                 res.Add(new ModelInput
                 {
@@ -68 +71 @@
                 });
             }
 
-            List<ActivityInfo> attendance = dataParser.AttendanceList;
+            List<ActivityInfo> attendance = DataParser.AttendanceList;
             foreach (ModelInput input in res)
             {
                 List<int> amounts = new List<int>();

Server/ServerApp/Predictor/NaiveBayesClassifier.cs

@@ -40 +40 @@
         {
             this._trainingDataView = _mlContext.Data.LoadFromEnumerable(trainInput);
             var pipeline = _mlContext.Transforms.Conversion.MapValueToKey(nameof(ModelInput.Label))
-                .Append(_mlContext.Transforms.Concatenate("Features", new[] { nameof(ModelInput.Temp), nameof(ModelInput.Rain), nameof(ModelInput.Wind) }))
+                .Append(_mlContext.Transforms.Conversion.ConvertType(nameof(ModelInput.Hour)))
+                .Append(_mlContext.Transforms.Concatenate("Features", 
+                new[] { nameof(ModelInput.Temp), nameof(ModelInput.Rain), nameof(ModelInput.Wind), nameof(ModelInput.Hour) }))
                 .Append(_mlContext.Transforms.NormalizeMinMax("Features", "Features"))
                 .AppendCacheCheckpoint(_mlContext)
                 .Append(_mlContext.MulticlassClassification.Trainers.NaiveBayes())
                 .Append(_mlContext.Transforms.Conversion.MapKeyToValue(nameof(ModelOutput.PredictedLabel)));
 
-            this._trainedModel = pipeline.Fit(this._trainingDataView);
+            var cvResults = _mlContext.MulticlassClassification.CrossValidate(this._trainingDataView, pipeline);
+            foreach (var result in cvResults)
+            {
+                var testMetrics = result.Metrics;
+                Console.WriteLine($"*************************************************************************************************************");
+                Console.WriteLine($"*       Metrics for Multi-class Classification model - Model #{result.Fold}    ");
+                Console.WriteLine($"*------------------------------------------------------------------------------------------------------------");
+                Console.WriteLine($"*       MicroAccuracy:    {testMetrics.MicroAccuracy:0.###}");
+                Console.WriteLine($"*       MacroAccuracy:    {testMetrics.MacroAccuracy:0.###}");
+                Console.WriteLine($"*       LogLoss:          {testMetrics.LogLoss:#.###}");
+                Console.WriteLine($"*       LogLossReduction: {testMetrics.LogLossReduction:#.###}");
+                Console.WriteLine($"*       Confusion Matrix: {testMetrics.ConfusionMatrix.GetFormattedConfusionTable()}");
+                Console.WriteLine($"*************************************************************************************************************");
+            }
+            this._trainedModel = cvResults.OrderByDescending(fold => fold.Metrics.MicroAccuracy).Select(fold => fold.Model).First();
+            Console.WriteLine($"Selected the model #{cvResults.OrderByDescending(fold => fold.Metrics.MicroAccuracy).Select(fold => fold.Fold).First()} as the best.");
             this._predictionEngine = _mlContext.Model.CreatePredictionEngine<ModelInput, ModelOutput>(this._trainedModel);
 
         }
@@ -69 +86 @@
             Console.WriteLine($"*       MacroAccuracy:    {testMetrics.MacroAccuracy:0.###}");
             Console.WriteLine($"*       LogLoss:          {testMetrics.LogLoss:#.###}");
             Console.WriteLine($"*       LogLossReduction: {testMetrics.LogLossReduction:#.###}");
+            Console.WriteLine($"*       Confusion Matrix: {testMetrics.ConfusionMatrix.GetFormattedConfusionTable()}");
             Console.WriteLine($"*************************************************************************************************************");
         }
     }

Server/ServerApp/Predictor/PredictionController.cs

@@ -6 +6 @@
 using System.Collections.Generic;
 using ServerApp.Connection.XMLProtocolHandler;
 using ServerApp.Parser.Parsers;
+using Newtonsoft.Json;
 
 namespace ServerApp.Predictor
 {
@@ -15 +16 @@
     class PredictionController : IPredictionController
     {
         /// <summary>
-        /// A dictionary for storing trained predictors.
+        /// Configuration of the <c>Predictor</c>
         /// </summary>
-        private Dictionary<string, int> buildingsToAreas;
+        private PredictorConfiguration Configuration;
 
-        private List<IPredictor> predictors;
+        private List<IPredictor> Predictors;
 
         /// <summary>
         /// A reference to a data parser.
         /// </summary>
-        private IDataParser dataParser;
+        private IDataParser DataParser;
 
         /// <summary>
         /// A feature extractor instance.
         /// </summary>
-        private FeatureExtractor featureExtractor;
+        private FeatureExtractor FeatureExtractor;
 
         /// <summary>
         /// Instantiates new prediction controller.
         /// </summary>
         /// <param name="dataParser">A data parser used to get training data.</param>
-        public PredictionController(IDataParser dataParser)
+        public PredictionController(IDataParser dataParser, string pathToConfig = null)
         {
-            this.dataParser = dataParser;
-            this.predictors = new List<IPredictor>();
-            this.buildingsToAreas = new Dictionary<string, int>();
-            this.featureExtractor = new FeatureExtractor(this.dataParser, buildingsToAreas);
+            // load config or get the default one
+            if (pathToConfig is null)
+            {
+                pathToConfig = PredictorConfiguration.DEFAULT_CONFIG_PATH;
+            }
+            try
+            {
+                string json = System.IO.File.ReadAllText(pathToConfig);
+                this.Configuration = JsonConvert.DeserializeObject<PredictorConfiguration>(json);
+            } catch (System.IO.IOException e)
+            {
+                Console.WriteLine(e.ToString());
+                this.Configuration = PredictorConfiguration.GetDefaultConfig();
+            }
+
+            this.DataParser = dataParser;
+            this.Predictors = new List<IPredictor>();
+            this.FeatureExtractor = new FeatureExtractor(this.DataParser, this.Configuration);
 
-            // fill predictors with all available locationKeys
-            // TODO Currently all locations use the same predictor. Try dividing locations into subareas with separate predictors.
-            var locationKeys = TagInfo.buildings;
-            foreach (string key in locationKeys)
+            for (int i = 0; i < this.Configuration.PredictorCount; i++)
             {
-                buildingsToAreas.Add(key, 0);
+                Predictors.Add(new NaiveBayesClassifier());
             }
-            IPredictor predictor = new NaiveBayesClassifier();
-            predictors.Add(predictor);
+            PredictorConfiguration.SaveConfig(PredictorConfiguration.DEFAULT_CONFIG_PATH, Configuration);
         }
         public List<string> GetPredictors()
         {
-            return new List<string>(buildingsToAreas.Keys);
+            return new List<string>(this.Configuration.BuildingsToAreas.Keys);
         }
 
         public void Load(string locationKey = null, string path = null)
@@ -81 +92 @@
             // train all predictors
             {
                 // TODO A single predictor is used for all areas, so training is done only once now.
-                for (int i = 0; i < this.predictors.Count; i++)
+                for (int i = 0; i < this.Predictors.Count; i++)
                 {
                     // train on all available data
                     // TODO the train/test split is used just temporarily for demonstration.
-                    List<ModelInput> data = featureExtractor.PrepareTrainingInput(i, DateTime.MinValue, DateTime.MaxValue);
-                    List<ModelInput> trainingData = data.GetRange(index: 0, count: 500);
-                    List<ModelInput> testData = data.GetRange(index: 500, count: 94);
-                    Console.WriteLine("Training predictor with {0} samples.", trainingData.Count);
-                    this.predictors[i].Fit(trainingData);
-
-                    Console.WriteLine("Evaluating predictor with {0} samples.", testData.Count);
-                    this.predictors[i].Evaluate(testData);
+                    List<ModelInput> data = FeatureExtractor.PrepareTrainingInput(i, DateTime.MinValue, DateTime.MaxValue);
+                    Console.WriteLine("Training predictor with {0} samples.", data.Count);
+                    this.Predictors[i].Fit(data);
                 }
             }
             else

Server/ServerApp/Predictor/PredictorConfiguration.cs

@@ +1 @@
+//
+// Author: Roman Kalivoda
+//
+
+using System.Collections.Generic;
+using System;
+using System.IO;
+using Newtonsoft.Json;
+
+namespace ServerApp.Predictor
+{
+    class PredictorConfiguration
+    {
+        public static readonly string DEFAULT_CONFIG_PATH = Path.GetFullPath(Path.GetDirectoryName(AppDomain.CurrentDomain.SetupInformation.ConfigurationFile) + @"\Predictor.config");
+
+        public int TimeResolution { get; set; }
+
+        public Dictionary<string, int> BuildingsToAreas { get; set; }
+
+        public int PredictorCount { get; set; }
+
+        public static PredictorConfiguration LoadConfig(string filename)
+        {
+            string json = System.IO.File.ReadAllText(filename);
+            PredictorConfiguration configuration = JsonConvert.DeserializeObject<PredictorConfiguration>(json);
+            return configuration;
+        }
+
+        public static void SaveConfig(string filename, PredictorConfiguration configuration)
+        {
+            string json = JsonConvert.SerializeObject(configuration);
+            System.IO.File.WriteAllText(filename, json);
+        }
+
+        public static PredictorConfiguration GetDefaultConfig()
+        {
+            Dictionary<string, int> dict = new Dictionary<string, int>();
+            var locationKeys = Parser.Parsers.TagInfo.buildings;
+            foreach (string key in locationKeys)
+            {
+                dict.Add(key, 0);
+            }
+
+            return new PredictorConfiguration
+            {
+                TimeResolution = 3,
+                PredictorCount = 3,
+                BuildingsToAreas = new Dictionary<string, int>
+                {
+                    { "FST+FEK", 0 },
+                    { "FDU", 0 },
+                    { "FAV", 0 },
+                    { "FEL", 0 },
+                    { "REK", 0 },
+                    { "MENZA", 0 },
+                    { "LIB", 0 },
+                    { "CIV", 0 },
+                    { "UNI14", 0 },
+                    { "DOM", 1 },
+                    { "HUS", 1 },
+                    { "CHOD", 1 },
+                    { "JUNG", 1 },
+                    { "KLAT", 1 },
+                    { "KOLL", 1 },
+                    { "RIEG", 1 },
+                    { "SADY", 1 },
+                    { "SED+VEL", 1 },
+                    { "TES", 1 },
+                    { "TYL", 1 },
+                    { "KARMA", 2 },
+                    { "KBORY", 2 },
+                    { "KLOCH", 2 },
+                    { "KKLAT", 2 }
+                }
+            };
+        }
+    }
+}

Server/ServerApp/ServerApp.csproj

@@ -179 +179 @@
     <Compile Include="Parser\Parsers\DataParser.cs" />
     <Compile Include="Parser\Parsers\IDataParser.cs" />
     <Compile Include="Parser\Parsers\JisParser.cs" />
+    <Compile Include="Predictor\PredictorConfiguration.cs" />
     <Compile Include="Predictor\FeatureExtractor.cs" />
     <Compile Include="Predictor\IPredictionController.cs" />
     <Compile Include="Predictor\PredictionController.cs" />