Revize ce0940b5

Re #9034 fixing files corrupted by merge

Server/ServerApp/Connection/XMLProtocolHandler/Request.cs

@@ -1 +1 @@
 using System;
-using System.Collections.Generic;
-using System.Linq;
-using System.Text;
-using System.Threading.Tasks;
 using System.Xml.Serialization;
 
 namespace ServerApp.Connection.XMLProtocolHandler

Server/ServerApp/Parser/Parsers/DataParser.cs

@@ -81 +81 @@
             var jisFiles = downloader.GetData(pathJis, start, end);
             var loginFiles = downloader.GetData(pathLogIn, start, end);
 
+            WeatherDataUsed = new List<string>();
+            ActivityDataUsed = new List<string>();
+
+            WeatherDataUsed.AddRange(weatherFiles);
+            ActivityDataUsed.AddRange(jisFiles);
+            ActivityDataUsed.AddRange(loginFiles);
+
             WeatherList = weatherParser.ParseWeatherData(weatherFiles, startTime, endTime, wholeDay, interval);
             jisList = jisParser.ParseJisData(jisFiles, startTime, endTime, wholeDay, interval);
             loginList = loginParser.ParseLogInData(loginFiles, startTime, endTime, wholeDay, interval);

Server/ServerApp/Parser/Parsers/IDataParser.cs

@@ -21 +21 @@
         List<ActivityInfo> attendanceList;
         public List<ActivityInfo> AttendanceList { get => attendanceList; internal set => attendanceList = value; }
 
+        /// <summary> List of weather file names the parser was last used on </summary>
+        List<String> weatherdataUsed;
+        public List<string> WeatherDataUsed { get => weatherdataUsed; set => weatherdataUsed = value; }
+
+        /// <summary> List of activity file names the parser was last used on </summary>
+        List<String> activitydataUsed;
+        public List<string> ActivityDataUsed { get => activitydataUsed; set => activitydataUsed = value; }
+
+
         /// <summary>
         /// Parse data
         /// </summary>

Server/ServerApp/Predictor/FeatureExtractor.cs

@@ -13 +13 @@
     /// <summary>
     /// A class responsible for preparation of features for classifiers.
     /// </summary>
-    public class FeatureExtractor
+    class FeatureExtractor
     {
         /// <summary>
         /// A DataParser instance used to access info objects.
         /// </summary>
-        private readonly IDataParser dataParser;
+        private readonly IDataParser DataParser;
 
-        private Dictionary<string, int> buildingsToAreas;
+        /// <summary>
+        /// A configuration object of the <c>Predictor</c> package
+        /// </summary>
+        private PredictorConfiguration Configuration;
 
         /// <summary>
         /// Instantiates new FeatureExtractor class.
         /// </summary>
         /// <param name="dataParser">Data parser used to access training data.</param>
-        public FeatureExtractor(IDataParser dataParser, Dictionary<string, int> buildingsToAreas)
+        public FeatureExtractor(IDataParser dataParser, PredictorConfiguration configuration)
         {
-            this.dataParser = dataParser;
-            this.buildingsToAreas = buildingsToAreas;
+            this.DataParser = dataParser;
+            this.Configuration = configuration;
         }
 
         /// <summary>
@@ -41 +44 @@
         /// <param name="interval"></param>
         /// <param name="wholeDay"></param>
         /// <returns></returns>
-        public List<ModelInput> PrepareTrainingInput(int area, DateTime startDate, DateTime endDate, int interval = 1, bool wholeDay = true)
+        public List<ModelInput> PrepareTrainingInput(int area)
         {
-            dataParser.Parse(startDate, endDate, interval, wholeDay);
             List<string> buildings = new List<string>();
 
             // find all buildings in area
-            foreach (KeyValuePair<string, int> kvp in buildingsToAreas)
+            foreach (KeyValuePair<string, int> kvp in Configuration.BuildingsToAreas)
             {
                 if (kvp.Value == area)
                 {
@@ -56 +58 @@
             }
 
             var res = new List<ModelInput>();
-            foreach (WeatherInfo val in dataParser.WeatherList)
+            foreach (WeatherInfo val in DataParser.WeatherList)
             {
                 res.Add(new ModelInput
                 {
-                    Temp = (float)val.temp,
                     Time = val.startTime,
+                    Temp = (float)val.temp,
+                    Hour = val.startTime.Hour,
                     Wind = (float)val.wind,
                     Rain = (float)val.rain,
                 });
             }
 
-            List<ActivityInfo> attendance = dataParser.AttendanceList;
+            List<ActivityInfo> attendance = DataParser.AttendanceList;
             foreach (ModelInput input in res)
             {
                 List<int> amounts = new List<int>();
@@ -90 +93 @@
             if (ratio < 0.1f)
             {
                 return "10%";
-            } else if (ratio < 0.2f)
+            }
+            else if (ratio < 0.2f)
             {
                 return "20%";
-            } else if (ratio < 0.3f)
+            }
+            else if (ratio < 0.3f)
             {
                 return "30%";
-            } else if (ratio < 0.4f)
+            }
+            else if (ratio < 0.4f)
             {
                 return "40%";
-            } else if (ratio < 0.5f)
+            }
+            else if (ratio < 0.5f)
             {
                 return "50%";
-            } else if (ratio < 0.6f)
+            }
+            else if (ratio < 0.6f)
             {
                 return "60%";
-            } else if(ratio < 0.7f) {
+            }
+            else if (ratio < 0.7f)
+            {
                 return "70%";
-            } else if (ratio < 0.8f)
+            }
+            else if (ratio < 0.8f)
             {
                 return "80%";
-            } else if (ratio < 0.9f)
+            }
+            else if (ratio < 0.9f)
             {
                 return "90%";
-            } else
+            }
+            else
             {
                 return "100%";
             }
         }
 
-        private double LabelToRatio(string label)
+        internal double LabelToRatio(string label)
         {
-            if (label.Equals("10%")) {
+            if (label.Equals("10%"))
+            {
                 return 0.1f;
             }
-            else if (label.Equals("20%"))  {
+            else if (label.Equals("20%"))
+            {
                 return 0.2f;
             }
-            else if (label.Equals("30%")) {
+            else if (label.Equals("30%"))
+            {
                 return 0.3f;
             }
-            else if (label.Equals("40%")) {
+            else if (label.Equals("40%"))
+            {
                 return 0.4f;
             }
-            else if (label.Equals("50%")) {
+            else if (label.Equals("50%"))
+            {
                 return 0.5f;
             }
-            else if (label.Equals("60%")) {
+            else if (label.Equals("60%"))
+            {
                 return 0.6f;
             }
-            else if (label.Equals("70%")) {
+            else if (label.Equals("70%"))
+            {
                 return 0.7f;
             }
-            else if (label.Equals("80%")) {
+            else if (label.Equals("80%"))
+            {
                 return 0.8f;
             }
-            else if (label.Equals("90%")) {
+            else if (label.Equals("90%"))
+            {
                 return 0.9f;
             }
             else

Server/ServerApp/Predictor/IPredictionController.cs

@@ -2 +2 @@
 // Author: Roman Kalivoda
 //
 
-using System;
 using System.Collections.Generic;
-using System.IO;
-using Microsoft.ML;
 using ServerApp.Connection.XMLProtocolHandler;
-using ServerApp.Parser.OutputInfo;
 
 namespace ServerApp.Predictor
 {
@@ -32 +28 @@
         /// </summary>
         /// <param name="locationKey">A string identifier of the location for which to load a predictor.</param>
         /// <param name="path">A path to folder with trained prediction model.</param>
-        void Load(string locationKey = null,string path = null);
+        void Load(string locationKey = null, string path = null);
 
         /// <summary>
         /// Predicts turnout level at given time supposing given weather conditions.

Server/ServerApp/Predictor/ModelInput.cs

@@ -13 +13 @@
     /// </summary>
     public class ModelInput
     {
+        /// <summary>
+        /// Start time of the information.
+        /// </summary>
+        public DateTime Time { get; set; }
+
         /// <summary>
         /// A label of this training input.
         /// </summary>
@@ -26 +31 @@
         public float Temp { get; set; }
 
         /// <summary>
-        /// Time of the predicted turnout.
+        /// Hour of the predicted turnout.
         /// </summary>
-        [ColumnName("Time"), LoadColumn(2)]
-        public DateTime Time { get; set; }
+        [ColumnName("Hour"), LoadColumn(2)]
+        public int Hour { get; set; }
 
         /// <summary>
-        /// Wind velocity in ? units
+        /// Wind velocity in m/s
         /// </summary>
         [ColumnName("Wind"), LoadColumn(3)]
         public float Wind { get; set; }
 
         /// <summary>
-        /// Precipitation
+        /// Precipitation in %
         /// </summary>
         [ColumnName("Rain"), LoadColumn(4)]
         public float Rain { get; set; }

Server/ServerApp/Predictor/ModelOutput.cs

@@ -2 +2 @@
 // Author: Roman Kalivoda
 //
 
-using System;
 using Microsoft.ML.Data;
 
 namespace ServerApp.Predictor
@@ -16 +15 @@
         /// A predicted class.
         /// </summary>
         [ColumnName("PredictedLabel")]
-        public String Prediction { get; set; }
+        public string PredictedLabel { get; set; }
 
     }
 }

Server/ServerApp/Predictor/NaiveBayesClassifier.cs

@@ -44 +44 @@
         {
             this._trainingDataView = _mlContext.Data.LoadFromEnumerable(trainInput);
             var pipeline = _mlContext.Transforms.Conversion.MapValueToKey(nameof(ModelInput.Label))
-                .Append(_mlContext.Transforms.Concatenate("Features", new[] { "Temp" }))
-                .Append(_mlContext.Transforms.NormalizeMinMax("Features", "Features"))
+                .Append(_mlContext.Transforms.Conversion.ConvertType(nameof(ModelInput.Hour)))
+                .Append(_mlContext.Transforms.Concatenate("Features", 
+                new[] { nameof(ModelInput.Temp), nameof(ModelInput.Rain), nameof(ModelInput.Wind), nameof(ModelInput.Hour) }))
+                .Append(_mlContext.Transforms.NormalizeMeanVariance("Features", useCdf:false))
                 .AppendCacheCheckpoint(_mlContext)
                 .Append(_mlContext.MulticlassClassification.Trainers.NaiveBayes())
-                .Append(_mlContext.Transforms.Conversion.MapKeyToValue("PredictedLabel")); ;
+                .Append(_mlContext.Transforms.Conversion.MapKeyToValue(nameof(ModelOutput.PredictedLabel)));
 
             var cvResults = _mlContext.MulticlassClassification.CrossValidate(this._trainingDataView, pipeline);
             _log.Debug("Cross-validated the trained model");
@@ -67 +69 @@
         public void Evaluate(IEnumerable<ModelInput> modelInputs)
         {
             var testDataView = this._mlContext.Data.LoadFromEnumerable(modelInputs);
-            var testMetrics = _mlContext.MulticlassClassification.Evaluate(_trainedModel.Transform(testDataView));
+            var data = _trainedModel.Transform(testDataView);
+            var testMetrics = _mlContext.MulticlassClassification.Evaluate(data);
 
             Console.WriteLine($"*************************************************************************************************************");
             Console.WriteLine($"*       Metrics for Multi-class Classification model - Test Data     ");
@@ -76 +79 @@
             Console.WriteLine($"*       MacroAccuracy:    {testMetrics.MacroAccuracy:0.###}");
             Console.WriteLine($"*       LogLoss:          {testMetrics.LogLoss:#.###}");
             Console.WriteLine($"*       LogLossReduction: {testMetrics.LogLossReduction:#.###}");
+            Console.WriteLine($"*       Confusion Matrix: {testMetrics.ConfusionMatrix.GetFormattedConfusionTable()}");
             Console.WriteLine($"*************************************************************************************************************");
         }
     }

Server/ServerApp/Predictor/PredictionController.cs

@@ -4 +4 @@
 
 using System;
 using System.Collections.Generic;
-using System.IO;
-using Microsoft.ML;
 using ServerApp.Connection.XMLProtocolHandler;
+using ServerApp.Parser.Parsers;
+using Newtonsoft.Json;
+using ServerApp.WeatherPredictionParser;
 using ServerApp.Parser.OutputInfo;
 using System.Reflection;
 using log4net;
@@ -16 +17 @@
     /// <summary>
     /// Implentation of the <c>IPredicitionController</c> interface.
     /// </summary>
-    class PredictionController : IPredictionController
+    public class PredictionController : IPredictionController
     {
         private static readonly ILog _log = LogManager.GetLogger(typeof(PredictionController));
 
         /// <summary>
-        /// A dictionary for storing trained predictors.
+        /// Configuration of the <c>Predictor</c>
         /// </summary>
-        private Dictionary<string, int> buildingsToAreas;
+        private PredictorConfiguration Configuration;
 
-        private List<IPredictor> predictors;
+        private List<IPredictor> Predictors;
 
         /// <summary>
         /// A reference to a data parser.
         /// </summary>
-        private IDataParser dataParser;
+        private IDataParser DataParser;
 
         /// <summary>
         /// A feature extractor instance.
         /// </summary>
-        private FeatureExtractor featureExtractor;
+        private FeatureExtractor FeatureExtractor;
+
+        /// <summary>
+        /// A weather prediction parser service
+        /// </summary>
+        private IJsonParser weatherService;
 
         /// <summary>
         /// Instantiates new prediction controller.
         /// </summary>
         /// <param name="dataParser">A data parser used to get training data.</param>
-        public PredictionController(IDataParser dataParser)
+        public PredictionController(IJsonParser weatherService, IDataParser dataParser, string pathToConfig = null)
         {
             _log.Info("Constructing a new PredictionController instance.");
             this.weatherService = weatherService;
@@ -68 +74 @@
 
             for (int i = 0; i < this.Configuration.PredictorCount; i++)
             {
-                buildingsToAreas.Add(key, 0);
+                Predictors.Add(new NaiveBayesClassifier());
             }
-            IPredictor predictor = new NaiveBayesClassifier();
-            predictors.Add(predictor);
+            PredictorConfiguration.SaveConfig(PredictorConfiguration.DEFAULT_CONFIG_PATH, Configuration);
         }
         public List<string> GetPredictors()
         {
-            return new List<string>(buildingsToAreas.Keys);
+            return new List<string>(this.Configuration.BuildingsToAreas.Keys);
         }
 
         public void Load(string locationKey = null, string path = null)
@@ -180 +185 @@
             if (locationKey is null)
             // train all predictors
             {
-                // TODO A single predictor is used for all areas, so training is done only once now.
-                for (int i = 0; i < this.predictors.Count; i++)
+                DataParser.Parse(DateTime.MinValue, DateTime.MaxValue, this.Configuration.TimeResolution, wholeDay: false);
+                for (int i = 0; i < this.Predictors.Count; i++)
                 {
                     // train on all available data
-                    // TODO the train/test split is used just temporarily for demonstration.
-                    List<ModelInput> data = featureExtractor.PrepareTrainingInput(i, DateTime.MinValue, DateTime.MaxValue);
-                    List<ModelInput> trainingData = data.GetRange(index: 0, count: 500);
-                    List<ModelInput> testData = data.GetRange(index: 500, count: 94);
-                    Console.WriteLine("Training predictor with {0} samples.", trainingData.Count);
-                    this.predictors[i].Fit(trainingData);
-
-                    Console.WriteLine("Evaluating predictor with {0} samples.", testData.Count);
-                    this.predictors[i].Evaluate(testData);
+                    List<ModelInput> data = FeatureExtractor.PrepareTrainingInput(i);
+                    Console.WriteLine("Training predictor with {0} samples.", data.Count);
+                    this.Predictors[i].Fit(data);
                 }
-            } else
+            }
+            else
             // train specified predictor only
             {
                 throw new NotImplementedException();

Server/ServerApp/WeatherPredictionParser/IJsonParser.cs

@@ -3 +3 @@
 //
 
 using ServerApp.Parser.OutputInfo;
+using System;
 using System.Collections.Generic;
 
 namespace ServerApp.WeatherPredictionParser
@@ -10 +11 @@
     /// <summary>
     /// Abstract class that every Json parser should inherit from
     /// </summary>
-    abstract class IJsonParser
+    public abstract class IJsonParser
     {
 
         /// <summary> Current weather </summary>
         WeatherInfo current;
-        public WeatherInfo Current { get => current; }
+        public WeatherInfo Current { get => current; set => current = value; }
 
         /// <summary> Prediction for today, tommorrow and day after tommorrow </summary>
         List<WeatherInfo> predictions;
@@ -27 +28 @@
         /// </summary>
         abstract public void ParsePrediction();
 
+        /// <summary>
+        /// Get predictions from Predictions that are within specified time span
+        /// </summary>
+        /// <param name="from">DateTime from</param>
+        /// <param name="to">DateTime to</param>
+        /// <returns>List of predictions that fit specified criteria</returns>
+        abstract public List<WeatherInfo> GetPredictionForTime(DateTime from, DateTime to);
+
     }
 
 }

Server/ServerApp/WeatherPredictionParser/JsonParser.cs

@@ -119 +119 @@
 
             DateTime now = DateTime.Now;
 
-            current = new WeatherInfo();
-            predictions = new List<WeatherInfo>();
+            Current = new WeatherInfo();
+            Predictions = new List<WeatherInfo>();
 
             // parse
             JsonDocument doc = JsonDocument.Parse(data);
@@ -138 +138 @@
                     case "current_condition":
                         {
                             ArrayEnumerator currentWeather = weatherP.Current.Value.EnumerateArray();
-                            current = ParseCurrentWeather(currentWeather);
+                            Current = ParseCurrentWeather(currentWeather);
 
                             break;
                         }
@@ -164 +164 @@
         /// </summary>
         private void TestConsoleOutput()
         {
-            Console.WriteLine(current);
-            foreach (WeatherInfo w in predictions)
+            Console.WriteLine(Current);
+            foreach (WeatherInfo w in Predictions)
                 Console.WriteLine(w);
         }
 
@@ -477 +477 @@
             return res;
         }
 
+      
     }
 }