Revize 0060a0ae

Re #9056 refactoring

Server/ServerApp/Predictor/FeatureExtractor.cs

@@ -7 +7 @@
 using ServerApp.Parser.Parsers;
 using ServerApp.Parser.OutputInfo;
 using System.Linq;
+using Microsoft.ML;
+using log4net;
 
 namespace ServerApp.Predictor
 {
@@ -15 +17 @@
     /// </summary>
     class FeatureExtractor
     {
+        private static readonly ILog _log = LogManager.GetLogger(typeof(FeatureExtractor));
+
         /// <summary>
         /// A DataParser instance used to access info objects.
         /// </summary>
@@ -46 +50 @@
         /// <returns></returns>
         public List<ModelInput> PrepareTrainingInput(int area)
         {
-            List<string> buildings = new List<string>();
+            List<string> buildings = new List<string>(); 
+            List<ActivityInfo> attendance = DataParser.AttendanceList;
 
             // find all buildings in area
             foreach (KeyValuePair<string, int> kvp in Configuration.BuildingsToAreas)
@@ -56 +61 @@
                     buildings.Add(kvp.Key);
                 }
             }
-
-            var res = new List<ModelInput>();
-            foreach (WeatherInfo val in DataParser.WeatherList)
+            List<ActivityInfo> activities = attendance.Where(e => buildings.Contains(e.building)).GroupBy(e => e.startTime).Select(g => g.Aggregate((a, b) => new ActivityInfo(null, a.amount + b.amount, a.startTime, -1))).OrderBy(e => e.startTime).ToList();
+           activities = RejectOutliers(activities);
+            var inputs = activities.Join(DataParser.WeatherList, activity => activity.startTime, weatherInfo => weatherInfo.startTime, (activity, weatherInfo) => new
             {
-                res.Add(new ModelInput
+                amount = activity.amount,
+                modelInput = new ModelInput
                 {
-                    Time = val.startTime,
-                    Temp = (float)val.temp,
-                    Hour = val.startTime.Hour,
-                    Wind = (float)val.wind,
-                    Rain = (float)val.rain,
-                });
+                    Hour = activity.startTime.Hour,
+                    Temp = (float)weatherInfo.temp,
+                    Rain = (float)weatherInfo.rain,
+                    Time = activity.startTime,
+                    Wind = (float)weatherInfo.wind
+                }
+            }).ToList();
+
+            int max = inputs.Select(e => e.amount).Max();
+            foreach (var input in inputs)
+            {
+                double ratio = input.amount / (double)max;
+                input.modelInput.Label = RatioToLabel(ratio);
             }
 
-            List<ActivityInfo> attendance = DataParser.AttendanceList;
-            foreach (ModelInput input in res)
+            return inputs.Select(e => e.modelInput).ToList();
+        }
+
+        private static List<ActivityInfo> RejectOutliers(List<ActivityInfo> data)
+        {
+            MLContext mlContext = new MLContext();
+            IDataView input = mlContext.Data.LoadFromEnumerable(data);
+            var pipeline = mlContext.Transforms.Conversion.ConvertType(nameof(ActivityInfo.amount)).Append(mlContext.Transforms.DetectIidSpike(nameof(AnomalyDetectionResult.Prediction), nameof(ActivityInfo.amount), 99.0, data.Count / 4));
+            ITransformer transformer = pipeline.Fit(mlContext.Data.LoadFromEnumerable(new List<ActivityInfo>()));
+            IDataView transformedData = transformer.Transform(input);
+            List<AnomalyDetectionResult> predictions = mlContext.Data.CreateEnumerable<AnomalyDetectionResult>(transformedData, false).ToList();
+            List<ActivityInfo> result = new List<ActivityInfo>();
+
+            for (int i=0; i<predictions.Count; i++)
             {
-                List<int> amounts = new List<int>();
-                List<int> maxima = new List<int>();
-                foreach (string building in buildings)
+                if(predictions[i].Prediction[0] == 1)
+                {
+                    _log.Debug($"Rejecting an outlier activity: {predictions[i].Prediction[1]}, p-value: {predictions[i].Prediction[2]}, from: {data[i].startTime}");
+                } else
                 {
-                    List<ActivityInfo> temp = attendance.Where(activity => activity.building.Equals(building)).ToList();
-                    amounts.Add(temp.Where(activity => activity.startTime.Equals(input.Time)).Select(activity => activity.amount).FirstOrDefault());
-                    maxima.Add(temp.Select(activity => activity.amount).Max());
+                    result.Add(data[i]);
                 }
-                double ratio = amounts.Sum() / (double)maxima.Sum();
-                input.Label = RatioToLabel(ratio);
             }
-
-            return res;
+            return result;
         }
 
         private string RatioToLabel(double ratio)
@@ -134 +155 @@
 
         internal double LabelToRatio(string label)
         {
-            if (label.Equals("10%"))
+            if (label is null)
+            {
+                return -1f;
+            }
+            else if (label.Equals("10%"))
             {
-                return 0.1f;
+                return 10f;
             }
             else if (label.Equals("20%"))
             {
-                return 0.2f;
+                return 20f;
             }
             else if (label.Equals("30%"))
             {
-                return 0.3f;
+                return 30f;
             }
             else if (label.Equals("40%"))
             {
-                return 0.4f;
+                return 40f;
             }
             else if (label.Equals("50%"))
             {
-                return 0.5f;
+                return 50f;
             }
             else if (label.Equals("60%"))
             {
-                return 0.6f;
+                return 60f;
             }
             else if (label.Equals("70%"))
             {
-                return 0.7f;
+                return 70f;
             }
             else if (label.Equals("80%"))
             {
-                return 0.8f;
+                return 80f;
             }
             else if (label.Equals("90%"))
             {
-                return 0.9f;
+                return 90f;
             }
             else
             {
-                return 1.0f;
+                return 100f;
             }
         }
     }