Revize 0060a0ae

Re #9056 refactoring

Server/ServerApp/Predictor/AnomalyDetectionResult.cs

@@ +1 @@
+//
+// Author: Roman Kalivoda
+//
+
+using Microsoft.ML.Data;
+
+namespace ServerApp.Predictor
+{
+    class AnomalyDetectionResult
+    {
+        [VectorType(3)]
+        public double[] Prediction { get; set; }
+    }
+}

Server/ServerApp/Predictor/FeatureExtractor.cs

@@ -7 +7 @@
 using ServerApp.Parser.Parsers;
 using ServerApp.Parser.OutputInfo;
 using System.Linq;
+using Microsoft.ML;
+using log4net;
 
 namespace ServerApp.Predictor
 {
@@ -15 +17 @@
     /// </summary>
     class FeatureExtractor
     {
+        private static readonly ILog _log = LogManager.GetLogger(typeof(FeatureExtractor));
+
         /// <summary>
         /// A DataParser instance used to access info objects.
         /// </summary>
@@ -46 +50 @@
         /// <returns></returns>
         public List<ModelInput> PrepareTrainingInput(int area)
         {
-            List<string> buildings = new List<string>();
+            List<string> buildings = new List<string>(); 
+            List<ActivityInfo> attendance = DataParser.AttendanceList;
 
             // find all buildings in area
             foreach (KeyValuePair<string, int> kvp in Configuration.BuildingsToAreas)
@@ -56 +61 @@
                     buildings.Add(kvp.Key);
                 }
             }
-
-            var res = new List<ModelInput>();
-            foreach (WeatherInfo val in DataParser.WeatherList)
+            List<ActivityInfo> activities = attendance.Where(e => buildings.Contains(e.building)).GroupBy(e => e.startTime).Select(g => g.Aggregate((a, b) => new ActivityInfo(null, a.amount + b.amount, a.startTime, -1))).OrderBy(e => e.startTime).ToList();
+           activities = RejectOutliers(activities);
+            var inputs = activities.Join(DataParser.WeatherList, activity => activity.startTime, weatherInfo => weatherInfo.startTime, (activity, weatherInfo) => new
             {
-                res.Add(new ModelInput
+                amount = activity.amount,
+                modelInput = new ModelInput
                 {
-                    Time = val.startTime,
-                    Temp = (float)val.temp,
-                    Hour = val.startTime.Hour,
-                    Wind = (float)val.wind,
-                    Rain = (float)val.rain,
-                });
+                    Hour = activity.startTime.Hour,
+                    Temp = (float)weatherInfo.temp,
+                    Rain = (float)weatherInfo.rain,
+                    Time = activity.startTime,
+                    Wind = (float)weatherInfo.wind
+                }
+            }).ToList();
+
+            int max = inputs.Select(e => e.amount).Max();
+            foreach (var input in inputs)
+            {
+                double ratio = input.amount / (double)max;
+                input.modelInput.Label = RatioToLabel(ratio);
             }
 
-            List<ActivityInfo> attendance = DataParser.AttendanceList;
-            foreach (ModelInput input in res)
+            return inputs.Select(e => e.modelInput).ToList();
+        }
+
+        private static List<ActivityInfo> RejectOutliers(List<ActivityInfo> data)
+        {
+            MLContext mlContext = new MLContext();
+            IDataView input = mlContext.Data.LoadFromEnumerable(data);
+            var pipeline = mlContext.Transforms.Conversion.ConvertType(nameof(ActivityInfo.amount)).Append(mlContext.Transforms.DetectIidSpike(nameof(AnomalyDetectionResult.Prediction), nameof(ActivityInfo.amount), 99.0, data.Count / 4));
+            ITransformer transformer = pipeline.Fit(mlContext.Data.LoadFromEnumerable(new List<ActivityInfo>()));
+            IDataView transformedData = transformer.Transform(input);
+            List<AnomalyDetectionResult> predictions = mlContext.Data.CreateEnumerable<AnomalyDetectionResult>(transformedData, false).ToList();
+            List<ActivityInfo> result = new List<ActivityInfo>();
+
+            for (int i=0; i<predictions.Count; i++)
             {
-                List<int> amounts = new List<int>();
-                List<int> maxima = new List<int>();
-                foreach (string building in buildings)
+                if(predictions[i].Prediction[0] == 1)
+                {
+                    _log.Debug($"Rejecting an outlier activity: {predictions[i].Prediction[1]}, p-value: {predictions[i].Prediction[2]}, from: {data[i].startTime}");
+                } else
                 {
-                    List<ActivityInfo> temp = attendance.Where(activity => activity.building.Equals(building)).ToList();
-                    amounts.Add(temp.Where(activity => activity.startTime.Equals(input.Time)).Select(activity => activity.amount).FirstOrDefault());
-                    maxima.Add(temp.Select(activity => activity.amount).Max());
+                    result.Add(data[i]);
                 }
-                double ratio = amounts.Sum() / (double)maxima.Sum();
-                input.Label = RatioToLabel(ratio);
             }
-
-            return res;
+            return result;
         }
 
         private string RatioToLabel(double ratio)
@@ -134 +155 @@
 
         internal double LabelToRatio(string label)
         {
-            if (label.Equals("10%"))
+            if (label is null)
+            {
+                return -1f;
+            }
+            else if (label.Equals("10%"))
             {
-                return 0.1f;
+                return 10f;
             }
             else if (label.Equals("20%"))
             {
-                return 0.2f;
+                return 20f;
             }
             else if (label.Equals("30%"))
             {
-                return 0.3f;
+                return 30f;
             }
             else if (label.Equals("40%"))
             {
-                return 0.4f;
+                return 40f;
             }
             else if (label.Equals("50%"))
             {
-                return 0.5f;
+                return 50f;
             }
             else if (label.Equals("60%"))
             {
-                return 0.6f;
+                return 60f;
             }
             else if (label.Equals("70%"))
             {
-                return 0.7f;
+                return 70f;
             }
             else if (label.Equals("80%"))
             {
-                return 0.8f;
+                return 80f;
             }
             else if (label.Equals("90%"))
             {
-                return 0.9f;
+                return 90f;
             }
             else
             {
-                return 1.0f;
+                return 100f;
             }
         }
     }

Server/ServerApp/Predictor/PredictionController.cs

@@ -129 +129 @@
             return response;
         }
 
-        private Prediction PredictSingle(Request request, DateTime current)
+        private Prediction PredictSingle(Request request, DateTime predictionTime)
         {
             double[] predictedValues = new double[this.Configuration.BuildingsToAreas.Count];
             string[] predictedLabels = new string[this.Predictors.Count];
@@ -143 +143 @@
                         Rain = (float)request.rain,
                         Temp = (float)request.temperature,
                         Wind = (float)request.wind,
-                        Hour = current.Hour,
-                        Time = current
+                        Hour = predictionTime.Hour,
+                        Time = predictionTime
                     });
                 }
                 else
                 {
                     _log.Debug("Retrieving weather info from the weather service.");
                     weatherService.ParsePrediction();
-                    WeatherInfo weatherInfo = weatherService.Predictions.Find(info => info.startTime.Equals(current));
+                    WeatherInfo weatherInfo = weatherService.Predictions.Find(info => info.startTime.Date.Equals(predictionTime.Date) && predictionTime.TimeOfDay.Subtract(info.startTime.TimeOfDay).Hours < info.intervalLength);
                     if (weatherInfo is null)
                     {
                         predictedLabels[i] = null;
@@ -163 +163 @@
                             Rain = weatherInfo.rain,
                             Temp = (float)weatherInfo.temp,
                             Wind = (float)weatherInfo.wind,
-                            Hour = current.Hour,
-                            Time = current
+                            Hour = predictionTime.Hour,
+                            Time = predictionTime
                         });
                     }
                 }
@@ -177 +177 @@
             Prediction prediction = new Prediction();
             prediction.dateTime = new Date
             {
-                year = current.Year,
-                month = current.Month,
-                day = current.Day,
-                hour = current.Hour
+                year = predictionTime.Year,
+                month = predictionTime.Month,
+                day = predictionTime.Day,
+                hour = predictionTime.Hour
             };
             prediction.predictions = predictedValues;
             _log.Debug($"Created prediction for DateTime: {prediction.dateTime}");

Server/ServerApp/Program.cs

@@ -34 +34 @@
 
         static void Main(string[] args)
         {
+            // setup logging service
             XmlConfigurator.Configure();
             // SETUP FOLDERS
             Config config = FillConfigInfo(args);

Server/ServerApp/ServerApp.csproj

@@ -10 +10 @@
     <ItemGroup>
         <PackageReference Include="log4net" Version="2.0.12" />
         <PackageReference Include="Microsoft.ML" Version="1.5.5" />
+        <PackageReference Include="Microsoft.ML.TimeSeries" Version="1.5.5" />
     </ItemGroup>
 
     <ItemGroup>

Server/ServerAppFunctionalTests/App.config

@@ +1 @@
+<?xml version="1.0" encoding="utf-8" ?>
+<configuration>
+    <configSections>
+        <section name="log4net" type="log4net.Config.Log4NetConfigurationSectionHandler, log4net" />
+    </configSections>
+    <log4net>
+        <appender name="LogFileAppender" type="log4net.Appender.RollingFileAppender">
+            <param name="File" value="server.log"/>
+            <lockingModel type="log4net.Appender.FileAppender+MinimalLock" />
+            <appendToFile value="true" />
+            <rollingStyle value="Size" />
+            <maxSizeRollBackups value="2" />
+            <maximumFileSize value="5MB" />
+            <staticLogFileName value="true" />
+            <threshold>DEBUG</threshold>
+            <layout type="log4net.Layout.PatternLayout">
+                <param name="ConversionPattern" value="%d [%t] %-5p %c %m%n"/>
+            </layout>
+        </appender>
+        <appender name="ConsoleAppender" type="log4net.Appender.ConsoleAppender">
+            <layout type="log4net.Layout.PatternLayout">
+                <param name="ConversionPattern" value="%d [%t] %-5p %c %m%n"/>
+            </layout>
+        </appender>
+        <appender name="DebugAppender" type="log4net.Appender.DebugAppender">
+            <layout type="log4net.Layout.PatternLayout">
+                <conversionPattern value="%date{dd.MM.yyyy HH:mm:ss.ffff} [%thread] %level %logger%exception - %message%newline" />
+            </layout>
+        </appender>
+        <root>
+            <level value="ALL" />
+            <appender-ref ref="LogFileAppender" />
+            <appender-ref ref="ConsoleAppender" />
+            <appender-ref ref="DebugAppender"/>
+        </root>
+    </log4net>
+</configuration>

Server/ServerAppFunctionalTests/Predictor/PredictionControllerTests.cs

@@ -14 +14 @@
 using ServerApp.Parser.InputData;
 using ServerApp.Connection.XMLProtocolHandler;
 using ServerApp.DataDownload;
+using log4net.Config;
 
 namespace ServerApp.Predictor.Tests
 {
     [TestClass()]
     public class PredictionControllerTests
     {
+        [AssemblyInitialize()]
+        public static void ClassInit(TestContext context)
+        {
+            // setup logging service
+            XmlConfigurator.Configure();
+        }
+
         [TestMethod()]
         public void PredictSingleTimeWeatherTest()
         {

Server/ServerAppFunctionalTests/ServerAppFunctionalTests.csproj

@@ -21 +21 @@
         <ProjectReference Include="..\ServerApp\ServerApp.csproj" />
     </ItemGroup>
 
+    <ItemGroup>
+      <None Update="App.config">
+        <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+      </None>
+    </ItemGroup>
+
 </Project>