aswi2021tri-musketyri-gitlab/Server/ServerApp/Predictor/FeatureExtractor.cs @ 9a529f10

//
// Author: Roman Kalivoda
//

using System.Collections.Generic;
using System;
using ServerApp.Parser.Parsers;
using ServerApp.Parser.OutputInfo;
using System.Linq;
using Microsoft.ML;
using log4net;

namespace ServerApp.Predictor
{
    /// <summary>
    /// A class responsible for preparation of features for classifiers.
    /// </summary>
    class FeatureExtractor
    {

        private const double Confidence = 99.0;

        private static readonly ILog _log = LogManager.GetLogger(typeof(FeatureExtractor));

        /// <summary>
        /// A DataParser instance used to access info objects.
        /// </summary>
        private readonly IDataParser DataParser;

        /// <summary>
        /// A configuration object of the <c>Predictor</c> package
        /// </summary>
        private PredictorConfiguration Configuration;

        /// <summary>
        /// Instantiates new FeatureExtractor class.
        /// </summary>
        /// <param name="dataParser">Data parser used to access training data.</param>
        public FeatureExtractor(IDataParser dataParser, PredictorConfiguration configuration)
        {
            this.DataParser = dataParser;
            this.Configuration = configuration;
        }

        /// <summary>
        /// TODO comment
        /// </summary>
        /// <param name="area"></param>
        /// <param name="startDate"></param>
        /// <param name="endDate"></param>
        /// <param name="interval"></param>
        /// <param name="wholeDay"></param>
        /// <returns></returns>
        public List<ModelInput> PrepareTrainingInput(int area)
        {
            List<string> buildings = new List<string>(); 
            List<ActivityInfo> attendance = DataParser.AttendanceList;

            // find all buildings in area
            foreach (KeyValuePair<string, int> kvp in Configuration.BuildingsToAreas)
            {
                if (kvp.Value == area)
                {
                    buildings.Add(kvp.Key);
                }
            }
            List<ActivityInfo> activities = attendance.Where(e => buildings.Contains(e.building)).GroupBy(e => e.startTime).Select(g => g.Aggregate((a, b) => new ActivityInfo(null, a.amount + b.amount, a.startTime, -1))).OrderBy(e => e.startTime).ToList();
           activities = RejectOutliers(activities);
            var inputs = activities.Join(DataParser.WeatherList, activity => activity.startTime, weatherInfo => weatherInfo.startTime, (activity, weatherInfo) => new
            {
                amount = activity.amount,
                modelInput = new ModelInput
                {
                    Hour = activity.startTime.Hour,
                    Temp = (float)weatherInfo.temp,
                    Rain = (float)weatherInfo.rain,
                    Time = activity.startTime,
                    Wind = (float)weatherInfo.wind
                }
            }).ToList();

            int max = inputs.Select(e => e.amount).Max();
            foreach (var input in inputs)
            {
                double ratio = input.amount / (double)max;
                input.modelInput.Label = RatioToLabel(ratio);
            }

            return inputs.Select(e => e.modelInput).ToList();
        }

        private static List<ActivityInfo> RejectOutliers(List<ActivityInfo> data)
        {
            MLContext mlContext = new MLContext();
            IDataView input = mlContext.Data.LoadFromEnumerable(data);
            var pipeline = mlContext.Transforms.Conversion.ConvertType(nameof(ActivityInfo.amount)).Append(mlContext.Transforms.DetectIidSpike(nameof(AnomalyDetectionResult.Prediction), nameof(ActivityInfo.amount), Confidence, data.Count / 4));
            ITransformer transformer = pipeline.Fit(mlContext.Data.LoadFromEnumerable(new List<ActivityInfo>()));
            IDataView transformedData = transformer.Transform(input);
            List<AnomalyDetectionResult> predictions = mlContext.Data.CreateEnumerable<AnomalyDetectionResult>(transformedData, false).ToList();
            List<ActivityInfo> result = new List<ActivityInfo>();

            for (int i=0; i<predictions.Count; i++)
            {
                if(predictions[i].Prediction[2] < (1 - Confidence))
                {
                    _log.Debug($"Rejecting an outlier activity: {predictions[i].Prediction[1]}, p-value: {predictions[i].Prediction[2]}, from: {data[i].startTime}");
                } else
                {
                    result.Add(data[i]);
                }
            }
            return result;
        }

        private string RatioToLabel(double ratio)
        {
            if (ratio < 0.1f)
            {
                return "10%";
            }
            else if (ratio < 0.2f)
            {
                return "20%";
            }
            else if (ratio < 0.3f)
            {
                return "30%";
            }
            else if (ratio < 0.4f)
            {
                return "40%";
            }
            else if (ratio < 0.5f)
            {
                return "50%";
            }
            else if (ratio < 0.6f)
            {
                return "60%";
            }
            else if (ratio < 0.7f)
            {
                return "70%";
            }
            else if (ratio < 0.8f)
            {
                return "80%";
            }
            else if (ratio < 0.9f)
            {
                return "90%";
            }
            else
            {
                return "100%";
            }
        }

        internal double LabelToRatio(string label)
        {
            if (label is null)
            {
                return -1f;
            }
            else if (label.Equals("10%"))
            {
                return 10f;
            }
            else if (label.Equals("20%"))
            {
                return 20f;
            }
            else if (label.Equals("30%"))
            {
                return 30f;
            }
            else if (label.Equals("40%"))
            {
                return 40f;
            }
            else if (label.Equals("50%"))
            {
                return 50f;
            }
            else if (label.Equals("60%"))
            {
                return 60f;
            }
            else if (label.Equals("70%"))
            {
                return 70f;
            }
            else if (label.Equals("80%"))
            {
                return 80f;
            }
            else if (label.Equals("90%"))
            {
                return 90f;
            }
            else
            {
                return 100f;
            }
        }
    }
}