commit 553486ad2f589b2cc149b720cb0bbf70e4f155b2
Author: Vojtěch Bartička <aitakaitov@gmail.com>
Date:   Wed Apr 27 09:51:56 2022 +0200

    Added caching and incremental updates to annotations

diff --git a/Backend/Core/Core.csproj b/Backend/Core/Core.csproj
index 717cd8b..1a8d20f 100644
--- a/Backend/Core/Core.csproj
+++ b/Backend/Core/Core.csproj
@@ -4,6 +4,7 @@
     <TargetFramework>net6.0</TargetFramework>
     <ImplicitUsings>enable</ImplicitUsings>
     <Nullable>enable</Nullable>
+	  <ServerGarbageCollection>false</ServerGarbageCollection>
   </PropertyGroup>
 
   <ItemGroup>
@@ -19,6 +20,7 @@
       <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets>
     </PackageReference>
     <PackageReference Include="Microsoft.IdentityModel.Tokens" Version="6.10.0" />
+    <PackageReference Include="Newtonsoft.Json" Version="13.0.1" />
     <PackageReference Include="Npgsql.EntityFrameworkCore.PostgreSQL" Version="6.0.3" />
     <PackageReference Include="Serilog" Version="2.10.0" />
     <PackageReference Include="Serilog.AspNetCore" Version="5.0.0" />
diff --git a/Backend/Core/Entities/Annotation.cs b/Backend/Core/Entities/Annotation.cs
index 2128354..e42f91f 100644
--- a/Backend/Core/Entities/Annotation.cs
+++ b/Backend/Core/Entities/Annotation.cs
@@ -28,6 +28,16 @@ namespace Core.Entities
         public DateTime DateLastChanged { get; set; }
         public string Note { get; set; } = "";
 
+        public string CachedDocumentHTML { get; set; } = "";
+        public string CachedStartPositions { get; set; } = "";
+        public string CachedLengths { get; set; } = "";
+        public string CachedClosingPositions { get; set; } = "";
+        public string CachedClosingLengths { get; set; } = "";
+        //public string CachedNodeDict { get; set; } = "";
+
+        public Guid? LastModifiedTagId { get; set; }
+        public EModified ModifiedType { get; set; } = EModified.NONE;
+
         public ICollection<Class> Classes { get; set; } = new List<Class>();
     }
 }
diff --git a/Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs b/Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs
index d3b1ec1..10e15a9 100644
--- a/Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs
+++ b/Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs
@@ -14,6 +14,7 @@ using Models.Tags;
 using Ganss.XSS;
 using HtmlAgilityPack;
 using System.Text.RegularExpressions;
+using Newtonsoft.Json;
 
 namespace Core.Services.AnnotationService
 {
@@ -25,6 +26,7 @@ namespace Core.Services.AnnotationService
 
         private const string TAG_ID_ATTRIBUTE_NAME = "aswi-tag-id";
         private const string TAG_INSTANCE_ATTRIBUTE_NAME = "aswi-tag-instance";
+        private const string TAG_EF_ID_ATTRIBUTE_NAME = "aswi-tag-ef-id";
 
         public AnnotationServiceEF(DatabaseContext context, ILogger logger, IMapper mapper)
         {
@@ -116,7 +118,8 @@ namespace Core.Services.AnnotationService
             var documentContent = context.Documents.Where(d => d.Id == annotation.Document.Id).Select(d => d.Content).First();
 
             var tags = context.AnnotationTags.Where(at => at.Annotation.Id == annotationId)
-                .Include(at => at.Tag).ThenInclude(t => t.Category)
+                .Include(at => at.Tag)
+                .ThenInclude(t => t.Category)
                 .Include(at => at.SubTag)
                 .ToList();
 
@@ -127,7 +130,50 @@ namespace Core.Services.AnnotationService
                 tagInstanceInfos.Add(tagInstance);
             }
 
-            var docToRender = PreprocessHTML(documentContent.Content, tags);
+            var docToRender = "";
+            if (annotation.CachedDocumentHTML == "")
+            {
+                docToRender = FullPreprocessHTML(documentContent.Content, tags);
+                annotation.CachedStartPositions = JsonConvert.SerializeObject(TagStartPositions);
+                annotation.CachedLengths = JsonConvert.SerializeObject(TagStartLengths);
+                annotation.CachedClosingPositions = JsonConvert.SerializeObject(TagClosingPositions);
+                annotation.CachedClosingLengths = JsonConvert.SerializeObject(TagClosingLengths);
+                //annotation.CachedNodeDict = JsonConvert.SerializeObject(NodeDict);
+                annotation.ModifiedType = EModified.NONE;
+                annotation.CachedDocumentHTML = docToRender;
+                context.SaveChanges();
+            }
+            else
+            {
+                docToRender = annotation.CachedDocumentHTML;
+                TagStartPositions = JsonConvert.DeserializeObject<List<int>>(annotation.CachedStartPositions);
+                TagStartLengths = JsonConvert.DeserializeObject<List<int>>(annotation.CachedLengths);
+                TagClosingPositions = JsonConvert.DeserializeObject<List<int>>(annotation.CachedClosingPositions);
+                TagClosingLengths = JsonConvert.DeserializeObject<List<int>>(annotation.CachedClosingLengths);
+
+                // The annotation has been modified and we need to either add the new tag or remove the tag
+                if (annotation.ModifiedType != EModified.NONE)
+                {
+                    if (annotation.ModifiedType == EModified.ADDED)
+                    {
+                        var lastModifiedTag = context.AnnotationTags.Where(at => at.Id == annotation.LastModifiedTagId).First();
+                        docToRender = PartialPreprocessHTMLAddTag(docToRender, documentContent.Content, lastModifiedTag, tags);
+                    }
+                    else if (annotation.ModifiedType == EModified.REMOVED)
+                    {
+                        docToRender = PartialPreprocessHTMLRemoveTag(docToRender, documentContent.Content, new AnnotationTag() { Id = annotation.LastModifiedTagId.Value }, tags);
+                    }
+
+                    annotation.ModifiedType = EModified.NONE;
+                    annotation.CachedStartPositions = JsonConvert.SerializeObject(TagStartPositions);
+                    annotation.CachedLengths = JsonConvert.SerializeObject(TagStartLengths);
+                    annotation.CachedClosingPositions = JsonConvert.SerializeObject(TagClosingPositions);
+                    annotation.CachedClosingLengths = JsonConvert.SerializeObject(TagClosingLengths);
+                    annotation.CachedDocumentHTML = docToRender;
+                    //annotation.CachedNodeDict = JsonConvert.SerializeObject(NodeDict);
+                    context.SaveChanges();
+                }
+            }
 
             // We probably cannot use AutoMapper since we are dealing with too many different entities
             AnnotationInfo annotationInfo = new()
@@ -142,6 +188,8 @@ namespace Core.Services.AnnotationService
                 TagInstances = tagInstanceInfos
             };
 
+            NodeDict.Clear();
+
             return annotationInfo;
         }
 
@@ -151,7 +199,12 @@ namespace Core.Services.AnnotationService
         private List<int> TagClosingLengths = new();
         private Dictionary<HtmlNode, HtmlNode> NodeDict = new();
 
-        private string PreprocessHTML(string htmlSource, List<AnnotationTag> tags)
+        /*
+         *      Full HTML Preprocessing -------------------------------------------------------------------------------
+         */
+
+
+        private string FullPreprocessHTML(string htmlSource, List<AnnotationTag> tags)
         {
             var docOriginal = new HtmlDocument();
             docOriginal.LoadHtml(htmlSource);
@@ -159,7 +212,7 @@ namespace Core.Services.AnnotationService
             docToEdit.LoadHtml(htmlSource);
 
             var descendantsOriginal = docOriginal.DocumentNode.DescendantsAndSelf();
-            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf();
+            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
 
             int currentId = 0;
 
@@ -168,13 +221,15 @@ namespace Core.Services.AnnotationService
 
             WrapTextInSpan(descendantsOriginal, docToEdit);
 
+            int descCount = descendantsToEdit.Count;
+
             foreach (var tag in tags)
             {
                 int i = 0;
                 List<HtmlNode> addedForSelection = new();
-                while (i < descendantsToEdit.Count())
+                while (i < descCount)
                 {
-                    for (; i < descendantsToEdit.Count(); i++)
+                    for (; i < descCount; i++)
                     {
                         var node = descendantsToEdit.ElementAt(i);
                         if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
@@ -209,6 +264,8 @@ namespace Core.Services.AnnotationService
                             {
                                 addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                             }
+                            descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+                            descCount = descendantsToEdit.Count;
                             break;
                         }
                     }
@@ -227,7 +284,7 @@ namespace Core.Services.AnnotationService
             if (sanitizer.AllowedTags.Contains("script"))
             {
                 sanitizer.AllowedTags.Remove("script");
-            }   
+            }
             if (!sanitizer.AllowedTags.Contains("style"))
             {
                 sanitizer.AllowedTags.Add("style");
@@ -257,8 +314,7 @@ namespace Core.Services.AnnotationService
                 markerPosition = EPosition.MARK_LEFT_RIGHT;
             }
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, start, markerPosition);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, start, markerPosition);
             parentNode.ChildNodes.Insert(nodeIndex, spanSelected);
 
             return spanSelected;
@@ -279,12 +335,10 @@ namespace Core.Services.AnnotationService
             int spanSelectedStart = start;
             int spanAfterStart = start + textSelected.Length;
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_RIGHT);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_RIGHT);
             parentNode.ChildNodes.Insert(nodeIndex, spanSelected);
 
-            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanAfterStart);
+            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
             parentNode.ChildNodes.Insert(nodeIndex + 1, spanAfter);
 
             return new() { spanSelected, spanAfter };
@@ -305,12 +359,10 @@ namespace Core.Services.AnnotationService
             int spanBeforeStart = start;
             int spanSelectedStart = start + textBefore.Length;
 
-            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanBeforeStart);
+            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
             parentNode.ChildNodes.Insert(nodeIndex, spanBefore);
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_LEFT);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT);
             parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);
 
             return new() { spanSelected, spanBefore };
@@ -333,23 +385,19 @@ namespace Core.Services.AnnotationService
             int spanSelectedStart = start + textBefore.Length;
             int spanAfterStart = start + textBefore.Length + textSelected.Length;
 
-            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanBeforeStart);
+            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
             parentNode.ChildNodes.Insert(nodeIndex, spanBefore);
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_LEFT_RIGHT);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT_RIGHT);
             parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);
 
-            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanAfterStart);
+            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
             parentNode.ChildNodes.Insert(nodeIndex + 2, spanAfter);
 
             return new() { spanSelected, spanBefore, spanAfter };
         }
 
-        private HtmlNode CreateSpan(HtmlDocument doc, string text, string tagIdAttributeName, int tagId, string tagInstanceAttributeName,
-                                            Guid? instanceId, int startPosition, EPosition position = EPosition.MARK_NONE)
+        private HtmlNode CreateSpan(HtmlDocument doc, string text, int tagId, Guid? instanceId, Guid? entityId, int startPosition, EPosition position = EPosition.MARK_NONE)
         {
             HtmlNode span = doc.CreateElement("span");
             span.InnerHtml = text;
@@ -357,12 +405,13 @@ namespace Core.Services.AnnotationService
             TagStartLengths.Add(0);
             TagClosingPositions.Add(startPosition + text.Length);
             TagClosingLengths.Add(0);
-            span.Attributes.Add(tagIdAttributeName, tagId.ToString());
+            span.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, tagId.ToString());
 
             if (instanceId != null)
             {
                 span.AddClass("annotation");
-                span.Attributes.Add(tagInstanceAttributeName, instanceId.Value.ToString());
+                span.Attributes.Add(TAG_INSTANCE_ATTRIBUTE_NAME, instanceId.Value.ToString());
+                span.Attributes.Add(TAG_EF_ID_ATTRIBUTE_NAME, entityId.ToString());
 
                 if (position == EPosition.MARK_LEFT || position == EPosition.MARK_LEFT_RIGHT)
                 {
@@ -505,6 +554,159 @@ namespace Core.Services.AnnotationService
             }
         }
 
+
+        /*
+         *      Full HTML Preprocessing -------------------------------------------------------------------------------
+         */
+
+        /*
+         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
+         */
+
+        private string PartialPreprocessHTMLAddTag(string htmlToEdit, string htmlOriginal, AnnotationTag tagToAdd, List<AnnotationTag> tags)
+        {
+            var docOriginal = new HtmlDocument();
+            docOriginal.LoadHtml(htmlOriginal);
+            var docToEdit = new HtmlDocument();
+            docToEdit.LoadHtml(htmlToEdit);
+
+            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+            RemoveCSS(docToEdit);
+
+            int i = 0;
+            List<HtmlNode> addedForSelection = new();
+
+            int descendantsCount = descendantsToEdit.Count();
+            while (i < descendantsCount)
+            {
+                for (; i < descendantsCount; i++)
+                {
+                    var node = descendantsToEdit.ElementAt(i);
+                    if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
+                        node.ParentNode.Name == "style")
+                    {
+                        continue;
+                    }
+
+                    int nodeId = node.ParentNode.GetAttributeValue(TAG_ID_ATTRIBUTE_NAME, -1);
+
+                    var start = TagStartPositions[nodeId] + TagStartLengths[nodeId];
+                    var end = TagClosingPositions[nodeId];
+
+                    int selectionStart = tagToAdd.Position;
+                    int selectionEnd = tagToAdd.Position + tagToAdd.Length;
+
+                    if (selectionStart < end && selectionEnd > start)
+                    {
+                        if (selectionStart <= start && selectionEnd >= end)
+                        {
+                            addedForSelection.Add(SolveFullFill(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        else if (selectionStart <= start)
+                        {
+                            addedForSelection.AddRange(SolveRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        else if (selectionEnd >= end)
+                        {
+                            addedForSelection.AddRange(SolveLeftGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        else
+                        {
+                            addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+                        descendantsCount = descendantsToEdit.Count();
+                        break;
+                    }
+                }
+            }
+
+            var cssNode = GenerateCSS(docToEdit, tags, new());
+            docToEdit.DocumentNode.ChildNodes.Insert(0, cssNode);
+
+            return docToEdit.DocumentNode.OuterHtml;
+        }
+
+
+        private string PartialPreprocessHTMLRemoveTag(string htmlToEdit, string htmlOriginal, AnnotationTag tagToRemove, List<AnnotationTag> tags)
+        {
+            var docOriginal = new HtmlDocument();
+            docOriginal.LoadHtml(htmlOriginal);
+            var docToEdit = new HtmlDocument();
+            docToEdit.LoadHtml(htmlToEdit);
+
+            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+            RemoveCSS(docToEdit);
+
+            int i = 0;
+            int descendantsCount = descendantsToEdit.Count();
+            while (i < descendantsCount)
+            {
+                for (; i < descendantsCount; i++)
+                {
+                    var node = descendantsToEdit.ElementAt(i);
+                    if (!node.Attributes.Contains(TAG_EF_ID_ATTRIBUTE_NAME))
+                    {
+                        continue;
+                    }
+                    else
+                    {
+                        if (node.Attributes[TAG_EF_ID_ATTRIBUTE_NAME].Value != tagToRemove.Id.ToString())
+                        {
+                            continue;
+                        }
+
+                        node.Attributes.Remove(TAG_EF_ID_ATTRIBUTE_NAME);
+                        node.Attributes.Remove(TAG_INSTANCE_ATTRIBUTE_NAME);
+                        node.Attributes.Remove("class");
+                        node.Attributes.Remove("start");
+                        node.Attributes.Remove("end");
+
+                        /*var parent = node.ParentNode;
+                        var contents = node.ChildNodes;
+                        int index = parent.ChildNodes.IndexOf(node);
+                        parent.ChildNodes.RemoveAt(index);
+
+                        List<HtmlNode> newChildren = new();
+                        for (int j = 0; j < index; j++)
+                        {
+                            newChildren.Add(parent.ChildNodes[j]);
+                        }
+                        for (int j = 0; j < contents.Count; j++)
+                        {
+                            newChildren.Add(contents[j]);
+                        }
+                        for (int j = index; j < parent.ChildNodes.Count; j++)
+                        {
+                            newChildren.Add(parent.ChildNodes[j]);
+                        }
+
+                        parent.ChildNodes.Clear();
+                        foreach (var child in newChildren) { parent.ChildNodes.Add(child); }*/
+                        
+                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+                        descendantsCount = descendantsToEdit.Count();
+                        break;
+                    }
+                }
+            }
+
+            var cssNode = GenerateCSS(docToEdit, tags, new());
+            docToEdit.DocumentNode.ChildNodes.Insert(0, cssNode);
+
+            return docToEdit.DocumentNode.OuterHtml;
+        }
+
+        private void RemoveCSS(HtmlDocument doc)
+        {
+            doc.DocumentNode.ChildNodes.RemoveAt(0);
+        }
+
+        /*
+         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
+         */
+
+
         // TODO temporary
         private bool IsHtml(string text)
         {
@@ -529,6 +731,7 @@ namespace Core.Services.AnnotationService
 
             AnnotationTag annotationTag = new()
             {
+                Id = Guid.NewGuid(),
                 Annotation = annotation,
                 Instance = request.InstanceId == null ? Guid.NewGuid() : request.InstanceId.Value,
                 Length = request.Length,
@@ -552,6 +755,9 @@ namespace Core.Services.AnnotationService
                 throw new ArgumentException($"Unknown tag type {request.Type}");
             }
 
+            annotation.LastModifiedTagId = annotationTag.Id;
+            annotation.ModifiedType = EModified.ADDED;
+
             context.AnnotationTags.Add(annotationTag);
             context.SaveChanges();
         }
@@ -587,9 +793,12 @@ namespace Core.Services.AnnotationService
                 throw new InvalidOperationException("Could not find tag instance");
             }
 
-            context.AnnotationTags
-                .Where(at => at.Id == tagInstanceId).ToList()
-                .ForEach(a => context.AnnotationTags.Remove(a));
+
+            var annotationTag = context.AnnotationTags.First(at => at.Id == tagInstanceId);
+            annotation.LastModifiedTagId = annotationTag.Id;
+            annotation.ModifiedType = EModified.REMOVED;
+
+            context.AnnotationTags.Remove(annotationTag);
 
             context.SaveChanges();
         }
diff --git a/Backend/Models/Enums/EModified.cs b/Backend/Models/Enums/EModified.cs
new file mode 100644
index 0000000..ed0a5ff
--- /dev/null
+++ b/Backend/Models/Enums/EModified.cs
@@ -0,0 +1,15 @@
+﻿using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Models.Enums
+{
+    public enum EModified
+    {
+        ADDED,
+        REMOVED,
+        NONE
+    }
+}
