Revize 553486ad

Added caching and incremental updates to annotations

Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs

@@ -14 +14 @@
 using Ganss.XSS;
 using HtmlAgilityPack;
 using System.Text.RegularExpressions;
+using Newtonsoft.Json;
 
 namespace Core.Services.AnnotationService
 {
@@ -25 +26 @@
 
         private const string TAG_ID_ATTRIBUTE_NAME = "aswi-tag-id";
         private const string TAG_INSTANCE_ATTRIBUTE_NAME = "aswi-tag-instance";
+        private const string TAG_EF_ID_ATTRIBUTE_NAME = "aswi-tag-ef-id";
 
         public AnnotationServiceEF(DatabaseContext context, ILogger logger, IMapper mapper)
         {
@@ -116 +118 @@
             var documentContent = context.Documents.Where(d => d.Id == annotation.Document.Id).Select(d => d.Content).First();
 
             var tags = context.AnnotationTags.Where(at => at.Annotation.Id == annotationId)
-                .Include(at => at.Tag).ThenInclude(t => t.Category)
+                .Include(at => at.Tag)
+                .ThenInclude(t => t.Category)
                 .Include(at => at.SubTag)
                 .ToList();
 
@@ -127 +130 @@
                 tagInstanceInfos.Add(tagInstance);
             }
 
-            var docToRender = PreprocessHTML(documentContent.Content, tags);
+            var docToRender = "";
+            if (annotation.CachedDocumentHTML == "")
+            {
+                docToRender = FullPreprocessHTML(documentContent.Content, tags);
+                annotation.CachedStartPositions = JsonConvert.SerializeObject(TagStartPositions);
+                annotation.CachedLengths = JsonConvert.SerializeObject(TagStartLengths);
+                annotation.CachedClosingPositions = JsonConvert.SerializeObject(TagClosingPositions);
+                annotation.CachedClosingLengths = JsonConvert.SerializeObject(TagClosingLengths);
+                //annotation.CachedNodeDict = JsonConvert.SerializeObject(NodeDict);
+                annotation.ModifiedType = EModified.NONE;
+                annotation.CachedDocumentHTML = docToRender;
+                context.SaveChanges();
+            }
+            else
+            {
+                docToRender = annotation.CachedDocumentHTML;
+                TagStartPositions = JsonConvert.DeserializeObject<List<int>>(annotation.CachedStartPositions);
+                TagStartLengths = JsonConvert.DeserializeObject<List<int>>(annotation.CachedLengths);
+                TagClosingPositions = JsonConvert.DeserializeObject<List<int>>(annotation.CachedClosingPositions);
+                TagClosingLengths = JsonConvert.DeserializeObject<List<int>>(annotation.CachedClosingLengths);
+
+                // The annotation has been modified and we need to either add the new tag or remove the tag
+                if (annotation.ModifiedType != EModified.NONE)
+                {
+                    if (annotation.ModifiedType == EModified.ADDED)
+                    {
+                        var lastModifiedTag = context.AnnotationTags.Where(at => at.Id == annotation.LastModifiedTagId).First();
+                        docToRender = PartialPreprocessHTMLAddTag(docToRender, documentContent.Content, lastModifiedTag, tags);
+                    }
+                    else if (annotation.ModifiedType == EModified.REMOVED)
+                    {
+                        docToRender = PartialPreprocessHTMLRemoveTag(docToRender, documentContent.Content, new AnnotationTag() { Id = annotation.LastModifiedTagId.Value }, tags);
+                    }
+
+                    annotation.ModifiedType = EModified.NONE;
+                    annotation.CachedStartPositions = JsonConvert.SerializeObject(TagStartPositions);
+                    annotation.CachedLengths = JsonConvert.SerializeObject(TagStartLengths);
+                    annotation.CachedClosingPositions = JsonConvert.SerializeObject(TagClosingPositions);
+                    annotation.CachedClosingLengths = JsonConvert.SerializeObject(TagClosingLengths);
+                    annotation.CachedDocumentHTML = docToRender;
+                    //annotation.CachedNodeDict = JsonConvert.SerializeObject(NodeDict);
+                    context.SaveChanges();
+                }
+            }
 
             // We probably cannot use AutoMapper since we are dealing with too many different entities
             AnnotationInfo annotationInfo = new()
@@ -142 +188 @@
                 TagInstances = tagInstanceInfos
             };
 
+            NodeDict.Clear();
+
             return annotationInfo;
         }
 
@@ -151 +199 @@
         private List<int> TagClosingLengths = new();
         private Dictionary<HtmlNode, HtmlNode> NodeDict = new();
 
-        private string PreprocessHTML(string htmlSource, List<AnnotationTag> tags)
+        /*
+         *      Full HTML Preprocessing -------------------------------------------------------------------------------
+         */
+
+
+        private string FullPreprocessHTML(string htmlSource, List<AnnotationTag> tags)
         {
             var docOriginal = new HtmlDocument();
             docOriginal.LoadHtml(htmlSource);
@@ -159 +212 @@
             docToEdit.LoadHtml(htmlSource);
 
             var descendantsOriginal = docOriginal.DocumentNode.DescendantsAndSelf();
-            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf();
+            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
 
             int currentId = 0;
 
@@ -168 +221 @@
 
             WrapTextInSpan(descendantsOriginal, docToEdit);
 
+            int descCount = descendantsToEdit.Count;
+
             foreach (var tag in tags)
             {
                 int i = 0;
                 List<HtmlNode> addedForSelection = new();
-                while (i < descendantsToEdit.Count())
+                while (i < descCount)
                 {
-                    for (; i < descendantsToEdit.Count(); i++)
+                    for (; i < descCount; i++)
                     {
                         var node = descendantsToEdit.ElementAt(i);
                         if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
@@ -209 +264 @@
                             {
                                 addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                             }
+                            descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+                            descCount = descendantsToEdit.Count;
                             break;
                         }
                     }
@@ -227 +284 @@
             if (sanitizer.AllowedTags.Contains("script"))
             {
                 sanitizer.AllowedTags.Remove("script");
-            }   
+            }
             if (!sanitizer.AllowedTags.Contains("style"))
             {
                 sanitizer.AllowedTags.Add("style");
@@ -257 +314 @@
                 markerPosition = EPosition.MARK_LEFT_RIGHT;
             }
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, start, markerPosition);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, start, markerPosition);
             parentNode.ChildNodes.Insert(nodeIndex, spanSelected);
 
             return spanSelected;
@@ -279 +335 @@
             int spanSelectedStart = start;
             int spanAfterStart = start + textSelected.Length;
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_RIGHT);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_RIGHT);
             parentNode.ChildNodes.Insert(nodeIndex, spanSelected);
 
-            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanAfterStart);
+            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
             parentNode.ChildNodes.Insert(nodeIndex + 1, spanAfter);
 
             return new() { spanSelected, spanAfter };
@@ -305 +359 @@
             int spanBeforeStart = start;
             int spanSelectedStart = start + textBefore.Length;
 
-            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanBeforeStart);
+            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
             parentNode.ChildNodes.Insert(nodeIndex, spanBefore);
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_LEFT);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT);
             parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);
 
             return new() { spanSelected, spanBefore };
@@ -333 +385 @@
             int spanSelectedStart = start + textBefore.Length;
             int spanAfterStart = start + textBefore.Length + textSelected.Length;
 
-            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanBeforeStart);
+            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
             parentNode.ChildNodes.Insert(nodeIndex, spanBefore);
 
-            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_LEFT_RIGHT);
+            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT_RIGHT);
             parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);
 
-            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
-                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanAfterStart);
+            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
             parentNode.ChildNodes.Insert(nodeIndex + 2, spanAfter);
 
             return new() { spanSelected, spanBefore, spanAfter };
         }
 
-        private HtmlNode CreateSpan(HtmlDocument doc, string text, string tagIdAttributeName, int tagId, string tagInstanceAttributeName,
-                                            Guid? instanceId, int startPosition, EPosition position = EPosition.MARK_NONE)
+        private HtmlNode CreateSpan(HtmlDocument doc, string text, int tagId, Guid? instanceId, Guid? entityId, int startPosition, EPosition position = EPosition.MARK_NONE)
         {
             HtmlNode span = doc.CreateElement("span");
             span.InnerHtml = text;
@@ -357 +405 @@
             TagStartLengths.Add(0);
             TagClosingPositions.Add(startPosition + text.Length);
             TagClosingLengths.Add(0);
-            span.Attributes.Add(tagIdAttributeName, tagId.ToString());
+            span.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, tagId.ToString());
 
             if (instanceId != null)
             {
                 span.AddClass("annotation");
-                span.Attributes.Add(tagInstanceAttributeName, instanceId.Value.ToString());
+                span.Attributes.Add(TAG_INSTANCE_ATTRIBUTE_NAME, instanceId.Value.ToString());
+                span.Attributes.Add(TAG_EF_ID_ATTRIBUTE_NAME, entityId.ToString());
 
                 if (position == EPosition.MARK_LEFT || position == EPosition.MARK_LEFT_RIGHT)
                 {
@@ -505 +554 @@
             }
         }
 
+
+        /*
+         *      Full HTML Preprocessing -------------------------------------------------------------------------------
+         */
+
+        /*
+         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
+         */
+
+        private string PartialPreprocessHTMLAddTag(string htmlToEdit, string htmlOriginal, AnnotationTag tagToAdd, List<AnnotationTag> tags)
+        {
+            var docOriginal = new HtmlDocument();
+            docOriginal.LoadHtml(htmlOriginal);
+            var docToEdit = new HtmlDocument();
+            docToEdit.LoadHtml(htmlToEdit);
+
+            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+            RemoveCSS(docToEdit);
+
+            int i = 0;
+            List<HtmlNode> addedForSelection = new();
+
+            int descendantsCount = descendantsToEdit.Count();
+            while (i < descendantsCount)
+            {
+                for (; i < descendantsCount; i++)
+                {
+                    var node = descendantsToEdit.ElementAt(i);
+                    if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
+                        node.ParentNode.Name == "style")
+                    {
+                        continue;
+                    }
+
+                    int nodeId = node.ParentNode.GetAttributeValue(TAG_ID_ATTRIBUTE_NAME, -1);
+
+                    var start = TagStartPositions[nodeId] + TagStartLengths[nodeId];
+                    var end = TagClosingPositions[nodeId];
+
+                    int selectionStart = tagToAdd.Position;
+                    int selectionEnd = tagToAdd.Position + tagToAdd.Length;
+
+                    if (selectionStart < end && selectionEnd > start)
+                    {
+                        if (selectionStart <= start && selectionEnd >= end)
+                        {
+                            addedForSelection.Add(SolveFullFill(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        else if (selectionStart <= start)
+                        {
+                            addedForSelection.AddRange(SolveRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        else if (selectionEnd >= end)
+                        {
+                            addedForSelection.AddRange(SolveLeftGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        else
+                        {
+                            addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
+                        }
+                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+                        descendantsCount = descendantsToEdit.Count();
+                        break;
+                    }
+                }
+            }
+
+            var cssNode = GenerateCSS(docToEdit, tags, new());
+            docToEdit.DocumentNode.ChildNodes.Insert(0, cssNode);
+
+            return docToEdit.DocumentNode.OuterHtml;
+        }
+
+
+        private string PartialPreprocessHTMLRemoveTag(string htmlToEdit, string htmlOriginal, AnnotationTag tagToRemove, List<AnnotationTag> tags)
+        {
+            var docOriginal = new HtmlDocument();
+            docOriginal.LoadHtml(htmlOriginal);
+            var docToEdit = new HtmlDocument();
+            docToEdit.LoadHtml(htmlToEdit);
+
+            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+            RemoveCSS(docToEdit);
+
+            int i = 0;
+            int descendantsCount = descendantsToEdit.Count();
+            while (i < descendantsCount)
+            {
+                for (; i < descendantsCount; i++)
+                {
+                    var node = descendantsToEdit.ElementAt(i);
+                    if (!node.Attributes.Contains(TAG_EF_ID_ATTRIBUTE_NAME))
+                    {
+                        continue;
+                    }
+                    else
+                    {
+                        if (node.Attributes[TAG_EF_ID_ATTRIBUTE_NAME].Value != tagToRemove.Id.ToString())
+                        {
+                            continue;
+                        }
+
+                        node.Attributes.Remove(TAG_EF_ID_ATTRIBUTE_NAME);
+                        node.Attributes.Remove(TAG_INSTANCE_ATTRIBUTE_NAME);
+                        node.Attributes.Remove("class");
+                        node.Attributes.Remove("start");
+                        node.Attributes.Remove("end");
+
+                        /*var parent = node.ParentNode;
+                        var contents = node.ChildNodes;
+                        int index = parent.ChildNodes.IndexOf(node);
+                        parent.ChildNodes.RemoveAt(index);
+
+                        List<HtmlNode> newChildren = new();
+                        for (int j = 0; j < index; j++)
+                        {
+                            newChildren.Add(parent.ChildNodes[j]);
+                        }
+                        for (int j = 0; j < contents.Count; j++)
+                        {
+                            newChildren.Add(contents[j]);
+                        }
+                        for (int j = index; j < parent.ChildNodes.Count; j++)
+                        {
+                            newChildren.Add(parent.ChildNodes[j]);
+                        }
+
+                        parent.ChildNodes.Clear();
+                        foreach (var child in newChildren) { parent.ChildNodes.Add(child); }*/
+                        
+                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
+                        descendantsCount = descendantsToEdit.Count();
+                        break;
+                    }
+                }
+            }
+
+            var cssNode = GenerateCSS(docToEdit, tags, new());
+            docToEdit.DocumentNode.ChildNodes.Insert(0, cssNode);
+
+            return docToEdit.DocumentNode.OuterHtml;
+        }
+
+        private void RemoveCSS(HtmlDocument doc)
+        {
+            doc.DocumentNode.ChildNodes.RemoveAt(0);
+        }
+
+        /*
+         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
+         */
+
+
         // TODO temporary
         private bool IsHtml(string text)
         {
@@ -529 +731 @@
 
             AnnotationTag annotationTag = new()
             {
+                Id = Guid.NewGuid(),
                 Annotation = annotation,
                 Instance = request.InstanceId == null ? Guid.NewGuid() : request.InstanceId.Value,
                 Length = request.Length,
@@ -552 +755 @@
                 throw new ArgumentException($"Unknown tag type {request.Type}");
             }
 
+            annotation.LastModifiedTagId = annotationTag.Id;
+            annotation.ModifiedType = EModified.ADDED;
+
             context.AnnotationTags.Add(annotationTag);
             context.SaveChanges();
         }
@@ -587 +793 @@
                 throw new InvalidOperationException("Could not find tag instance");
             }
 
-            context.AnnotationTags
-                .Where(at => at.Id == tagInstanceId).ToList()
-                .ForEach(a => context.AnnotationTags.Remove(a));
+
+            var annotationTag = context.AnnotationTags.First(at => at.Id == tagInstanceId);
+            annotation.LastModifiedTagId = annotationTag.Id;
+            annotation.ModifiedType = EModified.REMOVED;
+
+            context.AnnotationTags.Remove(annotationTag);
 
             context.SaveChanges();
         }