aswi20220x00-gitlab/Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs @ 4e12f0cc

using Core.Contexts;
using Core.Entities;
using Models.Annotations;
using Models.Enums;
using Serilog;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using AutoMapper;
using Models.Tags;
using Ganss.XSS;
using HtmlAgilityPack;
using System.Text.RegularExpressions;
using Newtonsoft.Json;
using Core.GraphUtils;

namespace Core.Services.AnnotationService
{
    public class AnnotationServiceEF : IAnnotationService
    {
        private readonly DatabaseContext context;
        private readonly ILogger logger;
        private readonly IMapper mapper;

        private const string TAG_ID_ATTRIBUTE_NAME = "aswi-tag-id";
        private const string TAG_INSTANCE_ATTRIBUTE_NAME = "aswi-tag-instance";
        private const string TAG_EF_ID_ATTRIBUTE_NAME = "aswi-tag-ef-id";

        public AnnotationServiceEF(DatabaseContext context, ILogger logger, IMapper mapper)
        {
            this.context = context;
            this.logger = logger;
            this.mapper = mapper;
        }

        public void CreateDocumentAnnotations(AnnotationsAddRequest request, Guid clientUserId)
        {
            User addingUser = context.Users.Single(u => u.Id == clientUserId);

            // Check the documents exist
            var documents = context.Documents.Where(d => request.DocumentIdList.Contains(d.Id)).ToList();
            if (documents.Count() != request.DocumentIdList.Count)
            {
                logger.Information($"Received a non-existent Document ID when assigning documents to users");
                throw new InvalidOperationException($"{request.DocumentIdList.Count - documents.Count()} of the received documents do not exist");
            }

            var users = context.Users.Where(u => request.UserIdList.Contains(u.Id)).ToList();
            foreach (var user in users)
            {
                var userAnnotatedDocuments = context.Annotations.Where(a => a.User == user).Select(a => a.Document).ToList();
                foreach (var doc in documents)
                {
                    if (userAnnotatedDocuments.Contains(doc))
                    {
                        logger.Information($"User {user.Username} has already been assigned the document {doc.Id}, ignoring");
                        continue;
                    }

                    context.Annotations.Add(new Annotation()
                    {
                        User = user,
                        UserAssigned = addingUser,
                        DateAssigned = DateTime.Now,
                        DateLastChanged = DateTime.Now,
                        Document = doc,
                        State = EState.NEW,
                        Note = ""
                    });
                }
            }

            context.SaveChanges();
        }

        public AnnotationListResponse GetUserAnnotations(Guid userId)
        {
            var annotations = context.Annotations.Where(a => a.User.Id == userId).Include(a => a.Document).ToList();
            var documentIds = annotations.Select(a => a.Document.Id).ToList();
            var documents = context.Documents.Where(d => documentIds.Contains(d.Id));
            var infos = new List<AnnotationListInfo>();

            var annotationsDocuments = annotations.Zip(documents, (a, d) => new { Annotation = a, Document = d });
            foreach (var ad in annotationsDocuments)
            {
                infos.Add(new AnnotationListInfo()
                {
                    AnnotationId = ad.Annotation.Id,
                    DocumentName = ad.Document.Name,
                    State = ad.Annotation.State
                });
            }

            return new AnnotationListResponse()
            {
                Annotations = infos
            };
        }

        public void AddNoteToAnnotation(Guid annotationId, Guid userId, ERole userRole, AddNoteToAnnotationRequest request)
        {
            Annotation annotation = null;
            try
            {
                annotation = context.Annotations.Include(a => a.User).First(a => a.Id == annotationId);
            }
            catch (Exception)
            {
                throw new InvalidOperationException("Annotation not found");
            }

            if (userRole < ERole.ADMINISTRATOR && annotation.User.Id != userId)
            {
                throw new UnauthorizedAccessException("User does not have access to this annotation");
            }

            annotation.Note = request.Note;
            context.SaveChanges();
        }


        public AnnotationInfo GetAnnotation(Guid annotationId, Guid userId, ERole userRole)
        {
            var annotation = context.Annotations
                .Where(a => a.Id == annotationId)
                .Include(a => a.User)
                .Include(a => a.Document).ThenInclude(d => d.Content)
                .First();

            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != userId)
                {
                    throw new UnauthorizedAccessException($"User {userId} does not have assigned annotation {annotationId}");
                }
            }

            var documentContent = context.Documents.Where(d => d.Id == annotation.Document.Id).Select(d => d.Content).First();

            var tags = context.AnnotationTags.Where(at => at.Annotation.Id == annotationId)
                .Include(at => at.Tag)
                .ThenInclude(t => t.Category)
                .Include(at => at.SubTag)
                .OrderBy(at => at.Position)
                .ToList();

            List<TagInstanceInfo> tagInstanceInfos = new();
            foreach (var tag in tags)
            {
                var tagInstance = mapper.Map<TagInstanceInfo>(tag);
                tagInstanceInfos.Add(tagInstance);
            }

            var docToRender = "";
            if (annotation.CachedDocumentHTML == "")
            {
                docToRender = FullPreprocessHTML(documentContent.Content, tags);
                annotation.CachedStartPositions = JsonConvert.SerializeObject(TagStartPositions);
                annotation.CachedLengths = JsonConvert.SerializeObject(TagStartLengths);
                annotation.CachedClosingPositions = JsonConvert.SerializeObject(TagClosingPositions);
                annotation.CachedClosingLengths = JsonConvert.SerializeObject(TagClosingLengths);
                annotation.CachedCSS = JsonConvert.SerializeObject(TagInstanceCSS);
                annotation.ModifiedType = EModified.NONE;
                annotation.CachedDocumentHTML = docToRender;
                context.SaveChanges();
            }
            else
            {
                docToRender = annotation.CachedDocumentHTML;
                TagStartPositions = JsonConvert.DeserializeObject<List<int>>(annotation.CachedStartPositions);
                TagStartLengths = JsonConvert.DeserializeObject<List<int>>(annotation.CachedLengths);
                TagClosingPositions = JsonConvert.DeserializeObject<List<int>>(annotation.CachedClosingPositions);
                TagClosingLengths = JsonConvert.DeserializeObject<List<int>>(annotation.CachedClosingLengths);

                // The annotation has been modified and we need to either add the new tag or remove the tag
                if (annotation.ModifiedType != EModified.NONE)
                {
                    if (annotation.ModifiedType == EModified.ADDED)
                    {
                        var lastModifiedTag = context.AnnotationTags.Where(at => at.Id == annotation.LastModifiedTagId).First();
                        docToRender = PartialPreprocessHTMLAddTag(docToRender, documentContent.Content, lastModifiedTag, tags);
                    }
                    else if (annotation.ModifiedType == EModified.REMOVED)
                    {
                        docToRender = PartialPreprocessHTMLRemoveTag(docToRender, documentContent.Content, new AnnotationTag() { Id = annotation.LastModifiedTagId.Value }, tags);
                    }

                    annotation.ModifiedType = EModified.NONE;
                    annotation.CachedStartPositions = JsonConvert.SerializeObject(TagStartPositions);
                    annotation.CachedLengths = JsonConvert.SerializeObject(TagStartLengths);
                    annotation.CachedClosingPositions = JsonConvert.SerializeObject(TagClosingPositions);
                    annotation.CachedClosingLengths = JsonConvert.SerializeObject(TagClosingLengths);
                    annotation.CachedDocumentHTML = docToRender;
                    //annotation.CachedNodeDict = JsonConvert.SerializeObject(NodeDict);
                    context.SaveChanges();
                }
            }

            // We probably cannot use AutoMapper since we are dealing with too many different entities
            AnnotationInfo annotationInfo = new()
            {
                SourceDocumentContent = documentContent.Content,
                DocumentToRender = docToRender,
                TagStartPositions = TagStartPositions.ToArray(),
                TagLengths = TagStartLengths.ToArray(),
                Note = annotation.Note,
                State = annotation.State,
                Type = IsHtml(documentContent.Content) ? EDocumentType.HTML : EDocumentType.TEXT,
                TagInstances = tagInstanceInfos
            };

            NodeDict.Clear();

            return annotationInfo;
        }

        private List<int> TagStartPositions = new();
        private List<int> TagStartLengths = new();
        private List<int> TagClosingPositions = new();
        private List<int> TagClosingLengths = new();
        private Dictionary<HtmlNode, HtmlNode> NodeDict = new();
        private List<TagInstanceCSSInfo> TagInstanceCSS = new();

        /*
         *      Full HTML Preprocessing -------------------------------------------------------------------------------
         */


        private string FullPreprocessHTML(string htmlSource, List<AnnotationTag> tags)
        {
            var docOriginal = new HtmlDocument();
            docOriginal.LoadHtml(htmlSource);
            var docToEdit = new HtmlDocument();
            docToEdit.LoadHtml(htmlSource);

            var descendantsOriginal = docOriginal.DocumentNode.DescendantsAndSelf();
            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();

            int currentId = 0;

            FillNodeDict(descendantsOriginal, descendantsToEdit);
            AssignIdsToOriginalDocument(descendantsOriginal, ref currentId);

            WrapTextInSpan(descendantsOriginal, docToEdit);

            int descCount = descendantsToEdit.Count;

            foreach (var tag in tags)
            {
                int i = 0;
                List<HtmlNode> addedForSelection = new();
                while (i < descCount)
                {
                    for (; i < descCount; i++)
                    {
                        var node = descendantsToEdit.ElementAt(i);
                        if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
                            node.ParentNode.Name == "style")
                        {
                            continue;
                        }

                        int nodeId = node.ParentNode.GetAttributeValue(TAG_ID_ATTRIBUTE_NAME, -1);

                        var start = TagStartPositions[nodeId] + TagStartLengths[nodeId];
                        var end = TagClosingPositions[nodeId];

                        int selectionStart = tag.Position;
                        int selectionEnd = tag.Position + tag.Length;

                        if (selectionStart < end && selectionEnd > start)
                        {
                            if (selectionStart <= start && selectionEnd >= end)
                            {
                                addedForSelection.Add(SolveFullFill(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else if (selectionStart <= start)
                            {
                                addedForSelection.AddRange(SolveRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else if (selectionEnd >= end)
                            {
                                addedForSelection.AddRange(SolveLeftGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else
                            {
                                addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
                            descCount = descendantsToEdit.Count;
                            break;
                        }
                    }
                }

            }

            string docToRender = docToEdit.DocumentNode.OuterHtml;
            HtmlSanitizer sanitizer = new HtmlSanitizer();
            sanitizer.AllowedAttributes.Clear();
            sanitizer.AllowedAttributes.Add(TAG_ID_ATTRIBUTE_NAME);
            sanitizer.AllowedAttributes.Add(TAG_INSTANCE_ATTRIBUTE_NAME);
            sanitizer.AllowedAttributes.Add("end");
            sanitizer.AllowedAttributes.Add("start");
            sanitizer.AllowedAttributes.Add("class");
            if (sanitizer.AllowedTags.Contains("script"))
            {
                sanitizer.AllowedTags.Remove("script");
            }
            if (!sanitizer.AllowedTags.Contains("style"))
            {
                sanitizer.AllowedTags.Add("style");
            }
            docToRender = sanitizer.Sanitize(docToRender);        
            GenerateCSS(tags);
            return docToRender;
        }

        private HtmlNode SolveFullFill(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit, AnnotationTag tag)
        {
            // full fill
            string textSelected = node.InnerText;

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            EPosition markerPosition = EPosition.MARK_NONE;
            if (selectionEnd == end && selectionStart == start)
            {
                markerPosition = EPosition.MARK_LEFT_RIGHT;
            }

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, start, markerPosition);
            parentNode.ChildNodes.Insert(nodeIndex, spanSelected);

            return spanSelected;
        }

        private List<HtmlNode> SolveRightGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                             AnnotationTag tag)
        {
            // partial fill, end gap
            string text = node.InnerText;
            string textAfter = text.Substring(Math.Min(selectionStart - start + tag.Length, text.Length));
            string textSelected = text.Substring(0, selectionEnd - start);

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanSelectedStart = start;
            int spanAfterStart = start + textSelected.Length;

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_RIGHT);
            parentNode.ChildNodes.Insert(nodeIndex, spanSelected);

            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanAfter);

            return new() { spanSelected, spanAfter };
        }

        private List<HtmlNode> SolveLeftGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                             AnnotationTag tag)
        {
            // partial fill, start gap
            string text = node.InnerText;
            string textBefore = text.Substring(0, selectionStart - start);
            string textSelected = text.Substring(selectionStart - start, Math.Min(tag.Length, text.Length - textBefore.Length));

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanBeforeStart = start;
            int spanSelectedStart = start + textBefore.Length;

            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
            parentNode.ChildNodes.Insert(nodeIndex, spanBefore);

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);

            return new() { spanSelected, spanBefore };
        }

        private List<HtmlNode> SolveLeftRightGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                                 AnnotationTag tag)
        {
            // partial fill, start gap end gap
            string text = node.InnerText;
            string textBefore = text.Substring(0, selectionStart - start);
            string textAfter = text.Substring(selectionStart - start + tag.Length);
            string textSelected = text.Substring(selectionStart - start, tag.Length);

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanBeforeStart = start;
            int spanSelectedStart = start + textBefore.Length;
            int spanAfterStart = start + textBefore.Length + textSelected.Length;

            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
            parentNode.ChildNodes.Insert(nodeIndex, spanBefore);

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT_RIGHT);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);

            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
            parentNode.ChildNodes.Insert(nodeIndex + 2, spanAfter);

            return new() { spanSelected, spanBefore, spanAfter };
        }

        private HtmlNode CreateSpan(HtmlDocument doc, string text, int tagId, Guid? instanceId, Guid? entityId, int startPosition, EPosition position = EPosition.MARK_NONE)
        {
            HtmlNode span = doc.CreateElement("span");
            span.InnerHtml = text;
            TagStartPositions.Add(startPosition);
            TagStartLengths.Add(0);
            TagClosingPositions.Add(startPosition + text.Length);
            TagClosingLengths.Add(0);
            span.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, tagId.ToString());

            if (instanceId != null)
            {
                span.AddClass("annotation");
                span.Attributes.Add(TAG_INSTANCE_ATTRIBUTE_NAME, instanceId.Value.ToString());
                span.Attributes.Add(TAG_EF_ID_ATTRIBUTE_NAME, entityId.ToString());

                if (position == EPosition.MARK_LEFT || position == EPosition.MARK_LEFT_RIGHT)
                {
                    span.Attributes.Add("start", "1");
                }
                if (position == EPosition.MARK_RIGHT || position == EPosition.MARK_LEFT_RIGHT)
                {
                    span.Attributes.Add("end", "1");
                }
            }

            return span;
        }

        private enum EPosition
        {
            MARK_LEFT = 5,
            MARK_RIGHT = 3,
            MARK_LEFT_RIGHT = 2,
            MARK_NONE = 0
        }

        private void WrapTextInSpan(IEnumerable<HtmlNode> descendantsOriginal, HtmlDocument docToEdit)
        {
            // Special case for non-html documents
            if (descendantsOriginal.Count() == 2)
            {
                var documentNode = descendantsOriginal.ElementAt(0);
                var childNode = descendantsOriginal.ElementAt(1);
                if (documentNode.Name == "#document" && childNode.Name == "#text")
                {
                    HtmlNode coveringSpan = docToEdit.CreateElement("span");
                    coveringSpan.InnerHtml = childNode.InnerHtml;
                    TagStartPositions.Add(childNode.InnerStartIndex);
                    TagStartLengths.Add(0);
                    TagClosingPositions.Add(childNode.InnerStartIndex + childNode.InnerLength);
                    TagClosingLengths.Add(0);
                    coveringSpan.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, (TagStartPositions.Count - 1).ToString());

                    var parent = NodeDict[documentNode];

                    parent.ChildNodes.RemoveAt(0);
                    parent.ChildNodes.Add(coveringSpan);

                    return;
                }
            }

            foreach (var node in descendantsOriginal)
            {
                var originalNode = node;
                var toEditNode = NodeDict[node];

                if (originalNode.Name.Contains("#"))
                {
                    continue;
                }
                else
                {
                    bool onlyText = true;
                    bool onlySubtags = true;

                    foreach (var child in node.ChildNodes)
                    {
                        if (child.Name.Contains("#"))
                        {
                            onlySubtags = false;
                        }
                        else
                        {
                            onlyText = false;
                        }
                    }

                    if (onlyText || onlySubtags)
                    {
                        continue;
                    }
                    else
                    {

                        foreach (var child in node.ChildNodes)
                        {
                            if (child.Name.Contains("#text"))
                            {
                                HtmlNode coveringSpan = docToEdit.CreateElement("span");
                                coveringSpan.InnerHtml = child.InnerHtml;
                                TagStartPositions.Add(child.InnerStartIndex);
                                TagStartLengths.Add(0);
                                TagClosingPositions.Add(child.InnerStartIndex + child.InnerLength);
                                TagClosingLengths.Add(0);
                                coveringSpan.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, (TagStartPositions.Count - 1).ToString());

                                var parent = NodeDict[node];
                                var index = parent.ChildNodes.IndexOf(NodeDict[child]);

                                parent.ChildNodes.RemoveAt(index);
                                parent.ChildNodes.Insert(index, coveringSpan);
                            }
                        }
                    }
                }
            }
        }

        private void GenerateCSS(List<AnnotationTag> tags)
        {
            /*string inner = "span.annotation {border-bottom: 2px solid;}";
            inner += "span {line-height: 30px}\n";*/

            var tagPaddingDict = Intersections.ColorGraph(Intersections.FindIntersections(tags));
            foreach (var tag in tags)
            {
                var padding = (tagPaddingDict[tag] + 1) * 2;
                TagInstanceCSS.Add(new()
                {
                    InstanceId = tag.Instance,
                    Color = tag.Tag.Color,
                    Padding = padding
                });             
            }
        }

        private void AssignIdsToOriginalDocument(IEnumerable<HtmlNode> descendantsOriginal, ref int currentId)
        {
            foreach (var node in descendantsOriginal)
            {
                var originalNode = node;
                var toEditNode = NodeDict[node];

                if (originalNode.Name.Contains("#"))
                {
                    continue;
                }
                else
                {
                    TagStartPositions.Add(originalNode.OuterStartIndex);
                    TagStartLengths.Add(originalNode.InnerStartIndex - originalNode.OuterStartIndex);
                    currentId = TagStartPositions.Count - 1;
                    toEditNode.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, currentId.ToString());

                    TagClosingPositions.Add(originalNode.InnerStartIndex + originalNode.InnerLength);
                    TagClosingLengths.Add((originalNode.OuterStartIndex + originalNode.OuterLength) - (originalNode.InnerStartIndex + originalNode.InnerLength));
                }
            }
        }

        private void FillNodeDict(IEnumerable<HtmlNode> descendantsOriginal, IEnumerable<HtmlNode> descendantsToEdit)
        {
            var zipped = descendantsOriginal.Zip(descendantsToEdit, (orig, toEdit) => new
            {
                Original = orig,
                ToEdit = toEdit
            });
            foreach (var node in zipped)
            {
                var originalNode = node.Original;
                var toEditNode = node.ToEdit;
                NodeDict.Add(originalNode, toEditNode);
            }
        }


        /*
         *      Full HTML Preprocessing -------------------------------------------------------------------------------
         */

        /*
         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
         */

        private string PartialPreprocessHTMLAddTag(string htmlToEdit, string htmlOriginal, AnnotationTag tagToAdd, List<AnnotationTag> tags)
        {
            var docOriginal = new HtmlDocument();
            docOriginal.LoadHtml(htmlOriginal);
            var docToEdit = new HtmlDocument();
            docToEdit.LoadHtml(htmlToEdit);

            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();

            int i = 0;
            List<HtmlNode> addedForSelection = new();

            int descendantsCount = descendantsToEdit.Count();
            while (i < descendantsCount)
            {
                for (; i < descendantsCount; i++)
                {
                    var node = descendantsToEdit.ElementAt(i);
                    if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
                        node.ParentNode.Name == "style" || node.ParentNode.Name.StartsWith("#"))
                    {
                        continue;
                    }

                    int nodeId = node.ParentNode.GetAttributeValue(TAG_ID_ATTRIBUTE_NAME, -1);

                    var start = TagStartPositions[nodeId] + TagStartLengths[nodeId];
                    var end = TagClosingPositions[nodeId];

                    int selectionStart = tagToAdd.Position;
                    int selectionEnd = tagToAdd.Position + tagToAdd.Length;

                    if (selectionStart < end && selectionEnd > start)
                    {
                        if (selectionStart <= start && selectionEnd >= end)
                        {
                            addedForSelection.Add(SolveFullFill(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        else if (selectionStart <= start)
                        {
                            addedForSelection.AddRange(SolveRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        else if (selectionEnd >= end)
                        {
                            addedForSelection.AddRange(SolveLeftGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        else
                        {
                            addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
                        descendantsCount = descendantsToEdit.Count();
                        break;
                    }
                }
            }

            GenerateCSS(tags);
            return docToEdit.DocumentNode.OuterHtml;
        }


        private string PartialPreprocessHTMLRemoveTag(string htmlToEdit, string htmlOriginal, AnnotationTag tagToRemove, List<AnnotationTag> tags)
        {
            var docOriginal = new HtmlDocument();
            docOriginal.LoadHtml(htmlOriginal);
            var docToEdit = new HtmlDocument();
            docToEdit.LoadHtml(htmlToEdit);

            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();

            int i = 0;
            int descendantsCount = descendantsToEdit.Count();
            while (i < descendantsCount)
            {
                for (; i < descendantsCount; i++)
                {
                    var node = descendantsToEdit.ElementAt(i);
                    if (!node.Attributes.Contains(TAG_EF_ID_ATTRIBUTE_NAME))
                    {
                        continue;
                    }
                    else
                    {
                        if (node.Attributes[TAG_EF_ID_ATTRIBUTE_NAME].Value != tagToRemove.Id.ToString())
                        {
                            continue;
                        }

                        node.Attributes.Remove(TAG_EF_ID_ATTRIBUTE_NAME);
                        node.Attributes.Remove(TAG_INSTANCE_ATTRIBUTE_NAME);
                        node.Attributes.Remove("class");
                        node.Attributes.Remove("start");
                        node.Attributes.Remove("end");

                        /*var parent = node.ParentNode;
                        var contents = node.ChildNodes;
                        int index = parent.ChildNodes.IndexOf(node);
                        parent.ChildNodes.RemoveAt(index);

                        List<HtmlNode> newChildren = new();
                        for (int j = 0; j < index; j++)
                        {
                            newChildren.Add(parent.ChildNodes[j]);
                        }
                        for (int j = 0; j < contents.Count; j++)
                        {
                            newChildren.Add(contents[j]);
                        }
                        for (int j = index; j < parent.ChildNodes.Count; j++)
                        {
                            newChildren.Add(parent.ChildNodes[j]);
                        }

                        parent.ChildNodes.Clear();
                        foreach (var child in newChildren) { parent.ChildNodes.Add(child); }*/

                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
                        descendantsCount = descendantsToEdit.Count();
                        break;
                    }
                }
            }

            GenerateCSS(tags);
            return docToEdit.DocumentNode.OuterHtml;
        }

        /*
         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
         */


        // TODO temporary
        private bool IsHtml(string text)
        {
            return text.Contains("<html>");
        }

        public void AddAnnotationInstance(Guid annotationId, Guid userId, ERole userRole, AnnotationInstanceAddRequest request)
        {
            var annotation = context.Annotations
               .Where(a => a.Id == annotationId)
               .Include(a => a.User)
               .Include(a => a.Document).ThenInclude(d => d.Content)
               .First();

            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != userId)
                {
                    throw new UnauthorizedAccessException($"User {userId} does not have assigned annotation {annotationId}");
                }
            }

            if (annotation.State == EState.NEW)
            {
                annotation.State = EState.IN_PROGRESS;
            }

            AnnotationTag annotationTag = new()
            {
                Id = Guid.NewGuid(),
                Annotation = annotation,
                Instance = request.InstanceId == null ? Guid.NewGuid() : request.InstanceId.Value,
                Length = request.Length,
                Position = request.Position,
                Note = ""
            };

            if (request.Type == ETagType.TAG)
            {
                annotationTag.Tag = context.Tags.Where(t => t.Id == request.Id).Single();
                annotationTag.SubTag = null;

                if (annotationTag.Tag.SentimentEnabled)
                {
                    annotationTag.Sentiment = ETagSentiment.NEUTRAL;
                }

                // If for the same annotation exists a tag with same position and length and of the same type, ignore
                if (context.AnnotationTags.Any(at =>
                at.Position == annotationTag.Position &&
                at.Length == annotationTag.Length &&
                at.Annotation == annotation &&
                at.Tag == annotationTag.Tag))
                {
                    throw new InvalidOperationException("Duplicate tag");
                }

            }
            else if (request.Type == ETagType.SUBTAG)
            {
                var subTag = context.SubTags.Where(st => st.Id == request.Id).Include(st => st.Tag).Single();
                annotationTag.SubTag = subTag;
                annotationTag.Tag = subTag.Tag;

                if (annotationTag.SubTag.SentimentEnabled)
                {
                    annotationTag.Sentiment = ETagSentiment.NEUTRAL;
                }

                if (context.AnnotationTags.Any(at =>
                at.Position == annotationTag.Position &&
                at.Length == annotationTag.Length &&
                at.Annotation == annotation &&
                at.Tag == annotationTag.Tag &&
                at.SubTag == annotationTag.SubTag))
                {
                    throw new InvalidOperationException("Duplicate tag");
                }
            }
            else
            {
                throw new ArgumentException($"Unknown tag type {request.Type}");
            }

            annotation.LastModifiedTagId = annotationTag.Id;
            annotation.ModifiedType = EModified.ADDED;

            context.AnnotationTags.Add(annotationTag);
            context.SaveChanges();
        }

        public void DeleteAnnotationInstance(Guid annotationId, Guid tagInstanceId, Guid loggedUserId, ERole userRole)
        {
            Annotation annotation = null;
            try
            {
                annotation = context.Annotations
                   .Where(a => a.Id == annotationId)
                   .Include(a => a.User)
                   .Include(a => a.Document).ThenInclude(d => d.Content)
                   .First();

            }
            catch (Exception ex)
            {
                throw new InvalidOperationException("Could not find annotation");
            }


            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != loggedUserId)
                {
                    throw new UnauthorizedAccessException($"User {loggedUserId} does not have assigned annotation {annotationId}");
                }
            }

            if (!context.AnnotationTags.Any(at => at.Id == tagInstanceId))
            {
                throw new InvalidOperationException("Could not find tag instance");
            }


            var annotationTag = context.AnnotationTags.First(at => at.Id == tagInstanceId);
            annotation.LastModifiedTagId = annotationTag.Id;
            annotation.ModifiedType = EModified.REMOVED;

            context.AnnotationTags.Remove(annotationTag);

            context.SaveChanges();
        }

        public void SetTagInstanceSentiment(Guid annotationId, Guid instanceId, Guid userId, ERole userRole, ETagSentiment sentiment)
        {
            Annotation annotation = null;
            try
            {
                annotation = context.Annotations
                   .Where(a => a.Id == annotationId)
                   .Include(a => a.User)
                   .Include(a => a.Document).ThenInclude(d => d.Content)
                   .First();

            }
            catch (Exception ex)
            {
                throw new InvalidOperationException("Could not find annotation");
            }


            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != userId)
                {
                    throw new UnauthorizedAccessException($"User {userId} does not have assigned annotation {annotationId}");
                }
            }


            var tagInstances = context.AnnotationTags.Where(at => at.Instance == instanceId).ToList();
            if (tagInstances.Count() == 0)
            {
                throw new InvalidOperationException("No such instance found");
            }

            foreach (var tagInstance in tagInstances)
            {
                tagInstance.Sentiment = sentiment;
            }

            context.SaveChanges();
        }

        public void MarkAnnotationAsDone(Guid annotationId, Guid userId, ERole userRole, bool done)
        {
            Annotation annotation = null;
            try
            {
                annotation = context.Annotations
                   .Where(a => a.Id == annotationId)
                   .Include(a => a.User)
                   .Include(a => a.Document).ThenInclude(d => d.Content)
                   .First();

            }
            catch (Exception ex)
            {
                throw new InvalidOperationException("Could not find annotation");
            }


            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != userId)
                {
                    throw new UnauthorizedAccessException($"User {userId} does not have assigned annotation {annotationId}");
                }
            }

            annotation.State = done ? EState.DONE : EState.IN_PROGRESS;
            context.SaveChanges();
        }
    }
}