aswi20220x00-gitlab/Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs @ c9762683

using Core.Contexts;
using Core.Entities;
using Models.Annotations;
using Models.Enums;
using Serilog;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;
using Microsoft.EntityFrameworkCore;
using AutoMapper;
using Models.Tags;
using Ganss.XSS;
using HtmlAgilityPack;
using System.Text.RegularExpressions;

namespace Core.Services.AnnotationService
{
    public class AnnotationServiceEF : IAnnotationService
    {
        private readonly DatabaseContext context;
        private readonly ILogger logger;
        private readonly IMapper mapper;

        private const string TAG_ID_ATTRIBUTE_NAME = "aswi-tag-id";
        private const string TAG_INSTANCE_ATTRIBUTE_NAME = "aswi-tag-instance";

        public AnnotationServiceEF(DatabaseContext context, ILogger logger, IMapper mapper)
        {
            this.context = context;
            this.logger = logger;
            this.mapper = mapper;
        }

        public void CreateDocumentAnnotations(AnnotationsAddRequest request, Guid clientUserId)
        {
            User addingUser = context.Users.Single(u => u.Id == clientUserId);

            // Check the documents exist
            var documents = context.Documents.Where(d => request.DocumentIdList.Contains(d.Id)).ToList();
            if (documents.Count() != request.DocumentIdList.Count)
            {
                logger.Information($"Received a non-existent Document ID when assigning documents to users");
                throw new InvalidOperationException($"{request.DocumentIdList.Count - documents.Count()} of the received documents do not exist");
            }

            var users = context.Users.Where(u => request.UserIdList.Contains(u.Id)).ToList();
            foreach (var user in users)
            {
                var userAnnotatedDocuments = context.Annotations.Where(a => a.User == user).Select(a => a.Document).ToList();
                foreach (var doc in documents)
                {
                    if (userAnnotatedDocuments.Contains(doc))
                    {
                        logger.Information($"User {user.Username} has already been assigned the document {doc.Id}, ignoring");
                        continue;
                    }

                    context.Annotations.Add(new Annotation()
                    {
                        User = user,
                        UserAssigned = addingUser,
                        DateAssigned = DateTime.Now,
                        DateLastChanged = DateTime.Now,
                        Document = doc,
                        State = EState.NEW,
                        Note = ""
                    });
                }
            }

            context.SaveChanges();
        }

        public AnnotationListResponse GetUserAnnotations(Guid userId)
        {
            var annotations = context.Annotations.Where(a => a.User.Id == userId).Include(a => a.Document).ToList();
            var documentIds = annotations.Select(a => a.Document.Id).ToList();
            var documents = context.Documents.Where(d => documentIds.Contains(d.Id));
            var infos = new List<AnnotationListInfo>();

            var annotationsDocuments = annotations.Zip(documents, (a, d) => new { Annotation = a, Document = d });
            foreach (var ad in annotationsDocuments)
            {
                infos.Add(new AnnotationListInfo()
                {
                    AnnotationId = ad.Annotation.Id,
                    DocumentName = ad.Document.Name,
                    State = ad.Annotation.State
                });
            }

            return new AnnotationListResponse()
            {
                Annotations = infos
            };
        }

        public AnnotationInfo GetAnnotation(Guid annotationId, Guid userId, ERole userRole)
        {
            var annotation = context.Annotations
                .Where(a => a.Id == annotationId)
                .Include(a => a.User)
                .Include(a => a.Document).ThenInclude(d => d.Content)
                .First();

            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != userId)
                {
                    throw new UnauthorizedAccessException($"User {userId} does not have assigned annotation {annotationId}");
                }
            }

            var documentContent = context.Documents.Where(d => d.Id == annotation.Document.Id).Select(d => d.Content).First();

            var tags = context.AnnotationTags.Where(at => at.Annotation.Id == annotationId)
                .Include(at => at.Tag).ThenInclude(t => t.Category)
                .Include(at => at.SubTag)
                .ToList();

            List<TagInstanceInfo> tagInstanceInfos = new();
            foreach (var tag in tags)
            {
                var tagInstance = mapper.Map<TagInstanceInfo>(tag);
                tagInstanceInfos.Add(tagInstance);
            }

            var docToRender = PreprocessHTML(documentContent.Content, tags);

            // We probably cannot use AutoMapper since we are dealing with too many different entities
            AnnotationInfo annotationInfo = new()
            {
                SourceDocumentContent = documentContent.Content,
                DocumentToRender = docToRender,
                TagStartPositions = TagStartPositions.ToArray(),
                TagLengths = TagStartLengths.ToArray(),
                Note = annotation.Note,
                State = annotation.State,
                Type = IsHtml(documentContent.Content) ? EDocumentType.HTML : EDocumentType.TEXT,
                TagInstances = tagInstanceInfos
            };

            return annotationInfo;
        }

        private List<int> TagStartPositions = new();
        private List<int> TagStartLengths = new();
        private List<int> TagClosingPositions = new();
        private List<int> TagClosingLengths = new();
        private Dictionary<HtmlNode, HtmlNode> NodeDict = new();

        private string PreprocessHTML(string htmlSource, List<AnnotationTag> tags)
        {
            var docOriginal = new HtmlDocument();
            docOriginal.LoadHtml(htmlSource);
            var docToEdit = new HtmlDocument();
            docToEdit.LoadHtml(htmlSource);

            var descendantsOriginal = docOriginal.DocumentNode.DescendantsAndSelf();
            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf();

            int currentId = 0;

            FillNodeDict(descendantsOriginal, descendantsToEdit);
            AssignIdsToOriginalDocument(descendantsOriginal, ref currentId);

            WrapTextInSpan(descendantsOriginal, docToEdit);

            foreach (var tag in tags)
            {
                int i = 0;
                List<HtmlNode> addedForSelection = new();
                while (i < descendantsToEdit.Count())
                {
                    for (; i < descendantsToEdit.Count(); i++)
                    {
                        var node = descendantsToEdit.ElementAt(i);
                        if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
                            node.ParentNode.Name == "style")
                        {
                            continue;
                        }

                        int nodeId = node.ParentNode.GetAttributeValue(TAG_ID_ATTRIBUTE_NAME, -1);

                        var start = TagStartPositions[nodeId] + TagStartLengths[nodeId];
                        var end = TagClosingPositions[nodeId];

                        int selectionStart = tag.Position;
                        int selectionEnd = tag.Position + tag.Length;

                        if (selectionStart < end && selectionEnd > start)
                        {
                            if (selectionStart <= start && selectionEnd >= end)
                            {
                                addedForSelection.Add(SolveFullFill(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else if (selectionStart <= start)
                            {
                                addedForSelection.AddRange(SolveRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else if (selectionEnd >= end)
                            {
                                addedForSelection.AddRange(SolveLeftGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else
                            {
                                addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            break;
                        }
                    }
                }

            }

            string docToRender = docToEdit.DocumentNode.OuterHtml;
            HtmlSanitizer sanitizer = new HtmlSanitizer();
            sanitizer.AllowedAttributes.Clear();
            sanitizer.AllowedAttributes.Add(TAG_ID_ATTRIBUTE_NAME);
            sanitizer.AllowedAttributes.Add(TAG_INSTANCE_ATTRIBUTE_NAME);
            sanitizer.AllowedAttributes.Add("end");
            sanitizer.AllowedAttributes.Add("start");
            sanitizer.AllowedAttributes.Add("class");
            if (sanitizer.AllowedTags.Contains("script"))
            {
                sanitizer.AllowedTags.Remove("script");
            }   
            if (!sanitizer.AllowedTags.Contains("style"))
            {
                sanitizer.AllowedTags.Add("style");
            }
            docToRender = sanitizer.Sanitize(docToRender);

            HtmlDocument doc = new HtmlDocument();
            doc.LoadHtml(docToRender);
            var cssNode = GenerateCSS(doc, tags, new());
            doc.DocumentNode.ChildNodes.Insert(0, cssNode);

            return doc.DocumentNode.OuterHtml;
        }

        private HtmlNode SolveFullFill(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit, AnnotationTag tag)
        {
            // full fill
            string textSelected = node.InnerText;

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            EPosition markerPosition = EPosition.MARK_NONE;
            if (selectionEnd == end && selectionStart == start)
            {
                markerPosition = EPosition.MARK_LEFT_RIGHT;
            }

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, start, markerPosition);
            parentNode.ChildNodes.Insert(nodeIndex, spanSelected);

            return spanSelected;
        }

        private List<HtmlNode> SolveRightGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                             AnnotationTag tag)
        {
            // partial fill, end gap
            string text = node.InnerText;
            string textAfter = text.Substring(Math.Min(selectionStart - start + tag.Length, text.Length));
            string textSelected = text.Substring(0, selectionEnd - start);

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanSelectedStart = start;
            int spanAfterStart = start + textSelected.Length;

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_RIGHT);
            parentNode.ChildNodes.Insert(nodeIndex, spanSelected);

            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanAfterStart);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanAfter);

            return new() { spanSelected, spanAfter };
        }

        private List<HtmlNode> SolveLeftGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                             AnnotationTag tag)
        {
            // partial fill, start gap
            string text = node.InnerText;
            string textBefore = text.Substring(0, selectionStart - start);
            string textSelected = text.Substring(selectionStart - start, Math.Min(tag.Length, text.Length - textBefore.Length));

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanBeforeStart = start;
            int spanSelectedStart = start + textBefore.Length;

            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanBeforeStart);
            parentNode.ChildNodes.Insert(nodeIndex, spanBefore);

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_LEFT);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);

            return new() { spanSelected, spanBefore };
        }

        private List<HtmlNode> SolveLeftRightGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                                 AnnotationTag tag)
        {
            // partial fill, start gap end gap
            string text = node.InnerText;
            string textBefore = text.Substring(0, selectionStart - start);
            string textAfter = text.Substring(selectionStart - start + tag.Length);
            string textSelected = text.Substring(selectionStart - start, tag.Length);

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanBeforeStart = start;
            int spanSelectedStart = start + textBefore.Length;
            int spanAfterStart = start + textBefore.Length + textSelected.Length;

            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanBeforeStart);
            parentNode.ChildNodes.Insert(nodeIndex, spanBefore);

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, tag.Instance, spanSelectedStart, EPosition.MARK_LEFT_RIGHT);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);

            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TAG_ID_ATTRIBUTE_NAME, TagStartPositions.Count,
                TAG_INSTANCE_ATTRIBUTE_NAME, null, spanAfterStart);
            parentNode.ChildNodes.Insert(nodeIndex + 2, spanAfter);

            return new() { spanSelected, spanBefore, spanAfter };
        }

        private HtmlNode CreateSpan(HtmlDocument doc, string text, string tagIdAttributeName, int tagId, string tagInstanceAttributeName,
                                            Guid? instanceId, int startPosition, EPosition position = EPosition.MARK_NONE)
        {
            HtmlNode span = doc.CreateElement("span");
            span.InnerHtml = text;
            TagStartPositions.Add(startPosition);
            TagStartLengths.Add(0);
            TagClosingPositions.Add(startPosition + text.Length);
            TagClosingLengths.Add(0);
            span.Attributes.Add(tagIdAttributeName, tagId.ToString());

            if (instanceId != null)
            {
                span.AddClass("annotation");
                span.Attributes.Add(tagInstanceAttributeName, instanceId.Value.ToString());

                if (position == EPosition.MARK_LEFT || position == EPosition.MARK_LEFT_RIGHT)
                {
                    span.Attributes.Add("start", "1");
                }
                if (position == EPosition.MARK_RIGHT || position == EPosition.MARK_LEFT_RIGHT)
                {
                    span.Attributes.Add("end", "1");
                }
            }

            return span;
        }

        private enum EPosition
        {
            MARK_LEFT = 5,
            MARK_RIGHT = 3,
            MARK_LEFT_RIGHT = 2,
            MARK_NONE = 0
        }

        private void WrapTextInSpan(IEnumerable<HtmlNode> descendantsOriginal, HtmlDocument docToEdit)
        {
            foreach (var node in descendantsOriginal)
            {
                var originalNode = node;
                var toEditNode = NodeDict[node];

                if (originalNode.Name.Contains("#"))
                {
                    continue;
                }
                else
                {
                    bool onlyText = true;
                    bool onlySubtags = true;

                    foreach (var child in node.ChildNodes)
                    {
                        if (child.Name.Contains("#"))
                        {
                            onlySubtags = false;
                        }
                        else
                        {
                            onlyText = false;
                        }
                    }

                    if (onlyText || onlySubtags)
                    {
                        continue;
                    }
                    else
                    {

                        foreach (var child in node.ChildNodes)
                        {
                            if (child.Name.Contains("#text"))
                            {
                                HtmlNode coveringSpan = docToEdit.CreateElement("span");
                                coveringSpan.InnerHtml = child.InnerHtml;
                                TagStartPositions.Add(child.InnerStartIndex);
                                TagStartLengths.Add(0);
                                TagClosingPositions.Add(child.InnerStartIndex + child.InnerLength);
                                TagClosingLengths.Add(0);
                                coveringSpan.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, (TagStartPositions.Count - 1).ToString());

                                var parent = NodeDict[node];
                                var index = parent.ChildNodes.IndexOf(NodeDict[child]);

                                parent.ChildNodes.RemoveAt(index);
                                parent.ChildNodes.Insert(index, coveringSpan);
                            }
                        }
                    }
                }
            }
        }

        private HtmlNode GenerateCSS(HtmlDocument docToEdit, List<AnnotationTag> tags, List<int> paddings)
        {
            HtmlNode style = docToEdit.CreateElement("style");

            string inner = "span.annotation {border-bottom: 2px solid;}";
            inner += "span {line-height: 30px}\n";

            // TODO temporary
            int lastPadding = 0;
            foreach (var tag in tags)
            {
                inner += $"span[{TAG_INSTANCE_ATTRIBUTE_NAME}=\"{tag.Instance}\"] {{ border-color:{tag.Tag.Color}; padding-bottom: {lastPadding % 5}px }}";
                lastPadding += 2;
            }

            inner += "span[end=\"1\"] {border-end-end-radius: 0px; border-right: 1px solid darkgray}\n";
            inner += "span[start=\"1\"] {border-end-start-radius: 0px; border-left: 1px solid darkgray}\n";

            style.InnerHtml = inner;
            return style;
        }

        private void AssignIdsToOriginalDocument(IEnumerable<HtmlNode> descendantsOriginal, ref int currentId)
        {
            foreach (var node in descendantsOriginal)
            {
                var originalNode = node;
                var toEditNode = NodeDict[node];

                if (originalNode.Name.Contains("#"))
                {
                    continue;
                }
                else
                {
                    TagStartPositions.Add(originalNode.OuterStartIndex);
                    TagStartLengths.Add(originalNode.InnerStartIndex - originalNode.OuterStartIndex);
                    currentId = TagStartPositions.Count - 1;
                    toEditNode.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, currentId.ToString());

                    TagClosingPositions.Add(originalNode.InnerStartIndex + originalNode.InnerLength);
                    TagClosingLengths.Add((originalNode.OuterStartIndex + originalNode.OuterLength) - (originalNode.InnerStartIndex + originalNode.InnerLength));
                }
            }
        }

        private void FillNodeDict(IEnumerable<HtmlNode> descendantsOriginal, IEnumerable<HtmlNode> descendantsToEdit)
        {
            var zipped = descendantsOriginal.Zip(descendantsToEdit, (orig, toEdit) => new
            {
                Original = orig,
                ToEdit = toEdit
            });
            foreach (var node in zipped)
            {
                var originalNode = node.Original;
                var toEditNode = node.ToEdit;
                NodeDict.Add(originalNode, toEditNode);
            }
        }

        // TODO temporary
        private bool IsHtml(string text)
        {
            return text.Contains("<html>");
        }

        public void AddAnnotationInstance(Guid annotationId, Guid userId, ERole userRole, AnnotationInstanceAddRequest request)
        {
            var annotation = context.Annotations
               .Where(a => a.Id == annotationId)
               .Include(a => a.User)
               .Include(a => a.Document).ThenInclude(d => d.Content)
               .First();

            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != userId)
                {
                    throw new UnauthorizedAccessException($"User {userId} does not have assigned annotation {annotationId}");
                }
            }

            AnnotationTag annotationTag = new()
            {
                Annotation = annotation,
                Instance = request.InstanceId == null ? Guid.NewGuid() : request.InstanceId.Value,
                Length = request.Length,
                Position = request.Position,
                Note = ""
            };

            if (request.Type == ETagType.TAG)
            {
                annotationTag.Tag = context.Tags.Where(t => t.Id == request.Id).Single();
                annotationTag.SubTag = null;
            }
            else if (request.Type == ETagType.SUBTAG)
            {
                var subTag = context.SubTags.Where(st => st.Id == request.Id).Include(st => st.Tag).Single();
                annotationTag.SubTag = subTag;
                annotationTag.Tag = subTag.Tag;
            }
            else
            {
                throw new ArgumentException($"Unknown tag type {request.Type}");
            }

            context.AnnotationTags.Add(annotationTag);
            context.SaveChanges();
        }

        public void DeleteAnnotationInstance(Guid annotationId, Guid tagInstanceId, Guid loggedUserId, ERole userRole)
        {
            Annotation annotation = null;
            try
            {
                annotation = context.Annotations
                   .Where(a => a.Id == annotationId)
                   .Include(a => a.User)
                   .Include(a => a.Document).ThenInclude(d => d.Content)
                   .First();

            }
            catch (Exception ex)
            {
                throw new InvalidOperationException("Could not find annotation");
            }


            if (userRole < ERole.ADMINISTRATOR)
            {
                if (annotation.User.Id != loggedUserId)
                {
                    throw new UnauthorizedAccessException($"User {loggedUserId} does not have assigned annotation {annotationId}");
                }
            }

            if (!context.AnnotationTags.Any(at => at.Id == tagInstanceId))
            {
                throw new InvalidOperationException("Could not find tag instance");
            }

            context.AnnotationTags
                .Where(at => at.Id == tagInstanceId).ToList()
                .ForEach(a => context.AnnotationTags.Remove(a));

            context.SaveChanges();
        }
    }
}