aswi20220x00-gitlab/Backend/Core/Services/HTMLService/HTMLService.cs @ 97442c93

using Core.Entities;
using Core.GraphUtils;
using Ganss.XSS;
using HtmlAgilityPack;
using Models.Tags;
using System;
using System.Collections.Generic;
using System.Linq;
using System.Text;
using System.Threading.Tasks;

namespace Core.Services
{
    public class HTMLService : IHTMLService
    {

        private const string TAG_ID_ATTRIBUTE_NAME = "aswi-tag-id";
        private const string TAG_INSTANCE_ATTRIBUTE_NAME = "aswi-tag-instance";
        private const string TAG_EF_ID_ATTRIBUTE_NAME = "aswi-tag-ef-id";

        public class CachedInfo
        {
            public List<int> TagStartPositions = new();
            public List<int> TagStartLengths = new();
            public List<int> TagClosingPositions = new();
            public List<int> TagClosingLengths = new();
            public Dictionary<HtmlNode, HtmlNode> NodeDict = new();
            public List<TagInstanceCSSInfo> TagInstanceCSS = new();
        }

        private List<int> TagStartPositions = new();
        private List<int> TagStartLengths = new();
        private List<int> TagClosingPositions = new();
        private List<int> TagClosingLengths = new();
        private Dictionary<HtmlNode, HtmlNode> NodeDict = new();
        private List<TagInstanceCSSInfo> TagInstanceCSS = new();

        private void UnpackCachedInfo(CachedInfo cachedInfo)
        {
            TagStartPositions = cachedInfo.TagStartPositions;
            TagStartLengths = cachedInfo.TagStartLengths;
            TagClosingPositions = cachedInfo.TagClosingPositions;
            TagClosingLengths = cachedInfo.TagClosingLengths;
            NodeDict = cachedInfo.NodeDict;
            TagInstanceCSS = cachedInfo.TagInstanceCSS;
        }


        /*
         *      Full HTML Preprocessing -------------------------------------------------------------------------------
         */

        public (string, CachedInfo) FullPreprocessHTML(string htmlSource, List<AnnotationTagGeneric> tags)
        {
            var docOriginal = new HtmlDocument();
            docOriginal.LoadHtml(htmlSource);
            var docToEdit = new HtmlDocument();
            docToEdit.LoadHtml(htmlSource);

            var descendantsOriginal = docOriginal.DocumentNode.DescendantsAndSelf();
            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();

            int currentId = 0;

            FillNodeDict(descendantsOriginal, descendantsToEdit);
            AssignIdsToOriginalDocument(descendantsOriginal, ref currentId);

            WrapTextInSpan(descendantsOriginal, docToEdit);

            descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
            int descCount = descendantsToEdit.Count;

            foreach (var tag in tags)
            {
                int i = 0;
                List<HtmlNode> addedForSelection = new();
                while (i < descCount)
                {
                    for (; i < descCount; i++)
                    {
                        var node = descendantsToEdit.ElementAt(i);
                        if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) || node.ParentNode.Name == "style")
                        {
                            continue;
                        }

                        int nodeId = node.ParentNode.GetAttributeValue(TAG_ID_ATTRIBUTE_NAME, -1);

                        var start = TagStartPositions[nodeId] + TagStartLengths[nodeId];
                        var end = TagClosingPositions[nodeId];

                        int selectionStart = tag.Position;
                        int selectionEnd = tag.Position + tag.Length;

                        if (selectionStart < end && selectionEnd > start)
                        {
                            if (selectionStart <= start && selectionEnd >= end)
                            {
                                addedForSelection.Add(SolveFullFill(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else if (selectionStart <= start)
                            {
                                addedForSelection.AddRange(SolveRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else if (selectionEnd >= end)
                            {
                                addedForSelection.AddRange(SolveLeftGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            else
                            {
                                addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tag));
                            }
                            descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
                            descCount = descendantsToEdit.Count;
                            break;
                        }
                    }
                }

            }

            ModifyLinks(descendantsToEdit);
            string docToRender = docToEdit.DocumentNode.OuterHtml;
            HtmlSanitizer sanitizer = new HtmlSanitizer();
            sanitizer.AllowedAttributes.Clear();
            sanitizer.AllowedAttributes.Add(TAG_ID_ATTRIBUTE_NAME);
            sanitizer.AllowedAttributes.Add(TAG_INSTANCE_ATTRIBUTE_NAME);
            sanitizer.AllowedAttributes.Add(TAG_EF_ID_ATTRIBUTE_NAME);
            sanitizer.AllowedAttributes.Add("href");
            sanitizer.AllowedAttributes.Add("end");
            sanitizer.AllowedAttributes.Add("start");
            sanitizer.AllowedAttributes.Add("class");
            sanitizer.AllowedAttributes.Add("target");
            sanitizer.AllowedAttributes.Add("id");
            if (sanitizer.AllowedTags.Contains("script"))
            {
                sanitizer.AllowedTags.Remove("script");
            }
            if (!sanitizer.AllowedTags.Contains("style"))
            {
                sanitizer.AllowedTags.Add("style");
            }
            sanitizer.AllowedTags.Add("a");
            docToRender = sanitizer.Sanitize(docToRender);
            GenerateCSS(tags);

            return (docToRender, new CachedInfo()
            {
                TagStartPositions = TagStartPositions,
                TagClosingPositions = TagClosingPositions,
                TagStartLengths = TagStartLengths,
                TagClosingLengths = TagClosingLengths,
                TagInstanceCSS = TagInstanceCSS,
                NodeDict = NodeDict
            });
        }

        private HtmlNode SolveFullFill(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit, AnnotationTagGeneric tag)
        {
            // full fill
            string textSelected = node.InnerText;

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            EPosition markerPosition = EPosition.MARK_NONE;
            if (selectionEnd == end && selectionStart == start)
            {
                markerPosition = EPosition.MARK_LEFT_RIGHT;
            }

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, start, markerPosition);
            parentNode.ChildNodes.Insert(nodeIndex, spanSelected);

            return spanSelected;
        }

        private List<HtmlNode> SolveRightGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                             AnnotationTagGeneric tag)
        {
            // partial fill, end gap
            string text = node.InnerText;
            string textAfter = text.Substring(Math.Min(selectionStart - start + tag.Length, text.Length));
            string textSelected = text.Substring(0, selectionEnd - start);

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanSelectedStart = start;
            int spanAfterStart = start + textSelected.Length;

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_RIGHT);
            parentNode.ChildNodes.Insert(nodeIndex, spanSelected);

            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanAfter);

            return new() { spanSelected, spanAfter };
        }

        private List<HtmlNode> SolveLeftGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                             AnnotationTagGeneric tag)
        {
            // partial fill, start gap
            string text = node.InnerText;
            string textBefore = text.Substring(0, selectionStart - start);
            string textSelected = text.Substring(selectionStart - start, Math.Min(tag.Length, text.Length - textBefore.Length));

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanBeforeStart = start;
            int spanSelectedStart = start + textBefore.Length;

            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
            parentNode.ChildNodes.Insert(nodeIndex, spanBefore);

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);

            return new() { spanSelected, spanBefore };
        }

        private List<HtmlNode> SolveLeftRightGap(HtmlNode node, int selectionStart, int selectionEnd, int start, int end, HtmlDocument docToEdit,
                                                 AnnotationTagGeneric tag)
        {
            // partial fill, start gap end gap
            string text = node.InnerText;
            string textBefore = text.Substring(0, selectionStart - start);
            string textAfter = text.Substring(selectionStart - start + tag.Length);
            string textSelected = text.Substring(selectionStart - start, tag.Length);

            var parentNode = node.ParentNode;
            int nodeIndex = parentNode.ChildNodes.IndexOf(node);
            parentNode.ChildNodes.RemoveAt(nodeIndex);

            int spanBeforeStart = start;
            int spanSelectedStart = start + textBefore.Length;
            int spanAfterStart = start + textBefore.Length + textSelected.Length;

            HtmlNode spanBefore = CreateSpan(docToEdit, textBefore, TagStartPositions.Count, null, null, spanBeforeStart);
            parentNode.ChildNodes.Insert(nodeIndex, spanBefore);

            HtmlNode spanSelected = CreateSpan(docToEdit, textSelected, TagStartPositions.Count, tag.Instance, tag.Id, spanSelectedStart, EPosition.MARK_LEFT_RIGHT);
            parentNode.ChildNodes.Insert(nodeIndex + 1, spanSelected);

            HtmlNode spanAfter = CreateSpan(docToEdit, textAfter, TagStartPositions.Count, null, null, spanAfterStart);
            parentNode.ChildNodes.Insert(nodeIndex + 2, spanAfter);

            return new() { spanSelected, spanBefore, spanAfter };
        }

        private HtmlNode CreateSpan(HtmlDocument doc, string text, int tagId, Guid? instanceId, Guid? entityId, int startPosition, EPosition position = EPosition.MARK_NONE)
        {
            HtmlNode span = doc.CreateElement("span");
            span.InnerHtml = text;
            TagStartPositions.Add(startPosition);
            TagStartLengths.Add(0);
            TagClosingPositions.Add(startPosition + text.Length);
            TagClosingLengths.Add(0);
            span.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, tagId.ToString());

            if (instanceId != null)
            {
                span.AddClass("annotation");
                span.Attributes.Add(TAG_INSTANCE_ATTRIBUTE_NAME, instanceId.Value.ToString());
                span.Attributes.Add(TAG_EF_ID_ATTRIBUTE_NAME, entityId.ToString());
                span.Attributes.Add("id", entityId.ToString());

                if (position == EPosition.MARK_LEFT || position == EPosition.MARK_LEFT_RIGHT)
                {
                    span.Attributes.Add("start", "1");
                }
                if (position == EPosition.MARK_RIGHT || position == EPosition.MARK_LEFT_RIGHT)
                {
                    span.Attributes.Add("end", "1");
                }
            }

            return span;
        }

        private enum EPosition
        {
            MARK_LEFT = 5,
            MARK_RIGHT = 3,
            MARK_LEFT_RIGHT = 2,
            MARK_NONE = 0
        }

        private void ModifyLinks(IEnumerable<HtmlNode> descendantsOriginal)
        {
            foreach (var descendant in descendantsOriginal)
            {
                if (descendant.Name == "a")
                {
                    if (descendant.Attributes.Contains("href"))
                    {
                        descendant.SetAttributeValue("href", "/link?url=" + descendant.Attributes["href"].Value);
                        descendant.SetAttributeValue("target", "_blank");
                    }
                }
            }
        }

        private void WrapTextInSpan(IEnumerable<HtmlNode> descendantsOriginal, HtmlDocument docToEdit)
        {
            // Special case for non-html documents
            if (descendantsOriginal.Count() == 2)
            {
                var documentNode = descendantsOriginal.ElementAt(0);
                var childNode = descendantsOriginal.ElementAt(1);
                if (documentNode.Name == "#document" && childNode.Name == "#text")
                {
                    HtmlNode coveringSpan = docToEdit.CreateElement("span");
                    coveringSpan.InnerHtml = childNode.InnerHtml;
                    TagStartPositions.Add(childNode.InnerStartIndex);
                    TagStartLengths.Add(0);
                    TagClosingPositions.Add(childNode.InnerStartIndex + childNode.InnerLength);
                    TagClosingLengths.Add(0);
                    coveringSpan.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, (TagStartPositions.Count - 1).ToString());

                    var parent = NodeDict[documentNode];

                    parent.ChildNodes.RemoveAt(0);
                    parent.ChildNodes.Add(coveringSpan);

                    return;
                }
            }

            foreach (var node in descendantsOriginal)
            {
                var originalNode = node;
                var toEditNode = NodeDict[node];

                if (originalNode.Name.Contains("#"))
                {
                    continue;
                }
                else
                {
                    bool onlyText = true;
                    bool onlySubtags = true;

                    foreach (var child in node.ChildNodes)
                    {
                        if (child.Name.Contains("#"))
                        {
                            onlySubtags = false;
                        }
                        else
                        {
                            onlyText = false;
                        }
                    }

                    if (onlyText || onlySubtags)
                    {
                        continue;
                    }
                    else
                    {

                        foreach (var child in node.ChildNodes)
                        {
                            if (child.Name.Contains("#text"))
                            {
                                HtmlNode coveringSpan = docToEdit.CreateElement("span");
                                coveringSpan.InnerHtml = child.InnerHtml;
                                TagStartPositions.Add(child.InnerStartIndex);
                                TagStartLengths.Add(0);
                                TagClosingPositions.Add(child.InnerStartIndex + child.InnerLength);
                                TagClosingLengths.Add(0);
                                coveringSpan.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, (TagStartPositions.Count - 1).ToString());

                                var parent = NodeDict[node];
                                var index = parent.ChildNodes.IndexOf(NodeDict[child]);

                                parent.ChildNodes.RemoveAt(index);
                                parent.ChildNodes.Insert(index, coveringSpan);
                            }
                        }
                    }
                }
            }
        }

        private void GenerateCSS(List<AnnotationTagGeneric> tags)
        {
            /*string inner = "span.annotation {border-bottom: 2px solid;}";
            inner += "span {line-height: 30px}\n";*/

            var tagPaddingDict = Intersections.ColorGraph(Intersections.FindIntersections(tags));
            foreach (var tag in tags.DistinctBy(t => t.Instance))
            {
                var padding = (tagPaddingDict[tag] + 1) * 2;
                TagInstanceCSS.Add(new()
                {
                    InstanceId = tag.Instance,
                    Color = tag.Tag.Color,
                    Padding = padding
                });
            }
        }

        private void AssignIdsToOriginalDocument(IEnumerable<HtmlNode> descendantsOriginal, ref int currentId)
        {
            foreach (var node in descendantsOriginal)
            {
                var originalNode = node;
                var toEditNode = NodeDict[node];

                if (originalNode.Name.Contains("#"))
                {
                    continue;
                }
                else
                {
                    TagStartPositions.Add(originalNode.OuterStartIndex);
                    TagStartLengths.Add(originalNode.InnerStartIndex - originalNode.OuterStartIndex);
                    currentId = TagStartPositions.Count - 1;
                    toEditNode.Attributes.Add(TAG_ID_ATTRIBUTE_NAME, currentId.ToString());

                    TagClosingPositions.Add(originalNode.InnerStartIndex + originalNode.InnerLength);
                    TagClosingLengths.Add((originalNode.OuterStartIndex + originalNode.OuterLength) - (originalNode.InnerStartIndex + originalNode.InnerLength));
                }
            }
        }

        private void FillNodeDict(IEnumerable<HtmlNode> descendantsOriginal, IEnumerable<HtmlNode> descendantsToEdit)
        {
            var zipped = descendantsOriginal.Zip(descendantsToEdit, (orig, toEdit) => new
            {
                Original = orig,
                ToEdit = toEdit
            });
            foreach (var node in zipped)
            {
                var originalNode = node.Original;
                var toEditNode = node.ToEdit;
                NodeDict.Add(originalNode, toEditNode);
            }
        }


        /*
         *      Full HTML Preprocessing -------------------------------------------------------------------------------
         */

        /*
         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
         */

        public (string, CachedInfo) PartialPreprocessHTMLAddTag(string htmlToEdit, string htmlOriginal, AnnotationTagGeneric tagToAdd, List<AnnotationTagGeneric> tags, CachedInfo cachedInfo)
        {
            UnpackCachedInfo(cachedInfo);

            var docOriginal = new HtmlDocument();
            docOriginal.LoadHtml(htmlOriginal);
            var docToEdit = new HtmlDocument();
            docToEdit.LoadHtml(htmlToEdit);

            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();

            int i = 0;
            List<HtmlNode> addedForSelection = new();

            int descendantsCount = descendantsToEdit.Count();
            while (i < descendantsCount)
            {
                for (; i < descendantsCount; i++)
                {
                    var node = descendantsToEdit.ElementAt(i);
                    if (!node.Name.Contains("#text") || addedForSelection.Contains(node) || addedForSelection.Contains(node.ParentNode) ||
                        node.ParentNode.Name == "style" || node.ParentNode.Name.StartsWith("#"))
                    {
                        continue;
                    }

                    int nodeId = node.ParentNode.GetAttributeValue(TAG_ID_ATTRIBUTE_NAME, -1);

                    var start = TagStartPositions[nodeId] + TagStartLengths[nodeId];
                    var end = TagClosingPositions[nodeId];

                    int selectionStart = tagToAdd.Position;
                    int selectionEnd = tagToAdd.Position + tagToAdd.Length;

                    if (selectionStart < end && selectionEnd > start)
                    {
                        if (selectionStart <= start && selectionEnd >= end)
                        {
                            addedForSelection.Add(SolveFullFill(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        else if (selectionStart <= start)
                        {
                            addedForSelection.AddRange(SolveRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        else if (selectionEnd >= end)
                        {
                            addedForSelection.AddRange(SolveLeftGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        else
                        {
                            addedForSelection.AddRange(SolveLeftRightGap(node, selectionStart, selectionEnd, start, end, docToEdit, tagToAdd));
                        }
                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
                        descendantsCount = descendantsToEdit.Count();
                        break;
                    }
                }
            }

            GenerateCSS(tags);
            return (docToEdit.DocumentNode.OuterHtml, new()
            {
                TagStartPositions = TagStartPositions,
                TagStartLengths = TagStartLengths,
                TagClosingPositions = TagClosingPositions,
                TagClosingLengths = TagClosingLengths,
                TagInstanceCSS = TagInstanceCSS,
                NodeDict = NodeDict
            });
        }


        public (string, CachedInfo) PartialPreprocessHTMLRemoveTag(string htmlToEdit, string htmlOriginal, AnnotationTagGeneric tagToRemove, List<AnnotationTagGeneric> tags, CachedInfo cachedInfo)
        {
            UnpackCachedInfo(cachedInfo);

            var docOriginal = new HtmlDocument();
            docOriginal.LoadHtml(htmlOriginal);
            var docToEdit = new HtmlDocument();
            docToEdit.LoadHtml(htmlToEdit);

            var descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();

            int i = 0;
            int descendantsCount = descendantsToEdit.Count();
            while (i < descendantsCount)
            {
                for (; i < descendantsCount; i++)
                {
                    var node = descendantsToEdit.ElementAt(i);
                    if (!node.Attributes.Contains(TAG_EF_ID_ATTRIBUTE_NAME))
                    {
                        continue;
                    }
                    else
                    {
                        if (node.Attributes[TAG_EF_ID_ATTRIBUTE_NAME].Value != tagToRemove.Id.ToString())
                        {
                            continue;
                        }

                        node.Attributes.Remove(TAG_EF_ID_ATTRIBUTE_NAME);
                        node.Attributes.Remove(TAG_INSTANCE_ATTRIBUTE_NAME);
                        node.Attributes.Remove("class");
                        node.Attributes.Remove("start");
                        node.Attributes.Remove("end");

                        descendantsToEdit = docToEdit.DocumentNode.DescendantsAndSelf().ToList();
                        descendantsCount = descendantsToEdit.Count();
                        break;
                    }
                }
            }

            GenerateCSS(tags);
            return (docToEdit.DocumentNode.OuterHtml, new()
            {
                TagStartPositions = TagStartPositions,
                TagStartLengths = TagStartLengths,
                TagClosingPositions = TagClosingPositions,
                TagClosingLengths = TagClosingLengths,
                TagInstanceCSS = TagInstanceCSS,
                NodeDict = NodeDict
            });
        }

        /*
         *      Partial HTML Preprocessing ----------------------------------------------------------------------------
         */
    }
}