Revize 830652de

Export in BE

1. Conflicts:
2. Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs
3. Backend/Core/Services/AnnotationService/IAnnotationService.cs

Backend/Backend/Controllers/DocumentController.cs

@@ -208 +208 @@
             throw new BadRequestException(e.Message);
         }
     }
+
+    [HttpPost("/documents/export")]
+    [Authorize(Models.Enums.ERole.ADMINISTRATOR)]
+    [ProducesResponseType((int)HttpStatusCode.OK)]
+    [ProducesResponseType((int)HttpStatusCode.Forbidden)]
+    public ActionResult<ExportResponse> ExportDocuments([FromServices] ClientInfo clientInfo, [FromBody] ExportRequest request)
+    {
+        if (clientInfo.LoggedUser == null)
+        {
+            logger.Warning("ClientInfo has null LoggerUser in [Authorized] controller");
+            return Problem();
+        }
+
+        try
+        {
+            var outputStream = annotationService.Export(request);
+
+            var cd = new System.Net.Mime.ContentDisposition()
+            {
+                FileName = "export.zip",
+                Inline = false,
+            };
+
+            Response.Headers.Add("Content-Disposition", cd.ToString());
+            return File(outputStream, "application/zip");
+        }
+        catch (InvalidOperationException e)
+        {
+            throw new BadRequestException(e.Message);
+        }
+    }
 }
 

Backend/Core/Services/AnnotationService/AnnotationServiceEF.cs

@@ -16 +16 @@
 using System.Text.RegularExpressions;
 using Newtonsoft.Json;
 using Core.GraphUtils;
+using Models.Documents;
 
 namespace Core.Services.AnnotationService
 {
@@ -775 +776 @@
 
             return true;
         }
-
+        
         public void SetTagIsFinal(Guid annotationId, Guid occurenceId, bool isFinal)
         {
             Annotation annotation;
@@ -802 +803 @@
 
             context.SaveChanges();
         }
+
+        public MemoryStream Export(ExportRequest request)
+        {
+            if (request.ExportAllDone && request.DocumentIds == null)
+            {
+                throw new InvalidOperationException("No documents specified");
+            }
+
+            // Get documents
+            IEnumerable<Document> documentsToExport = null;
+            if (request.ExportAllDone)
+            {
+                documentsToExport = context.Documents
+                    .Include(d => d.Content);
+            }
+            else
+            {
+                documentsToExport = context.Documents
+                    .Include(d => d.Content)
+                    .Where(d => request.DocumentIds.Contains(d.Id));
+            }
+
+            // Get annotations
+            Dictionary<Document, List<Annotation>> annotationsToExport = new();
+            foreach (var document in documentsToExport)
+            {
+                annotationsToExport[document] = context.Annotations
+                    .Where(a => a.Document == document && a.State == EState.DONE)
+                    .Include(a => a.Document)
+                    .Include(a => a.User)
+                    .ToList();
+            }
+
+            // Get final annotations
+            Dictionary<Document, FinalAnnotation> finalAnnotationsToExport = new();
+            foreach (var document in documentsToExport)
+            {
+                if (!context.FinalAnnotations.Any(fa => fa.Document == document))
+                {
+                    finalAnnotationsToExport[document] = null;
+                }
+                else
+                {
+                    finalAnnotationsToExport[document] = context.FinalAnnotations
+                        .Single(fa => fa.Document == document);
+                }
+            }
+
+            // Get tags for each annotation
+            Dictionary<Annotation, List<AnnotationTag>> tagsToExport = new();
+            foreach (var document in annotationsToExport.Keys)
+            {
+                foreach (var annotation in annotationsToExport[document])
+                {
+                    tagsToExport[annotation] = context.AnnotationTags
+                        .Include(at => at.Annotation)
+                        .Include(at => at.Tag).ThenInclude(t => t.Category)
+                        .Include(at => at.SubTag)
+                        .Where(at => at.Annotation == annotation)
+                        .ToList();
+                }
+            }
+
+            // Get tags for each final annotation
+            Dictionary<FinalAnnotation, List<FinalAnnotationTag>> finalTagsToExport = new();
+            foreach (var document in finalAnnotationsToExport.Keys)
+            {
+                var finalAnnotation = finalAnnotationsToExport[document];
+                finalTagsToExport[finalAnnotation] = context.FinalAnnotationTags
+                    .Include(at => at.Annotation)
+                    .Include(at => at.Tag).ThenInclude(t => t.Category)
+                    .Include(at => at.SubTag)
+                    .Where(at => at.Annotation == finalAnnotation)
+                    .ToList();
+            }
+
+            MemoryStream output = ZipUtils.Export.FullExport(documentsToExport.ToList(), annotationsToExport, finalAnnotationsToExport, tagsToExport, finalTagsToExport);
+            return output;
+        }
     }
 }

Backend/Core/Services/AnnotationService/IAnnotationService.cs

@@ -1 +1 @@
 using Models.Annotations;
+using Models.Documents;
 using Models.Enums;
 using System;
 using System.Collections.Generic;
@@ -19 +20 @@
         public void SetTagInstanceSentiment(Guid annotationId, Guid instanceId, Guid userId, ERole userRole, ETagSentiment sentiment, bool isFinal);
         public void MarkAnnotationAsDone(Guid annotationId, Guid userId, ERole userRole, bool done, bool isFinal);
         public Guid CreateFinalAnnotation(Guid documentId, Guid userId);
+        public MemoryStream Export(ExportRequest request);
         public void SetTagIsFinal(Guid annotationId, Guid occurenceId, bool isFinal);
     }
 }

Backend/Core/ZipUtils/Export.cs

@@ +1 @@
+using Core.Entities;
+using Models.Enums;
+using Newtonsoft.Json;
+using System;
+using System.Collections.Generic;
+using System.IO.Compression;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Core.ZipUtils
+{
+    public class Export
+    {
+        public static MemoryStream FullExport(List<Document> documents, Dictionary<Document, List<Annotation>> documentAnnotations, Dictionary<Document, FinalAnnotation> documentFinalAnnotations,
+            Dictionary<Annotation, List<AnnotationTag>> annotationTags, Dictionary<FinalAnnotation, List<FinalAnnotationTag>> finalAnnotationTags)
+        {
+            MemoryStream ms = new MemoryStream();
+            var archive = new ZipArchive(ms);
+
+            for (int docIndex = 0; docIndex < documents.Count; docIndex++)
+            {
+                var document = documents[docIndex];
+                string documentDir = $"{docIndex:00000}";
+
+                // Write document content and name into a JSON file
+                string documentInfoJson = DumpDocument(document);
+                CreateFile(documentDir + "/document.json", documentInfoJson, archive);
+
+                // Deal with annotations
+                var annotations = documentAnnotations[document];
+                for (int annotationIndex = 0; annotationIndex < annotations.Count; annotationIndex++)
+                {
+                    var annotation = annotations[annotationIndex];
+                    var tags = annotationTags[annotation].Select(at => at as AnnotationTagGeneric).ToList();
+                    string annotationInfoJson = DumpAnnotation(annotation, tags, document.Id);
+                    CreateFile(documentDir + $"/annotation_{annotationIndex:00000}.json", annotationInfoJson, archive);
+                }
+
+                var finalAnnotation = documentFinalAnnotations[document];
+                var finalTags = finalAnnotationTags[finalAnnotation].Select(ft => ft as AnnotationTagGeneric).ToList();
+                string finalAnnotationInfoJson = DumpAnnotation(finalAnnotation, finalTags, document.Id);
+                CreateFile(documentDir + "/final.json", finalAnnotationInfoJson, archive);
+            }
+
+            archive.Dispose();
+            ms.Position = 0;
+
+            return ms;
+        }
+
+        private static void CreateFile(string path, string fileContent, ZipArchive archive)
+        {
+            var entry = archive.CreateEntry(path);
+            var entryStream = entry.Open();
+            entryStream.Write(Encoding.UTF8.GetBytes(fileContent));
+            entryStream.Close();
+        }
+
+        private static string DumpDocument(Document document)
+        {
+            ExportDocumentInfo documentInfo = new();
+            documentInfo.Name = document.Name;
+            documentInfo.Content = document.Content.Content;
+            documentInfo.DocumentId = document.Id;
+
+            return JsonConvert.SerializeObject(documentInfo);
+        }
+
+        private static string DumpAnnotation(Annotation annotation, List<AnnotationTagGeneric> tags, Guid documentId)
+        {
+            ExportAnnotationInfo annotationInfo = new();
+            annotationInfo.AnnotatorId = annotation.User.Id;
+            annotationInfo.DocumentId = documentId;
+
+            foreach (var tag in tags)
+            {
+                ExportTagInstanceInfo tagInfo = new()
+                {
+
+                    Category = tag.Tag.Category.Name,
+                    TagName = tag.Tag.Name,
+                    InstanceId = tag.Instance,
+                    FirstCharPosition = tag.Position,
+                    Length = tag.Length,
+                };
+
+                if (tag.Tag.SentimentEnabled && tagInfo.Sentiment != null)
+                {
+                    tagInfo.Sentiment = tag.Sentiment.ToString().ToLower();
+                }
+
+                if (tag.SubTag != null)
+                {
+                    tagInfo.SubTagName = tag.SubTag.Name;
+                }
+
+                annotationInfo.Tags.Add(tagInfo);
+            }
+
+            return JsonConvert.SerializeObject(annotationInfo);
+        }
+    }
+
+    public class ExportDocumentInfo
+    {
+        public string Name { get; set; }
+        public string Content { get; set; }
+        public Guid DocumentId { get; set; }
+    }
+
+    public class ExportAnnotationInfo
+    {
+        public Guid AnnotatorId { get; set; }
+        public Guid DocumentId { get; set; }
+        public List<ExportTagInstanceInfo> Tags { get; set; } = new();
+    }
+
+    public class ExportTagInstanceInfo
+    {
+        public string Category { get; set; }
+        public string TagName { get; set; }
+        public string? SubTagName { get; set; }
+        public Guid InstanceId { get; set; }
+        public int FirstCharPosition { get; set; }
+        public int Length { get; set; }
+        public string? Sentiment { get; set; }
+    }
+}

Backend/Models/Documents/ExportRequest.cs

@@ +1 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Models.Documents
+{
+    public class ExportRequest
+    {
+        public List<Guid>? DocumentIds { get; set; }
+        public bool ExportAllDone { get; set; }
+    }
+}

Backend/Models/Documents/ExportResponse.cs

@@ +1 @@
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Models.Documents
+{
+    public class ExportResponse
+    {
+
+    }
+}