Include inline emoji in JSON export (#1311)

DiscordChatExporter.Cli.Tests/Infra/ChannelIds.cs

@@ -10,6 +10,8 @@ public static class ChannelIds
 
     public static Snowflake EmbedTestCases { get; } = Snowflake.Parse("866472452459462687");
 
+    public static Snowflake EmojiTestCases { get; } = Snowflake.Parse("866768438290415636");
+
     public static Snowflake GroupingTestCases { get; } = Snowflake.Parse("992092091545034842");
 
     public static Snowflake FilterTestCases { get; } = Snowflake.Parse("866744075033641020");

DiscordChatExporter.Cli.Tests/Specs/JsonEmojiSpecs.cs

@@ -0,0 +1,69 @@
+using System.Linq;
+using System.Threading.Tasks;
+using DiscordChatExporter.Cli.Tests.Infra;
+using DiscordChatExporter.Core.Discord;
+using FluentAssertions;
+using Xunit;
+
+namespace DiscordChatExporter.Cli.Tests.Specs;
+
+public class JsonEmojiSpecs
+{
+    [Fact]
+    public async Task I_can_export_a_channel_that_contains_a_message_with_inline_emoji_and_have_them_listed_separately()
+    {
+        // Act
+        var message = await ExportWrapper.GetMessageAsJsonAsync(
+            ChannelIds.EmojiTestCases,
+            Snowflake.Parse("866768521052553216")
+        );
+
+        // Assert
+        var inlineEmojis = message.GetProperty("inlineEmojis").EnumerateArray().ToArray();
+        inlineEmojis.Should().HaveCount(4);
+
+        inlineEmojis[0].GetProperty("id").GetString().Should().BeNullOrEmpty();
+        inlineEmojis[0].GetProperty("name").GetString().Should().Be("🙂");
+        inlineEmojis[0].GetProperty("code").GetString().Should().Be("slight_smile");
+        inlineEmojis[0].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
+        inlineEmojis[0].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
+
+        inlineEmojis[1].GetProperty("id").GetString().Should().BeNullOrEmpty();
+        inlineEmojis[1].GetProperty("name").GetString().Should().Be("😦");
+        inlineEmojis[1].GetProperty("code").GetString().Should().Be("frowning");
+        inlineEmojis[1].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
+        inlineEmojis[1].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
+
+        inlineEmojis[2].GetProperty("id").GetString().Should().BeNullOrEmpty();
+        inlineEmojis[2].GetProperty("name").GetString().Should().Be("😔");
+        inlineEmojis[2].GetProperty("code").GetString().Should().Be("pensive");
+        inlineEmojis[2].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
+        inlineEmojis[2].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
+
+        inlineEmojis[3].GetProperty("id").GetString().Should().BeNullOrEmpty();
+        inlineEmojis[3].GetProperty("name").GetString().Should().Be("😂");
+        inlineEmojis[3].GetProperty("code").GetString().Should().Be("joy");
+        inlineEmojis[3].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
+        inlineEmojis[3].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
+    }
+
+    [Fact]
+    public async Task I_can_export_a_channel_that_contains_a_message_with_custom_inline_emoji_and_have_them_listed_separately()
+    {
+        // Act
+        var message = await ExportWrapper.GetMessageAsJsonAsync(
+            ChannelIds.EmojiTestCases,
+            Snowflake.Parse("1299804867447230594")
+        );
+
+        // Assert
+        var inlineEmojis = message.GetProperty("inlineEmojis").EnumerateArray().ToArray();
+        inlineEmojis.Should().HaveCount(1);
+
+        inlineEmojis[0].GetProperty("id").GetString().Should().Be("754441880066064584");
+        inlineEmojis[0].GetProperty("name").GetString().Should().Be("lemon_blush");
+        inlineEmojis[0].GetProperty("code").GetString().Should().Be("lemon_blush");
+        inlineEmojis[0].GetProperty("isAnimated").GetBoolean().Should().BeFalse();
+        inlineEmojis[0].GetProperty("imageUrl").GetString().Should().NotBeNullOrWhiteSpace();
+    }
+}

DiscordChatExporter.Core/Discord/Data/Emoji.cs

@@ -1,5 +1,4 @@
-using System;
+using System.Text.Json;
-using System.Text.Json;
 using DiscordChatExporter.Core.Discord.Data.Common;
 using DiscordChatExporter.Core.Utils;
 using DiscordChatExporter.Core.Utils.Extensions;
@@ -13,29 +12,22 @@ public partial record Emoji(
     Snowflake? Id,
     // Name of a custom emoji (e.g. LUL) or actual representation of a standard emoji (e.g. 🙂)
     string Name,
-    bool IsAnimated,
+    bool IsAnimated
-    string ImageUrl
 )
 {
+    public bool IsCustomEmoji { get; } = Id is not null;
+
     // Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
-    public string Code => Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
+    public string Code { get; } = Id is not null ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
+
+    public string ImageUrl { get; } =
+        Id is not null
+            ? ImageCdn.GetCustomEmojiUrl(Id.Value, IsAnimated)
+            : ImageCdn.GetStandardEmojiUrl(Name);
 }
 
 public partial record Emoji
 {
-    public static string GetImageUrl(Snowflake? id, string? name, bool isAnimated)
-    {
-        // Custom emoji
-        if (id is not null)
-            return ImageCdn.GetCustomEmojiUrl(id.Value, isAnimated);
-
-        // Standard emoji
-        if (!string.IsNullOrWhiteSpace(name))
-            return ImageCdn.GetStandardEmojiUrl(name);
-
-        throw new InvalidOperationException("Either the emoji ID or name should be provided.");
-    }
-
     public static Emoji Parse(JsonElement json)
     {
         var id = json.GetPropertyOrNull("id")
@@ -47,8 +39,7 @@ public partial record Emoji
             json.GetPropertyOrNull("name")?.GetNonWhiteSpaceStringOrNull() ?? "Unknown Emoji";
 
         var isAnimated = json.GetPropertyOrNull("animated")?.GetBooleanOrNull() ?? false;
-        var imageUrl = GetImageUrl(id, name, isAnimated);
 
-        return new Emoji(id, name, isAnimated, imageUrl);
+        return new Emoji(id, name, isAnimated);
     }
 }

DiscordChatExporter.Core/Exporting/HtmlMarkdownVisitor.cs

@@ -5,7 +5,6 @@ using System.Text;
 using System.Text.RegularExpressions;
 using System.Threading;
 using System.Threading.Tasks;
-using DiscordChatExporter.Core.Discord.Data;
 using DiscordChatExporter.Core.Markdown;
 using DiscordChatExporter.Core.Markdown.Parsing;
 using DiscordChatExporter.Core.Utils.Extensions;
@@ -210,7 +209,6 @@ internal partial class HtmlMarkdownVisitor(
         CancellationToken cancellationToken = default
     )
     {
-        var emojiImageUrl = Emoji.GetImageUrl(emoji.Id, emoji.Name, emoji.IsAnimated);
         var jumboClass = isJumbo ? "chatlog__emoji--large" : "";
 
         buffer.Append(
@@ -221,7 +219,7 @@ internal partial class HtmlMarkdownVisitor(
                 class="chatlog__emoji {jumboClass}"
                 alt="{emoji.Name}"
                 title="{emoji.Code}"
-                src="{await context.ResolveAssetUrlAsync(emojiImageUrl, cancellationToken)}">
+                src="{await context.ResolveAssetUrlAsync(emoji.ImageUrl, cancellationToken)}">
             """
         );
     }

DiscordChatExporter.Core/Exporting/JsonMessageWriter.cs

@@ -1,12 +1,14 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
+using System.Linq;
 using System.Text.Encodings.Web;
 using System.Text.Json;
 using System.Threading;
 using System.Threading.Tasks;
 using DiscordChatExporter.Core.Discord.Data;
 using DiscordChatExporter.Core.Discord.Data.Embeds;
+using DiscordChatExporter.Core.Markdown.Parsing;
 using DiscordChatExporter.Core.Utils.Extensions;
 using JsonExtensions.Writing;
 
@@ -37,22 +39,31 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
             ? await PlainTextMarkdownVisitor.FormatAsync(Context, markdown, cancellationToken)
             : markdown;
 
-    private async ValueTask WriteUserAsync(User user, CancellationToken cancellationToken = default)
+    private async ValueTask WriteUserAsync(
+        User user,
+        bool includeRoles = true,
+        CancellationToken cancellationToken = default
+    )
     {
         _writer.WriteStartObject();
 
         _writer.WriteString("id", user.Id.ToString());
         _writer.WriteString("name", user.Name);
         _writer.WriteString("discriminator", user.DiscriminatorFormatted);
+
         _writer.WriteString(
             "nickname",
             Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
         );
+
         _writer.WriteString("color", Context.TryGetUserColor(user.Id)?.ToHex());
         _writer.WriteBoolean("isBot", user.IsBot);
 
-        _writer.WritePropertyName("roles");
+        if (includeRoles)
-        await WriteRolesAsync(Context.GetUserRoles(user.Id), cancellationToken);
+        {
+            _writer.WritePropertyName("roles");
+            await WriteRolesAsync(Context.GetUserRoles(user.Id), cancellationToken);
+        }
 
         _writer.WriteString(
             "avatarUrl",
@@ -66,6 +77,26 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
         await _writer.FlushAsync(cancellationToken);
     }
 
+    private async ValueTask WriteEmojiAsync(
+        Emoji emoji,
+        CancellationToken cancellationToken = default
+    )
+    {
+        _writer.WriteStartObject();
+
+        _writer.WriteString("id", emoji.Id.ToString());
+        _writer.WriteString("name", emoji.Name);
+        _writer.WriteString("code", emoji.Code);
+        _writer.WriteBoolean("isAnimated", emoji.IsAnimated);
+        _writer.WriteString(
+            "imageUrl",
+            await Context.ResolveAssetUrlAsync(emoji.ImageUrl, cancellationToken)
+        );
+
+        _writer.WriteEndObject();
+        await _writer.FlushAsync(cancellationToken);
+    }
+
     private async ValueTask WriteRolesAsync(
         IReadOnlyList<Role> roles,
         CancellationToken cancellationToken = default
@@ -273,6 +304,26 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
 
         _writer.WriteEndArray();
 
+        // Inline emoji
+        _writer.WriteStartArray("inlineEmojis");
+
+        if (!string.IsNullOrWhiteSpace(embed.Description))
+        {
+            foreach (
+                var emoji in MarkdownParser
+                    .ExtractEmojis(embed.Description)
+                    .DistinctBy(e => e.Name, StringComparer.Ordinal)
+            )
+            {
+                await WriteEmojiAsync(
+                    new Emoji(emoji.Id, emoji.Name, emoji.IsAnimated),
+                    cancellationToken
+                );
+            }
+        }
+
+        _writer.WriteEndArray();
+
         _writer.WriteEndObject();
         await _writer.FlushAsync(cancellationToken);
     }
@@ -373,7 +424,7 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
 
         // Author
         _writer.WritePropertyName("author");
-        await WriteUserAsync(message.Author, cancellationToken);
+        await WriteUserAsync(message.Author, true, cancellationToken);
 
         // Attachments
         _writer.WriteStartArray("attachments");
@@ -431,20 +482,14 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
             _writer.WriteStartObject();
 
             // Emoji
-            _writer.WriteStartObject("emoji");
+            _writer.WritePropertyName("emoji");
-            _writer.WriteString("id", reaction.Emoji.Id.ToString());
+            await WriteEmojiAsync(reaction.Emoji, cancellationToken);
-            _writer.WriteString("name", reaction.Emoji.Name);
-            _writer.WriteString("code", reaction.Emoji.Code);
-            _writer.WriteBoolean("isAnimated", reaction.Emoji.IsAnimated);
-            _writer.WriteString(
-                "imageUrl",
-                await Context.ResolveAssetUrlAsync(reaction.Emoji.ImageUrl, cancellationToken)
-            );
-            _writer.WriteEndObject();
 
             _writer.WriteNumber("count", reaction.Count);
 
+            // Reaction authors
             _writer.WriteStartArray("users");
+
             await foreach (
                 var user in Context.Discord.GetMessageReactionsAsync(
                     Context.Request.Channel.Id,
@@ -454,28 +499,7 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
                 )
             )
             {
-                _writer.WriteStartObject();
+                await WriteUserAsync(user, false, cancellationToken);
-
-                // Write limited user information without color and roles,
-                // so we can avoid fetching guild member information for each user.
-                _writer.WriteString("id", user.Id.ToString());
-                _writer.WriteString("name", user.Name);
-                _writer.WriteString("discriminator", user.DiscriminatorFormatted);
-                _writer.WriteString(
-                    "nickname",
-                    Context.TryGetMember(user.Id)?.DisplayName ?? user.DisplayName
-                );
-                _writer.WriteBoolean("isBot", user.IsBot);
-
-                _writer.WriteString(
-                    "avatarUrl",
-                    await Context.ResolveAssetUrlAsync(
-                        Context.TryGetMember(user.Id)?.AvatarUrl ?? user.AvatarUrl,
-                        cancellationToken
-                    )
-                );
-
-                _writer.WriteEndObject();
             }
 
             _writer.WriteEndArray();
@@ -487,9 +511,8 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
 
         // Mentions
         _writer.WriteStartArray("mentions");
-
         foreach (var user in message.MentionedUsers)
-            await WriteUserAsync(user, cancellationToken);
+            await WriteUserAsync(user, true, cancellationToken);
 
         _writer.WriteEndArray();
 
@@ -512,11 +535,28 @@ internal class JsonMessageWriter(Stream stream, ExportContext context)
             _writer.WriteString("name", message.Interaction.Name);
 
             _writer.WritePropertyName("user");
-            await WriteUserAsync(message.Interaction.User, cancellationToken);
+            await WriteUserAsync(message.Interaction.User, true, cancellationToken);
 
             _writer.WriteEndObject();
         }
 
+        // Inline emoji
+        _writer.WriteStartArray("inlineEmojis");
+
+        foreach (
+            var emoji in MarkdownParser
+                .ExtractEmojis(message.Content)
+                .DistinctBy(e => e.Name, StringComparer.Ordinal)
+        )
+        {
+            await WriteEmojiAsync(
+                new Emoji(emoji.Id, emoji.Name, emoji.IsAnimated),
+                cancellationToken
+            );
+        }
+
+        _writer.WriteEndArray();
+
         _writer.WriteEndObject();
         await _writer.FlushAsync(cancellationToken);
     }

DiscordChatExporter.Core/Markdown/EmojiNode.cs

@@ -1,5 +1,5 @@
 using DiscordChatExporter.Core.Discord;
-using DiscordChatExporter.Core.Utils;
+using DiscordChatExporter.Core.Discord.Data;
 
 namespace DiscordChatExporter.Core.Markdown;
 
@@ -11,11 +11,17 @@ internal record EmojiNode(
     bool IsAnimated
 ) : MarkdownNode
 {
-    public bool IsCustomEmoji => Id is not null;
+    // This coupling is unsound from the domain-design perspective, but it helps us reuse
-
+    // some code for now. We can refactor this later, if the coupling becomes a problem.
-    // Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
+    private readonly Emoji _emoji = new(Id, Name, IsAnimated);
-    public string Code => IsCustomEmoji ? Name : EmojiIndex.TryGetCode(Name) ?? Name;
 
     public EmojiNode(string name)
         : this(null, name, false) { }
+
+    public bool IsCustomEmoji => _emoji.IsCustomEmoji;
+
+    // Name of a custom emoji (e.g. LUL) or name of a standard emoji (e.g. slight_smile)
+    public string Code => _emoji.Code;
+
+    public string ImageUrl => _emoji.ImageUrl;
 }

DiscordChatExporter.Core/Markdown/Parsing/MarkdownParser.cs

@@ -484,6 +484,37 @@ internal static partial class MarkdownParser
 
 internal static partial class MarkdownParser
 {
+    private static void Extract<TNode>(
+        IEnumerable<MarkdownNode> nodes,
+        ICollection<TNode> extractedNodes
+    )
+        where TNode : MarkdownNode
+    {
+        foreach (var node in nodes)
+        {
+            if (node is TNode extractedNode)
+                extractedNodes.Add(extractedNode);
+
+            if (node is IContainerNode containerNode)
+                Extract(containerNode.Children, extractedNodes);
+        }
+    }
+
+    public static IReadOnlyList<TNode> Extract<TNode>(string markdown)
+        where TNode : MarkdownNode
+    {
+        var extractedNodes = new List<TNode>();
+        Extract(Parse(markdown), extractedNodes);
+
+        return extractedNodes;
+    }
+
+    public static IReadOnlyList<LinkNode> ExtractLinks(string markdown) =>
+        Extract<LinkNode>(markdown);
+
+    public static IReadOnlyList<EmojiNode> ExtractEmojis(string markdown) =>
+        Extract<EmojiNode>(markdown);
+
     private static IReadOnlyList<MarkdownNode> Parse(
         MarkdownContext context,
         StringSegment segment
@@ -499,24 +530,4 @@ internal static partial class MarkdownParser
 
     public static IReadOnlyList<MarkdownNode> ParseMinimal(string markdown) =>
         ParseMinimal(new MarkdownContext(), new StringSegment(markdown));
-
-    private static void ExtractLinks(IEnumerable<MarkdownNode> nodes, ICollection<LinkNode> links)
-    {
-        foreach (var node in nodes)
-        {
-            if (node is LinkNode linkNode)
-                links.Add(linkNode);
-
-            if (node is IContainerNode containerNode)
-                ExtractLinks(containerNode.Children, links);
-        }
-    }
-
-    public static IReadOnlyList<LinkNode> ExtractLinks(string markdown)
-    {
-        var links = new List<LinkNode>();
-        ExtractLinks(Parse(markdown), links);
-
-        return links;
-    }
 }