Merge dc8c74bcc8 into 30ba273fb1

DiscordChatExporter.Cli/Commands/Base/ExportCommandBase.cs

@@ -8,6 +8,7 @@ using CliFx.Attributes;
 using CliFx.Exceptions;
 using CliFx.Infrastructure;
 using DiscordChatExporter.Cli.Commands.Converters;
+using DiscordChatExporter.Cli.Commands.Shared;
 using DiscordChatExporter.Cli.Utils.Extensions;
 using DiscordChatExporter.Core.Discord;
 using DiscordChatExporter.Core.Discord.Data;
@@ -64,6 +65,13 @@ public abstract class ExportCommandBase : DiscordCommandBase
     )]
     public PartitionLimit PartitionLimit { get; init; } = PartitionLimit.Null;
 
+    [CommandOption(
+        "include-threads",
+        Description = "Which types of threads should be included.",
+        Converter = typeof(ThreadInclusionModeBindingConverter)
+    )]
+    public ThreadInclusionMode ThreadInclusionMode { get; init; } = ThreadInclusionMode.None;
+
     [CommandOption(
         "filter",
         Description = "Only include messages that satisfy this filter. "

DiscordChatExporter.Cli/Commands/ExportAllCommand.cs

@@ -27,13 +27,6 @@ public class ExportAllCommand : ExportCommandBase
     [CommandOption("include-vc", Description = "Include voice channels.")]
     public bool IncludeVoiceChannels { get; init; } = true;
 
-    [CommandOption(
-        "include-threads",
-        Description = "Which types of threads should be included.",
-        Converter = typeof(ThreadInclusionModeBindingConverter)
-    )]
-    public ThreadInclusionMode ThreadInclusionMode { get; init; } = ThreadInclusionMode.None;
-
     [CommandOption(
         "data-package",
         Description = "Path to the personal data package (ZIP file) requested from Discord. "

DiscordChatExporter.Cli/Commands/ExportChannelsCommand.cs

@@ -1,9 +1,16 @@
 using System.Collections.Generic;
+using System.Linq;
 using System.Threading.Tasks;
 using CliFx.Attributes;
 using CliFx.Infrastructure;
 using DiscordChatExporter.Cli.Commands.Base;
+using DiscordChatExporter.Cli.Commands.Converters;
+using DiscordChatExporter.Cli.Commands.Shared;
+using DiscordChatExporter.Cli.Utils.Extensions;
 using DiscordChatExporter.Core.Discord;
+using DiscordChatExporter.Core.Discord.Data;
+using DiscordChatExporter.Core.Utils.Extensions;
+using Spectre.Console;
 
 namespace DiscordChatExporter.Cli.Commands;
 
@@ -22,6 +29,76 @@ public class ExportChannelsCommand : ExportCommandBase
     public override async ValueTask ExecuteAsync(IConsole console)
     {
         await base.ExecuteAsync(console);
-        await ExportAsync(console, ChannelIds);
+
+        var cancellationToken = console.RegisterCancellationHandler();
+
+        await console.Output.WriteLineAsync("Resolving channel(s)...");
+
+        var channels = new List<Channel>();
+        var channelsByGuild = new Dictionary<Snowflake, IReadOnlyList<Channel>>();
+
+        foreach (var channelId in ChannelIds)
+        {
+            var channel = await Discord.GetChannelAsync(channelId, cancellationToken);
+
+            // Unwrap categories
+            if (channel.IsCategory)
+            {
+                var guildChannels =
+                    channelsByGuild.GetValueOrDefault(channel.GuildId)
+                    ?? await Discord.GetGuildChannelsAsync(channel.GuildId, cancellationToken);
+
+                foreach (var guildChannel in guildChannels)
+                {
+                    if (guildChannel.Parent?.Id == channel.Id)
+                        channels.Add(guildChannel);
+                }
+
+                // Cache the guild channels to avoid redundant work
+                channelsByGuild[channel.GuildId] = guildChannels;
+            }
+            else
+            {
+                channels.Add(channel);
+            }
+        }
+
+        // Threads
+        if (ThreadInclusionMode != ThreadInclusionMode.None)
+        {
+            await console.Output.WriteLineAsync("Fetching threads...");
+
+            var fetchedThreadsCount = 0;
+            await console
+                .CreateStatusTicker()
+                .StartAsync(
+                    "...",
+                    async ctx =>
+                    {
+                        await foreach (
+                            var thread in Discord.GetChannelThreadsAsync(
+                                channels.ToArray(),
+                                ThreadInclusionMode == ThreadInclusionMode.All,
+                                After,
+                                cancellationToken
+                            )
+                        )
+                        {
+                            channels.Add(thread);
+
+                            ctx.Status(Markup.Escape($"Fetched '{thread.GetHierarchicalName()}'."));
+
+                            fetchedThreadsCount++;
+                        }
+                    }
+                );
+
+            // Remove unneeded forums, as they cannot be crawled directly.
+            channels.RemoveAll(channel => channel.Kind == ChannelKind.GuildForum);
+
+            await console.Output.WriteLineAsync($"Fetched {fetchedThreadsCount} thread(s).");
+        }
+
+        await ExportAsync(console, channels);
     }
 }

DiscordChatExporter.Cli/Commands/ExportGuildCommand.cs

@@ -21,13 +21,6 @@ public class ExportGuildCommand : ExportCommandBase
     [CommandOption("include-vc", Description = "Include voice channels.")]
     public bool IncludeVoiceChannels { get; init; } = true;
 
-    [CommandOption(
-        "include-threads",
-        Description = "Which types of threads should be included.",
-        Converter = typeof(ThreadInclusionModeBindingConverter)
-    )]
-    public ThreadInclusionMode ThreadInclusionMode { get; init; } = ThreadInclusionMode.None;
-
     public override async ValueTask ExecuteAsync(IConsole console)
     {
         await base.ExecuteAsync(console);

DiscordChatExporter.Core/Discord/DiscordClient.cs

@@ -308,6 +308,26 @@ public class DiscordClient(string token)
             .Where(c => before is null || c.MayHaveMessagesBefore(before.Value))
             .ToArray();
 
+        foreach (
+            var channel in await GetChannelThreadsAsync(
+                channels,
+                includeArchived,
+                after,
+                cancellationToken
+            )
+        )
+        {
+            yield return channel;
+        }
+    }
+
+    public async IAsyncEnumerable<Channel> GetChannelThreadsAsync(
+        Channel[] channels,
+        bool includeArchived = false,
+        Snowflake? after = null,
+        [EnumeratorCancellation] CancellationToken cancellationToken = default
+    )
+    {
         // User accounts can only fetch threads using the search endpoint
         if (await ResolveTokenKindAsync(cancellationToken) == TokenKind.User)
         {
@@ -367,7 +387,12 @@ public class DiscordClient(string token)
         // Bot accounts can only fetch threads using the threads endpoint
         else
         {
+            var guilds = new HashSet<Snowflake>();
+            foreach (var channel in channels)
+                guilds.Add(channel.GuildId);
+
             // Active threads
+            foreach (var guildId in guilds)
             {
                 var parentsById = channels.ToDictionary(c => c.Id);
 
@@ -384,7 +409,8 @@ public class DiscordClient(string token)
                         ?.Pipe(Snowflake.Parse)
                         .Pipe(parentsById.GetValueOrDefault);
 
-                    yield return Channel.Parse(threadJson, parent);
+                    if (channels.Contains(parent))
+                        yield return Channel.Parse(threadJson, parent);
                 }
             }