mirrors/DiscordChatExporter
1
0
Fork 0
mirror of https://github.com/Tyrrrz/DiscordChatExporter.git synced 2025-05-11 18:36:45 +02:00
Code Issues Projects Releases Packages Wiki Activity Actions 2

Refactor after last changes

Browse source
This commit is contained in:
Tyrrrz 2025-04-21 00:45:26 +03:00
parent 7ddd55951c
commit 1fadc0755b
1 changed files with 54 additions and 114 deletions
Download patch file Download diff file Expand all files Collapse all files

168
DiscordChatExporter.Core/Discord/DiscordClient.cs
View file

@ -148,14 +148,14 @@ public class DiscordClient(string token)
_ => throw new DiscordChatExporterException( _ => throw new DiscordChatExporterException(
$""" $"""
Request to '{url}' failed: {response Request to '{url}' failed: {response
.StatusCode.ToString() .StatusCode.ToString()
.ToSpaceSeparatedWords() .ToSpaceSeparatedWords()
.ToLowerInvariant()}. .ToLowerInvariant()}.
Response content: {await response.Content.ReadAsStringAsync( Response content: {await response.Content.ReadAsStringAsync(
cancellationToken cancellationToken
)} )}
""", """,
true true
), ),
}; };
@ -311,58 +311,14 @@ public class DiscordClient(string token)
// User accounts can only fetch threads using the search endpoint // User accounts can only fetch threads using the search endpoint
if (await ResolveTokenKindAsync(cancellationToken) == TokenKind.User) if (await ResolveTokenKindAsync(cancellationToken) == TokenKind.User)
{ {
// Active threads
foreach (var channel in channels) foreach (var channel in channels)
{ {
var currentOffset = 0; // Either include both active and archived threads, or only active threads
while (true) foreach (
{ var isArchived in includeArchived ? new[] { false, true } : new[] { false }
var url = new UrlBuilder() )
.SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "false")
.SetQueryParameter("offset", currentOffset.ToString())
.Build();
// Can be null on channels that the user cannot access or channels without threads
var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
break;
var breakOuter = false;
foreach (
var threadJson in response.Value.GetProperty("threads").EnumerateArray()
)
{
var thread = Channel.Parse(threadJson, channel);
// If the 'after' boundary is specified, we can break early,
// because threads are sorted by last message time.
if (after is not null && !thread.MayHaveMessagesAfter(after.Value))
{
breakOuter = true;
break;
}
yield return thread;
currentOffset++;
}
if (breakOuter)
break;
if (!response.Value.GetProperty("has_more").GetBoolean())
break;
}
}
// Archived threads
if (includeArchived)
{
foreach (var channel in channels)
{ {
// Offset is just the index of the last thread in the previous batch
var currentOffset = 0; var currentOffset = 0;
while (true) while (true)
{ {
@ -370,7 +326,7 @@ public class DiscordClient(string token)
.SetPath($"channels/{channel.Id}/threads/search") .SetPath($"channels/{channel.Id}/threads/search")
.SetQueryParameter("sort_by", "last_message_time") .SetQueryParameter("sort_by", "last_message_time")
.SetQueryParameter("sort_order", "desc") .SetQueryParameter("sort_order", "desc")
.SetQueryParameter("archived", "true") .SetQueryParameter("archived", isArchived.ToString().ToLowerInvariant())
.SetQueryParameter("offset", currentOffset.ToString()) .SetQueryParameter("offset", currentOffset.ToString())
.Build(); .Build();
@ -388,7 +344,7 @@ public class DiscordClient(string token)
var thread = Channel.Parse(threadJson, channel); var thread = Channel.Parse(threadJson, channel);
// If the 'after' boundary is specified, we can break early, // If the 'after' boundary is specified, we can break early,
// because threads are sorted by last message time. // because threads are sorted by last message timestamp.
if (after is not null && !thread.MayHaveMessagesAfter(after.Value)) if (after is not null && !thread.MayHaveMessagesAfter(after.Value))
{ {
breakOuter = true; breakOuter = true;
@ -437,64 +393,50 @@ public class DiscordClient(string token)
{ {
foreach (var channel in channels) foreach (var channel in channels)
{ {
// Public archived threads foreach (var archiveType in new[] { "public", "private" })
await foreach ( {
var th in GetAllArchivedThreadsAsync(channel, "public", cancellationToken) // This endpoint parameter expects an ISO8601 timestamp, not a snowflake
) var currentBefore = before
yield return th; ?.ToDate()
.ToString("O", CultureInfo.InvariantCulture);
// Private archived threads while (true)
await foreach ( {
var th in GetAllArchivedThreadsAsync(channel, "private", cancellationToken) // Threads are sorted by archive timestamp, not by last message timestamp
) var url = new UrlBuilder()
yield return th; .SetPath($"channels/{channel.Id}/threads/archived/{archiveType}")
.SetQueryParameter("before", currentBefore)
.Build();
// Can be null on certain channels
var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
break;
foreach (
var threadJson in response
.Value.GetProperty("threads")
.EnumerateArray()
)
{
var thread = Channel.Parse(threadJson, channel);
yield return thread;
currentBefore = threadJson
.GetProperty("thread_metadata")
.GetProperty("archive_timestamp")
.GetString();
}
if (!response.Value.GetProperty("has_more").GetBoolean())
break;
}
}
} }
} }
} }
} }
private async IAsyncEnumerable<Channel> GetAllArchivedThreadsAsync(
Channel channel,
string archiveType,
[EnumeratorCancellation] CancellationToken cancellationToken
)
{
// Base endpoint: "public" or "private"
var endpointBase = $"channels/{channel.Id}/threads/archived/{archiveType}";
// Cursor parameter: ISO8601 timestamp string
string? beforeTimestamp = null;
bool hasMorePages = true;
while (hasMorePages && !cancellationToken.IsCancellationRequested)
{
// Build URL with optional before= parameter
var url = beforeTimestamp is null
? endpointBase
: $"{endpointBase}?before={Uri.EscapeDataString(beforeTimestamp)}";
var response = await TryGetJsonResponseAsync(url, cancellationToken);
if (response is null)
yield break;
// Parse out the threads array
var threadsJson = response.Value.GetProperty("threads").EnumerateArray().ToList();
foreach (var threadJson in threadsJson)
{
yield return Channel.Parse(threadJson, channel);
}
// Check pagination flag
hasMorePages = response.Value.GetProperty("has_more").GetBoolean();
if (hasMorePages && threadsJson.Count > 0)
{
// Prepare next cursor: the archived timestamp of the last thread
var lastThreadMeta = threadsJson.Last().GetProperty("thread_metadata");
beforeTimestamp = lastThreadMeta.GetProperty("archive_timestamp").GetString();
}
}
}
public async IAsyncEnumerable<Role> GetGuildRolesAsync( public async IAsyncEnumerable<Role> GetGuildRolesAsync(
Snowflake guildId, Snowflake guildId,
[EnumeratorCancellation] CancellationToken cancellationToken = default [EnumeratorCancellation] CancellationToken cancellationToken = default
@ -708,9 +650,7 @@ public class DiscordClient(string token)
count++; count++;
} }
// Each batch can contain up to 100 users. if (count <= 0)
// If we got fewer, then it's definitely the last batch.
if (count < 100)
yield break; yield break;
} }
} }