diff options
47 files changed, 1297 insertions, 527 deletions
diff --git a/ExampleBots/LibMatrix.ExampleBot/Bot/Commands/CmdCommand.cs b/ExampleBots/LibMatrix.ExampleBot/Bot/Commands/CmdCommand.cs index f3b4dde..ea42597 100644 --- a/ExampleBots/LibMatrix.ExampleBot/Bot/Commands/CmdCommand.cs +++ b/ExampleBots/LibMatrix.ExampleBot/Bot/Commands/CmdCommand.cs @@ -1,4 +1,5 @@ using ArcaneLibs.Extensions; +using ArcaneLibs.StringNormalisation; using LibMatrix.EventTypes.Spec; using LibMatrix.ExampleBot.Bot.Interfaces; diff --git a/ExampleBots/LibMatrix.ExampleBot/LibMatrix.ExampleBot.csproj b/ExampleBots/LibMatrix.ExampleBot/LibMatrix.ExampleBot.csproj index 3e33be8..94c9045 100644 --- a/ExampleBots/LibMatrix.ExampleBot/LibMatrix.ExampleBot.csproj +++ b/ExampleBots/LibMatrix.ExampleBot/LibMatrix.ExampleBot.csproj @@ -22,7 +22,8 @@ </ItemGroup> <ItemGroup> - <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" /> + <PackageReference Include="ArcaneLibs.StringNormalisation" Version="1.0.0-preview6876311209.3b815f2" /> + <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" /> </ItemGroup> <ItemGroup> <Content Include="appsettings*.json"> diff --git a/ExampleBots/MediaModeratorPoC/MediaModBot.cs b/ExampleBots/MediaModeratorPoC/MediaModBot.cs deleted file mode 100644 index 0aacf61..0000000 --- a/ExampleBots/MediaModeratorPoC/MediaModBot.cs +++ /dev/null @@ -1,339 +0,0 @@ -using System.Security.Cryptography; -using System.Text.RegularExpressions; -using ArcaneLibs.Extensions; -using LibMatrix; -using LibMatrix.EventTypes.Spec; -using LibMatrix.EventTypes.Spec.State; -using LibMatrix.Helpers; -using LibMatrix.Homeservers; -using LibMatrix.Responses; -using LibMatrix.RoomTypes; -using LibMatrix.Services; -using LibMatrix.Utilities.Bot.Interfaces; -using MediaModeratorPoC.AccountData; -using MediaModeratorPoC.StateEventTypes; -using Microsoft.Extensions.Hosting; -using Microsoft.Extensions.Logging; - -namespace MediaModeratorPoC; - -public class MediaModBot(AuthenticatedHomeserverGeneric hs, ILogger<MediaModBot> logger, MediaModBotConfiguration configuration, - HomeserverResolverService hsResolver) : IHostedService { - private readonly IEnumerable<ICommand> _commands; - - private Task _listenerTask; - - private GenericRoom _policyRoom; - private GenericRoom _logRoom; - private GenericRoom _controlRoom; - - - - /// <summary>Triggered when the application host is ready to start the service.</summary> - /// <param name="cancellationToken">Indicates that the start process has been aborted.</param> - public async Task StartAsync(CancellationToken cancellationToken) { - _listenerTask = Run(cancellationToken); - logger.LogInformation("Bot started!"); - } - - private async Task Run(CancellationToken cancellationToken) { - Directory.GetFiles("bot_data/cache").ToList().ForEach(File.Delete); - - BotData botData; - - try { - botData = await hs.GetAccountDataAsync<BotData>("gay.rory.modbot_data"); - } - catch (Exception e) { - if (e is not MatrixException { ErrorCode: "M_NOT_FOUND" }) { - logger.LogError("{}", e.ToString()); - throw; - } - - botData = new BotData(); - var creationContent = CreateRoomRequest.CreatePrivate(hs, name: "Media Moderator PoC - Control room", roomAliasName: "media-moderator-poc-control-room"); - creationContent.Invite = configuration.Admins; - creationContent.CreationContent["type"] = "gay.rory.media_moderator_poc.control_room"; - - botData.ControlRoom = (await hs.CreateRoom(creationContent)).RoomId; - - //set access rules to allow joining via control room - creationContent.InitialState.Add(new StateEvent { - Type = "m.room.join_rules", - StateKey = "", - TypedContent = new RoomJoinRulesEventContent { - JoinRule = "knock_restricted", - Allow = new() { - new RoomJoinRulesEventContent.AllowEntry { - Type = "m.room_membership", - RoomId = botData.ControlRoom - } - } - } - }); - - creationContent.Name = "Media Moderator PoC - Log room"; - creationContent.RoomAliasName = "media-moderator-poc-log-room"; - creationContent.CreationContent["type"] = "gay.rory.media_moderator_poc.log_room"; - botData.LogRoom = (await hs.CreateRoom(creationContent)).RoomId; - - creationContent.Name = "Media Moderator PoC - Policy room"; - creationContent.RoomAliasName = "media-moderator-poc-policy-room"; - creationContent.CreationContent["type"] = "gay.rory.media_moderator_poc.policy_room"; - botData.PolicyRoom = (await hs.CreateRoom(creationContent)).RoomId; - - await hs.SetAccountData("gay.rory.modbot_data", botData); - } - - _policyRoom = hs.GetRoom(botData.PolicyRoom ?? botData.ControlRoom); - _logRoom = hs.GetRoom(botData.LogRoom ?? botData.ControlRoom); - _controlRoom = hs.GetRoom(botData.ControlRoom); - var syncHelper = new SyncHelper(hs); - - List<string> admins = new(); - -#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - Task.Run(async () => { - while (!cancellationToken.IsCancellationRequested) { - var controlRoomMembers = _controlRoom.GetMembersAsync(); - await foreach (var member in controlRoomMembers) { - if ((member.TypedContent as RoomMemberEventContent)? - - .Membership == "join") admins.Add(member.UserId); - } - - await Task.Delay(TimeSpan.FromSeconds(30), cancellationToken); - } - }, cancellationToken); -#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed - - syncHelper.InviteReceivedHandlers.Add(async Task (args) => { - var inviteEvent = - args.Value.InviteState.Events.FirstOrDefault(x => - x.Type == "m.room.member" && x.StateKey == hs.UserId); - logger.LogInformation("Got invite to {RoomId} by {Sender} with reason: {Reason}", args.Key, inviteEvent!.Sender, (inviteEvent.TypedContent as RoomMemberEventContent)!.Reason); - if (inviteEvent.Sender.EndsWith(":rory.gay") || inviteEvent!.Sender.EndsWith(":conduit.rory.gay")) { - try { - var senderProfile = await hs.GetProfileAsync(inviteEvent.Sender); - await hs.GetRoom(args.Key).JoinAsync(reason: $"I was invited by {senderProfile.DisplayName ?? inviteEvent.Sender}!"); - } - catch (Exception e) { - logger.LogError("{}", e.ToString()); - await hs.GetRoom(args.Key).LeaveAsync(reason: "I was unable to join the room: " + e); - } - } - }); - - syncHelper.TimelineEventHandlers.Add(async @event => { - var room = hs.GetRoom(@event.RoomId); - try { - logger.LogInformation( - "Got timeline event in {}: {}", @event.RoomId, @event.ToJson(indent: true, ignoreNull: true)); - - if (@event is { Type: "m.room.message", TypedContent: RoomMessageEventContent message }) { - if (message is { MessageType: "m.image" }) { - //check media - var matchedPolicy = await CheckMedia(@event); - if (matchedPolicy is null) return; - var matchedpolicyData = matchedPolicy.TypedContent as MediaPolicyEventContent; - var recommendation = matchedpolicyData.Recommendation; - await _logRoom.SendMessageEventAsync( - new RoomMessageEventContent( - body: - $"User {MessageFormatter.HtmlFormatMention(@event.Sender)} posted an image in {MessageFormatter.HtmlFormatMention(room.RoomId)} that matched rule {matchedPolicy.StateKey}, applying action {matchedpolicyData.Recommendation}, as described in rule: {matchedPolicy.RawContent!.ToJson(ignoreNull: true)}", - messageType: "m.text") { - Format = "org.matrix.custom.html", - FormattedBody = - $"<font color=\"#FFFF00\">User {MessageFormatter.HtmlFormatMention(@event.Sender)} posted an image in {MessageFormatter.HtmlFormatMention(room.RoomId)} that matched rule {matchedPolicy.StateKey}, applying action {matchedpolicyData.Recommendation}, as described in rule: <pre>{matchedPolicy.RawContent!.ToJson(ignoreNull: true)}</pre></font>" - }); - switch (recommendation) { - case "warn_admins": { - await _controlRoom.SendMessageEventAsync( - new RoomMessageEventContent( - body: $"{string.Join(' ', admins)}\nUser {MessageFormatter.HtmlFormatMention(@event.Sender)} posted a banned image {message.Url}", - messageType: "m.text") { - Format = "org.matrix.custom.html", - FormattedBody = $"{string.Join(' ', admins.Select(u => MessageFormatter.HtmlFormatMention(u)))}\n" + - $"<font color=\"#FF0000\">User {MessageFormatter.HtmlFormatMention(@event.Sender)} posted a banned image <a href=\"{message.Url}\">{message.Url}</a></font>" - }); - break; - } - case "warn": { - await room.SendMessageEventAsync( - new RoomMessageEventContent( - body: $"Please be careful when posting this image: {matchedpolicyData.Reason ?? "No reason specified"}", - messageType: "m.text") { - Format = "org.matrix.custom.html", - FormattedBody = - $"<font color=\"#FFFF00\">Please be careful when posting this image: {matchedpolicyData.Reason ?? "No reason specified"}</a></font>" - }); - break; - } - case "redact": { - await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason ?? "No reason specified"); - break; - } - case "spoiler": { - // <blockquote> - // <a href=\"https://matrix.to/#/@emma:rory.gay\">@emma:rory.gay</a><br> - // <a href=\"https://codeberg.org/crimsonfork/CN\"></a> - // <font color=\"#dc143c\" data-mx-color=\"#dc143c\"> - // <b>CN</b> - // </font>: - // <a href=\"https://the-apothecary.club/_matrix/media/v3/download/rory.gay/sLkdxUhipiQaFwRkXcPSRwdg\">test</a><br> - // <span data-mx-spoiler=\"\"><a href=\"https://the-apothecary.club/_matrix/media/v3/download/rory.gay/sLkdxUhipiQaFwRkXcPSRwdg\"> - // <img src=\"mxc://rory.gay/sLkdxUhipiQaFwRkXcPSRwdg\" height=\"69\"></a> - // </span> - // </blockquote> - await room.SendMessageEventAsync( - new RoomMessageEventContent( - body: - $"Please be careful when posting this image: {matchedpolicyData.Reason}, I have spoilered it for you:", - messageType: "m.text") { - Format = "org.matrix.custom.html", - FormattedBody = - $"<font color=\"#FFFF00\">Please be careful when posting this image: {matchedpolicyData.Reason}, I have spoilered it for you:</a></font>" - }); - var imageUrl = message.Url; - await room.SendMessageEventAsync( - new RoomMessageEventContent(body: $"CN: {imageUrl}", - messageType: "m.text") { - Format = "org.matrix.custom.html", - FormattedBody = $""" - <blockquote> - <font color=\"#dc143c\" data-mx-color=\"#dc143c\"> - <b>CN</b> - </font>: - <a href=\"{imageUrl}\">{matchedpolicyData.Reason}</a><br> - <span data-mx-spoiler=\"\"> - <a href=\"{imageUrl}\"> - <img src=\"{imageUrl}\" height=\"69\"> - </a> - </span> - </blockquote> - """ - }); - await room.RedactEventAsync(@event.EventId, "Automatically spoilered: " + matchedpolicyData.Reason); - break; - } - case "mute": { - await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason); - //change powerlevel to -1 - var currentPls = await room.GetPowerLevelsAsync(); - if(currentPls is null) { - logger.LogWarning("Unable to get power levels for {room}", room.RoomId); - await _logRoom.SendMessageEventAsync( - MessageFormatter.FormatError($"Unable to get power levels for {MessageFormatter.HtmlFormatMention(room.RoomId)}")); - return; - } - currentPls.Users ??= new(); - currentPls.Users[@event.Sender] = -1; - await room.SendStateEventAsync("m.room.power_levels", currentPls); - break; - } - case "kick": { - await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason); - await room.KickAsync(@event.Sender, matchedpolicyData.Reason); - break; - } - case "ban": { - await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason); - await room.BanAsync(@event.Sender, matchedpolicyData.Reason); - break; - } - default: { - throw new ArgumentOutOfRangeException("recommendation", - $"Unknown response type {matchedpolicyData.Recommendation}!"); - } - } - } - } - } - catch (Exception e) { - logger.LogError("{}", e.ToString()); - await _controlRoom.SendMessageEventAsync( - MessageFormatter.FormatException($"Unable to ban user in {MessageFormatter.HtmlFormatMention(room.RoomId)}", e)); - await _logRoom.SendMessageEventAsync( - MessageFormatter.FormatException($"Unable to ban user in {MessageFormatter.HtmlFormatMention(room.RoomId)}", e)); - await using var stream = new MemoryStream(e.ToString().AsBytes().ToArray()); - await _controlRoom.SendFileAsync("error.log.cs", stream); - await _logRoom.SendFileAsync("error.log.cs", stream); - } - }); - } - - /// <summary>Triggered when the application host is performing a graceful shutdown.</summary> - /// <param name="cancellationToken">Indicates that the shutdown process should no longer be graceful.</param> - public async Task StopAsync(CancellationToken cancellationToken) { - logger.LogInformation("Shutting down bot!"); - } - - private async Task<StateEventResponse?> CheckMedia(StateEventResponse @event) { - var stateList = _policyRoom.GetFullStateAsync(); - var hashAlgo = SHA3_256.Create(); - - var mxcUri = @event.RawContent["url"].GetValue<string>(); - var resolvedUri = await hsResolver.ResolveMediaUri(mxcUri.Split('/')[2], mxcUri); - var uriHash = hashAlgo.ComputeHash(mxcUri.AsBytes().ToArray()); - byte[]? fileHash = null; - - try { - fileHash = await hashAlgo.ComputeHashAsync(await hs._httpClient.GetStreamAsync(resolvedUri)); - } - catch (Exception ex) { - await _logRoom.SendMessageEventAsync( - MessageFormatter.FormatException($"Error calculating file hash for {mxcUri} via {mxcUri.Split('/')[2]} ({resolvedUri}), retrying via {hs.BaseUrl}...", - ex)); - try { - resolvedUri = await hsResolver.ResolveMediaUri(hs.BaseUrl, mxcUri); - fileHash = await hashAlgo.ComputeHashAsync(await hs._httpClient.GetStreamAsync(resolvedUri)); - } - catch (Exception ex2) { - await _logRoom.SendMessageEventAsync( - MessageFormatter.FormatException($"Error calculating file hash via {hs.BaseUrl} ({resolvedUri})!", ex2)); - } - } - - logger.LogInformation("Checking media {url} with hash {hash}", resolvedUri, fileHash); - - await foreach (var state in stateList) { - if (state.Type != "gay.rory.media_moderator_poc.rule.media" && state.Type != "gay.rory.media_moderator_poc.rule.server") continue; - if (!state.RawContent.ContainsKey("entity")) { - logger.LogWarning("Rule {rule} has no entity, this event was probably redacted!", state.StateKey); - continue; - } - - logger.LogInformation("Checking rule {rule}: {data}", state.StateKey, state.TypedContent.ToJson(ignoreNull: true, indent: false)); - var rule = state.TypedContent as MediaPolicyEventContent; - if (state.Type == "gay.rory.media_moderator_poc.rule.server" && rule.ServerEntity is not null) { - rule.ServerEntity = rule.ServerEntity.Replace("\\*", ".*").Replace("\\?", "."); - var regex = new Regex($"mxc://({rule.ServerEntity})/.*", RegexOptions.Compiled | RegexOptions.IgnoreCase); - if (regex.IsMatch(@event.RawContent["url"]!.GetValue<string>())) { - logger.LogInformation("{url} matched rule {rule}", @event.RawContent["url"], rule.ToJson(ignoreNull: true)); - return state; - } - } - - if (rule.Entity is not null && uriHash.SequenceEqual(rule.Entity)) { - logger.LogInformation("{url} matched rule {rule} by uri hash", @event.RawContent["url"], rule.ToJson(ignoreNull: true)); - return state; - } - - logger.LogInformation("uri hash {uriHash} did not match rule's {ruleUriHash}", Convert.ToBase64String(uriHash), Convert.ToBase64String(rule.Entity)); - - if (rule.FileHash is not null && fileHash is not null && rule.FileHash.SequenceEqual(fileHash)) { - logger.LogInformation("{url} matched rule {rule} by file hash", @event.RawContent["url"], rule.ToJson(ignoreNull: true)); - return state; - } - - logger.LogInformation("file hash {fileHash} did not match rule's {ruleFileHash}", Convert.ToBase64String(fileHash), Convert.ToBase64String(rule.FileHash)); - - //check pixels every 10% of the way through the image using ImageSharp - // var image = Image.Load(await _hs._httpClient.GetStreamAsync(resolvedUri)); - } - - logger.LogInformation("{url} did not match any rules", @event.RawContent["url"]); - - return null; - } -} diff --git a/ExampleBots/MediaModeratorPoC/MediaModBotConfiguration.cs b/ExampleBots/MediaModeratorPoC/MediaModBotConfiguration.cs deleted file mode 100644 index cb5b596..0000000 --- a/ExampleBots/MediaModeratorPoC/MediaModBotConfiguration.cs +++ /dev/null @@ -1,10 +0,0 @@ -using Microsoft.Extensions.Configuration; - -namespace MediaModeratorPoC; - -public class MediaModBotConfiguration { - public MediaModBotConfiguration(IConfiguration config) => config.GetRequiredSection("MediaMod").Bind(this); - - public List<string> Admins { get; set; } = new(); - public bool DemoMode { get; set; } = false; -} diff --git a/ExampleBots/MediaModeratorPoC/PolicyEngine.cs b/ExampleBots/MediaModeratorPoC/PolicyEngine.cs deleted file mode 100644 index 0a0a565..0000000 --- a/ExampleBots/MediaModeratorPoC/PolicyEngine.cs +++ /dev/null @@ -1,86 +0,0 @@ -using LibMatrix.EventTypes.Spec; -using LibMatrix.Helpers; -using LibMatrix.Homeservers; -using LibMatrix.RoomTypes; -using LibMatrix.Services; -using MediaModeratorPoC.AccountData; -using MediaModeratorPoC.StateEventTypes; -using Microsoft.Extensions.Logging; - -namespace MediaModeratorPoC; - -public class PolicyEngine(AuthenticatedHomeserverGeneric hs, ILogger<MediaModBot> logger, MediaModBotConfiguration configuration, - HomeserverResolverService hsResolver) { - public List<PolicyList> ActivePolicyLists { get; set; } = new(); - private GenericRoom? _logRoom; - private GenericRoom? _controlRoom; - - public async Task ReloadActivePolicyLists() { - // first time init - if (_logRoom is null || _controlRoom is null) { - var botData = await hs.GetAccountDataAsync<BotData>("gay.rory.modbot_data"); - _logRoom ??= hs.GetRoom(botData.LogRoom ?? botData.ControlRoom); - _controlRoom ??= hs.GetRoom(botData.ControlRoom); - } - - await _controlRoom?.SendMessageEventAsync(MessageFormatter.FormatSuccess("Reloading policy lists!"))!; - await _logRoom?.SendMessageEventAsync( - new RoomMessageEventContent( - body: "Reloading policy lists!", - messageType: "m.text"))!; - - await _controlRoom?.SendMessageEventAsync(MessageFormatter.FormatSuccess("0/? policy lists loaded"))!; - - var policyLists = new List<PolicyList>(); - var policyListAccountData = await hs.GetAccountDataAsync<Dictionary<string, PolicyList>>("gay.rory.modbot.policy_lists"); - foreach (var (roomId, policyList) in policyListAccountData) { - _logRoom?.SendMessageEventAsync( - new RoomMessageEventContent( - body: $"Loading policy list {MessageFormatter.HtmlFormatMention(roomId)}!", - messageType: "m.text")); - var room = hs.GetRoom(roomId); - - policyList.Room = room; - - var stateEvents = room.GetFullStateAsync(); - await foreach (var stateEvent in stateEvents) { - if (stateEvent != null && stateEvent.GetType.IsAssignableTo(typeof(BasePolicy))) { - policyList.Policies.Add(stateEvent); - } - } - - //html table of policy count by type - var policyCount = policyList.Policies.GroupBy(x => x.Type).ToDictionary(x => x.Key, x => x.Count()); - var policyCountTable = policyCount.Aggregate( - "<table><tr><th>Policy Type</th><th>Count</th></tr>", - (current, policy) => current + $"<tr><td>{policy.Key}</td><td>{policy.Value}</td></tr>"); - policyCountTable += "</table>"; - - var policyCountTablePlainText = policyCount.Aggregate( - "Policy Type | Count\n", - (current, policy) => current + $"{policy.Key,-16} | {policy.Value}\n"); - await _logRoom?.SendMessageEventAsync( - new RoomMessageEventContent() { - MessageType = "org.matrix.custom.html", - Body = $"Policy count for {roomId}:\n{policyCountTablePlainText}", - FormattedBody = $"Policy count for {MessageFormatter.HtmlFormatMention(roomId)}:\n{policyCountTable}", - })!; - - await _logRoom?.SendMessageEventAsync( - new RoomMessageEventContent( - body: $"Loaded {policyList.Policies.Count} policies for {MessageFormatter.HtmlFormatMention(roomId)}!", - messageType: "m.text"))!; - - policyLists.Add(policyList); - - var progressMsgContent = MessageFormatter.FormatSuccess($"{policyLists.Count}/{policyListAccountData.Count} policy lists loaded"); - //edit old message - progressMsgContent.RelatesTo = new() { - - }; - _controlRoom?.SendMessageEventAsync(progressMsgContent); - } - - ActivePolicyLists = policyLists; - } -} diff --git a/ExampleBots/MediaModeratorPoC/AccountData/BotData.cs b/ExampleBots/ModerationBot/AccountData/BotData.cs index 0fee4eb..df86589 100644 --- a/ExampleBots/MediaModeratorPoC/AccountData/BotData.cs +++ b/ExampleBots/ModerationBot/AccountData/BotData.cs @@ -1,6 +1,6 @@ using System.Text.Json.Serialization; -namespace MediaModeratorPoC.AccountData; +namespace ModerationBot.AccountData; public class BotData { [JsonPropertyName("control_room")] @@ -8,4 +8,7 @@ public class BotData { [JsonPropertyName("log_room")] public string? LogRoom { get; set; } = ""; -} + + [JsonPropertyName("default_policy_room")] + public string? DefaultPolicyRoom { get; set; } +} \ No newline at end of file diff --git a/ExampleBots/MediaModeratorPoC/Commands/BanMediaCommand.cs b/ExampleBots/ModerationBot/Commands/BanMediaCommand.cs index 5dfa706..21e0a94 100644 --- a/ExampleBots/MediaModeratorPoC/Commands/BanMediaCommand.cs +++ b/ExampleBots/ModerationBot/Commands/BanMediaCommand.cs @@ -6,18 +6,19 @@ using LibMatrix.EventTypes.Spec; using LibMatrix.Helpers; using LibMatrix.Services; using LibMatrix.Utilities.Bot.Interfaces; -using MediaModeratorPoC.AccountData; -using MediaModeratorPoC.StateEventTypes; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; +using ModerationBot.StateEventTypes.Policies.Implementations; -namespace MediaModeratorPoC.Commands; +namespace ModerationBot.Commands; -public class BanMediaCommand(IServiceProvider services, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver) : ICommand { +public class BanMediaCommand(IServiceProvider services, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver, PolicyEngine engine) : ICommand { public string Name { get; } = "banmedia"; public string Description { get; } = "Create a policy banning a piece of media, must be used in reply to a message"; public async Task<bool> CanInvoke(CommandContext ctx) { //check if user is admin in control room - var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.modbot_data"); + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); var controlRoom = ctx.Homeserver.GetRoom(botData.ControlRoom); var isAdmin = (await controlRoom.GetPowerLevelsAsync())!.UserHasPermission(ctx.MessageEvent.Sender, "m.room.ban"); if (!isAdmin) { @@ -30,8 +31,9 @@ public class BanMediaCommand(IServiceProvider services, HomeserverProviderServic } public async Task Invoke(CommandContext ctx) { - var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.modbot_data"); - var policyRoom = ctx.Homeserver.GetRoom(botData.PolicyRoom ?? botData.ControlRoom); + + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + var policyRoom = ctx.Homeserver.GetRoom(botData.DefaultPolicyRoom ?? botData.ControlRoom); var logRoom = ctx.Homeserver.GetRoom(botData.LogRoom ?? botData.ControlRoom); //check if reply @@ -69,7 +71,7 @@ public class BanMediaCommand(IServiceProvider services, HomeserverProviderServic byte[]? fileHash = null; try { - fileHash = await hashAlgo.ComputeHashAsync(await ctx.Homeserver._httpClient.GetStreamAsync(resolvedUri)); + fileHash = await hashAlgo.ComputeHashAsync(await ctx.Homeserver.ClientHttpClient.GetStreamAsync(resolvedUri)); } catch (Exception ex) { await logRoom.SendMessageEventAsync( @@ -77,7 +79,7 @@ public class BanMediaCommand(IServiceProvider services, HomeserverProviderServic ex)); try { resolvedUri = await hsResolver.ResolveMediaUri(ctx.Homeserver.BaseUrl, mxcUri); - fileHash = await hashAlgo.ComputeHashAsync(await ctx.Homeserver._httpClient.GetStreamAsync(resolvedUri)); + fileHash = await hashAlgo.ComputeHashAsync(await ctx.Homeserver.ClientHttpClient.GetStreamAsync(resolvedUri)); } catch (Exception ex2) { await ctx.Room.SendMessageEventAsync(MessageFormatter.FormatException("Error calculating file hash", ex2)); @@ -86,10 +88,10 @@ public class BanMediaCommand(IServiceProvider services, HomeserverProviderServic } } - MediaPolicyEventContent policy; - await policyRoom.SendStateEventAsync("gay.rory.media_moderator_poc.rule.media", Guid.NewGuid().ToString(), policy = new MediaPolicyEventContent { - // Entity = uriHash, - FileHash = fileHash, + MediaPolicyFile policy; + await policyRoom.SendStateEventAsync("gay.rory.moderation.rule.media", Guid.NewGuid().ToString(), policy = new MediaPolicyFile { + Entity = Convert.ToBase64String(uriHash), + FileHash = Convert.ToBase64String(fileHash), Reason = string.Join(' ', ctx.Args[1..]), Recommendation = recommendation, }); diff --git a/ExampleBots/ModerationBot/Commands/DbgAllRoomsArePolicyListsCommand.cs b/ExampleBots/ModerationBot/Commands/DbgAllRoomsArePolicyListsCommand.cs new file mode 100644 index 0000000..09d3caf --- /dev/null +++ b/ExampleBots/ModerationBot/Commands/DbgAllRoomsArePolicyListsCommand.cs @@ -0,0 +1,63 @@ +using System.Buffers.Text; +using System.Security.Cryptography; +using ArcaneLibs.Extensions; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using LibMatrix.Helpers; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using LibMatrix.Utilities.Bot.Interfaces; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; + +namespace ModerationBot.Commands; + +public class DbgAllRoomsArePolicyListsCommand + (IServiceProvider services, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver, PolicyEngine engine) : ICommand { + public string Name { get; } = "dbg-allroomsarepolicy"; + public string Description { get; } = "[Debug] mark all rooms as trusted policy rooms"; + private GenericRoom logRoom { get; set; } + + public async Task<bool> CanInvoke(CommandContext ctx) { +#if !DEBUG + return false; +#endif + + //check if user is admin in control room + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + var controlRoom = ctx.Homeserver.GetRoom(botData.ControlRoom); + var isAdmin = (await controlRoom.GetPowerLevelsAsync())!.UserHasPermission(ctx.MessageEvent.Sender, "m.room.ban"); + if (!isAdmin) { + // await ctx.Reply("You do not have permission to use this command!"); + await ctx.Homeserver.GetRoom(botData.LogRoom!).SendMessageEventAsync( + new RoomMessageEventContent(body: $"User {ctx.MessageEvent.Sender} tried to use command {Name} but does not have permission!", messageType: "m.text")); + } + + return isAdmin; + } + + public async Task Invoke(CommandContext ctx) { + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + logRoom = ctx.Homeserver.GetRoom(botData.LogRoom ?? botData.ControlRoom); + + var joinedRooms = await ctx.Homeserver.GetJoinedRooms(); + + await ctx.Homeserver.SetAccountDataAsync("gay.rory.moderation_bot.policy_lists", joinedRooms.ToDictionary(x => x.RoomId, x => new PolicyList() { + Trusted = true + })); + + await engine.ReloadActivePolicyLists(); + } + + private async Task<bool> JoinRoom(GenericRoom memberRoom, string reason, List<string> servers) { + try { + await memberRoom.JoinAsync(servers.ToArray(), reason); + await logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Joined room {memberRoom.RoomId}")); + } + catch (Exception e) { + await logRoom.SendMessageEventAsync(MessageFormatter.FormatException($"Failed to join {memberRoom.RoomId}", e)); + } + + return true; + } +} \ No newline at end of file diff --git a/ExampleBots/ModerationBot/Commands/DbgDumpActivePoliciesCommand.cs b/ExampleBots/ModerationBot/Commands/DbgDumpActivePoliciesCommand.cs new file mode 100644 index 0000000..395c87c --- /dev/null +++ b/ExampleBots/ModerationBot/Commands/DbgDumpActivePoliciesCommand.cs @@ -0,0 +1,43 @@ +using System.Buffers.Text; +using System.Security.Cryptography; +using ArcaneLibs.Extensions; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using LibMatrix.Helpers; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using LibMatrix.Utilities.Bot.Interfaces; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; + +namespace ModerationBot.Commands; + +public class DbgDumpActivePoliciesCommand + (IServiceProvider services, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver, PolicyEngine engine) : ICommand { + public string Name { get; } = "dbg-dumppolicies"; + public string Description { get; } = "[Debug] Dump all active policies"; + private GenericRoom logRoom { get; set; } + + public async Task<bool> CanInvoke(CommandContext ctx) { +#if !DEBUG + return false; +#endif + + //check if user is admin in control room + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + var controlRoom = ctx.Homeserver.GetRoom(botData.ControlRoom); + var isAdmin = (await controlRoom.GetPowerLevelsAsync())!.UserHasPermission(ctx.MessageEvent.Sender, "m.room.ban"); + if (!isAdmin) { + // await ctx.Reply("You do not have permission to use this command!"); + await ctx.Homeserver.GetRoom(botData.LogRoom!).SendMessageEventAsync( + new RoomMessageEventContent(body: $"User {ctx.MessageEvent.Sender} tried to use command {Name} but does not have permission!", messageType: "m.text")); + } + + return isAdmin; + } + + public async Task Invoke(CommandContext ctx) { + await ctx.Room.SendFileAsync("all.json", new MemoryStream(engine.ActivePolicies.ToJson().AsBytes().ToArray()), contentType: "application/json"); + await ctx.Room.SendFileAsync("by-type.json", new MemoryStream(engine.ActivePoliciesByType.ToJson().AsBytes().ToArray()), contentType: "application/json"); + } +} \ No newline at end of file diff --git a/ExampleBots/ModerationBot/Commands/DbgDumpAllStateTypesCommand.cs b/ExampleBots/ModerationBot/Commands/DbgDumpAllStateTypesCommand.cs new file mode 100644 index 0000000..e9a645e --- /dev/null +++ b/ExampleBots/ModerationBot/Commands/DbgDumpAllStateTypesCommand.cs @@ -0,0 +1,73 @@ +using System.Buffers.Text; +using System.Security.Cryptography; +using ArcaneLibs.Extensions; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using LibMatrix.Helpers; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using LibMatrix.Utilities.Bot.Interfaces; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; + +namespace ModerationBot.Commands; + +public class DbgDumpAllStateTypesCommand + (IServiceProvider services, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver, PolicyEngine engine) : ICommand { + public string Name { get; } = "dbg-dumpstatetypes"; + public string Description { get; } = "[Debug] Dump all state types we can find"; + private GenericRoom logRoom { get; set; } + + public async Task<bool> CanInvoke(CommandContext ctx) { +#if !DEBUG + return false; +#endif + + //check if user is admin in control room + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + var controlRoom = ctx.Homeserver.GetRoom(botData.ControlRoom); + var isAdmin = (await controlRoom.GetPowerLevelsAsync())!.UserHasPermission(ctx.MessageEvent.Sender, "m.room.ban"); + if (!isAdmin) { + // await ctx.Reply("You do not have permission to use this command!"); + await ctx.Homeserver.GetRoom(botData.LogRoom!).SendMessageEventAsync( + new RoomMessageEventContent(body: $"User {ctx.MessageEvent.Sender} tried to use command {Name} but does not have permission!", messageType: "m.text")); + } + + return isAdmin; + } + + public async Task Invoke(CommandContext ctx) { + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + logRoom = ctx.Homeserver.GetRoom(botData.LogRoom ?? botData.ControlRoom); + + + var joinedRooms = await ctx.Homeserver.GetJoinedRooms(); + + var tasks = joinedRooms.Select(GetStateTypes).ToAsyncEnumerable(); + await foreach (var (room, (raw, html)) in tasks) { + await ctx.Room.SendMessageEventAsync(new RoomMessageEventContent("m.text") { + Body = $"States for {room.RoomId}:\n{raw}", + FormattedBody = $"States for {room.RoomId}:\n{html}", + Format = "org.matrix.custom.html" + }); + } + } + + private async Task<(GenericRoom room, (string raw, string html))> GetStateTypes(GenericRoom memberRoom) { + var states = await memberRoom.GetFullStateAsListAsync(); + + return (memberRoom, SummariseStateTypeCounts(states)); + } + + private static (string Raw, string Html) SummariseStateTypeCounts(IList<StateEventResponse> states) { + string raw = "Count | State type | Mapped type", html = "<table><tr><th>Count</th><th>State type</th><th>Mapped type</th></tr>"; + var groupedStates = states.GroupBy(x => x.Type).ToDictionary(x => x.Key, x => x.ToList()).OrderByDescending(x => x.Value.Count); + foreach (var (type, stateGroup) in groupedStates) { + raw += $"{stateGroup.Count} | {type} | {stateGroup[0].GetType.Name}"; + html += $"<tr><td>{stateGroup.Count}</td><td>{type}</td><td>{stateGroup[0].GetType.Name}</td></tr>"; + } + + html += "</table>"; + return (raw, html); + } +} \ No newline at end of file diff --git a/ExampleBots/ModerationBot/Commands/JoinRoomCommand.cs b/ExampleBots/ModerationBot/Commands/JoinRoomCommand.cs new file mode 100644 index 0000000..19a2c54 --- /dev/null +++ b/ExampleBots/ModerationBot/Commands/JoinRoomCommand.cs @@ -0,0 +1,54 @@ +using System.Buffers.Text; +using System.Security.Cryptography; +using ArcaneLibs.Extensions; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using LibMatrix.Helpers; +using LibMatrix.Services; +using LibMatrix.Utilities.Bot.Interfaces; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; + +namespace ModerationBot.Commands; + +public class JoinRoomCommand(IServiceProvider services, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver, PolicyEngine engine) : ICommand { + public string Name { get; } = "join"; + public string Description { get; } = "Join arbitrary rooms"; + + public async Task<bool> CanInvoke(CommandContext ctx) { + //check if user is admin in control room + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + var controlRoom = ctx.Homeserver.GetRoom(botData.ControlRoom); + var isAdmin = (await controlRoom.GetPowerLevelsAsync())!.UserHasPermission(ctx.MessageEvent.Sender, "m.room.ban"); + if (!isAdmin) { + // await ctx.Reply("You do not have permission to use this command!"); + await ctx.Homeserver.GetRoom(botData.LogRoom!).SendMessageEventAsync( + new RoomMessageEventContent(body: $"User {ctx.MessageEvent.Sender} tried to use command {Name} but does not have permission!", messageType: "m.text")); + } + + return isAdmin; + } + + public async Task Invoke(CommandContext ctx) { + + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + var policyRoom = ctx.Homeserver.GetRoom(botData.DefaultPolicyRoom ?? botData.ControlRoom); + var logRoom = ctx.Homeserver.GetRoom(botData.LogRoom ?? botData.ControlRoom); + + await logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Joining room {ctx.Args[0]} with reason: {string.Join(' ', ctx.Args[1..])}")); + var roomId = ctx.Args[0]; + var servers = new List<string>() {ctx.Homeserver.ServerName}; + if (roomId.StartsWith('[')) { + + } + + if (roomId.StartsWith('#')) { + var res = await ctx.Homeserver.ResolveRoomAliasAsync(roomId); + roomId = res.RoomId; + servers.AddRange(servers); + } + + await ctx.Homeserver.JoinRoomAsync(roomId, servers, string.Join(' ', ctx.Args[1..])); + await logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Resolved room {ctx.Args[0]} to {roomId} with servers: {string.Join(", ", servers)}")); + } +} diff --git a/ExampleBots/ModerationBot/Commands/JoinSpaceMembersCommand.cs b/ExampleBots/ModerationBot/Commands/JoinSpaceMembersCommand.cs new file mode 100644 index 0000000..c3b7d12 --- /dev/null +++ b/ExampleBots/ModerationBot/Commands/JoinSpaceMembersCommand.cs @@ -0,0 +1,75 @@ +using System.Buffers.Text; +using System.Security.Cryptography; +using ArcaneLibs.Extensions; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using LibMatrix.Helpers; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using LibMatrix.Utilities.Bot.Interfaces; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; + +namespace ModerationBot.Commands; + +public class JoinSpaceMembersCommand(IServiceProvider services, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver, PolicyEngine engine) : ICommand { + public string Name { get; } = "joinspacemembers"; + public string Description { get; } = "Join all rooms in space"; + private GenericRoom logRoom { get; set; } + + public async Task<bool> CanInvoke(CommandContext ctx) { + //check if user is admin in control room + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + var controlRoom = ctx.Homeserver.GetRoom(botData.ControlRoom); + var isAdmin = (await controlRoom.GetPowerLevelsAsync())!.UserHasPermission(ctx.MessageEvent.Sender, "m.room.ban"); + if (!isAdmin) { + // await ctx.Reply("You do not have permission to use this command!"); + await ctx.Homeserver.GetRoom(botData.LogRoom!).SendMessageEventAsync( + new RoomMessageEventContent(body: $"User {ctx.MessageEvent.Sender} tried to use command {Name} but does not have permission!", messageType: "m.text")); + } + + return isAdmin; + } + + public async Task Invoke(CommandContext ctx) { + var botData = await ctx.Homeserver.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + logRoom = ctx.Homeserver.GetRoom(botData.LogRoom ?? botData.ControlRoom); + + await logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Joining space children of {ctx.Args[0]} with reason: {string.Join(' ', ctx.Args[1..])}")); + var roomId = ctx.Args[0]; + var servers = new List<string>() {ctx.Homeserver.ServerName}; + if (roomId.StartsWith('[')) { + + } + + if (roomId.StartsWith('#')) { + var res = await ctx.Homeserver.ResolveRoomAliasAsync(roomId); + roomId = res.RoomId; + servers.AddRange(servers); + } + + var room = ctx.Homeserver.GetRoom(roomId); + var tasks = new List<Task<bool>>(); + await foreach (var memberRoom in room.AsSpace.GetChildrenAsync()) { + servers.Add(room.RoomId.Split(':', 2)[1]); + servers = servers.Distinct().ToList(); + tasks.Add(JoinRoom(memberRoom, string.Join(' ', ctx.Args[1..]), servers)); + } + + await foreach (var b in tasks.ToAsyncEnumerable()) { + await Task.Delay(50); + } + } + + private async Task<bool> JoinRoom(GenericRoom memberRoom, string reason, List<string> servers) { + try { + await memberRoom.JoinAsync(servers.ToArray(), reason); + await logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Joined room {memberRoom.RoomId}")); + } + catch (Exception e) { + await logRoom.SendMessageEventAsync(MessageFormatter.FormatException($"Failed to join {memberRoom.RoomId}", e)); + } + + return true; + } +} diff --git a/ExampleBots/ModerationBot/FirstRunTasks.cs b/ExampleBots/ModerationBot/FirstRunTasks.cs new file mode 100644 index 0000000..ebbdc81 --- /dev/null +++ b/ExampleBots/ModerationBot/FirstRunTasks.cs @@ -0,0 +1,84 @@ +using LibMatrix; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.Homeservers; +using LibMatrix.Responses; +using ModerationBot.AccountData; + +namespace ModerationBot; + +public class FirstRunTasks { + public static async Task<BotData> ConstructBotData(AuthenticatedHomeserverGeneric hs, ModerationBotConfiguration configuration, BotData? botdata) { + botdata ??= new BotData(); + var creationContent = CreateRoomRequest.CreatePrivate(hs, name: "Rory&::ModerationBot - Control room", roomAliasName: "moderation-bot-control-room"); + creationContent.Invite = configuration.Admins; + creationContent.CreationContent["type"] = "gay.rory.moderation_bot.control_room"; + + if (botdata.ControlRoom is null) + try { + botdata.ControlRoom = (await hs.CreateRoom(creationContent)).RoomId; + } + catch (Exception e) { + if (e is not MatrixException { ErrorCode: "M_ROOM_IN_USE" }) { + Console.WriteLine(e); + throw; + } + + creationContent.RoomAliasName += $"-{Guid.NewGuid()}"; + botdata.ControlRoom = (await hs.CreateRoom(creationContent)).RoomId; + } + //set access rules to allow joining via control room + // creationContent.InitialState.Add(new StateEvent { + // Type = "m.room.join_rules", + // StateKey = "", + // TypedContent = new RoomJoinRulesEventContent { + // JoinRule = "knock_restricted", + // Allow = new() { + // new RoomJoinRulesEventContent.AllowEntry { + // Type = "m.room_membership", + // RoomId = botdata.ControlRoom + // } + // } + // } + // }); + + creationContent.Name = "Rory&::ModerationBot - Log room"; + creationContent.RoomAliasName = "moderation-bot-log-room"; + creationContent.CreationContent["type"] = "gay.rory.moderation_bot.log_room"; + + if (botdata.LogRoom is null) + try { + botdata.LogRoom = (await hs.CreateRoom(creationContent)).RoomId; + } + catch (Exception e) { + if (e is not MatrixException { ErrorCode: "M_ROOM_IN_USE" }) { + Console.WriteLine(e); + throw; + } + + creationContent.RoomAliasName += $"-{Guid.NewGuid()}"; + botdata.LogRoom = (await hs.CreateRoom(creationContent)).RoomId; + } + + creationContent.Name = "Rory&::ModerationBot - Policy room"; + creationContent.RoomAliasName = "moderation-bot-policy-room"; + creationContent.CreationContent["type"] = "gay.rory.moderation_bot.policy_room"; + + if (botdata.DefaultPolicyRoom is null) + try { + botdata.DefaultPolicyRoom = (await hs.CreateRoom(creationContent)).RoomId; + } + catch (Exception e) { + if (e is not MatrixException { ErrorCode: "M_ROOM_IN_USE" }) { + Console.WriteLine(e); + throw; + } + + creationContent.RoomAliasName += $"-{Guid.NewGuid()}"; + botdata.DefaultPolicyRoom = (await hs.CreateRoom(creationContent)).RoomId; + } + + await hs.SetAccountDataAsync("gay.rory.moderation_bot_data", botdata); + + return botdata; + } +} \ No newline at end of file diff --git a/ExampleBots/ModerationBot/ModerationBot.cs b/ExampleBots/ModerationBot/ModerationBot.cs new file mode 100644 index 0000000..79b05bf --- /dev/null +++ b/ExampleBots/ModerationBot/ModerationBot.cs @@ -0,0 +1,275 @@ +using System.Security.Cryptography; +using System.Text.RegularExpressions; +using ArcaneLibs.Extensions; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.Helpers; +using LibMatrix.Homeservers; +using LibMatrix.Responses; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using LibMatrix.Utilities.Bot.Interfaces; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging; +using ModerationBot.StateEventTypes.Policies; + +namespace ModerationBot; + +public class ModerationBot(AuthenticatedHomeserverGeneric hs, ILogger<ModerationBot> logger, ModerationBotConfiguration configuration, + HomeserverResolverService hsResolver, PolicyEngine engine) : IHostedService { + private readonly IEnumerable<ICommand> _commands; + + private Task _listenerTask; + + // private GenericRoom _policyRoom; + private GenericRoom _logRoom; + private GenericRoom _controlRoom; + + /// <summary>Triggered when the application host is ready to start the service.</summary> + /// <param name="cancellationToken">Indicates that the start process has been aborted.</param> + public async Task StartAsync(CancellationToken cancellationToken) { + _listenerTask = Run(cancellationToken); + logger.LogInformation("Bot started!"); + } + + private async Task Run(CancellationToken cancellationToken) { + if (Directory.Exists("bot_data/cache")) + Directory.GetFiles("bot_data/cache").ToList().ForEach(File.Delete); + + BotData botData; + + try { + logger.LogInformation("Fetching bot account data..."); + botData = await hs.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + logger.LogInformation("Got bot account data..."); + } + catch (Exception e) { + logger.LogInformation("Could not fetch bot account data... {}", e.Message); + if (e is not MatrixException { ErrorCode: "M_NOT_FOUND" }) { + logger.LogError("{}", e.ToString()); + throw; + } + + botData = null; + } + + botData = await FirstRunTasks.ConstructBotData(hs, configuration, botData); + + // _policyRoom = hs.GetRoom(botData.PolicyRoom ?? botData.ControlRoom); + _logRoom = hs.GetRoom(botData.LogRoom ?? botData.ControlRoom); + _controlRoom = hs.GetRoom(botData.ControlRoom); + foreach (var configurationAdmin in configuration.Admins) { + var pls = await _controlRoom.GetPowerLevelsAsync(); + pls.SetUserPowerLevel(configurationAdmin, pls.GetUserPowerLevel(hs.UserId)); + await _controlRoom.SendStateEventAsync(RoomPowerLevelEventContent.EventId, pls); + } + var syncHelper = new SyncHelper(hs); + + List<string> admins = new(); + +#pragma warning disable CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed + Task.Run(async () => { + while (!cancellationToken.IsCancellationRequested) { + var controlRoomMembers = _controlRoom.GetMembersAsync(); + await foreach (var member in controlRoomMembers) { + if ((member.TypedContent as RoomMemberEventContent)? + .Membership == "join") admins.Add(member.StateKey); + } + + await Task.Delay(TimeSpan.FromMinutes(5), cancellationToken); + } + }, cancellationToken); +#pragma warning restore CS4014 // Because this call is not awaited, execution of the current method continues before the call is completed + + syncHelper.InviteReceivedHandlers.Add(async Task (args) => { + var inviteEvent = + args.Value.InviteState.Events.FirstOrDefault(x => + x.Type == "m.room.member" && x.StateKey == hs.UserId); + logger.LogInformation("Got invite to {RoomId} by {Sender} with reason: {Reason}", args.Key, inviteEvent!.Sender, + (inviteEvent.TypedContent as RoomMemberEventContent)!.Reason); + await _logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Bot invited to {MessageFormatter.HtmlFormatMention(args.Key)} by {MessageFormatter.HtmlFormatMention(inviteEvent.Sender)}")); + if (admins.Contains(inviteEvent.Sender)) { + try { + await _logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Joining {MessageFormatter.HtmlFormatMention(args.Key)}...")); + var senderProfile = await hs.GetProfileAsync(inviteEvent.Sender); + await hs.GetRoom(args.Key).JoinAsync(reason: $"I was invited by {senderProfile.DisplayName ?? inviteEvent.Sender}!"); + } + catch (Exception e) { + logger.LogError("{}", e.ToString()); + await _logRoom.SendMessageEventAsync(MessageFormatter.FormatException("Could not join room", e)); + await hs.GetRoom(args.Key).LeaveAsync(reason: "I was unable to join the room: " + e); + } + } + }); + + syncHelper.TimelineEventHandlers.Add(async @event => { + var room = hs.GetRoom(@event.RoomId); + try { + logger.LogInformation( + "Got timeline event in {}: {}", @event.RoomId, @event.ToJson(indent: true, ignoreNull: true)); + + if (@event != null && ( + @event.GetType.IsAssignableTo(typeof(BasePolicy)) + || @event.GetType.IsAssignableTo(typeof(PolicyRuleEventContent)) + )) + await engine.ReloadActivePolicyListById(@event.RoomId); + + var rules = await engine.GetMatchingPolicies(@event); + foreach (var matchedRule in rules) { + await _logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccessJson( + $"{MessageFormatter.HtmlFormatMessageLink(eventId: @event.EventId, roomId: room.RoomId, displayName: "Event")} matched {MessageFormatter.HtmlFormatMessageLink(eventId: @matchedRule.OriginalEvent.EventId, roomId: matchedRule.PolicyList.Room.RoomId, displayName: "rule")}", @matchedRule.OriginalEvent.RawContent)); + } + + if (configuration.DemoMode) { + // foreach (var matchedRule in rules) { + // await room.SendMessageEventAsync(MessageFormatter.FormatSuccessJson( + // $"{MessageFormatter.HtmlFormatMessageLink(eventId: @event.EventId, roomId: room.RoomId, displayName: "Event")} matched {MessageFormatter.HtmlFormatMessageLink(eventId: @matchedRule.EventId, roomId: matchedRule.RoomId, displayName: "rule")}", @matchedRule.RawContent)); + // } + return; + } +// +// if (@event is { Type: "m.room.message", TypedContent: RoomMessageEventContent message }) { +// if (message is { MessageType: "m.image" }) { +// //check media +// // var matchedPolicy = await CheckMedia(@event); +// var matchedPolicy = rules.FirstOrDefault(); +// if (matchedPolicy is null) return; +// var matchedpolicyData = matchedPolicy.TypedContent as MediaPolicyEventContent; +// await _logRoom.SendMessageEventAsync( +// new RoomMessageEventContent( +// body: +// $"User {MessageFormatter.HtmlFormatMention(@event.Sender)} posted an image in {MessageFormatter.HtmlFormatMention(room.RoomId)} that matched rule {matchedPolicy.StateKey}, applying action {matchedpolicyData.Recommendation}, as described in rule: {matchedPolicy.RawContent!.ToJson(ignoreNull: true)}", +// messageType: "m.text") { +// Format = "org.matrix.custom.html", +// FormattedBody = +// $"<font color=\"#FFFF00\">User {MessageFormatter.HtmlFormatMention(@event.Sender)} posted an image in {MessageFormatter.HtmlFormatMention(room.RoomId)} that matched rule {matchedPolicy.StateKey}, applying action {matchedpolicyData.Recommendation}, as described in rule: <pre>{matchedPolicy.RawContent!.ToJson(ignoreNull: true)}</pre></font>" +// }); +// switch (matchedpolicyData.Recommendation) { +// case "warn_admins": { +// await _controlRoom.SendMessageEventAsync( +// new RoomMessageEventContent( +// body: $"{string.Join(' ', admins)}\nUser {MessageFormatter.HtmlFormatMention(@event.Sender)} posted a banned image {message.Url}", +// messageType: "m.text") { +// Format = "org.matrix.custom.html", +// FormattedBody = $"{string.Join(' ', admins.Select(u => MessageFormatter.HtmlFormatMention(u)))}\n" + +// $"<font color=\"#FF0000\">User {MessageFormatter.HtmlFormatMention(@event.Sender)} posted a banned image <a href=\"{message.Url}\">{message.Url}</a></font>" +// }); +// break; +// } +// case "warn": { +// await room.SendMessageEventAsync( +// new RoomMessageEventContent( +// body: $"Please be careful when posting this image: {matchedpolicyData.Reason ?? "No reason specified"}", +// messageType: "m.text") { +// Format = "org.matrix.custom.html", +// FormattedBody = +// $"<font color=\"#FFFF00\">Please be careful when posting this image: {matchedpolicyData.Reason ?? "No reason specified"}</a></font>" +// }); +// break; +// } +// case "redact": { +// await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason ?? "No reason specified"); +// break; +// } +// case "spoiler": { +// // <blockquote> +// // <a href=\"https://matrix.to/#/@emma:rory.gay\">@emma:rory.gay</a><br> +// // <a href=\"https://codeberg.org/crimsonfork/CN\"></a> +// // <font color=\"#dc143c\" data-mx-color=\"#dc143c\"> +// // <b>CN</b> +// // </font>: +// // <a href=\"https://the-apothecary.club/_matrix/media/v3/download/rory.gay/sLkdxUhipiQaFwRkXcPSRwdg\">test</a><br> +// // <span data-mx-spoiler=\"\"><a href=\"https://the-apothecary.club/_matrix/media/v3/download/rory.gay/sLkdxUhipiQaFwRkXcPSRwdg\"> +// // <img src=\"mxc://rory.gay/sLkdxUhipiQaFwRkXcPSRwdg\" height=\"69\"></a> +// // </span> +// // </blockquote> +// await room.SendMessageEventAsync( +// new RoomMessageEventContent( +// body: +// $"Please be careful when posting this image: {matchedpolicyData.Reason}, I have spoilered it for you:", +// messageType: "m.text") { +// Format = "org.matrix.custom.html", +// FormattedBody = +// $"<font color=\"#FFFF00\">Please be careful when posting this image: {matchedpolicyData.Reason}, I have spoilered it for you:</a></font>" +// }); +// var imageUrl = message.Url; +// await room.SendMessageEventAsync( +// new RoomMessageEventContent(body: $"CN: {imageUrl}", +// messageType: "m.text") { +// Format = "org.matrix.custom.html", +// FormattedBody = $""" +// <blockquote> +// <font color=\"#dc143c\" data-mx-color=\"#dc143c\"> +// <b>CN</b> +// </font>: +// <a href=\"{imageUrl}\">{matchedpolicyData.Reason}</a><br> +// <span data-mx-spoiler=\"\"> +// <a href=\"{imageUrl}\"> +// <img src=\"{imageUrl}\" height=\"69\"> +// </a> +// </span> +// </blockquote> +// """ +// }); +// await room.RedactEventAsync(@event.EventId, "Automatically spoilered: " + matchedpolicyData.Reason); +// break; +// } +// case "mute": { +// await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason); +// //change powerlevel to -1 +// var currentPls = await room.GetPowerLevelsAsync(); +// if (currentPls is null) { +// logger.LogWarning("Unable to get power levels for {room}", room.RoomId); +// await _logRoom.SendMessageEventAsync( +// MessageFormatter.FormatError($"Unable to get power levels for {MessageFormatter.HtmlFormatMention(room.RoomId)}")); +// return; +// } +// +// currentPls.Users ??= new(); +// currentPls.Users[@event.Sender] = -1; +// await room.SendStateEventAsync("m.room.power_levels", currentPls); +// break; +// } +// case "kick": { +// await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason); +// await room.KickAsync(@event.Sender, matchedpolicyData.Reason); +// break; +// } +// case "ban": { +// await room.RedactEventAsync(@event.EventId, matchedpolicyData.Reason); +// await room.BanAsync(@event.Sender, matchedpolicyData.Reason); +// break; +// } +// default: { +// throw new ArgumentOutOfRangeException("recommendation", +// $"Unknown response type {matchedpolicyData.Recommendation}!"); +// } +// } +// } +// } + } + catch (Exception e) { + logger.LogError("{}", e.ToString()); + await _controlRoom.SendMessageEventAsync( + MessageFormatter.FormatException($"Unable to process event in {MessageFormatter.HtmlFormatMention(room.RoomId)}", e)); + await _logRoom.SendMessageEventAsync( + MessageFormatter.FormatException($"Unable to process event in {MessageFormatter.HtmlFormatMention(room.RoomId)}", e)); + await using var stream = new MemoryStream(e.ToString().AsBytes().ToArray()); + await _controlRoom.SendFileAsync("error.log.cs", stream); + await _logRoom.SendFileAsync("error.log.cs", stream); + } + }); + await engine.ReloadActivePolicyLists(); + await syncHelper.RunSyncLoopAsync(); + } + + /// <summary>Triggered when the application host is performing a graceful shutdown.</summary> + /// <param name="cancellationToken">Indicates that the shutdown process should no longer be graceful.</param> + public async Task StopAsync(CancellationToken cancellationToken) { + logger.LogInformation("Shutting down bot!"); + } + +} \ No newline at end of file diff --git a/ExampleBots/MediaModeratorPoC/MediaModeratorPoC.csproj b/ExampleBots/ModerationBot/ModerationBot.csproj index c3a5d87..5c8f8ff 100644 --- a/ExampleBots/MediaModeratorPoC/MediaModeratorPoC.csproj +++ b/ExampleBots/ModerationBot/ModerationBot.csproj @@ -23,7 +23,7 @@ </ItemGroup> <ItemGroup> - <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0-preview.7.23375.6" /> + <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" /> </ItemGroup> <ItemGroup> <Content Include="appsettings*.json"> diff --git a/ExampleBots/ModerationBot/ModerationBotConfiguration.cs b/ExampleBots/ModerationBot/ModerationBotConfiguration.cs new file mode 100644 index 0000000..415b581 --- /dev/null +++ b/ExampleBots/ModerationBot/ModerationBotConfiguration.cs @@ -0,0 +1,10 @@ +using Microsoft.Extensions.Configuration; + +namespace ModerationBot; + +public class ModerationBotConfiguration { + public ModerationBotConfiguration(IConfiguration config) => config.GetRequiredSection("ModerationBot").Bind(this); + + public List<string> Admins { get; set; } = new(); + public bool DemoMode { get; set; } = false; +} diff --git a/ExampleBots/ModerationBot/PolicyEngine.cs b/ExampleBots/ModerationBot/PolicyEngine.cs new file mode 100644 index 0000000..5311637 --- /dev/null +++ b/ExampleBots/ModerationBot/PolicyEngine.cs @@ -0,0 +1,269 @@ +using System.Diagnostics; +using System.Security.Cryptography; +using System.Text.Json; +using System.Text.RegularExpressions; +using ArcaneLibs.Extensions; +using LibMatrix; +using LibMatrix.EventTypes.Spec; +using LibMatrix.EventTypes.Spec.State; +using LibMatrix.Helpers; +using LibMatrix.Homeservers; +using LibMatrix.Interfaces; +using LibMatrix.RoomTypes; +using LibMatrix.Services; +using ModerationBot.AccountData; +using ModerationBot.StateEventTypes; +using Microsoft.Extensions.Logging; +using ModerationBot.StateEventTypes.Policies; +using ModerationBot.StateEventTypes.Policies.Implementations; + +namespace ModerationBot; + +public class PolicyEngine(AuthenticatedHomeserverGeneric hs, ILogger<ModerationBot> logger, ModerationBotConfiguration configuration, HomeserverResolverService hsResolver) { + private Dictionary<string, PolicyList> PolicyListAccountData { get; set; } = new(); + public List<PolicyList> ActivePolicyLists { get; set; } = new(); + public List<BasePolicy> ActivePolicies { get; set; } = new(); + public Dictionary<string, List<BasePolicy>> ActivePoliciesByType { get; set; } = new(); + private GenericRoom? _logRoom; + private GenericRoom? _controlRoom; + + public async Task ReloadActivePolicyLists() { + var sw = Stopwatch.StartNew(); + + var botData = await hs.GetAccountDataAsync<BotData>("gay.rory.moderation_bot_data"); + _logRoom ??= hs.GetRoom(botData.LogRoom ?? botData.ControlRoom); + _controlRoom ??= hs.GetRoom(botData.ControlRoom); + + await _controlRoom?.SendMessageEventAsync(MessageFormatter.FormatSuccess("Reloading policy lists!"))!; + await _logRoom?.SendMessageEventAsync(MessageFormatter.FormatSuccess("Reloading policy lists!"))!; + + var progressMessage = await _logRoom?.SendMessageEventAsync(MessageFormatter.FormatSuccess("0/? policy lists loaded"))!; + + var policyLists = new List<PolicyList>(); + try { + PolicyListAccountData = await hs.GetAccountDataAsync<Dictionary<string, PolicyList>>("gay.rory.moderation_bot.policy_lists"); + } + catch (MatrixException e) { + if (e is not { ErrorCode: "M_NOT_FOUND" }) throw; + } + + if (!PolicyListAccountData.ContainsKey(botData.DefaultPolicyRoom)) { + PolicyListAccountData.Add(botData.DefaultPolicyRoom, new PolicyList() { + Trusted = true + }); + await hs.SetAccountDataAsync("gay.rory.moderation_bot.policy_lists", PolicyListAccountData); + } + + var loadTasks = new List<Task<PolicyList>>(); + foreach (var (roomId, policyList) in PolicyListAccountData) { + var room = hs.GetRoom(roomId); + loadTasks.Add(LoadPolicyListAsync(room, policyList)); + } + + await foreach (var policyList in loadTasks.ToAsyncEnumerable()) { + policyLists.Add(policyList); + + if (policyList.Policies.Count >= 256 || policyLists.Count == PolicyListAccountData.Count) { + var progressMsgContent = MessageFormatter.FormatSuccess($"{policyLists.Count}/{PolicyListAccountData.Count} policy lists loaded, " + + $"{policyLists.Sum(x => x.Policies.Count)} policies total, {sw.Elapsed} elapsed.") + .SetReplaceRelation<RoomMessageEventContent>(progressMessage.EventId); + + _logRoom?.SendMessageEventAsync(progressMsgContent); + } + } + + // Console.WriteLine($"Reloaded policy list data in {sw.Elapsed}"); + // await _logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Done fetching {policyLists.Count} policy lists in {sw.Elapsed}!")); + + ActivePolicyLists = policyLists; + ActivePolicies = await GetActivePolicies(); + } + + private async Task<PolicyList> LoadPolicyListAsync(GenericRoom room, PolicyList policyList) { + policyList.Room = room; + policyList.Policies.Clear(); + + var stateEvents = room.GetFullStateAsync(); + await foreach (var stateEvent in stateEvents) { + if (stateEvent != null && ( + stateEvent.GetType.IsAssignableTo(typeof(BasePolicy)) + || stateEvent.GetType.IsAssignableTo(typeof(PolicyRuleEventContent)) + )) { + policyList.Policies.Add(stateEvent); + } + } + + // if (policyList.Policies.Count >= 1) + // await _logRoom?.SendMessageEventAsync( + // MessageFormatter.FormatSuccess($"Loaded {policyList.Policies.Count} policies for {MessageFormatter.HtmlFormatMention(room.RoomId)}!"))!; + + return policyList; + } + + + public async Task ReloadActivePolicyListById(string roomId) { + if (!ActivePolicyLists.Any(x => x.Room.RoomId == roomId)) return; + await LoadPolicyListAsync(hs.GetRoom(roomId), ActivePolicyLists.Single(x => x.Room.RoomId == roomId)); + ActivePolicies = await GetActivePolicies(); + } + + public async Task<List<BasePolicy>> GetActivePolicies() { + var sw = Stopwatch.StartNew(); + List<BasePolicy> activePolicies = new(); + + foreach (var activePolicyList in ActivePolicyLists) { + foreach (var policyEntry in activePolicyList.Policies) { + // TODO: implement rule translation + BasePolicy policy = policyEntry.TypedContent is BasePolicy ? policyEntry.TypedContent as BasePolicy : policyEntry.RawContent.Deserialize<UnknownPolicy>(); + if (policy.Entity is null) continue; + policy.PolicyList = activePolicyList; + policy.OriginalEvent = policyEntry; + activePolicies.Add(policy); + } + } + + Console.WriteLine($"Translated policy list data in {sw.Elapsed}"); + ActivePoliciesByType = activePolicies.GroupBy(x => x.GetType().Name).ToDictionary(x => x.Key, x => x.ToList()); + await _logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Translated policy list data in {sw.GetElapsedAndRestart()}")); + // await _logRoom.SendMessageEventAsync(MessageFormatter.FormatSuccess($"Built policy type map in {sw.GetElapsedAndRestart()}")); + + var summary = SummariseStateTypeCounts(activePolicies.Select(x => x.OriginalEvent).ToList()); + await _logRoom?.SendMessageEventAsync(new RoomMessageEventContent() { + Body = summary.Raw, + FormattedBody = summary.Html, + Format = "org.matrix.custom.html" + })!; + + return activePolicies; + } + + public async Task<List<BasePolicy>> GetMatchingPolicies(StateEventResponse @event) { + List<BasePolicy> matchingPolicies = new(); + if (@event.Sender == @hs.UserId) return matchingPolicies; //ignore self at all costs + + if (ActivePoliciesByType.TryGetValue(nameof(ServerPolicyRuleEventContent), out var serverPolicies)) { + var userServer = @event.Sender.Split(':', 2)[1]; + matchingPolicies.AddRange(serverPolicies.Where(x => x.Entity == userServer)); + } + + if (ActivePoliciesByType.TryGetValue(nameof(UserPolicyRuleEventContent), out var userPolicies)) { + matchingPolicies.AddRange(userPolicies.Where(x => x.Entity == @event.Sender)); + } + + if (@event.TypedContent is RoomMessageEventContent msgContent) { + matchingPolicies.AddRange(await CheckMessageContent(@event)); + if (msgContent.MessageType == "m.text" || msgContent.MessageType == "m.notice") ; //TODO: implement word etc. filters + if (msgContent.MessageType == "m.image" || msgContent.MessageType == "m.file" || msgContent.MessageType == "m.audio" || msgContent.MessageType == "m.video") + matchingPolicies.AddRange(await CheckMedia(@event)); + } + + return matchingPolicies; + } + +#region Policy matching + + private async Task<List<BasePolicy>> CheckMessageContent(StateEventResponse @event) { + var matchedRules = new List<BasePolicy>(); + var msgContent = @event.TypedContent as RoomMessageEventContent; + + if (ActivePoliciesByType.TryGetValue(nameof(MessagePolicyContainsText), out var messageContainsPolicies)) + foreach (var policy in messageContainsPolicies) { + if((@msgContent?.Body?.ToLowerInvariant().Contains(policy.Entity.ToLowerInvariant()) ?? false) || (@msgContent?.FormattedBody?.ToLowerInvariant().Contains(policy.Entity.ToLowerInvariant()) ?? false)) + matchedRules.Add(policy); + } + + + return matchedRules; + } + + private async Task<List<BasePolicy>> CheckMedia(StateEventResponse @event) { + var matchedRules = new List<BasePolicy>(); + var hashAlgo = SHA3_256.Create(); + + var mxcUri = @event.RawContent["url"].GetValue<string>(); + + //check server policies before bothering with hashes + if (ActivePoliciesByType.TryGetValue(nameof(MediaPolicyHomeserver), out var mediaHomeserverPolicies)) + foreach (var policy in mediaHomeserverPolicies) { + logger.LogInformation("Checking rule {rule}: {data}", policy.OriginalEvent.StateKey, policy.OriginalEvent.TypedContent.ToJson(ignoreNull: true, indent: false)); + policy.Entity = policy.Entity.Replace("\\*", ".*").Replace("\\?", "."); + var regex = new Regex($"mxc://({policy.Entity})/.*", RegexOptions.Compiled | RegexOptions.IgnoreCase); + if (regex.IsMatch(@event.RawContent["url"]!.GetValue<string>())) { + logger.LogInformation("{url} matched rule {rule}", @event.RawContent["url"], policy.ToJson(ignoreNull: true)); + matchedRules.Add(policy); + // continue; + } + } + + var resolvedUri = await hsResolver.ResolveMediaUri(mxcUri.Split('/')[2], mxcUri); + var uriHash = hashAlgo.ComputeHash(mxcUri.AsBytes().ToArray()); + byte[]? fileHash = null; + + try { + fileHash = await hashAlgo.ComputeHashAsync(await hs.ClientHttpClient.GetStreamAsync(resolvedUri)); + } + catch (Exception ex) { + await _logRoom.SendMessageEventAsync( + MessageFormatter.FormatException($"Error calculating file hash for {mxcUri} via {mxcUri.Split('/')[2]} ({resolvedUri}), retrying via {hs.BaseUrl}...", + ex)); + try { + resolvedUri = await hsResolver.ResolveMediaUri(hs.BaseUrl, mxcUri); + fileHash = await hashAlgo.ComputeHashAsync(await hs.ClientHttpClient.GetStreamAsync(resolvedUri)); + } + catch (Exception ex2) { + await _logRoom.SendMessageEventAsync( + MessageFormatter.FormatException($"Error calculating file hash via {hs.BaseUrl} ({resolvedUri})!", ex2)); + } + } + + logger.LogInformation("Checking media {url} with hash {hash}", resolvedUri, fileHash); + + if (ActivePoliciesByType.ContainsKey(nameof(MediaPolicyFile))) + foreach (MediaPolicyFile policy in ActivePoliciesByType[nameof(MediaPolicyFile)]) { + logger.LogInformation("Checking rule {rule}: {data}", policy.OriginalEvent.StateKey, policy.OriginalEvent.TypedContent.ToJson(ignoreNull: true, indent: false)); + if (policy.Entity is not null && Convert.ToBase64String(uriHash).SequenceEqual(policy.Entity)) { + logger.LogInformation("{url} matched rule {rule} by uri hash", @event.RawContent["url"], policy.ToJson(ignoreNull: true)); + matchedRules.Add(policy); + // continue; + } + else logger.LogInformation("uri hash {uriHash} did not match rule's {ruleUriHash}", Convert.ToHexString(uriHash), policy.Entity); + + if (policy.FileHash is not null && fileHash is not null && policy.FileHash == Convert.ToBase64String(fileHash)) { + logger.LogInformation("{url} matched rule {rule} by file hash", @event.RawContent["url"], policy.ToJson(ignoreNull: true)); + matchedRules.Add(policy); + // continue; + } + else logger.LogInformation("file hash {fileHash} did not match rule's {ruleFileHash}", Convert.ToBase64String(fileHash), policy.FileHash); + + //check pixels every 10% of the way through the image using ImageSharp + // var image = Image.Load(await _hs._httpClient.GetStreamAsync(resolvedUri)); + } + else logger.LogInformation("No active media file policies"); + // logger.LogInformation("{url} did not match any rules", @event.RawContent["url"]); + + return matchedRules; + } + +#endregion + +#region Internal code + +#region Summarisation + + private static (string Raw, string Html) SummariseStateTypeCounts(IList<StateEventResponse> states) { + string raw = "Count | State type | Mapped type", html = "<table><tr><th>Count</th><th>State type</th><th>Mapped type</th></tr>"; + var groupedStates = states.GroupBy(x => x.Type).ToDictionary(x => x.Key, x => x.ToList()).OrderByDescending(x => x.Value.Count); + foreach (var (type, stateGroup) in groupedStates) { + raw += $"{stateGroup.Count} | {type} | {stateGroup[0].GetType.Name}"; + html += $"<tr><td>{stateGroup.Count}</td><td>{type}</td><td>{stateGroup[0].GetType.Name}</td></tr>"; + } + + html += "</table>"; + return (raw, html); + } + +#endregion + +#endregion + +} \ No newline at end of file diff --git a/ExampleBots/MediaModeratorPoC/PolicyList.cs b/ExampleBots/ModerationBot/PolicyList.cs index 0f49c97..a3052bd 100644 --- a/ExampleBots/MediaModeratorPoC/PolicyList.cs +++ b/ExampleBots/ModerationBot/PolicyList.cs @@ -1,9 +1,9 @@ using System.Text.Json.Serialization; using LibMatrix; using LibMatrix.RoomTypes; -using MediaModeratorPoC.StateEventTypes; +using ModerationBot.StateEventTypes; -namespace MediaModeratorPoC; +namespace ModerationBot; public class PolicyList { [JsonIgnore] @@ -13,5 +13,5 @@ public class PolicyList { public bool Trusted { get; set; } = false; [JsonIgnore] - public List<StateEvent> Policies { get; set; } = new(); + public List<StateEventResponse> Policies { get; set; } = new(); } diff --git a/ExampleBots/MediaModeratorPoC/Program.cs b/ExampleBots/ModerationBot/Program.cs index 5b8e734..b41b0be 100644 --- a/ExampleBots/MediaModeratorPoC/Program.cs +++ b/ExampleBots/ModerationBot/Program.cs @@ -2,7 +2,7 @@ using LibMatrix.Services; using LibMatrix.Utilities.Bot; -using MediaModeratorPoC; +using ModerationBot; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Hosting; @@ -15,12 +15,14 @@ var host = Host.CreateDefaultBuilder(args).ConfigureServices((_, services) => { dataStorageProvider: new FileStorageProvider("bot_data/data/") ) ); - services.AddSingleton<MediaModBotConfiguration>(); + services.AddSingleton<ModerationBotConfiguration>(); services.AddRoryLibMatrixServices(); services.AddBot(withCommands: true); - services.AddHostedService<MediaModBot>(); + services.AddSingleton<PolicyEngine>(); + + services.AddHostedService<ModerationBot.ModerationBot>(); }).UseConsoleLifetime().Build(); await host.RunAsync(); diff --git a/ExampleBots/MediaModeratorPoC/Properties/launchSettings.json b/ExampleBots/ModerationBot/Properties/launchSettings.json index 997e294..997e294 100644 --- a/ExampleBots/MediaModeratorPoC/Properties/launchSettings.json +++ b/ExampleBots/ModerationBot/Properties/launchSettings.json diff --git a/ExampleBots/MediaModeratorPoC/StateEventTypes/BasePolicy.cs b/ExampleBots/ModerationBot/StateEventTypes/Policies/BasePolicy.cs index 7735314..94b2f63 100644 --- a/ExampleBots/MediaModeratorPoC/StateEventTypes/BasePolicy.cs +++ b/ExampleBots/ModerationBot/StateEventTypes/Policies/BasePolicy.cs @@ -3,14 +3,14 @@ using System.Text.Json.Serialization; using LibMatrix; using LibMatrix.Interfaces; -namespace MediaModeratorPoC.StateEventTypes; +namespace ModerationBot.StateEventTypes.Policies; public abstract class BasePolicy : EventContent { /// <summary> - /// Entity this policy applies to + /// Entity this policy applies to, null if event was redacted /// </summary> [JsonPropertyName("entity")] - public string Entity { get; set; } + public string? Entity { get; set; } /// <summary> /// Reason this policy exists @@ -40,4 +40,13 @@ public abstract class BasePolicy : EventContent { get => Expiry == null ? null : DateTimeOffset.FromUnixTimeMilliseconds(Expiry.Value).DateTime; set => Expiry = value is null ? null : ((DateTimeOffset)value).ToUnixTimeMilliseconds(); } + +#region Internal metadata + + [JsonIgnore] + public PolicyList PolicyList { get; set; } + + public StateEventResponse OriginalEvent { get; set; } + +#endregion } diff --git a/ExampleBots/MediaModeratorPoC/StateEventTypes/MediaPolicyStateEventData.cs b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MediaPolicyFile.cs index 603a858..c5b6ef2 100644 --- a/ExampleBots/MediaModeratorPoC/StateEventTypes/MediaPolicyStateEventData.cs +++ b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MediaPolicyFile.cs @@ -1,17 +1,16 @@ using System.Text.Json.Serialization; using LibMatrix.EventTypes; -namespace MediaModeratorPoC.StateEventTypes; +namespace ModerationBot.StateEventTypes.Policies.Implementations; /// <summary> /// File policy event, entity is the MXC URI of the file, hashed with SHA3-256. /// </summary> -[MatrixEvent(EventName = "gay.rory.media_moderator_poc.rule.homeserver")] -[MatrixEvent(EventName = "gay.rory.media_moderator_poc.rule.media")] -public class MediaPolicyEventContent : BasePolicy { +[MatrixEvent(EventName = "gay.rory.moderation.rule.media")] +public class MediaPolicyFile : BasePolicy { /// <summary> /// Hash of the file /// </summary> [JsonPropertyName("file_hash")] - public byte[]? FileHash { get; set; } + public string? FileHash { get; set; } } diff --git a/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MediaPolicyHomeserver.cs b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MediaPolicyHomeserver.cs new file mode 100644 index 0000000..3dfd937 --- /dev/null +++ b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MediaPolicyHomeserver.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; +using LibMatrix.EventTypes; + +namespace ModerationBot.StateEventTypes.Policies.Implementations; + +/// <summary> +/// Homeserver media policy event, entity is the MXC URI of the file, hashed with SHA3-256. +/// </summary> +[MatrixEvent(EventName = "gay.rory.moderation.rule.media.homeserver")] +public class MediaPolicyHomeserver : BasePolicy { +} diff --git a/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MessagePolicyContainsText.cs b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MessagePolicyContainsText.cs new file mode 100644 index 0000000..daac162 --- /dev/null +++ b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/MessagePolicyContainsText.cs @@ -0,0 +1,11 @@ +using System.Text.Json.Serialization; +using LibMatrix.EventTypes; + +namespace ModerationBot.StateEventTypes.Policies.Implementations; + +/// <summary> +/// Text contains policy event, entity is the text to contain. +/// </summary> +[MatrixEvent(EventName = "gay.rory.moderation.rule.text.contains")] +public class MessagePolicyContainsText : BasePolicy { +} diff --git a/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/UnknownPolicy.cs b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/UnknownPolicy.cs new file mode 100644 index 0000000..8dc8258 --- /dev/null +++ b/ExampleBots/ModerationBot/StateEventTypes/Policies/Implementations/UnknownPolicy.cs @@ -0,0 +1,10 @@ +using System.Text.Json.Serialization; +using LibMatrix.EventTypes; + +namespace ModerationBot.StateEventTypes.Policies.Implementations; + +/// <summary> +/// Unknown policy event, usually used for handling unknown cases +/// </summary> +public class UnknownPolicy : BasePolicy { +} diff --git a/ExampleBots/MediaModeratorPoC/appsettings.Development.json b/ExampleBots/ModerationBot/appsettings.Development.json index 600efc3..224d0da 100644 --- a/ExampleBots/MediaModeratorPoC/appsettings.Development.json +++ b/ExampleBots/ModerationBot/appsettings.Development.json @@ -14,7 +14,7 @@ // The command prefix "Prefix": "?" }, - "MediaMod": { + "ModerationBot": { // List of people who should be invited to the control room "Admins": [ "@emma:conduit.rory.gay", diff --git a/ExampleBots/MediaModeratorPoC/appsettings.json b/ExampleBots/ModerationBot/appsettings.json index 6ba02f3..6ba02f3 100644 --- a/ExampleBots/MediaModeratorPoC/appsettings.json +++ b/ExampleBots/ModerationBot/appsettings.json diff --git a/ExampleBots/PluralContactBotPoC/Bot/Commands/CreateSystemCommand.cs b/ExampleBots/PluralContactBotPoC/Bot/Commands/CreateSystemCommand.cs index 4a7d646..6d2cb3a 100644 --- a/ExampleBots/PluralContactBotPoC/Bot/Commands/CreateSystemCommand.cs +++ b/ExampleBots/PluralContactBotPoC/Bot/Commands/CreateSystemCommand.cs @@ -37,7 +37,7 @@ public class CreateSystemCommand(IServiceProvider services, HomeserverProviderSe var state = ctx.Room.GetMembersAsync(); await foreach (var member in state) { - sysData.Members.Add(member.UserId); + sysData.Members.Add(member.StateKey); } await ctx.Room.SendStateEventAsync("m.room.name", new RoomNameEventContent() { diff --git a/ExampleBots/PluralContactBotPoC/PluralContactBotPoC.csproj b/ExampleBots/PluralContactBotPoC/PluralContactBotPoC.csproj index cd549f9..664ff49 100644 --- a/ExampleBots/PluralContactBotPoC/PluralContactBotPoC.csproj +++ b/ExampleBots/PluralContactBotPoC/PluralContactBotPoC.csproj @@ -23,7 +23,7 @@ </ItemGroup> <ItemGroup> - <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0-preview.7.23375.6" /> + <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" /> </ItemGroup> <ItemGroup> <Content Include="appsettings*.json"> diff --git a/LibMatrix.sln b/LibMatrix.sln index fd241b3..87398f3 100644 --- a/LibMatrix.sln +++ b/LibMatrix.sln @@ -13,12 +13,20 @@ Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix.MxApiExtensions", EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix.ExampleBot", "ExampleBots\LibMatrix.ExampleBot\LibMatrix.ExampleBot.csproj", "{1B1B2197-61FB-416F-B6C8-845F2E5A0442}" EndProject -Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "MediaModeratorPoC", "ExampleBots\MediaModeratorPoC\MediaModeratorPoC.csproj", "{8F0A820E-F6AE-45A2-970E-7A3759693919}" +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "ModerationBot", "ExampleBots\ModerationBot\ModerationBot.csproj", "{8F0A820E-F6AE-45A2-970E-7A3759693919}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix.DebugDataValidationApi", "Utilities\LibMatrix.DebugDataValidationApi\LibMatrix.DebugDataValidationApi.csproj", "{35DF9A1A-D988-4225-AFA3-06BB8EDEB559}" EndProject Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix.Utilities.Bot", "Utilities\LibMatrix.Utilities.Bot\LibMatrix.Utilities.Bot.csproj", "{3ACF2613-E23F-42C2-925E-0BB4FC3AB1F7}" EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "PluralContactBotPoC", "ExampleBots\PluralContactBotPoC\PluralContactBotPoC.csproj", "{FB8FE4EB-B53B-464B-A5FD-9BF9D0F3EF9B}" +EndProject +Project("{2150E333-8FDC-42A3-9474-1A3956D46DE8}") = "Tests", "Tests", "{BFE16D8E-EFC5-49F6-9854-DB001309B3B4}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "LibMatrix.Tests", "Tests\LibMatrix.Tests\LibMatrix.Tests.csproj", "{345934FF-CA81-4A4B-B137-9F198102C65F}" +EndProject +Project("{FAE04EC0-301F-11D3-BF4B-00C04F79EFBC}") = "TestDataGenerator", "Tests\TestDataGenerator\TestDataGenerator.csproj", "{0B9B34D1-9362-45A9-9C21-816FD6959110}" +EndProject Global GlobalSection(SolutionConfigurationPlatforms) = preSolution Debug|Any CPU = Debug|Any CPU @@ -52,11 +60,26 @@ Global {3ACF2613-E23F-42C2-925E-0BB4FC3AB1F7}.Debug|Any CPU.Build.0 = Debug|Any CPU {3ACF2613-E23F-42C2-925E-0BB4FC3AB1F7}.Release|Any CPU.ActiveCfg = Release|Any CPU {3ACF2613-E23F-42C2-925E-0BB4FC3AB1F7}.Release|Any CPU.Build.0 = Release|Any CPU + {FB8FE4EB-B53B-464B-A5FD-9BF9D0F3EF9B}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {FB8FE4EB-B53B-464B-A5FD-9BF9D0F3EF9B}.Debug|Any CPU.Build.0 = Debug|Any CPU + {FB8FE4EB-B53B-464B-A5FD-9BF9D0F3EF9B}.Release|Any CPU.ActiveCfg = Release|Any CPU + {FB8FE4EB-B53B-464B-A5FD-9BF9D0F3EF9B}.Release|Any CPU.Build.0 = Release|Any CPU + {345934FF-CA81-4A4B-B137-9F198102C65F}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {345934FF-CA81-4A4B-B137-9F198102C65F}.Debug|Any CPU.Build.0 = Debug|Any CPU + {345934FF-CA81-4A4B-B137-9F198102C65F}.Release|Any CPU.ActiveCfg = Release|Any CPU + {345934FF-CA81-4A4B-B137-9F198102C65F}.Release|Any CPU.Build.0 = Release|Any CPU + {0B9B34D1-9362-45A9-9C21-816FD6959110}.Debug|Any CPU.ActiveCfg = Debug|Any CPU + {0B9B34D1-9362-45A9-9C21-816FD6959110}.Debug|Any CPU.Build.0 = Debug|Any CPU + {0B9B34D1-9362-45A9-9C21-816FD6959110}.Release|Any CPU.ActiveCfg = Release|Any CPU + {0B9B34D1-9362-45A9-9C21-816FD6959110}.Release|Any CPU.Build.0 = Release|Any CPU EndGlobalSection GlobalSection(NestedProjects) = preSolution {1B1B2197-61FB-416F-B6C8-845F2E5A0442} = {840309F0-435B-43A7-8471-8C2BE643889D} {8F0A820E-F6AE-45A2-970E-7A3759693919} = {840309F0-435B-43A7-8471-8C2BE643889D} {35DF9A1A-D988-4225-AFA3-06BB8EDEB559} = {A6345ECE-4C5E-400F-9130-886E343BF314} {3ACF2613-E23F-42C2-925E-0BB4FC3AB1F7} = {A6345ECE-4C5E-400F-9130-886E343BF314} + {FB8FE4EB-B53B-464B-A5FD-9BF9D0F3EF9B} = {840309F0-435B-43A7-8471-8C2BE643889D} + {345934FF-CA81-4A4B-B137-9F198102C65F} = {BFE16D8E-EFC5-49F6-9854-DB001309B3B4} + {0B9B34D1-9362-45A9-9C21-816FD6959110} = {BFE16D8E-EFC5-49F6-9854-DB001309B3B4} EndGlobalSection EndGlobal diff --git a/LibMatrix/EventTypes/Spec/Ephemeral/PresenceStateEventContent.cs b/LibMatrix/EventTypes/Spec/Ephemeral/PresenceStateEventContent.cs index 3e4a5cd..558e4fc 100644 --- a/LibMatrix/EventTypes/Spec/Ephemeral/PresenceStateEventContent.cs +++ b/LibMatrix/EventTypes/Spec/Ephemeral/PresenceStateEventContent.cs @@ -4,17 +4,17 @@ using LibMatrix.Interfaces; namespace LibMatrix.EventTypes.Spec.State; [MatrixEvent(EventName = "m.presence")] -public class PresenceEventContent : TimelineEventContent { - [JsonPropertyName("presence")] - public string Presence { get; set; } +public class PresenceEventContent : EventContent { + [JsonPropertyName("presence"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? Presence { get; set; } [JsonPropertyName("last_active_ago")] public long LastActiveAgo { get; set; } [JsonPropertyName("currently_active")] public bool CurrentlyActive { get; set; } - [JsonPropertyName("status_msg")] - public string StatusMessage { get; set; } - [JsonPropertyName("avatar_url")] - public string AvatarUrl { get; set; } - [JsonPropertyName("displayname")] - public string DisplayName { get; set; } + [JsonPropertyName("status_msg"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? StatusMessage { get; set; } + [JsonPropertyName("avatar_url"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? AvatarUrl { get; set; } + [JsonPropertyName("displayname"), JsonIgnore(Condition = JsonIgnoreCondition.WhenWritingNull)] + public string? DisplayName { get; set; } } diff --git a/LibMatrix/EventTypes/Spec/RoomMessageEventContent.cs b/LibMatrix/EventTypes/Spec/RoomMessageEventContent.cs index 1938b3e..8a22489 100644 --- a/LibMatrix/EventTypes/Spec/RoomMessageEventContent.cs +++ b/LibMatrix/EventTypes/Spec/RoomMessageEventContent.cs @@ -17,16 +17,29 @@ public class RoomMessageEventContent : TimelineEventContent { public string MessageType { get; set; } = "m.notice"; [JsonPropertyName("formatted_body")] - public string FormattedBody { get; set; } + public string? FormattedBody { get; set; } [JsonPropertyName("format")] - public string Format { get; set; } + public string? Format { get; set; } /// <summary> /// Media URI for this message, if any /// </summary> [JsonPropertyName("url")] public string? Url { get; set; } - + public string? FileName { get; set; } + + [JsonPropertyName("info")] + public FileInfoStruct? FileInfo { get; set; } + + public class FileInfoStruct { + [JsonPropertyName("mimetype")] + public string? MimeType { get; set; } + [JsonPropertyName("size")] + public long Size { get; set; } + [JsonPropertyName("thumbnail_url")] + public string? ThumbnailUrl { get; set; } + } + } diff --git a/LibMatrix/EventTypes/Spec/State/Policy/PolicyRuleStateEventContent.cs b/LibMatrix/EventTypes/Spec/State/Policy/PolicyRuleStateEventContent.cs index 63f148c..757a9e9 100644 --- a/LibMatrix/EventTypes/Spec/State/Policy/PolicyRuleStateEventContent.cs +++ b/LibMatrix/EventTypes/Spec/State/Policy/PolicyRuleStateEventContent.cs @@ -3,10 +3,23 @@ using LibMatrix.Interfaces; namespace LibMatrix.EventTypes.Spec.State; -[MatrixEvent(EventName = "m.policy.rule.user")] -[MatrixEvent(EventName = "m.policy.rule.server")] -[MatrixEvent(EventName = "org.matrix.mjolnir.rule.server")] -public class PolicyRuleEventContent : TimelineEventContent { +//spec +[MatrixEvent(EventName = "m.policy.rule.server")] //spec +[MatrixEvent(EventName = "m.room.rule.server")] //??? +[MatrixEvent(EventName = "org.matrix.mjolnir.rule.server")] //legacy +public class ServerPolicyRuleEventContent : PolicyRuleEventContent { } + +[MatrixEvent(EventName = "m.policy.rule.user")] //spec +[MatrixEvent(EventName = "m.room.rule.user")] //??? +[MatrixEvent(EventName = "org.matrix.mjolnir.rule.user")] //legacy +public class UserPolicyRuleEventContent : PolicyRuleEventContent { } + +[MatrixEvent(EventName = "m.policy.rule.room")] //spec +[MatrixEvent(EventName = "m.room.rule.room")] //??? +[MatrixEvent(EventName = "org.matrix.mjolnir.rule.room")] //legacy +public class RoomPolicyRuleEventContent : PolicyRuleEventContent { } + +public abstract class PolicyRuleEventContent : EventContent { /// <summary> /// Entity this ban applies to, can use * and ? as globs. /// </summary> @@ -52,4 +65,4 @@ public static class PolicyRecommendationTypes { /// Mute this user /// </summary> public static string Mute = "support.feline.policy.recommendation_mute"; //stable prefix: m.mute, msc pending -} +} \ No newline at end of file diff --git a/LibMatrix/Extensions/HttpClientExtensions.cs b/LibMatrix/Extensions/HttpClientExtensions.cs index 913864e..6f27f71 100644 --- a/LibMatrix/Extensions/HttpClientExtensions.cs +++ b/LibMatrix/Extensions/HttpClientExtensions.cs @@ -36,12 +36,11 @@ public class MatrixHttpClient : HttpClient { return options; } - public override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, - CancellationToken cancellationToken) { + public override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) { if (request.RequestUri is null) throw new NullReferenceException("RequestUri is null"); if (AssertedUserId is not null) request.RequestUri = request.RequestUri.AddQuery("user_id", AssertedUserId); - // Console.WriteLine($"Sending request to {request.RequestUri}"); + Console.WriteLine($"Sending request to {request.RequestUri}"); try { var webAssemblyEnableStreamingResponseKey = @@ -60,7 +59,7 @@ public class MatrixHttpClient : HttpClient { catch (Exception e) { typeof(HttpRequestMessage).GetField("_sendStatus", BindingFlags.NonPublic | BindingFlags.Instance) ?.SetValue(request, 0); - await Task.Delay(2500); + await Task.Delay(2500, cancellationToken); return await SendAsync(request, cancellationToken); } @@ -123,7 +122,7 @@ public class MatrixHttpClient : HttpClient { var request = new HttpRequestMessage(HttpMethod.Put, requestUri); request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); Console.WriteLine($"Sending PUT {requestUri}"); - Console.WriteLine($"Content: {value.ToJson()}"); + Console.WriteLine($"Content: {JsonSerializer.Serialize(value, value.GetType(), options)}"); Console.WriteLine($"Type: {value.GetType().FullName}"); request.Content = new StringContent(JsonSerializer.Serialize(value, value.GetType(), options), Encoding.UTF8, "application/json"); diff --git a/LibMatrix/Helpers/MessageFormatter.cs b/LibMatrix/Helpers/MessageFormatter.cs index d252e85..3ebc9a2 100644 --- a/LibMatrix/Helpers/MessageFormatter.cs +++ b/LibMatrix/Helpers/MessageFormatter.cs @@ -35,6 +35,11 @@ public static class MessageFormatter { public static string HtmlFormatMention(string id, string? displayName = null) { return $"<a href=\"https://matrix.to/#/{id}\">{displayName ?? id}</a>"; } + + public static string HtmlFormatMessageLink(string roomId, string eventId, string[]? servers = null, string? displayName = null) { + if (servers is not { Length: > 0 }) servers = new[] { roomId.Split(':', 2)[1] }; + return $"<a href=\"https://matrix.to/#/{roomId}/{eventId}?via={string.Join("&via=", servers)}\">{displayName ?? eventId}</a>"; + } #region Extension functions diff --git a/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs b/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs index e4b7483..288608d 100644 --- a/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs +++ b/LibMatrix/Homeservers/AuthenticatedHomeserverGeneric.cs @@ -3,6 +3,7 @@ using System.Net.Http.Json; using System.Text.Json; using System.Text.Json.Nodes; using System.Text.Json.Serialization; +using System.Web; using ArcaneLibs.Extensions; using LibMatrix.EventTypes.Spec.State; using LibMatrix.Filters; @@ -18,7 +19,7 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke var instance = Activator.CreateInstance(typeof(T), serverName, accessToken) as T ?? throw new InvalidOperationException($"Failed to create instance of {typeof(T).Name}"); HomeserverResolverService.WellKnownUris? urls = null; - if(proxy is null) + if (proxy is null) urls = await new HomeserverResolverService().ResolveHomeserverFromWellKnown(serverName); instance.ClientHttpClient = new() { @@ -78,7 +79,11 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke } public virtual async Task<string> UploadFile(string fileName, Stream fileStream, string contentType = "application/octet-stream") { - var res = await ClientHttpClient.PostAsync($"/_matrix/media/v3/upload?filename={fileName}", new StreamContent(fileStream)); + var req = new HttpRequestMessage(HttpMethod.Post, $"/_matrix/media/v3/upload?filename={fileName}"); + req.Content = new StreamContent(fileStream); + req.Content.Headers.ContentType = new MediaTypeHeaderValue(contentType); + var res = await ClientHttpClient.SendAsync(req); + if (!res.IsSuccessStatusCode) { Console.WriteLine($"Failed to upload file: {await res.Content.ReadAsStringAsync()}"); throw new InvalidDataException($"Failed to upload file: {await res.Content.ReadAsStringAsync()}"); @@ -283,6 +288,17 @@ public class AuthenticatedHomeserverGeneric(string serverName, string accessToke } } + public async Task<RoomIdResponse> JoinRoomAsync(string roomId, List<string> homeservers = null, string? reason = null) { + var join_url = $"/_matrix/client/v3/join/{HttpUtility.UrlEncode(roomId)}"; + Console.WriteLine($"Calling {join_url} with {homeservers?.Count ?? 0} via's..."); + if (homeservers == null || homeservers.Count == 0) homeservers = new() { roomId.Split(':')[1] }; + var fullJoinUrl = $"{join_url}?server_name=" + string.Join("&server_name=", homeservers); + var res = await ClientHttpClient.PostAsJsonAsync(fullJoinUrl, new { + reason + }); + return await res.Content.ReadFromJsonAsync<RoomIdResponse>() ?? throw new Exception("Failed to join room?"); + } + #region Room Profile Utility private async Task<KeyValuePair<string, RoomMemberEventContent>> GetOwnRoomProfileWithIdAsync(GenericRoom room) { diff --git a/LibMatrix/Interfaces/EventContent.cs b/LibMatrix/Interfaces/EventContent.cs index ec09c7e..1fb6974 100644 --- a/LibMatrix/Interfaces/EventContent.cs +++ b/LibMatrix/Interfaces/EventContent.cs @@ -1,21 +1,42 @@ +using System.Text.Json; +using System.Text.Json.Nodes; using System.Text.Json.Serialization; namespace LibMatrix.Interfaces; -public abstract class EventContent { - -} +public abstract class EventContent { } + public abstract class TimelineEventContent : EventContent { [JsonPropertyName("m.relates_to")] public MessageRelatesTo? RelatesTo { get; set; } - // [JsonPropertyName("m.new_content")] - // public TimelineEventContent? NewContent { get; set; } + [JsonPropertyName("m.new_content")] + public JsonObject? NewContent { get; set; } + + public TimelineEventContent SetReplaceRelation(string eventId) { + NewContent = JsonSerializer.SerializeToNode(this, GetType()).AsObject(); + // NewContent = JsonSerializer.Deserialize(jsonText, GetType()); + RelatesTo = new() { + RelationType = "m.replace", + EventId = eventId + }; + return this; + } + + public T SetReplaceRelation<T>(string eventId) where T : TimelineEventContent { + return SetReplaceRelation(eventId) as T ?? throw new InvalidOperationException(); + } public class MessageRelatesTo { [JsonPropertyName("m.in_reply_to")] public EventInReplyTo? InReplyTo { get; set; } + [JsonPropertyName("event_id")] + public string? EventId { get; set; } + + [JsonPropertyName("rel_type")] + public string? RelationType { get; set; } + public class EventInReplyTo { [JsonPropertyName("event_id")] public string EventId { get; set; } @@ -24,4 +45,4 @@ public abstract class TimelineEventContent : EventContent { public string RelType { get; set; } } } -} +} \ No newline at end of file diff --git a/LibMatrix/LibMatrix.csproj b/LibMatrix/LibMatrix.csproj index bfb3069..690556f 100644 --- a/LibMatrix/LibMatrix.csproj +++ b/LibMatrix/LibMatrix.csproj @@ -11,8 +11,8 @@ </PropertyGroup> <ItemGroup> - <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0"/> - <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1"/> + <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" /> + <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" /> </ItemGroup> <ItemGroup> diff --git a/LibMatrix/Responses/CreateRoomRequest.cs b/LibMatrix/Responses/CreateRoomRequest.cs index 4267094..85db517 100644 --- a/LibMatrix/Responses/CreateRoomRequest.cs +++ b/LibMatrix/Responses/CreateRoomRequest.cs @@ -81,9 +81,48 @@ public class CreateRoomRequest { return errors; } + public static CreateRoomRequest CreatePublic(AuthenticatedHomeserverGeneric hs, string? name = null, string? roomAliasName = null) { + var request = new CreateRoomRequest { + Name = name ?? "New public Room", + Visibility = "public", + CreationContent = new(), + PowerLevelContentOverride = new() { + EventsDefault = 0, + UsersDefault = 0, + Kick = 50, + Ban = 50, + Invite = 25, + StateDefault = 10, + Redact = 50, + NotificationsPl = new() { + Room = 10 + }, + Events = new() { + { "m.room.avatar", 50 }, + { "m.room.canonical_alias", 50 }, + { "m.room.encryption", 100 }, + { "m.room.history_visibility", 100 }, + { "m.room.name", 50 }, + { "m.room.power_levels", 100 }, + { "m.room.server_acl", 100 }, + { "m.room.tombstone", 100 } + }, + Users = new() { + { + hs.UserId, + 101 + } + } + }, + RoomAliasName = roomAliasName, + InitialState = new() + }; + + return request; + } public static CreateRoomRequest CreatePrivate(AuthenticatedHomeserverGeneric hs, string? name = null, string? roomAliasName = null) { var request = new CreateRoomRequest { - Name = name ?? "Private Room", + Name = name ?? "New private Room", Visibility = "private", CreationContent = new(), PowerLevelContentOverride = new() { diff --git a/LibMatrix/RoomTypes/GenericRoom.cs b/LibMatrix/RoomTypes/GenericRoom.cs index b81713a..d26b1f8 100644 --- a/LibMatrix/RoomTypes/GenericRoom.cs +++ b/LibMatrix/RoomTypes/GenericRoom.cs @@ -29,13 +29,16 @@ public class GenericRoom { public string RoomId { get; set; } public async IAsyncEnumerable<StateEventResponse?> GetFullStateAsync() { - var result = _httpClient.GetAsyncEnumerableFromJsonAsync<StateEventResponse>( - $"/_matrix/client/v3/rooms/{RoomId}/state"); + var result = _httpClient.GetAsyncEnumerableFromJsonAsync<StateEventResponse>($"/_matrix/client/v3/rooms/{RoomId}/state"); await foreach (var resp in result) { yield return resp; } } + public async Task<List<StateEventResponse>> GetFullStateAsListAsync() { + return await _httpClient.GetFromJsonAsync<List<StateEventResponse>>($"/_matrix/client/v3/rooms/{RoomId}/state"); + } + public async Task<T?> GetStateAsync<T>(string type, string stateKey = "") { var url = $"/_matrix/client/v3/rooms/{RoomId}/state"; if (!string.IsNullOrEmpty(type)) url += $"/{type}"; @@ -77,17 +80,82 @@ public class GenericRoom { } } - public async Task<MessagesResponse> GetMessagesAsync(string from = "", int limit = 10, string dir = "b", - string filter = "") { - var url = $"/_matrix/client/v3/rooms/{RoomId}/messages?from={from}&limit={limit}&dir={dir}"; - if (!string.IsNullOrEmpty(filter)) url += $"&filter={filter}"; + public async Task<MessagesResponse> GetMessagesAsync(string from = "", int? limit = null, string dir = "b", string filter = "") { + var url = $"/_matrix/client/v3/rooms/{RoomId}/messages?dir={dir}"; + if (!string.IsNullOrWhiteSpace(from)) url += $"&from={from}"; + if (limit is not null) url += $"&limit={limit}"; + if (!string.IsNullOrWhiteSpace(filter)) url += $"&filter={filter}"; var res = await _httpClient.GetFromJsonAsync<MessagesResponse>(url); return res ?? new MessagesResponse(); } + /// <summary> + /// Same as <see cref="GetMessagesAsync"/>, except keeps fetching more responses until the beginning of the room is found, or the target message limit is reached + /// </summary> + public async IAsyncEnumerable<MessagesResponse> GetManyMessagesAsync(string from = "", int limit = 100, string dir = "b", string filter = "", bool includeState = true, + bool fixForward = false) { + if (dir == "f" && fixForward) { + var concat = new List<MessagesResponse>(); + while (true) { + var resp = await GetMessagesAsync(from, int.MaxValue, "b", filter); + concat.Add(resp); + if (!includeState) + resp.State.Clear(); + from = resp.End; + if (resp.End is null) break; + } + + concat.Reverse(); + foreach (var eventResponse in concat) { + limit -= eventResponse.State.Count + eventResponse.Chunk.Count; + while (limit < 0) { + if (eventResponse.State.Count > 0 && eventResponse.State.Max(x => x.OriginServerTs) > eventResponse.Chunk.Max(x => x.OriginServerTs)) + eventResponse.State.Remove(eventResponse.State.MaxBy(x => x.OriginServerTs)); + else + eventResponse.Chunk.Remove(eventResponse.Chunk.MaxBy(x => x.OriginServerTs)); + + limit++; + } + + eventResponse.Chunk.Reverse(); + eventResponse.State.Reverse(); + yield return eventResponse; + if (limit <= 0) yield break; + } + } + else { + while (limit > 0) { + var resp = await GetMessagesAsync(from, limit, dir, filter); + + if (!includeState) + resp.State.Clear(); + + limit -= resp.Chunk.Count + resp.State.Count; + from = resp.End; + yield return resp; + if (resp.End is null) { + Console.WriteLine("End is null"); + yield break; + } + } + } + + Console.WriteLine("End of GetManyAsync"); + } + public async Task<string?> GetNameAsync() => (await GetStateAsync<RoomNameEventContent>("m.room.name"))?.Name; - public async Task<RoomIdResponse> JoinAsync(string[]? homeservers = null, string? reason = null) { + public async Task<RoomIdResponse> JoinAsync(string[]? homeservers = null, string? reason = null, bool checkIfAlreadyMember = true) { + if (checkIfAlreadyMember) { + try { + var ce = await GetCreateEventAsync(); + return new() { + RoomId = RoomId + }; + } + catch { } //ignore + } + var join_url = $"/_matrix/client/v3/join/{HttpUtility.UrlEncode(RoomId)}"; Console.WriteLine($"Calling {join_url} with {homeservers?.Length ?? 0} via's..."); if (homeservers == null || homeservers.Length == 0) homeservers = new[] { RoomId.Split(':')[1] }; @@ -235,13 +303,17 @@ public class GenericRoom { return await res.Content.ReadFromJsonAsync<EventIdResponse>(); } - public async Task<EventIdResponse?> SendFileAsync(string fileName, Stream fileStream, string messageType = "m.file") { + public async Task<EventIdResponse?> SendFileAsync(string fileName, Stream fileStream, string messageType = "m.file", string contentType = "application/octet-stream") { var url = await Homeserver.UploadFile(fileName, fileStream); var content = new RoomMessageEventContent() { MessageType = messageType, Url = url, Body = fileName, FileName = fileName, + FileInfo = new() { + Size = fileStream.Length, + MimeType = contentType + } }; return await SendTimelineEventAsync("m.room.message", content); } diff --git a/LibMatrix/Services/HomeserverResolverService.cs b/LibMatrix/Services/HomeserverResolverService.cs index c8b6bb7..556bf86 100644 --- a/LibMatrix/Services/HomeserverResolverService.cs +++ b/LibMatrix/Services/HomeserverResolverService.cs @@ -7,7 +7,9 @@ using Microsoft.Extensions.Logging; namespace LibMatrix.Services; public class HomeserverResolverService(ILogger<HomeserverResolverService>? logger = null) { - private readonly MatrixHttpClient _httpClient = new(); + private readonly MatrixHttpClient _httpClient = new() { + Timeout = TimeSpan.FromMilliseconds(10000) + }; private static readonly ConcurrentDictionary<string, WellKnownUris> _wellKnownCache = new(); private static readonly ConcurrentDictionary<string, SemaphoreSlim> _wellKnownSemaphores = new(); @@ -20,7 +22,7 @@ public class HomeserverResolverService(ILogger<HomeserverResolverService>? logge _wellKnownSemaphores[homeserver].Release(); return known; } - + logger?.LogInformation("Resolving homeserver: {}", homeserver); var res = new WellKnownUris { Client = await _tryResolveFromClientWellknown(homeserver), @@ -33,11 +35,12 @@ public class HomeserverResolverService(ILogger<HomeserverResolverService>? logge private async Task<string?> _tryResolveFromClientWellknown(string homeserver) { if (!homeserver.StartsWith("http")) homeserver = "https://" + homeserver; - if (await _httpClient.CheckSuccessStatus($"{homeserver}/.well-known/matrix/client")) { + try { var resp = await _httpClient.GetFromJsonAsync<JsonElement>($"{homeserver}/.well-known/matrix/client"); var hs = resp.GetProperty("m.homeserver").GetProperty("base_url").GetString(); return hs; } + catch { } logger?.LogInformation("No client well-known..."); return null; @@ -45,13 +48,14 @@ public class HomeserverResolverService(ILogger<HomeserverResolverService>? logge private async Task<string?> _tryResolveFromServerWellknown(string homeserver) { if (!homeserver.StartsWith("http")) homeserver = "https://" + homeserver; - if (await _httpClient.CheckSuccessStatus($"{homeserver}/.well-known/matrix/server")) { + try { var resp = await _httpClient.GetFromJsonAsync<JsonElement>($"{homeserver}/.well-known/matrix/server"); var hs = resp.GetProperty("m.server").GetString(); if (!hs.StartsWithAnyOf("http://", "https://")) hs = $"https://{hs}"; return hs; } + catch { } // fallback: most servers host these on the same location var clientUrl = await _tryResolveFromClientWellknown(homeserver); @@ -74,4 +78,4 @@ public class HomeserverResolverService(ILogger<HomeserverResolverService>? logge public string? Client { get; set; } public string? Server { get; set; } } -} +} \ No newline at end of file diff --git a/Tests/LibMatrix.Tests/LibMatrix.Tests.csproj b/Tests/LibMatrix.Tests/LibMatrix.Tests.csproj index a0b1421..3f7bc87 100644 --- a/Tests/LibMatrix.Tests/LibMatrix.Tests.csproj +++ b/Tests/LibMatrix.Tests/LibMatrix.Tests.csproj @@ -10,12 +10,12 @@ </PropertyGroup> <ItemGroup> - <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" /> + <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" /> - <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.6.0"/> - <PackageReference Include="xunit" Version="2.5.0" /> - <PackageReference Include="Xunit.Microsoft.DependencyInjection" Version="7.0.6"/> - <PackageReference Include="xunit.runner.visualstudio" Version="2.4.5"> + <PackageReference Include="Microsoft.NET.Test.Sdk" Version="17.9.0-preview-23531-01" /> + <PackageReference Include="xunit" Version="2.6.1" /> + <PackageReference Include="Xunit.Microsoft.DependencyInjection" Version="7.0.10" /> + <PackageReference Include="xunit.runner.visualstudio" Version="2.5.3"> <IncludeAssets>runtime; build; native; contentfiles; analyzers; buildtransitive</IncludeAssets> <PrivateAssets>all</PrivateAssets> </PackageReference> diff --git a/Tests/TestDataGenerator/TestDataGenerator.csproj b/Tests/TestDataGenerator/TestDataGenerator.csproj index 9e0d6b1..cf212de 100644 --- a/Tests/TestDataGenerator/TestDataGenerator.csproj +++ b/Tests/TestDataGenerator/TestDataGenerator.csproj @@ -17,7 +17,7 @@ </PropertyGroup> <ItemGroup> - <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" /> + <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" /> </ItemGroup> <ItemGroup> <Content Include="appsettings*.json"> diff --git a/Utilities/LibMatrix.DebugDataValidationApi/LibMatrix.DebugDataValidationApi.csproj b/Utilities/LibMatrix.DebugDataValidationApi/LibMatrix.DebugDataValidationApi.csproj index f285b59..e14b7a1 100644 --- a/Utilities/LibMatrix.DebugDataValidationApi/LibMatrix.DebugDataValidationApi.csproj +++ b/Utilities/LibMatrix.DebugDataValidationApi/LibMatrix.DebugDataValidationApi.csproj @@ -9,7 +9,7 @@ </PropertyGroup> <ItemGroup> - <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="7.0.10" /> + <PackageReference Include="Microsoft.AspNetCore.OpenApi" Version="8.0.0" /> <PackageReference Include="Swashbuckle.AspNetCore" Version="6.5.0" /> </ItemGroup> diff --git a/Utilities/LibMatrix.Utilities.Bot/LibMatrix.Utilities.Bot.csproj b/Utilities/LibMatrix.Utilities.Bot/LibMatrix.Utilities.Bot.csproj index dd248b9..e16c00c 100644 --- a/Utilities/LibMatrix.Utilities.Bot/LibMatrix.Utilities.Bot.csproj +++ b/Utilities/LibMatrix.Utilities.Bot/LibMatrix.Utilities.Bot.csproj @@ -12,9 +12,9 @@ </ItemGroup> <ItemGroup> - <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="7.0.0" /> - <PackageReference Include="Microsoft.Extensions.Hosting" Version="7.0.1" /> - <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="7.0.1" /> + <PackageReference Include="Microsoft.Extensions.DependencyInjection.Abstractions" Version="8.0.0" /> + <PackageReference Include="Microsoft.Extensions.Hosting" Version="8.0.0" /> + <PackageReference Include="Microsoft.Extensions.Logging.Abstractions" Version="8.0.0" /> </ItemGroup> diff --git a/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs b/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs index 6d2cba2..9949631 100644 --- a/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs +++ b/Utilities/LibMatrix.Utilities.Bot/Services/CommandListenerHostedService.cs @@ -37,7 +37,9 @@ public class CommandListenerHostedService : IHostedService { private async Task? Run(CancellationToken cancellationToken) { _logger.LogInformation("Starting command listener!"); - var syncHelper = new SyncHelper(_hs, _logger); + var syncHelper = new SyncHelper(_hs, _logger) { + Timeout = 300_000 + }; syncHelper.TimelineEventHandlers.Add(async @event => { try { var room = _hs.GetRoom(@event.RoomId); diff --git a/release.sh b/release.sh new file mode 100755 index 0000000..53bd050 --- /dev/null +++ b/release.sh @@ -0,0 +1,3 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i "bash -x" -p bash git dotnet-sdk_8 nix curl jq yq +dotnet test \ No newline at end of file |