Added a lot of utilities

24 files changed, 1050 insertions, 271 deletions

diff --git a/MxApiExtensions.Classes.LibMatrix/MxApiExtensions.Classes.LibMatrix.csproj b/MxApiExtensions.Classes.LibMatrix/MxApiExtensions.Classes.LibMatrix.csproj
new file mode 100644
index 0000000..db9f354
--- /dev/null
+++ b/MxApiExtensions.Classes.LibMatrix/MxApiExtensions.Classes.LibMatrix.csproj
@@ -0,0 +1,14 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+    <ItemGroup>
+      <ProjectReference Include="..\..\LibMatrix\LibMatrix\LibMatrix.csproj" />
+      <ProjectReference Include="..\MxApiExtensions.Classes\MxApiExtensions.Classes.csproj" />
+    </ItemGroup>
+
+</Project>
diff --git a/MxApiExtensions.Classes.LibMatrix/MxApiMatrixException.cs b/MxApiExtensions.Classes.LibMatrix/MxApiMatrixException.cs
new file mode 100644
index 0000000..e5f434a
--- /dev/null
+++ b/MxApiExtensions.Classes.LibMatrix/MxApiMatrixException.cs
@@ -0,0 +1,48 @@
+using ArcaneLibs.Extensions;
+using LibMatrix;
+
+namespace MxApiExtensions;
+
+public class MxApiMatrixException : MatrixException {
+    public string? GetAsJson() => new { ErrorCode, Error, SoftLogout, RetryAfterMs }.ToJson(ignoreNull: true);
+    public override string Message =>
+        base.Message.StartsWith("Unknown error: ")
+            ? $"{ErrorCode}: {ErrorCode switch {
+                // common
+                "M_FORBIDDEN" => $"You do not have permission to perform this action: {Error}",
+                "M_UNKNOWN_TOKEN" => $"The access token specified was not recognised: {Error}{(SoftLogout == true ? " (soft logout)" : "")}",
+                "M_MISSING_TOKEN" => $"No access token was specified: {Error}",
+                "M_BAD_JSON" => $"Request contained valid JSON, but it was malformed in some way: {Error}",
+                "M_NOT_JSON" => $"Request did not contain valid JSON: {Error}",
+                "M_NOT_FOUND" => $"The requested resource was not found: {Error}",
+                "M_LIMIT_EXCEEDED" => $"Too many requests have been sent in a short period of time. Wait a while then try again: {Error}",
+                "M_UNRECOGNISED" => $"The server did not recognise the request: {Error}",
+                "M_UNKOWN" => $"The server encountered an unexpected error: {Error}",
+                // endpoint specific
+                "M_UNAUTHORIZED" => $"The request did not contain valid authentication information for the target of the request: {Error}",
+                "M_USER_DEACTIVATED" => $"The user ID associated with the request has been deactivated: {Error}",
+                "M_USER_IN_USE" => $"The user ID associated with the request is already in use: {Error}",
+                "M_INVALID_USERNAME" => $"The requested user ID is not valid: {Error}",
+                "M_ROOM_IN_USE" => $"The room alias requested is already taken: {Error}",
+                "M_INVALID_ROOM_STATE" => $"The room associated with the request is not in a valid state to perform the request: {Error}",
+                "M_THREEPID_IN_USE" => $"The threepid requested is already associated with a user ID on this server: {Error}",
+                "M_THREEPID_NOT_FOUND" => $"The threepid requested is not associated with any user ID: {Error}",
+                "M_THREEPID_AUTH_FAILED" => $"The provided threepid and/or token was invalid: {Error}",
+                "M_THREEPID_DENIED" => $"The homeserver does not permit the third party identifier in question: {Error}",
+                "M_SERVER_NOT_TRUSTED" => $"The homeserver does not trust the identity server: {Error}",
+                "M_UNSUPPORTED_ROOM_VERSION" => $"The room version is not supported: {Error}",
+                "M_INCOMPATIBLE_ROOM_VERSION" => $"The room version is incompatible: {Error}",
+                "M_BAD_STATE" => $"The request was invalid because the state was invalid: {Error}",
+                "M_GUEST_ACCESS_FORBIDDEN" => $"Guest access is forbidden: {Error}",
+                "M_CAPTCHA_NEEDED" => $"Captcha needed: {Error}",
+                "M_CAPTCHA_INVALID" => $"Captcha invalid: {Error}",
+                "M_MISSING_PARAM" => $"Missing parameter: {Error}",
+                "M_INVALID_PARAM" => $"Invalid parameter: {Error}",
+                "M_TOO_LARGE" => $"The request or entity was too large: {Error}",
+                "M_EXCLUSIVE" => $"The resource being requested is reserved by an application service, or the application service making the request has not created the resource: {Error}",
+                "M_RESOURCE_LIMIT_EXCEEDED" => $"Exceeded resource limit: {Error}",
+                "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" => $"Cannot leave server notice room: {Error}",
+                _ => $"Unknown error: {new { ErrorCode, Error, SoftLogout, RetryAfterMs }.ToJson(ignoreNull: true)}"
+            }}"
+            : base.Message;
+}
diff --git a/MxApiExtensions.Classes.LibMatrix/RoomInfoEntry.cs b/MxApiExtensions.Classes.LibMatrix/RoomInfoEntry.cs
new file mode 100644
index 0000000..6ea188e
--- /dev/null
+++ b/MxApiExtensions.Classes.LibMatrix/RoomInfoEntry.cs
@@ -0,0 +1,20 @@
+using System.Text.Json.Serialization;
+using LibMatrix.Responses;
+
+namespace LibMatrix.MxApiExtensions;
+
+/// <summary>
+/// Generic room info, this will most likely be out of date due to caching!
+/// This is only useful for giving a rough idea of the room state.
+/// </summary>
+public class RoomInfoEntry {
+    public string RoomId { get; set; }
+    public List<StateEventResponse?> RoomState { get; set; }
+
+    public int StateCount { get; set; }
+
+    public Dictionary<string, int> MemberCounts { get; set; } = new();
+
+    // [JsonIgnore]
+    public DateTime ExpiresAt { get; set; } = DateTime.Now.AddMinutes(1);
+}
diff --git a/MxApiExtensions.Classes/MxApiExtensions.Classes.csproj b/MxApiExtensions.Classes/MxApiExtensions.Classes.csproj
new file mode 100644
index 0000000..6836c68
--- /dev/null
+++ b/MxApiExtensions.Classes/MxApiExtensions.Classes.csproj
@@ -0,0 +1,9 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+    <PropertyGroup>
+        <TargetFramework>net7.0</TargetFramework>
+        <ImplicitUsings>enable</ImplicitUsings>
+        <Nullable>enable</Nullable>
+    </PropertyGroup>
+
+</Project>
diff --git a/MxApiExtensions/Auth.cs b/MxApiExtensions/Auth.cs
deleted file mode 100644
index cc60a99..0000000
--- a/MxApiExtensions/Auth.cs
+++ /dev/null
@@ -1,82 +0,0 @@
-using System.Net.Http.Headers;
-using System.Text.Json;
-using MatrixRoomUtils.Core.Extensions;
-
-namespace MxApiExtensions;
-
-public class Auth {
-    private readonly ILogger<Auth> _logger;
-    private readonly CacheConfiguration _config;
-    private readonly HttpRequest _request;
-
-    private static Dictionary<string, string> _tokenMap = new();
-
-    public Auth(ILogger<Auth> logger, CacheConfiguration config, IHttpContextAccessor request) {
-        _logger = logger;
-        _config = config;
-        _request = request.HttpContext.Request;
-    }
-
-    internal string? GetToken(bool fail = true) {
-        string? token;
-        if (_request.Headers.TryGetValue("Authorization", out var tokens)) {
-            token = tokens.FirstOrDefault()?[7..];
-        }
-        else {
-            token = _request.Query["access_token"];
-        }
-
-        if (token == null && fail) {
-            throw new MatrixException {
-                ErrorCode = "M_MISSING_TOKEN",
-                Error = "Missing access token"
-            };
-        }
-
-        return token;
-    }
-
-    public string GetUserId(bool fail = true) {
-        var token = GetToken(fail);
-        if (token == null) {
-            if(fail) {
-                throw new MatrixException {
-                    ErrorCode = "M_MISSING_TOKEN",
-                    Error = "Missing access token"
-                };
-            }
-            return "@anonymous:*";
-        }
-        try {
-            return _tokenMap.GetOrCreate(token, GetMxidFromToken);
-        }
-        catch {
-            return GetUserId();
-        }
-    }
-
-    private string GetMxidFromToken(string token) {
-        using var hc = new HttpClient();
-        hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
-        var resp = hc.GetAsync($"{_config.Homeserver}/_matrix/client/v3/account/whoami").Result;
-        if (!resp.IsSuccessStatusCode) {
-            throw new MatrixException {
-                ErrorCode = "M_UNKNOWN",
-                Error = "[Rory&::MxSyncCache] Whoami request failed"
-            };
-        }
-
-        if (resp.Content is null) {
-            throw new MatrixException {
-                ErrorCode = "M_UNKNOWN",
-                Error = "No content in response"
-            };
-        }
-
-        var json = JsonDocument.Parse(resp.Content.ReadAsStream()).RootElement;
-        var mxid = json.GetProperty("user_id").GetString()!;
-        _logger.LogInformation("Got mxid {} from token {}", mxid, token);
-        return mxid;
-    }
-
-}
diff --git a/MxApiExtensions/CacheConfiguration.cs b/MxApiExtensions/CacheConfiguration.cs
deleted file mode 100644
index f1da404..0000000
--- a/MxApiExtensions/CacheConfiguration.cs
+++ /dev/null
@@ -1,9 +0,0 @@
-namespace MxApiExtensions;
-
-public class CacheConfiguration {
-    public CacheConfiguration(IConfiguration config) {
-        config.GetRequiredSection("MxSyncCache").Bind(this);
-    }
-
-    public string Homeserver { get; set; } = "";
-}
diff --git a/MxApiExtensions/Classes/SyncState.cs b/MxApiExtensions/Classes/SyncState.cs
new file mode 100644
index 0000000..7f07894
--- /dev/null
+++ b/MxApiExtensions/Classes/SyncState.cs
@@ -0,0 +1,14 @@
+using System.Collections.Concurrent;
+using LibMatrix.Helpers;
+using LibMatrix.Homeservers;
+
+namespace MxApiExtensions.Classes;
+
+public class SyncState {
+    public string? NextBatch { get; set; }
+    public ConcurrentQueue<SyncResult> SyncQueue { get; set; } = new();
+    public bool IsInitialSync { get; set; }
+    public Task? NextSyncResult { get; set; }
+    public DateTime NextSyncResultStartedAt { get; set; } = DateTime.Now;
+    public AuthenticatedHomeserverGeneric Homeserver { get; set; }
+}
diff --git a/MxApiExtensions/Controllers/ClientVersionsController.cs b/MxApiExtensions/Controllers/ClientVersionsController.cs
new file mode 100644
index 0000000..60a3364
--- /dev/null
+++ b/MxApiExtensions/Controllers/ClientVersionsController.cs
@@ -0,0 +1,52 @@
+using System.Net.Http.Headers;
+using LibMatrix.Responses;
+using Microsoft.AspNetCore.Mvc;
+using MxApiExtensions.Services;
+
+namespace MxApiExtensions.Controllers;
+
+[ApiController]
+[Route("/")]
+public class ClientVersionsController : ControllerBase {
+    private readonly ILogger _logger;
+    private readonly AuthenticatedHomeserverProviderService _authenticatedHomeserverProviderService;
+    private static Dictionary<string, string> _tokenMap = new();
+
+    public ClientVersionsController(ILogger<ClientVersionsController> logger, MxApiExtensionsConfiguration config, AuthenticationService authenticationService, AuthenticatedHomeserverProviderService authenticatedHomeserverProviderService) {
+        _logger = logger;
+        _authenticatedHomeserverProviderService = authenticatedHomeserverProviderService;
+    }
+
+    [HttpGet("/_matrix/client/versions")]
+    public async Task<ClientVersionsResponse> Proxy([FromQuery] string? access_token, string? _) {
+        var clientVersions = new ClientVersionsResponse() {
+            Versions = new() {
+                "r0.0.1",
+                "r0.1.0",
+                "r0.2.0",
+                "r0.3.0",
+                "r0.4.0",
+                "r0.5.0",
+                "r0.6.0",
+                "r0.6.1",
+                "v1.1",
+                "v1.2",
+                "v1.3",
+                "v1.4",
+                "v1.5",
+                "v1.6"
+            },
+            UnstableFeatures = new()
+        };
+        try {
+            var hs = await _authenticatedHomeserverProviderService.GetHomeserver();
+            clientVersions = await hs.GetClientVersions();
+
+            _logger.LogInformation("Fetching client versions for {}: {}{}", hs.WhoAmI.UserId, Request.Path, Request.QueryString);
+        }
+        catch { }
+
+        clientVersions.UnstableFeatures.Add("gay.rory.mxapiextensions.v0", true);
+        return clientVersions;
+    }
+}
diff --git a/MxApiExtensions/Controllers/Extensions/JoinedRoomListController.cs b/MxApiExtensions/Controllers/Extensions/JoinedRoomListController.cs
new file mode 100644
index 0000000..3c4161d
--- /dev/null
+++ b/MxApiExtensions/Controllers/Extensions/JoinedRoomListController.cs
@@ -0,0 +1,144 @@
+using System.Collections.Concurrent;
+using System.Net.Http.Headers;
+using ArcaneLibs.Extensions;
+using LibMatrix.Homeservers;
+using LibMatrix.MxApiExtensions;
+using LibMatrix.RoomTypes;
+using LibMatrix.StateEventTypes.Spec;
+using Microsoft.AspNetCore.Mvc;
+using MxApiExtensions.Services;
+
+namespace MxApiExtensions.Controllers.Extensions;
+
+[ApiController]
+[Route("/_matrix/client/unstable/gay.rory.mxapiextensions")]
+public class JoinedRoomListController : ControllerBase {
+    private static ILogger _logger;
+    private static MxApiExtensionsConfiguration _config;
+    private readonly AuthenticationService _authenticationService;
+    private readonly AuthenticatedHomeserverProviderService _authenticatedHomeserverProviderService;
+
+    private static ConcurrentDictionary<string, RoomInfoEntry> _roomInfoCache = new();
+
+    public JoinedRoomListController(ILogger<JoinedRoomListController> logger, MxApiExtensionsConfiguration config, AuthenticationService authenticationService,
+        AuthenticatedHomeserverProviderService authenticatedHomeserverProviderService) {
+        _logger = logger;
+        _config = config;
+        _authenticationService = authenticationService;
+        _authenticatedHomeserverProviderService = authenticatedHomeserverProviderService;
+    }
+
+    [HttpGet("joined_rooms_with_info")]
+    public async IAsyncEnumerable<RoomInfoEntry> GetJoinedRooms([FromQuery] string? access_token) {
+        List<GenericRoom> rooms = new();
+        AuthenticatedHomeserverGeneric? hs = null;
+        try {
+            hs = await _authenticatedHomeserverProviderService.GetHomeserver();
+            _logger.LogInformation("Got room list with info request for {user} ({hs})", hs.UserId, hs.FullHomeServerDomain);
+            rooms = await hs.GetJoinedRooms();
+        }
+        catch (MxApiMatrixException e) {
+            _logger.LogError(e, "Matrix error");
+            Response.StatusCode = StatusCodes.Status500InternalServerError;
+            Response.ContentType = "application/json";
+
+            await Response.WriteAsJsonAsync(e.GetAsJson());
+            await Response.CompleteAsync();
+        }
+        catch (Exception e) {
+            _logger.LogError(e, "Unhandled error");
+            Response.StatusCode = StatusCodes.Status500InternalServerError;
+            Response.ContentType = "text/plain";
+
+            await Response.WriteAsJsonAsync(e.ToString());
+            await Response.CompleteAsync();
+        }
+
+        if (hs is not null) {
+            Response.ContentType = "application/json";
+            Response.Headers.Add("Cache-Control", "public, max-age=60");
+            Response.Headers.Add("Expires", DateTime.Now.AddMinutes(1).ToString("R"));
+            Response.Headers.Add("Last-Modified", DateTime.Now.ToString("R"));
+            Response.Headers.Add("X-Matrix-Server", hs.FullHomeServerDomain);
+            Response.Headers.Add("X-Matrix-User", hs.UserId);
+            // await Response.StartAsync();
+
+            var cachedRooms = _roomInfoCache
+                .Where(cr => rooms.Any(r => r.RoomId == cr.Key) && cr.Value.ExpiresAt > DateTime.Now)
+                .ToList();
+            rooms.RemoveAll(r => cachedRooms.Any(cr => cr.Key == r.RoomId));
+
+            foreach (var room in cachedRooms) {
+                yield return room.Value;
+                _logger.LogInformation("Sent cached room info for {room} for {user} ({hs})", room.Key, hs.UserId, hs.FullHomeServerDomain);
+            }
+
+            var tasks = rooms.Select(r => GetRoomInfo(hs, r.RoomId)).ToAsyncEnumerable();
+
+            await foreach (var result in tasks) {
+                yield return result;
+                _logger.LogInformation("Sent room info for {room} for {user} ({hs})", result.RoomId, hs.UserId, hs.FullHomeServerDomain);
+            }
+        }
+    }
+
+    private SemaphoreSlim _roomInfoSemaphore = new(100, 100);
+
+    private async Task<RoomInfoEntry> GetRoomInfo(AuthenticatedHomeserverGeneric hs, string roomId) {
+        _logger.LogInformation("Getting room info for {room} for {user} ({hs})", roomId, hs.UserId, hs.FullHomeServerDomain);
+        var room = await hs.GetRoom(roomId);
+        var state = room.GetFullStateAsync();
+        var result = new RoomInfoEntry {
+            RoomId = roomId,
+            RoomState = new(),
+            MemberCounts = new(),
+            StateCount = 0,
+            ExpiresAt = DateTime.Now.AddMinutes(5)
+        };
+
+        await foreach (var @event in state) {
+            // result.ExpiresAt = result.ExpiresAt.AddMilliseconds(100);
+            result.StateCount++;
+            if (@event.Type != "m.room.member") result.RoomState.Add(@event);
+            else {
+                if(!result.MemberCounts.ContainsKey((@event.TypedContent as RoomMemberEventData)?.Membership)) result.MemberCounts.Add((@event.TypedContent as RoomMemberEventData)?.Membership, 0);
+                    result.MemberCounts[(@event.TypedContent as RoomMemberEventData)?.Membership]++;
+            }
+        }
+
+        result.ExpiresAt = result.ExpiresAt.AddMilliseconds(100 * result.StateCount);
+
+        _logger.LogInformation("Got room info for {room} for {user} ({hs})", roomId, hs.UserId, hs.FullHomeServerDomain);
+        while (!_roomInfoCache.TryAdd(roomId, result)) {
+            _logger.LogWarning("Failed to add room info for {room} to cache, retrying...", roomId);
+            await Task.Delay(100);
+            if (_roomInfoCache.ContainsKey(roomId)) break;
+        }
+
+        return result;
+    }
+
+    [HttpGet("joined_rooms_with_info_cache")]
+    public async Task<object> GetRoomInfoCache() {
+        var mxid = await _authenticationService.GetMxidFromToken();
+        if(!_config.Admins.Contains(mxid)) {
+            Response.StatusCode = StatusCodes.Status403Forbidden;
+            Response.ContentType = "application/json";
+
+            await Response.WriteAsJsonAsync(new {
+                ErrorCode = "M_FORBIDDEN",
+                Error = "You are not an admin"
+            });
+            await Response.CompleteAsync();
+            return null;
+        }
+
+        return _roomInfoCache.Select(x => new {
+            x.Key,
+            x.Value.ExpiresAt,
+            ExpiresIn = x.Value.ExpiresAt - DateTime.Now,
+            x.Value.MemberCounts,
+            x.Value.StateCount
+        }).OrderByDescending(x => x.ExpiresAt);
+    }
+}
diff --git a/MxApiExtensions/Controllers/Extensions/ProxyConfigurationController.cs b/MxApiExtensions/Controllers/Extensions/ProxyConfigurationController.cs
new file mode 100644
index 0000000..71bf167
--- /dev/null
+++ b/MxApiExtensions/Controllers/Extensions/ProxyConfigurationController.cs
@@ -0,0 +1,43 @@
+using System.Collections.Concurrent;
+using LibMatrix.MxApiExtensions;
+using Microsoft.AspNetCore.Mvc;
+using MxApiExtensions.Services;
+
+namespace MxApiExtensions.Controllers.Extensions;
+
+[ApiController]
+[Route("/_matrix/client/unstable/gay.rory.mxapiextensions")]
+public class ProxyConfigurationController : ControllerBase {
+    private readonly ILogger _logger;
+    private readonly MxApiExtensionsConfiguration _config;
+    private readonly AuthenticationService _authenticationService;
+
+    private static ConcurrentDictionary<string, RoomInfoEntry> _roomInfoCache = new();
+
+    public ProxyConfigurationController(ILogger<ProxyConfigurationController> logger, MxApiExtensionsConfiguration config, AuthenticationService authenticationService,
+        AuthenticatedHomeserverProviderService authenticatedHomeserverProviderService) {
+        _logger = logger;
+        _config = config;
+        _authenticationService = authenticationService;
+    }
+
+    [HttpGet("proxy_config")]
+    public async Task<MxApiExtensionsConfiguration> GetConfig() {
+        var mxid = await _authenticationService.GetMxidFromToken();
+        if(!_config.Admins.Contains(mxid)) {
+            _logger.LogWarning("Got proxy config request for {user}, but they are not an admin", mxid);
+            Response.StatusCode = StatusCodes.Status403Forbidden;
+            Response.ContentType = "application/json";
+
+            await Response.WriteAsJsonAsync(new {
+                ErrorCode = "M_FORBIDDEN",
+                Error = "You are not an admin"
+            });
+            await Response.CompleteAsync();
+            return null;
+        }
+
+        _logger.LogInformation("Got proxy config request for {user}", mxid);
+        return _config;
+    }
+}
diff --git a/MxApiExtensions/Controllers/GenericProxyController.cs b/MxApiExtensions/Controllers/GenericProxyController.cs
index f0ad4e7..4e27b4a 100644
--- a/MxApiExtensions/Controllers/GenericProxyController.cs
+++ b/MxApiExtensions/Controllers/GenericProxyController.cs
@@ -1,5 +1,6 @@
 using System.Net.Http.Headers;
 using Microsoft.AspNetCore.Mvc;
+using MxApiExtensions.Services;
 
 namespace MxApiExtensions.Controllers;
 
@@ -7,21 +8,75 @@ namespace MxApiExtensions.Controllers;
 [Route("/{*_}")]
 public class GenericController : ControllerBase {
     private readonly ILogger<GenericController> _logger;
-    private readonly CacheConfiguration _config;
-    private readonly Auth _auth;
+    private readonly MxApiExtensionsConfiguration _config;
+    private readonly AuthenticationService _authenticationService;
+    private readonly AuthenticatedHomeserverProviderService _authenticatedHomeserverProviderService;
     private static Dictionary<string, string> _tokenMap = new();
 
-    public GenericController(ILogger<GenericController> logger, CacheConfiguration config, Auth auth) {
+    public GenericController(ILogger<GenericController> logger, MxApiExtensionsConfiguration config, AuthenticationService authenticationService,
+        AuthenticatedHomeserverProviderService authenticatedHomeserverProviderService) {
         _logger = logger;
         _config = config;
-        _auth = auth;
+        _authenticationService = authenticationService;
+        _authenticatedHomeserverProviderService = authenticatedHomeserverProviderService;
     }
 
     [HttpGet]
-    public async Task Proxy([FromQuery] string? access_token, string _) {
+    public async Task Proxy([FromQuery] string? access_token, string? _) {
         try {
-            access_token ??= _auth.GetToken(fail: false);
-            var mxid = _auth.GetUserId(fail: false);
+            access_token ??= _authenticationService.GetToken(fail: false);
+            var mxid = await _authenticationService.GetMxidFromToken(fail: false);
+            var hs = await _authenticatedHomeserverProviderService.GetHomeserver();
+
+            _logger.LogInformation("Proxying request for {}: {}{}", mxid, Request.Path, Request.QueryString);
+
+            //remove access_token from query string
+            Request.QueryString = new QueryString(
+                Request.QueryString.Value?.Replace("&access_token", "access_token")
+                    .Replace($"access_token={access_token}", "")
+            );
+
+            var resp = await hs._httpClient.GetAsync($"{Request.Path}{Request.QueryString}");
+
+            if (resp.Content is null) {
+                throw new MxApiMatrixException {
+                    ErrorCode = "M_UNKNOWN",
+                    Error = "No content in response"
+                };
+            }
+
+            Response.StatusCode = (int)resp.StatusCode;
+            Response.ContentType = resp.Content.Headers.ContentType?.ToString() ?? "application/json";
+            await Response.StartAsync();
+            await using var stream = await resp.Content.ReadAsStreamAsync();
+            await stream.CopyToAsync(Response.Body);
+            await Response.Body.FlushAsync();
+            await Response.CompleteAsync();
+        }
+        catch (MxApiMatrixException e) {
+            _logger.LogError(e, "Matrix error");
+            Response.StatusCode = StatusCodes.Status500InternalServerError;
+            Response.ContentType = "application/json";
+
+            await Response.WriteAsync(e.GetAsJson());
+            await Response.CompleteAsync();
+        }
+        catch (Exception e) {
+            _logger.LogError(e, "Unhandled error");
+            Response.StatusCode = StatusCodes.Status500InternalServerError;
+            Response.ContentType = "text/plain";
+
+            await Response.WriteAsync(e.ToString());
+            await Response.CompleteAsync();
+        }
+    }
+
+    [HttpPost]
+    public async Task ProxyPost([FromQuery] string? access_token, string _) {
+        try {
+            access_token ??= _authenticationService.GetToken(fail: false);
+            var mxid = await _authenticationService.GetMxidFromToken(fail: false);
+            var hs = await _authenticatedHomeserverProviderService.GetHomeserver();
 
             _logger.LogInformation("Proxying request for {}: {}{}", mxid, Request.Path, Request.QueryString);
 
@@ -35,10 +90,13 @@ public class GenericController : ControllerBase {
                     .Replace($"access_token={access_token}", "")
             );
 
-            var resp = await hc.GetAsync($"{_config.Homeserver}{Request.Path}{Request.QueryString}");
+            var resp = await hs._httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Post, $"{Request.Path}{Request.QueryString}") {
+                Method = HttpMethod.Post,
+                Content = new StreamContent(Request.Body)
+            });
 
             if (resp.Content is null) {
-                throw new MatrixException {
+                throw new MxApiMatrixException {
                     ErrorCode = "M_UNKNOWN",
                     Error = "No content in response"
                 };
@@ -51,9 +109,8 @@ public class GenericController : ControllerBase {
             await stream.CopyToAsync(Response.Body);
             await Response.Body.FlushAsync();
             await Response.CompleteAsync();
-
         }
-        catch (MatrixException e) {
+        catch (MxApiMatrixException e) {
             _logger.LogError(e, "Matrix error");
             Response.StatusCode = StatusCodes.Status500InternalServerError;
             Response.ContentType = "application/json";
@@ -71,11 +128,12 @@ public class GenericController : ControllerBase {
         }
     }
 
-    [HttpPost]
-    public async Task ProxyPost([FromQuery] string? access_token, string _) {
+    [HttpPut]
+    public async Task ProxyPut([FromQuery] string? access_token, string _) {
         try {
-            access_token ??= _auth.GetToken(fail: false);
-            var mxid = _auth.GetUserId(fail: false);
+            access_token ??= _authenticationService.GetToken(fail: false);
+            var mxid = await _authenticationService.GetMxidFromToken(fail: false);
+            var hs = await _authenticatedHomeserverProviderService.GetHomeserver();
 
             _logger.LogInformation("Proxying request for {}: {}{}", mxid, Request.Path, Request.QueryString);
 
@@ -89,14 +147,13 @@ public class GenericController : ControllerBase {
                     .Replace($"access_token={access_token}", "")
             );
 
-            var resp = await hc.SendAsync(new HttpRequestMessage {
-                Method = HttpMethod.Post,
-                RequestUri = new Uri($"{_config.Homeserver}{Request.Path}{Request.QueryString}"),
+            var resp = await hs._httpClient.SendAsync(new HttpRequestMessage(HttpMethod.Put, $"{Request.Path}{Request.QueryString}") {
+                Method = HttpMethod.Put,
                 Content = new StreamContent(Request.Body)
             });
 
             if (resp.Content is null) {
-                throw new MatrixException {
+                throw new MxApiMatrixException {
                     ErrorCode = "M_UNKNOWN",
                     Error = "No content in response"
                 };
@@ -109,9 +166,8 @@ public class GenericController : ControllerBase {
             await stream.CopyToAsync(Response.Body);
             await Response.Body.FlushAsync();
             await Response.CompleteAsync();
-
         }
-        catch (MatrixException e) {
+        catch (MxApiMatrixException e) {
             _logger.LogError(e, "Matrix error");
             Response.StatusCode = StatusCodes.Status500InternalServerError;
             Response.ContentType = "application/json";
diff --git a/MxApiExtensions/Controllers/LoginController.cs b/MxApiExtensions/Controllers/LoginController.cs
new file mode 100644
index 0000000..1a7970a
--- /dev/null
+++ b/MxApiExtensions/Controllers/LoginController.cs
@@ -0,0 +1,69 @@
+using System.Net.Http.Headers;
+using LibMatrix;
+using LibMatrix.Extensions;
+using LibMatrix.Responses;
+using LibMatrix.Services;
+using Microsoft.AspNetCore.Mvc;
+using MxApiExtensions.Services;
+
+namespace MxApiExtensions.Controllers;
+
+[ApiController]
+[Route("/")]
+public class LoginController : ControllerBase {
+    private readonly ILogger _logger;
+    private readonly HomeserverProviderService _hsProvider;
+    private readonly HomeserverResolverService _hsResolver;
+    private readonly AuthenticationService _auth;
+    private readonly MxApiExtensionsConfiguration _conf;
+
+    public LoginController(ILogger<LoginController> logger, HomeserverProviderService hsProvider, HomeserverResolverService hsResolver, AuthenticationService auth, MxApiExtensionsConfiguration conf) {
+        _logger = logger;
+        _hsProvider = hsProvider;
+        _hsResolver = hsResolver;
+        _auth = auth;
+        _conf = conf;
+    }
+
+    [HttpPost("/_matrix/client/{_}/login")]
+    public async Task Proxy([FromBody] LoginRequest request, string _) {
+        if (!request.Identifier.User.Contains("#")) {
+            Response.StatusCode = (int)StatusCodes.Status403Forbidden;
+            Response.ContentType = "application/json";
+            await Response.StartAsync();
+            await Response.WriteAsync(new MxApiMatrixException() {
+                ErrorCode = "M_FORBIDDEN",
+                Error = "[MxApiExtensions] Invalid username, must be of the form @user#domain:" + Request.Host.Value
+            }.GetAsJson() ?? "");
+            await Response.CompleteAsync();
+        }
+        var hsCanonical = request.Identifier.User.Split('#')[1].Split(':')[0];
+        request.Identifier.User = request.Identifier.User.Split(':')[0].Replace('#', ':');
+        if(!request.Identifier.User.StartsWith('@')) request.Identifier.User = '@' + request.Identifier.User;
+        var hs = await _hsResolver.ResolveHomeserverFromWellKnown(hsCanonical);
+        //var hs = await _hsProvider.Login(hsCanonical, mxid, request.Password);
+        var hsClient = new MatrixHttpClient { BaseAddress = new Uri(hs) };
+        //hsClient.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", hsClient.DefaultRequestHeaders.Authorization!.Parameter);
+        var resp = await hsClient.PostAsJsonAsync("/_matrix/client/r0/login", request);
+        var loginResp = await resp.Content.ReadAsStringAsync();
+        Response.StatusCode = (int)resp.StatusCode;
+        Response.ContentType = resp.Content.Headers.ContentType?.ToString() ?? "application/json";
+        await Response.StartAsync();
+        await Response.WriteAsync(loginResp);
+        await Response.CompleteAsync();
+        var token = (await resp.Content.ReadFromJsonAsync<LoginResponse>())!.AccessToken;
+        await _auth.SaveMxidForToken(token, request.Identifier.User);
+    }
+
+
+    [HttpGet("/_matrix/client/{_}/login")]
+    public async Task<object> Proxy(string? _) {
+        return new {
+            flows = new[] {
+                new {
+                    type = "m.login.password"
+                }
+            }
+        };
+    }
+}
diff --git a/MxApiExtensions/Controllers/SyncController.cs b/MxApiExtensions/Controllers/SyncController.cs
index d883377..382d670 100644
--- a/MxApiExtensions/Controllers/SyncController.cs
+++ b/MxApiExtensions/Controllers/SyncController.cs
@@ -1,5 +1,19 @@
+using System.Collections.Concurrent;
 using System.Net.Http.Headers;
+using System.Text.Encodings.Web;
+using System.Text.Json;
+using System.Text.Json.Serialization;
+using System.Web;
+using LibMatrix;
+using LibMatrix.Helpers;
+using LibMatrix.Homeservers;
+using LibMatrix.Responses;
+using LibMatrix.RoomTypes;
+using LibMatrix.StateEventTypes.Spec;
 using Microsoft.AspNetCore.Mvc;
+using MxApiExtensions.Classes;
+using MxApiExtensions.Extensions;
+using MxApiExtensions.Services;
 
 namespace MxApiExtensions.Controllers;
 
@@ -7,103 +21,255 @@ namespace MxApiExtensions.Controllers;
 [Route("/")]
 public class SyncController : ControllerBase {
     private readonly ILogger<SyncController> _logger;
-    private readonly CacheConfiguration _config;
-    private readonly Auth _auth;
+    private readonly MxApiExtensionsConfiguration _config;
+    private readonly AuthenticationService _auth;
+    private readonly AuthenticatedHomeserverProviderService _hs;
 
-    public SyncController(ILogger<SyncController> logger, CacheConfiguration config, Auth auth) {
+    private static readonly ConcurrentDictionary<string, SyncState> _syncStates = new();
+
+    public SyncController(ILogger<SyncController> logger, MxApiExtensionsConfiguration config, AuthenticationService auth, AuthenticatedHomeserverProviderService hs) {
         _logger = logger;
         _config = config;
         _auth = auth;
+        _hs = hs;
     }
 
     [HttpGet("/_matrix/client/v3/sync")]
-    public async Task Sync([FromQuery] string? since, [FromQuery] string? access_token) {
+    public async Task Sync([FromQuery] string? since, [FromQuery] int timeout = 1000) {
+        Task? preloadTask = null;
+        AuthenticatedHomeserverGeneric? hs = null;
+        try {
+            hs = await _hs.GetHomeserver();
+        }
+        catch (Exception e) {
+            Console.WriteLine();
+        }
+        var qs = HttpUtility.ParseQueryString(Request.QueryString.Value!);
+        qs.Remove("access_token");
+
+        if (!_config.FastInitialSync.Enabled) {
+            _logger.LogInformation("Starting sync for {} on {} ({})", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken);
+            var result = await hs._httpClient.GetAsync($"{Request.Path}?{qs}");
+            await Response.WriteHttpResponse(result);
+            return;
+        }
+
         try {
-            access_token ??= _auth.GetToken();
-            var mxid = _auth.GetUserId();
-            var cacheFile = GetFilePath(mxid, since);
-
-            if (!await TrySendCached(cacheFile)) {
-                using var hc = new HttpClient();
-                hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", access_token);
-                hc.Timeout = TimeSpan.FromMinutes(10);
-                //remove access_token from query string
-                Request.QueryString = new QueryString(
-                    Request.QueryString.Value
-                        .Replace("&access_token", "access_token")
-                        .Replace($"access_token={access_token}", "")
-                );
-
-                var resp = hc.GetAsync($"{_config.Homeserver}{Request.Path}{Request.QueryString}").Result;
-                // var resp = await hs._httpClient.GetAsync($"/_matrix/client/v3/sync?since={since}");
-
-                if (resp.Content is null) {
-                    throw new MatrixException {
-                        ErrorCode = "M_UNKNOWN",
-                        Error = "No content in response"
-                    };
+            var syncState = _syncStates.GetOrAdd(hs.AccessToken, _ => {
+                _logger.LogInformation("Started tracking sync state for {} on {} ({})", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken);
+                return new SyncState {
+                    IsInitialSync = string.IsNullOrWhiteSpace(since),
+                    Homeserver = hs
+                };
+            });
+
+            if (syncState.NextSyncResult is null) {
+                _logger.LogInformation("Starting sync for {} on {} ({})", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken);
+
+                if (syncState.IsInitialSync) {
+                    preloadTask = EnqueuePreloadData(syncState);
                 }
 
-                Response.StatusCode = (int)resp.StatusCode;
+                syncState.NextSyncResultStartedAt = DateTime.Now;
+                syncState.NextSyncResult = Task.Delay(30_000);
+                syncState.NextSyncResult.ContinueWith(x => {
+                    _logger.LogInformation("Sync for {} on {} ({}) starting", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken);
+                    syncState.NextSyncResult = hs._httpClient.GetAsync($"{Request.Path}?{qs}");
+                });
+            }
+
+            if (syncState.SyncQueue.Count > 0) {
+                _logger.LogInformation("Sync for {} on {} ({}) has {} queued results", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken, syncState.SyncQueue.Count);
+                syncState.SyncQueue.TryDequeue(out var result);
+
+                Response.StatusCode = StatusCodes.Status200OK;
                 Response.ContentType = "application/json";
                 await Response.StartAsync();
-                await using var stream = await resp.Content.ReadAsStreamAsync();
-                await using var target = System.IO.File.OpenWrite(cacheFile);
-                var buffer = new byte[1];
-
-                int bytesRead;
-                while ((bytesRead = await stream.ReadAsync(buffer, 0, buffer.Length)) > 0) {
-                    await Response.Body.WriteAsync(buffer.AsMemory(0, bytesRead));
-                    target.Write(buffer, 0, bytesRead);
-                }
-
-                await target.FlushAsync();
+                await JsonSerializer.SerializeAsync(Response.Body, result, new JsonSerializerOptions {
+                    WriteIndented = true,
+                    DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
+                });
                 await Response.CompleteAsync();
+                return;
+            }
+
+            timeout = Math.Clamp(timeout, 0, 100);
+            _logger.LogInformation("Sync for {} on {} ({}) is still running, waiting for {}ms, {} elapsed", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken, timeout,
+                DateTime.Now.Subtract(syncState.NextSyncResultStartedAt));
+
+            try {
+                await syncState.NextSyncResult.WaitAsync(TimeSpan.FromMilliseconds(timeout));
+            }
+            catch { }
+
+            if (syncState.NextSyncResult is Task<HttpResponseMessage> { IsCompleted: true } response) {
+                _logger.LogInformation("Sync for {} on {} ({}) completed", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken);
+                var resp = await response;
+                await Response.WriteHttpResponse(resp);
+                return;
             }
+
+            // await Task.Delay(timeout);
+            _logger.LogInformation("Sync for {} on {} ({}): sending bogus response", hs.WhoAmI.UserId, hs.HomeServerDomain, hs.AccessToken);
+            Response.StatusCode = StatusCodes.Status200OK;
+            Response.ContentType = "application/json";
+            await Response.StartAsync();
+            var syncResult = new SyncResult {
+                // NextBatch = "MxApiExtensions::Next" + Random.Shared.NextInt64(),
+                NextBatch = since ?? "",
+                Presence = new() {
+                    Events = new() {
+                        await GetStatusMessage(syncState, $"{DateTime.Now.Subtract(syncState.NextSyncResultStartedAt)} {syncState.NextSyncResult.Status}")
+                    }
+                },
+                Rooms = new() {
+                    Invite = new(),
+                    Join = new()
+                }
+            };
+            await JsonSerializer.SerializeAsync(Response.Body, syncResult, new JsonSerializerOptions {
+                WriteIndented = true,
+                DefaultIgnoreCondition = JsonIgnoreCondition.WhenWritingNull
+            });
+            await Response.CompleteAsync();
         }
-        catch (MatrixException e) {
+        catch (MxApiMatrixException e) {
+            _logger.LogError(e, "Error while syncing for {} on {} ({})", _hs.GetHomeserver().Result.WhoAmI.UserId,
+                _hs.GetHomeserver().Result.HomeServerDomain, _hs.GetHomeserver().Result.AccessToken);
+
             Response.StatusCode = StatusCodes.Status500InternalServerError;
             Response.ContentType = "application/json";
 
             await Response.WriteAsJsonAsync(e.GetAsJson());
             await Response.CompleteAsync();
         }
+
         catch (Exception e) {
+            //catch SSL connection errors and retry
+            if (e.InnerException is HttpRequestException && e.InnerException.Message.Contains("The SSL connection could not be established")) {
+                _logger.LogWarning("Caught SSL connection error, retrying sync for {} on {} ({})", _hs.GetHomeserver().Result.WhoAmI.UserId,
+                    _hs.GetHomeserver().Result.HomeServerDomain, _hs.GetHomeserver().Result.AccessToken);
+                await Sync(since, timeout);
+                return;
+            }
+
+            _logger.LogError(e, "Error while syncing for {} on {} ({})", _hs.GetHomeserver().Result.WhoAmI.UserId,
+                _hs.GetHomeserver().Result.HomeServerDomain, _hs.GetHomeserver().Result.AccessToken);
+
             Response.StatusCode = StatusCodes.Status500InternalServerError;
             Response.ContentType = "text/plain";
 
             await Response.WriteAsync(e.ToString());
             await Response.CompleteAsync();
         }
+
+        Response.Body.Close();
+        if (preloadTask is not null)
+            await preloadTask;
     }
 
-    private async Task<bool> TrySendCached(string cacheFile) {
-        if (!System.IO.File.Exists(cacheFile)) return false;
+    private async Task EnqueuePreloadData(SyncState syncState) {
+        var rooms = await syncState.Homeserver.GetJoinedRooms();
+        var dm_rooms = (await syncState.Homeserver.GetAccountData<Dictionary<string, List<string>>>("m.direct")).Aggregate(new List<string>(), (list, entry) => {
+            list.AddRange(entry.Value);
+            return list;
+        });
+
+        var ownHs = syncState.Homeserver.WhoAmI.UserId.Split(':')[1];
+        rooms = rooms.OrderBy(x => {
+            if (dm_rooms.Contains(x.RoomId)) return -1;
+            var parts = x.RoomId.Split(':');
+            if (parts[1] == ownHs) return 200;
+            if (HomeserverWeightEstimation.EstimatedSize.ContainsKey(parts[1])) return HomeserverWeightEstimation.EstimatedSize[parts[1]] + parts[0].Length;
+            return 5000;
+        }).ToList();
+        var roomDataTasks = rooms.Select(room => EnqueueRoomData(syncState, room)).ToList();
+        _logger.LogInformation("Preloading data for {} rooms on {} ({})", roomDataTasks.Count, syncState.Homeserver.HomeServerDomain, syncState.Homeserver.AccessToken);
 
-        Response.StatusCode = 200;
-        Response.ContentType = "application/json";
-        await Response.StartAsync();
-        await using var stream = System.IO.File.OpenRead(cacheFile);
-        await stream.CopyToAsync(Response.Body);
-        await Response.CompleteAsync();
-        return true;
+        await Task.WhenAll(roomDataTasks);
     }
 
-#region Cache management
+    private SemaphoreSlim _roomDataSemaphore = new(4, 4);
 
-    public string GetFilePath(string mxid, string since) {
-        var cacheDir = Path.Join("cache", mxid);
-        Directory.CreateDirectory(cacheDir);
-        var cacheFile = Path.Join(cacheDir, $"sync-{since}.json");
-        if (!Path.GetFullPath(cacheFile).StartsWith(Path.GetFullPath(cacheDir))) {
-            throw new MatrixException {
-                ErrorCode = "M_UNKNOWN",
-                Error = "[Rory&::MxSyncCache] Cache file path is not in cache directory"
-            };
+    private async Task EnqueueRoomData(SyncState syncState, GenericRoom room) {
+        await _roomDataSemaphore.WaitAsync();
+        var roomState = room.GetFullStateAsync();
+        var timeline = await room.GetMessagesAsync(limit: 100, dir: "b");
+        timeline.Chunk.Reverse();
+        var syncResult = new SyncResult {
+            Rooms = new() {
+                Join = new() {
+                    {
+                        room.RoomId,
+                        new SyncResult.RoomsDataStructure.JoinedRoomDataStructure() {
+                            AccountData = new() {
+                                Events = new()
+                            },
+                            Ephemeral = new() {
+                                Events = new()
+                            },
+                            State = new() {
+                                Events = timeline.State
+                            },
+                            UnreadNotifications = new() {
+                                HighlightCount = 0,
+                                NotificationCount = 0
+                            },
+                            Timeline = new() {
+                                Events = timeline.Chunk,
+                                Limited = false,
+                                PrevBatch = timeline.Start
+                            },
+                            Summary = new() {
+                                Heroes = new(),
+                                InvitedMemberCount = 0,
+                                JoinedMemberCount = 1
+                            }
+                        }
+                    }
+                }
+            },
+            Presence = new() {
+                Events = new() {
+                    await GetStatusMessage(syncState, $"{DateTime.Now.Subtract(syncState.NextSyncResultStartedAt)} {syncState.NextSyncResult.Status} {room.RoomId}")
+                }
+            },
+            NextBatch = ""
+        };
+
+        await foreach (var stateEvent in roomState) {
+            syncResult.Rooms.Join[room.RoomId].State.Events.Add(stateEvent);
         }
 
-        return cacheFile;
+        var joinRoom = syncResult.Rooms.Join[room.RoomId];
+        joinRoom.Summary.Heroes.AddRange(joinRoom.State.Events
+            .Where(x =>
+                x.Type == "m.room.member"
+                && x.StateKey != syncState.Homeserver.WhoAmI.UserId
+                && (x.TypedContent as RoomMemberEventData).Membership == "join"
+                )
+            .Select(x => x.StateKey));
+        joinRoom.Summary.JoinedMemberCount = joinRoom.Summary.Heroes.Count;
+
+        syncState.SyncQueue.Enqueue(syncResult);
+        _roomDataSemaphore.Release();
     }
 
-#endregion
+    private async Task<StateEventResponse> GetStatusMessage(SyncState syncState, string message) {
+        return new StateEventResponse() {
+            TypedContent = new PresenceStateEventData() {
+                DisplayName = "MxApiExtensions",
+                Presence = "online",
+                StatusMessage = message,
+                // AvatarUrl = (await syncState.Homeserver.GetProfile(syncState.Homeserver.WhoAmI.UserId)).AvatarUrl
+                AvatarUrl = ""
+            },
+            Type = "m.presence",
+            StateKey = syncState.Homeserver.WhoAmI.UserId,
+            Sender = syncState.Homeserver.WhoAmI.UserId,
+            UserId = syncState.Homeserver.WhoAmI.UserId,
+            EventId = Guid.NewGuid().ToString(),
+            OriginServerTs = 0
+        };
+    }
 }
diff --git a/MxApiExtensions/Controllers/WellKnownController.cs b/MxApiExtensions/Controllers/WellKnownController.cs
new file mode 100644
index 0000000..b27451f
--- /dev/null
+++ b/MxApiExtensions/Controllers/WellKnownController.cs
@@ -0,0 +1,23 @@
+using System.Text.Json.Nodes;
+using Microsoft.AspNetCore.Mvc;
+
+namespace MxApiExtensions.Controllers;
+
+[ApiController]
+[Route("/")]
+public class WellKnownController : ControllerBase {
+    private readonly MxApiExtensionsConfiguration _config;
+
+    public WellKnownController(MxApiExtensionsConfiguration config) {
+        _config = config;
+    }
+
+    [HttpGet("/.well-known/matrix/client")]
+    public object GetWellKnown() {
+        var res = new JsonObject();
+        res.Add("m.homeserver", new JsonObject {
+            { "base_url", Request.Scheme + "://" + Request.Host + "/" },
+        });
+        return res;
+    }
+}
diff --git a/MxApiExtensions/Extensions/HttpResponseExtensions.cs b/MxApiExtensions/Extensions/HttpResponseExtensions.cs
new file mode 100644
index 0000000..1dd1c00
--- /dev/null
+++ b/MxApiExtensions/Extensions/HttpResponseExtensions.cs
@@ -0,0 +1,17 @@
+namespace MxApiExtensions.Extensions;
+
+public static class HttpResponseExtensions {
+    public static async Task WriteHttpResponse(this HttpResponse response, HttpResponseMessage message) {
+        response.StatusCode = (int) message.StatusCode;
+        //copy all headers
+        foreach (var header in message.Headers) {
+            response.Headers.Add(header.Key, header.Value.ToArray());
+        }
+
+        await response.StartAsync();
+        var content = await message.Content.ReadAsStreamAsync();
+        await content.CopyToAsync(response.Body);
+        await response.CompleteAsync();
+        // await content.DisposeAsync();
+    }
+}
diff --git a/MxApiExtensions/FileStorageProvider.cs b/MxApiExtensions/FileStorageProvider.cs
new file mode 100644
index 0000000..c2bb267
--- /dev/null
+++ b/MxApiExtensions/FileStorageProvider.cs
@@ -0,0 +1,37 @@
+using System.Text.Json;
+using ArcaneLibs.Extensions;
+using LibMatrix.Interfaces.Services;
+
+namespace MxApiExtensions;
+
+public class FileStorageProvider : IStorageProvider {
+    private readonly ILogger<FileStorageProvider> _logger;
+
+    public string TargetPath { get; }
+
+    /// <summary>
+    /// Creates a new instance of <see cref="FileStorageProvider" />.
+    /// </summary>
+    /// <param name="targetPath"></param>
+    public FileStorageProvider(string targetPath) {
+        new Logger<FileStorageProvider>(new LoggerFactory()).LogInformation("test");
+        Console.WriteLine($"Initialised FileStorageProvider with path {targetPath}");
+        TargetPath = targetPath;
+        if(!Directory.Exists(targetPath)) {
+            Directory.CreateDirectory(targetPath);
+        }
+    }
+
+    public async Task SaveObjectAsync<T>(string key, T value) => await File.WriteAllTextAsync(Path.Join(TargetPath, key), value?.ToJson());
+
+    public async Task<T?> LoadObjectAsync<T>(string key) => JsonSerializer.Deserialize<T>(await File.ReadAllTextAsync(Path.Join(TargetPath, key)));
+
+    public Task<bool> ObjectExistsAsync(string key) => Task.FromResult(File.Exists(Path.Join(TargetPath, key)));
+
+    public Task<List<string>> GetAllKeysAsync() => Task.FromResult(Directory.GetFiles(TargetPath).Select(Path.GetFileName).ToList());
+
+    public Task DeleteObjectAsync(string key) {
+        File.Delete(Path.Join(TargetPath, key));
+        return Task.CompletedTask;
+    }
+}
diff --git a/MxApiExtensions/MatrixException.cs b/MxApiExtensions/MatrixException.cs
deleted file mode 100644
index 568c5a9..0000000
--- a/MxApiExtensions/MatrixException.cs
+++ /dev/null
@@ -1,72 +0,0 @@
-using System.Text.Json.Nodes;
-using System.Text.Json.Serialization;
-using ArcaneLibs.Extensions;
-
-namespace MxApiExtensions;
-
-public class MatrixException : Exception {
-    [JsonPropertyName("errcode")]
-    public string ErrorCode { get; set; }
-
-    [JsonPropertyName("error")]
-    public string Error { get; set; }
-
-    [JsonPropertyName("soft_logout")]
-    public bool? SoftLogout { get; set; }
-
-    [JsonPropertyName("retry_after_ms")]
-    public int? RetryAfterMs { get; set; }
-
-    public string RawContent { get; set; }
-
-    //turn this into json
-    public JsonObject GetAsJson() {
-        var jsonObject = new JsonObject {
-            ["errcode"] = ErrorCode,
-            ["error"] = Error
-        };
-        if(SoftLogout is not null) jsonObject["soft_logout"] = SoftLogout;
-        if(RetryAfterMs is not null) jsonObject["retry_after_ms"] = RetryAfterMs;
-        return jsonObject;
-    }
-
-
-    public override string Message =>
-        $"{ErrorCode}: {ErrorCode switch {
-            // common
-            "M_FORBIDDEN" => $"You do not have permission to perform this action: {Error}",
-            "M_UNKNOWN_TOKEN" => $"The access token specified was not recognised: {Error}{(SoftLogout == true ? " (soft logout)" : "")}",
-            "M_MISSING_TOKEN" => $"No access token was specified: {Error}",
-            "M_BAD_JSON" => $"Request contained valid JSON, but it was malformed in some way: {Error}",
-            "M_NOT_JSON" => $"Request did not contain valid JSON: {Error}",
-            "M_NOT_FOUND" => $"The requested resource was not found: {Error}",
-            "M_LIMIT_EXCEEDED" => $"Too many requests have been sent in a short period of time. Wait a while then try again: {Error}",
-            "M_UNRECOGNISED" => $"The server did not recognise the request: {Error}",
-            "M_UNKOWN" => $"The server encountered an unexpected error: {Error}",
-            // endpoint specific
-            "M_UNAUTHORIZED" => $"The request did not contain valid authentication information for the target of the request: {Error}",
-            "M_USER_DEACTIVATED" => $"The user ID associated with the request has been deactivated: {Error}",
-            "M_USER_IN_USE" => $"The user ID associated with the request is already in use: {Error}",
-            "M_INVALID_USERNAME" => $"The requested user ID is not valid: {Error}",
-            "M_ROOM_IN_USE" => $"The room alias requested is already taken: {Error}",
-            "M_INVALID_ROOM_STATE" => $"The room associated with the request is not in a valid state to perform the request: {Error}",
-            "M_THREEPID_IN_USE" => $"The threepid requested is already associated with a user ID on this server: {Error}",
-            "M_THREEPID_NOT_FOUND" => $"The threepid requested is not associated with any user ID: {Error}",
-            "M_THREEPID_AUTH_FAILED" => $"The provided threepid and/or token was invalid: {Error}",
-            "M_THREEPID_DENIED" => $"The homeserver does not permit the third party identifier in question: {Error}",
-            "M_SERVER_NOT_TRUSTED" => $"The homeserver does not trust the identity server: {Error}",
-            "M_UNSUPPORTED_ROOM_VERSION" => $"The room version is not supported: {Error}",
-            "M_INCOMPATIBLE_ROOM_VERSION" => $"The room version is incompatible: {Error}",
-            "M_BAD_STATE" => $"The request was invalid because the state was invalid: {Error}",
-            "M_GUEST_ACCESS_FORBIDDEN" => $"Guest access is forbidden: {Error}",
-            "M_CAPTCHA_NEEDED" => $"Captcha needed: {Error}",
-            "M_CAPTCHA_INVALID" => $"Captcha invalid: {Error}",
-            "M_MISSING_PARAM" => $"Missing parameter: {Error}",
-            "M_INVALID_PARAM" => $"Invalid parameter: {Error}",
-            "M_TOO_LARGE" => $"The request or entity was too large: {Error}",
-            "M_EXCLUSIVE" => $"The resource being requested is reserved by an application service, or the application service making the request has not created the resource: {Error}",
-            "M_RESOURCE_LIMIT_EXCEEDED" => $"Exceeded resource limit: {Error}",
-            "M_CANNOT_LEAVE_SERVER_NOTICE_ROOM" => $"Cannot leave server notice room: {Error}",
-            _ => $"Unknown error: {new { ErrorCode, Error, SoftLogout, RetryAfterMs }.ToJson(ignoreNull: true)}"
-        }}";
-}
diff --git a/MxApiExtensions/MxApiExtensions.csproj b/MxApiExtensions/MxApiExtensions.csproj
index 0b88ce4..86bc290 100644
--- a/MxApiExtensions/MxApiExtensions.csproj
+++ b/MxApiExtensions/MxApiExtensions.csproj
@@ -13,4 +13,22 @@
         <PackageReference Include="Swashbuckle.AspNetCore" Version="6.4.0" />
     </ItemGroup>
 
+
+    <ItemGroup>
+      <ProjectReference Include="..\..\ArcaneLibs\ArcaneLibs\ArcaneLibs.csproj" />
+      <ProjectReference Include="..\..\LibMatrix\LibMatrix\LibMatrix.csproj" />
+      <ProjectReference Include="..\MxApiExtensions.Classes.LibMatrix\MxApiExtensions.Classes.LibMatrix.csproj" />
+      <ProjectReference Include="..\MxApiExtensions.Classes\MxApiExtensions.Classes.csproj" />
+
+
+
+
+    </ItemGroup>
+
+
+
+
+
+
+
 </Project>
diff --git a/MxApiExtensions/MxApiExtensionsConfiguration.cs b/MxApiExtensions/MxApiExtensionsConfiguration.cs
new file mode 100644
index 0000000..c3b6297
--- /dev/null
+++ b/MxApiExtensions/MxApiExtensionsConfiguration.cs
@@ -0,0 +1,29 @@
+namespace MxApiExtensions;
+
+public class MxApiExtensionsConfiguration {
+    public MxApiExtensionsConfiguration(IConfiguration config) {
+        config.GetRequiredSection("MxApiExtensions").Bind(this);
+    }
+
+    public List<string> AuthHomeservers { get; set; } = new();
+    public List<string> Admins { get; set; } = new();
+
+    public FastInitialSyncConfiguration FastInitialSync { get; set; } = new();
+
+    public CacheConfiguration Cache { get; set; } = new();
+
+
+    public class FastInitialSyncConfiguration {
+        public bool Enabled { get; set; } = true;
+        public bool UseRoomInfoCache { get; set; } = true;
+    }
+
+    public class CacheConfiguration {
+        public RoomInfoCacheConfiguration RoomInfo { get; set; } = new();
+
+        public class RoomInfoCacheConfiguration {
+            public TimeSpan BaseTtl { get; set; } = TimeSpan.FromMinutes(1);
+            public TimeSpan ExtraTtlPerState { get; set; } = TimeSpan.FromMilliseconds(100);
+        }
+    }
+}
diff --git a/MxApiExtensions/Program.cs b/MxApiExtensions/Program.cs
index 11fe114..a219e83 100644
--- a/MxApiExtensions/Program.cs
+++ b/MxApiExtensions/Program.cs
@@ -1,19 +1,34 @@
+using LibMatrix.Services;
 using Microsoft.AspNetCore.Http.Timeouts;
 using MxApiExtensions;
+using MxApiExtensions.Services;
 
 var builder = WebApplication.CreateBuilder(args);
 
 // Add services to the container.
 
-builder.Services.AddControllers();
+builder.Services.AddControllers().AddJsonOptions(options => {
+    options.JsonSerializerOptions.WriteIndented = true;
+});
 // Learn more about configuring Swagger/OpenAPI at https://aka.ms/aspnetcore/swashbuckle
 builder.Services.AddEndpointsApiExplorer();
 builder.Services.AddSwaggerGen();
 
 builder.Services.AddSingleton<IHttpContextAccessor, HttpContextAccessor>();
 
-builder.Services.AddSingleton<CacheConfiguration>();
-builder.Services.AddScoped<Auth>();
+builder.Services.AddSingleton<MxApiExtensionsConfiguration>();
+
+builder.Services.AddScoped<AuthenticationService>();
+builder.Services.AddScoped<AuthenticatedHomeserverProviderService>();
+
+builder.Services.AddSingleton<TieredStorageService>(x => {
+    var config = x.GetRequiredService<MxApiExtensionsConfiguration>();
+    return new TieredStorageService(
+        cacheStorageProvider: new FileStorageProvider("/run"),
+        dataStorageProvider: new FileStorageProvider("/run")
+    );
+});
+builder.Services.AddRoryLibMatrixServices();
 
 builder.Services.AddRequestTimeouts(x => {
     x.DefaultPolicy = new RequestTimeoutPolicy {
@@ -22,7 +37,7 @@ builder.Services.AddRequestTimeouts(x => {
             context.Response.StatusCode = 504;
             context.Response.ContentType = "application/json";
             await context.Response.StartAsync();
-            await context.Response.WriteAsJsonAsync(new MatrixException {
+            await context.Response.WriteAsJsonAsync(new MxApiMatrixException {
                 ErrorCode = "M_TIMEOUT",
                 Error = "Request timed out"
             }.GetAsJson());
@@ -34,8 +49,7 @@ builder.Services.AddRequestTimeouts(x => {
 var app = builder.Build();
 
 // Configure the HTTP request pipeline.
-if (app.Environment.IsDevelopment())
-{
+if (app.Environment.IsDevelopment()) {
     app.UseSwagger();
     app.UseSwaggerUI();
 }
diff --git a/MxApiExtensions/Properties/launchSettings.json b/MxApiExtensions/Properties/launchSettings.json
index bf22b91..c6392a4 100644
--- a/MxApiExtensions/Properties/launchSettings.json
+++ b/MxApiExtensions/Properties/launchSettings.json
@@ -1,40 +1,31 @@
 {
-  "$schema": "http://json.schemastore.org/launchsettings.json",
+  "$schema": "https://json.schemastore.org/launchsettings.json",
   "iisSettings": {
     "windowsAuthentication": false,
     "anonymousAuthentication": true,
     "iisExpress": {
-      "applicationUrl": "http://localhost:33875",
-      "sslPort": 44326
+      "applicationUrl": "http://localhost:9169",
+      "sslPort": 44321
     }
   },
   "profiles": {
-    "http": {
+    "Development": {
       "commandName": "Project",
       "dotnetRunMessages": true,
       "launchBrowser": false,
       "launchUrl": "swagger",
-      "applicationUrl": "http://localhost:5258",
+      "applicationUrl": "http://localhost:5119",
       "environmentVariables": {
         "ASPNETCORE_ENVIRONMENT": "Development"
       }
     },
-    "https": {
+    "Local": {
       "commandName": "Project",
       "dotnetRunMessages": true,
-      "launchBrowser": true,
-      "launchUrl": "swagger",
-      "applicationUrl": "https://localhost:7049;http://localhost:5258",
-      "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
-      }
-    },
-    "IIS Express": {
-      "commandName": "IISExpress",
-      "launchBrowser": true,
-      "launchUrl": "swagger",
+      "launchBrowser": false,
+      "applicationUrl": "http://localhost:5119",
       "environmentVariables": {
-        "ASPNETCORE_ENVIRONMENT": "Development"
+        "DOTNET_ENVIRONMENT": "Local"
       }
     }
   }
diff --git a/MxApiExtensions/Services/AuthenticatedHomeserverProviderService.cs b/MxApiExtensions/Services/AuthenticatedHomeserverProviderService.cs
new file mode 100644
index 0000000..96b0254
--- /dev/null
+++ b/MxApiExtensions/Services/AuthenticatedHomeserverProviderService.cs
@@ -0,0 +1,35 @@
+using LibMatrix.Homeservers;
+using LibMatrix.Services;
+
+namespace MxApiExtensions.Services;
+
+public class AuthenticatedHomeserverProviderService {
+    private readonly AuthenticationService _authenticationService;
+    private readonly HomeserverProviderService _homeserverProviderService;
+
+    public AuthenticatedHomeserverProviderService(AuthenticationService authenticationService, HomeserverProviderService homeserverProviderService) {
+        _authenticationService = authenticationService;
+        _homeserverProviderService = homeserverProviderService;
+    }
+
+    public async Task<AuthenticatedHomeserverGeneric> GetHomeserver() {
+        var token = _authenticationService.GetToken();
+        if (token == null) {
+            throw new MxApiMatrixException {
+                ErrorCode = "M_MISSING_TOKEN",
+                Error = "Missing access token"
+            };
+        }
+
+        var mxid = await _authenticationService.GetMxidFromToken(token);
+        if (mxid == "@anonymous:*") {
+            throw new MxApiMatrixException {
+                ErrorCode = "M_MISSING_TOKEN",
+                Error = "Missing access token"
+            };
+        }
+
+        var hsCanonical = string.Join(":", mxid.Split(':').Skip(1));
+        return await _homeserverProviderService.GetAuthenticatedWithToken(hsCanonical, token);
+    }
+}
diff --git a/MxApiExtensions/Services/AuthenticationService.cs b/MxApiExtensions/Services/AuthenticationService.cs
new file mode 100644
index 0000000..09a6e70
--- /dev/null
+++ b/MxApiExtensions/Services/AuthenticationService.cs
@@ -0,0 +1,121 @@
+using LibMatrix.Services;
+
+namespace MxApiExtensions.Services;
+
+public class AuthenticationService {
+    private readonly ILogger<AuthenticationService> _logger;
+    private readonly MxApiExtensionsConfiguration _config;
+    private readonly HomeserverProviderService _homeserverProviderService;
+    private readonly HttpRequest _request;
+
+    private static Dictionary<string, string> _tokenMap = new();
+
+    public AuthenticationService(ILogger<AuthenticationService> logger, MxApiExtensionsConfiguration config, IHttpContextAccessor request, HomeserverProviderService homeserverProviderService) {
+        _logger = logger;
+        _config = config;
+        _homeserverProviderService = homeserverProviderService;
+        _request = request.HttpContext!.Request;
+    }
+
+    internal string? GetToken(bool fail = true) {
+        string? token;
+        if (_request.Headers.TryGetValue("Authorization", out var tokens)) {
+            token = tokens.FirstOrDefault()?[7..];
+        }
+        else {
+            token = _request.Query["access_token"];
+        }
+
+        if (token == null && fail) {
+            throw new MxApiMatrixException {
+                ErrorCode = "M_MISSING_TOKEN",
+                Error = "Missing access token"
+            };
+        }
+
+        return token;
+    }
+
+    public async Task<string> GetMxidFromToken(string? token = null, bool fail = true) {
+        token ??= GetToken(fail);
+        if (token == null) {
+            if (fail) {
+                throw new MxApiMatrixException {
+                    ErrorCode = "M_MISSING_TOKEN",
+                    Error = "Missing access token"
+                };
+            }
+
+            return "@anonymous:*";
+        }
+
+        if(_tokenMap is not { Count: >0 } && File.Exists("token_map")) {
+            _tokenMap = (await File.ReadAllLinesAsync("token_map"))
+                .Select(l => l.Split('\t'))
+                .ToDictionary(l => l[0], l => l[1]);
+        }
+
+        if (_tokenMap.TryGetValue(token, out var mxid)) return mxid;
+
+        var lookupTasks = new Dictionary<string, Task<string?>>();
+        foreach (var homeserver in _config.AuthHomeservers) {
+            lookupTasks.Add(homeserver, GetMxidFromToken(token, homeserver));
+            await lookupTasks[homeserver].WaitAsync(TimeSpan.FromMilliseconds(250));
+            if(lookupTasks[homeserver].IsCompletedSuccessfully && !string.IsNullOrWhiteSpace(lookupTasks[homeserver].Result)) break;
+        }
+        await Task.WhenAll(lookupTasks.Values);
+
+        mxid = lookupTasks.Values.FirstOrDefault(x => x.Result != null)?.Result;
+        if(mxid is null) {
+            throw new MxApiMatrixException {
+                ErrorCode = "M_UNKNOWN_TOKEN",
+                Error = "Token not found on any configured homeservers: " + string.Join(", ", _config.AuthHomeservers)
+            };
+        }
+
+        // using var hc = new HttpClient();
+        // hc.DefaultRequestHeaders.Authorization = new AuthenticationHeaderValue("Bearer", token);
+        // var resp = hc.GetAsync($"{_config.Homeserver}/_matrix/client/v3/account/whoami").Result;
+        // if (!resp.IsSuccessStatusCode) {
+        //     throw new MatrixException {
+        //         ErrorCode = "M_UNKNOWN",
+        //         Error = "[Rory&::MxSyncCache] Whoami request failed"
+        //     };
+        // }
+        //
+        // if (resp.Content is null) {
+        //     throw new MatrixException {
+        //         ErrorCode = "M_UNKNOWN",
+        //         Error = "No content in response"
+        //     };
+        // }
+        //
+        // var json = (await JsonDocument.ParseAsync(await resp.Content.ReadAsStreamAsync())).RootElement;
+        // var mxid = json.GetProperty("user_id").GetString()!;
+        _logger.LogInformation("Got mxid {} from token {}", mxid, token);
+        await SaveMxidForToken(token, mxid);
+        return mxid;
+    }
+
+    private async Task<string?> GetMxidFromToken(string token, string hsDomain) {
+        _logger.LogInformation("Looking up mxid for token {} on {}", token, hsDomain);
+        var hs = await _homeserverProviderService.GetAuthenticatedWithToken(hsDomain, token);
+        try {
+            var res = hs.WhoAmI.UserId;
+            _logger.LogInformation("Got mxid {} for token {} on {}", res, token, hsDomain);
+            return res;
+        }
+        catch (MxApiMatrixException e) {
+            if (e.ErrorCode == "M_UNKNOWN_TOKEN") {
+                return null;
+            }
+
+            throw;
+        }
+    }
+
+    public async Task SaveMxidForToken(string token, string mxid) {
+        _tokenMap.Add(token, mxid);
+        await File.AppendAllLinesAsync("token_map", new[] { $"{token}\t{mxid}" });
+    }
+}
diff --git a/MxApiExtensions/appsettings.json b/MxApiExtensions/appsettings.json
index 0163bad..5a73cbe 100644
--- a/MxApiExtensions/appsettings.json
+++ b/MxApiExtensions/appsettings.json
@@ -6,7 +6,29 @@
     }
   },
   "AllowedHosts": "*",
-  "MxSyncCache": {
-    "Homeserver": "https://matrix.rory.gay"
+  // Configuration for the proxy
+  "MxApiExtensions": {
+    // WARNING: this exposes user tokens to servers listed here, which could be a security risk
+    // Only list servers you trust!
+    // Keep in mind that token conflicts can occur between servers!
+    "AuthHomeservers": [
+      "rory.gay",
+      "conduit.rory.gay"
+    ],
+    // List of administrator MXIDs for the proxy, this allows them to use administrative and debug endpoints
+    "Admins": [
+      "@emma:rory.gay",
+      "@emma:conduit.rory.gay"
+    ],
+    "FastInitialSync": {
+      "Enabled": true,
+      "UseRoomInfoCache": true
+    },
+    "Cache": {
+      "RoomInfo": {
+        "BaseTtl": "00:01:00",
+        "ExtraTtlPerState": "00:00:00.1000000"
+      }
+    }
   }
 }