diff --git a/desktop/go.mod b/desktop/go.mod index 927bee3..e7919b3 100644 --- a/desktop/go.mod +++ b/desktop/go.mod @@ -8,7 +8,7 @@ require github.com/wailsapp/wails/v3 v3.0.0-alpha.9 require ( go.mau.fi/gomuks v0.4.0 - go.mau.fi/util v0.8.5 + go.mau.fi/util v0.8.6-0.20250227184636-7ff63b0b9d95 ) require ( @@ -79,7 +79,7 @@ require ( gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/warnings.v0 v0.1.2 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - maunium.net/go/mautrix v0.23.2-0.20250226205639-b72caa948c18 // indirect + maunium.net/go/mautrix v0.23.2-0.20250304004736-7d3791ace3b7 // indirect mvdan.cc/xurls/v2 v2.6.0 // indirect ) diff --git a/desktop/go.sum b/desktop/go.sum index 7d4501d..9b6e2b7 100644 --- a/desktop/go.sum +++ b/desktop/go.sum @@ -166,8 +166,8 @@ github.com/xanzy/ssh-agent v0.3.3/go.mod h1:6dzNDKs0J9rVPHPhaGCukekBHKqfl+L3KghI github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= -go.mau.fi/util v0.8.5 h1:PwCAAtcfK0XxZ4sdErJyfBMkTEWoQU33aB7QqDDzQRI= -go.mau.fi/util v0.8.5/go.mod h1:Ycug9mrbztlahHPEJ6H5r8Nu/xqZaWbE5vPHVWmfz6M= +go.mau.fi/util v0.8.6-0.20250227184636-7ff63b0b9d95 h1:5EfVWWjU2Hte9uE6B/hBgvjnVfBx/7SYDZBnsuo+EBs= +go.mau.fi/util v0.8.6-0.20250227184636-7ff63b0b9d95/go.mod h1:Ycug9mrbztlahHPEJ6H5r8Nu/xqZaWbE5vPHVWmfz6M= go.mau.fi/webp v0.2.0 h1:QVMenHw7JDb4vall5sV75JNBQj9Hw4u8AKbi1QetHvg= go.mau.fi/webp v0.2.0/go.mod h1:VSg9MyODn12Mb5pyG0NIyNFhujrmoFSsZBs8syOZD1Q= go.mau.fi/zeroconfig v0.1.3 h1:As9wYDKmktjmNZW5i1vn8zvJlmGKHeVxHVIBMXsm4kM= @@ -261,7 +261,7 @@ gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -maunium.net/go/mautrix v0.23.2-0.20250226205639-b72caa948c18 h1:1JVivuS1whIdai/Yurqe1OXiHAarCh0UgR/zh61coiQ= -maunium.net/go/mautrix v0.23.2-0.20250226205639-b72caa948c18/go.mod h1:kldoZQDneV/jquIhwG1MmMw5j2A2M/MnQYRSWt863cY= +maunium.net/go/mautrix v0.23.2-0.20250304004736-7d3791ace3b7 h1:AeNHqITptzOpmfMxnqmQRw6xN7DUDCgsN00BaPyRd4k= +maunium.net/go/mautrix v0.23.2-0.20250304004736-7d3791ace3b7/go.mod h1:IHMaSJh7YIxMrZSDVefS+nLdr3RbeLowsCSa6ibONZ0= mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI= mvdan.cc/xurls/v2 v2.6.0/go.mod h1:bCvEZ1XvdA6wDnxY7jPPjEmigDtvtvPXAD/Exa9IMSk= diff --git a/go.mod b/go.mod index efd1233..5f3a138 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/tidwall/gjson v1.18.0 github.com/tidwall/sjson v1.2.5 github.com/yuin/goldmark v1.7.8 - go.mau.fi/util v0.8.5 + go.mau.fi/util v0.8.6-0.20250227184636-7ff63b0b9d95 go.mau.fi/webp v0.2.0 go.mau.fi/zeroconfig v0.1.3 golang.org/x/crypto v0.34.0 @@ -27,7 +27,7 @@ require ( golang.org/x/text v0.22.0 gopkg.in/yaml.v3 v3.0.1 maunium.net/go/mauflag v1.0.0 - maunium.net/go/mautrix v0.23.2-0.20250226205639-b72caa948c18 + maunium.net/go/mautrix v0.23.2-0.20250304004736-7d3791ace3b7 mvdan.cc/xurls/v2 v2.6.0 ) diff --git a/go.sum b/go.sum index 44b9c84..12ea087 100644 --- a/go.sum +++ b/go.sum @@ -68,8 +68,8 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/yuin/goldmark v1.7.8 h1:iERMLn0/QJeHFhxSt3p6PeN9mGnvIKSpG9YYorDMnic= github.com/yuin/goldmark v1.7.8/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E= -go.mau.fi/util v0.8.5 h1:PwCAAtcfK0XxZ4sdErJyfBMkTEWoQU33aB7QqDDzQRI= -go.mau.fi/util v0.8.5/go.mod h1:Ycug9mrbztlahHPEJ6H5r8Nu/xqZaWbE5vPHVWmfz6M= +go.mau.fi/util v0.8.6-0.20250227184636-7ff63b0b9d95 h1:5EfVWWjU2Hte9uE6B/hBgvjnVfBx/7SYDZBnsuo+EBs= +go.mau.fi/util v0.8.6-0.20250227184636-7ff63b0b9d95/go.mod h1:Ycug9mrbztlahHPEJ6H5r8Nu/xqZaWbE5vPHVWmfz6M= go.mau.fi/webp v0.2.0 h1:QVMenHw7JDb4vall5sV75JNBQj9Hw4u8AKbi1QetHvg= go.mau.fi/webp v0.2.0/go.mod h1:VSg9MyODn12Mb5pyG0NIyNFhujrmoFSsZBs8syOZD1Q= go.mau.fi/zeroconfig v0.1.3 h1:As9wYDKmktjmNZW5i1vn8zvJlmGKHeVxHVIBMXsm4kM= @@ -100,7 +100,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M= maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA= -maunium.net/go/mautrix v0.23.2-0.20250226205639-b72caa948c18 h1:1JVivuS1whIdai/Yurqe1OXiHAarCh0UgR/zh61coiQ= -maunium.net/go/mautrix v0.23.2-0.20250226205639-b72caa948c18/go.mod h1:kldoZQDneV/jquIhwG1MmMw5j2A2M/MnQYRSWt863cY= +maunium.net/go/mautrix v0.23.2-0.20250304004736-7d3791ace3b7 h1:AeNHqITptzOpmfMxnqmQRw6xN7DUDCgsN00BaPyRd4k= +maunium.net/go/mautrix v0.23.2-0.20250304004736-7d3791ace3b7/go.mod h1:IHMaSJh7YIxMrZSDVefS+nLdr3RbeLowsCSa6ibONZ0= mvdan.cc/xurls/v2 v2.6.0 h1:3NTZpeTxYVWNSokW3MKeyVkz/j7uYXYiMtXRUfmjbgI= mvdan.cc/xurls/v2 v2.6.0/go.mod h1:bCvEZ1XvdA6wDnxY7jPPjEmigDtvtvPXAD/Exa9IMSk= diff --git a/pkg/hicli/events.go b/pkg/hicli/events.go index 31cb9bb..328027a 100644 --- a/pkg/hicli/events.go +++ b/pkg/hicli/events.go @@ -7,6 +7,8 @@ package hicli import ( + "encoding/json" + "go.mau.fi/util/jsontime" "maunium.net/go/mautrix/event" "maunium.net/go/mautrix/id" @@ -35,6 +37,13 @@ type SyncNotification struct { Room *database.Room `json:"-"` } +type SyncToDevice struct { + Sender id.UserID `json:"sender"` + Type event.Type `json:"type"` + Content json.RawMessage `json:"content"` + Encrypted bool `json:"encrypted"` +} + type SyncComplete struct { Since *string `json:"since,omitempty"` ClearState bool `json:"clear_state,omitempty"` @@ -44,6 +53,8 @@ type SyncComplete struct { InvitedRooms []*database.InvitedRoom `json:"invited_rooms"` SpaceEdges map[id.RoomID][]*database.SpaceEdge `json:"space_edges"` TopLevelSpaces []id.RoomID `json:"top_level_spaces"` + + ToDevice []*SyncToDevice `json:"to_device,omitempty"` } func (c *SyncComplete) Notifications(yield func(SyncNotification) bool) { diff --git a/pkg/hicli/hicli.go b/pkg/hicli/hicli.go index 3b41577..4c1a97e 100644 --- a/pkg/hicli/hicli.go +++ b/pkg/hicli/hicli.go @@ -50,6 +50,8 @@ type HiClient struct { syncErrors int lastSync time.Time + ToDeviceInSync atomic.Bool + EventHandler func(evt any) LogoutFunc func(context.Context) error diff --git a/pkg/hicli/json-commands.go b/pkg/hicli/json-commands.go index 40b57b0..c39b312 100644 --- a/pkg/hicli/json-commands.go +++ b/pkg/hicli/json-commands.go @@ -186,6 +186,11 @@ func (h *HiClient) handleJSONCommand(ctx context.Context, req *JSONCommand) (any return unmarshalAndCall(req.Data, func(params *ensureGroupSessionSharedParams) (bool, error) { return true, h.EnsureGroupSessionShared(ctx, params.RoomID) }) + case "send_to_device": + return unmarshalAndCall(req.Data, func(params *sendToDeviceParams) (*mautrix.RespSendToDevice, error) { + params.EventType.Class = event.ToDeviceEventType + return h.SendToDevice(ctx, params.EventType, params.ReqSendToDevice, params.Encrypted) + }) case "resolve_alias": return unmarshalAndCall(req.Data, func(params *resolveAliasParams) (*mautrix.RespAliasResolve, error) { return h.Client.ResolveAlias(ctx, params.Alias) @@ -238,6 +243,14 @@ func (h *HiClient) handleJSONCommand(ctx context.Context, req *JSONCommand) (any return unmarshalAndCall(req.Data, func(params *database.PushRegistration) (bool, error) { return true, h.DB.PushRegistration.Put(ctx, params) }) + case "listen_to_device": + return unmarshalAndCall(req.Data, func(listen *bool) (bool, error) { + return h.ToDeviceInSync.Swap(*listen), nil + }) + case "get_turn_servers": + return h.Client.TurnServer(ctx) + case "get_media_config": + return h.Client.GetMediaConfig(ctx) default: return nil, fmt.Errorf("unknown command %q", req.Command) } @@ -357,6 +370,12 @@ type ensureGroupSessionSharedParams struct { RoomID id.RoomID `json:"room_id"` } +type sendToDeviceParams struct { + *mautrix.ReqSendToDevice + EventType event.Type `json:"event_type"` + Encrypted bool `json:"encrypted"` +} + type resolveAliasParams struct { Alias id.RoomAlias `json:"alias"` } diff --git a/pkg/hicli/send.go b/pkg/hicli/send.go index 061ca0a..8a46230 100644 --- a/pkg/hicli/send.go +++ b/pkg/hicli/send.go @@ -247,6 +247,10 @@ func (h *HiClient) Send( content any, disableEncryption bool, ) (*database.Event, error) { + if evtType == event.EventRedaction { + // TODO implement + return nil, fmt.Errorf("redaction is not supported") + } return h.send(ctx, roomID, evtType, content, "", disableEncryption) } @@ -425,6 +429,18 @@ func (h *HiClient) EnsureGroupSessionShared(ctx context.Context, roomID id.RoomI } } +func (h *HiClient) SendToDevice(ctx context.Context, evtType event.Type, content *mautrix.ReqSendToDevice, encrypt bool) (*mautrix.RespSendToDevice, error) { + if encrypt { + var err error + content, err = h.Crypto.EncryptToDevices(ctx, evtType, content) + if err != nil { + return nil, fmt.Errorf("failed to encrypt: %w", err) + } + evtType = event.ToDeviceEncrypted + } + return h.Client.SendToDevice(ctx, evtType, content) +} + func (h *HiClient) loadMembers(ctx context.Context, room *database.Room) error { if room.HasMemberList { return nil diff --git a/pkg/hicli/sync.go b/pkg/hicli/sync.go index 074d7e0..e39d384 100644 --- a/pkg/hicli/sync.go +++ b/pkg/hicli/sync.go @@ -66,6 +66,9 @@ func (h *HiClient) markSyncOK() { func (h *HiClient) preProcessSyncResponse(ctx context.Context, resp *mautrix.RespSync, since string) error { log := zerolog.Ctx(ctx) + listenToDevice := h.ToDeviceInSync.Load() + var syncTD []*SyncToDevice + postponedToDevices := resp.ToDevice.Events[:0] for _, evt := range resp.ToDevice.Events { evt.Type.Class = event.ToDeviceEventType @@ -80,7 +83,15 @@ func (h *HiClient) preProcessSyncResponse(ctx context.Context, resp *mautrix.Res switch content := evt.Content.Parsed.(type) { case *event.EncryptedEventContent: - h.Crypto.HandleEncryptedEvent(ctx, evt) + unhandledDecrypted := h.Crypto.HandleEncryptedEvent(ctx, evt) + if unhandledDecrypted != nil && listenToDevice { + syncTD = append(syncTD, &SyncToDevice{ + Sender: evt.Sender, + Type: unhandledDecrypted.Type, + Content: unhandledDecrypted.Content.VeryRaw, + Encrypted: true, + }) + } case *event.RoomKeyWithheldEventContent: // TODO move this check to mautrix-go? if evt.Sender == h.Account.UserID && content.Code == event.RoomKeyWithheldUnavailable { @@ -88,11 +99,22 @@ func (h *HiClient) preProcessSyncResponse(ctx context.Context, resp *mautrix.Res } else { h.Crypto.HandleRoomKeyWithheld(ctx, content) } - default: + case *event.SecretRequestEventContent, *event.RoomKeyRequestEventContent: postponedToDevices = append(postponedToDevices, evt) + default: + if listenToDevice { + syncTD = append(syncTD, &SyncToDevice{ + Sender: evt.Sender, + Type: evt.Type, + Content: evt.Content.VeryRaw, + }) + } } } resp.ToDevice.Events = postponedToDevices + if len(syncTD) > 0 { + ctx.Value(syncContextKey).(*syncContext).evt.ToDevice = syncTD + } h.Crypto.MarkOlmHashSavePoint(ctx) return nil diff --git a/web/package-lock.json b/web/package-lock.json index 015f5ba..a5ad218 100644 --- a/web/package-lock.json +++ b/web/package-lock.json @@ -11,8 +11,10 @@ "dependencies": { "@wailsio/runtime": "^3.0.0-alpha.29", "blurhash": "^2.0.5", + "fast-deep-equal": "^3.1.3", "katex": "^0.16.11", "leaflet": "^1.9.4", + "matrix-widget-api": "^1.13.1", "monaco-editor": "^0.52.0", "react": "^19.0.0", "react-blurhash": "^0.3.0", @@ -1796,6 +1798,12 @@ "dev": true, "license": "MIT" }, + "node_modules/@types/events": { + "version": "3.0.3", + "resolved": "https://registry.npmjs.org/@types/events/-/events-3.0.3.tgz", + "integrity": "sha512-trOc4AAUThEz9hapPtSd7wf5tiQKvTtu5b371UxXdTuqzIh0ArcRspRP0i0Viu+LXstIQ1z96t1nsPxT9ol01g==", + "license": "MIT" + }, "node_modules/@types/geojson": { "version": "7946.0.16", "resolved": "https://registry.npmjs.org/@types/geojson/-/geojson-7946.0.16.tgz", @@ -3314,11 +3322,19 @@ "node": ">=0.10.0" } }, + "node_modules/events": { + "version": "3.3.0", + "resolved": "https://registry.npmjs.org/events/-/events-3.3.0.tgz", + "integrity": "sha512-mQw+2fkQbALzQ7V0MY0IqdnXNOeTtP4r0lN9z7AAawCXgqea7bDii20AYrIBrFd/Hx0M2Ocz6S111CaFkUcb0Q==", + "license": "MIT", + "engines": { + "node": ">=0.8.x" + } + }, "node_modules/fast-deep-equal": { "version": "3.1.3", "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-3.1.3.tgz", "integrity": "sha512-f3qQ9oQy9j2AhBe/H9VC91wLmKBCCU/gDOnKNAYG5hswO7BLKj09Hc5HYNz9cGI++xlpDCIgDaitVs03ATR84Q==", - "dev": true, "license": "MIT" }, "node_modules/fast-glob": { @@ -4386,6 +4402,16 @@ "node": ">= 0.4" } }, + "node_modules/matrix-widget-api": { + "version": "1.13.1", + "resolved": "https://registry.npmjs.org/matrix-widget-api/-/matrix-widget-api-1.13.1.tgz", + "integrity": "sha512-mkOHUVzaN018TCbObfGOSaMW2GoUxOfcxNNlTVx5/HeMk3OSQPQM0C9oEME5Liiv/dBUoSrEB64V8wF7e/gb1w==", + "license": "Apache-2.0", + "dependencies": { + "@types/events": "^3.0.0", + "events": "^3.2.0" + } + }, "node_modules/merge2": { "version": "1.4.1", "resolved": "https://registry.npmjs.org/merge2/-/merge2-1.4.1.tgz", diff --git a/web/package.json b/web/package.json index 76b4398..536721f 100644 --- a/web/package.json +++ b/web/package.json @@ -13,8 +13,10 @@ "dependencies": { "@wailsio/runtime": "^3.0.0-alpha.29", "blurhash": "^2.0.5", + "fast-deep-equal": "^3.1.3", "katex": "^0.16.11", "leaflet": "^1.9.4", + "matrix-widget-api": "^1.13.1", "monaco-editor": "^0.52.0", "react": "^19.0.0", "react-blurhash": "^0.3.0", diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 6f5611b..2b7ad24 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -16,7 +16,7 @@ import type { MouseEvent } from "react" import { CachedEventDispatcher, NonNullCachedEventDispatcher } from "../util/eventdispatcher.ts" import RPCClient, { SendMessageParams } from "./rpc.ts" -import { RoomStateStore, StateStore } from "./statestore" +import { RoomStateStore, StateStore, WidgetListener } from "./statestore" import type { ClientState, ElementRecentEmoji, @@ -41,6 +41,7 @@ export default class Client { #stateRequests: RoomStateGUID[] = [] #stateRequestPromise: Promise | null = null #gcInterval: number | undefined + #toDeviceRequested = false constructor(readonly rpc: RPCClient) { this.rpc.event.listen(this.#handleEvent) @@ -154,6 +155,22 @@ export default class Client { navigator.registerProtocolHandler("matrix", "#/uri/%s") } + addWidgetListener(listener: WidgetListener): () => void { + this.store.widgetListeners.add(listener) + // TODO only request to-device events if there are widgets that need them? + if (!this.#toDeviceRequested) { + this.#toDeviceRequested = true + this.rpc.setListenToDevice(true) + } + return () => { + this.store.widgetListeners.delete(listener) + if (this.store.widgetListeners.size === 0 && this.#toDeviceRequested) { + this.#toDeviceRequested = false + this.rpc.setListenToDevice(false) + } + } + } + start(): () => void { const abort = new AbortController() if (window.gomuksAndroid) { diff --git a/web/src/api/rpc.ts b/web/src/api/rpc.ts index 6b443e4..a9be534 100644 --- a/web/src/api/rpc.ts +++ b/web/src/api/rpc.ts @@ -37,8 +37,10 @@ import type { ReqCreateRoom, ResolveAliasResponse, RespCreateRoom, + RespMediaConfig, RespOpenIDToken, RespRoomJoin, + RespTurnServer, RoomAlias, RoomID, RoomStateGUID, @@ -212,6 +214,14 @@ export default abstract class RPCClient { return this.request("ensure_group_session_shared", { room_id }) } + sendToDevice( + event_type: EventType, + messages: { [userId: string]: { [deviceId: string]: object } }, + encrypted: boolean = false, + ): Promise { + return this.request("send_to_device", { event_type, messages, encrypted }) + } + getSpecificRoomState(keys: RoomStateGUID[]): Promise { return this.request("get_specific_room_state", { keys }) } @@ -289,4 +299,16 @@ export default abstract class RPCClient { registerPush(reg: DBPushRegistration): Promise { return this.request("register_push", reg) } + + getTurnServers(): Promise { + return this.request("get_turn_servers", {}) + } + + getMediaConfig(): Promise { + return this.request("get_media_config", {}) + } + + setListenToDevice(listen: boolean): Promise { + return this.request("listen_to_device", listen) + } } diff --git a/web/src/api/statestore/main.ts b/web/src/api/statestore/main.ts index 5e808a4..804d118 100644 --- a/web/src/api/statestore/main.ts +++ b/web/src/api/statestore/main.ts @@ -32,6 +32,7 @@ import { SendCompleteData, SyncCompleteData, SyncRoom, + SyncToDevice, TypingEventData, UnknownEventContent, UserID, @@ -61,6 +62,13 @@ export interface GCSettings { lastOpenedCutoff: number, } +export interface WidgetListener { + onTimelineEvent(evt: MemDBEvent): void + onStateEvent(evt: MemDBEvent): void + onToDeviceEvent(evt: SyncToDevice): void + onRoomChange(roomID: RoomID | null): void +} + window.gcSettings ??= { // Run garbage collection every 15 minutes. interval: 15 * 60 * 1000, @@ -98,9 +106,19 @@ export class StateStore { readonly localPreferenceCache: Preferences = getLocalStoragePreferences("global_prefs", this.preferenceSub.notify) serverPreferenceCache: Preferences = {} switchRoom?: (roomID: RoomID | null) => void - activeRoomID: RoomID | null = null + #activeRoomID: RoomID | null = null activeRoomIsPreview: boolean = false imageAuthToken?: string + readonly widgetListeners: Set = new Set() + + get activeRoomID(): RoomID | null { + return this.#activeRoomID + } + + set activeRoomID(roomID: RoomID | null) { + this.#activeRoomID = roomID + this.widgetListeners.forEach(listener => listener.onRoomChange(roomID)) + } #roomListFilterFunc = (entry: RoomListEntry) => { if (this.currentRoomListQuery && !entry.search_name.includes(this.currentRoomListQuery)) { @@ -243,6 +261,11 @@ export class StateStore { } const resyncRoomList = this.roomList.current.length === 0 const changedRoomListEntries = new Map() + if (sync.to_device?.length && this.widgetListeners.size > 0) { + for (const listener of this.widgetListeners) { + sync.to_device.forEach(listener.onToDeviceEvent) + } + } for (const data of sync.invited_rooms ?? []) { const room = new InvitedRoomStore(data, this) this.inviteRooms.set(room.room_id, room) diff --git a/web/src/api/statestore/room.ts b/web/src/api/statestore/room.ts index 02a53a4..78cd474 100644 --- a/web/src/api/statestore/room.ts +++ b/web/src/api/statestore/room.ts @@ -457,6 +457,8 @@ export class RoomStateStore { for (const evt of sync.events ?? []) { this.applyEvent(evt) } + const hasWidgets = this.parent.widgetListeners.size > 0 + const newState: MemDBEvent[] = [] for (const [evtType, changedEvts] of Object.entries(sync.state ?? {})) { let stateMap = this.state.get(evtType) if (!stateMap) { @@ -466,6 +468,12 @@ export class RoomStateStore { for (const [key, rowID] of Object.entries(changedEvts)) { stateMap.set(key, rowID) this.invalidateStateCaches(evtType, key) + if (hasWidgets) { + const evt = this.eventsByRowID.get(rowID) + if (evt) { + newState.push(evt) + } + } } this.stateSubs.notify(evtType) } @@ -485,6 +493,13 @@ export class RoomStateStore { for (const [evtID, receipts] of Object.entries(sync.receipts ?? {})) { this.applyReceipts(receipts, evtID, false) } + if (hasWidgets && ((sync.timeline && sync.timeline.length > 0) || newState.length > 0)) { + const evts = sync.timeline?.map(evt => this.eventsByRowID.get(evt.event_rowid)).filter(evt => !!evt) + this.parent.widgetListeners.forEach(listener => { + evts?.forEach(listener.onTimelineEvent) + newState.forEach(listener.onStateEvent) + }) + } } applyState(evt: RawDBEvent) { diff --git a/web/src/api/types/hievents.ts b/web/src/api/types/hievents.ts index 125eb57..8efbbaf 100644 --- a/web/src/api/types/hievents.ts +++ b/web/src/api/types/hievents.ts @@ -86,6 +86,13 @@ export interface SyncNotification { sound: boolean } +export interface SyncToDevice { + sender: UserID + type: EventType + content: Record + encrypted: boolean +} + export interface SyncCompleteData { rooms: Record | null invited_rooms: DBInvitedRoom[] | null @@ -95,6 +102,7 @@ export interface SyncCompleteData { top_level_spaces: RoomID[] | null since?: string clear_state?: boolean + to_device?: SyncToDevice[] | null } export interface SyncCompleteEvent extends BaseRPCCommand { diff --git a/web/src/api/types/mxtypes.ts b/web/src/api/types/mxtypes.ts index 9413e49..e576127 100644 --- a/web/src/api/types/mxtypes.ts +++ b/web/src/api/types/mxtypes.ts @@ -346,3 +346,15 @@ export interface ReqCreateRoom { export interface RespCreateRoom { room_id: RoomID } + +export interface RespTurnServer { + username: string + password: string + ttl: number + uris: string[] +} + +export interface RespMediaConfig { + "m.upload.size": number + [key: string]: unknown +} diff --git a/web/src/api/types/preferences/preferences.ts b/web/src/api/types/preferences/preferences.ts index 94e8764..696083a 100644 --- a/web/src/api/types/preferences/preferences.ts +++ b/web/src/api/types/preferences/preferences.ts @@ -139,6 +139,12 @@ export const preferences = { allowedContexts: anyContext, defaultValue: "https://tile.openstreetmap.org/{z}/{x}/{y}.png", }), + element_call_base_url: new Preference({ + displayName: "Element call base URL", + description: "The widget base URL for Element calls.", + allowedContexts: anyContext, + defaultValue: "https://call.element.io", + }), gif_provider: new Preference({ displayName: "GIF provider", description: "The service to use to search for GIFs", diff --git a/web/src/icons/widgets.svg b/web/src/icons/widgets.svg new file mode 100644 index 0000000..139afad --- /dev/null +++ b/web/src/icons/widgets.svg @@ -0,0 +1 @@ + diff --git a/web/src/ui/MainScreen.tsx b/web/src/ui/MainScreen.tsx index bd7c4ed..35804b5 100644 --- a/web/src/ui/MainScreen.tsx +++ b/web/src/ui/MainScreen.tsx @@ -13,6 +13,7 @@ // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +import equal from "fast-deep-equal" import { JSX, use, useEffect, useMemo, useReducer, useRef, useState } from "react" import { SyncLoader } from "react-spinners" import Client from "@/api/client.ts" @@ -32,19 +33,6 @@ import RoomView from "./roomview/RoomView.tsx" import { useResizeHandle } from "./util/useResizeHandle.tsx" import "./MainScreen.css" -function objectIsEqual(a: RightPanelProps | null, b: RightPanelProps | null): boolean { - if (a === null || b === null) { - return a === null && b === null - } - for (const key of Object.keys(a)) { - // @ts-expect-error 3:< - if (a[key] !== b[key]) { - return false - } - } - return true -} - class ContextFields implements MainScreenContextFields { public keybindings: Keybindings private rightPanelStack: RightPanelProps[] = [] @@ -64,10 +52,10 @@ class ContextFields implements MainScreenContextFields { } setRightPanel = (props: RightPanelProps | null, pushState = true) => { - if ((props?.type === "members" || props?.type === "pinned-messages") && !this.client.store.activeRoomID) { + if ((props?.type !== "user") && !this.client.store.activeRoomID) { props = null } - const isEqual = objectIsEqual(this.currentRightPanel, props) + const isEqual = equal(this.currentRightPanel, props) if (isEqual && !pushState) { return } @@ -81,7 +69,7 @@ class ContextFields implements MainScreenContextFields { } else { this.directSetRightPanel(props) for (let i = this.rightPanelStack.length - 1; i >= 0; i--) { - if (objectIsEqual(this.rightPanelStack[i], props)) { + if (equal(this.rightPanelStack[i], props)) { this.rightPanelStack = this.rightPanelStack.slice(0, i + 1) if (pushState) { history.go(i - this.rightPanelStack.length) @@ -219,7 +207,7 @@ class ContextFields implements MainScreenContextFields { clickRightPanelOpener = (evt: React.MouseEvent) => { evt.preventDefault() const type = evt.currentTarget.getAttribute("data-target-panel") - if (type === "pinned-messages" || type === "members") { + if (type === "pinned-messages" || type === "members" || type === "widgets") { this.setRightPanel({ type }) } else if (type === "user") { this.setRightPanel({ type, userID: evt.currentTarget.getAttribute("data-target-user")! }) diff --git a/web/src/ui/rightpanel/RightPanel.css b/web/src/ui/rightpanel/RightPanel.css index 597f4af..065bc06 100644 --- a/web/src/ui/rightpanel/RightPanel.css +++ b/web/src/ui/rightpanel/RightPanel.css @@ -51,6 +51,22 @@ div.right-panel-content.pinned-messages { } } +div.right-panel-content.widgets { + display: flex; + flex-direction: column; + gap: .5rem; + padding: .5rem; + + > button { + padding: .5rem; + width: 100%; + } + + > div.separator { + flex: 1; + } +} + div.right-panel-content.user { display: flex; flex-direction: column; diff --git a/web/src/ui/rightpanel/RightPanel.tsx b/web/src/ui/rightpanel/RightPanel.tsx index a891ee1..79f0029 100644 --- a/web/src/ui/rightpanel/RightPanel.tsx +++ b/web/src/ui/rightpanel/RightPanel.tsx @@ -13,21 +13,30 @@ // // You should have received a copy of the GNU Affero General Public License // along with this program. If not, see . +import type { IWidget } from "matrix-widget-api" import { JSX, use } from "react" import type { UserID } from "@/api/types" import MainScreenContext from "../MainScreenContext.ts" import ErrorBoundary from "../util/ErrorBoundary.tsx" +import ElementCall from "../widget/ElementCall.tsx" +import LazyWidget from "../widget/LazyWidget.tsx" import MemberList from "./MemberList.tsx" import PinnedMessages from "./PinnedMessages.tsx" import UserInfo from "./UserInfo.tsx" +import WidgetList from "./WidgetList.tsx" import BackIcon from "@/icons/back.svg?react" import CloseIcon from "@/icons/close.svg?react" import "./RightPanel.css" -export type RightPanelType = "pinned-messages" | "members" | "user" +export type RightPanelType = "pinned-messages" | "members" | "widgets" | "widget" | "user" | "element-call" interface RightPanelSimpleProps { - type: "pinned-messages" | "members" + type: "pinned-messages" | "members" | "widgets" | "element-call" +} + +interface RightPanelWidgetProps { + type: "widget" + info: IWidget } interface RightPanelUserProps { @@ -35,14 +44,20 @@ interface RightPanelUserProps { userID: UserID } -export type RightPanelProps = RightPanelUserProps | RightPanelSimpleProps +export type RightPanelProps = RightPanelUserProps | RightPanelWidgetProps | RightPanelSimpleProps -function getTitle(type: RightPanelType): string { - switch (type) { +function getTitle(props: RightPanelProps): string { + switch (props.type) { case "pinned-messages": return "Pinned Messages" case "members": return "Room Members" + case "widgets": + return "Widgets in room" + case "widget": + return props.info.name || "Widget" + case "element-call": + return "Element Call" case "user": return "User Info" } @@ -54,6 +69,12 @@ function renderRightPanelContent(props: RightPanelProps): JSX.Element | null { return case "members": return + case "widgets": + return + case "element-call": + return + case "widget": + return case "user": return } @@ -67,12 +88,17 @@ const RightPanel = (props: RightPanelProps) => { data-target-panel="members" onClick={mainScreen.clickRightPanelOpener} > + } else if (props.type === "element-call" || props.type === "widget") { + backButton = } return
{backButton} -
{getTitle(props.type)}
+
{getTitle(props)}
diff --git a/web/src/ui/rightpanel/WidgetList.tsx b/web/src/ui/rightpanel/WidgetList.tsx new file mode 100644 index 0000000..7d6a861 --- /dev/null +++ b/web/src/ui/rightpanel/WidgetList.tsx @@ -0,0 +1,55 @@ +// gomuks - A Matrix client written in Go. +// Copyright (C) 2025 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +import type { IWidget } from "matrix-widget-api" +import { use } from "react" +import MainScreenContext from "../MainScreenContext.ts" +import { RoomContext } from "../roomview/roomcontext.ts" + +const WidgetList = () => { + const roomCtx = use(RoomContext) + const mainScreen = use(MainScreenContext) + const widgets = roomCtx?.store.state.get("im.vector.modular.widgets") ?? new Map() + const widgetElements = [] + for (const [stateKey, rowid] of widgets.entries()) { + const evt = roomCtx?.store.eventsByRowID.get(rowid) + if (!evt || !evt.content.url) { + continue + } + const onClick = () => mainScreen.setRightPanel({ + type: "widget", + info: { + id: stateKey, + creatorUserId: evt.sender, + ...evt.content, + } as IWidget, + }) + widgetElements.push() + } + + const openElementCall = () => { + mainScreen.setRightPanel({ type: "element-call" }) + } + + return <> + {widgetElements} +
+ + +} + +export default WidgetList diff --git a/web/src/ui/roomview/RoomViewHeader.tsx b/web/src/ui/roomview/RoomViewHeader.tsx index a30d8e2..7f1948e 100644 --- a/web/src/ui/roomview/RoomViewHeader.tsx +++ b/web/src/ui/roomview/RoomViewHeader.tsx @@ -26,6 +26,7 @@ import CodeIcon from "@/icons/code.svg?react" import PeopleIcon from "@/icons/group.svg?react" import PinIcon from "@/icons/pin.svg?react" import SettingsIcon from "@/icons/settings.svg?react" +import WidgetIcon from "@/icons/widgets.svg?react" import "./RoomViewHeader.css" interface RoomViewHeaderProps { @@ -81,6 +82,11 @@ const RoomViewHeader = ({ room }: RoomViewHeaderProps) => { onClick={mainScreen.clickRightPanelOpener} title="Room Members" > +
diff --git a/web/src/ui/widget/ElementCall.tsx b/web/src/ui/widget/ElementCall.tsx new file mode 100644 index 0000000..802fe65 --- /dev/null +++ b/web/src/ui/widget/ElementCall.tsx @@ -0,0 +1,60 @@ +// gomuks - A Matrix client written in Go. +// Copyright (C) 2025 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +import { use, useMemo } from "react" +import { usePreference } from "@/api/statestore" +import ClientContext from "../ClientContext" +import { RoomContext } from "../roomview/roomcontext" +import LazyWidget from "./LazyWidget" + +const elementCallParams = new URLSearchParams({ + roomId: "$matrix_room_id", + theme: "$org.matrix.msc2873.client_theme", + userId: "$matrix_user_id", + deviceId: "$org.matrix.msc3819.matrix_device_id", + widgetId: "$matrix_widget_id", + perParticipantE2EE: "$perParticipantE2EE", + baseUrl: "$homeserverBaseURL", + intent: "join_existing", + hideHeader: "true", + confineToRoom: "true", +}).toString().replaceAll("%24", "$") + +const ElementCall = () => { + const room = use(RoomContext)?.store ?? null + const client = use(ClientContext)! + const baseURL = usePreference(client.store, room, "element_call_base_url") + const widgetInfo = useMemo(() => ({ + id: `fi.mau.gomuks.call.${crypto.randomUUID().replaceAll("-", "")}`, + creatorUserId: client.userID, + type: "m.call", + url: `${baseURL}/room?${elementCallParams}`, + waitForIframeLoad: false, + data: { + perParticipantE2EE: !!room?.meta.current.encryption_event, + // Note: this won't actually work because matrix-js-sdk drops the path prefix for media requests. + homeserverBaseURL: new URL( + `_gomuks/matrixcompat/${client.store.imageAuthToken}`, + window.location.href, + ).toString(), + }, + }), [room, client, baseURL]) + if (!room || !client) { + return null + } + return +} + +export default ElementCall diff --git a/web/src/ui/widget/LazyWidget.tsx b/web/src/ui/widget/LazyWidget.tsx new file mode 100644 index 0000000..75e981a --- /dev/null +++ b/web/src/ui/widget/LazyWidget.tsx @@ -0,0 +1,46 @@ +// gomuks - A Matrix client written in Go. +// Copyright (C) 2025 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +import type { IWidget } from "matrix-widget-api" +import { Suspense, lazy, use } from "react" +import { GridLoader } from "react-spinners" +import ClientContext from "../ClientContext" +import { RoomContext } from "../roomview/roomcontext" + +const Widget = lazy(() => import("./widget")) + +const widgetLoader =
+ +
+ + +export interface LazyWidgetProps { + info: IWidget +} + +const LazyWidget = ({ info }: LazyWidgetProps) => { + const room = use(RoomContext)?.store + const client = use(ClientContext) + if (!room || !client) { + return null + } + return ( + + + + ) +} + +export default LazyWidget diff --git a/web/src/ui/widget/Widget.css b/web/src/ui/widget/Widget.css new file mode 100644 index 0000000..9563544 --- /dev/null +++ b/web/src/ui/widget/Widget.css @@ -0,0 +1,9 @@ +div.right-panel-content.widget, div.right-panel-content.element-call { + overflow: hidden !important; + + > iframe.widget-iframe { + width: 100%; + height: 100%; + border: none; + } +} diff --git a/web/src/ui/widget/util.ts b/web/src/ui/widget/util.ts new file mode 100644 index 0000000..f463e2b --- /dev/null +++ b/web/src/ui/widget/util.ts @@ -0,0 +1,56 @@ +// gomuks - A Matrix client written in Go. +// Copyright (C) 2025 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +import type { IRoomEvent } from "matrix-widget-api" +import type { RoomStateStore } from "@/api/statestore" +import type { MemDBEvent } from "@/api/types" + +export function isRecord(value: unknown): value is Record { + return typeof value === "object" && value !== null +} + +export function notNull(value: T | null | undefined): value is T { + return value !== null && value !== undefined +} + +export function memDBEventToIRoomEvent(evt: MemDBEvent): IRoomEvent { + return { + type: evt.type, + sender: evt.sender, + event_id: evt.event_id, + room_id: evt.room_id, + state_key: evt.state_key, + origin_server_ts: evt.timestamp, + content: evt.content, + unsigned: evt.unsigned, + } +} + +export function * iterRoomTimeline(room: RoomStateStore, since: string | undefined) { + const tc = room.timelineCache + for (let i = tc.length - 1; i >= 0; i--) { + const evt = tc[i]! + if (evt.event_id === since) { + return + } + yield evt + } +} + +export function filterEvent(eventType: string, msgtype: string | undefined, stateKey: string | undefined) { + return (evt: MemDBEvent) => evt.type === eventType + && (!msgtype || evt.content.msgtype === msgtype) + && (!stateKey || evt.state_key === stateKey) +} diff --git a/web/src/ui/widget/widget.tsx b/web/src/ui/widget/widget.tsx new file mode 100644 index 0000000..9589414 --- /dev/null +++ b/web/src/ui/widget/widget.tsx @@ -0,0 +1,122 @@ +// gomuks - A Matrix client written in Go. +// Copyright (C) 2025 Tulir Asokan +// +// This program is free software: you can redistribute it and/or modify +// it under the terms of the GNU Affero General Public License as published by +// the Free Software Foundation, either version 3 of the License, or +// (at your option) any later version. +// +// This program is distributed in the hope that it will be useful, +// but WITHOUT ANY WARRANTY; without even the implied warranty of +// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the +// GNU Affero General Public License for more details. +// +// You should have received a copy of the GNU Affero General Public License +// along with this program. If not, see . +import { ClientWidgetApi, IWidget, Widget as WrappedWidget } from "matrix-widget-api" +import { memo } from "react" +import type Client from "@/api/client" +import type { RoomStateStore, WidgetListener } from "@/api/statestore" +import type { MemDBEvent, RoomID, SyncToDevice } from "@/api/types" +import { getDisplayname } from "@/util/validation" +import { memDBEventToIRoomEvent } from "./util" +import GomuksWidgetDriver from "./widgetDriver" +import "./Widget.css" + +export interface WidgetProps { + info: IWidget + room: RoomStateStore + client: Client +} + +// TODO remove this after widgets start using a parameter for it +const addLegacyParams = (url: string, widgetID: string) => { + const urlObj = new URL(url) + urlObj.searchParams.set("parentUrl", window.location.href) + urlObj.searchParams.set("widgetId", widgetID) + return urlObj.toString() +} + +class WidgetListenerImpl implements WidgetListener { + constructor(private api: ClientWidgetApi) {} + + onTimelineEvent = (evt: MemDBEvent) => { + this.api.feedEvent(memDBEventToIRoomEvent(evt)) + .catch(err => console.error("Failed to feed event", memDBEventToIRoomEvent(evt), err)) + } + + onStateEvent = (evt: MemDBEvent) => { + this.api.feedStateUpdate(memDBEventToIRoomEvent(evt)) + .catch(err => console.error("Failed to feed state update", memDBEventToIRoomEvent(evt), err)) + } + + onRoomChange = (roomID: RoomID | null) => { + this.api.setViewedRoomId(roomID) + } + + onToDeviceEvent = (evt: SyncToDevice) => { + this.api.feedToDevice({ + sender: evt.sender, + type: evt.type, + content: evt.content, + // Why does this use the IRoomEvent interface?? + event_id: "", + room_id: "", + origin_server_ts: 0, + unsigned: {}, + }, evt.encrypted).catch(err => console.error("Failed to feed to-device event", evt, err)) + } +} + +const ReactWidget = ({ room, info, client }: WidgetProps) => { + const wrappedWidget = new WrappedWidget(info) + const driver = new GomuksWidgetDriver(client, room) + const widgetURL = addLegacyParams(wrappedWidget.getCompleteUrl({ + widgetRoomId: room.roomID, + currentUserId: client.userID, + deviceId: client.state.current?.is_logged_in ? client.state.current.device_id : "", + userDisplayName: getDisplayname(client.userID, room.getStateEvent("m.room.member", client.userID)?.content), + clientId: "fi.mau.gomuks", + clientTheme: window.matchMedia("(prefers-color-scheme: dark)").matches ? "dark" : "light", + clientLanguage: navigator.language, + }), wrappedWidget.id) + + const handleIframe = (iframe: HTMLIFrameElement) => { + console.info("Setting up widget API for", iframe) + const clientAPI = new ClientWidgetApi(wrappedWidget, iframe, driver) + clientAPI.setViewedRoomId(room.roomID) + + clientAPI.on("ready", () => console.info("Widget is ready")) + // Suppress unnecessary events to avoid errors + const noopReply = (evt: CustomEvent) => { + evt.preventDefault() + clientAPI.transport.reply(evt.detail, {}) + } + clientAPI.on("action:io.element.join", noopReply) + clientAPI.on("action:im.vector.hangup", noopReply) + clientAPI.on("action:io.element.device_mute", noopReply) + clientAPI.on("action:io.element.tile_layout", noopReply) + clientAPI.on("action:io.element.spotlight_layout", noopReply) + // TODO handle this one? + clientAPI.on("action:io.element.close", noopReply) + clientAPI.on("action:set_always_on_screen", noopReply) + const removeListener = client.addWidgetListener(new WidgetListenerImpl(clientAPI)) + + return () => { + console.info("Removing widget API") + removeListener() + clientAPI.stop() + clientAPI.removeAllListeners() + } + } + + return