From 05f64edeaf86b33984e72fc1b91c521a18645af3 Mon Sep 17 00:00:00 2001 From: Tulir Asokan Date: Mon, 18 Nov 2024 02:19:21 +0200 Subject: [PATCH] web/statestore: add garbage collection Fixes #490 --- web/src/api/client.ts | 10 +++- web/src/api/statestore/main.ts | 27 +++++++++++ web/src/api/statestore/room.ts | 70 +++++++++++++++++++++++++--- web/src/ui/MainScreen.tsx | 14 +++--- web/src/ui/timeline/TimelineView.tsx | 5 +- web/src/util/polyfill.js | 11 +++++ web/src/vite-env.d.ts | 3 +- 7 files changed, 124 insertions(+), 16 deletions(-) diff --git a/web/src/api/client.ts b/web/src/api/client.ts index 24f2f9c..8c3c9fa 100644 --- a/web/src/api/client.ts +++ b/web/src/api/client.ts @@ -35,6 +35,7 @@ export default class Client { readonly store = new StateStore() #stateRequests: RoomStateGUID[] = [] #stateRequestQueued = false + #gcInterval: number | undefined constructor(readonly rpc: RPCClient) { this.rpc.event.listen(this.#handleEvent) @@ -71,9 +72,13 @@ export default class Client { start(): () => void { const abort = new AbortController() this.#reallyStart(abort.signal) + this.#gcInterval = setInterval(() => { + console.log("Garbage collection completed:", this.store.doGarbageCollection()) + }, window.gcSettings.interval) return () => { abort.abort() this.rpc.stop() + clearInterval(this.#gcInterval) } } @@ -276,7 +281,10 @@ export default class Client { room.paginating = true try { const oldestRowID = room.timeline[0]?.timeline_rowid - const resp = await this.rpc.paginate(roomID, oldestRowID ?? 0, 100) + // Request 50 messages at a time first, increase batch size when going further + const count = room.timeline.length < 100 ? 50 : 100 + console.log("Requesting", count, "messages of history in", roomID) + const resp = await this.rpc.paginate(roomID, oldestRowID ?? 0, count) if (room.timeline[0]?.timeline_rowid !== oldestRowID) { throw new Error("Timeline changed while loading history") } diff --git a/web/src/api/statestore/main.ts b/web/src/api/statestore/main.ts index d03dde2..4f064b8 100644 --- a/web/src/api/statestore/main.ts +++ b/web/src/api/statestore/main.ts @@ -53,6 +53,18 @@ export interface RoomListEntry { marked_unread: boolean } +export interface GCSettings { + interval: number, + lastOpenedCutoff: number, +} + +window.gcSettings ??= { + // Run garbage collection every 15 minutes. + interval: 15 * 60 * 1000, + // Run garbage collection to rooms not opened in the past 30 minutes. + lastOpenedCutoff: 30 * 60 * 1000, +} + export class StateStore { readonly rooms: Map = new Map() readonly roomList = new NonNullCachedEventDispatcher([]) @@ -359,4 +371,19 @@ export class StateStore { } } } + + doGarbageCollection() { + const maxLastOpened = Date.now() - window.gcSettings.lastOpenedCutoff + let deletedEvents = 0 + let deletedState = 0 + for (const room of this.rooms.values()) { + if (room.roomID === this.activeRoomID || room.lastOpened > maxLastOpened) { + continue + } + const [de, ds] = room.doGarbageCollection() + deletedEvents += de + deletedState += ds + } + return { deletedEvents, deletedState } as const + } } diff --git a/web/src/api/statestore/room.ts b/web/src/api/statestore/room.ts index 893b145..50e9f3f 100644 --- a/web/src/api/statestore/room.ts +++ b/web/src/api/statestore/room.ts @@ -98,7 +98,7 @@ export class RoomStateStore { readonly accountData: Map = new Map() readonly accountDataSubs = new MultiSubscribable() readonly openNotifications: Map = new Map() - readonly emojiPacks: Map = new Map() + readonly #emojiPacksCache: Map = new Map() readonly preferences: Preferences readonly localPreferenceCache: Preferences readonly preferenceSub = new NoDataSubscribable() @@ -107,6 +107,7 @@ export class RoomStateStore { #autocompleteMembersCache: AutocompleteMemberEntry[] | null = null membersRequested: boolean = false #allPacksCache: Record | null = null + lastOpened: number = 0 readonly pendingEvents: EventRowID[] = [] paginating = false paginationRequestedForRow = -1 @@ -146,10 +147,10 @@ export class RoomStateStore { } getEmojiPack(key: string): CustomEmojiPack | null { - if (!this.emojiPacks.has(key)) { + if (!this.#emojiPacksCache.has(key)) { const pack = this.getStateEvent("im.ponies.room_emotes", key)?.content if (!pack || !pack.images) { - this.emojiPacks.set(key, null) + this.#emojiPacksCache.set(key, null) return null } const fallbackName = key === "" @@ -159,9 +160,9 @@ export class RoomStateStore { type: "im.ponies.room_emotes", state_key: key, }) - this.emojiPacks.set(key, parseCustomEmojiPack(pack as ImagePack, packID, fallbackName)) + this.#emojiPacksCache.set(key, parseCustomEmojiPack(pack as ImagePack, packID, fallbackName)) } - return this.emojiPacks.get(key) ?? null + return this.#emojiPacksCache.get(key) ?? null } getAllEmojiPacks(): Record { @@ -303,7 +304,7 @@ export class RoomStateStore { invalidateStateCaches(evtType: string, key: string) { if (evtType === "im.ponies.room_emotes") { - this.emojiPacks.delete(key) + this.#emojiPacksCache.delete(key) this.#allPacksCache = null this.parent.invalidateEmojiPacksCache() } else if (evtType === "m.room.member") { @@ -390,7 +391,7 @@ export class RoomStateStore { } stateMap.set(evt.state_key, evt.rowid) } - this.emojiPacks.clear() + this.#emojiPacksCache.clear() this.#allPacksCache = null if (omitMembers) { newStateMap.set("m.room.member", this.state.get("m.room.member") ?? new Map()) @@ -425,4 +426,59 @@ export class RoomStateStore { this.meta.current.preview_event_rowid = decrypted.preview_event_rowid } } + + doGarbageCollection() { + const memberEventsToKeep = new Set() + const eventsToKeep = new Set() + if (this.meta.current.preview_event_rowid) { + eventsToKeep.add(this.meta.current.preview_event_rowid) + const previewEvt = this.eventsByRowID.get(this.meta.current.preview_event_rowid) + if (previewEvt) { + if (previewEvt.last_edit_rowid) { + eventsToKeep.add(previewEvt.last_edit_rowid) + } + memberEventsToKeep.add(previewEvt?.sender ?? "") + } + } + const newState = new Map>() + let deletedState = 0 + newState.set("m.room.member", new Map( + this.state.get("m.room.member")?.entries().filter(([key, eventRowID]) => { + if (memberEventsToKeep.has(key)) { + eventsToKeep.add(eventRowID) + return true + } else { + deletedState++ + return false + } + }) ?? [], + )) + const emotes = this.state.get("im.ponies.room_emotes") + if (emotes) { + newState.set("im.ponies.room_emotes", emotes) + for (const rowid of emotes.values()) { + eventsToKeep.add(rowid) + } + } + this.state = newState + this.stateLoaded = false + this.fullMembersLoaded = false + this.membersRequested = false + this.#membersCache = null + this.#autocompleteMembersCache = null + this.paginationRequestedForRow = -1 + this.timeline = [] + this.notifyTimelineSubscribers() + const eventsToKeepList = this.eventsByRowID.values() + .filter(evt => eventsToKeep.has(evt.rowid)) + .toArray() + const deletedEvents = this.eventsByRowID.size - eventsToKeep.size + this.eventsByRowID.clear() + this.eventsByID.clear() + for (const evt of eventsToKeepList) { + this.eventsByRowID.set(evt.rowid, evt) + this.eventsByID.set(evt.event_id, evt) + } + return [deletedEvents, deletedState] as const + } } diff --git a/web/src/ui/MainScreen.tsx b/web/src/ui/MainScreen.tsx index a704fcc..8272bb5 100644 --- a/web/src/ui/MainScreen.tsx +++ b/web/src/ui/MainScreen.tsx @@ -68,14 +68,16 @@ class ContextFields implements MainScreenContextFields { window.activeRoom = room this.directSetActiveRoom(room) this.setRightPanel(null) - if (room?.stateLoaded === false) { - this.client.loadRoomState(room.roomID) - .catch(err => console.error("Failed to load room state", err)) - } this.client.store.activeRoomID = room?.roomID this.keybindings.activeRoom = room - if (roomID) { - document.querySelector(`div.room-entry[data-room-id="${CSS.escape(roomID)}"]`) + if (room) { + room.lastOpened = Date.now() + if (!room.stateLoaded) { + this.client.loadRoomState(room.roomID) + .catch(err => console.error("Failed to load room state", err)) + } + document + .querySelector(`div.room-entry[data-room-id="${CSS.escape(room.roomID)}"]`) ?.scrollIntoView({ block: "nearest" }) } } diff --git a/web/src/ui/timeline/TimelineView.tsx b/web/src/ui/timeline/TimelineView.tsx index 1d74272..2021679 100644 --- a/web/src/ui/timeline/TimelineView.tsx +++ b/web/src/ui/timeline/TimelineView.tsx @@ -84,7 +84,10 @@ const TimelineView = () => { const receiptType = roomCtx.store.preferences.send_read_receipts ? "m.read" : "m.read.private" client.rpc.markRead(room.roomID, newestEvent.event_id, receiptType).then( () => console.log("Marked read up to", newestEvent.event_id, newestEvent.timeline_rowid), - err => console.error(`Failed to send read receipt for ${newestEvent.event_id}:`, err), + err => { + console.error(`Failed to send read receipt for ${newestEvent.event_id}:`, err) + room.readUpToRow = -1 + }, ) } }, [focused, client, roomCtx, room, timeline]) diff --git a/web/src/util/polyfill.js b/web/src/util/polyfill.js index 830f731..b079e68 100644 --- a/web/src/util/polyfill.js +++ b/web/src/util/polyfill.js @@ -24,6 +24,17 @@ if (!window.Iterator?.prototype.map) { } return output } + (new Map([])).keys().__proto__.filter = function(callbackFn) { + const output = [] + let i = 0 + for (const item of this) { + if (callbackFn(item, i)) { + output.push(item) + } + i++ + } + return output + } Array.prototype.toArray = function() { return this } diff --git a/web/src/vite-env.d.ts b/web/src/vite-env.d.ts index 475ffd3..5edc4d9 100644 --- a/web/src/vite-env.d.ts +++ b/web/src/vite-env.d.ts @@ -2,7 +2,7 @@ /// import type Client from "@/api/client.ts" -import type { RoomStateStore } from "@/api/statestore" +import type { GCSettings, RoomStateStore } from "@/api/statestore" import type { MainScreenContextFields } from "@/ui/MainScreenContext.ts" declare global { @@ -11,5 +11,6 @@ declare global { activeRoom?: RoomStateStore | null mainScreenContext: MainScreenContextFields openLightbox: (params: { src: string, alt: string }) => void + gcSettings: GCSettings } }