web/statestore: add garbage collection

Fixes #490

web/src/api/client.ts

@@ -35,6 +35,7 @@ export default class Client {
 	readonly store = new StateStore()
 	#stateRequests: RoomStateGUID[] = []
 	#stateRequestQueued = false
+	#gcInterval: number | undefined
 
 	constructor(readonly rpc: RPCClient) {
 		this.rpc.event.listen(this.#handleEvent)
@@ -71,9 +72,13 @@ export default class Client {
 	start(): () => void {
 		const abort = new AbortController()
 		this.#reallyStart(abort.signal)
+		this.#gcInterval = setInterval(() => {
+			console.log("Garbage collection completed:", this.store.doGarbageCollection())
+		}, window.gcSettings.interval)
 		return () => {
 			abort.abort()
 			this.rpc.stop()
+			clearInterval(this.#gcInterval)
 		}
 	}
 
@@ -276,7 +281,10 @@ export default class Client {
 		room.paginating = true
 		try {
 			const oldestRowID = room.timeline[0]?.timeline_rowid
-			const resp = await this.rpc.paginate(roomID, oldestRowID ?? 0, 100)
+			// Request 50 messages at a time first, increase batch size when going further
+			const count = room.timeline.length < 100 ? 50 : 100
+			console.log("Requesting", count, "messages of history in", roomID)
+			const resp = await this.rpc.paginate(roomID, oldestRowID ?? 0, count)
 			if (room.timeline[0]?.timeline_rowid !== oldestRowID) {
 				throw new Error("Timeline changed while loading history")
 			}

web/src/api/statestore/main.ts

@@ -53,6 +53,18 @@ export interface RoomListEntry {
 	marked_unread: boolean
 }
 
+export interface GCSettings {
+	interval: number,
+	lastOpenedCutoff: number,
+}
+
+window.gcSettings ??= {
+	// Run garbage collection every 15 minutes.
+	interval: 15 * 60 * 1000,
+	// Run garbage collection to rooms not opened in the past 30 minutes.
+	lastOpenedCutoff: 30 * 60 * 1000,
+}
+
 export class StateStore {
 	readonly rooms: Map<RoomID, RoomStateStore> = new Map()
 	readonly roomList = new NonNullCachedEventDispatcher<RoomListEntry[]>([])
@@ -359,4 +371,19 @@ export class StateStore {
 			}
 		}
 	}
+
+	doGarbageCollection() {
+		const maxLastOpened = Date.now() - window.gcSettings.lastOpenedCutoff
+		let deletedEvents = 0
+		let deletedState = 0
+		for (const room of this.rooms.values()) {
+			if (room.roomID === this.activeRoomID || room.lastOpened > maxLastOpened) {
+				continue
+			}
+			const [de, ds] = room.doGarbageCollection()
+			deletedEvents += de
+			deletedState += ds
+		}
+		return { deletedEvents, deletedState } as const
+	}
 }

web/src/api/statestore/room.ts

@@ -98,7 +98,7 @@ export class RoomStateStore {
 	readonly accountData: Map<string, UnknownEventContent> = new Map()
 	readonly accountDataSubs = new MultiSubscribable()
 	readonly openNotifications: Map<EventRowID, Notification> = new Map()
-	readonly emojiPacks: Map<string, CustomEmojiPack | null> = new Map()
+	readonly #emojiPacksCache: Map<string, CustomEmojiPack | null> = new Map()
 	readonly preferences: Preferences
 	readonly localPreferenceCache: Preferences
 	readonly preferenceSub = new NoDataSubscribable()
@@ -107,6 +107,7 @@ export class RoomStateStore {
 	#autocompleteMembersCache: AutocompleteMemberEntry[] | null = null
 	membersRequested: boolean = false
 	#allPacksCache: Record<string, CustomEmojiPack> | null = null
+	lastOpened: number = 0
 	readonly pendingEvents: EventRowID[] = []
 	paginating = false
 	paginationRequestedForRow = -1
@@ -146,10 +147,10 @@ export class RoomStateStore {
 	}
 
 	getEmojiPack(key: string): CustomEmojiPack | null {
-		if (!this.emojiPacks.has(key)) {
+		if (!this.#emojiPacksCache.has(key)) {
 			const pack = this.getStateEvent("im.ponies.room_emotes", key)?.content
 			if (!pack || !pack.images) {
-				this.emojiPacks.set(key, null)
+				this.#emojiPacksCache.set(key, null)
 				return null
 			}
 			const fallbackName = key === ""
@@ -159,9 +160,9 @@ export class RoomStateStore {
 				type: "im.ponies.room_emotes",
 				state_key: key,
 			})
-			this.emojiPacks.set(key, parseCustomEmojiPack(pack as ImagePack, packID, fallbackName))
+			this.#emojiPacksCache.set(key, parseCustomEmojiPack(pack as ImagePack, packID, fallbackName))
 		}
-		return this.emojiPacks.get(key) ?? null
+		return this.#emojiPacksCache.get(key) ?? null
 	}
 
 	getAllEmojiPacks(): Record<string, CustomEmojiPack> {
@@ -303,7 +304,7 @@ export class RoomStateStore {
 
 	invalidateStateCaches(evtType: string, key: string) {
 		if (evtType === "im.ponies.room_emotes") {
-			this.emojiPacks.delete(key)
+			this.#emojiPacksCache.delete(key)
 			this.#allPacksCache = null
 			this.parent.invalidateEmojiPacksCache()
 		} else if (evtType === "m.room.member") {
@@ -390,7 +391,7 @@ export class RoomStateStore {
 			}
 			stateMap.set(evt.state_key, evt.rowid)
 		}
-		this.emojiPacks.clear()
+		this.#emojiPacksCache.clear()
 		this.#allPacksCache = null
 		if (omitMembers) {
 			newStateMap.set("m.room.member", this.state.get("m.room.member") ?? new Map())
@@ -425,4 +426,59 @@ export class RoomStateStore {
 			this.meta.current.preview_event_rowid = decrypted.preview_event_rowid
 		}
 	}
+
+	doGarbageCollection() {
+		const memberEventsToKeep = new Set<UserID>()
+		const eventsToKeep = new Set<EventRowID>()
+		if (this.meta.current.preview_event_rowid) {
+			eventsToKeep.add(this.meta.current.preview_event_rowid)
+			const previewEvt = this.eventsByRowID.get(this.meta.current.preview_event_rowid)
+			if (previewEvt) {
+				if (previewEvt.last_edit_rowid) {
+					eventsToKeep.add(previewEvt.last_edit_rowid)
+				}
+				memberEventsToKeep.add(previewEvt?.sender ?? "")
+			}
+		}
+		const newState = new Map<EventType, Map<string, EventRowID>>()
+		let deletedState = 0
+		newState.set("m.room.member", new Map<string, EventRowID>(
+			this.state.get("m.room.member")?.entries().filter(([key, eventRowID]) => {
+				if (memberEventsToKeep.has(key)) {
+					eventsToKeep.add(eventRowID)
+					return true
+				} else {
+					deletedState++
+					return false
+				}
+			}) ?? [],
+		))
+		const emotes = this.state.get("im.ponies.room_emotes")
+		if (emotes) {
+			newState.set("im.ponies.room_emotes", emotes)
+			for (const rowid of emotes.values()) {
+				eventsToKeep.add(rowid)
+			}
+		}
+		this.state = newState
+		this.stateLoaded = false
+		this.fullMembersLoaded = false
+		this.membersRequested = false
+		this.#membersCache = null
+		this.#autocompleteMembersCache = null
+		this.paginationRequestedForRow = -1
+		this.timeline = []
+		this.notifyTimelineSubscribers()
+		const eventsToKeepList = this.eventsByRowID.values()
+			.filter(evt => eventsToKeep.has(evt.rowid))
+			.toArray()
+		const deletedEvents = this.eventsByRowID.size - eventsToKeep.size
+		this.eventsByRowID.clear()
+		this.eventsByID.clear()
+		for (const evt of eventsToKeepList) {
+			this.eventsByRowID.set(evt.rowid, evt)
+			this.eventsByID.set(evt.event_id, evt)
+		}
+		return [deletedEvents, deletedState] as const
+	}
 }

web/src/ui/MainScreen.tsx

@@ -68,14 +68,16 @@ class ContextFields implements MainScreenContextFields {
 		window.activeRoom = room
 		this.directSetActiveRoom(room)
 		this.setRightPanel(null)
-		if (room?.stateLoaded === false) {
-			this.client.loadRoomState(room.roomID)
-				.catch(err => console.error("Failed to load room state", err))
-		}
 		this.client.store.activeRoomID = room?.roomID
 		this.keybindings.activeRoom = room
-		if (roomID) {
+		if (room) {
-			document.querySelector(`div.room-entry[data-room-id="${CSS.escape(roomID)}"]`)
+			room.lastOpened = Date.now()
+			if (!room.stateLoaded) {
+				this.client.loadRoomState(room.roomID)
+					.catch(err => console.error("Failed to load room state", err))
+			}
+			document
+				.querySelector(`div.room-entry[data-room-id="${CSS.escape(room.roomID)}"]`)
 				?.scrollIntoView({ block: "nearest" })
 		}
 	}

web/src/ui/timeline/TimelineView.tsx

@@ -84,7 +84,10 @@ const TimelineView = () => {
 			const receiptType = roomCtx.store.preferences.send_read_receipts ? "m.read" : "m.read.private"
 			client.rpc.markRead(room.roomID, newestEvent.event_id, receiptType).then(
 				() => console.log("Marked read up to", newestEvent.event_id, newestEvent.timeline_rowid),
-				err => console.error(`Failed to send read receipt for ${newestEvent.event_id}:`, err),
+				err => {
+					console.error(`Failed to send read receipt for ${newestEvent.event_id}:`, err)
+					room.readUpToRow = -1
+				},
 			)
 		}
 	}, [focused, client, roomCtx, room, timeline])

web/src/util/polyfill.js

@@ -24,6 +24,17 @@ if (!window.Iterator?.prototype.map) {
 		}
 		return output
 	}
+	(new Map([])).keys().__proto__.filter = function(callbackFn) {
+		const output = []
+		let i = 0
+		for (const item of this) {
+			if (callbackFn(item, i)) {
+				output.push(item)
+			}
+			i++
+		}
+		return output
+	}
 	Array.prototype.toArray = function() {
 		return this
 	}

web/src/vite-env.d.ts

@@ -2,7 +2,7 @@
 /// <reference types="vite-plugin-svgr/client" />
 
 import type Client from "@/api/client.ts"
-import type { RoomStateStore } from "@/api/statestore"
+import type { GCSettings, RoomStateStore } from "@/api/statestore"
 import type { MainScreenContextFields } from "@/ui/MainScreenContext.ts"
 
 declare global {
@@ -11,5 +11,6 @@ declare global {
 		activeRoom?: RoomStateStore | null
 		mainScreenContext: MainScreenContextFields
 		openLightbox: (params: { src: string, alt: string }) => void
+		gcSettings: GCSettings
 	}
 }