web/store: only fetch full member list when needed

pkg/hicli/database/state.go

@@ -38,6 +38,7 @@ const (
 		JOIN event ON cs.event_rowid = event.rowid
 	`
 	getCurrentRoomStateQuery               = getCurrentRoomStateBaseQuery + `WHERE cs.room_id = $1`
+	getCurrentRoomStateWithoutMembersQuery = getCurrentRoomStateBaseQuery + `WHERE cs.room_id = $1 AND type<>'m.room.member'`
 	getManyCurrentRoomStateQuery           = getCurrentRoomStateBaseQuery + `WHERE (cs.room_id, cs.event_type, cs.state_key) IN (%s)`
 	getCurrentStateEventQuery              = getCurrentRoomStateBaseQuery + `WHERE cs.room_id = $1 AND cs.event_type = $2 AND cs.state_key = $3`
 )
@@ -113,3 +114,7 @@ func (csq *CurrentStateQuery) Get(ctx context.Context, roomID id.RoomID, eventTy
 func (csq *CurrentStateQuery) GetAll(ctx context.Context, roomID id.RoomID) ([]*Event, error) {
 	return csq.QueryMany(ctx, getCurrentRoomStateQuery, roomID)
 }
+
+func (csq *CurrentStateQuery) GetAllExceptMembers(ctx context.Context, roomID id.RoomID) ([]*Event, error) {
+	return csq.QueryMany(ctx, getCurrentRoomStateWithoutMembersQuery, roomID)
+}

pkg/hicli/json-commands.go

@@ -91,7 +91,7 @@ func (h *HiClient) handleJSONCommand(ctx context.Context, req *JSONCommand) (any
 		})
 	case "get_room_state":
 		return unmarshalAndCall(req.Data, func(params *getRoomStateParams) ([]*database.Event, error) {
-			return h.GetRoomState(ctx, params.RoomID, params.FetchMembers, params.Refetch)
+			return h.GetRoomState(ctx, params.RoomID, params.IncludeMembers, params.FetchMembers, params.Refetch)
 		})
 	case "get_specific_room_state":
 		return unmarshalAndCall(req.Data, func(params *getSpecificRoomStateParams) ([]*database.Event, error) {
@@ -215,6 +215,7 @@ type getRoomStateParams struct {
 	RoomID         id.RoomID `json:"room_id"`
 	Refetch        bool      `json:"refetch"`
 	FetchMembers   bool      `json:"fetch_members"`
+	IncludeMembers bool      `json:"include_members"`
 }
 
 type getSpecificRoomStateParams struct {

pkg/hicli/paginate.go

@@ -65,23 +65,25 @@ func (h *HiClient) GetEvent(ctx context.Context, roomID id.RoomID, eventID id.Ev
 	}
 }
 
-func (h *HiClient) GetRoomState(ctx context.Context, roomID id.RoomID, fetchMembers, refetch bool) ([]*database.Event, error) {
+func (h *HiClient) processGetRoomState(ctx context.Context, roomID id.RoomID, fetchMembers, refetch, dispatchEvt bool) error {
 	var evts []*event.Event
 	if refetch {
 		resp, err := h.Client.StateAsArray(ctx, roomID)
 		if err != nil {
-			return nil, fmt.Errorf("failed to refetch state: %w", err)
+			return fmt.Errorf("failed to refetch state: %w", err)
 		}
 		evts = resp
 	} else if fetchMembers {
 		resp, err := h.Client.Members(ctx, roomID)
 		if err != nil {
-			return nil, fmt.Errorf("failed to fetch members: %w", err)
+			return fmt.Errorf("failed to fetch members: %w", err)
 		}
 		evts = resp.Chunk
 	}
-	if evts != nil {
-		err := h.DB.DoTxn(ctx, nil, func(ctx context.Context) error {
+	if evts == nil {
+		return nil
+	}
+	return h.DB.DoTxn(ctx, nil, func(ctx context.Context) error {
 		room, err := h.DB.Room.Get(ctx, roomID)
 		if err != nil {
 			return fmt.Errorf("failed to get room from database: %w", err)
@@ -117,13 +119,47 @@ func (h *HiClient) GetRoomState(ctx context.Context, roomID id.RoomID, fetchMemb
 			if err != nil {
 				return fmt.Errorf("failed to save room data: %w", err)
 			}
+			if dispatchEvt {
+				h.EventHandler(&SyncComplete{
+					Rooms: map[id.RoomID]*SyncRoom{
+						roomID: {
+							Meta:          room,
+							Timeline:      make([]database.TimelineRowTuple, 0),
+							State:         make(map[event.Type]map[string]database.EventRowID),
+							AccountData:   make(map[event.Type]*database.AccountData),
+							Events:        make([]*database.Event, 0),
+							Reset:         false,
+							Notifications: make([]SyncNotification, 0),
+						},
+					},
+					AccountData: make(map[event.Type]*database.AccountData),
+					LeftRooms:   make([]id.RoomID, 0),
+				})
+			}
 		}
 		return nil
 	})
+}
+
+func (h *HiClient) GetRoomState(ctx context.Context, roomID id.RoomID, includeMembers, fetchMembers, refetch bool) ([]*database.Event, error) {
+	if fetchMembers || refetch {
+		if !includeMembers {
+			go func(ctx context.Context) {
+				err := h.processGetRoomState(ctx, roomID, fetchMembers, refetch, true)
+				if err != nil {
+					zerolog.Ctx(ctx).Err(err).Msg("Failed to fetch room state in background")
+				}
+			}(context.WithoutCancel(ctx))
+		} else {
+			err := h.processGetRoomState(ctx, roomID, fetchMembers, refetch, false)
 			if err != nil {
 				return nil, err
 			}
 		}
+	}
+	if !includeMembers {
+		return h.DB.CurrentState.GetAllExceptMembers(ctx, roomID)
+	}
 	return h.DB.CurrentState.GetAll(ctx, roomID)
 }
 

web/src/api/client.ts

@@ -31,6 +31,8 @@ import type {
 export default class Client {
 	readonly state = new CachedEventDispatcher<ClientState>()
 	readonly store = new StateStore()
+	#stateRequests: RoomStateGUID[] = []
+	#stateRequestQueued = false
 
 	constructor(readonly rpc: RPCClient) {
 		this.rpc.event.listen(this.#handleEvent)
@@ -91,6 +93,28 @@ export default class Client {
 		}
 	}
 
+	requestMemberEvent(room: RoomStateStore | RoomID | undefined, userID: UserID) {
+		if (typeof room === "string") {
+			room = this.store.rooms.get(room)
+		}
+		if (!room || room.state.get("m.room.member")?.has(userID) || room.requestedMembers.has(userID)) {
+			return
+		}
+		room.requestedMembers.add(userID)
+		this.#stateRequests.push({ room_id: room.roomID, type: "m.room.member", state_key: userID })
+		if (!this.#stateRequestQueued) {
+			this.#stateRequestQueued = true
+			window.queueMicrotask(this.doStateRequests)
+		}
+	}
+
+	doStateRequests = () => {
+		const reqs = this.#stateRequests
+		this.#stateRequestQueued = false
+		this.#stateRequests = []
+		this.loadSpecificRoomState(reqs).catch(err => console.error("Failed to load room state", reqs, err))
+	}
+
 	requestEvent(room: RoomStateStore | RoomID | undefined, eventID: EventID) {
 		if (typeof room === "string") {
 			room = this.store.rooms.get(room)
@@ -219,13 +243,22 @@ export default class Client {
 		}
 	}
 
-	async loadRoomState(roomID: RoomID, refetch = false): Promise<void> {
+	async loadRoomState(
+		roomID: RoomID, { omitMembers, refetch } = { omitMembers: true, refetch: false },
+	): Promise<void> {
 		const room = this.store.rooms.get(roomID)
 		if (!room) {
 			throw new Error("Room not found")
 		}
-		const state = await this.rpc.getRoomState(roomID, !room.meta.current.has_member_list, refetch)
-		room.applyFullState(state)
+		if (!omitMembers) {
+			room.membersRequested = true
+			console.log("Requesting full member list for", roomID)
+		}
+		const state = await this.rpc.getRoomState(roomID, !omitMembers, !room.meta.current.has_member_list, refetch)
+		room.applyFullState(state, omitMembers)
+		if (!omitMembers && !room.meta.current.has_member_list) {
+			room.meta.current.has_member_list = true
+		}
 	}
 
 	async loadMoreHistory(roomID: RoomID): Promise<void> {

web/src/api/rpc.ts

@@ -172,8 +172,10 @@ export default abstract class RPCClient {
 		return this.request("get_specific_room_state", { keys })
 	}
 
-	getRoomState(room_id: RoomID, fetch_members = false, refetch = false): Promise<RawDBEvent[]> {
-		return this.request("get_room_state", { room_id, fetch_members, refetch })
+	getRoomState(
+		room_id: RoomID, include_members = false, fetch_members = false, refetch = false,
+	): Promise<RawDBEvent[]> {
+		return this.request("get_room_state", { room_id, include_members, fetch_members, refetch })
 	}
 
 	getEvent(room_id: RoomID, event_id: EventID): Promise<RawDBEvent> {

web/src/api/statestore/room.ts

@@ -15,9 +15,11 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import { CustomEmojiPack, parseCustomEmojiPack } from "@/util/emoji"
 import { NonNullCachedEventDispatcher } from "@/util/eventdispatcher.ts"
+import toSearchableString from "@/util/searchablestring.ts"
 import Subscribable, { MultiSubscribable } from "@/util/subscribable.ts"
 import { getDisplayname } from "@/util/validation.ts"
 import {
+	ContentURI,
 	DBRoom,
 	EncryptedEventContent,
 	EventID,
@@ -69,6 +71,13 @@ function visibleMetaIsEqual(meta1: DBRoom, meta2: DBRoom): boolean {
 		meta1.has_member_list === meta2.has_member_list
 }
 
+export interface AutocompleteMemberEntry {
+	userID: UserID
+	displayName: string
+	avatarURL?: ContentURI
+	searchString: string
+}
+
 export class RoomStateStore {
 	readonly roomID: RoomID
 	readonly meta: NonNullCachedEventDispatcher<DBRoom>
@@ -76,15 +85,19 @@ export class RoomStateStore {
 	timelineCache: (MemDBEvent | null)[] = []
 	state: Map<EventType, Map<string, EventRowID>> = new Map()
 	stateLoaded = false
+	fullMembersLoaded = false
 	readonly eventsByRowID: Map<EventRowID, MemDBEvent> = new Map()
 	readonly eventsByID: Map<EventID, MemDBEvent> = new Map()
 	readonly timelineSub = new Subscribable()
 	readonly stateSubs = new MultiSubscribable()
 	readonly eventSubs = new MultiSubscribable()
 	readonly requestedEvents: Set<EventID> = new Set()
+	readonly requestedMembers: Set<UserID> = new Set()
 	readonly openNotifications: Map<EventRowID, Notification> = new Map()
 	readonly emojiPacks: Map<string, CustomEmojiPack | null> = new Map()
 	#membersCache: MemDBEvent[] | null = null
+	#autocompleteMembersCache: AutocompleteMemberEntry[] | null = null
+	membersRequested: boolean = false
 	#allPacksCache: Record<string, CustomEmojiPack> | null = null
 	readonly pendingEvents: EventRowID[] = []
 	paginating = false
@@ -155,18 +168,17 @@ export class RoomStateStore {
 		return this.#allPacksCache
 	}
 
-	getMembers = (): MemDBEvent[] => {
-		if (this.#membersCache === null) {
+	#fillMembersCache() {
 		const memberEvtIDs = this.state.get("m.room.member")
 		if (!memberEvtIDs) {
-				return []
+			return
 		}
 		const powerLevels: PowerLevelEventContent = this.getStateEvent("m.room.power_levels", "")?.content ?? {}
-			this.#membersCache = memberEvtIDs.values()
+		const membersCache = memberEvtIDs.values()
 			.map(rowID => this.eventsByRowID.get(rowID))
-				.filter(evt => !!evt)
+			.filter((evt): evt is MemDBEvent => !!evt && evt.content.membership === "join")
 			.toArray()
-			this.#membersCache.sort((a, b) => {
+		membersCache.sort((a, b) => {
 			const aUserID = a.state_key as UserID
 			const bUserID = b.state_key as UserID
 			const aPower = powerLevels.users?.[aUserID] ?? powerLevels.users_default ?? 0
@@ -181,8 +193,28 @@ export class RoomStateStore {
 			}
 			return aName.localeCompare(bName)
 		})
+		this.#autocompleteMembersCache = membersCache.map(evt => ({
+			userID: evt.state_key!,
+			displayName: getDisplayname(evt.state_key!, evt.content as MemberEventContent),
+			avatarURL: evt.content?.avatar_url,
+			searchString: toSearchableString(`${evt.content?.displayname ?? ""}${evt.state_key!.slice(1)}`),
+		}))
+		this.#membersCache = membersCache
+		return membersCache
 	}
-		return this.#membersCache
+
+	getMembers = (): MemDBEvent[] => {
+		if (this.#membersCache === null) {
+			this.#fillMembersCache()
+		}
+		return this.#membersCache ?? []
+	}
+
+	getAutocompleteMembers = (): AutocompleteMemberEntry[] => {
+		if (this.#autocompleteMembersCache === null) {
+			this.#fillMembersCache()
+		}
+		return this.#autocompleteMembersCache ?? []
 	}
 
 	getPinnedEvents(): EventID[] {
@@ -264,8 +296,13 @@ export class RoomStateStore {
 			this.emojiPacks.delete(key)
 			this.#allPacksCache = null
 			this.parent.invalidateEmojiPacksCache()
-		} else if (evtType === "m.room.member" || evtType === "m.room.power_levels") {
+		} else if (evtType === "m.room.member") {
 			this.#membersCache = null
+			this.#autocompleteMembersCache = null
+			this.requestedMembers.delete(key as UserID)
+		} else if (evtType === "m.room.power_levels") {
+			this.#membersCache = null
+			this.#autocompleteMembersCache = null
 		}
 		this.stateSubs.notify(this.stateSubKey(evtType, key))
 	}
@@ -321,7 +358,7 @@ export class RoomStateStore {
 		this.stateSubs.notify(evt.type)
 	}
 
-	applyFullState(state: RawDBEvent[]) {
+	applyFullState(state: RawDBEvent[], omitMembers: boolean) {
 		const newStateMap: Map<EventType, Map<string, EventRowID>> = new Map()
 		for (const evt of state) {
 			if (evt.state_key === undefined) {
@@ -337,9 +374,19 @@ export class RoomStateStore {
 		}
 		this.emojiPacks.clear()
 		this.#allPacksCache = null
+		if (omitMembers) {
+			newStateMap.set("m.room.member", this.state.get("m.room.member") ?? new Map())
+		} else {
+			this.#membersCache = null
+			this.#autocompleteMembersCache = null
+		}
 		this.state = newStateMap
 		this.stateLoaded = true
+		this.fullMembersLoaded = this.fullMembersLoaded || !omitMembers
 		for (const [evtType, stateMap] of newStateMap) {
+			if (omitMembers && evtType === "m.room.member") {
+				continue
+			}
 			for (const [key] of stateMap) {
 				this.stateSubs.notify(this.stateSubKey(evtType, key))
 			}

web/src/ui/composer/Autocompleter.css

@@ -11,6 +11,7 @@ div.autocompletions {
 	background-color: var(--background-color);
 	padding: .5rem;
 	max-height: 20rem;
+	max-width: 30rem;
 	overflow: auto;
 
 	> .autocompletion-item {
@@ -20,6 +21,10 @@ div.autocompletions {
 		align-items: center;
 		gap: .25rem;
 		cursor: var(--clickable-cursor);
+		overflow: hidden;
+		height: 2rem;
+		contain-intrinsic-height: 2rem;
+		content-visibility: auto;
 
 		&.selected, &:hover {
 			background-color: var(--light-hover-color);

web/src/ui/composer/Autocompleter.tsx

@@ -87,7 +87,10 @@ function useAutocompleter<T>({
 		}
 	}, [onSelect, setAutocomplete, params.selected, params.close])
 	const selected = params.selected !== undefined ? positiveMod(params.selected, items.length) : -1
-	return <div className={`autocompletions ${items.length === 0 ? "empty" : "has-items"}`} id="composer-autocompletions">
+	return <div
+		className={`autocompletions ${items.length === 0 ? "empty" : "has-items"}`}
+		id="composer-autocompletions"
+	>
 		{items.map((item, i) => <div
 			onClick={onClick}
 			data-index={i}

web/src/ui/composer/MessageComposer.tsx

@@ -378,7 +378,7 @@ const MessageComposer = () => {
 			onClose: () => textInput.current?.focus(),
 		})
 	})
-	const Autocompleter = getAutocompleter(autocomplete)
+	const Autocompleter = getAutocompleter(autocomplete, client, room)
 	return <>
 		{Autocompleter && autocomplete && <div className="autocompletions-wrapper"><Autocompleter
 			params={autocomplete}

web/src/ui/composer/getAutocompleter.ts

@@ -13,6 +13,8 @@
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import Client from "@/api/client.ts"
+import { RoomStateStore } from "@/api/statestore"
 import {
 	AutocompleteQuery,
 	AutocompleterProps,
@@ -36,10 +38,23 @@ export function charToAutocompleteType(newChar?: string): AutocompleteQuery["typ
 
 export const emojiQueryRegex = /[a-zA-Z0-9_+-]*$/
 
-export function getAutocompleter(params: AutocompleteQuery | null): React.ElementType<AutocompleterProps> | null {
+export function getAutocompleter(
+	params: AutocompleteQuery | null, client: Client, room: RoomStateStore,
+): React.ElementType<AutocompleterProps> | null {
 	switch (params?.type) {
-	case "user":
+	case "user": {
+		const memberCount = room.state.get("m.room.member")?.size ?? 0
+		if (memberCount > 500 && params.query.length < 2) {
+			return null
+		} else if (memberCount > 5000 && params.query.length < 3) {
+			return null
+		}
+		if (!room.membersRequested) {
+			room.membersRequested = true
+			client.loadRoomState(room.roomID, { omitMembers: false, refetch: false })
+		}
 		return UserAutocompleter
+	}
 	case "emoji":
 		if (params.query.length < 3) {
 			return null

web/src/ui/composer/userautocomplete.ts

@@ -15,7 +15,7 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import { useMemo, useRef } from "react"
 import { RoomStateStore } from "@/api/statestore"
-import type { ContentURI, MemberEventContent, UserID } from "@/api/types"
+import type { ContentURI, UserID } from "@/api/types"
 import toSearchableString from "@/util/searchablestring.ts"
 
 export interface AutocompleteUser {
@@ -34,35 +34,18 @@ export function filterAndSort(users: AutocompleteUser[], query: string): Autocom
 		.map(({ user }) => user)
 }
 
-export function getAutocompleteMemberList(room: RoomStateStore) {
-	const states = room.state.get("m.room.member")
-	if (!states) {
-		return []
-	}
-	const output = []
-	for (const [stateKey, rowID] of states) {
-		const memberEvt = room.eventsByRowID.get(rowID)
-		if (!memberEvt) {
-			continue
-		}
-		const content = memberEvt.content as MemberEventContent
-		output.push({
-			userID: stateKey,
-			displayName: content.displayname ?? stateKey,
-			avatarURL: content.avatar_url,
-			searchString: toSearchableString(`${content.displayname ?? ""}${stateKey.slice(1)}`),
-		})
-	}
-	return output
-}
-
 interface filteredUserCache {
 	query: string
 	result: AutocompleteUser[]
 }
 
 export function useFilteredMembers(room: RoomStateStore, query: string): AutocompleteUser[] {
-	const allMembers = useMemo(() => getAutocompleteMemberList(room), [room])
+	const allMembers = useMemo(
+		() => room.getAutocompleteMembers(),
+		// fullMembersLoaded needs to be monitored for when the member list loads
+		// eslint-disable-next-line react-hooks/exhaustive-deps
+		[room, room.fullMembersLoaded],
+	)
 	const prev = useRef<filteredUserCache>({ query: "", result: allMembers })
 	if (!query) {
 		prev.current.query = ""
@@ -72,6 +55,9 @@ export function useFilteredMembers(room: RoomStateStore, query: string): Autocom
 			query.startsWith(prev.current.query) ? prev.current.result : allMembers,
 			query,
 		)
+		if (prev.current.result.length > 100) {
+			prev.current.result = prev.current.result.slice(0, 100)
+		}
 		prev.current.query = query
 	}
 	return prev.current.result

web/src/ui/rightpanel/MemberList.tsx

@@ -18,6 +18,7 @@ import { getAvatarURL } from "@/api/media.ts"
 import { useRoomMembers } from "@/api/statestore"
 import { MemDBEvent, MemberEventContent } from "@/api/types"
 import { getDisplayname } from "@/util/validation.ts"
+import ClientContext from "../ClientContext.ts"
 import MainScreenContext from "../MainScreenContext.ts"
 import { RoomContext } from "../roomview/roomcontext.ts"
 
@@ -44,6 +45,10 @@ const MemberList = () => {
 	const [limit, setLimit] = useState(50)
 	const increaseLimit = useCallback(() => setLimit(limit => limit + 50), [])
 	const roomCtx = use(RoomContext)
+	if (roomCtx?.store && !roomCtx?.store.membersRequested && !roomCtx?.store.fullMembersLoaded) {
+		roomCtx.store.membersRequested = true
+		use(ClientContext)?.loadRoomState(roomCtx.store.roomID, { omitMembers: false, refetch: false })
+	}
 	const memberEvents = useRoomMembers(roomCtx?.store)
 	if (!roomCtx) {
 		return null

web/src/ui/rightpanel/UserInfo.tsx

@@ -31,6 +31,9 @@ const UserInfo = ({ userID }: UserInfoProps) => {
 	const roomCtx = use(RoomContext)
 	const openLightbox = use(LightboxContext)!
 	const memberEvt = useRoomState(roomCtx?.store, "m.room.member", userID)
+	if (!memberEvt) {
+		use(ClientContext)?.requestMemberEvent(roomCtx?.store, userID)
+	}
 	const memberEvtContent = memberEvt?.content as MemberEventContent
 	if (!memberEvtContent) {
 		return <NonMemberInfo userID={userID}/>

web/src/ui/roomlist/RoomList.tsx

@@ -62,7 +62,7 @@ const RoomList = ({ activeRoomID }: RoomListProps) => {
 			evt.stopPropagation()
 			evt.preventDefault()
 		}
-	}, [clearQuery])
+	}, [mainScreen, client.store, clearQuery])
 
 	return <div className="room-list-wrapper">
 		<div className="room-search-wrapper">

web/src/ui/timeline/ReplyBody.tsx

@@ -70,6 +70,9 @@ const onClickReply = (evt: React.MouseEvent) => {
 
 export const ReplyBody = ({ room, event, onClose, isThread, isEditing }: ReplyBodyProps) => {
 	const memberEvt = useRoomState(room, "m.room.member", event.sender)
+	if (!memberEvt) {
+		use(ClientContext)?.requestMemberEvent(room, event.sender)
+	}
 	const memberEvtContent = memberEvt?.content as MemberEventContent | undefined
 	const BodyType = getBodyType(event, true)
 	const classNames = ["reply-body"]

web/src/ui/timeline/TimelineEvent.tsx

@@ -74,6 +74,9 @@ const TimelineEvent = ({ evt, prevEvt, disableMenu }: TimelineEventProps) => {
 	const mainScreen = use(MainScreenContext)
 	const [forceContextMenuOpen, setForceContextMenuOpen] = useState(false)
 	const memberEvt = useRoomState(roomCtx.store, "m.room.member", evt.sender)
+	if (!memberEvt) {
+		client.requestMemberEvent(roomCtx.store, evt.sender)
+	}
 	const memberEvtContent = memberEvt?.content as MemberEventContent | undefined
 	const BodyType = getBodyType(evt)
 	const eventTS = new Date(evt.timestamp)