web/emoji: implement MSC2545

cmd/gomuks/gomuks.go

@@ -145,7 +145,7 @@ func (gmx *Gomuks) SetupLog() {
 }
 
 func (gmx *Gomuks) StartClient() {
-	hicli.HTMLSanitizerImgSrcTemplate = "_gomuks/media/%s/%s"
+	hicli.HTMLSanitizerImgSrcTemplate = "_gomuks/media/%s/%s?encrypted=false"
 	rawDB, err := dbutil.NewFromConfig("gomuks", dbutil.Config{
 		PoolConfig: dbutil.PoolConfig{
 			Type:         "sqlite3-fk-wal",

go.mod

@@ -24,7 +24,7 @@ require (
 	golang.org/x/text v0.19.0
 	gopkg.in/yaml.v3 v3.0.1
 	maunium.net/go/mauflag v1.0.0
-	maunium.net/go/mautrix v0.21.2-0.20241020164413-e17cb8385518
+	maunium.net/go/mautrix v0.21.2-0.20241026115159-a59d4d78677f
 	mvdan.cc/xurls/v2 v2.5.0
 )
 

go.sum

@@ -89,7 +89,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
 maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
-maunium.net/go/mautrix v0.21.2-0.20241020164413-e17cb8385518 h1:ROFV2DnfWw6wzNGXJP+SVSsd+cHoyeqHvVvaW5PQyk0=
+maunium.net/go/mautrix v0.21.2-0.20241026115159-a59d4d78677f h1:fZL9ASp9m4KaC0QUEDkv5ptPwVvRjigy9uPI6NYZAD0=
-maunium.net/go/mautrix v0.21.2-0.20241020164413-e17cb8385518/go.mod h1:sjCZR1R/3NET/WjkcXPL6WpAHlWKku9HjRsdOkbM8Qw=
+maunium.net/go/mautrix v0.21.2-0.20241026115159-a59d4d78677f/go.mod h1:sjCZR1R/3NET/WjkcXPL6WpAHlWKku9HjRsdOkbM8Qw=
 mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8=
 mvdan.cc/xurls/v2 v2.5.0/go.mod h1:yQgaGQ1rFtJUzkmKiHYSSfuQxqfYmd//X6PxvholpeE=

pkg/hicli/send.go

@@ -52,7 +52,7 @@ func (h *HiClient) SendMessage(
 		text = strings.TrimPrefix(text, "/html ")
 		content = format.RenderMarkdown(text, false, true)
 	} else if text != "" {
-		content = format.RenderMarkdown(text, true, false)
+		content = format.RenderMarkdown(text, true, true)
 	}
 	if base != nil {
 		if text != "" {

web/eslint.config.js

@@ -81,7 +81,7 @@ export default tseslint.config(
 				"asyncArrow": "always",
 			}],
 			"func-style": ["warn", "declaration", {"allowArrowFunctions": true}],
-			"id-length": ["warn", {"max": 40, "exceptions": ["i", "j", "x", "y", "_"]}],
+			"id-length": ["warn", {"min": 1, "max": 40, "exceptions": ["i", "j", "x", "y", "_"]}],
 			"new-cap": ["warn", {
 				"newIsCap": true,
 				"capIsNew": true,

web/src/api/client.ts

@@ -24,6 +24,8 @@ export default class Client {
 
 	constructor(readonly rpc: RPCClient) {
 		this.rpc.event.listen(this.#handleEvent)
+		this.store.accountDataSubs.getSubscriber("im.ponies.emote_rooms")(() =>
+			queueMicrotask(() => this.#handleEmoteRoomsChange))
 	}
 
 	get userID(): UserID {
@@ -102,6 +104,9 @@ export default class Client {
 	}
 
 	async incrementFrequentlyUsedEmoji(targetEmoji: string) {
+		if (targetEmoji.startsWith("mxc://")) {
+			return
+		}
 		let recentEmoji = this.store.accountData.get("io.element.recent_emoji")?.recent_emoji as
 			[string, number][] | undefined
 		if (!Array.isArray(recentEmoji)) {
@@ -127,6 +132,14 @@ export default class Client {
 		await this.rpc.setAccountData("io.element.recent_emoji", newContent)
 	}
 
+	#handleEmoteRoomsChange() {
+		this.store.invalidateEmojiPackKeyCache()
+		this.loadSpecificRoomState(this.store.getEmojiPackKeys()).then(
+			() => this.store.emojiRoomsSub.notify(),
+			err => console.error("Failed to load emote rooms", err),
+		)
+	}
+
 	async loadSpecificRoomState(keys: RoomStateGUID[]): Promise<void> {
 		const missingKeys = keys.filter(key => {
 			const room = this.store.rooms.get(key.room_id)

web/src/api/statestore/hooks.ts

@@ -13,7 +13,8 @@
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import { useSyncExternalStore } from "react"
+import { useMemo, useSyncExternalStore } from "react"
+import { CustomEmojiPack } from "@/util/emoji"
 import type { EventID, EventType, MemDBEvent, UnknownEventContent } from "../types"
 import { StateStore } from "./main.ts"
 import { RoomStateStore } from "./room.ts"
@@ -50,3 +51,27 @@ export function useAccountData(ss: StateStore, type: EventType): UnknownEventCon
 		() => ss.accountData.get(type) ?? null,
 	)
 }
+
+export function useCustomEmojis(
+	ss: StateStore, room: RoomStateStore,
+): CustomEmojiPack[] {
+	const personalPack = useSyncExternalStore(
+		ss.accountDataSubs.getSubscriber("im.ponies.user_emotes"),
+		() => ss.getPersonalEmojiPack(),
+	)
+	const watchedRoomPacks = useSyncExternalStore(
+		ss.emojiRoomsSub.subscribe,
+		() => ss.getRoomEmojiPacks(),
+	)
+	const specialRoomPacks = useSyncExternalStore(
+		room.stateSubs.getSubscriber("im.ponies.room_emotes"),
+		() => room.getAllEmojiPacks(),
+	)
+	return useMemo(() => {
+		const allPacksObject = { ...watchedRoomPacks, ...specialRoomPacks }
+		if (personalPack) {
+			allPacksObject.personal = personalPack
+		}
+		return Object.values(allPacksObject)
+	}, [personalPack, watchedRoomPacks, specialRoomPacks])
+}

web/src/api/statestore/main.ts

@@ -14,21 +14,26 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import { getAvatarURL } from "@/api/media.ts"
+import { CustomEmojiPack, parseCustomEmojiPack } from "@/util/emoji"
 import { NonNullCachedEventDispatcher } from "@/util/eventdispatcher.ts"
 import { focused } from "@/util/focus.ts"
 import toSearchableString from "@/util/searchablestring.ts"
-import { MultiSubscribable } from "@/util/subscribable.ts"
+import Subscribable, { MultiSubscribable } from "@/util/subscribable.ts"
-import type {
+import {
 	ContentURI,
 	EventRowID,
 	EventsDecryptedData,
+	ImagePack,
+	ImagePackRooms,
 	MemDBEvent,
 	RoomID,
+	RoomStateGUID,
 	SendCompleteData,
 	SyncCompleteData,
 	SyncRoom,
 	UnknownEventContent,
 	UserID,
+	roomStateGUIDToString,
 } from "../types"
 import { RoomStateStore } from "./room.ts"
 
@@ -51,7 +56,11 @@ export class StateStore {
 	readonly roomList = new NonNullCachedEventDispatcher<RoomListEntry[]>([])
 	readonly accountData: Map<string, UnknownEventContent> = new Map()
 	readonly accountDataSubs = new MultiSubscribable()
+	readonly emojiRoomsSub = new Subscribable()
 	#frequentlyUsedEmoji: Map<string, number> | null = null
+	#emojiPackKeys: RoomStateGUID[] | null = null
+	#watchedRoomEmojiPacks: Record<string, CustomEmojiPack> | null = null
+	#personalEmojiPack: CustomEmojiPack | null = null
 	switchRoom?: (roomID: RoomID) => void
 	imageAuthToken?: string
 
@@ -116,7 +125,7 @@ export class StateStore {
 			let isNewRoom = false
 			let room = this.rooms.get(roomID)
 			if (!room) {
-				room = new RoomStateStore(data.meta)
+				room = new RoomStateStore(data.meta, this)
 				this.rooms.set(roomID, room)
 				isNewRoom = true
 			}
@@ -184,6 +193,66 @@ export class StateStore {
 		}
 	}
 
+	invalidateEmojiPackKeyCache() {
+		this.#emojiPackKeys = null
+	}
+
+	invalidateEmojiPacksCache() {
+		this.#watchedRoomEmojiPacks = null
+		this.emojiRoomsSub.notify()
+	}
+
+	getPersonalEmojiPack(): CustomEmojiPack | null {
+		if (this.#personalEmojiPack === null) {
+			const pack = this.accountData.get("im.ponies.user_emotes")
+			if (!pack) {
+				return null
+			}
+			this.#personalEmojiPack = parseCustomEmojiPack(pack as ImagePack, "personal", "Personal pack")
+		}
+		return this.#personalEmojiPack
+	}
+
+	getEmojiPackKeys(): RoomStateGUID[] {
+		if (this.#emojiPackKeys === null) {
+			const emoteRooms = this.accountData.get("im.ponies.emote_rooms") as ImagePackRooms | undefined
+			try {
+				const emojiPacks: RoomStateGUID[] = []
+				for (const [roomID, packs] of Object.entries(emoteRooms?.rooms ?? {})) {
+					for (const pack of Object.keys(packs)) {
+						emojiPacks.push({ room_id: roomID, type: "im.ponies.room_emotes", state_key: pack })
+					}
+				}
+				this.#emojiPackKeys = emojiPacks
+			} catch (err) {
+				console.warn("Failed to parse emote rooms data", err, emoteRooms)
+				this.#emojiPackKeys = []
+			}
+		}
+		return this.#emojiPackKeys
+	}
+
+	getRoomEmojiPacks() {
+		if (this.#watchedRoomEmojiPacks === null) {
+			this.#watchedRoomEmojiPacks = Object.fromEntries(
+				this.getEmojiPackKeys()
+					.map(key => {
+						const room = this.rooms.get(key.room_id)
+						if (!room) {
+							return null
+						}
+						const pack = room.getEmojiPack(key.state_key)
+						if (!pack) {
+							return null
+						}
+						return [roomStateGUIDToString(key), pack]
+					})
+					.filter(pack => !!pack),
+			)
+		}
+		return this.#watchedRoomEmojiPacks ?? {}
+	}
+
 	get frequentlyUsedEmoji(): Map<string, number> {
 		if (this.#frequentlyUsedEmoji === null) {
 			const emojiData = this.accountData.get("io.element.recent_emoji")

web/src/api/statestore/room.ts

@@ -13,22 +13,26 @@
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import { CustomEmojiPack, parseCustomEmojiPack } from "@/util/emoji"
 import { NonNullCachedEventDispatcher } from "@/util/eventdispatcher.ts"
 import Subscribable, { MultiSubscribable } from "@/util/subscribable.ts"
-import type {
+import {
 	DBRoom,
 	EncryptedEventContent,
 	EventID,
 	EventRowID,
 	EventType,
 	EventsDecryptedData,
+	ImagePack,
 	LazyLoadSummary,
 	MemDBEvent,
 	RawDBEvent,
 	RoomID,
 	SyncRoom,
 	TimelineRowTuple,
+	roomStateGUIDToString,
 } from "../types"
+import type { StateStore } from "./main.ts"
 
 function arraysAreEqual<T>(arr1?: T[], arr2?: T[]): boolean {
 	if (!arr1 || !arr2) {
@@ -75,12 +79,14 @@ export class RoomStateStore {
 	readonly eventSubs = new MultiSubscribable()
 	readonly requestedEvents: Set<EventID> = new Set()
 	readonly openNotifications: Map<EventRowID, Notification> = new Map()
+	readonly emojiPacks: Map<string, CustomEmojiPack | null> = new Map()
+	#allPacksCache: Record<string, CustomEmojiPack> | null = null
 	readonly pendingEvents: EventRowID[] = []
 	paginating = false
 	paginationRequestedForRow = -1
 	readUpToRow = -1
 
-	constructor(meta: DBRoom) {
+	constructor(meta: DBRoom, private parent: StateStore) {
 		this.roomID = meta.room_id
 		this.meta = new NonNullCachedEventDispatcher(meta)
 	}
@@ -111,6 +117,39 @@ export class RoomStateStore {
 		return this.eventsByRowID.get(rowID)
 	}
 
+	getEmojiPack(key: string): CustomEmojiPack | null {
+		if (!this.emojiPacks.has(key)) {
+			const pack = this.getStateEvent("im.ponies.room_emotes", key)?.content
+			if (!pack) {
+				this.emojiPacks.set(key, null)
+				return null
+			}
+			const fallbackName = key === ""
+				? this.meta.current.name : `${this.meta.current.name} - ${key}`
+			const packID = roomStateGUIDToString({
+				room_id: this.roomID,
+				type: "im.ponies.room_emotes",
+				state_key: key,
+			})
+			this.emojiPacks.set(key, parseCustomEmojiPack(pack as ImagePack, packID, fallbackName))
+		}
+		return this.emojiPacks.get(key) ?? null
+	}
+
+	getAllEmojiPacks(): Record<string, CustomEmojiPack> {
+		if (this.#allPacksCache === null) {
+			this.#allPacksCache = Object.fromEntries(
+				this.state.get("im.ponies.room_emotes")?.keys()
+					.map(stateKey => {
+						const pack = this.getEmojiPack(stateKey)
+						return pack ? [pack.id, pack] : null
+					})
+					.filter((res): res is [string, CustomEmojiPack] => !!res) ?? [],
+			)
+		}
+		return this.#allPacksCache
+	}
+
 	getPinnedEvents(): EventID[] {
 		const pinnedList = this.getStateEvent("m.room.pinned_events", "")?.content?.pinned
 		if (Array.isArray(pinnedList)) {
@@ -185,6 +224,15 @@ export class RoomStateStore {
 		this.notifyTimelineSubscribers()
 	}
 
+	invalidateStateCaches(evtType: string, key: string) {
+		if (evtType === "im.ponies.room_emotes") {
+			this.emojiPacks.delete(key)
+			this.#allPacksCache = null
+			this.parent.invalidateEmojiPacksCache()
+		}
+		this.stateSubs.notify(this.stateSubKey(evtType, key))
+	}
+
 	applySync(sync: SyncRoom) {
 		if (visibleMetaIsEqual(this.meta.current, sync.meta)) {
 			this.meta.current = sync.meta
@@ -202,8 +250,9 @@ export class RoomStateStore {
 			}
 			for (const [key, rowID] of Object.entries(changedEvts)) {
 				stateMap.set(key, rowID)
-				this.stateSubs.notify(this.stateSubKey(evtType, key))
+				this.invalidateStateCaches(evtType, key)
 			}
+			this.stateSubs.notify(evtType)
 		}
 		if (sync.reset) {
 			this.timeline = sync.timeline
@@ -231,7 +280,8 @@ export class RoomStateStore {
 			this.state.set(evt.type, stateMap)
 		}
 		stateMap.set(evt.state_key, evt.rowid)
-		this.stateSubs.notify(this.stateSubKey(evt.type, evt.state_key))
+		this.invalidateStateCaches(evt.type, evt.state_key)
+		this.stateSubs.notify(evt.type)
 	}
 
 	applyFullState(state: RawDBEvent[]) {
@@ -248,12 +298,15 @@ export class RoomStateStore {
 			}
 			stateMap.set(evt.state_key, evt.rowid)
 		}
+		this.emojiPacks.clear()
+		this.#allPacksCache = null
 		this.state = newStateMap
 		this.stateLoaded = true
 		for (const [evtType, stateMap] of newStateMap) {
 			for (const [key] of stateMap) {
 				this.stateSubs.notify(this.stateSubKey(evtType, key))
 			}
+			this.stateSubs.notify(evtType)
 		}
 	}
 

web/src/api/types/hitypes.ts

@@ -164,6 +164,10 @@ export interface ClientWellKnown {
 	}
 }
 
+export function roomStateGUIDToString(guid: RoomStateGUID): string {
+	return `${guid.room_id}/${guid.type}/${guid.state_key}`
+}
+
 export interface RoomStateGUID {
 	room_id: RoomID
 	type: EventType

web/src/api/types/mxtypes.ts

@@ -169,6 +169,7 @@ export type ImagePackUsage = "emoticon" | "sticker"
 
 export interface ImagePackEntry {
 	url: ContentURI
+	body?: string
 	info?: MediaInfo
 	usage?: ImagePackUsage[]
 }

web/src/icons/category.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 -960 960 960" width="24px" fill="#5f6368"><path d="m260-520 220-360 220 360H260ZM700-80q-75 0-127.5-52.5T520-260q0-75 52.5-127.5T700-440q75 0 127.5 52.5T880-260q0 75-52.5 127.5T700-80Zm-580-20v-320h320v320H120Zm580-60q42 0 71-29t29-71q0-42-29-71t-71-29q-42 0-71 29t-29 71q0 42 29 71t71 29Zm-500-20h160v-160H200v160Zm202-420h156l-78-126-78 126Zm78 0ZM360-340Zm340 80Z"/></svg>

web/src/ui/composer/Autocompleter.css

@@ -23,5 +23,10 @@ div.autocompletions {
 		&.selected, &:hover {
 			background-color: #f0f0f0;
 		}
+
+		> img {
+			width: 1.5rem;
+			height: 1.5rem;
+		}
 	}
 }

web/src/ui/composer/Autocompleter.tsx

@@ -14,9 +14,9 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import { JSX, use, useEffect } from "react"
-import { getAvatarURL } from "@/api/media.ts"
+import { getAvatarURL, getMediaURL } from "@/api/media.ts"
-import { RoomStateStore } from "@/api/statestore"
+import { RoomStateStore, useCustomEmojis } from "@/api/statestore"
-import { Emoji, useFilteredEmojis } from "@/util/emoji"
+import { Emoji, emojiToMarkdown, useSortedAndFilteredEmojis } from "@/util/emoji"
 import useEvent from "@/util/useEvent.ts"
 import { ClientContext } from "../ClientContext.ts"
 import type { ComposerState } from "./MessageComposer.tsx"
@@ -81,7 +81,7 @@ function useAutocompleter<T>({
 			onSelect(params.selected)
 		}
 	}, [onSelect, params.selected])
-	const selected = params.selected !== undefined	 ? positiveMod(params.selected, items.length) : -1
+	const selected = params.selected !== undefined ? positiveMod(params.selected, items.length) : -1
 	return <div className="autocompletions">
 		{items.map((item, i) => <div
 			onClick={onClick}
@@ -93,18 +93,22 @@ function useAutocompleter<T>({
 }
 
 const emojiFuncs = {
-	getText: (emoji: Emoji) => emoji.u,
+	getText: (emoji: Emoji) => emojiToMarkdown(emoji),
-	getKey: (emoji: Emoji) => emoji.u,
+	getKey: (emoji: Emoji) => `${emoji.c}-${emoji.u}`,
-	render: (emoji: Emoji) => <>{emoji.u} :{emoji.n}:</>,
+	render: (emoji: Emoji) => <>{emoji.u.startsWith("mxc://")
+		? <img loading="lazy" src={getMediaURL(emoji.u)} alt={`:${emoji.n}:`}/>
+		: emoji.u
+	} :{emoji.n}:</>,
 }
 
-export const EmojiAutocompleter = ({ params, ...rest }: AutocompleterProps) => {
+export const EmojiAutocompleter = ({ params, room, ...rest }: AutocompleterProps) => {
 	const client = use(ClientContext)!
-	const items = useFilteredEmojis((params.frozenQuery ?? params.query).slice(1), {
+	const customEmojiPacks = useCustomEmojis(client.store, room)
-		sorted: true,
+	const items = useSortedAndFilteredEmojis((params.frozenQuery ?? params.query).slice(1), {
 		frequentlyUsed: client.store.frequentlyUsedEmoji,
+		customEmojiPacks,
 	})
-	return useAutocompleter({ params, ...rest, items, ...emojiFuncs })
+	return useAutocompleter({ params, room, ...rest, items, ...emojiFuncs })
 }
 
 const escapeDisplayname = (input: string) => input

web/src/ui/composer/MessageComposer.tsx

@@ -25,7 +25,7 @@ import type {
 	RelatesTo,
 	RoomID,
 } from "@/api/types"
-import { emojiToMarkdown } from "@/util/emoji"
+import { PartialEmoji, emojiToMarkdown } from "@/util/emoji"
 import useEvent from "@/util/useEvent.ts"
 import { ClientContext } from "../ClientContext.ts"
 import EmojiPicker from "../emojipicker/EmojiPicker.tsx"
@@ -320,17 +320,19 @@ const MessageComposer = () => {
 		evt.stopPropagation()
 		roomCtx.setEditing(null)
 	}, [roomCtx])
+	const onSelectEmoji = useEvent((emoji: PartialEmoji) => {
+		setState({
+			text: state.text.slice(0, textInput.current?.selectionStart ?? 0)
+				+ emojiToMarkdown(emoji)
+				+ state.text.slice(textInput.current?.selectionEnd ?? 0),
+		})
+	})
 	const openEmojiPicker = useEvent(() => {
 		openModal({
 			content: <EmojiPicker
 				style={{ bottom: (composerRef.current?.clientHeight ?? 32) + 2, right: "1rem" }}
-				onSelect={emoji => {
+				room={roomCtx.store}
-					setState({
+				onSelect={onSelectEmoji}
-						text: state.text.slice(0, textInput.current?.selectionStart ?? 0)
-							+ emojiToMarkdown(emoji)
-							+ state.text.slice(textInput.current?.selectionEnd ?? 0),
-					})
-				}}
 			/>,
 			onClose: () => textInput.current?.focus(),
 		})

web/src/ui/emojipicker/EmojiPicker.css

@@ -2,14 +2,14 @@ div.emoji-picker {
 	position: fixed;
 	background-color: white;
 	width: 22rem;
-	height: 30rem;
+	height: 34rem;
 	border-radius: 1rem;
 	border: 1px solid #ccc;
 	display: flex;
 	flex-direction: column;
 
 	div.emoji-category-bar {
-		height: 2.5rem;
+		/*height: 2.5rem;*/
 		display: flex;
 		justify-content: center;
 		flex-wrap: wrap;
@@ -80,6 +80,11 @@ div.emoji-picker {
 			display: flex;
 			justify-content: center;
 			align-items: center;
+
+			> img {
+				width: 3rem;
+				height: 3rem;
+			}
 		}
 
 		> div.emoji-name {
@@ -97,6 +102,7 @@ div.emoji-picker {
 
 	div.emoji-category {
 		width: 100%;
+		content-visibility: auto;
 	}
 
 	div.emoji-category-list {
@@ -109,6 +115,11 @@ div.emoji-picker {
 		margin: 0;
 	}
 
+	button.emoji-category-icon > img, button.emoji > img {
+		width: 1.5rem;
+		height: 1.5rem;
+	}
+
 	button.emoji {
 		font-size: 1.25rem;
 		padding: 0;

web/src/ui/emojipicker/EmojiPicker.tsx

@@ -15,9 +15,12 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import { CSSProperties, JSX, use, useCallback, useState } from "react"
 import { getMediaURL } from "@/api/media.ts"
+import { RoomStateStore, useCustomEmojis } from "@/api/statestore"
 import { CATEGORY_FREQUENTLY_USED, Emoji, PartialEmoji, categories, useFilteredEmojis } from "@/util/emoji"
+import useEvent from "@/util/useEvent.ts"
 import { ClientContext } from "../ClientContext.ts"
 import { ModalCloseContext } from "../modal/Modal.tsx"
+import FallbackPackIcon from "@/icons/category.svg?react"
 import CloseIcon from "@/icons/close.svg?react"
 import ActivitiesIcon from "@/icons/emoji-categories/activities.svg?react"
 import AnimalsNatureIcon from "@/icons/emoji-categories/animals-nature.svg?react"
@@ -52,7 +55,7 @@ const sortedEmojiCategories: EmojiCategory[] = [
 
 function renderEmoji(emoji: Emoji): JSX.Element | string {
 	if (emoji.u.startsWith("mxc://")) {
-		return <img src={getMediaURL(emoji.u)} alt={`:${emoji.n}:`}/>
+		return <img loading="lazy" src={getMediaURL(emoji.u)} alt={`:${emoji.n}:`}/>
 	}
 	return emoji.u
 }
@@ -60,25 +63,27 @@ function renderEmoji(emoji: Emoji): JSX.Element | string {
 interface EmojiPickerProps {
 	style: CSSProperties
 	onSelect: (emoji: PartialEmoji, isSelected?: boolean) => void
+	room: RoomStateStore
 	allowFreeform?: boolean
 	closeOnSelect?: boolean
 	selected?: string[]
 }
 
-export const EmojiPicker = ({ style, selected, onSelect, allowFreeform, closeOnSelect }: EmojiPickerProps) => {
+export const EmojiPicker = ({ style, selected, onSelect, room, allowFreeform, closeOnSelect }: EmojiPickerProps) => {
 	const client = use(ClientContext)!
 	const [query, setQuery] = useState("")
+	const customEmojiPacks = useCustomEmojis(client.store, room)
 	const emojis = useFilteredEmojis(query, {
 		frequentlyUsed: client.store.frequentlyUsedEmoji,
-		frequentlyUsedAsCategory: true,
+		customEmojiPacks,
 	})
 	const [previewEmoji, setPreviewEmoji] = useState<Emoji>()
 	const clearQuery = useCallback(() => setQuery(""), [])
-	const cats: JSX.Element[] = []
-	let currentCat: JSX.Element[] = []
-	let currentCatNum: number | string = -1
 	const close = use(ModalCloseContext)
-	const onSelectWrapped = (emoji: PartialEmoji) => {
+	const onSelectWrapped = (emoji?: PartialEmoji) => {
+		if (!emoji) {
+			return
+		}
 		onSelect(emoji, selected?.includes(emoji.u))
 		if (emoji.c) {
 			client.incrementFrequentlyUsedEmoji(emoji.u)
@@ -88,39 +93,72 @@ export const EmojiPicker = ({ style, selected, onSelect, allowFreeform, closeOnS
 			close()
 		}
 	}
-	for (const emoji of emojis) {
+	const getEmojiFromAttrs = (elem: HTMLButtonElement) => {
-		if (emoji.c === 2) {
+		const groupIdx = elem.getAttribute("data-emoji-group-index")
-			continue
+		if (!groupIdx) {
+			return
 		}
-		if (emoji.c !== currentCatNum) {
+		const idx = elem.getAttribute("data-emoji-index")
-			if (currentCat.length) {
+		if (!idx) {
-				const categoryName = typeof currentCatNum === "number" ? categories[currentCatNum] : currentCatNum
+			return
-				cats.push(<div className="emoji-category" key={currentCatNum} id={`emoji-category-${categoryName}`}>
-					<h4 className="emoji-category-name">{categoryName}</h4>
-					<div className="emoji-category-list">
-						{currentCat}
-					</div>
-				</div>)
-			}
-			currentCatNum = emoji.c
-			currentCat = []
 		}
-		currentCat.push(<button
+		const emoji = emojis[+groupIdx]?.[+idx]
-			key={emoji.c === CATEGORY_FREQUENTLY_USED ? `freq-${emoji.u}` : emoji.u}
+		if (!emoji) {
-			className={`emoji ${selected?.includes(emoji.u) ? "selected" : ""}`}
+			return
-			onMouseOver={() => setPreviewEmoji(emoji)}
+		}
-			onMouseOut={() => setPreviewEmoji(undefined)}
+		return emoji
-			onClick={() => onSelectWrapped(emoji)}
-		>{renderEmoji(emoji)}</button>)
 	}
-	if (currentCat.length) {
+	const onClickEmoji = useEvent((evt: React.MouseEvent<HTMLButtonElement>) =>
+		onSelectWrapped(getEmojiFromAttrs(evt.currentTarget)))
+	const onMouseOverEmoji = useEvent((evt: React.MouseEvent<HTMLButtonElement>) =>
+		setPreviewEmoji(getEmojiFromAttrs(evt.currentTarget)))
+	const onMouseOutEmoji = useCallback(() => setPreviewEmoji(undefined), [])
+	const onClickFreeformReact = useEvent(() => onSelectWrapped({ u: query }))
+
+	const renderedCats: JSX.Element[] = []
+	let currentCatRender: JSX.Element[] = []
+	let currentCatNum: number | string = -1
+	const renderCurrentCategory = () => {
+		if (!currentCatRender.length) {
+			return
+		}
 		const categoryName = typeof currentCatNum === "number" ? categories[currentCatNum] : currentCatNum
-		cats.push(<div className="emoji-category" key={currentCatNum} id={`emoji-category-${categoryName}`}>
+		renderedCats.push(<div
+			className="emoji-category"
+			key={currentCatNum}
+			id={`emoji-category-${categoryName}`}
+			style={{ containIntrinsicHeight: `${1.5 + Math.ceil(currentCatRender.length / 8) * 2.5}rem` }}
+		>
 			<h4 className="emoji-category-name">{categoryName}</h4>
 			<div className="emoji-category-list">
-				{currentCat}
+				{currentCatRender}
 			</div>
 		</div>)
+		currentCatRender = []
+		currentCatNum = -1
+	}
+	for (let catIdx = 0; catIdx < emojis.length; catIdx++) {
+		const cat = emojis[catIdx]
+		for (let emojiIdx = 0; emojiIdx < cat.length; emojiIdx++) {
+			const emoji = cat[emojiIdx]
+			if (emoji.c === 2) {
+				continue
+			}
+			if (emoji.c !== currentCatNum) {
+				renderCurrentCategory()
+				currentCatNum = emoji.c
+			}
+			currentCatRender.push(<button
+				key={`${emoji.c}-${emoji.u}`}
+				className={`emoji ${selected?.includes(emoji.u) ? "selected" : ""}`}
+				data-emoji-group-index={catIdx}
+				data-emoji-index={emojiIdx}
+				onMouseOver={onMouseOverEmoji}
+				onMouseOut={onMouseOutEmoji}
+				onClick={onClickEmoji}
+			>{renderEmoji(emoji)}</button>)
+		}
+		renderCurrentCategory()
 	}
 	const onChangeQuery = useCallback((evt: React.ChangeEvent<HTMLInputElement>) => setQuery(evt.target.value), [])
 	const onClickCategoryButton = useCallback((evt: React.MouseEvent) => {
@@ -131,6 +169,7 @@ export const EmojiPicker = ({ style, selected, onSelect, allowFreeform, closeOnS
 		<div className="emoji-category-bar">
 			<button
 				className="emoji-category-icon"
+				data-category-id={CATEGORY_FREQUENTLY_USED}
 				title={CATEGORY_FREQUENTLY_USED}
 				onClick={onClickCategoryButton}
 			>{<RecentIcon/>}</button>
@@ -138,10 +177,22 @@ export const EmojiPicker = ({ style, selected, onSelect, allowFreeform, closeOnS
 				<button
 					key={cat.index}
 					className="emoji-category-icon"
+					data-category-id={cat.index}
 					title={cat.name ?? categories[cat.index]}
 					onClick={onClickCategoryButton}
 				>{cat.icon}</button>,
 			)}
+			{customEmojiPacks.map(customPack =>
+				<button
+					key={customPack.id}
+					className="emoji-category-icon custom-emoji"
+					data-category-id={customPack.id}
+					title={customPack.name}
+					onClick={onClickCategoryButton}
+				>
+					{customPack.icon ? <img src={getMediaURL(customPack.icon)} alt="" /> : <FallbackPackIcon/>}
+				</button>,
+			)}
 		</div>
 		<div className="emoji-search">
 			<input autoFocus onChange={onChangeQuery} value={query} type="search" placeholder="Search emojis"/>
@@ -150,10 +201,10 @@ export const EmojiPicker = ({ style, selected, onSelect, allowFreeform, closeOnS
 			</button>
 		</div>
 		<div className="emoji-list">
-			{cats}
+			{renderedCats}
 			{allowFreeform && query && <button
 				className="freeform-react"
-				onClick={() => onSelectWrapped({ u: query })}
+				onClick={onClickFreeformReact}
 			>React with "{query}"</button>}
 		</div>
 		{previewEmoji ? <div className="emoji-preview">

web/src/ui/timeline/EventMenu.tsx

@@ -71,12 +71,13 @@ const EventMenu = ({ evt, setForceOpen }: EventHoverMenuProps) => {
 					client.sendEvent(evt.room_id, "m.reaction", emojiToReactionContent(emoji, evt.event_id))
 						.catch(err => window.alert(`Failed to send reaction: ${err}`))
 				}}
+				room={roomCtx.store}
 				closeOnSelect={true}
 				allowFreeform={true}
 			/>,
 			onClose: () => setForceOpen(false),
 		})
-	}, [client, evt, setForceOpen, openModal])
+	}, [client, roomCtx, evt, setForceOpen, openModal])
 	const onClickEdit = useCallback(() => {
 		roomCtx.setEditing(evt)
 	}, [roomCtx, evt])

web/src/util/emoji/index.ts

@@ -14,7 +14,7 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import { useMemo, useRef } from "react"
-import { EventID, ReactionEventContent } from "@/api/types"
+import { ContentURI, EventID, ImagePack, ImagePackUsage, ReactionEventContent } from "@/api/types"
 import data from "./data.json"
 
 export interface EmojiMetadata {
@@ -45,8 +45,13 @@ function filter(emojis: Emoji[], query: string): Emoji[] {
 	return emojis.filter(emoji => emoji.s.some(shortcode => shortcode.includes(query)))
 }
 
-function filterAndSort(emojis: Emoji[], query: string, frequentlyUsed?: Map<string, number>): Emoji[] {
+function filterAndSort(
-	return emojis
+	emojis: Emoji[],
+	query: string,
+	frequentlyUsed?: Map<string, number>,
+	customEmojis?: CustomEmojiPack[],
+): Emoji[] {
+	const filteredStandardEmojis = emojis
 		.map(emoji => {
 			const matchIndex = emoji.s.reduce((minIndex, shortcode) => {
 				const index = shortcode.indexOf(query)
@@ -55,6 +60,20 @@ function filterAndSort(emojis: Emoji[], query: string, frequentlyUsed?: Map<stri
 			return { emoji, matchIndex }
 		})
 		.filter(({ matchIndex }) => matchIndex !== -1)
+	const filteredCustomEmojis = customEmojis
+		?.flatMap(pack => pack.emojis
+			.map(emoji => {
+				const matchIndex = emoji.s.reduce((minIndex, shortcode) => {
+					const index = shortcode.indexOf(query)
+					return index !== -1 && (minIndex === -1 || index < minIndex) ? index : minIndex
+				}, -1)
+				return { emoji, matchIndex }
+			})
+			.filter(({ matchIndex }) => matchIndex !== -1)) ?? []
+	const allEmojis = filteredCustomEmojis.length
+		? filteredStandardEmojis.concat(filteredCustomEmojis)
+		: filteredStandardEmojis
+	return allEmojis
 		.sort((e1, e2) =>
 			e1.matchIndex === e2.matchIndex
 				? (frequentlyUsed?.get(e2.emoji.u) ?? 0) - (frequentlyUsed?.get(e1.emoji.u) ?? 0)
@@ -62,15 +81,9 @@ function filterAndSort(emojis: Emoji[], query: string, frequentlyUsed?: Map<stri
 		.map(({ emoji }) => emoji)
 }
 
-export function search(query: string, sorted = false, prev?: Emoji[]): Emoji[] {
-	query = query.toLowerCase().replaceAll("_", "")
-	if (!query) return emojis
-	return (sorted ? filterAndSort : filter)(prev ?? emojis, query)
-}
-
 export function emojiToMarkdown(emoji: PartialEmoji): string {
 	if (emoji.u.startsWith("mxc://")) {
-		return `<img data-mx-emoticon src="${emoji.u}" alt=":${emoji.n}:" title=":${emoji.n}:"/>`
+		return `<img data-mx-emoticon height="32" src="${emoji.u}" alt=":${emoji.n}:" title=":${emoji.n}:"/>`
 	}
 	return emoji.u
 }
@@ -89,48 +102,121 @@ export function emojiToReactionContent(emoji: PartialEmoji, evtID: EventID): Rea
 	return content
 }
 
+export interface CustomEmojiPack {
+	id: string
+	name: string
+	icon?: ContentURI
+	emojis: Emoji[]
+	emojiMap: Map<string, Emoji>
+}
+
+export function parseCustomEmojiPack(
+	pack: ImagePack,
+	id: string,
+	fallbackName?: string,
+	usage: ImagePackUsage = "emoticon",
+): CustomEmojiPack | null {
+	try {
+		if (pack.pack.usage && !pack.pack.usage.includes(usage)) {
+			return null
+		}
+		const name = pack.pack.display_name || fallbackName || "Unnamed pack"
+		const emojiMap = new Map<string, Emoji>()
+		for (const [shortcode, image] of Object.entries(pack.images)) {
+			if (!image.url || (image.usage && !image.usage.includes(usage))) {
+				continue
+			}
+			let converted = emojiMap.get(image.url)
+			if (converted) {
+				converted.s.push(shortcode.toLowerCase().replaceAll("_", ""))
+			} else {
+				converted = {
+					c: name,
+					u: image.url,
+					n: shortcode,
+					s: [shortcode.toLowerCase().replaceAll("_", "")],
+					t: image.body || shortcode,
+				}
+				emojiMap.set(image.url, converted)
+			}
+		}
+		const emojis = Array.from(emojiMap.values())
+		const icon = pack.pack.avatar_url || emojis[0]?.u
+		return {
+			id,
+			name,
+			icon,
+			emojis,
+			emojiMap,
+		}
+	} catch (err) {
+		console.warn("Failed to parse custom emoji pack", pack, err)
+		return null
+	}
+}
+
 interface filteredEmojiCache {
 	query: string
-	result: Emoji[]
+	result: Emoji[][]
 }
 
-interface useFilteredEmojisParams {
+interface filteredAndSortedEmojiCache {
-	sorted?: boolean
+	query: string
+	result: Emoji[] | null
+}
+
+interface useEmojisParams {
 	frequentlyUsed?: Map<string, number>
-	frequentlyUsedAsCategory?: boolean
+	customEmojiPacks?: CustomEmojiPack[]
 }
 
-export function useFilteredEmojis(query: string, params: useFilteredEmojisParams = {}): Emoji[] {
+export function useFilteredEmojis(query: string, params: useEmojisParams = {}): Emoji[][] {
 	query = query.toLowerCase().replaceAll("_", "")
-	const allEmojis: Emoji[] = useMemo(() => {
+	const frequentlyUsedCategory: Emoji[] = useMemo(() => {
-		let output: Emoji[] = []
+		if (!params.frequentlyUsed?.size) {
-		if (params.frequentlyUsedAsCategory && params.frequentlyUsed) {
+			return []
-			output = Array.from(params.frequentlyUsed.keys()
-				.map(key => {
-					const emoji = emojiMap.get(key)
-					if (!emoji) {
-						return undefined
-					}
-					return { ...emoji, c: CATEGORY_FREQUENTLY_USED } as Emoji
-				})
-				.filter((emoji, index): emoji is Emoji => emoji !== undefined && index < 24))
 		}
-		if (output.length === 0) {
+		return Array.from(params.frequentlyUsed.keys()
-			return emojis
+			.map(key => {
-		}
+				const emoji = emojiMap.get(key)
-		return output.concat(emojis)
+				if (!emoji) {
-	}, [params.frequentlyUsed, params.frequentlyUsedAsCategory])
+					return undefined
-	const prev = useRef<filteredEmojiCache>({ query: "", result: allEmojis })
+				}
+				return { ...emoji, c: CATEGORY_FREQUENTLY_USED } as Emoji
+			})
+			.filter(emoji => emoji !== undefined))
+			.filter((_emoji, index) => index < 24)
+	}, [params.frequentlyUsed])
+	const allPacks = [frequentlyUsedCategory, emojis, ...(params.customEmojiPacks?.map(pack => pack.emojis) ?? [])]
+	const prev = useRef<filteredEmojiCache>({ query: "", result: allPacks })
 	if (!query) {
 		prev.current.query = ""
-		prev.current.result = allEmojis
+		prev.current.result = allPacks
 	} else if (prev.current.query !== query) {
-		prev.current.result = (params.sorted ? filterAndSort : filter)(
+		if (query.startsWith(prev.current.query) && allPacks.length === prev.current.result.length) {
-			query.startsWith(prev.current.query) ? prev.current.result : allEmojis,
+			prev.current.result = prev.current.result.map(pack => filter(pack, query))
-			query,
+		} else {
-			params.frequentlyUsed,
+			prev.current.result = allPacks.map(pack => filter(pack, query))
-		)
+		}
 		prev.current.query = query
 	}
 	return prev.current.result
 }
+
+export function useSortedAndFilteredEmojis(query: string, params: useEmojisParams = {}): Emoji[] {
+	if (!query) {
+		throw new Error("useSortedAndFilteredEmojis requires a query")
+	}
+	query = query.toLowerCase().replaceAll("_", "")
+
+	const prev = useRef<filteredAndSortedEmojiCache>({ query: "", result: null })
+	if (prev.current.query !== query) {
+		if (prev.current.result != null && query.startsWith(prev.current.query)) {
+			prev.current.result = filterAndSort(prev.current.result, query, params.frequentlyUsed)
+		} else {
+			prev.current.result = filterAndSort(emojis, query, params.frequentlyUsed, params.customEmojiPacks)
+		}
+		prev.current.query = query
+	}
+	return prev.current.result ?? []
+}