server,web/composer: add support for sending media

cmd/gomuks/media.go

@@ -22,17 +22,31 @@ import (
 	"encoding/hex"
 	"errors"
 	"fmt"
+	"image"
+	_ "image/gif"
+	_ "image/jpeg"
+	_ "image/png"
 	"io"
 	"mime"
 	"net/http"
 	"os"
 	"path/filepath"
 	"strconv"
+	"strings"
 
+	"github.com/gabriel-vasile/mimetype"
 	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/hlog"
+	_ "golang.org/x/image/webp"
+
+	"go.mau.fi/util/exhttp"
+	"go.mau.fi/util/ffmpeg"
 	"go.mau.fi/util/jsontime"
 	"go.mau.fi/util/ptr"
+	"go.mau.fi/util/random"
 	"maunium.net/go/mautrix"
+	"maunium.net/go/mautrix/crypto/attachment"
+	"maunium.net/go/mautrix/event"
 	"maunium.net/go/mautrix/id"
 
 	"go.mau.fi/gomuks/pkg/hicli/database"
@@ -57,7 +71,7 @@ func (gmx *Gomuks) downloadMediaFromCache(ctx context.Context, w http.ResponseWr
 		return true
 	}
 	log := zerolog.Ctx(ctx)
-	cacheFile, err := os.Open(gmx.cacheEntryToPath(entry))
+	cacheFile, err := os.Open(gmx.cacheEntryToPath(entry.Hash[:]))
 	if err != nil {
 		if errors.Is(err, os.ErrNotExist) && !force {
 			return false
@@ -78,8 +92,8 @@ func (gmx *Gomuks) downloadMediaFromCache(ctx context.Context, w http.ResponseWr
 	return true
 }
 
-func (gmx *Gomuks) cacheEntryToPath(entry *database.CachedMedia) string {
+func (gmx *Gomuks) cacheEntryToPath(hash []byte) string {
-	hashPath := hex.EncodeToString(entry.Hash[:])
+	hashPath := hex.EncodeToString(hash[:])
 	return filepath.Join(gmx.CacheDir, "media", hashPath[0:2], hashPath[2:4], hashPath[4:])
 }
 
@@ -223,7 +237,7 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) {
 		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to save cache entry: %v", err)).Write(w)
 		return
 	}
-	cachePath := gmx.cacheEntryToPath(cacheEntry)
+	cachePath := gmx.cacheEntryToPath(cacheEntry.Hash[:])
 	err = os.MkdirAll(filepath.Dir(cachePath), 0700)
 	if err != nil {
 		log.Err(err).Msg("Failed to create cache directory")
@@ -238,3 +252,269 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) {
 	}
 	gmx.downloadMediaFromCache(ctx, w, cacheEntry, true)
 }
+
+func (gmx *Gomuks) UploadMedia(w http.ResponseWriter, r *http.Request) {
+	log := hlog.FromRequest(r)
+	tempFile, err := os.CreateTemp(gmx.TempDir, "upload-*")
+	if err != nil {
+		log.Err(err).Msg("Failed to create temporary file")
+		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to create temp file: %v", err)).Write(w)
+		return
+	}
+	defer func() {
+		_ = tempFile.Close()
+		_ = os.Remove(tempFile.Name())
+	}()
+	hasher := sha256.New()
+	_, err = io.Copy(tempFile, io.TeeReader(r.Body, hasher))
+	if err != nil {
+		log.Err(err).Msg("Failed to copy upload media to temporary file")
+		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to copy media to temp file: %v", err)).Write(w)
+		return
+	}
+	_ = tempFile.Close()
+
+	checksum := hasher.Sum(nil)
+	cachePath := gmx.cacheEntryToPath(checksum)
+	if _, err = os.Stat(cachePath); err == nil {
+		log.Debug().Str("path", cachePath).Msg("Media already exists in cache, removing temp file")
+	} else {
+		err = os.MkdirAll(filepath.Dir(cachePath), 0700)
+		if err != nil {
+			log.Err(err).Msg("Failed to create cache directory")
+			mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to create cache directory: %v", err)).Write(w)
+			return
+		}
+		err = os.Rename(tempFile.Name(), cachePath)
+		if err != nil {
+			log.Err(err).Msg("Failed to rename temporary file")
+			mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to rename temp file: %v", err)).Write(w)
+			return
+		}
+	}
+
+	cacheFile, err := os.Open(cachePath)
+	if err != nil {
+		log.Err(err).Msg("Failed to open cache file")
+		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to open cache file: %v", err)).Write(w)
+		return
+	}
+
+	msgType, info, defaultFileName, err := gmx.generateFileInfo(r.Context(), cacheFile)
+	if err != nil {
+		log.Err(err).Msg("Failed to generate file info")
+		mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to generate file info: %v", err)).Write(w)
+		return
+	}
+	encrypt, _ := strconv.ParseBool(r.URL.Query().Get("encrypt"))
+	if msgType == event.MsgVideo {
+		err = gmx.generateVideoThumbnail(r.Context(), cacheFile.Name(), encrypt, info)
+		if err != nil {
+			log.Warn().Err(err).Msg("Failed to generate video thumbnail")
+		}
+	}
+	fileName := r.URL.Query().Get("filename")
+	if fileName == "" {
+		fileName = defaultFileName
+	}
+	content := &event.MessageEventContent{
+		MsgType:  msgType,
+		Body:     fileName,
+		Info:     info,
+		FileName: fileName,
+	}
+	content.File, content.URL, err = gmx.uploadFile(r.Context(), checksum, cacheFile, encrypt, int64(info.Size), info.MimeType, fileName)
+	if err != nil {
+		log.Err(err).Msg("Failed to upload media")
+		writeMaybeRespError(err, w)
+		return
+	}
+	exhttp.WriteJSONResponse(w, http.StatusOK, content)
+}
+
+func (gmx *Gomuks) uploadFile(ctx context.Context, checksum []byte, cacheFile *os.File, encrypt bool, fileSize int64, mimeType, fileName string) (*event.EncryptedFileInfo, id.ContentURIString, error) {
+	cm := &database.CachedMedia{
+		FileName: fileName,
+		MimeType: mimeType,
+		Size:     fileSize,
+		Hash:     (*[32]byte)(checksum),
+	}
+	var cacheReader io.ReadSeekCloser = cacheFile
+	if encrypt {
+		cm.EncFile = attachment.NewEncryptedFile()
+		cacheReader = cm.EncFile.EncryptStream(cacheReader)
+		mimeType = "application/octet-stream"
+		fileName = ""
+	}
+	resp, err := gmx.Client.Client.UploadMedia(ctx, mautrix.ReqUploadMedia{
+		Content:       cacheReader,
+		ContentLength: fileSize,
+		ContentType:   mimeType,
+		FileName:      fileName,
+	})
+	err2 := cacheReader.Close()
+	if err != nil {
+		return nil, "", err
+	} else if err2 != nil {
+		return nil, "", fmt.Errorf("failed to close cache reader: %w", err)
+	}
+	cm.MXC = resp.ContentURI
+	err = gmx.Client.DB.CachedMedia.Put(ctx, cm)
+	if err != nil {
+		zerolog.Ctx(ctx).Err(err).
+			Stringer("mxc", cm.MXC).
+			Hex("checksum", checksum).
+			Msg("Failed to save cache entry")
+	}
+	if cm.EncFile != nil {
+		return &event.EncryptedFileInfo{
+			EncryptedFile: *cm.EncFile,
+			URL:           resp.ContentURI.CUString(),
+		}, "", nil
+	} else {
+		return nil, resp.ContentURI.CUString(), nil
+	}
+}
+
+func (gmx *Gomuks) generateFileInfo(ctx context.Context, file *os.File) (event.MessageType, *event.FileInfo, string, error) {
+	fileInfo, err := file.Stat()
+	if err != nil {
+		return "", nil, "", fmt.Errorf("failed to stat cache file: %w", err)
+	}
+	mimeType, err := mimetype.DetectReader(file)
+	if err != nil {
+		return "", nil, "", fmt.Errorf("failed to detect mime type: %w", err)
+	}
+	_, err = file.Seek(0, io.SeekStart)
+	if err != nil {
+		return "", nil, "", fmt.Errorf("failed to seek to start of file: %w", err)
+	}
+	info := &event.FileInfo{
+		MimeType: mimeType.String(),
+		Size:     int(fileInfo.Size()),
+	}
+	var msgType event.MessageType
+	var defaultFileName string
+	switch strings.Split(mimeType.String(), "/")[0] {
+	case "image":
+		msgType = event.MsgImage
+		defaultFileName = "image" + mimeType.Extension()
+		cfg, _, err := image.DecodeConfig(file)
+		if err != nil {
+			zerolog.Ctx(ctx).Warn().Err(err).Msg("Failed to decode image config")
+		}
+		info.Width = cfg.Width
+		info.Height = cfg.Height
+	case "video":
+		msgType = event.MsgVideo
+		defaultFileName = "video" + mimeType.Extension()
+	case "audio":
+		msgType = event.MsgAudio
+		defaultFileName = "audio" + mimeType.Extension()
+	default:
+		msgType = event.MsgFile
+		defaultFileName = "file" + mimeType.Extension()
+	}
+	if msgType == event.MsgVideo || msgType == event.MsgAudio {
+		probe, err := ffmpeg.Probe(ctx, file.Name())
+		if err != nil {
+			zerolog.Ctx(ctx).Warn().Err(err).Msg("Failed to probe video")
+		} else if probe != nil && probe.Format != nil {
+			info.Duration = int(probe.Format.Duration * 1000)
+			for _, stream := range probe.Streams {
+				if stream.Width != 0 {
+					info.Width = stream.Width
+					info.Height = stream.Height
+					break
+				}
+			}
+		}
+	}
+	_, err = file.Seek(0, io.SeekStart)
+	if err != nil {
+		return "", nil, "", fmt.Errorf("failed to seek to start of file: %w", err)
+	}
+	return msgType, info, defaultFileName, nil
+}
+
+func (gmx *Gomuks) generateVideoThumbnail(ctx context.Context, filePath string, encrypt bool, saveInto *event.FileInfo) error {
+	tempPath := filepath.Join(gmx.TempDir, "thumbnail-"+random.String(12)+".jpeg")
+	defer os.Remove(tempPath)
+	err := ffmpeg.ConvertPathWithDestination(
+		ctx, filePath, tempPath, nil,
+		[]string{"-frames:v", "1", "-update", "1", "-f", "image2"},
+		false,
+	)
+	if err != nil {
+		return err
+	}
+	tempFile, err := os.Open(tempPath)
+	if err != nil {
+		return fmt.Errorf("failed to open file: %w", err)
+	}
+	defer tempFile.Close()
+	fileInfo, err := tempFile.Stat()
+	if err != nil {
+		return fmt.Errorf("failed to stat file: %w", err)
+	}
+	hasher := sha256.New()
+	_, err = io.Copy(hasher, tempFile)
+	if err != nil {
+		return fmt.Errorf("failed to hash file: %w", err)
+	}
+	thumbnailInfo := &event.FileInfo{
+		MimeType: "image/jpeg",
+		Size:     int(fileInfo.Size()),
+	}
+	_, err = tempFile.Seek(0, io.SeekStart)
+	if err != nil {
+		return fmt.Errorf("failed to seek to start of file: %w", err)
+	}
+	cfg, _, err := image.DecodeConfig(tempFile)
+	if err != nil {
+		zerolog.Ctx(ctx).Warn().Err(err).Msg("Failed to decode thumbnail image config")
+	} else {
+		thumbnailInfo.Width = cfg.Width
+		thumbnailInfo.Height = cfg.Height
+	}
+	_ = tempFile.Close()
+	checksum := hasher.Sum(nil)
+	cachePath := gmx.cacheEntryToPath(checksum)
+	if _, err = os.Stat(cachePath); err == nil {
+		zerolog.Ctx(ctx).Debug().Str("path", cachePath).Msg("Media already exists in cache, removing temp file")
+	} else {
+		err = os.MkdirAll(filepath.Dir(cachePath), 0700)
+		if err != nil {
+			return fmt.Errorf("failed to create cache directory: %w", err)
+		}
+		err = os.Rename(tempPath, cachePath)
+		if err != nil {
+			return fmt.Errorf("failed to rename file: %w", err)
+		}
+	}
+	tempFile, err = os.Open(cachePath)
+	if err != nil {
+		return fmt.Errorf("failed to open renamed file: %w", err)
+	}
+	saveInto.ThumbnailFile, saveInto.ThumbnailURL, err = gmx.uploadFile(ctx, checksum, tempFile, encrypt, fileInfo.Size(), "image/jpeg", "thumbnail.jpeg")
+	if err != nil {
+		return fmt.Errorf("failed to upload: %w", err)
+	}
+	saveInto.ThumbnailInfo = thumbnailInfo
+	return nil
+}
+
+func writeMaybeRespError(err error, w http.ResponseWriter) {
+	var httpErr mautrix.HTTPError
+	if errors.As(err, &httpErr) {
+		if httpErr.WrappedError != nil {
+			ErrBadGateway.WithMessage(httpErr.WrappedError.Error()).Write(w)
+		} else if httpErr.RespError != nil {
+			httpErr.RespError.Write(w)
+		} else {
+			mautrix.MUnknown.WithMessage("Server returned non-JSON error").Write(w)
+		}
+	} else {
+		mautrix.MUnknown.WithMessage(err.Error()).Write(w)
+	}
+}

cmd/gomuks/server.go

@@ -43,6 +43,7 @@ func (gmx *Gomuks) StartServer() {
 	api := http.NewServeMux()
 	api.HandleFunc("GET /websocket", gmx.HandleWebsocket)
 	api.HandleFunc("POST /auth", gmx.Authenticate)
+	api.HandleFunc("POST /upload", gmx.UploadMedia)
 	api.HandleFunc("GET /media/{server}/{media_id}", gmx.DownloadMedia)
 	apiHandler := exhttp.ApplyMiddleware(
 		api,

go.mod

@@ -7,6 +7,7 @@ toolchain go1.23.2
 require (
 	github.com/chzyer/readline v1.5.1
 	github.com/coder/websocket v1.8.12
+	github.com/gabriel-vasile/mimetype v1.4.6
 	github.com/lucasb-eyer/go-colorful v1.2.0
 	github.com/mattn/go-sqlite3 v1.14.24
 	github.com/rivo/uniseg v0.4.7
@@ -14,13 +15,14 @@ require (
 	github.com/tidwall/gjson v1.18.0
 	github.com/tidwall/sjson v1.2.5
 	github.com/yuin/goldmark v1.7.7
-	go.mau.fi/util v0.8.1
+	go.mau.fi/util v0.8.2-0.20241018231932-9da45c4e6e04
 	go.mau.fi/zeroconfig v0.1.3
 	golang.org/x/crypto v0.28.0
+	golang.org/x/image v0.21.0
 	golang.org/x/net v0.30.0
 	gopkg.in/yaml.v3 v3.0.1
 	maunium.net/go/mauflag v1.0.0
-	maunium.net/go/mautrix v0.21.2-0.20241018110725-6c07832ed7b5
+	maunium.net/go/mautrix v0.21.2-0.20241018174905-3277c529a2e5
 	mvdan.cc/xurls/v2 v2.5.0
 )
 

go.sum

@@ -14,6 +14,8 @@ github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8
 github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc=
 github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c=
 github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
+github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
+github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/lucasb-eyer/go-colorful v1.2.0 h1:1nnpGOrhyZZuNyfu1QjKiUICQ74+3FNCN69Aj6K7nkY=
 github.com/lucasb-eyer/go-colorful v1.2.0/go.mod h1:R4dSotOR9KMtayYi1e77YzuveK+i7ruzyGqttikkLy0=
@@ -49,14 +51,16 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY=
 github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28=
 github.com/yuin/goldmark v1.7.7 h1:5m9rrB1sW3JUMToKFQfb+FGt1U7r57IHu5GrYrG2nqU=
 github.com/yuin/goldmark v1.7.7/go.mod h1:uzxRWxtg69N339t3louHJ7+O03ezfj6PlliRlaOzY1E=
-go.mau.fi/util v0.8.1 h1:Ga43cz6esQBYqcjZ/onRoVnYWoUwjWbsxVeJg2jOTSo=
+go.mau.fi/util v0.8.2-0.20241018231932-9da45c4e6e04 h1:sfTKol1VyOfKb/QilgmtTT8GlZcL790IPWX60W1EEKU=
-go.mau.fi/util v0.8.1/go.mod h1:T1u/rD2rzidVrBLyaUdPpZiJdP/rsyi+aTzn0D+Q6wc=
+go.mau.fi/util v0.8.2-0.20241018231932-9da45c4e6e04/go.mod h1:T1u/rD2rzidVrBLyaUdPpZiJdP/rsyi+aTzn0D+Q6wc=
 go.mau.fi/zeroconfig v0.1.3 h1:As9wYDKmktjmNZW5i1vn8zvJlmGKHeVxHVIBMXsm4kM=
 go.mau.fi/zeroconfig v0.1.3/go.mod h1:NcSJkf180JT+1IId76PcMuLTNa1CzsFFZ0nBygIQM70=
 golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
 golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c h1:7dEasQXItcW1xKJ2+gg5VOiBnqWrJc+rq0DPKyvvdbY=
 golang.org/x/exp v0.0.0-20241009180824-f66d83c29e7c/go.mod h1:NQtJDoLvd6faHhE7m4T/1IY708gDefGGjR/iUW8yQQ8=
+golang.org/x/image v0.21.0 h1:c5qV36ajHpdj4Qi0GnE0jUc/yuo33OLFaa0d+crTD5s=
+golang.org/x/image v0.21.0/go.mod h1:vUbsLavqK/W303ZroQQVKQ+Af3Yl6Uz1Ppu5J/cLz78=
 golang.org/x/net v0.30.0 h1:AcW1SDZMkb8IpzCdQUaIq2sP4sZ4zw+55h6ynffypl4=
 golang.org/x/net v0.30.0/go.mod h1:2wGyMJ5iFasEhkwi13ChkO/t1ECNC4X4eBKkVFyYFlU=
 golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
@@ -73,7 +77,7 @@ gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
 gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 maunium.net/go/mauflag v1.0.0 h1:YiaRc0tEI3toYtJMRIfjP+jklH45uDHtT80nUamyD4M=
 maunium.net/go/mauflag v1.0.0/go.mod h1:nLivPOpTpHnpzEh8jEdSL9UqO9+/KBJFmNRlwKfkPeA=
-maunium.net/go/mautrix v0.21.2-0.20241018110725-6c07832ed7b5 h1:tbBKkNxpRabCFcekRMDTaV+z+ZZUAG5zNpOMI4zk/sQ=
+maunium.net/go/mautrix v0.21.2-0.20241018174905-3277c529a2e5 h1:ITR15W0vCyMiX5ZO6JLCPmHrdSL2+4TLBuDaISwGsFE=
-maunium.net/go/mautrix v0.21.2-0.20241018110725-6c07832ed7b5/go.mod h1:sjCZR1R/3NET/WjkcXPL6WpAHlWKku9HjRsdOkbM8Qw=
+maunium.net/go/mautrix v0.21.2-0.20241018174905-3277c529a2e5/go.mod h1:sjCZR1R/3NET/WjkcXPL6WpAHlWKku9HjRsdOkbM8Qw=
 mvdan.cc/xurls/v2 v2.5.0 h1:lyBNOm8Wo71UknhUs4QTFUNNMyxy2JEIaKKo0RWOh+8=
 mvdan.cc/xurls/v2 v2.5.0/go.mod h1:yQgaGQ1rFtJUzkmKiHYSSfuQxqfYmd//X6PxvholpeE=

pkg/hicli/json-commands.go

@@ -41,7 +41,7 @@ func (h *HiClient) handleJSONCommand(ctx context.Context, req *JSONCommand) (any
 		})
 	case "send_message":
 		return unmarshalAndCall(req.Data, func(params *sendMessageParams) (*database.Event, error) {
-			return h.SendMessage(ctx, params.RoomID, params.Text, params.MediaPath, params.ReplyTo, params.Mentions)
+			return h.SendMessage(ctx, params.RoomID, params.BaseContent, params.Text, params.ReplyTo, params.Mentions)
 		})
 	case "send_event":
 		return unmarshalAndCall(req.Data, func(params *sendEventParams) (*database.Event, error) {
@@ -115,11 +115,11 @@ type cancelRequestParams struct {
 }
 
 type sendMessageParams struct {
-	RoomID    id.RoomID       `json:"room_id"`
+	RoomID      id.RoomID                  `json:"room_id"`
-	Text      string          `json:"text"`
+	BaseContent *event.MessageEventContent `json:"base_content"`
-	MediaPath string          `json:"media_path"`
+	Text        string                     `json:"text"`
-	ReplyTo   id.EventID      `json:"reply_to"`
+	ReplyTo     id.EventID                 `json:"reply_to"`
-	Mentions  *event.Mentions `json:"mentions"`
+	Mentions    *event.Mentions            `json:"mentions"`
 }
 
 type sendEventParams struct {

pkg/hicli/send.go

@@ -32,7 +32,7 @@ var (
 	rainbowWithHTML = goldmark.New(format.Extensions, format.HTMLOptions, goldmark.WithExtensions(rainbow.Extension))
 )
 
-func (h *HiClient) SendMessage(ctx context.Context, roomID id.RoomID, text, mediaPath string, replyTo id.EventID, mentions *event.Mentions) (*database.Event, error) {
+func (h *HiClient) SendMessage(ctx context.Context, roomID id.RoomID, base *event.MessageEventContent, text string, replyTo id.EventID, mentions *event.Mentions) (*database.Event, error) {
 	var content event.MessageEventContent
 	if strings.HasPrefix(text, "/rainbow ") {
 		text = strings.TrimPrefix(text, "/rainbow ")
@@ -44,9 +44,20 @@ func (h *HiClient) SendMessage(ctx context.Context, roomID id.RoomID, text, medi
 	} else if strings.HasPrefix(text, "/html ") {
 		text = strings.TrimPrefix(text, "/html ")
 		content = format.RenderMarkdown(text, false, true)
-	} else {
+	} else if text != "" {
 		content = format.RenderMarkdown(text, true, false)
 	}
+	if base != nil {
+		if text != "" {
+			base.Body = content.Body
+			base.Format = content.Format
+			base.FormattedBody = content.FormattedBody
+		}
+		content = *base
+	}
+	if content.Mentions == nil {
+		content.Mentions = &event.Mentions{}
+	}
 	if mentions != nil {
 		content.Mentions.Room = mentions.Room
 		for _, userID := range mentions.UserIDs {
@@ -84,16 +95,21 @@ func (h *HiClient) SetTyping(ctx context.Context, roomID id.RoomID, timeout time
 	return err
 }
 
-func (h *HiClient) Send(ctx context.Context, roomID id.RoomID, evtType event.Type, content any) (*database.Event, error) {
+func (h *HiClient) Send(
-	roomMeta, err := h.DB.Room.Get(ctx, roomID)
+	ctx context.Context,
+	roomID id.RoomID,
+	evtType event.Type,
+	content any,
+) (*database.Event, error) {
+	room, err := h.DB.Room.Get(ctx, roomID)
 	if err != nil {
 		return nil, fmt.Errorf("failed to get room metadata: %w", err)
-	} else if roomMeta == nil {
+	} else if room == nil {
 		return nil, fmt.Errorf("unknown room")
 	}
 	txnID := "hicli-" + h.Client.TxnID()
 	dbEvt := &database.Event{
-		RoomID:          roomID,
+		RoomID:          room.ID,
 		ID:              id.EventID(fmt.Sprintf("~%s", txnID)),
 		Sender:          h.Account.UserID,
 		Timestamp:       jsontime.UnixMilliNow(),
@@ -104,7 +120,7 @@ func (h *HiClient) Send(ctx context.Context, roomID id.RoomID, evtType event.Typ
 		Reactions:       map[string]int{},
 		LastEditRowID:   ptr.Ptr(database.EventRowID(0)),
 	}
-	if roomMeta.EncryptionEvent != nil && evtType != event.EventReaction {
+	if room.EncryptionEvent != nil && evtType != event.EventReaction {
 		dbEvt.Type = event.EventEncrypted.Type
 		dbEvt.DecryptedType = evtType.Type
 		dbEvt.Decrypted, err = json.Marshal(content)
@@ -128,7 +144,7 @@ func (h *HiClient) Send(ctx context.Context, roomID id.RoomID, evtType event.Typ
 	}
 	ctx = context.WithoutCancel(ctx)
 	go func() {
-		err := h.SetTyping(ctx, roomID, 0)
+		err := h.SetTyping(ctx, room.ID, 0)
 		if err != nil {
 			zerolog.Ctx(ctx).Err(err).Msg("Failed to stop typing while sending message")
 		}
@@ -150,7 +166,7 @@ func (h *HiClient) Send(ctx context.Context, roomID id.RoomID, evtType event.Typ
 		}()
 		if dbEvt.Decrypted != nil {
 			var encryptedContent *event.EncryptedEventContent
-			encryptedContent, err = h.Encrypt(ctx, roomMeta, evtType, dbEvt.Decrypted)
+			encryptedContent, err = h.Encrypt(ctx, room, evtType, dbEvt.Decrypted)
 			if err != nil {
 				dbEvt.SendError = fmt.Sprintf("failed to encrypt: %v", err)
 				zerolog.Ctx(ctx).Err(err).Msg("Failed to encrypt event")
@@ -172,7 +188,7 @@ func (h *HiClient) Send(ctx context.Context, roomID id.RoomID, evtType event.Typ
 			}
 		}
 		var resp *mautrix.RespSendEvent
-		resp, err = h.Client.SendMessageEvent(ctx, roomID, evtType, dbEvt.Content, mautrix.ReqSendEvent{
+		resp, err = h.Client.SendMessageEvent(ctx, room.ID, evtType, dbEvt.Content, mautrix.ReqSendEvent{
 			Timestamp:     dbEvt.Timestamp.UnixMilli(),
 			TransactionID: txnID,
 			DontEncrypt:   true,

web/src/api/rpc.ts

@@ -21,6 +21,7 @@ import type {
 	EventRowID,
 	EventType,
 	Mentions,
+	MessageEventContent,
 	PaginationResponse,
 	RPCCommand,
 	RPCEvent,
@@ -44,6 +45,7 @@ export class ErrorResponse extends Error {
 
 export interface SendMessageParams {
 	room_id: RoomID
+	base_content?: MessageEventContent
 	text: string
 	media_path?: string
 	reply_to?: EventID

web/src/icons/attach.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 -960 960 960" width="24px" fill="#5f6368"><path d="M720-330q0 104-73 177T470-80q-104 0-177-73t-73-177v-370q0-75 52.5-127.5T400-880q75 0 127.5 52.5T580-700v350q0 46-32 78t-78 32q-46 0-78-32t-32-78v-370h80v370q0 13 8.5 21.5T470-320q13 0 21.5-8.5T500-350v-350q-1-42-29.5-71T400-800q-42 0-71 29t-29 71v370q-1 71 49 120.5T470-160q70 0 119-49.5T640-330v-390h80v390Z"/></svg>

web/src/icons/send.svg

@@ -0,0 +1 @@
+<svg xmlns="http://www.w3.org/2000/svg" height="24px" viewBox="0 -960 960 960" width="24px" fill="#5f6368"><path d="M120-160v-640l760 320-760 320Zm80-120 474-200-474-200v140l240 60-240 60v140Zm0 0v-400 400Z"/></svg>

web/src/index.css

@@ -25,6 +25,15 @@ button {
 	font-size: 1em;
 	background: none;
 	border: none;
+	border-radius: .25rem;
+
+	&:disabled {
+		cursor: default;
+	}
+
+	&:hover:not(:disabled) {
+		background-color: rgba(0, 0, 0, .2);
+	}
 }
 
 :root {

web/src/ui/MessageComposer.css

@@ -3,6 +3,9 @@ div.message-composer {
 
 	> div.input-area {
 		display: flex;
+		align-items: center;
+		line-height: 1.25;
+		margin-right: .25rem;
 
 		> textarea {
 			flex: 1;
@@ -15,6 +18,20 @@ div.message-composer {
 		}
 
 		> button {
+			height: 2rem;
+			width: 2rem;
+			padding: .25rem;
+		}
+	}
+
+	> div.composer-media {
+		display: flex;
+		padding: .5rem;
+		justify-content: space-between;
+
+		> button {
+			height: 2.5rem;
+			width: 2.5rem;
 			padding: .5rem;
 		}
 	}

web/src/ui/MessageComposer.tsx

@@ -14,10 +14,15 @@
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import React, { use, useCallback, useLayoutEffect, useRef, useState } from "react"
+import { ScaleLoader } from "react-spinners"
 import { RoomStateStore } from "@/api/statestore"
-import { MemDBEvent, Mentions, RoomID } from "@/api/types"
+import { MediaMessageEventContent, MemDBEvent, Mentions, RoomID } from "@/api/types"
 import { ClientContext } from "./ClientContext.ts"
 import { ReplyBody } from "./timeline/ReplyBody.tsx"
+import { useMediaContent } from "./timeline/content/useMediaContent.tsx"
+import AttachIcon from "@/icons/attach.svg?react"
+import CloseIcon from "@/icons/close.svg?react"
+import SendIcon from "@/icons/send.svg?react"
 import "./MessageComposer.css"
 
 interface MessageComposerProps {
@@ -36,6 +41,9 @@ const draftStore = {
 const MessageComposer = ({ room, replyTo, setTextRows, closeReply }: MessageComposerProps) => {
 	const client = use(ClientContext)!
 	const [text, setText] = useState("")
+	const [media, setMedia] = useState(null)
+	const [loadingMedia, setLoadingMedia] = useState(false)
+	const fileInput = useRef<HTMLInputElement>(null)
 	const textRows = useRef(1)
 	const typingSentAt = useRef(0)
 	const fullSetText = useCallback((text: string, setDraft: boolean) => {
@@ -52,10 +60,11 @@ const MessageComposer = ({ room, replyTo, setTextRows, closeReply }: MessageComp
 	}, [setTextRows, room.roomID])
 	const sendMessage = useCallback((evt: React.FormEvent) => {
 		evt.preventDefault()
-		if (text === "") {
+		if (text === "" && !media) {
 			return
 		}
 		fullSetText("", true)
+		setMedia(null)
 		closeReply()
 		const room_id = room.roomID
 		const mentions: Mentions = {
@@ -65,9 +74,9 @@ const MessageComposer = ({ room, replyTo, setTextRows, closeReply }: MessageComp
 		if (replyTo) {
 			mentions.user_ids.push(replyTo.sender)
 		}
-		client.sendMessage({ room_id, text, reply_to: replyTo?.event_id, mentions })
+		client.sendMessage({ room_id, base_content: media ?? undefined, text, reply_to: replyTo?.event_id, mentions })
 			.catch(err => window.alert("Failed to send message: " + err))
-	}, [fullSetText, closeReply, replyTo, text, room, client])
+	}, [fullSetText, closeReply, replyTo, media, text, room, client])
 	const onKeyDown = useCallback((evt: React.KeyboardEvent) => {
 		if (evt.key === "Enter" && !evt.shiftKey) {
 			sendMessage(evt)
@@ -86,6 +95,31 @@ const MessageComposer = ({ room, replyTo, setTextRows, closeReply }: MessageComp
 				.catch(err => console.error("Failed to send stop typing notification:", err))
 		}
 	}, [client, room.roomID, fullSetText])
+	const openFilePicker = useCallback(() => {
+		fileInput.current!.click()
+	}, [])
+	const clearMedia = useCallback(() => {
+		setMedia(null)
+	}, [])
+	const onAttachFile = useCallback((evt: React.ChangeEvent<HTMLInputElement>) => {
+		setLoadingMedia(true)
+		const file = evt.target.files![0]
+		const encrypt = !!room.meta.current.encryption_event
+		fetch(`/_gomuks/upload?encrypt=${encrypt}&filename=${encodeURIComponent(file.name)}`, {
+			method: "POST",
+			body: file,
+		})
+			.then(async res => {
+				const json = await res.json()
+				if (!res.ok) {
+					throw new Error(json.error)
+				} else {
+					setMedia(json)
+				}
+			})
+			.catch(err => window.alert("Failed to upload file: " + err))
+			.finally(() => setLoadingMedia(false))
+	}, [room])
 	// To ensure the cursor jumps to the end, do this in an effect rather than as the initial value of useState
 	// To try to avoid the input bar flashing, use useLayoutEffect instead of useEffect
 	useLayoutEffect(() => {
@@ -100,6 +134,8 @@ const MessageComposer = ({ room, replyTo, setTextRows, closeReply }: MessageComp
 	}, [client, room.roomID, fullSetText])
 	return <div className="message-composer">
 		{replyTo && <ReplyBody room={room} event={replyTo} onClose={closeReply}/>}
+		{loadingMedia && <div className="composer-media"><ScaleLoader/></div>}
+		{media && <ComposerMedia content={media} clearMedia={clearMedia}/>}
 		<div className="input-area">
 			<textarea
 				autoFocus
@@ -110,9 +146,33 @@ const MessageComposer = ({ room, replyTo, setTextRows, closeReply }: MessageComp
 				placeholder="Send a message"
 				id="message-composer"
 			/>
-			<button onClick={sendMessage}>Send</button>
+			<button
+				onClick={openFilePicker}
+				disabled={!!media || loadingMedia}
+				title={media ? "You can only attach one file at a time" : ""}
+			><AttachIcon/></button>
+			<button onClick={sendMessage} disabled={(!text && !media) || loadingMedia}><SendIcon/></button>
+			<input ref={fileInput} onChange={onAttachFile} type="file" value="" style={{ display: "none" }}/>
 		</div>
 	</div>
 }
 
+interface ComposerMediaProps {
+	content: MediaMessageEventContent
+	clearMedia: () => void
+}
+
+const ComposerMedia = ({ content, clearMedia }: ComposerMediaProps) => {
+	// TODO stickers?
+	const [mediaContent, containerClass, containerStyle] = useMediaContent(
+		content, "m.room.message", { height: 120, width: 360 },
+	)
+	return <div className="composer-media">
+		<div className={`media-container ${containerClass}`} style={containerStyle}>
+			{mediaContent}
+		</div>
+		<button onClick={clearMedia}><CloseIcon/></button>
+	</div>
+}
+
 export default MessageComposer

web/src/ui/timeline/content/MessageBody.tsx

@@ -13,13 +13,9 @@
 //
 // You should have received a copy of the GNU Affero General Public License
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
-import { CSSProperties, use } from "react"
+import type { MediaMessageEventContent, MessageEventContent } from "@/api/types"
-import { getEncryptedMediaURL, getMediaURL } from "@/api/media.ts"
-import type { EventType, MediaMessageEventContent, MessageEventContent } from "@/api/types"
-import { calculateMediaSize } from "@/util/mediasize.ts"
-import { LightboxContext } from "../../Lightbox.tsx"
 import { EventContentProps } from "./props.ts"
-import DownloadIcon from "@/icons/download.svg?react"
+import { useMediaContent } from "./useMediaContent.tsx"
 
 const onClickHTML = (evt: React.MouseEvent<HTMLDivElement>) => {
 	if ((evt.target as HTMLElement).closest("span.hicli-spoiler")?.classList.toggle("spoiler-revealed")) {
@@ -42,64 +38,6 @@ export const TextMessageBody = ({ event }: EventContentProps) => {
 	return <div className="message-text plaintext-body">{content.body}</div>
 }
 
-const useMediaContent = (
-	content: MediaMessageEventContent, evtType: EventType,
-): [React.ReactElement | null, string, CSSProperties] => {
-	const mediaURL = content.url ? getMediaURL(content.url) : getEncryptedMediaURL(content.file?.url)
-	const thumbnailURL = content.info?.thumbnail_url
-		? getMediaURL(content.info.thumbnail_url) : getEncryptedMediaURL(content.info?.thumbnail_file?.url)
-	if (content.msgtype === "m.image" || evtType === "m.sticker") {
-		const style = calculateMediaSize(content.info?.w, content.info?.h)
-		return [<img
-			loading="lazy"
-			style={style.media}
-			src={mediaURL}
-			alt={content.filename ?? content.body}
-			onClick={use(LightboxContext)}
-		/>, "image-container", style.container]
-	} else if (content.msgtype === "m.video") {
-		const autoplay = false
-		const controls = !content.info?.["fi.mau.hide_controls"]
-		const loop = !!content.info?.["fi.mau.loop"]
-		let onMouseOver: React.MouseEventHandler<HTMLVideoElement> | undefined
-		let onMouseOut: React.MouseEventHandler<HTMLVideoElement> | undefined
-		if (!autoplay && !controls) {
-			onMouseOver = (event: React.MouseEvent<HTMLVideoElement>) => event.currentTarget.play()
-			onMouseOut = (event: React.MouseEvent<HTMLVideoElement>) => {
-				event.currentTarget.pause()
-				event.currentTarget.currentTime = 0
-			}
-		}
-		return [<video
-			autoPlay={autoplay}
-			controls={controls}
-			loop={loop}
-			poster={thumbnailURL}
-			onMouseOver={onMouseOver}
-			onMouseOut={onMouseOut}
-			preload="none"
-		>
-			<source src={mediaURL} type={content.info?.mimetype} />
-		</video>, "video-container", {}]
-	} else if (content.msgtype === "m.audio") {
-		return [<audio controls src={mediaURL} preload="none"/>, "audio-container", {}]
-	} else if (content.msgtype === "m.file") {
-		return [
-			<>
-				<a
-					href={mediaURL}
-					target="_blank"
-					rel="noopener noreferrer"
-					download={content.filename ?? content.body}
-				><DownloadIcon height={32} width={32}/> {content.filename ?? content.body}</a>
-			</>,
-			"file-container",
-			{},
-		]
-	}
-	return [null, "unknown-container", {}]
-}
-
 export const MediaMessageBody = ({ event, room }: EventContentProps) => {
 	const content = event.content as MediaMessageEventContent
 	let caption = null

web/src/ui/timeline/content/useMediaContent.tsx

@@ -0,0 +1,79 @@
+// gomuks - A Matrix client written in Go.
+// Copyright (C) 2024 Tulir Asokan
+//
+// This program is free software: you can redistribute it and/or modify
+// it under the terms of the GNU Affero General Public License as published by
+// the Free Software Foundation, either version 3 of the License, or
+// (at your option) any later version.
+//
+// This program is distributed in the hope that it will be useful,
+// but WITHOUT ANY WARRANTY; without even the implied warranty of
+// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+// GNU Affero General Public License for more details.
+//
+// You should have received a copy of the GNU Affero General Public License
+// along with this program.  If not, see <https://www.gnu.org/licenses/>.
+import { CSSProperties, use } from "react"
+import { getEncryptedMediaURL, getMediaURL } from "@/api/media.ts"
+import type { EventType, MediaMessageEventContent } from "@/api/types"
+import { ImageContainerSize, calculateMediaSize } from "@/util/mediasize.ts"
+import { LightboxContext } from "../../Lightbox.tsx"
+import DownloadIcon from "@/icons/download.svg?react"
+
+export const useMediaContent = (
+	content: MediaMessageEventContent, evtType: EventType, containerSize?: ImageContainerSize,
+): [React.ReactElement | null, string, CSSProperties] => {
+	const mediaURL = content.url ? getMediaURL(content.url) : getEncryptedMediaURL(content.file?.url)
+	const thumbnailURL = content.info?.thumbnail_url
+		? getMediaURL(content.info.thumbnail_url) : getEncryptedMediaURL(content.info?.thumbnail_file?.url)
+	if (content.msgtype === "m.image" || evtType === "m.sticker") {
+		const style = calculateMediaSize(content.info?.w, content.info?.h, containerSize)
+		return [<img
+			loading="lazy"
+			style={style.media}
+			src={mediaURL}
+			alt={content.filename ?? content.body}
+			onClick={use(LightboxContext)}
+		/>, "image-container", style.container]
+	} else if (content.msgtype === "m.video") {
+		const autoplay = false
+		const controls = !content.info?.["fi.mau.hide_controls"]
+		const loop = !!content.info?.["fi.mau.loop"]
+		let onMouseOver: React.MouseEventHandler<HTMLVideoElement> | undefined
+		let onMouseOut: React.MouseEventHandler<HTMLVideoElement> | undefined
+		if (!autoplay && !controls) {
+			onMouseOver = (event: React.MouseEvent<HTMLVideoElement>) => event.currentTarget.play()
+			onMouseOut = (event: React.MouseEvent<HTMLVideoElement>) => {
+				event.currentTarget.pause()
+				event.currentTarget.currentTime = 0
+			}
+		}
+		return [<video
+			autoPlay={autoplay}
+			controls={controls}
+			loop={loop}
+			poster={thumbnailURL}
+			onMouseOver={onMouseOver}
+			onMouseOut={onMouseOut}
+			preload="none"
+		>
+			<source src={mediaURL} type={content.info?.mimetype}/>
+		</video>, "video-container", {}]
+	} else if (content.msgtype === "m.audio") {
+		return [<audio controls src={mediaURL} preload="none"/>, "audio-container", {}]
+	} else if (content.msgtype === "m.file") {
+		return [
+			<>
+				<a
+					href={mediaURL}
+					target="_blank"
+					rel="noopener noreferrer"
+					download={content.filename ?? content.body}
+				><DownloadIcon height={32} width={32}/> {content.filename ?? content.body}</a>
+			</>,
+			"file-container",
+			{},
+		]
+	}
+	return [null, "unknown-container", {}]
+}

web/src/util/mediasize.ts

@@ -15,16 +15,24 @@
 // along with this program.  If not, see <https://www.gnu.org/licenses/>.
 import { CSSProperties } from "react"
 
-const imageContainerWidth = 320
-const imageContainerHeight = 240
-const imageContainerAspectRatio = imageContainerWidth / imageContainerHeight
-
 export interface CalculatedMediaSize {
 	container: CSSProperties
 	media: CSSProperties
 }
 
-export function calculateMediaSize(width?: number, height?: number): CalculatedMediaSize {
+export interface ImageContainerSize {
+	width: number
+	height: number
+}
+
+export const defaultImageContainerSize: ImageContainerSize = { width: 320, height: 240 }
+
+export function calculateMediaSize(
+	width?: number,
+	height?: number,
+	imageContainer: ImageContainerSize | undefined = defaultImageContainerSize,
+): CalculatedMediaSize {
+	const { width: imageContainerWidth, height: imageContainerHeight } = imageContainer ?? defaultImageContainerSize
 	if (!width || !height) {
 		return {
 			container: {
@@ -34,6 +42,8 @@ export function calculateMediaSize(width?: number, height?: number): CalculatedM
 			media: {},
 		}
 	}
+	const imageContainerAspectRatio = imageContainerWidth / imageContainerHeight
+
 	const origWidth = width
 	const origHeight = height
 	if (width > imageContainerWidth || height > imageContainerHeight) {