diff --git a/cmd/gomuks/media.go b/cmd/gomuks/media.go
index ff5d481..063eceb 100644
--- a/cmd/gomuks/media.go
+++ b/cmd/gomuks/media.go
@@ -72,6 +72,9 @@ func (gmx *Gomuks) downloadMediaFromCache(ctx context.Context, w http.ResponseWr
 	} else if r.Header.Get("If-None-Match") == entry.ETag() {
 		w.WriteHeader(http.StatusNotModified)
 		return true
+	} else if entry.MimeType != "" && r.URL.Query().Has("fallback") && !isAllowedAvatarMime(entry.MimeType) {
+		w.WriteHeader(http.StatusUnsupportedMediaType)
+		return true
 	}
 	log := zerolog.Ctx(ctx)
 	cacheFile, err := os.Open(gmx.cacheEntryToPath(entry.Hash[:]))
@@ -130,30 +133,35 @@ type avatarResponseWriter struct {
 	http.ResponseWriter
 	bgColor   string
 	character string
-	data      []byte
 	errored   bool
 }
 
+func isAllowedAvatarMime(mime string) bool {
+	switch mime {
+	case "image/png", "image/jpeg", "image/gif", "image/webp":
+		return true
+	default:
+		return false
+	}
+}
+
 func (w *avatarResponseWriter) WriteHeader(statusCode int) {
-	if statusCode != http.StatusOK {
-		w.data = []byte(fmt.Sprintf(fallbackAvatarTemplate, w.bgColor, w.character))
+	if statusCode != http.StatusOK && statusCode != http.StatusNotModified {
+		data := []byte(fmt.Sprintf(fallbackAvatarTemplate, w.bgColor, w.character))
 		w.Header().Set("Content-Type", "image/svg+xml")
-		w.Header().Set("Content-Length", strconv.Itoa(len(w.data)))
+		w.Header().Set("Content-Length", strconv.Itoa(len(data)))
 		w.Header().Del("Content-Disposition")
+		w.ResponseWriter.WriteHeader(http.StatusOK)
+		_, _ = w.ResponseWriter.Write(data)
 		w.errored = true
-		statusCode = http.StatusOK
+		return
 	}
 	w.ResponseWriter.WriteHeader(statusCode)
 }
 
 func (w *avatarResponseWriter) Write(p []byte) (n int, err error) {
 	if w.errored {
-		if w.data != nil {
-			_, err = w.ResponseWriter.Write(w.data)
-			w.data = nil
-		}
-		n = len(p)
-		return
+		return len(p), nil
 	}
 	return w.ResponseWriter.Write(p)
 }