diff --git a/cmd/gomuks/media.go b/cmd/gomuks/media.go index 0bce71d..a6f0cc8 100644 --- a/cmd/gomuks/media.go +++ b/cmd/gomuks/media.go @@ -57,7 +57,7 @@ var ErrBadGateway = mautrix.RespError{ StatusCode: http.StatusBadGateway, } -func (gmx *Gomuks) downloadMediaFromCache(ctx context.Context, w http.ResponseWriter, entry *database.CachedMedia, force bool) bool { +func (gmx *Gomuks) downloadMediaFromCache(ctx context.Context, w http.ResponseWriter, entry *database.Media, force bool) bool { if !entry.UseCache() { if force { mautrix.MNotFound.WithMessage("Media not found in cache").Write(w) @@ -97,7 +97,7 @@ func (gmx *Gomuks) cacheEntryToPath(hash []byte) string { return filepath.Join(gmx.CacheDir, "media", hashPath[0:2], hashPath[2:4], hashPath[4:]) } -func cacheEntryToHeaders(w http.ResponseWriter, entry *database.CachedMedia) { +func cacheEntryToHeaders(w http.ResponseWriter, entry *database.Media) { w.Header().Set("Content-Type", entry.MimeType) w.Header().Set("Content-Length", strconv.FormatInt(entry.Size, 10)) w.Header().Set("Content-Disposition", mime.FormatMediaType(entry.ContentDisposition(), map[string]string{"filename": entry.FileName})) @@ -122,7 +122,7 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) { Logger() log := &logVal ctx := log.WithContext(r.Context()) - cacheEntry, err := gmx.Client.DB.CachedMedia.Get(ctx, mxc) + cacheEntry, err := gmx.Client.DB.Media.Get(ctx, mxc) if err != nil { log.Err(err).Msg("Failed to get cached media entry") mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to get cached media entry: %v", err)).Write(w) @@ -152,7 +152,7 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) { log.Err(err).Msg("Failed to download media") var httpErr mautrix.HTTPError if cacheEntry == nil { - cacheEntry = &database.CachedMedia{ + cacheEntry = &database.Media{ MXC: mxc, } } @@ -179,7 +179,7 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) { cacheEntry.Error.Matrix = ptr.Ptr(ErrBadGateway.WithMessage(err.Error())) cacheEntry.Error.StatusCode = http.StatusBadGateway } - err = gmx.Client.DB.CachedMedia.Put(ctx, cacheEntry) + err = gmx.Client.DB.Media.Put(ctx, cacheEntry) if err != nil { log.Err(err).Msg("Failed to save errored cache entry") } @@ -190,7 +190,7 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) { _ = resp.Body.Close() }() if cacheEntry == nil { - cacheEntry = &database.CachedMedia{ + cacheEntry = &database.Media{ MXC: mxc, MimeType: resp.Header.Get("Content-Type"), Size: resp.ContentLength, @@ -231,7 +231,7 @@ func (gmx *Gomuks) DownloadMedia(w http.ResponseWriter, r *http.Request) { } cacheEntry.Hash = (*[32]byte)(fileHasher.Sum(nil)) cacheEntry.Error = nil - err = gmx.Client.DB.CachedMedia.Put(ctx, cacheEntry) + err = gmx.Client.DB.Media.Put(ctx, cacheEntry) if err != nil { log.Err(err).Msg("Failed to save cache entry") mautrix.MUnknown.WithMessage(fmt.Sprintf("Failed to save cache entry: %v", err)).Write(w) @@ -333,7 +333,7 @@ func (gmx *Gomuks) UploadMedia(w http.ResponseWriter, r *http.Request) { } func (gmx *Gomuks) uploadFile(ctx context.Context, checksum []byte, cacheFile *os.File, encrypt bool, fileSize int64, mimeType, fileName string) (*event.EncryptedFileInfo, id.ContentURIString, error) { - cm := &database.CachedMedia{ + cm := &database.Media{ FileName: fileName, MimeType: mimeType, Size: fileSize, @@ -359,7 +359,7 @@ func (gmx *Gomuks) uploadFile(ctx context.Context, checksum []byte, cacheFile *o return nil, "", fmt.Errorf("failed to close cache reader: %w", err) } cm.MXC = resp.ContentURI - err = gmx.Client.DB.CachedMedia.Put(ctx, cm) + err = gmx.Client.DB.Media.Put(ctx, cm) if err != nil { zerolog.Ctx(ctx).Err(err). Stringer("mxc", cm.MXC). diff --git a/pkg/hicli/database/cachedmedia.go b/pkg/hicli/database/cachedmedia.go deleted file mode 100644 index 35cb9a9..0000000 --- a/pkg/hicli/database/cachedmedia.go +++ /dev/null @@ -1,149 +0,0 @@ -// Copyright (c) 2024 Tulir Asokan -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this -// file, You can obtain one at http://mozilla.org/MPL/2.0/. - -package database - -import ( - "context" - "database/sql" - "net/http" - "slices" - "time" - - "go.mau.fi/util/dbutil" - "go.mau.fi/util/jsontime" - "maunium.net/go/mautrix" - "maunium.net/go/mautrix/crypto/attachment" - "maunium.net/go/mautrix/id" -) - -const ( - insertCachedMediaQuery = ` - INSERT INTO cached_media (mxc, event_rowid, enc_file, file_name, mime_type, size, hash, error) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (mxc) DO NOTHING - ` - upsertCachedMediaQuery = ` - INSERT INTO cached_media (mxc, event_rowid, enc_file, file_name, mime_type, size, hash, error) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8) - ON CONFLICT (mxc) DO UPDATE - SET enc_file = excluded.enc_file, - file_name = excluded.file_name, - mime_type = excluded.mime_type, - size = excluded.size, - hash = excluded.hash, - error = excluded.error - WHERE excluded.error IS NULL OR cached_media.hash IS NULL - ` - getCachedMediaQuery = ` - SELECT mxc, event_rowid, enc_file, file_name, mime_type, size, hash, error - FROM cached_media - WHERE mxc = $1 - ` -) - -type CachedMediaQuery struct { - *dbutil.QueryHelper[*CachedMedia] -} - -func (cmq *CachedMediaQuery) Add(ctx context.Context, cm *CachedMedia) error { - return cmq.Exec(ctx, insertCachedMediaQuery, cm.sqlVariables()...) -} - -func (cmq *CachedMediaQuery) Put(ctx context.Context, cm *CachedMedia) error { - return cmq.Exec(ctx, upsertCachedMediaQuery, cm.sqlVariables()...) -} - -func (cmq *CachedMediaQuery) Get(ctx context.Context, mxc id.ContentURI) (*CachedMedia, error) { - return cmq.QueryOne(ctx, getCachedMediaQuery, &mxc) -} - -type MediaError struct { - Matrix *mautrix.RespError `json:"data"` - StatusCode int `json:"status_code"` - ReceivedAt jsontime.UnixMilli `json:"received_at"` - Attempts int `json:"attempts"` -} - -const MaxMediaBackoff = 7 * 24 * time.Hour - -func (me *MediaError) backoff() time.Duration { - return min(time.Duration(2< v4 (compatible with v1+): Latest revision +-- v0 -> v5 (compatible with v5+): Latest revision CREATE TABLE account ( user_id TEXT NOT NULL PRIMARY KEY, device_id TEXT NOT NULL, @@ -192,17 +192,23 @@ BEGIN AND reactions IS NOT NULL; END; -CREATE TABLE cached_media ( - mxc TEXT NOT NULL PRIMARY KEY, - event_rowid INTEGER, - enc_file TEXT, - file_name TEXT, - mime_type TEXT, - size INTEGER, - hash BLOB, - error TEXT, +CREATE TABLE media ( + mxc TEXT NOT NULL PRIMARY KEY, + enc_file TEXT, + file_name TEXT, + mime_type TEXT, + size INTEGER, + hash BLOB, + error TEXT +) STRICT; - CONSTRAINT cached_media_event_fkey FOREIGN KEY (event_rowid) REFERENCES event (rowid) ON DELETE SET NULL +CREATE TABLE media_reference ( + event_rowid INTEGER NOT NULL, + media_mxc TEXT NOT NULL, + + PRIMARY KEY (event_rowid, media_mxc), + CONSTRAINT media_reference_event_fkey FOREIGN KEY (event_rowid) REFERENCES event (rowid) ON DELETE CASCADE, + CONSTRAINT media_reference_media_fkey FOREIGN KEY (media_mxc) REFERENCES media (mxc) ON DELETE CASCADE ) STRICT; CREATE TABLE session_request ( diff --git a/pkg/hicli/database/upgrades/05-refactor-media-cache.sql b/pkg/hicli/database/upgrades/05-refactor-media-cache.sql new file mode 100644 index 0000000..7434bc7 --- /dev/null +++ b/pkg/hicli/database/upgrades/05-refactor-media-cache.sql @@ -0,0 +1,29 @@ +-- v5: Refactor media cache +CREATE TABLE media ( + mxc TEXT NOT NULL PRIMARY KEY, + enc_file TEXT, + file_name TEXT, + mime_type TEXT, + size INTEGER, + hash BLOB, + error TEXT +) STRICT; + +INSERT INTO media (mxc, enc_file, file_name, mime_type, size, hash, error) +SELECT mxc, enc_file, file_name, mime_type, size, hash, error +FROM cached_media; + +CREATE TABLE media_reference ( + event_rowid INTEGER NOT NULL, + media_mxc TEXT NOT NULL, + + PRIMARY KEY (event_rowid, media_mxc), + CONSTRAINT media_reference_event_fkey FOREIGN KEY (event_rowid) REFERENCES event (rowid) ON DELETE CASCADE, + CONSTRAINT media_reference_media_fkey FOREIGN KEY (media_mxc) REFERENCES media (mxc) ON DELETE CASCADE +) STRICT; + +INSERT INTO media_reference (event_rowid, media_mxc) +SELECT event_rowid, mxc +FROM cached_media WHERE event_rowid IS NOT NULL; + +DROP TABLE cached_media; diff --git a/pkg/hicli/sync.go b/pkg/hicli/sync.go index d32a24a..d0fbda2 100644 --- a/pkg/hicli/sync.go +++ b/pkg/hicli/sync.go @@ -282,10 +282,9 @@ func (h *HiClient) addMediaCache( if !parsedMXC.IsValid() { return } - cm := &database.CachedMedia{ - MXC: parsedMXC, - EventRowID: eventRowID, - FileName: fileName, + cm := &database.Media{ + MXC: parsedMXC, + FileName: fileName, } if file != nil { cm.EncFile = &file.EncryptedFile @@ -293,12 +292,20 @@ func (h *HiClient) addMediaCache( if info != nil { cm.MimeType = info.MimeType } - err := h.DB.CachedMedia.Put(ctx, cm) + err := h.DB.Media.Put(ctx, cm) if err != nil { zerolog.Ctx(ctx).Warn().Err(err). Stringer("mxc", parsedMXC). Int64("event_rowid", int64(eventRowID)). - Msg("Failed to add cached media entry") + Msg("Failed to add database media entry") + return + } + err = h.DB.Media.AddReference(ctx, eventRowID, parsedMXC) + if err != nil { + zerolog.Ctx(ctx).Warn().Err(err). + Stringer("mxc", parsedMXC). + Int64("event_rowid", int64(eventRowID)). + Msg("Failed to add database media reference") } }