gabehf/Koito
1
0
Fork 0
mirror of https://github.com/gabehf/Koito.git synced 2026-04-22 12:01:52 -07:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add bulk import optimization: track_lookup cache, batch inserts, BulkSubmitter

Browse source
This commit is contained in:
safierinx-a 2026-03-27 03:29:33 +05:30
parent 0ec7b458cc
commit ae373a7090
21 changed files with 1296 additions and 125 deletions
Download patch file Download diff file Expand all files Collapse all files

49
internal/catalog/associate_album.go
View file

@ -122,17 +122,18 @@ func createOrUpdateAlbumWithMbzReleaseID(ctx context.Context, d db.DB, opts Asso
}
}
l.Debug().Msg("Searching for album images...")
var imgid uuid.UUID
imgUrl, err := images.GetAlbumImage(ctx, images.AlbumImageOpts{
Artists: utils.UniqueIgnoringCase(slices.Concat(utils.FlattenMbzArtistCreditNames(release.ArtistCredit), utils.FlattenArtistNames(opts.Artists))),
Album: release.Title,
ReleaseMbzID: &opts.ReleaseMbzID,
})
var imgUrl string
if !opts.SkipCacheImage {
l.Debug().Msg("Searching for album images...")
imgUrl, err = images.GetAlbumImage(ctx, images.AlbumImageOpts{
Artists: utils.UniqueIgnoringCase(slices.Concat(utils.FlattenMbzArtistCreditNames(release.ArtistCredit), utils.FlattenArtistNames(opts.Artists))),
Album: release.Title,
ReleaseMbzID: &opts.ReleaseMbzID,
})
if err == nil && imgUrl != "" {
imgid = uuid.New()
if !opts.SkipCacheImage {
if err == nil && imgUrl != "" {
imgid = uuid.New()
var size ImageSize
if cfg.FullImageCacheEnabled() {
size = ImageSizeFull
@ -144,13 +145,11 @@ func createOrUpdateAlbumWithMbzReleaseID(ctx context.Context, d db.DB, opts Asso
if err != nil {
l.Err(err).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image")
}
} else if err != nil {
l.Debug().Msgf("createOrUpdateAlbumWithMbzReleaseID: failed to get album images for %s: %s", release.Title, err.Error())
}
}
if err != nil {
l.Debug().Msgf("createOrUpdateAlbumWithMbzReleaseID: failed to get album images for %s: %s", release.Title, err.Error())
}
album, err = d.SaveAlbum(ctx, db.SaveAlbumOpts{
Title: release.Title,
MusicBrainzID: opts.ReleaseMbzID,
@ -217,14 +216,15 @@ func matchAlbumByTitle(ctx context.Context, d db.DB, opts AssociateAlbumOpts) (*
return nil, fmt.Errorf("matchAlbumByTitle: %w", err)
} else {
var imgid uuid.UUID
imgUrl, err := images.GetAlbumImage(ctx, images.AlbumImageOpts{
Artists: utils.FlattenArtistNames(opts.Artists),
Album: opts.ReleaseName,
ReleaseMbzID: &opts.ReleaseMbzID,
})
if err == nil && imgUrl != "" {
imgid = uuid.New()
if !opts.SkipCacheImage {
var imgUrl string
if !opts.SkipCacheImage {
imgUrl, err = images.GetAlbumImage(ctx, images.AlbumImageOpts{
Artists: utils.FlattenArtistNames(opts.Artists),
Album: opts.ReleaseName,
ReleaseMbzID: &opts.ReleaseMbzID,
})
if err == nil && imgUrl != "" {
imgid = uuid.New()
var size ImageSize
if cfg.FullImageCacheEnabled() {
size = ImageSizeFull
@ -234,13 +234,12 @@ func matchAlbumByTitle(ctx context.Context, d db.DB, opts AssociateAlbumOpts) (*
l.Debug().Msg("Downloading album image from source...")
err = DownloadAndCacheImage(ctx, imgid, imgUrl, size)
if err != nil {
l.Err(err).Msg("createOrUpdateAlbumWithMbzReleaseID: failed to cache image")
l.Err(err).Msg("matchAlbumByTitle: failed to cache image")
}
} else if err != nil {
l.Debug().AnErr("error", err).Msgf("matchAlbumByTitle: failed to get album images for %s", opts.ReleaseName)
}
}
if err != nil {
l.Debug().AnErr("error", err).Msgf("matchAlbumByTitle: failed to get album images for %s", opts.ReleaseName)
}
a, err = d.SaveAlbum(ctx, db.SaveAlbumOpts{
Title: releaseName,

52
internal/catalog/associate_artists.go
View file

@ -127,12 +127,14 @@ func matchArtistsByMBIDMappings(ctx context.Context, d db.DB, opts AssociateArti
l.Warn().AnErr("error", err).Msg("matchArtistsByMBIDMappings: MusicBrainz unreachable, creating new artist with provided MusicBrainz ID mapping")
var imgid uuid.UUID
imgUrl, imgErr := images.GetArtistImage(ctx, images.ArtistImageOpts{
Aliases: []string{a.Artist},
})
if imgErr == nil && imgUrl != "" {
imgid = uuid.New()
if !opts.SkipCacheImage {
var imgUrl string
if !opts.SkipCacheImage {
var imgErr error
imgUrl, imgErr = images.GetArtistImage(ctx, images.ArtistImageOpts{
Aliases: []string{a.Artist},
})
if imgErr == nil && imgUrl != "" {
imgid = uuid.New()
var size ImageSize
if cfg.FullImageCacheEnabled() {
size = ImageSizeFull
@ -144,9 +146,9 @@ func matchArtistsByMBIDMappings(ctx context.Context, d db.DB, opts AssociateArti
if err != nil {
l.Err(err).Msg("Failed to cache image")
}
} else if imgErr != nil {
l.Err(imgErr).Msgf("matchArtistsByMBIDMappings: Failed to get artist image for artist '%s'", a.Artist)
}
} else {
l.Err(imgErr).Msgf("matchArtistsByMBIDMappings: Failed to get artist image for artist '%s'", a.Artist)
}
artist, err = d.SaveArtist(ctx, db.SaveArtistOpts{
@ -246,12 +248,13 @@ func resolveAliasOrCreateArtist(ctx context.Context, mbzID uuid.UUID, names []st
}
var imgid uuid.UUID
imgUrl, err := images.GetArtistImage(ctx, images.ArtistImageOpts{
Aliases: aliases,
})
if err == nil && imgUrl != "" {
imgid = uuid.New()
if !opts.SkipCacheImage {
var imgUrl string
if !opts.SkipCacheImage {
imgUrl, err = images.GetArtistImage(ctx, images.ArtistImageOpts{
Aliases: aliases,
})
if err == nil && imgUrl != "" {
imgid = uuid.New()
var size ImageSize
if cfg.FullImageCacheEnabled() {
size = ImageSizeFull
@ -263,9 +266,9 @@ func resolveAliasOrCreateArtist(ctx context.Context, mbzID uuid.UUID, names []st
if err != nil {
l.Err(err).Msg("Failed to cache image")
}
} else if err != nil {
l.Warn().AnErr("error", err).Msg("Failed to get artist image from ImageSrc")
}
} else if err != nil {
l.Warn().AnErr("error", err).Msg("Failed to get artist image from ImageSrc")
}
u, err := d.SaveArtist(ctx, db.SaveArtistOpts{
@ -301,12 +304,13 @@ func matchArtistsByNames(ctx context.Context, names []string, existing []*models
}
if errors.Is(err, pgx.ErrNoRows) {
var imgid uuid.UUID
imgUrl, err := images.GetArtistImage(ctx, images.ArtistImageOpts{
Aliases: []string{name},
})
if err == nil && imgUrl != "" {
imgid = uuid.New()
if !opts.SkipCacheImage {
var imgUrl string
if !opts.SkipCacheImage {
imgUrl, err = images.GetArtistImage(ctx, images.ArtistImageOpts{
Aliases: []string{name},
})
if err == nil && imgUrl != "" {
imgid = uuid.New()
var size ImageSize
if cfg.FullImageCacheEnabled() {
size = ImageSizeFull
@ -318,9 +322,9 @@ func matchArtistsByNames(ctx context.Context, names []string, existing []*models
if err != nil {
l.Err(err).Msg("Failed to cache image")
}
} else if err != nil {
l.Debug().AnErr("error", err).Msgf("Failed to get artist images for %s", name)
}
} else if err != nil {
l.Debug().AnErr("error", err).Msgf("Failed to get artist images for %s", name)
}
a, err = d.SaveArtist(ctx, db.SaveArtistOpts{Name: name, Image: imgid, ImageSrc: imgUrl})
if err != nil {

25
internal/catalog/catalog.go
View file

@ -77,6 +77,21 @@ func SubmitListen(ctx context.Context, store db.DB, opts SubmitListenOpts) error
// bandaid to ensure new activity does not have sub-second precision
opts.Time = opts.Time.Truncate(time.Second)
// Fast path: check lookup cache for known entity combo
if !opts.SkipSaveListen {
key := TrackLookupKey(opts.Artist, opts.TrackTitle, opts.ReleaseTitle)
cached, err := store.GetTrackLookup(ctx, key)
if err == nil && cached != nil {
l.Debug().Msg("Track lookup cache hit — skipping entity resolution")
return store.SaveListen(ctx, db.SaveListenOpts{
TrackID: cached.TrackID,
Time: opts.Time,
UserID: opts.UserID,
Client: opts.Client,
})
}
}
artists, err := AssociateArtists(
ctx,
store,
@ -168,6 +183,16 @@ func SubmitListen(ctx context.Context, store db.DB, opts SubmitListenOpts) error
}
}
// Populate lookup cache for future fast-path hits
if len(artists) > 0 {
store.SaveTrackLookup(ctx, db.SaveTrackLookupOpts{
Key: TrackLookupKey(opts.Artist, opts.TrackTitle, opts.ReleaseTitle),
ArtistID: artists[0].ID,
AlbumID: rg.ID,
TrackID: track.ID,
})
}
if opts.IsNowPlaying {
if track.Duration == 0 {
memkv.Store.Set(strconv.Itoa(int(opts.UserID)), track.ID)

9
internal/catalog/lookup_key.go Normal file
View file

@ -0,0 +1,9 @@
package catalog
import "strings"
// TrackLookupKey builds a normalized cache key for entity resolution.
// Uses null-byte separators to avoid collisions between field values.
func TrackLookupKey(artist, track, album string) string {
return strings.ToLower(artist) + "\x00" + strings.ToLower(track) + "\x00" + strings.ToLower(album)
}