mirror of
https://github.com/gabehf/Koito.git
synced 2026-04-22 12:01:52 -07:00
8ce6ec494d
Adopts ListenBrainz-inspired patterns to speed up imports from ~24h to under 30 minutes for 49k scrobbles. Phase 1 - track_lookup cache table: - New migration (000006) adds persistent entity lookup cache - Maps normalized (artist, track, album) → (artist_id, album_id, track_id) - SubmitListen fast path: cache hit skips 18 DB queries → 2 queries - Cache populated after entity resolution, invalidated on merge/delete - Benefits both live scrobbles and imports Phase 2 - SaveListensBatch: - New batch listen insert using pgx CopyFrom → temp table → INSERT ON CONFLICT - Thousands of inserts per second vs one-at-a-time Phase 3 - BulkSubmitter: - Reusable import accelerator for all importers - Pre-deduplicates scrobbles by (artist, track, album) in memory - Worker pool (4 goroutines) for parallel entity creation on cache miss - Batch listen insertion via SaveListensBatch Phase 4 - Migrate importers: - Maloja, Spotify, LastFM, ListenBrainz importers use BulkSubmitter - Koito importer left as-is (already fast with pre-resolved IDs) Phase 5 - Skip image lookups during import: - GetArtistImage/GetAlbumImage calls fully skipped when SkipCacheImage=true - Background tasks (FetchMissingArtistImages/FetchMissingAlbumImages) backfill Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
181 lines
2.9 KiB
Go
181 lines
2.9 KiB
Go
package db
|
|
|
|
import (
|
|
"time"
|
|
|
|
"github.com/gabehf/koito/internal/models"
|
|
"github.com/google/uuid"
|
|
)
|
|
|
|
type GetAlbumOpts struct {
|
|
ID int32
|
|
MusicBrainzID uuid.UUID
|
|
ArtistID int32
|
|
Title string
|
|
Titles []string
|
|
Image uuid.UUID
|
|
}
|
|
|
|
type GetArtistOpts struct {
|
|
ID int32
|
|
MusicBrainzID uuid.UUID
|
|
Name string
|
|
Image uuid.UUID
|
|
}
|
|
|
|
type GetTrackOpts struct {
|
|
ID int32
|
|
MusicBrainzID uuid.UUID
|
|
Title string
|
|
ReleaseID int32
|
|
ArtistIDs []int32
|
|
}
|
|
|
|
type SaveTrackOpts struct {
|
|
Title string
|
|
AlbumID int32
|
|
ArtistIDs []int32
|
|
RecordingMbzID uuid.UUID
|
|
Duration int32
|
|
}
|
|
|
|
type SaveAlbumOpts struct {
|
|
Title string
|
|
MusicBrainzID uuid.UUID
|
|
Type string
|
|
ArtistIDs []int32
|
|
VariousArtists bool
|
|
Image uuid.UUID
|
|
ImageSrc string
|
|
Aliases []string
|
|
}
|
|
|
|
type SaveArtistOpts struct {
|
|
Name string
|
|
MusicBrainzID uuid.UUID
|
|
Aliases []string
|
|
Image uuid.UUID
|
|
ImageSrc string
|
|
}
|
|
|
|
type UpdateApiKeyLabelOpts struct {
|
|
UserID int32
|
|
ID int32
|
|
Label string
|
|
}
|
|
|
|
type SaveUserOpts struct {
|
|
Username string
|
|
Password string
|
|
Role models.UserRole
|
|
}
|
|
|
|
type SaveApiKeyOpts struct {
|
|
Key string
|
|
UserID int32
|
|
Label string
|
|
}
|
|
|
|
type SaveListenOpts struct {
|
|
TrackID int32
|
|
Time time.Time
|
|
UserID int32
|
|
Client string
|
|
}
|
|
|
|
type UpdateTrackOpts struct {
|
|
ID int32
|
|
MusicBrainzID uuid.UUID
|
|
Duration int32
|
|
}
|
|
|
|
type UpdateArtistOpts struct {
|
|
ID int32
|
|
MusicBrainzID uuid.UUID
|
|
Image uuid.UUID
|
|
ImageSrc string
|
|
}
|
|
|
|
type UpdateAlbumOpts struct {
|
|
ID int32
|
|
MusicBrainzID uuid.UUID
|
|
Image uuid.UUID
|
|
ImageSrc string
|
|
VariousArtistsUpdate bool
|
|
VariousArtistsValue bool
|
|
}
|
|
|
|
type UpdateUserOpts struct {
|
|
ID int32
|
|
Username string
|
|
Password string
|
|
}
|
|
|
|
type AddArtistsToAlbumOpts struct {
|
|
AlbumID int32
|
|
ArtistIDs []int32
|
|
}
|
|
|
|
type GetItemsOpts struct {
|
|
Limit int
|
|
Page int
|
|
Timeframe Timeframe
|
|
|
|
// Used only for getting top tracks
|
|
ArtistID int
|
|
AlbumID int
|
|
|
|
// Used for getting listens
|
|
TrackID int
|
|
}
|
|
|
|
type ListenActivityOpts struct {
|
|
Step StepInterval
|
|
Range int
|
|
Month int
|
|
Year int
|
|
Timezone *time.Location
|
|
AlbumID int32
|
|
ArtistID int32
|
|
TrackID int32
|
|
}
|
|
|
|
type TimeListenedOpts struct {
|
|
Timeframe Timeframe
|
|
AlbumID int32
|
|
ArtistID int32
|
|
TrackID int32
|
|
}
|
|
|
|
type GetExportPageOpts struct {
|
|
UserID int32
|
|
ListenedAt time.Time
|
|
TrackID int32
|
|
Limit int32
|
|
}
|
|
|
|
type GetInterestOpts struct {
|
|
Buckets int
|
|
AlbumID int32
|
|
ArtistID int32
|
|
TrackID int32
|
|
}
|
|
|
|
type TrackLookupResult struct {
|
|
ArtistID int32
|
|
AlbumID int32
|
|
TrackID int32
|
|
}
|
|
|
|
type SaveTrackLookupOpts struct {
|
|
Key string
|
|
ArtistID int32
|
|
AlbumID int32
|
|
TrackID int32
|
|
}
|
|
|
|
type InvalidateTrackLookupOpts struct {
|
|
ArtistID int32
|
|
AlbumID int32
|
|
TrackID int32
|
|
}
|