diff --git a/engine/engine.go b/engine/engine.go index fb518f2..f53355a 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -224,6 +224,12 @@ func RunImporter(l *zerolog.Logger, store db.DB, mbzc mbz.MusicBrainzCaller) { if err != nil { l.Err(err).Msgf("Failed to import file: %s", file.Name()) } + } else if strings.Contains(file.Name(), "listenbrainz") { + l.Info().Msgf("Import file %s detecting as being ListenBrainz export", file.Name()) + err := importer.ImportListenBrainzExport(logger.NewContext(l), store, mbzc, file.Name()) + if err != nil { + l.Err(err).Msgf("Failed to import file: %s", file.Name()) + } } else { l.Warn().Msgf("File %s not recognized as a valid import file; make sure it is valid and named correctly", file.Name()) } diff --git a/engine/import_test.go b/engine/import_test.go index 7dff162..d0447dc 100644 --- a/engine/import_test.go +++ b/engine/import_test.go @@ -79,12 +79,29 @@ func TestImportLastFM(t *testing.T) { require.NoError(t, os.WriteFile(dest, input, os.ModePerm)) - engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{}) + mbzcMock := &mbz.MbzMockCaller{ + Artists: map[uuid.UUID]*mbz.MusicBrainzArtist{ + uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a"): &mbz.MusicBrainzArtist{ + Name: "OurR", + Aliases: []mbz.MusicBrainzArtistAlias{ + { + Name: "OurR", + Primary: true, + }, + }, + }, + }, + } + + engine.RunImporter(logger.Get(), store, mbzcMock) album, err := store.GetAlbum(context.Background(), db.GetAlbumOpts{MusicBrainzID: uuid.MustParse("e9e78802-0bf8-4ca3-9655-1d943d2d2fa0")}) require.NoError(t, err) assert.Equal(t, "ZOO!!", album.Title) - artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "CHUU"}) + artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a")}) + require.NoError(t, err) + assert.Equal(t, "OurR", artist.Name) + artist, err = store.GetArtist(context.Background(), db.GetArtistOpts{Name: "CHUU"}) require.NoError(t, err) track, err := store.GetTrack(context.Background(), db.GetTrackOpts{Title: "because I'm stupid?", ArtistIDs: []int32{artist.ID}}) require.NoError(t, err) @@ -93,3 +110,71 @@ func TestImportLastFM(t *testing.T) { assert.Len(t, listens.Items, 1) assert.WithinDuration(t, time.Unix(1749776100, 0), listens.Items[0].Time, 1*time.Second) } + +func TestImportListenBrainz(t *testing.T) { + + src := "../static/listenbrainz_shoko1_1749780844.zip" + destDir := filepath.Join(cfg.ConfigDir(), "import") + dest := filepath.Join(destDir, "listenbrainz_shoko1_1749780844.zip") + + // not going to make the dest dir because engine should make it already + + input, err := os.ReadFile(src) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(dest, input, os.ModePerm)) + + mbzcMock := &mbz.MbzMockCaller{ + Artists: map[uuid.UUID]*mbz.MusicBrainzArtist{ + uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a"): { + Name: "OurR", + Aliases: []mbz.MusicBrainzArtistAlias{ + { + Name: "OurR", + Primary: true, + }, + }, + }, + uuid.MustParse("09887aa7-226e-4ecc-9a0c-02d2ae5777e1"): { + Name: "Carly Rae Jepsen", + Aliases: []mbz.MusicBrainzArtistAlias{ + { + Name: "Carly Rae Jepsen", + Primary: true, + }, + }, + }, + uuid.MustParse("78e46ae5-9bfd-433b-be3f-19e993d67ecc"): &mbz.MusicBrainzArtist{ + Name: "Rufus Wainwright", + Aliases: []mbz.MusicBrainzArtistAlias{ + { + Name: "OurR", + Primary: true, + }, + }, + }, + }, + } + + engine.RunImporter(logger.Get(), store, mbzcMock) + + album, err := store.GetAlbum(context.Background(), db.GetAlbumOpts{MusicBrainzID: uuid.MustParse("ce330d67-9c46-4a3b-9d62-08406370f234")}) + require.NoError(t, err) + assert.Equal(t, "酸欠少女", album.Title) + artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a")}) + require.NoError(t, err) + assert.Equal(t, "OurR", artist.Name) + artist, err = store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("09887aa7-226e-4ecc-9a0c-02d2ae5777e1")}) + require.NoError(t, err) + assert.Equal(t, "Carly Rae Jepsen", artist.Name) + artist, err = store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("78e46ae5-9bfd-433b-be3f-19e993d67ecc")}) + require.NoError(t, err) + assert.Equal(t, "Rufus Wainwright", artist.Name) + track, err := store.GetTrack(context.Background(), db.GetTrackOpts{MusicBrainzID: uuid.MustParse("08e8f55b-f1a4-46b8-b2d1-fab4c592165c")}) + require.NoError(t, err) + assert.Equal(t, "Desert", track.Title) + listens, err := store.GetListensPaginated(context.Background(), db.GetItemsOpts{TrackID: int(track.ID)}) + require.NoError(t, err) + assert.Len(t, listens.Items, 1) + assert.WithinDuration(t, time.Unix(1749780612, 0), listens.Items[0].Time, 1*time.Second) +} diff --git a/internal/importer/importer.go b/internal/importer/importer.go index 1e7025d..306f7a6 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -22,6 +22,8 @@ func finishImport(ctx context.Context, filename string, numImported int) error { if err != nil { l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start") } - l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported) + if numImported != 0 { + l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported) + } return nil } diff --git a/internal/importer/lastfm.go b/internal/importer/lastfm.go index 07a412a..2e457cc 100644 --- a/internal/importer/lastfm.go +++ b/internal/importer/lastfm.go @@ -48,6 +48,7 @@ func ImportLastFMFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCall l.Err(err).Msgf("Failed to read import file: %s", filename) return err } + defer file.Close() var throttleFunc = func() {} if ms := cfg.ThrottleImportMs(); ms > 0 { throttleFunc = func() { diff --git a/internal/importer/listenbrainz.go b/internal/importer/listenbrainz.go new file mode 100644 index 0000000..7e0df1a --- /dev/null +++ b/internal/importer/listenbrainz.go @@ -0,0 +1,142 @@ +package importer + +import ( + "archive/zip" + "bufio" + "context" + "encoding/json" + "fmt" + "io" + "log" + "path" + "strings" + "time" + + "github.com/gabehf/koito/engine/handlers" + "github.com/gabehf/koito/internal/catalog" + "github.com/gabehf/koito/internal/cfg" + "github.com/gabehf/koito/internal/db" + "github.com/gabehf/koito/internal/logger" + "github.com/gabehf/koito/internal/mbz" + "github.com/gabehf/koito/internal/utils" + "github.com/google/uuid" +) + +// first, unzip zip file with name "listenbrainz_username_unix.zip" +// then, enter the listens folder +// then, recursively traverse all files in folder +// then, import all .jsonl files found in folders +// then, cleanup folder recursively +// finally, move zip to complete dir +func ImportListenBrainzExport(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, filename string) error { + l := logger.FromContext(ctx) + + r, err := zip.OpenReader(path.Join(path.Join(cfg.ConfigDir(), "import", filename))) + if err != nil { + return err + } + defer r.Close() + + for _, f := range r.File { + + if f.FileInfo().IsDir() { + continue + } + + if strings.HasPrefix(f.Name, "listens/") && strings.HasSuffix(f.Name, ".jsonl") { + fmt.Println("Found:", f.Name) + + rc, err := f.Open() + if err != nil { + log.Printf("Failed to open %s: %v\n", f.Name, err) + continue + } + + err = ImportListenBrainzFile(ctx, store, mbzc, rc, f.Name) + if err != nil { + l.Err(err).Msgf("Failed to import listens from file: %s", f.Name) + } + + rc.Close() + } + } + return finishImport(ctx, filename, 0) +} + +func ImportListenBrainzFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, r io.Reader, filename string) error { + l := logger.FromContext(ctx) + l.Info().Msgf("Beginning ListenBrainz import on file: %s", filename) + + scanner := bufio.NewScanner(r) + + var throttleFunc = func() {} + if ms := cfg.ThrottleImportMs(); ms > 0 { + throttleFunc = func() { + time.Sleep(time.Duration(ms) * time.Millisecond) + } + } + count := 0 + for scanner.Scan() { + line := scanner.Bytes() + payload := new(handlers.LbzSubmitListenPayload) + err := json.Unmarshal(line, payload) + if err != nil { + fmt.Println("Error unmarshaling JSON:", err) + continue + } + artistMbzIDs, err := utils.ParseUUIDSlice(payload.TrackMeta.AdditionalInfo.ArtistMBIDs) + if err != nil { + l.Debug().Err(err).Msg("Failed to parse one or more uuids") + } + rgMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.ReleaseGroupMBID) + if err != nil { + rgMbzID = uuid.Nil + } + releaseMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.ReleaseMBID) + if err != nil { + releaseMbzID = uuid.Nil + } + recordingMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.RecordingMBID) + if err != nil { + recordingMbzID = uuid.Nil + } + + var client string + if payload.TrackMeta.AdditionalInfo.MediaPlayer != "" { + client = payload.TrackMeta.AdditionalInfo.MediaPlayer + } else if payload.TrackMeta.AdditionalInfo.SubmissionClient != "" { + client = payload.TrackMeta.AdditionalInfo.SubmissionClient + } + + var duration int32 + if payload.TrackMeta.AdditionalInfo.Duration != 0 { + duration = payload.TrackMeta.AdditionalInfo.Duration + } else if payload.TrackMeta.AdditionalInfo.DurationMs != 0 { + duration = payload.TrackMeta.AdditionalInfo.DurationMs / 1000 + } + opts := catalog.SubmitListenOpts{ + MbzCaller: mbzc, + ArtistNames: payload.TrackMeta.AdditionalInfo.ArtistNames, + Artist: payload.TrackMeta.ArtistName, + ArtistMbzIDs: artistMbzIDs, + TrackTitle: payload.TrackMeta.TrackName, + RecordingMbzID: recordingMbzID, + ReleaseTitle: payload.TrackMeta.ReleaseName, + ReleaseMbzID: releaseMbzID, + ReleaseGroupMbzID: rgMbzID, + Duration: duration, + Time: time.Unix(payload.ListenedAt, 0), + UserID: 1, + Client: client, + } + err = catalog.SubmitListen(ctx, store, opts) + if err != nil { + l.Err(err).Msg("Failed to import LastFM playback item") + return err + } + count++ + throttleFunc() + } + l.Info().Msgf("Finished importing %s; imported %d items", filename, count) + return nil +} diff --git a/internal/importer/maloja.go b/internal/importer/maloja.go index fdd647b..f14eeb5 100644 --- a/internal/importer/maloja.go +++ b/internal/importer/maloja.go @@ -39,6 +39,7 @@ func ImportMalojaFile(ctx context.Context, store db.DB, filename string) error { l.Err(err).Msgf("Failed to read import file: %s", filename) return err } + defer file.Close() var throttleFunc = func() {} if ms := cfg.ThrottleImportMs(); ms > 0 { throttleFunc = func() { diff --git a/internal/importer/spotify.go b/internal/importer/spotify.go index 0f338ff..25f24d0 100644 --- a/internal/importer/spotify.go +++ b/internal/importer/spotify.go @@ -31,6 +31,7 @@ func ImportSpotifyFile(ctx context.Context, store db.DB, filename string) error l.Err(err).Msgf("Failed to read import file: %s", filename) return err } + defer file.Close() var throttleFunc = func() {} if ms := cfg.ThrottleImportMs(); ms > 0 { throttleFunc = func() { diff --git a/internal/mbz/mock.go b/internal/mbz/mock.go index b70e237..2b5d7d8 100644 --- a/internal/mbz/mock.go +++ b/internal/mbz/mock.go @@ -57,10 +57,14 @@ func (m *MbzMockCaller) GetTrack(ctx context.Context, id uuid.UUID) (*MusicBrain } func (m *MbzMockCaller) GetArtistPrimaryAliases(ctx context.Context, id uuid.UUID) ([]string, error) { - name := m.Artists[id].Name - ss := make([]string, len(m.Artists[id].Aliases)+1) + artist, exists := m.Artists[id] + if !exists { + return nil, fmt.Errorf("artist with ID %s not found", id) + } + name := artist.Name + ss := make([]string, len(artist.Aliases)+1) ss[0] = name - for i, alias := range m.Artists[id].Aliases { + for i, alias := range artist.Aliases { ss[i+1] = alias.Name } return ss, nil diff --git a/static/listenbrainz_shoko1_1749780844.zip b/static/listenbrainz_shoko1_1749780844.zip new file mode 100644 index 0000000..be96a22 Binary files /dev/null and b/static/listenbrainz_shoko1_1749780844.zip differ