feat: listenbrainz import

engine/engine.go

@@ -224,6 +224,12 @@ func RunImporter(l *zerolog.Logger, store db.DB, mbzc mbz.MusicBrainzCaller) {
 			if err != nil {
 				l.Err(err).Msgf("Failed to import file: %s", file.Name())
 			}
+		} else if strings.Contains(file.Name(), "listenbrainz") {
+			l.Info().Msgf("Import file %s detecting as being ListenBrainz export", file.Name())
+			err := importer.ImportListenBrainzExport(logger.NewContext(l), store, mbzc, file.Name())
+			if err != nil {
+				l.Err(err).Msgf("Failed to import file: %s", file.Name())
+			}
 		} else {
 			l.Warn().Msgf("File %s not recognized as a valid import file; make sure it is valid and named correctly", file.Name())
 		}

engine/import_test.go

@@ -79,12 +79,29 @@ func TestImportLastFM(t *testing.T) {
 
 	require.NoError(t, os.WriteFile(dest, input, os.ModePerm))
 
-	engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{})
+	mbzcMock := &mbz.MbzMockCaller{
+		Artists: map[uuid.UUID]*mbz.MusicBrainzArtist{
+			uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a"): &mbz.MusicBrainzArtist{
+				Name: "OurR",
+				Aliases: []mbz.MusicBrainzArtistAlias{
+					{
+						Name:    "OurR",
+						Primary: true,
+					},
+				},
+			},
+		},
+	}
+
+	engine.RunImporter(logger.Get(), store, mbzcMock)
 
 	album, err := store.GetAlbum(context.Background(), db.GetAlbumOpts{MusicBrainzID: uuid.MustParse("e9e78802-0bf8-4ca3-9655-1d943d2d2fa0")})
 	require.NoError(t, err)
 	assert.Equal(t, "ZOO!!", album.Title)
-	artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "CHUU"})
+	artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a")})
+	require.NoError(t, err)
+	assert.Equal(t, "OurR", artist.Name)
+	artist, err = store.GetArtist(context.Background(), db.GetArtistOpts{Name: "CHUU"})
 	require.NoError(t, err)
 	track, err := store.GetTrack(context.Background(), db.GetTrackOpts{Title: "because I'm stupid?", ArtistIDs: []int32{artist.ID}})
 	require.NoError(t, err)
@@ -93,3 +110,71 @@ func TestImportLastFM(t *testing.T) {
 	assert.Len(t, listens.Items, 1)
 	assert.WithinDuration(t, time.Unix(1749776100, 0), listens.Items[0].Time, 1*time.Second)
 }
+
+func TestImportListenBrainz(t *testing.T) {
+
+	src := "../static/listenbrainz_shoko1_1749780844.zip"
+	destDir := filepath.Join(cfg.ConfigDir(), "import")
+	dest := filepath.Join(destDir, "listenbrainz_shoko1_1749780844.zip")
+
+	// not going to make the dest dir because engine should make it already
+
+	input, err := os.ReadFile(src)
+	require.NoError(t, err)
+
+	require.NoError(t, os.WriteFile(dest, input, os.ModePerm))
+
+	mbzcMock := &mbz.MbzMockCaller{
+		Artists: map[uuid.UUID]*mbz.MusicBrainzArtist{
+			uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a"): {
+				Name: "OurR",
+				Aliases: []mbz.MusicBrainzArtistAlias{
+					{
+						Name:    "OurR",
+						Primary: true,
+					},
+				},
+			},
+			uuid.MustParse("09887aa7-226e-4ecc-9a0c-02d2ae5777e1"): {
+				Name: "Carly Rae Jepsen",
+				Aliases: []mbz.MusicBrainzArtistAlias{
+					{
+						Name:    "Carly Rae Jepsen",
+						Primary: true,
+					},
+				},
+			},
+			uuid.MustParse("78e46ae5-9bfd-433b-be3f-19e993d67ecc"): &mbz.MusicBrainzArtist{
+				Name: "Rufus Wainwright",
+				Aliases: []mbz.MusicBrainzArtistAlias{
+					{
+						Name:    "OurR",
+						Primary: true,
+					},
+				},
+			},
+		},
+	}
+
+	engine.RunImporter(logger.Get(), store, mbzcMock)
+
+	album, err := store.GetAlbum(context.Background(), db.GetAlbumOpts{MusicBrainzID: uuid.MustParse("ce330d67-9c46-4a3b-9d62-08406370f234")})
+	require.NoError(t, err)
+	assert.Equal(t, "酸欠少女", album.Title)
+	artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("4b00640f-3be6-43f8-9b34-ff81bd89320a")})
+	require.NoError(t, err)
+	assert.Equal(t, "OurR", artist.Name)
+	artist, err = store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("09887aa7-226e-4ecc-9a0c-02d2ae5777e1")})
+	require.NoError(t, err)
+	assert.Equal(t, "Carly Rae Jepsen", artist.Name)
+	artist, err = store.GetArtist(context.Background(), db.GetArtistOpts{MusicBrainzID: uuid.MustParse("78e46ae5-9bfd-433b-be3f-19e993d67ecc")})
+	require.NoError(t, err)
+	assert.Equal(t, "Rufus Wainwright", artist.Name)
+	track, err := store.GetTrack(context.Background(), db.GetTrackOpts{MusicBrainzID: uuid.MustParse("08e8f55b-f1a4-46b8-b2d1-fab4c592165c")})
+	require.NoError(t, err)
+	assert.Equal(t, "Desert", track.Title)
+	listens, err := store.GetListensPaginated(context.Background(), db.GetItemsOpts{TrackID: int(track.ID)})
+	require.NoError(t, err)
+	assert.Len(t, listens.Items, 1)
+	assert.WithinDuration(t, time.Unix(1749780612, 0), listens.Items[0].Time, 1*time.Second)
+}

internal/importer/importer.go

@@ -22,6 +22,8 @@ func finishImport(ctx context.Context, filename string, numImported int) error {
 	if err != nil {
 		l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
 	}
-	l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported)
+	if numImported != 0 {
+		l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported)
+	}
 	return nil
 }

internal/importer/lastfm.go

@@ -48,6 +48,7 @@ func ImportLastFMFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCall
 		l.Err(err).Msgf("Failed to read import file: %s", filename)
 		return err
 	}
+	defer file.Close()
 	var throttleFunc = func() {}
 	if ms := cfg.ThrottleImportMs(); ms > 0 {
 		throttleFunc = func() {

internal/importer/listenbrainz.go

@@ -0,0 +1,142 @@
+package importer
+
+import (
+	"archive/zip"
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/gabehf/koito/engine/handlers"
+	"github.com/gabehf/koito/internal/catalog"
+	"github.com/gabehf/koito/internal/cfg"
+	"github.com/gabehf/koito/internal/db"
+	"github.com/gabehf/koito/internal/logger"
+	"github.com/gabehf/koito/internal/mbz"
+	"github.com/gabehf/koito/internal/utils"
+	"github.com/google/uuid"
+)
+
+// first, unzip zip file with name "listenbrainz_username_unix.zip"
+// then, enter the listens folder
+// then, recursively traverse all files in folder
+// then, import all .jsonl files found in folders
+// then, cleanup folder recursively
+// finally, move zip to complete dir
+func ImportListenBrainzExport(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, filename string) error {
+	l := logger.FromContext(ctx)
+
+	r, err := zip.OpenReader(path.Join(path.Join(cfg.ConfigDir(), "import", filename)))
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+
+		if f.FileInfo().IsDir() {
+			continue
+		}
+
+		if strings.HasPrefix(f.Name, "listens/") && strings.HasSuffix(f.Name, ".jsonl") {
+			fmt.Println("Found:", f.Name)
+
+			rc, err := f.Open()
+			if err != nil {
+				log.Printf("Failed to open %s: %v\n", f.Name, err)
+				continue
+			}
+
+			err = ImportListenBrainzFile(ctx, store, mbzc, rc, f.Name)
+			if err != nil {
+				l.Err(err).Msgf("Failed to import listens from file: %s", f.Name)
+			}
+
+			rc.Close()
+		}
+	}
+	return finishImport(ctx, filename, 0)
+}
+
+func ImportListenBrainzFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, r io.Reader, filename string) error {
+	l := logger.FromContext(ctx)
+	l.Info().Msgf("Beginning ListenBrainz import on file: %s", filename)
+
+	scanner := bufio.NewScanner(r)
+
+	var throttleFunc = func() {}
+	if ms := cfg.ThrottleImportMs(); ms > 0 {
+		throttleFunc = func() {
+			time.Sleep(time.Duration(ms) * time.Millisecond)
+		}
+	}
+	count := 0
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		payload := new(handlers.LbzSubmitListenPayload)
+		err := json.Unmarshal(line, payload)
+		if err != nil {
+			fmt.Println("Error unmarshaling JSON:", err)
+			continue
+		}
+		artistMbzIDs, err := utils.ParseUUIDSlice(payload.TrackMeta.AdditionalInfo.ArtistMBIDs)
+		if err != nil {
+			l.Debug().Err(err).Msg("Failed to parse one or more uuids")
+		}
+		rgMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.ReleaseGroupMBID)
+		if err != nil {
+			rgMbzID = uuid.Nil
+		}
+		releaseMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.ReleaseMBID)
+		if err != nil {
+			releaseMbzID = uuid.Nil
+		}
+		recordingMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.RecordingMBID)
+		if err != nil {
+			recordingMbzID = uuid.Nil
+		}
+
+		var client string
+		if payload.TrackMeta.AdditionalInfo.MediaPlayer != "" {
+			client = payload.TrackMeta.AdditionalInfo.MediaPlayer
+		} else if payload.TrackMeta.AdditionalInfo.SubmissionClient != "" {
+			client = payload.TrackMeta.AdditionalInfo.SubmissionClient
+		}
+
+		var duration int32
+		if payload.TrackMeta.AdditionalInfo.Duration != 0 {
+			duration = payload.TrackMeta.AdditionalInfo.Duration
+		} else if payload.TrackMeta.AdditionalInfo.DurationMs != 0 {
+			duration = payload.TrackMeta.AdditionalInfo.DurationMs / 1000
+		}
+		opts := catalog.SubmitListenOpts{
+			MbzCaller:         mbzc,
+			ArtistNames:       payload.TrackMeta.AdditionalInfo.ArtistNames,
+			Artist:            payload.TrackMeta.ArtistName,
+			ArtistMbzIDs:      artistMbzIDs,
+			TrackTitle:        payload.TrackMeta.TrackName,
+			RecordingMbzID:    recordingMbzID,
+			ReleaseTitle:      payload.TrackMeta.ReleaseName,
+			ReleaseMbzID:      releaseMbzID,
+			ReleaseGroupMbzID: rgMbzID,
+			Duration:          duration,
+			Time:              time.Unix(payload.ListenedAt, 0),
+			UserID:            1,
+			Client:            client,
+		}
+		err = catalog.SubmitListen(ctx, store, opts)
+		if err != nil {
+			l.Err(err).Msg("Failed to import LastFM playback item")
+			return err
+		}
+		count++
+		throttleFunc()
+	}
+	l.Info().Msgf("Finished importing %s; imported %d items", filename, count)
+	return nil
+}

internal/importer/maloja.go

@@ -39,6 +39,7 @@ func ImportMalojaFile(ctx context.Context, store db.DB, filename string) error {
 		l.Err(err).Msgf("Failed to read import file: %s", filename)
 		return err
 	}
+	defer file.Close()
 	var throttleFunc = func() {}
 	if ms := cfg.ThrottleImportMs(); ms > 0 {
 		throttleFunc = func() {

internal/importer/spotify.go

@@ -31,6 +31,7 @@ func ImportSpotifyFile(ctx context.Context, store db.DB, filename string) error
 		l.Err(err).Msgf("Failed to read import file: %s", filename)
 		return err
 	}
+	defer file.Close()
 	var throttleFunc = func() {}
 	if ms := cfg.ThrottleImportMs(); ms > 0 {
 		throttleFunc = func() {

internal/mbz/mock.go

@@ -57,10 +57,14 @@ func (m *MbzMockCaller) GetTrack(ctx context.Context, id uuid.UUID) (*MusicBrain
 }
 
 func (m *MbzMockCaller) GetArtistPrimaryAliases(ctx context.Context, id uuid.UUID) ([]string, error) {
-	name := m.Artists[id].Name
-	ss := make([]string, len(m.Artists[id].Aliases)+1)
+	artist, exists := m.Artists[id]
+	if !exists {
+		return nil, fmt.Errorf("artist with ID %s not found", id)
+	}
+	name := artist.Name
+	ss := make([]string, len(artist.Aliases)+1)
 	ss[0] = name
-	for i, alias := range m.Artists[id].Aliases {
+	for i, alias := range artist.Aliases {
 		ss[i+1] = alias.Name
 	}
 	return ss, nil