feat: listenbrainz import

internal/importer/importer.go

@@ -22,6 +22,8 @@ func finishImport(ctx context.Context, filename string, numImported int) error {
 	if err != nil {
 		l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
 	}
-	l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported)
+	if numImported != 0 {
+		l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported)
+	}
 	return nil
 }

internal/importer/lastfm.go

@@ -48,6 +48,7 @@ func ImportLastFMFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCall
 		l.Err(err).Msgf("Failed to read import file: %s", filename)
 		return err
 	}
+	defer file.Close()
 	var throttleFunc = func() {}
 	if ms := cfg.ThrottleImportMs(); ms > 0 {
 		throttleFunc = func() {

internal/importer/listenbrainz.go

@@ -0,0 +1,142 @@
+package importer
+
+import (
+	"archive/zip"
+	"bufio"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"log"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/gabehf/koito/engine/handlers"
+	"github.com/gabehf/koito/internal/catalog"
+	"github.com/gabehf/koito/internal/cfg"
+	"github.com/gabehf/koito/internal/db"
+	"github.com/gabehf/koito/internal/logger"
+	"github.com/gabehf/koito/internal/mbz"
+	"github.com/gabehf/koito/internal/utils"
+	"github.com/google/uuid"
+)
+
+// first, unzip zip file with name "listenbrainz_username_unix.zip"
+// then, enter the listens folder
+// then, recursively traverse all files in folder
+// then, import all .jsonl files found in folders
+// then, cleanup folder recursively
+// finally, move zip to complete dir
+func ImportListenBrainzExport(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, filename string) error {
+	l := logger.FromContext(ctx)
+
+	r, err := zip.OpenReader(path.Join(path.Join(cfg.ConfigDir(), "import", filename)))
+	if err != nil {
+		return err
+	}
+	defer r.Close()
+
+	for _, f := range r.File {
+
+		if f.FileInfo().IsDir() {
+			continue
+		}
+
+		if strings.HasPrefix(f.Name, "listens/") && strings.HasSuffix(f.Name, ".jsonl") {
+			fmt.Println("Found:", f.Name)
+
+			rc, err := f.Open()
+			if err != nil {
+				log.Printf("Failed to open %s: %v\n", f.Name, err)
+				continue
+			}
+
+			err = ImportListenBrainzFile(ctx, store, mbzc, rc, f.Name)
+			if err != nil {
+				l.Err(err).Msgf("Failed to import listens from file: %s", f.Name)
+			}
+
+			rc.Close()
+		}
+	}
+	return finishImport(ctx, filename, 0)
+}
+
+func ImportListenBrainzFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, r io.Reader, filename string) error {
+	l := logger.FromContext(ctx)
+	l.Info().Msgf("Beginning ListenBrainz import on file: %s", filename)
+
+	scanner := bufio.NewScanner(r)
+
+	var throttleFunc = func() {}
+	if ms := cfg.ThrottleImportMs(); ms > 0 {
+		throttleFunc = func() {
+			time.Sleep(time.Duration(ms) * time.Millisecond)
+		}
+	}
+	count := 0
+	for scanner.Scan() {
+		line := scanner.Bytes()
+		payload := new(handlers.LbzSubmitListenPayload)
+		err := json.Unmarshal(line, payload)
+		if err != nil {
+			fmt.Println("Error unmarshaling JSON:", err)
+			continue
+		}
+		artistMbzIDs, err := utils.ParseUUIDSlice(payload.TrackMeta.AdditionalInfo.ArtistMBIDs)
+		if err != nil {
+			l.Debug().Err(err).Msg("Failed to parse one or more uuids")
+		}
+		rgMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.ReleaseGroupMBID)
+		if err != nil {
+			rgMbzID = uuid.Nil
+		}
+		releaseMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.ReleaseMBID)
+		if err != nil {
+			releaseMbzID = uuid.Nil
+		}
+		recordingMbzID, err := uuid.Parse(payload.TrackMeta.AdditionalInfo.RecordingMBID)
+		if err != nil {
+			recordingMbzID = uuid.Nil
+		}
+
+		var client string
+		if payload.TrackMeta.AdditionalInfo.MediaPlayer != "" {
+			client = payload.TrackMeta.AdditionalInfo.MediaPlayer
+		} else if payload.TrackMeta.AdditionalInfo.SubmissionClient != "" {
+			client = payload.TrackMeta.AdditionalInfo.SubmissionClient
+		}
+
+		var duration int32
+		if payload.TrackMeta.AdditionalInfo.Duration != 0 {
+			duration = payload.TrackMeta.AdditionalInfo.Duration
+		} else if payload.TrackMeta.AdditionalInfo.DurationMs != 0 {
+			duration = payload.TrackMeta.AdditionalInfo.DurationMs / 1000
+		}
+		opts := catalog.SubmitListenOpts{
+			MbzCaller:         mbzc,
+			ArtistNames:       payload.TrackMeta.AdditionalInfo.ArtistNames,
+			Artist:            payload.TrackMeta.ArtistName,
+			ArtistMbzIDs:      artistMbzIDs,
+			TrackTitle:        payload.TrackMeta.TrackName,
+			RecordingMbzID:    recordingMbzID,
+			ReleaseTitle:      payload.TrackMeta.ReleaseName,
+			ReleaseMbzID:      releaseMbzID,
+			ReleaseGroupMbzID: rgMbzID,
+			Duration:          duration,
+			Time:              time.Unix(payload.ListenedAt, 0),
+			UserID:            1,
+			Client:            client,
+		}
+		err = catalog.SubmitListen(ctx, store, opts)
+		if err != nil {
+			l.Err(err).Msg("Failed to import LastFM playback item")
+			return err
+		}
+		count++
+		throttleFunc()
+	}
+	l.Info().Msgf("Finished importing %s; imported %d items", filename, count)
+	return nil
+}

internal/importer/maloja.go

@@ -39,6 +39,7 @@ func ImportMalojaFile(ctx context.Context, store db.DB, filename string) error {
 		l.Err(err).Msgf("Failed to read import file: %s", filename)
 		return err
 	}
+	defer file.Close()
 	var throttleFunc = func() {}
 	if ms := cfg.ThrottleImportMs(); ms > 0 {
 		throttleFunc = func() {

internal/importer/spotify.go

@@ -31,6 +31,7 @@ func ImportSpotifyFile(ctx context.Context, store db.DB, filename string) error
 		l.Err(err).Msgf("Failed to read import file: %s", filename)
 		return err
 	}
+	defer file.Close()
 	var throttleFunc = func() {}
 	if ms := cfg.ThrottleImportMs(); ms > 0 {
 		throttleFunc = func() {

internal/mbz/mock.go

@@ -57,10 +57,14 @@ func (m *MbzMockCaller) GetTrack(ctx context.Context, id uuid.UUID) (*MusicBrain
 }
 
 func (m *MbzMockCaller) GetArtistPrimaryAliases(ctx context.Context, id uuid.UUID) ([]string, error) {
-	name := m.Artists[id].Name
-	ss := make([]string, len(m.Artists[id].Aliases)+1)
+	artist, exists := m.Artists[id]
+	if !exists {
+		return nil, fmt.Errorf("artist with ID %s not found", id)
+	}
+	name := artist.Name
+	ss := make([]string, len(artist.Aliases)+1)
 	ss[0] = name
-	for i, alias := range m.Artists[id].Aliases {
+	for i, alias := range artist.Aliases {
 		ss[i+1] = alias.Name
 	}
 	return ss, nil