feat: add last fm importer

engine/engine.go

@@ -163,7 +163,7 @@ func Run(
 	// Import
 	if !cfg.SkipImport() {
 		go func() {
-			RunImporter(l, store)
+			RunImporter(l, store, mbzC)
 		}()
 	}
 
@@ -186,7 +186,7 @@ func Run(
 	return nil
 }
 
-func RunImporter(l *zerolog.Logger, store db.DB) {
+func RunImporter(l *zerolog.Logger, store db.DB, mbzc mbz.MusicBrainzCaller) {
 	l.Debug().Msg("Checking for import files...")
 	files, err := os.ReadDir(path.Join(cfg.ConfigDir(), "import"))
 	if err != nil {
@@ -218,6 +218,12 @@ func RunImporter(l *zerolog.Logger, store db.DB) {
 			if err != nil {
 				l.Err(err).Msgf("Failed to import file: %s", file.Name())
 			}
+		} else if strings.Contains(file.Name(), "recenttracks") {
+			l.Info().Msgf("Import file %s detecting as being ghan.nl LastFM export", file.Name())
+			err := importer.ImportLastFMFile(logger.NewContext(l), store, mbzc, file.Name())
+			if err != nil {
+				l.Err(err).Msgf("Failed to import file: %s", file.Name())
+			}
 		} else {
 			l.Warn().Msgf("File %s not recognized as a valid import file; make sure it is valid and named correctly", file.Name())
 		}

engine/import_test.go

@@ -5,11 +5,14 @@ import (
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
 	"github.com/gabehf/koito/engine"
 	"github.com/gabehf/koito/internal/cfg"
 	"github.com/gabehf/koito/internal/db"
 	"github.com/gabehf/koito/internal/logger"
+	"github.com/gabehf/koito/internal/mbz"
+	"github.com/google/uuid"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 )
@@ -27,7 +30,7 @@ func TestImportMaloja(t *testing.T) {
 
 	require.NoError(t, os.WriteFile(dest, input, os.ModePerm))
 
-	engine.RunImporter(logger.Get(), store)
+	engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{})
 
 	// maloja test import is 38 Magnify Tokyo streams
 	a, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "Magnify Tokyo"})
@@ -50,7 +53,7 @@ func TestImportSpotify(t *testing.T) {
 
 	require.NoError(t, os.WriteFile(dest, input, os.ModePerm))
 
-	engine.RunImporter(logger.Get(), store)
+	engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{})
 
 	a, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "The Story So Far"})
 	require.NoError(t, err)
@@ -62,3 +65,31 @@ func TestImportSpotify(t *testing.T) {
 	// this is the only track with valid duration data
 	assert.EqualValues(t, 181, track.Duration)
 }
+
+func TestImportLastFM(t *testing.T) {
+
+	src := "../static/recenttracks-shoko2-1749776100.json"
+	destDir := filepath.Join(cfg.ConfigDir(), "import")
+	dest := filepath.Join(destDir, "recenttracks-shoko2-1749776100.json")
+
+	// not going to make the dest dir because engine should make it already
+
+	input, err := os.ReadFile(src)
+	require.NoError(t, err)
+
+	require.NoError(t, os.WriteFile(dest, input, os.ModePerm))
+
+	engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{})
+
+	album, err := store.GetAlbum(context.Background(), db.GetAlbumOpts{MusicBrainzID: uuid.MustParse("e9e78802-0bf8-4ca3-9655-1d943d2d2fa0")})
+	require.NoError(t, err)
+	assert.Equal(t, "ZOO!!", album.Title)
+	artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "CHUU"})
+	require.NoError(t, err)
+	track, err := store.GetTrack(context.Background(), db.GetTrackOpts{Title: "because I'm stupid?", ArtistIDs: []int32{artist.ID}})
+	require.NoError(t, err)
+	listens, err := store.GetListensPaginated(context.Background(), db.GetItemsOpts{TrackID: int(track.ID)})
+	require.NoError(t, err)
+	assert.Len(t, listens.Items, 1)
+	assert.WithinDuration(t, time.Unix(1749776100, 0), listens.Items[0].Time, 1*time.Second)
+}

internal/importer/importer.go

@@ -1 +1,27 @@
 package importer
+
+import (
+	"context"
+	"os"
+	"path"
+
+	"github.com/gabehf/koito/internal/cfg"
+	"github.com/gabehf/koito/internal/logger"
+)
+
+func finishImport(ctx context.Context, filename string, numImported int) error {
+	l := logger.FromContext(ctx)
+	_, err := os.Stat(path.Join(cfg.ConfigDir(), "import_complete"))
+	if err != nil {
+		err = os.Mkdir(path.Join(cfg.ConfigDir(), "import_complete"), 0744)
+		if err != nil {
+			l.Err(err).Msg("Failed to create import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
+		}
+	}
+	err = os.Rename(path.Join(cfg.ConfigDir(), "import", filename), path.Join(cfg.ConfigDir(), "import_complete", filename))
+	if err != nil {
+		l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
+	}
+	l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported)
+	return nil
+}

internal/importer/lastfm.go

@@ -0,0 +1,116 @@
+package importer
+
+import (
+	"context"
+	"encoding/json"
+	"os"
+	"path"
+	"strconv"
+	"time"
+
+	"github.com/gabehf/koito/internal/catalog"
+	"github.com/gabehf/koito/internal/cfg"
+	"github.com/gabehf/koito/internal/db"
+	"github.com/gabehf/koito/internal/logger"
+	"github.com/gabehf/koito/internal/mbz"
+	"github.com/google/uuid"
+)
+
+type LastFMExportPage struct {
+	Track []LastFMTrack `json:"track"`
+}
+type LastFMTrack struct {
+	Artist LastFMItem    `json:"artist"`
+	Images []LastFMImage `json:"image"`
+	MBID   string        `json:"mbid"`
+	Album  LastFMItem    `json:"album"`
+	Name   string        `json:"name"`
+	Date   LastFMDate    `json:"date"`
+}
+type LastFMItem struct {
+	MBID string `json:"mbid"`
+	Text string `json:"#text"`
+}
+type LastFMDate struct {
+	Unix string `json:"uts"`
+	Text string `json:"#text"`
+}
+type LastFMImage struct {
+	Size string `json:"size"`
+	Url  string `json:"#text"`
+}
+
+func ImportLastFMFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, filename string) error {
+	l := logger.FromContext(ctx)
+	l.Info().Msgf("Beginning LastFM import on file: %s", filename)
+	file, err := os.Open(path.Join(cfg.ConfigDir(), "import", filename))
+	if err != nil {
+		l.Err(err).Msgf("Failed to read import file: %s", filename)
+		return err
+	}
+	var throttleFunc = func() {}
+	if ms := cfg.ThrottleImportMs(); ms > 0 {
+		throttleFunc = func() {
+			time.Sleep(time.Duration(ms) * time.Millisecond)
+		}
+	}
+	export := make([]LastFMExportPage, 0)
+	err = json.NewDecoder(file).Decode(&export)
+	if err != nil {
+		return err
+	}
+	count := 0
+	for _, item := range export {
+		for _, track := range item.Track {
+			album := track.Album.Text
+			if album == "" {
+				album = track.Name
+			}
+			if track.Name == "" || track.Artist.Text == "" {
+				l.Debug().Msg("Skipping invalid LastFM import item")
+				continue
+			}
+			albumMbzID, err := uuid.Parse(track.Album.MBID)
+			if err != nil {
+				albumMbzID = uuid.Nil
+			}
+			artistMbzID, err := uuid.Parse(track.Artist.MBID)
+			if err != nil {
+				artistMbzID = uuid.Nil
+			}
+			trackMbzID, err := uuid.Parse(track.MBID)
+			if err != nil {
+				trackMbzID = uuid.Nil
+			}
+			var ts time.Time
+			unix, err := strconv.ParseInt(track.Date.Unix, 10, 64)
+			if err != nil {
+				ts, err = time.Parse("02 Jan 2006, 15:04", track.Date.Text)
+				if err != nil {
+					ts = time.Now()
+				}
+			} else {
+				ts = time.Unix(unix, 0).UTC()
+			}
+			opts := catalog.SubmitListenOpts{
+				MbzCaller:      mbzc,
+				Artist:         track.Artist.Text,
+				ArtistMbzIDs:   []uuid.UUID{artistMbzID},
+				TrackTitle:     track.Name,
+				RecordingMbzID: trackMbzID,
+				ReleaseTitle:   album,
+				ReleaseMbzID:   albumMbzID,
+				Time:           ts,
+				UserID:         1,
+			}
+			err = catalog.SubmitListen(ctx, store, opts)
+			if err != nil {
+				l.Err(err).Msg("Failed to import LastFM playback item")
+				return err
+			}
+			count++
+			throttleFunc()
+		}
+	}
+	return finishImport(ctx, filename, count)
+}

internal/importer/maloja.go

@@ -81,17 +81,5 @@ func ImportMalojaFile(ctx context.Context, store db.DB, filename string) error {
 		}
 		throttleFunc()
 	}
-	_, err = os.Stat(path.Join(cfg.ConfigDir(), "import_complete"))
+	return finishImport(ctx, filename, len(export.Scrobbles))
-	if err != nil {
-		err = os.Mkdir(path.Join(cfg.ConfigDir(), "import_complete"), 0744)
-		if err != nil {
-			l.Err(err).Msg("Failed to create import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
-		}
-	}
-	err = os.Rename(path.Join(cfg.ConfigDir(), "import", filename), path.Join(cfg.ConfigDir(), "import_complete", filename))
-	if err != nil {
-		l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
-	}
-	l.Info().Msgf("Finished importing %s; imported %d items", filename, len(export.Scrobbles))
-	return nil
 }

internal/importer/spotify.go

@@ -67,17 +67,5 @@ func ImportSpotifyFile(ctx context.Context, store db.DB, filename string) error
 		}
 		throttleFunc()
 	}
-	_, err = os.Stat(path.Join(cfg.ConfigDir(), "import_complete"))
+	return finishImport(ctx, filename, len(export))
-	if err != nil {
-		err = os.Mkdir(path.Join(cfg.ConfigDir(), "import_complete"), 0744)
-		if err != nil {
-			l.Err(err).Msg("Failed to create import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
-		}
-	}
-	err = os.Rename(path.Join(cfg.ConfigDir(), "import", filename), path.Join(cfg.ConfigDir(), "import_complete", filename))
-	if err != nil {
-		l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start")
-	}
-	l.Info().Msgf("Finished importing %s; imported %d items", filename, len(export))
-	return nil
 }