From d2277aea3293ceed173b726cd92c216f5a5cbab0 Mon Sep 17 00:00:00 2001 From: Gabe Farrell Date: Thu, 12 Jun 2025 21:54:35 -0400 Subject: [PATCH] feat: add last fm importer --- engine/engine.go | 10 +- engine/import_test.go | 35 ++++++- internal/importer/importer.go | 26 +++++ internal/importer/lastfm.go | 116 +++++++++++++++++++++ internal/importer/maloja.go | 14 +-- internal/importer/spotify.go | 14 +-- static/recenttracks-shoko2-1749776100.json | 1 + 7 files changed, 186 insertions(+), 30 deletions(-) create mode 100644 internal/importer/lastfm.go create mode 100644 static/recenttracks-shoko2-1749776100.json diff --git a/engine/engine.go b/engine/engine.go index a7133b9..7886efc 100644 --- a/engine/engine.go +++ b/engine/engine.go @@ -163,7 +163,7 @@ func Run( // Import if !cfg.SkipImport() { go func() { - RunImporter(l, store) + RunImporter(l, store, mbzC) }() } @@ -186,7 +186,7 @@ func Run( return nil } -func RunImporter(l *zerolog.Logger, store db.DB) { +func RunImporter(l *zerolog.Logger, store db.DB, mbzc mbz.MusicBrainzCaller) { l.Debug().Msg("Checking for import files...") files, err := os.ReadDir(path.Join(cfg.ConfigDir(), "import")) if err != nil { @@ -218,6 +218,12 @@ func RunImporter(l *zerolog.Logger, store db.DB) { if err != nil { l.Err(err).Msgf("Failed to import file: %s", file.Name()) } + } else if strings.Contains(file.Name(), "recenttracks") { + l.Info().Msgf("Import file %s detecting as being ghan.nl LastFM export", file.Name()) + err := importer.ImportLastFMFile(logger.NewContext(l), store, mbzc, file.Name()) + if err != nil { + l.Err(err).Msgf("Failed to import file: %s", file.Name()) + } } else { l.Warn().Msgf("File %s not recognized as a valid import file; make sure it is valid and named correctly", file.Name()) } diff --git a/engine/import_test.go b/engine/import_test.go index 09c6f39..7dff162 100644 --- a/engine/import_test.go +++ b/engine/import_test.go @@ -5,11 +5,14 @@ import ( "os" "path/filepath" "testing" + "time" "github.com/gabehf/koito/engine" "github.com/gabehf/koito/internal/cfg" "github.com/gabehf/koito/internal/db" "github.com/gabehf/koito/internal/logger" + "github.com/gabehf/koito/internal/mbz" + "github.com/google/uuid" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" ) @@ -27,7 +30,7 @@ func TestImportMaloja(t *testing.T) { require.NoError(t, os.WriteFile(dest, input, os.ModePerm)) - engine.RunImporter(logger.Get(), store) + engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{}) // maloja test import is 38 Magnify Tokyo streams a, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "Magnify Tokyo"}) @@ -50,7 +53,7 @@ func TestImportSpotify(t *testing.T) { require.NoError(t, os.WriteFile(dest, input, os.ModePerm)) - engine.RunImporter(logger.Get(), store) + engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{}) a, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "The Story So Far"}) require.NoError(t, err) @@ -62,3 +65,31 @@ func TestImportSpotify(t *testing.T) { // this is the only track with valid duration data assert.EqualValues(t, 181, track.Duration) } + +func TestImportLastFM(t *testing.T) { + + src := "../static/recenttracks-shoko2-1749776100.json" + destDir := filepath.Join(cfg.ConfigDir(), "import") + dest := filepath.Join(destDir, "recenttracks-shoko2-1749776100.json") + + // not going to make the dest dir because engine should make it already + + input, err := os.ReadFile(src) + require.NoError(t, err) + + require.NoError(t, os.WriteFile(dest, input, os.ModePerm)) + + engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{}) + + album, err := store.GetAlbum(context.Background(), db.GetAlbumOpts{MusicBrainzID: uuid.MustParse("e9e78802-0bf8-4ca3-9655-1d943d2d2fa0")}) + require.NoError(t, err) + assert.Equal(t, "ZOO!!", album.Title) + artist, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "CHUU"}) + require.NoError(t, err) + track, err := store.GetTrack(context.Background(), db.GetTrackOpts{Title: "because I'm stupid?", ArtistIDs: []int32{artist.ID}}) + require.NoError(t, err) + listens, err := store.GetListensPaginated(context.Background(), db.GetItemsOpts{TrackID: int(track.ID)}) + require.NoError(t, err) + assert.Len(t, listens.Items, 1) + assert.WithinDuration(t, time.Unix(1749776100, 0), listens.Items[0].Time, 1*time.Second) +} diff --git a/internal/importer/importer.go b/internal/importer/importer.go index ead7eca..1e7025d 100644 --- a/internal/importer/importer.go +++ b/internal/importer/importer.go @@ -1 +1,27 @@ package importer + +import ( + "context" + "os" + "path" + + "github.com/gabehf/koito/internal/cfg" + "github.com/gabehf/koito/internal/logger" +) + +func finishImport(ctx context.Context, filename string, numImported int) error { + l := logger.FromContext(ctx) + _, err := os.Stat(path.Join(cfg.ConfigDir(), "import_complete")) + if err != nil { + err = os.Mkdir(path.Join(cfg.ConfigDir(), "import_complete"), 0744) + if err != nil { + l.Err(err).Msg("Failed to create import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start") + } + } + err = os.Rename(path.Join(cfg.ConfigDir(), "import", filename), path.Join(cfg.ConfigDir(), "import_complete", filename)) + if err != nil { + l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start") + } + l.Info().Msgf("Finished importing %s; imported %d items", filename, numImported) + return nil +} diff --git a/internal/importer/lastfm.go b/internal/importer/lastfm.go new file mode 100644 index 0000000..07a412a --- /dev/null +++ b/internal/importer/lastfm.go @@ -0,0 +1,116 @@ +package importer + +import ( + "context" + "encoding/json" + "os" + "path" + "strconv" + "time" + + "github.com/gabehf/koito/internal/catalog" + "github.com/gabehf/koito/internal/cfg" + "github.com/gabehf/koito/internal/db" + "github.com/gabehf/koito/internal/logger" + "github.com/gabehf/koito/internal/mbz" + "github.com/google/uuid" +) + +type LastFMExportPage struct { + Track []LastFMTrack `json:"track"` +} +type LastFMTrack struct { + Artist LastFMItem `json:"artist"` + Images []LastFMImage `json:"image"` + MBID string `json:"mbid"` + Album LastFMItem `json:"album"` + Name string `json:"name"` + Date LastFMDate `json:"date"` +} +type LastFMItem struct { + MBID string `json:"mbid"` + Text string `json:"#text"` +} +type LastFMDate struct { + Unix string `json:"uts"` + Text string `json:"#text"` +} +type LastFMImage struct { + Size string `json:"size"` + Url string `json:"#text"` +} + +func ImportLastFMFile(ctx context.Context, store db.DB, mbzc mbz.MusicBrainzCaller, filename string) error { + l := logger.FromContext(ctx) + l.Info().Msgf("Beginning LastFM import on file: %s", filename) + file, err := os.Open(path.Join(cfg.ConfigDir(), "import", filename)) + if err != nil { + l.Err(err).Msgf("Failed to read import file: %s", filename) + return err + } + var throttleFunc = func() {} + if ms := cfg.ThrottleImportMs(); ms > 0 { + throttleFunc = func() { + time.Sleep(time.Duration(ms) * time.Millisecond) + } + } + export := make([]LastFMExportPage, 0) + err = json.NewDecoder(file).Decode(&export) + if err != nil { + return err + } + count := 0 + for _, item := range export { + for _, track := range item.Track { + album := track.Album.Text + if album == "" { + album = track.Name + } + if track.Name == "" || track.Artist.Text == "" { + l.Debug().Msg("Skipping invalid LastFM import item") + continue + } + albumMbzID, err := uuid.Parse(track.Album.MBID) + if err != nil { + albumMbzID = uuid.Nil + } + artistMbzID, err := uuid.Parse(track.Artist.MBID) + if err != nil { + artistMbzID = uuid.Nil + } + trackMbzID, err := uuid.Parse(track.MBID) + if err != nil { + trackMbzID = uuid.Nil + } + var ts time.Time + unix, err := strconv.ParseInt(track.Date.Unix, 10, 64) + if err != nil { + ts, err = time.Parse("02 Jan 2006, 15:04", track.Date.Text) + if err != nil { + ts = time.Now() + } + } else { + ts = time.Unix(unix, 0).UTC() + } + opts := catalog.SubmitListenOpts{ + MbzCaller: mbzc, + Artist: track.Artist.Text, + ArtistMbzIDs: []uuid.UUID{artistMbzID}, + TrackTitle: track.Name, + RecordingMbzID: trackMbzID, + ReleaseTitle: album, + ReleaseMbzID: albumMbzID, + Time: ts, + UserID: 1, + } + err = catalog.SubmitListen(ctx, store, opts) + if err != nil { + l.Err(err).Msg("Failed to import LastFM playback item") + return err + } + count++ + throttleFunc() + } + } + return finishImport(ctx, filename, count) +} diff --git a/internal/importer/maloja.go b/internal/importer/maloja.go index b0a6d16..fdd647b 100644 --- a/internal/importer/maloja.go +++ b/internal/importer/maloja.go @@ -81,17 +81,5 @@ func ImportMalojaFile(ctx context.Context, store db.DB, filename string) error { } throttleFunc() } - _, err = os.Stat(path.Join(cfg.ConfigDir(), "import_complete")) - if err != nil { - err = os.Mkdir(path.Join(cfg.ConfigDir(), "import_complete"), 0744) - if err != nil { - l.Err(err).Msg("Failed to create import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start") - } - } - err = os.Rename(path.Join(cfg.ConfigDir(), "import", filename), path.Join(cfg.ConfigDir(), "import_complete", filename)) - if err != nil { - l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start") - } - l.Info().Msgf("Finished importing %s; imported %d items", filename, len(export.Scrobbles)) - return nil + return finishImport(ctx, filename, len(export.Scrobbles)) } diff --git a/internal/importer/spotify.go b/internal/importer/spotify.go index 275d47e..0f338ff 100644 --- a/internal/importer/spotify.go +++ b/internal/importer/spotify.go @@ -67,17 +67,5 @@ func ImportSpotifyFile(ctx context.Context, store db.DB, filename string) error } throttleFunc() } - _, err = os.Stat(path.Join(cfg.ConfigDir(), "import_complete")) - if err != nil { - err = os.Mkdir(path.Join(cfg.ConfigDir(), "import_complete"), 0744) - if err != nil { - l.Err(err).Msg("Failed to create import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start") - } - } - err = os.Rename(path.Join(cfg.ConfigDir(), "import", filename), path.Join(cfg.ConfigDir(), "import_complete", filename)) - if err != nil { - l.Err(err).Msg("Failed to move file to import_complete dir! Import files must be removed from the import directory manually, or else the importer will run on every app start") - } - l.Info().Msgf("Finished importing %s; imported %d items", filename, len(export)) - return nil + return finishImport(ctx, filename, len(export)) } diff --git a/static/recenttracks-shoko2-1749776100.json b/static/recenttracks-shoko2-1749776100.json new file mode 100644 index 0000000..c8e647d --- /dev/null +++ b/static/recenttracks-shoko2-1749776100.json @@ -0,0 +1 @@ +[{"track":[{"artist":{"mbid":"","#text":"CHUU"},"streamable":"0","image":[{"size":"small","#text":""},{"size":"medium","#text":""},{"size":"large","#text":""},{"size":"extralarge","#text":""}],"mbid":"","album":{"mbid":"","#text":""},"name":"because I'm stupid?","url":"https://www.last.fm/music/CHUU/_/because+I%27m+stupid%3F","date":{"uts":"1749776100","#text":"13 Jun 2025, 00:55"}},{"artist":{"mbid":"","#text":"Carly Rae Jepsen"},"streamable":"0","image":[{"size":"small","#text":""},{"size":"medium","#text":""},{"size":"large","#text":""},{"size":"extralarge","#text":""}],"mbid":"","album":{"mbid":"","#text":""},"name":"The Loneliest Time","url":"https://www.last.fm/music/Carly+Rae+Jepsen/_/The+Loneliest+Time","date":{"uts":"1749775800","#text":"13 Jun 2025, 00:50"}},{"artist":{"mbid":"","#text":"Minami"},"streamable":"0","image":[{"size":"small","#text":""},{"size":"medium","#text":""},{"size":"large","#text":""},{"size":"extralarge","#text":""}],"mbid":"","album":{"mbid":"","#text":""},"name":"Kawaki wo Ameku","url":"https://www.last.fm/music/Minami/_/Kawaki+wo+Ameku","date":{"uts":"1749775500","#text":"13 Jun 2025, 00:45"}},{"artist":{"mbid":"","#text":"Younha"},"streamable":"0","image":[{"size":"small","#text":"https://lastfm.freetls.fastly.net/i/u/34s/fdc57ed2af0201d30069cd66b446f749.png"},{"size":"medium","#text":"https://lastfm.freetls.fastly.net/i/u/64s/fdc57ed2af0201d30069cd66b446f749.png"},{"size":"large","#text":"https://lastfm.freetls.fastly.net/i/u/174s/fdc57ed2af0201d30069cd66b446f749.png"},{"size":"extralarge","#text":"https://lastfm.freetls.fastly.net/i/u/300x300/fdc57ed2af0201d30069cd66b446f749.png"}],"mbid":"","album":{"mbid":"","#text":"YOUNHA 6th Album Repackage 'END THEORY : Final Edition'"},"name":"Event Horizon","url":"https://www.last.fm/music/Younha/_/Event+Horizon","date":{"uts":"1749775200","#text":"13 Jun 2025, 00:40"}},{"artist":{"mbid":"","#text":"Necry Talkie"},"streamable":"0","image":[{"size":"small","#text":"https://lastfm.freetls.fastly.net/i/u/34s/7a4b1ccf2ab64548c0fce8e488fc181f.jpg"},{"size":"medium","#text":"https://lastfm.freetls.fastly.net/i/u/64s/7a4b1ccf2ab64548c0fce8e488fc181f.jpg"},{"size":"large","#text":"https://lastfm.freetls.fastly.net/i/u/174s/7a4b1ccf2ab64548c0fce8e488fc181f.jpg"},{"size":"extralarge","#text":"https://lastfm.freetls.fastly.net/i/u/300x300/7a4b1ccf2ab64548c0fce8e488fc181f.jpg"}],"mbid":"","album":{"mbid":"e9e78802-0bf8-4ca3-9655-1d943d2d2fa0","#text":"ZOO!!"},"name":"放課後の記憶","url":"https://www.last.fm/music/Necry+Talkie/_/%E6%94%BE%E8%AA%B2%E5%BE%8C%E3%81%AE%E8%A8%98%E6%86%B6","date":{"uts":"1749774900","#text":"13 Jun 2025, 00:35"}},{"artist":{"mbid":"","#text":"Necry Talkie"},"streamable":"0","image":[{"size":"small","#text":""},{"size":"medium","#text":""},{"size":"large","#text":""},{"size":"extralarge","#text":""}],"mbid":"","album":{"mbid":"","#text":""},"name":"Bloom","url":"https://www.last.fm/music/Necry+Talkie/_/Bloom","date":{"uts":"1749774600","#text":"13 Jun 2025, 00:30"}},{"artist":{"mbid":"","#text":"Masayuki Suzuki"},"streamable":"0","image":[{"size":"small","#text":"https://lastfm.freetls.fastly.net/i/u/34s/c35142e24bd8c301c9d4fbd78a1d32a0.jpg"},{"size":"medium","#text":"https://lastfm.freetls.fastly.net/i/u/64s/c35142e24bd8c301c9d4fbd78a1d32a0.jpg"},{"size":"large","#text":"https://lastfm.freetls.fastly.net/i/u/174s/c35142e24bd8c301c9d4fbd78a1d32a0.jpg"},{"size":"extralarge","#text":"https://lastfm.freetls.fastly.net/i/u/300x300/c35142e24bd8c301c9d4fbd78a1d32a0.jpg"}],"mbid":"","album":{"mbid":"","#text":"GIRI GIRI"},"name":"GIRI GIRI","url":"https://www.last.fm/music/Masayuki+Suzuki/_/GIRI+GIRI","date":{"uts":"1749774300","#text":"13 Jun 2025, 00:25"}},{"artist":{"mbid":"","#text":"Tota"},"streamable":"0","image":[{"size":"small","#text":""},{"size":"medium","#text":""},{"size":"large","#text":""},{"size":"extralarge","#text":""}],"mbid":"","album":{"mbid":"","#text":""},"name":"Tsumugu","url":"https://www.last.fm/music/Tota/_/Tsumugu","date":{"uts":"1749774000","#text":"13 Jun 2025, 00:20"}},{"artist":{"mbid":"4b00640f-3be6-43f8-9b34-ff81bd89320a","#text":"OurR"},"streamable":"0","image":[{"size":"small","#text":"https://lastfm.freetls.fastly.net/i/u/34s/a4e182b7382c6b631a2b4a073323408b.png"},{"size":"medium","#text":"https://lastfm.freetls.fastly.net/i/u/64s/a4e182b7382c6b631a2b4a073323408b.png"},{"size":"large","#text":"https://lastfm.freetls.fastly.net/i/u/174s/a4e182b7382c6b631a2b4a073323408b.png"},{"size":"extralarge","#text":"https://lastfm.freetls.fastly.net/i/u/300x300/a4e182b7382c6b631a2b4a073323408b.png"}],"mbid":"","album":{"mbid":"","#text":"Desert"},"name":"Desert","url":"https://www.last.fm/music/OurR/_/Desert","date":{"uts":"1749773700","#text":"13 Jun 2025, 00:15"}},{"artist":{"mbid":"4ebb5ad3-9018-407d-8c24-c03011ab9ac6","#text":"American Football"},"streamable":"0","image":[{"size":"small","#text":"https://lastfm.freetls.fastly.net/i/u/34s/3cc807cfbb0eaaf0a2d1999af7305c09.png"},{"size":"medium","#text":"https://lastfm.freetls.fastly.net/i/u/64s/3cc807cfbb0eaaf0a2d1999af7305c09.png"},{"size":"large","#text":"https://lastfm.freetls.fastly.net/i/u/174s/3cc807cfbb0eaaf0a2d1999af7305c09.png"},{"size":"extralarge","#text":"https://lastfm.freetls.fastly.net/i/u/300x300/3cc807cfbb0eaaf0a2d1999af7305c09.png"}],"mbid":"f3a4bd1d-8b3c-4da8-8ee4-e984fe79d14f","album":{"mbid":"","#text":"Uncomfortably Numb"},"name":"Uncomfortably Numb","url":"https://www.last.fm/music/American+Football/_/Uncomfortably+Numb","date":{"uts":"1749773400","#text":"13 Jun 2025, 00:10"}}],"@attr":{"perPage":"200","totalPages":"1","page":"1","total":"10","user":"shoko2"}}] \ No newline at end of file