Fix Maloja importer resilience: nil MBZ client, null album, error handling

Root cause of the panic at ~1,693 items: importers created
`&mbz.MusicBrainzClient{}` (nil requestQueue) instead of using the
engine's properly initialized client. When any code path called an MBZ
method, it panicked on the nil channel.

Changes:
- Pass engine's MBZ client to Maloja and Spotify importers
- Change MalojaTrack.Album to pointer type to handle null album JSON
- Continue on error instead of aborting the entire import
- Accept both Maloja export formats ("scrobbles" and "list" keys)
- Extract per-file import into importFile() with its own defer/recover
- Add progress logging every 500 items

Test fixtures:
- maloja_import_null_album_test.json (null album, valid album, empty artists)
- maloja_api_format_test.json (API "list" format)

New tests: TestImportMaloja_NullAlbum, TestImportMaloja_ApiFormat

Co-Authored-By: Claude Opus 4.6 (1M context) <noreply@anthropic.com>
This commit is contained in:
safierinx-a 2026-03-24 23:58:45 +05:30
parent 0ec7b458cc
commit c2d393aa03
6 changed files with 224 additions and 55 deletions

View file

@ -263,47 +263,51 @@ func RunImporter(l *zerolog.Logger, store db.DB, mbzc mbz.MusicBrainzCaller) {
} else {
return
}
defer func() {
if r := recover(); r != nil {
l.Error().Interface("recover", r).Msg("Importer: Panic when importing files")
}
}()
for _, file := range files {
if file.IsDir() {
continue
}
if strings.Contains(file.Name(), "Streaming_History_Audio") {
l.Info().Msgf("Importer: Import file %s detecting as being Spotify export", file.Name())
err := importer.ImportSpotifyFile(logger.NewContext(l), store, file.Name())
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", file.Name())
}
} else if strings.Contains(file.Name(), "maloja") {
l.Info().Msgf("Importer: Import file %s detecting as being Maloja export", file.Name())
err := importer.ImportMalojaFile(logger.NewContext(l), store, file.Name())
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", file.Name())
}
} else if strings.Contains(file.Name(), "recenttracks") {
l.Info().Msgf("Importer: Import file %s detecting as being ghan.nl LastFM export", file.Name())
err := importer.ImportLastFMFile(logger.NewContext(l), store, mbzc, file.Name())
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", file.Name())
}
} else if strings.Contains(file.Name(), "listenbrainz") {
l.Info().Msgf("Importer: Import file %s detecting as being ListenBrainz export", file.Name())
err := importer.ImportListenBrainzExport(logger.NewContext(l), store, mbzc, file.Name())
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", file.Name())
}
} else if strings.Contains(file.Name(), "koito") {
l.Info().Msgf("Importer: Import file %s detecting as being Koito export", file.Name())
err := importer.ImportKoitoFile(logger.NewContext(l), store, file.Name())
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", file.Name())
}
} else {
l.Warn().Msgf("Importer: File %s not recognized as a valid import file; make sure it is valid and named correctly", file.Name())
}
importFile(l, store, mbzc, file.Name())
}
}
func importFile(l *zerolog.Logger, store db.DB, mbzc mbz.MusicBrainzCaller, filename string) {
defer func() {
if r := recover(); r != nil {
l.Error().Interface("recover", r).Msgf("Importer: Panic when importing file: %s", filename)
}
}()
if strings.Contains(filename, "Streaming_History_Audio") {
l.Info().Msgf("Importer: Import file %s detecting as being Spotify export", filename)
err := importer.ImportSpotifyFile(logger.NewContext(l), store, mbzc, filename)
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", filename)
}
} else if strings.Contains(filename, "maloja") {
l.Info().Msgf("Importer: Import file %s detecting as being Maloja export", filename)
err := importer.ImportMalojaFile(logger.NewContext(l), store, mbzc, filename)
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", filename)
}
} else if strings.Contains(filename, "recenttracks") {
l.Info().Msgf("Importer: Import file %s detecting as being ghan.nl LastFM export", filename)
err := importer.ImportLastFMFile(logger.NewContext(l), store, mbzc, filename)
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", filename)
}
} else if strings.Contains(filename, "listenbrainz") {
l.Info().Msgf("Importer: Import file %s detecting as being ListenBrainz export", filename)
err := importer.ImportListenBrainzExport(logger.NewContext(l), store, mbzc, filename)
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", filename)
}
} else if strings.Contains(filename, "koito") {
l.Info().Msgf("Importer: Import file %s detecting as being Koito export", filename)
err := importer.ImportKoitoFile(logger.NewContext(l), store, filename)
if err != nil {
l.Err(err).Msgf("Importer: Failed to import file: %s", filename)
}
} else {
l.Warn().Msgf("Importer: File %s not recognized as a valid import file; make sure it is valid and named correctly", filename)
}
}

View file

@ -44,6 +44,61 @@ func TestImportMaloja(t *testing.T) {
truncateTestData(t)
}
func TestImportMaloja_NullAlbum(t *testing.T) {
src := path.Join("..", "test_assets", "maloja_import_null_album_test.json")
destDir := filepath.Join(cfg.ConfigDir(), "import")
dest := filepath.Join(destDir, "maloja_import_null_album_test.json")
input, err := os.ReadFile(src)
require.NoError(t, err)
require.NoError(t, os.WriteFile(dest, input, os.ModePerm))
engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{})
// The null-album track should be imported with album falling back to track name
a, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "Null Album Artist"})
require.NoError(t, err)
assert.Equal(t, "Null Album Artist", a.Name)
assert.EqualValues(t, 1, a.ListenCount)
// The valid-album track should also be imported
a, err = store.GetArtist(context.Background(), db.GetArtistOpts{Name: "Valid Album Artist"})
require.NoError(t, err)
assert.Equal(t, "Valid Album Artist", a.Name)
assert.EqualValues(t, 1, a.ListenCount)
// The empty artists item should be skipped
count, err := store.CountArtists(context.Background(), db.Timeframe{Period: db.PeriodAllTime})
require.NoError(t, err)
assert.EqualValues(t, 2, count)
truncateTestData(t)
}
func TestImportMaloja_ApiFormat(t *testing.T) {
src := path.Join("..", "test_assets", "maloja_api_format_test.json")
destDir := filepath.Join(cfg.ConfigDir(), "import")
dest := filepath.Join(destDir, "maloja_api_format_test.json")
input, err := os.ReadFile(src)
require.NoError(t, err)
require.NoError(t, os.WriteFile(dest, input, os.ModePerm))
engine.RunImporter(logger.Get(), store, &mbz.MbzErrorCaller{})
// API format uses "list" key instead of "scrobbles"
a, err := store.GetArtist(context.Background(), db.GetArtistOpts{Name: "API Format Artist"})
require.NoError(t, err)
assert.Equal(t, "API Format Artist", a.Name)
assert.EqualValues(t, 2, a.ListenCount)
truncateTestData(t)
}
func TestImportSpotify(t *testing.T) {
src := path.Join("..", "test_assets", "Streaming_History_Audio_spotify_import_test.json")