feat: native import & export

pull/23/head
Gabe Farrell 6 months ago
parent 1d02cede49
commit 7b0cff0a07

@ -200,3 +200,70 @@ WHERE track_id = $1;
-- name: DeleteListen :exec -- name: DeleteListen :exec
DELETE FROM listens WHERE track_id = $1 AND listened_at = $2; DELETE FROM listens WHERE track_id = $1 AND listened_at = $2;
-- name: GetListensExportPage :many
SELECT
l.listened_at,
l.user_id,
l.client,
-- Track info
t.id AS track_id,
t.musicbrainz_id AS track_mbid,
t.duration AS track_duration,
(
SELECT json_agg(json_build_object(
'alias', ta.alias,
'source', ta.source,
'is_primary', ta.is_primary
))
FROM track_aliases ta
WHERE ta.track_id = t.id
) AS track_aliases,
-- Release info
r.id AS release_id,
r.musicbrainz_id AS release_mbid,
r.image AS release_image,
r.image_source AS release_image_source,
r.various_artists,
(
SELECT json_agg(json_build_object(
'alias', ra.alias,
'source', ra.source,
'is_primary', ra.is_primary
))
FROM release_aliases ra
WHERE ra.release_id = r.id
) AS release_aliases,
-- Artists
(
SELECT json_agg(json_build_object(
'id', a.id,
'musicbrainz_id', a.musicbrainz_id,
'image', a.image,
'image_source', a.image_source,
'aliases', (
SELECT json_agg(json_build_object(
'alias', aa.alias,
'source', aa.source,
'is_primary', aa.is_primary
))
FROM artist_aliases aa
WHERE aa.artist_id = a.id
)
))
FROM artist_tracks at
JOIN artists a ON a.id = at.artist_id
WHERE at.track_id = t.id
) AS artists
FROM listens l
JOIN tracks t ON l.track_id = t.id
JOIN releases r ON t.release_id = r.id
WHERE l.user_id = @user_id::int
AND (l.listened_at, l.track_id) > (@listened_at::timestamptz, @track_id::int)
ORDER BY l.listened_at, l.track_id
LIMIT $1;

@ -190,6 +190,14 @@ func Run(
}() }()
} }
// l.Info().Msg("Creating test export file")
// go func() {
// err := export.ExportData(ctx, "koito", store)
// if err != nil {
// l.Err(err).Msg("Failed to generate export file")
// }
// }()
l.Info().Msg("Engine: Pruning orphaned images") l.Info().Msg("Engine: Pruning orphaned images")
go catalog.PruneOrphanedImages(logger.NewContext(l), store) go catalog.PruneOrphanedImages(logger.NewContext(l), store)
@ -255,6 +263,12 @@ func RunImporter(l *zerolog.Logger, store db.DB, mbzc mbz.MusicBrainzCaller) {
if err != nil { if err != nil {
l.Err(err).Msgf("Failed to import file: %s", file.Name()) l.Err(err).Msgf("Failed to import file: %s", file.Name())
} }
} else if strings.Contains(file.Name(), "koito") {
l.Info().Msgf("Import file %s detecting as being Koito export", file.Name())
err := importer.ImportKoitoFile(logger.NewContext(l), store, file.Name())
if err != nil {
l.Err(err).Msgf("Failed to import file: %s", file.Name())
}
} else { } else {
l.Warn().Msgf("File %s not recognized as a valid import file; make sure it is valid and named correctly", file.Name()) l.Warn().Msgf("File %s not recognized as a valid import file; make sure it is valid and named correctly", file.Name())
} }

@ -82,6 +82,7 @@ type DB interface {
ImageHasAssociation(ctx context.Context, image uuid.UUID) (bool, error) ImageHasAssociation(ctx context.Context, image uuid.UUID) (bool, error)
GetImageSource(ctx context.Context, image uuid.UUID) (string, error) GetImageSource(ctx context.Context, image uuid.UUID) (string, error)
AlbumsWithoutImages(ctx context.Context, from int32) ([]*models.Album, error) AlbumsWithoutImages(ctx context.Context, from int32) ([]*models.Album, error)
GetExportPage(ctx context.Context, opts GetExportPageOpts) ([]*ExportItem, error)
Ping(ctx context.Context) error Ping(ctx context.Context) error
Close(ctx context.Context) Close(ctx context.Context)
} }

@ -147,3 +147,10 @@ type TimeListenedOpts struct {
ArtistID int32 ArtistID int32
TrackID int32 TrackID int32
} }
type GetExportPageOpts struct {
UserID int32
ListenedAt time.Time
TrackID int32
Limit int32
}

@ -128,6 +128,7 @@ func (d *Psql) SaveArtistAliases(ctx context.Context, id int32, aliases []string
} }
defer tx.Rollback(ctx) defer tx.Rollback(ctx)
qtx := d.q.WithTx(tx) qtx := d.q.WithTx(tx)
l.Debug().Msgf("Fetching existing artist aliases for artist %d...", id)
existing, err := qtx.GetAllArtistAliases(ctx, id) existing, err := qtx.GetAllArtistAliases(ctx, id)
if err != nil { if err != nil {
return fmt.Errorf("SaveArtistAliases: GetAllArtistAliases: %w", err) return fmt.Errorf("SaveArtistAliases: GetAllArtistAliases: %w", err)
@ -135,8 +136,10 @@ func (d *Psql) SaveArtistAliases(ctx context.Context, id int32, aliases []string
for _, v := range existing { for _, v := range existing {
aliases = append(aliases, v.Alias) aliases = append(aliases, v.Alias)
} }
l.Debug().Msgf("Ensuring aliases are unique...")
utils.Unique(&aliases) utils.Unique(&aliases)
for _, alias := range aliases { for _, alias := range aliases {
l.Debug().Msgf("Inserting alias %s for artist with id %d", alias, id)
alias = strings.TrimSpace(alias) alias = strings.TrimSpace(alias)
if alias == "" { if alias == "" {
return errors.New("SaveArtistAliases: aliases cannot be blank") return errors.New("SaveArtistAliases: aliases cannot be blank")

@ -0,0 +1,59 @@
package psql
import (
"context"
"encoding/json"
"fmt"
"github.com/gabehf/koito/internal/db"
"github.com/gabehf/koito/internal/models"
"github.com/gabehf/koito/internal/repository"
)
func (d *Psql) GetExportPage(ctx context.Context, opts db.GetExportPageOpts) ([]*db.ExportItem, error) {
rows, err := d.q.GetListensExportPage(ctx, repository.GetListensExportPageParams{
UserID: opts.UserID,
TrackID: opts.TrackID,
Limit: opts.Limit,
ListenedAt: opts.ListenedAt,
})
if err != nil {
return nil, fmt.Errorf("GetExportPage: %w", err)
}
ret := make([]*db.ExportItem, len(rows))
for i, row := range rows {
var trackAliases []models.Alias
err = json.Unmarshal(row.TrackAliases, &trackAliases)
if err != nil {
return nil, fmt.Errorf("GetExportPage: json.Unmarshal trackAliases: %w", err)
}
var albumAliases []models.Alias
err = json.Unmarshal(row.ReleaseAliases, &albumAliases)
if err != nil {
return nil, fmt.Errorf("GetExportPage: json.Unmarshal albumAliases: %w", err)
}
var artists []models.ArtistWithFullAliases
err = json.Unmarshal(row.Artists, &artists)
if err != nil {
return nil, fmt.Errorf("GetExportPage: json.Unmarshal artists: %w", err)
}
ret[i] = &db.ExportItem{
TrackID: row.TrackID,
ListenedAt: row.ListenedAt,
UserID: row.UserID,
Client: row.Client,
TrackMbid: row.TrackMbid,
TrackDuration: row.TrackDuration,
TrackAliases: trackAliases,
ReleaseID: row.ReleaseID,
ReleaseMbid: row.ReleaseMbid,
ReleaseImageSource: row.ReleaseImageSource.String,
VariousArtists: row.VariousArtists,
ReleaseAliases: albumAliases,
Artists: artists,
}
}
return ret, nil
}

@ -2,6 +2,9 @@ package db
import ( import (
"time" "time"
"github.com/gabehf/koito/internal/models"
"github.com/google/uuid"
) )
type InformationSource string type InformationSource string
@ -24,3 +27,20 @@ type PaginatedResponse[T any] struct {
HasNextPage bool `json:"has_next_page"` HasNextPage bool `json:"has_next_page"`
CurrentPage int32 `json:"current_page"` CurrentPage int32 `json:"current_page"`
} }
type ExportItem struct {
ListenedAt time.Time
UserID int32
Client *string
TrackID int32
TrackMbid *uuid.UUID
TrackDuration int32
TrackAliases []models.Alias
ReleaseID int32
ReleaseMbid *uuid.UUID
ReleaseImage *uuid.UUID
ReleaseImageSource string
VariousArtists bool
ReleaseAliases []models.Alias
Artists []models.ArtistWithFullAliases
}

@ -0,0 +1,147 @@
package export
import (
"context"
"encoding/json"
"fmt"
"os"
"path"
"time"
"github.com/gabehf/koito/internal/cfg"
"github.com/gabehf/koito/internal/db"
"github.com/gabehf/koito/internal/logger"
"github.com/gabehf/koito/internal/models"
"github.com/google/uuid"
)
type KoitoExport struct {
Version string `json:"version"`
ExportedAt time.Time `json:"exported_at"` // RFC3339
User string `json:"user"` // username
Listens []KoitoListen `json:"listens"`
}
type KoitoListen struct {
ListenedAt time.Time `json:"listened_at"`
Track KoitoTrack `json:"track"`
Album KoitoAlbum `json:"album"`
Artists []KoitoArtist `json:"artists"`
}
type KoitoTrack struct {
MBID *uuid.UUID `json:"mbid"`
Duration int `json:"duration"`
Aliases []models.Alias `json:"aliases"`
}
type KoitoAlbum struct {
ImageUrl string `json:"image_url"`
MBID *uuid.UUID `json:"mbid"`
Aliases []models.Alias `json:"aliases"`
VariousArtists bool `json:"various_artists"`
}
type KoitoArtist struct {
ImageUrl string `json:"image_url"`
MBID *uuid.UUID `json:"mbid"`
IsPrimary bool `json:"is_primary"`
Aliases []models.Alias `json:"aliases"`
}
func ExportData(ctx context.Context, username string, store db.DB) error {
lastTime := time.Unix(0, 0)
lastTrackId := int32(0)
pageSize := int32(1000)
l := logger.FromContext(ctx)
l.Info().Msg("ExportData: Generating Koito export file...")
exportedAt := time.Now()
exportFile := path.Join(cfg.ConfigDir(), fmt.Sprintf("koito_export_%d.json", exportedAt.Unix()))
f, err := os.Create(exportFile)
if err != nil {
return fmt.Errorf("ExportData: %w", err)
}
defer f.Close()
// Write the opening of the JSON manually
_, err = fmt.Fprintf(f, "{\n \"version\": \"1\",\n \"exported_at\": \"%s\",\n \"user\": \"%s\",\n \"listens\": [\n", exportedAt.UTC().Format(time.RFC3339), username)
if err != nil {
return fmt.Errorf("ExportData: %w", err)
}
first := true
for {
rows, err := store.GetExportPage(ctx, db.GetExportPageOpts{
UserID: 1,
ListenedAt: lastTime,
TrackID: lastTrackId,
Limit: pageSize,
})
if err != nil {
return fmt.Errorf("ExportData: %w", err)
}
if len(rows) == 0 {
break
}
for _, r := range rows {
// Adds a comma after each listen item
if !first {
_, _ = f.Write([]byte(",\n"))
}
first = false
exported := convertToExportFormat(r)
raw, err := json.MarshalIndent(exported, " ", " ")
// needed to make the listen item start at the right indent level
f.Write([]byte(" "))
if err != nil {
return fmt.Errorf("ExportData: marshal: %w", err)
}
_, _ = f.Write(raw)
if r.TrackID > lastTrackId {
lastTrackId = r.TrackID
}
if r.ListenedAt.After(lastTime) {
lastTime = r.ListenedAt
}
}
}
// Write closing of the JSON array and object
_, err = f.Write([]byte("\n ]\n}\n"))
if err != nil {
return fmt.Errorf("ExportData: f.Write: %w", err)
}
l.Info().Msgf("Export successful! File can be found at %s", exportFile)
return nil
}
func convertToExportFormat(item *db.ExportItem) *KoitoListen {
ret := &KoitoListen{
ListenedAt: item.ListenedAt.UTC(),
Track: KoitoTrack{
MBID: item.TrackMbid,
Duration: int(item.TrackDuration),
Aliases: item.TrackAliases,
},
Album: KoitoAlbum{
MBID: item.ReleaseMbid,
ImageUrl: item.ReleaseImageSource,
VariousArtists: item.VariousArtists,
Aliases: item.ReleaseAliases,
},
}
for i := range item.Artists {
ret.Artists = append(ret.Artists, KoitoArtist{
IsPrimary: item.Artists[i].IsPrimary,
MBID: item.Artists[i].MbzID,
Aliases: item.Artists[i].Aliases,
ImageUrl: item.Artists[i].ImageSource,
})
}
return ret
}

@ -0,0 +1,172 @@
package importer
import (
"context"
"encoding/json"
"errors"
"fmt"
"os"
"path"
"strings"
"github.com/gabehf/koito/internal/cfg"
"github.com/gabehf/koito/internal/db"
"github.com/gabehf/koito/internal/export"
"github.com/gabehf/koito/internal/logger"
"github.com/gabehf/koito/internal/models"
"github.com/gabehf/koito/internal/utils"
"github.com/google/uuid"
"github.com/jackc/pgx/v5"
)
func ImportKoitoFile(ctx context.Context, store db.DB, filename string) error {
l := logger.FromContext(ctx)
l.Info().Msgf("Beginning Koito import on file: %s", filename)
data := new(export.KoitoExport)
f, err := os.Open(path.Join(cfg.ConfigDir(), "import", filename))
if err != nil {
return fmt.Errorf("ImportKoitoFile: os.Open: %w", err)
}
defer f.Close()
err = json.NewDecoder(f).Decode(data)
if err != nil {
return fmt.Errorf("ImportKoitoFile: Decode: %w", err)
}
if data.Version != "1" {
return fmt.Errorf("ImportKoitoFile: unupported version: %s", data.Version)
}
l.Info().Msgf("Beginning data import for user: %s", data.User)
count := 0
for i := range data.Listens {
// use this for save/get mbid for all artist/album/track
mbid := uuid.Nil
artistIds := make([]int32, 0)
for _, ia := range data.Listens[i].Artists {
if ia.MBID != nil {
mbid = *ia.MBID
}
artist, err := store.GetArtist(ctx, db.GetArtistOpts{
MusicBrainzID: mbid,
Name: getPrimaryAliasFromAliasSlice(ia.Aliases),
})
if errors.Is(err, pgx.ErrNoRows) {
var imgid = uuid.Nil
// not a perfect way to check if the image url is an actual source vs manual upload but
// im like 99% sure it will work perfectly
if strings.HasPrefix(ia.ImageUrl, "http") {
imgid = uuid.New()
}
// save artist
artist, err := store.SaveArtist(ctx, db.SaveArtistOpts{
Name: getPrimaryAliasFromAliasSlice(ia.Aliases),
Image: imgid,
ImageSrc: ia.ImageUrl,
MusicBrainzID: mbid,
Aliases: utils.FlattenAliases(ia.Aliases),
})
if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
}
artistIds = append(artistIds, artist.ID)
} else if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
} else {
artistIds = append(artistIds, artist.ID)
}
}
// call associate album
albumId := int32(0)
if data.Listens[i].Album.MBID != nil {
mbid = *data.Listens[i].Album.MBID
}
album, err := store.GetAlbum(ctx, db.GetAlbumOpts{
MusicBrainzID: mbid,
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Album.Aliases),
ArtistID: artistIds[0],
})
if errors.Is(err, pgx.ErrNoRows) {
var imgid = uuid.Nil
// not a perfect way to check if the image url is an actual source vs manual upload but
// im like 99% sure it will work perfectly
if strings.HasPrefix(data.Listens[i].Album.ImageUrl, "http") {
imgid = uuid.New()
}
// save album
album, err = store.SaveAlbum(ctx, db.SaveAlbumOpts{
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Album.Aliases),
Image: imgid,
ImageSrc: data.Listens[i].Album.ImageUrl,
MusicBrainzID: mbid,
Aliases: utils.FlattenAliases(data.Listens[i].Album.Aliases),
ArtistIDs: artistIds,
VariousArtists: data.Listens[i].Album.VariousArtists,
})
if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
}
albumId = album.ID
} else if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
} else {
albumId = album.ID
}
// call associate track
if data.Listens[i].Track.MBID != nil {
mbid = *data.Listens[i].Track.MBID
}
track, err := store.GetTrack(ctx, db.GetTrackOpts{
MusicBrainzID: mbid,
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Track.Aliases),
ArtistIDs: artistIds,
})
if errors.Is(err, pgx.ErrNoRows) {
// save track
track, err = store.SaveTrack(ctx, db.SaveTrackOpts{
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Track.Aliases),
RecordingMbzID: mbid,
Duration: int32(data.Listens[i].Track.Duration),
ArtistIDs: artistIds,
AlbumID: albumId,
})
if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
}
// save track aliases
err = store.SaveTrackAliases(ctx, track.ID, utils.FlattenAliases(data.Listens[i].Track.Aliases), "Import")
if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
}
} else if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
}
// save listen
err = store.SaveListen(ctx, db.SaveListenOpts{
TrackID: track.ID,
Time: data.Listens[i].ListenedAt,
UserID: 1,
})
if err != nil {
return fmt.Errorf("ImportKoitoFile: %w", err)
}
l.Info().Msgf("ImportKoitoFile: Imported listen for track %s", track.Title)
count++
}
return finishImport(ctx, filename, count)
}
func getPrimaryAliasFromAliasSlice(aliases []models.Alias) string {
for _, a := range aliases {
if a.Primary {
return a.Alias
}
}
return ""
}

@ -1,7 +1,7 @@
package models package models
type Alias struct { type Alias struct {
ID int32 `json:"id"` ID int32 `json:"id,omitempty"`
Alias string `json:"alias"` Alias string `json:"alias"`
Source string `json:"source"` Source string `json:"source"`
Primary bool `json:"is_primary"` Primary bool `json:"is_primary"`

@ -17,3 +17,15 @@ type SimpleArtist struct {
ID int32 `json:"id"` ID int32 `json:"id"`
Name string `json:"name"` Name string `json:"name"`
} }
type ArtistWithFullAliases struct {
ID int32 `json:"id"`
MbzID *uuid.UUID `json:"musicbrainz_id"`
Name string `json:"name"`
Aliases []Alias `json:"aliases"`
Image *uuid.UUID `json:"image"`
ImageSource string `json:"image_source,omitempty"`
ListenCount int64 `json:"listen_count"`
TimeListened int64 `json:"time_listened"`
IsPrimary bool `json:"is_primary,omitempty"`
}

@ -9,6 +9,7 @@ import (
"context" "context"
"time" "time"
"github.com/google/uuid"
"github.com/jackc/pgx/v5/pgtype" "github.com/jackc/pgx/v5/pgtype"
) )
@ -451,6 +452,138 @@ func (q *Queries) GetLastListensPaginated(ctx context.Context, arg GetLastListen
return items, nil return items, nil
} }
const getListensExportPage = `-- name: GetListensExportPage :many
SELECT
l.listened_at,
l.user_id,
l.client,
-- Track info
t.id AS track_id,
t.musicbrainz_id AS track_mbid,
t.duration AS track_duration,
(
SELECT json_agg(json_build_object(
'alias', ta.alias,
'source', ta.source,
'is_primary', ta.is_primary
))
FROM track_aliases ta
WHERE ta.track_id = t.id
) AS track_aliases,
-- Release info
r.id AS release_id,
r.musicbrainz_id AS release_mbid,
r.image AS release_image,
r.image_source AS release_image_source,
r.various_artists,
(
SELECT json_agg(json_build_object(
'alias', ra.alias,
'source', ra.source,
'is_primary', ra.is_primary
))
FROM release_aliases ra
WHERE ra.release_id = r.id
) AS release_aliases,
-- Artists
(
SELECT json_agg(json_build_object(
'id', a.id,
'musicbrainz_id', a.musicbrainz_id,
'image', a.image,
'image_source', a.image_source,
'aliases', (
SELECT json_agg(json_build_object(
'alias', aa.alias,
'source', aa.source,
'is_primary', aa.is_primary
))
FROM artist_aliases aa
WHERE aa.artist_id = a.id
)
))
FROM artist_tracks at
JOIN artists a ON a.id = at.artist_id
WHERE at.track_id = t.id
) AS artists
FROM listens l
JOIN tracks t ON l.track_id = t.id
JOIN releases r ON t.release_id = r.id
WHERE l.user_id = $2::int
AND (l.listened_at, l.track_id) > ($3::timestamptz, $4::int)
ORDER BY l.listened_at, l.track_id
LIMIT $1
`
type GetListensExportPageParams struct {
Limit int32
UserID int32
ListenedAt time.Time
TrackID int32
}
type GetListensExportPageRow struct {
ListenedAt time.Time
UserID int32
Client *string
TrackID int32
TrackMbid *uuid.UUID
TrackDuration int32
TrackAliases []byte
ReleaseID int32
ReleaseMbid *uuid.UUID
ReleaseImage *uuid.UUID
ReleaseImageSource pgtype.Text
VariousArtists bool
ReleaseAliases []byte
Artists []byte
}
func (q *Queries) GetListensExportPage(ctx context.Context, arg GetListensExportPageParams) ([]GetListensExportPageRow, error) {
rows, err := q.db.Query(ctx, getListensExportPage,
arg.Limit,
arg.UserID,
arg.ListenedAt,
arg.TrackID,
)
if err != nil {
return nil, err
}
defer rows.Close()
var items []GetListensExportPageRow
for rows.Next() {
var i GetListensExportPageRow
if err := rows.Scan(
&i.ListenedAt,
&i.UserID,
&i.Client,
&i.TrackID,
&i.TrackMbid,
&i.TrackDuration,
&i.TrackAliases,
&i.ReleaseID,
&i.ReleaseMbid,
&i.ReleaseImage,
&i.ReleaseImageSource,
&i.VariousArtists,
&i.ReleaseAliases,
&i.Artists,
); err != nil {
return nil, err
}
items = append(items, i)
}
if err := rows.Err(); err != nil {
return nil, err
}
return items, nil
}
const insertListen = `-- name: InsertListen :exec const insertListen = `-- name: InsertListen :exec
INSERT INTO listens (track_id, listened_at, user_id, client) INSERT INTO listens (track_id, listened_at, user_id, client)
VALUES ($1, $2, $3, $4) VALUES ($1, $2, $3, $4)

@ -327,3 +327,11 @@ func ParseBool(s string) (value, ok bool) {
return return
} }
} }
func FlattenAliases(aliases []models.Alias) []string {
ret := make([]string, len(aliases))
for i := range aliases {
ret[i] = aliases[i].Alias
}
return ret
}

Loading…
Cancel
Save