mirror of https://github.com/gabehf/Koito.git
parent
1d02cede49
commit
7b0cff0a07
@ -0,0 +1,59 @@
|
||||
package psql
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
|
||||
"github.com/gabehf/koito/internal/db"
|
||||
"github.com/gabehf/koito/internal/models"
|
||||
"github.com/gabehf/koito/internal/repository"
|
||||
)
|
||||
|
||||
func (d *Psql) GetExportPage(ctx context.Context, opts db.GetExportPageOpts) ([]*db.ExportItem, error) {
|
||||
rows, err := d.q.GetListensExportPage(ctx, repository.GetListensExportPageParams{
|
||||
UserID: opts.UserID,
|
||||
TrackID: opts.TrackID,
|
||||
Limit: opts.Limit,
|
||||
ListenedAt: opts.ListenedAt,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetExportPage: %w", err)
|
||||
}
|
||||
ret := make([]*db.ExportItem, len(rows))
|
||||
for i, row := range rows {
|
||||
|
||||
var trackAliases []models.Alias
|
||||
err = json.Unmarshal(row.TrackAliases, &trackAliases)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetExportPage: json.Unmarshal trackAliases: %w", err)
|
||||
}
|
||||
var albumAliases []models.Alias
|
||||
err = json.Unmarshal(row.ReleaseAliases, &albumAliases)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetExportPage: json.Unmarshal albumAliases: %w", err)
|
||||
}
|
||||
var artists []models.ArtistWithFullAliases
|
||||
err = json.Unmarshal(row.Artists, &artists)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("GetExportPage: json.Unmarshal artists: %w", err)
|
||||
}
|
||||
|
||||
ret[i] = &db.ExportItem{
|
||||
TrackID: row.TrackID,
|
||||
ListenedAt: row.ListenedAt,
|
||||
UserID: row.UserID,
|
||||
Client: row.Client,
|
||||
TrackMbid: row.TrackMbid,
|
||||
TrackDuration: row.TrackDuration,
|
||||
TrackAliases: trackAliases,
|
||||
ReleaseID: row.ReleaseID,
|
||||
ReleaseMbid: row.ReleaseMbid,
|
||||
ReleaseImageSource: row.ReleaseImageSource.String,
|
||||
VariousArtists: row.VariousArtists,
|
||||
ReleaseAliases: albumAliases,
|
||||
Artists: artists,
|
||||
}
|
||||
}
|
||||
return ret, nil
|
||||
}
|
||||
@ -0,0 +1,147 @@
|
||||
package export
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"time"
|
||||
|
||||
"github.com/gabehf/koito/internal/cfg"
|
||||
"github.com/gabehf/koito/internal/db"
|
||||
"github.com/gabehf/koito/internal/logger"
|
||||
"github.com/gabehf/koito/internal/models"
|
||||
"github.com/google/uuid"
|
||||
)
|
||||
|
||||
type KoitoExport struct {
|
||||
Version string `json:"version"`
|
||||
ExportedAt time.Time `json:"exported_at"` // RFC3339
|
||||
User string `json:"user"` // username
|
||||
Listens []KoitoListen `json:"listens"`
|
||||
}
|
||||
type KoitoListen struct {
|
||||
ListenedAt time.Time `json:"listened_at"`
|
||||
Track KoitoTrack `json:"track"`
|
||||
Album KoitoAlbum `json:"album"`
|
||||
Artists []KoitoArtist `json:"artists"`
|
||||
}
|
||||
type KoitoTrack struct {
|
||||
MBID *uuid.UUID `json:"mbid"`
|
||||
Duration int `json:"duration"`
|
||||
Aliases []models.Alias `json:"aliases"`
|
||||
}
|
||||
type KoitoAlbum struct {
|
||||
ImageUrl string `json:"image_url"`
|
||||
MBID *uuid.UUID `json:"mbid"`
|
||||
Aliases []models.Alias `json:"aliases"`
|
||||
VariousArtists bool `json:"various_artists"`
|
||||
}
|
||||
type KoitoArtist struct {
|
||||
ImageUrl string `json:"image_url"`
|
||||
MBID *uuid.UUID `json:"mbid"`
|
||||
IsPrimary bool `json:"is_primary"`
|
||||
Aliases []models.Alias `json:"aliases"`
|
||||
}
|
||||
|
||||
func ExportData(ctx context.Context, username string, store db.DB) error {
|
||||
lastTime := time.Unix(0, 0)
|
||||
lastTrackId := int32(0)
|
||||
pageSize := int32(1000)
|
||||
|
||||
l := logger.FromContext(ctx)
|
||||
l.Info().Msg("ExportData: Generating Koito export file...")
|
||||
|
||||
exportedAt := time.Now()
|
||||
exportFile := path.Join(cfg.ConfigDir(), fmt.Sprintf("koito_export_%d.json", exportedAt.Unix()))
|
||||
f, err := os.Create(exportFile)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ExportData: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
|
||||
// Write the opening of the JSON manually
|
||||
_, err = fmt.Fprintf(f, "{\n \"version\": \"1\",\n \"exported_at\": \"%s\",\n \"user\": \"%s\",\n \"listens\": [\n", exportedAt.UTC().Format(time.RFC3339), username)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ExportData: %w", err)
|
||||
}
|
||||
|
||||
first := true
|
||||
for {
|
||||
rows, err := store.GetExportPage(ctx, db.GetExportPageOpts{
|
||||
UserID: 1,
|
||||
ListenedAt: lastTime,
|
||||
TrackID: lastTrackId,
|
||||
Limit: pageSize,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ExportData: %w", err)
|
||||
}
|
||||
if len(rows) == 0 {
|
||||
break
|
||||
}
|
||||
|
||||
for _, r := range rows {
|
||||
// Adds a comma after each listen item
|
||||
if !first {
|
||||
_, _ = f.Write([]byte(",\n"))
|
||||
}
|
||||
first = false
|
||||
|
||||
exported := convertToExportFormat(r)
|
||||
|
||||
raw, err := json.MarshalIndent(exported, " ", " ")
|
||||
|
||||
// needed to make the listen item start at the right indent level
|
||||
f.Write([]byte(" "))
|
||||
|
||||
if err != nil {
|
||||
return fmt.Errorf("ExportData: marshal: %w", err)
|
||||
}
|
||||
_, _ = f.Write(raw)
|
||||
|
||||
if r.TrackID > lastTrackId {
|
||||
lastTrackId = r.TrackID
|
||||
}
|
||||
if r.ListenedAt.After(lastTime) {
|
||||
lastTime = r.ListenedAt
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Write closing of the JSON array and object
|
||||
_, err = f.Write([]byte("\n ]\n}\n"))
|
||||
if err != nil {
|
||||
return fmt.Errorf("ExportData: f.Write: %w", err)
|
||||
}
|
||||
|
||||
l.Info().Msgf("Export successful! File can be found at %s", exportFile)
|
||||
return nil
|
||||
}
|
||||
|
||||
func convertToExportFormat(item *db.ExportItem) *KoitoListen {
|
||||
ret := &KoitoListen{
|
||||
ListenedAt: item.ListenedAt.UTC(),
|
||||
Track: KoitoTrack{
|
||||
MBID: item.TrackMbid,
|
||||
Duration: int(item.TrackDuration),
|
||||
Aliases: item.TrackAliases,
|
||||
},
|
||||
Album: KoitoAlbum{
|
||||
MBID: item.ReleaseMbid,
|
||||
ImageUrl: item.ReleaseImageSource,
|
||||
VariousArtists: item.VariousArtists,
|
||||
Aliases: item.ReleaseAliases,
|
||||
},
|
||||
}
|
||||
for i := range item.Artists {
|
||||
ret.Artists = append(ret.Artists, KoitoArtist{
|
||||
IsPrimary: item.Artists[i].IsPrimary,
|
||||
MBID: item.Artists[i].MbzID,
|
||||
Aliases: item.Artists[i].Aliases,
|
||||
ImageUrl: item.Artists[i].ImageSource,
|
||||
})
|
||||
}
|
||||
return ret
|
||||
}
|
||||
@ -0,0 +1,172 @@
|
||||
package importer
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"os"
|
||||
"path"
|
||||
"strings"
|
||||
|
||||
"github.com/gabehf/koito/internal/cfg"
|
||||
"github.com/gabehf/koito/internal/db"
|
||||
"github.com/gabehf/koito/internal/export"
|
||||
"github.com/gabehf/koito/internal/logger"
|
||||
"github.com/gabehf/koito/internal/models"
|
||||
"github.com/gabehf/koito/internal/utils"
|
||||
"github.com/google/uuid"
|
||||
"github.com/jackc/pgx/v5"
|
||||
)
|
||||
|
||||
func ImportKoitoFile(ctx context.Context, store db.DB, filename string) error {
|
||||
l := logger.FromContext(ctx)
|
||||
l.Info().Msgf("Beginning Koito import on file: %s", filename)
|
||||
data := new(export.KoitoExport)
|
||||
f, err := os.Open(path.Join(cfg.ConfigDir(), "import", filename))
|
||||
if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: os.Open: %w", err)
|
||||
}
|
||||
defer f.Close()
|
||||
err = json.NewDecoder(f).Decode(data)
|
||||
if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: Decode: %w", err)
|
||||
}
|
||||
|
||||
if data.Version != "1" {
|
||||
return fmt.Errorf("ImportKoitoFile: unupported version: %s", data.Version)
|
||||
}
|
||||
|
||||
l.Info().Msgf("Beginning data import for user: %s", data.User)
|
||||
|
||||
count := 0
|
||||
|
||||
for i := range data.Listens {
|
||||
// use this for save/get mbid for all artist/album/track
|
||||
mbid := uuid.Nil
|
||||
|
||||
artistIds := make([]int32, 0)
|
||||
for _, ia := range data.Listens[i].Artists {
|
||||
if ia.MBID != nil {
|
||||
mbid = *ia.MBID
|
||||
}
|
||||
artist, err := store.GetArtist(ctx, db.GetArtistOpts{
|
||||
MusicBrainzID: mbid,
|
||||
Name: getPrimaryAliasFromAliasSlice(ia.Aliases),
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
var imgid = uuid.Nil
|
||||
// not a perfect way to check if the image url is an actual source vs manual upload but
|
||||
// im like 99% sure it will work perfectly
|
||||
if strings.HasPrefix(ia.ImageUrl, "http") {
|
||||
imgid = uuid.New()
|
||||
}
|
||||
// save artist
|
||||
artist, err := store.SaveArtist(ctx, db.SaveArtistOpts{
|
||||
Name: getPrimaryAliasFromAliasSlice(ia.Aliases),
|
||||
Image: imgid,
|
||||
ImageSrc: ia.ImageUrl,
|
||||
MusicBrainzID: mbid,
|
||||
Aliases: utils.FlattenAliases(ia.Aliases),
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
}
|
||||
artistIds = append(artistIds, artist.ID)
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
} else {
|
||||
artistIds = append(artistIds, artist.ID)
|
||||
}
|
||||
}
|
||||
// call associate album
|
||||
albumId := int32(0)
|
||||
if data.Listens[i].Album.MBID != nil {
|
||||
mbid = *data.Listens[i].Album.MBID
|
||||
}
|
||||
album, err := store.GetAlbum(ctx, db.GetAlbumOpts{
|
||||
MusicBrainzID: mbid,
|
||||
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Album.Aliases),
|
||||
ArtistID: artistIds[0],
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
var imgid = uuid.Nil
|
||||
// not a perfect way to check if the image url is an actual source vs manual upload but
|
||||
// im like 99% sure it will work perfectly
|
||||
if strings.HasPrefix(data.Listens[i].Album.ImageUrl, "http") {
|
||||
imgid = uuid.New()
|
||||
}
|
||||
// save album
|
||||
album, err = store.SaveAlbum(ctx, db.SaveAlbumOpts{
|
||||
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Album.Aliases),
|
||||
Image: imgid,
|
||||
ImageSrc: data.Listens[i].Album.ImageUrl,
|
||||
MusicBrainzID: mbid,
|
||||
Aliases: utils.FlattenAliases(data.Listens[i].Album.Aliases),
|
||||
ArtistIDs: artistIds,
|
||||
VariousArtists: data.Listens[i].Album.VariousArtists,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
}
|
||||
albumId = album.ID
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
} else {
|
||||
albumId = album.ID
|
||||
}
|
||||
|
||||
// call associate track
|
||||
if data.Listens[i].Track.MBID != nil {
|
||||
mbid = *data.Listens[i].Track.MBID
|
||||
}
|
||||
track, err := store.GetTrack(ctx, db.GetTrackOpts{
|
||||
MusicBrainzID: mbid,
|
||||
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Track.Aliases),
|
||||
ArtistIDs: artistIds,
|
||||
})
|
||||
if errors.Is(err, pgx.ErrNoRows) {
|
||||
// save track
|
||||
track, err = store.SaveTrack(ctx, db.SaveTrackOpts{
|
||||
Title: getPrimaryAliasFromAliasSlice(data.Listens[i].Track.Aliases),
|
||||
RecordingMbzID: mbid,
|
||||
Duration: int32(data.Listens[i].Track.Duration),
|
||||
ArtistIDs: artistIds,
|
||||
AlbumID: albumId,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
}
|
||||
// save track aliases
|
||||
err = store.SaveTrackAliases(ctx, track.ID, utils.FlattenAliases(data.Listens[i].Track.Aliases), "Import")
|
||||
if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
}
|
||||
} else if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
}
|
||||
|
||||
// save listen
|
||||
err = store.SaveListen(ctx, db.SaveListenOpts{
|
||||
TrackID: track.ID,
|
||||
Time: data.Listens[i].ListenedAt,
|
||||
UserID: 1,
|
||||
})
|
||||
if err != nil {
|
||||
return fmt.Errorf("ImportKoitoFile: %w", err)
|
||||
}
|
||||
|
||||
l.Info().Msgf("ImportKoitoFile: Imported listen for track %s", track.Title)
|
||||
count++
|
||||
}
|
||||
|
||||
return finishImport(ctx, filename, count)
|
||||
}
|
||||
func getPrimaryAliasFromAliasSlice(aliases []models.Alias) string {
|
||||
for _, a := range aliases {
|
||||
if a.Primary {
|
||||
return a.Alias
|
||||
}
|
||||
}
|
||||
return ""
|
||||
}
|
||||
Loading…
Reference in new issue