diff --git a/db/elasticsearch.go b/db/elasticsearch.go index 0d832aa..bb7e89c 100644 --- a/db/elasticsearch.go +++ b/db/elasticsearch.go @@ -8,7 +8,6 @@ import ( var ElasticSearch *elasticsearch.Client -const elasticMapsetIndex = "mapsets-v2" const elasticMapSearchIndex = "maps" // InitializeElasticSearch Initializes the ElasticSearch client diff --git a/db/elasticsearch_mapsets.go b/db/elasticsearch_mapsets.go index a926d8c..a208f92 100644 --- a/db/elasticsearch_mapsets.go +++ b/db/elasticsearch_mapsets.go @@ -197,6 +197,7 @@ func IndexElasticSearchMapset(mapset Mapset) error { for _, mapQua := range mapset.Maps { elasticMap := ElasticMap{ MapQua: mapQua, + PackageMD5: mapset.PackageMD5, DateSubmitted: mapset.DateSubmitted, DateLastUpdated: mapset.DateLastUpdated, Explicit: mapset.IsExplicit, @@ -210,13 +211,13 @@ func IndexElasticSearchMapset(mapset Mapset) error { } resp, err := ElasticSearch.Create(elasticMapSearchIndex, - fmt.Sprintf("%v", mapQua.Id), bytes.NewReader(data)) + strconv.Itoa(mapQua.Id), bytes.NewReader(data)) if err != nil { return err } - defer resp.Body.Close() + resp.Body.Close() } return nil @@ -229,19 +230,23 @@ func UpdateElasticSearchMapset(mapset Mapset) error { for _, mapQua := range mapset.Maps { elasticMap := ElasticMap{ MapQua: mapQua, + PackageMD5: mapset.PackageMD5, DateSubmitted: mapset.DateSubmitted, DateLastUpdated: mapset.DateLastUpdated, Explicit: mapset.IsExplicit, MapsetPlayCount: mapsetPlayCount, } - data, err := json.Marshal(&elasticMap) + data, err := json.Marshal(struct { + Doc ElasticMap `json:"doc"` + DocAsUpsert bool `json:"doc_as_upsert"` + }{Doc: elasticMap, DocAsUpsert: true}) if err != nil { return err } - resp, err := ElasticSearch.Update(elasticMapsetIndex, fmt.Sprintf("%v", mapQua.Id), bytes.NewReader(data)) + resp, err := ElasticSearch.Update(elasticMapSearchIndex, strconv.Itoa(mapQua.Id), bytes.NewReader(data)) if err != nil { return err @@ -253,6 +258,28 @@ func UpdateElasticSearchMapset(mapset Mapset) error { return nil } +// UpdateElasticSearchMapDifficulty Updates only the rating of the matching uploaded map version. +func UpdateElasticSearchMapDifficulty(id int, md5 string, difficultyRating float64) error { + data, err := json.Marshal(map[string]interface{}{ + "script": map[string]interface{}{ + "source": "if (ctx._source.md5 == params.md5) { ctx._source.difficulty_rating = params.difficulty_rating } else { ctx.op = 'noop' }", + "params": map[string]interface{}{ + "md5": md5, + "difficulty_rating": difficultyRating, + }, + }, + }) + if err != nil { + return err + } + resp, err := ElasticSearch.Update(elasticMapSearchIndex, strconv.Itoa(id), bytes.NewReader(data), ElasticSearch.Update.WithRetryOnConflict(3)) + if err != nil { + return err + } + resp.Body.Close() + return nil +} + // DeleteElasticSearchMapset Deletes an individual mapset in elastic func DeleteElasticSearchMapset(id int) error { queryMap := map[string]interface{}{ diff --git a/db/maps.go b/db/maps.go index db7dc7c..3a455b7 100644 --- a/db/maps.go +++ b/db/maps.go @@ -49,13 +49,7 @@ func (m *MapQua) TableName() string { } func (m *MapQua) AfterFind(*gorm.DB) error { - m.MaxCombo = m.CountHitObjectLong*2 + m.CountHitObjectNormal - m.PlayAttempts = m.PlayCount + m.FailCount - - if m.CountHitObjectLong != 0 { - m.LongNotePercentage = float32(m.CountHitObjectLong) / - float32(m.CountHitObjectNormal+m.CountHitObjectLong) * 100 - } + m.UpdateComputedValues() if m.DateClanRanked != nil { t := time.UnixMilli(*m.DateClanRanked) @@ -65,6 +59,18 @@ func (m *MapQua) AfterFind(*gorm.DB) error { return nil } +// UpdateComputedValues Calculates display and search fields from the map's stored counts. +func (m *MapQua) UpdateComputedValues() { + m.MaxCombo = m.CountHitObjectLong*2 + m.CountHitObjectNormal + m.PlayAttempts = m.PlayCount + m.FailCount + + m.LongNotePercentage = 0 + if m.CountHitObjectNormal+m.CountHitObjectLong > 0 { + m.LongNotePercentage = float32(m.CountHitObjectLong) / + float32(m.CountHitObjectNormal+m.CountHitObjectLong) * 100 + } +} + func (m *MapQua) String() string { return fmt.Sprintf("%v - %v [%v]", m.Artist, m.Title, m.DifficultyName) } @@ -151,13 +157,16 @@ func DeleteMap(id int) error { return result.Error } -// UpdateMapDifficultyRating Updates the difficulty rating of a map -func UpdateMapDifficultyRating(id int, difficultyRating float64) error { +// UpdateMapDifficultyRating Saves and indexes a calculation only for the matching uploaded map version. +func UpdateMapDifficultyRating(id int, md5 string, difficultyRating float64) error { result := SQL.Model(&MapQua{}). - Where("id = ?", id). + Where("id = ? AND md5 = ?", id, md5). Update("difficulty_rating", difficultyRating) - return result.Error + if result.Error != nil { + return result.Error + } + return UpdateElasticSearchMapDifficulty(id, md5, difficultyRating) } // UpdateMapClanRanked Updates the clan ranked status of a map diff --git a/db/mapsets.go b/db/mapsets.go index 96819c4..a1d3fbd 100644 --- a/db/mapsets.go +++ b/db/mapsets.go @@ -186,24 +186,31 @@ func GetMapsetOnlineOffsets() (interface{}, error) { return offsets, nil } -// RankMapset Ranks all maps in a mapset -func RankMapset(id int) error { +// RankMapset Ranks all maps in SQL and synchronizes the loaded mapset for indexing. +func RankMapset(mapset *Mapset) error { result := SQL.Model(&MapQua{}). - Where("mapset_id = ?", id). + Where("mapset_id = ?", mapset.Id). Update("ranked_status", enums.RankedStatusRanked) if result.Error != nil { return result.Error } + dateLastUpdated := time.Now().UnixMilli() result = SQL.Model(&Mapset{}). - Where("id = ?", id). - Update("date_last_updated", time.Now().UnixMilli()) + Where("id = ?", mapset.Id). + Update("date_last_updated", dateLastUpdated) if result.Error != nil { return result.Error } + for _, songMap := range mapset.Maps { + songMap.RankedStatus = enums.RankedStatusRanked + } + mapset.DateLastUpdated = dateLastUpdated + mapset.DateLastUpdatedJSON = time.UnixMilli(dateLastUpdated) + return nil } @@ -249,6 +256,7 @@ func UpdateMapsetPackageMD5(id int, md5 string) error { // UpdateMetadata Updates the metadata of a given mapset (username, artist, title, etc) func (m *Mapset) UpdateMetadata() error { + dateLastUpdated := time.Now().UnixMilli() result := SQL.Model(&Mapset{}). Where("id = ?", m.Id). Updates(map[string]interface{}{ @@ -257,10 +265,15 @@ func (m *Mapset) UpdateMetadata() error { "title": m.Title, "source": m.Source, "tags": m.Tags, - "date_last_updated": time.Now().UnixMilli(), + "date_last_updated": dateLastUpdated, }) - return result.Error + if result.Error != nil { + return result.Error + } + m.DateLastUpdated = dateLastUpdated + m.DateLastUpdatedJSON = time.UnixMilli(dateLastUpdated) + return nil } // UpdateExplicit Sets the explicit state of the mapset diff --git a/handlers/mapset_submission.go b/handlers/mapset_submission.go index e5e9b6d..45c197f 100644 --- a/handlers/mapset_submission.go +++ b/handlers/mapset_submission.go @@ -13,7 +13,6 @@ import ( "github.com/Quaver/api2/qua" "github.com/Quaver/api2/sliceutil" "github.com/Quaver/api2/tools" - v1 "github.com/Quaver/api2/v1" "github.com/Quaver/api2/webhooks" "github.com/gabriel-vasile/mimetype" "github.com/gin-gonic/gin" @@ -53,6 +52,8 @@ const ( errAudioPreviewFileNoExists string = "could not create audio preview (file no exists)" ) +type mapDifficultyFiles map[*db.MapQua]string + // HandleMapsetSubmission Handles the uploading/updating of a mapset archive (.qp) file // Endpoint: POST /v2/mapset func HandleMapsetSubmission(c *gin.Context) *APIError { @@ -62,6 +63,15 @@ func HandleMapsetSubmission(c *gin.Context) *APIError { return nil } + difficultyFiles := make(mapDifficultyFiles) + defer func() { + for _, filePath := range difficultyFiles { + if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) { + logrus.Error("Error removing map difficulty file: ", filePath, err) + } + } + }() + zipReader, apiErr := checkValidRequestMapset(c, user) if apiErr != nil { @@ -94,9 +104,9 @@ func HandleMapsetSubmission(c *gin.Context) *APIError { var mapset *db.Mapset if isUploadingNewMapset { - mapset, apiErr = uploadNewMapset(user, quaFiles) + mapset, apiErr = uploadNewMapset(user, quaFiles, difficultyFiles) } else { - mapset, apiErr = updateExistingMapset(user, quaFiles) + mapset, apiErr = updateExistingMapset(user, quaFiles, difficultyFiles) } if apiErr != nil { @@ -123,8 +133,9 @@ func HandleMapsetSubmission(c *gin.Context) *APIError { return APIErrorServerError("Error updating elastic search", err) } - if err := v1.UpdateElasticSearchMapset(mapset.Id); err != nil { - logrus.Error(err) + for songMap, filePath := range difficultyFiles { + delete(difficultyFiles, songMap) + go calcMapDifficulty(songMap.Id, songMap.MD5, filePath) } if apiErr := resolveMapsetInRankingQueue(user, mapset); apiErr != nil { @@ -314,7 +325,7 @@ func validateQuaFiles(user *db.User, quaFiles map[*zip.File]*qua.Qua) *APIError } // Handles the uploading of a brand new mapset -func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset, *APIError) { +func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua, difficultyFiles mapDifficultyFiles) (*db.Mapset, *APIError) { if apiErr := checkUserUploadEligibility(user); apiErr != nil { return nil, apiErr } @@ -335,7 +346,7 @@ func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset } for _, quaFile := range quaFiles { - songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile) + songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile, difficultyFiles) if apiErr != nil { return nil, apiErr @@ -356,7 +367,7 @@ func uploadNewMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset } // Handles the updating of an existing mapset -func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.Mapset, *APIError) { +func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua, difficultyFiles mapDifficultyFiles) (*db.Mapset, *APIError) { quaSlice := sliceutil.Values(quaFiles) if !sliceutil.All(quaSlice, func(q *qua.Qua) bool { @@ -400,7 +411,7 @@ func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.M // Insert new maps & update existing ones for _, quaFile := range quaFiles { - songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile) + songMap, apiErr := InsertOrUpdateMap(user, mapset, quaFile, difficultyFiles) if apiErr != nil { return nil, apiErr @@ -440,7 +451,7 @@ func updateExistingMapset(user *db.User, quaFiles map[*zip.File]*qua.Qua) (*db.M } // InsertOrUpdateMap Inserts/Updates a map in the database -func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db.MapQua, *APIError) { +func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua, difficultyFiles mapDifficultyFiles) (*db.MapQua, *APIError) { songMap := &db.MapQua{ MapsetId: mapset.Id, CreatorId: user.Id, @@ -457,8 +468,8 @@ func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db. BPM: quaFile.CommonBPM(), CountHitObjectNormal: quaFile.CountHitObjectNormal(), CountHitObjectLong: quaFile.CountHitObjectLong(), - MaxCombo: quaFile.MaxCombo(), } + songMap.UpdateComputedValues() if quaFile.MapId != -1 { songMap.Id = quaFile.MapId @@ -475,7 +486,15 @@ func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db. return nil, APIErrorServerError("Error saving map in db", err) } - filePath := fmt.Sprintf("%v/%v.qua", files.GetTempDirectory(), songMap.Id) + tempMapFile, err := os.CreateTemp(files.GetTempDirectory(), fmt.Sprintf("%d-*.qua", songMap.Id)) + if err != nil { + return nil, APIErrorServerError("Error creating temporary map file", err) + } + filePath := tempMapFile.Name() + difficultyFiles[songMap] = filePath + if err := tempMapFile.Close(); err != nil { + return nil, APIErrorServerError("Error closing temporary map file", err) + } if err := quaFile.Write(filePath); err != nil { return nil, APIErrorServerError("Error writing .qua file to disk", err) @@ -485,14 +504,6 @@ func InsertOrUpdateMap(user *db.User, mapset *db.Mapset, quaFile *qua.Qua) (*db. return nil, APIErrorServerError("Error uploading .qua file to azure", err) } - go func() { - calcMapDifficulty(songMap, filePath) - - if err := os.Remove(filePath); err != nil { - logrus.Error("Error removing file: ", filePath) - } - }() - return songMap, nil } @@ -560,7 +571,13 @@ func getUserMaxUploadsPerMonth(user *db.User) int { } // Calculates a map's difficulty rating -func calcMapDifficulty(songMap *db.MapQua, filePath string) { +func calcMapDifficulty(id int, md5 string, filePath string) { + defer func() { + if err := os.Remove(filePath); err != nil && !os.IsNotExist(err) { + logrus.Error("Error removing map difficulty file: ", filePath, err) + } + }() + calc, err := tools.RunDifficultyCalculator(filePath, 0) if err != nil { @@ -568,8 +585,8 @@ func calcMapDifficulty(songMap *db.MapQua, filePath string) { return } - if err := db.UpdateMapDifficultyRating(songMap.Id, calc.Difficulty.OverallDifficulty); err != nil { - logrus.Error("Error updating map difficulty rating in DB: ", err) + if err := db.UpdateMapDifficultyRating(id, md5, calc.Difficulty.OverallDifficulty); err != nil { + logrus.Errorf("Error saving difficulty rating for map %d: %v", id, err) return } } diff --git a/handlers/ranking_actions.go b/handlers/ranking_actions.go index 566cfc2..a34fd81 100644 --- a/handlers/ranking_actions.go +++ b/handlers/ranking_actions.go @@ -4,10 +4,8 @@ import ( "github.com/Quaver/api2/config" "github.com/Quaver/api2/db" "github.com/Quaver/api2/enums" - v1 "github.com/Quaver/api2/v1" "github.com/Quaver/api2/webhooks" "github.com/gin-gonic/gin" - "github.com/sirupsen/logrus" "gorm.io/gorm" "net/http" "strconv" @@ -144,7 +142,7 @@ func VoteForRankingQueueMapset(c *gin.Context) *APIError { return APIErrorServerError("Error updating ranking queue status mapset status", err) } - if err := db.RankMapset(data.MapsetId); err != nil { + if err := db.RankMapset(data.QueueMapset.Mapset); err != nil { return APIErrorServerError("Failed to rank mapset", err) } @@ -165,10 +163,6 @@ func VoteForRankingQueueMapset(c *gin.Context) *APIError { return APIErrorServerError("Error inserting ranked mapset notification", err) } - if err := v1.UpdateElasticSearchMapset(data.QueueMapset.MapsetId); err != nil { - logrus.Warn("Error updating v1 elastic search for mapset: ", data.QueueMapset.MapsetId, err) - } - _ = webhooks.SendRankedWebhook(data.QueueMapset.Mapset, existingVotes) }