From 2d0e28ba7d2532c3c5ad1809de39d19024879dd5 Mon Sep 17 00:00:00 2001 From: jeffyanta Date: Tue, 16 Jun 2026 14:53:41 -0400 Subject: [PATCH 1/5] Implement DynamoDB currency stores --- ocp/data/currency/exchange/dynamodb/store.go | 432 +++++++++++++++ .../currency/exchange/dynamodb/store_test.go | 151 +++++ ocp/data/currency/exchange/dynamodb/table.go | 87 +++ ocp/data/currency/exchange/memory/store.go | 152 ++++++ .../currency/exchange/memory/store_test.go | 15 + ocp/data/currency/exchange/store.go | 46 ++ ocp/data/currency/exchange/tests/tests.go | 143 +++++ ocp/data/currency/holder/dynamodb/store.go | 514 ++++++++++++++++++ .../currency/holder/dynamodb/store_test.go | 106 ++++ ocp/data/currency/holder/dynamodb/table.go | 106 ++++ ocp/data/currency/holder/memory/store.go | 198 +++++++ ocp/data/currency/holder/memory/store_test.go | 15 + ocp/data/currency/holder/store.go | 76 +++ ocp/data/currency/holder/tests/tests.go | 256 +++++++++ ocp/data/currency/reserve/dynamodb/store.go | 448 +++++++++++++++ .../currency/reserve/dynamodb/store_test.go | 106 ++++ ocp/data/currency/reserve/dynamodb/table.go | 106 ++++ ocp/data/currency/reserve/memory/store.go | 169 ++++++ .../currency/reserve/memory/store_test.go | 15 + ocp/data/currency/reserve/store.go | 65 +++ ocp/data/currency/reserve/tests/tests.go | 231 ++++++++ ocp/rpc/currency/historical_data.go | 4 +- 22 files changed, 3439 insertions(+), 2 deletions(-) create mode 100644 ocp/data/currency/exchange/dynamodb/store.go create mode 100644 ocp/data/currency/exchange/dynamodb/store_test.go create mode 100644 ocp/data/currency/exchange/dynamodb/table.go create mode 100644 ocp/data/currency/exchange/memory/store.go create mode 100644 ocp/data/currency/exchange/memory/store_test.go create mode 100644 ocp/data/currency/exchange/store.go create mode 100644 ocp/data/currency/exchange/tests/tests.go create mode 100644 ocp/data/currency/holder/dynamodb/store.go create mode 100644 ocp/data/currency/holder/dynamodb/store_test.go create mode 100644 ocp/data/currency/holder/dynamodb/table.go create mode 100644 ocp/data/currency/holder/memory/store.go create mode 100644 ocp/data/currency/holder/memory/store_test.go create mode 100644 ocp/data/currency/holder/store.go create mode 100644 ocp/data/currency/holder/tests/tests.go create mode 100644 ocp/data/currency/reserve/dynamodb/store.go create mode 100644 ocp/data/currency/reserve/dynamodb/store_test.go create mode 100644 ocp/data/currency/reserve/dynamodb/table.go create mode 100644 ocp/data/currency/reserve/memory/store.go create mode 100644 ocp/data/currency/reserve/memory/store_test.go create mode 100644 ocp/data/currency/reserve/store.go create mode 100644 ocp/data/currency/reserve/tests/tests.go diff --git a/ocp/data/currency/exchange/dynamodb/store.go b/ocp/data/currency/exchange/dynamodb/store.go new file mode 100644 index 0000000..658a5d9 --- /dev/null +++ b/ocp/data/currency/exchange/dynamodb/store.go @@ -0,0 +1,432 @@ +// Package dynamodb implements the exchange.Store interface on top of DynamoDB. +// +// Data model (single table, no secondary indexes). Items are grouped into one +// partition per time resolution: +// +// pk = "rates#raw" sk = rates (every sample) +// pk = "rates#hour" sk = ts, rates (close per hour) +// pk = "rates#day" sk = ts, rates (close per day) +// pk = "rates#week" sk = ts, rates (close per week, Sunday-start) +// pk = "rates#month" sk = ts, rates (close per month) +// +// Every item carries the full symbol->rate map for that instant; rollup items +// hold the map from the most recent sample in their bucket (the "close"). The +// sort key is the bucket-start time as unix-nanos (a Number), which sorts in +// chronological order. Rollup items also store `ts`, the close +// sample's actual time, which both drives the monotonic last-write-wins guard +// and is the timestamp returned for the point. Raw items omit `ts` because their +// sort key already is the sample time. +// +// Writes fan out to all resolutions in a single transaction: the raw point is a +// conditional Put that enforces the once-per-timestamp contract, and each rollup +// is a monotonic last-write-wins Update. Reads serve the resolution matching the +// requested interval (see resolutionForInterval): the caller chooses an interval +// appropriate to the range and the store honors it directly. Point lookups +// always read the raw resolution. +package dynamodb + +import ( + "context" + "errors" + "fmt" + "strconv" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" +) + +const ( + attrPK = "pk" + attrSK = "sk" + attrTS = "ts" + attrRates = "rates" + + resRaw = "raw" + resHour = "hour" + resDay = "day" + resWeek = "week" + resMonth = "month" + + partitionPrefix = "rates#" + rawPK = partitionPrefix + resRaw + + // codeConditionalCheckFailed is the DynamoDB cancellation reason code for a + // transaction item whose ConditionExpression evaluated false. + codeConditionalCheckFailed = "ConditionalCheckFailed" +) + +// rollupResolutions are the coarse resolutions maintained alongside the raw +// points on every write. +var rollupResolutions = []string{resHour, resDay, resWeek, resMonth} + +type store struct { + client *dynamodb.Client + table string +} + +// New returns an exchange.Store backed by the given DynamoDB table. +// Use CreateTables to provision it. +func New(client *dynamodb.Client, table string) exchange.Store { + return &store{ + client: client, + table: table, + } +} + +// PutExchangeRates writes the raw sample and refreshes every rollup bucket that +// contains it, atomically, in a single transaction. The raw item's +// attribute_not_exists condition enforces the once-per-timestamp contract: a +// duplicate timestamp cancels the transaction and returns currency.ErrExists. +// +// The rollup updates carry a monotonic guard (only advance a bucket to a newer +// sample). Because a transaction is all-or-nothing, this assumes monotonically +// increasing write timestamps — the case for the periodic rate worker. An +// out-of-order (older) sample would fail a rollup guard and cancel the whole +// transaction, surfacing as an error rather than partially applying. +func (s *store) PutExchangeRates(ctx context.Context, record *currency.MultiRateRecord) error { + if len(record.Rates) == 0 { + return nil + } + + rates := make(map[string]types.AttributeValue, len(record.Rates)) + for symbol, rate := range record.Rates { + rates[symbol] = avF(rate) + } + ratesAttr := &types.AttributeValueMemberM{Value: rates} + ts := avN(record.Time.UTC().UnixNano()) + + // The raw item is first so that a duplicate timestamp is identifiable as the + // zeroth cancellation reason. + transactItems := []types.TransactWriteItem{ + {Put: &types.Put{ + TableName: aws.String(s.table), + Item: map[string]types.AttributeValue{ + attrPK: avS(rawPK), + attrSK: skN(record.Time), + attrRates: ratesAttr, + }, + ConditionExpression: aws.String(fmt.Sprintf("attribute_not_exists(%s)", attrPK)), + }}, + } + for _, res := range rollupResolutions { + transactItems = append(transactItems, types.TransactWriteItem{ + Update: &types.Update{ + TableName: aws.String(s.table), + Key: map[string]types.AttributeValue{ + attrPK: avS(partition(res)), + attrSK: skN(bucketStart(record.Time, res)), + }, + UpdateExpression: aws.String("SET #r = :rates, #ts = :ts"), + ConditionExpression: aws.String("attribute_not_exists(#pk) OR #ts < :ts"), + ExpressionAttributeNames: map[string]string{ + "#pk": attrPK, + "#r": attrRates, + "#ts": attrTS, + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":rates": ratesAttr, + ":ts": ts, + }, + }, + }) + } + + _, err := s.client.TransactWriteItems(ctx, &dynamodb.TransactWriteItemsInput{ + TransactItems: transactItems, + }) + if err != nil { + // A cancellation whose raw item (index 0) failed its condition means a + // record already exists for this timestamp. + var tce *types.TransactionCanceledException + if errors.As(err, &tce) && len(tce.CancellationReasons) > 0 && + aws.ToString(tce.CancellationReasons[0].Code) == codeConditionalCheckFailed { + return currency.ErrExists + } + return err + } + return nil +} + +func (s *store) GetExchangeRate(ctx context.Context, symbol string, t time.Time) (*currency.ExchangeRateRecord, error) { + // Only this symbol's rate is needed, so project it out of the full rate map. + projection, names := symbolProjection(symbol) + item, err := s.latestItemAtOrBefore(ctx, t, projection, names) + if err != nil { + return nil, err + } + if item == nil { + return nil, currency.ErrNotFound + } + + rate, ok, err := rateForSymbol(item, symbol) + if err != nil { + return nil, err + } + if !ok { + return nil, currency.ErrNotFound + } + return rate, nil +} + +func (s *store) GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) { + // The whole rate map is returned, so no projection. + item, err := s.latestItemAtOrBefore(ctx, t, "", nil) + if err != nil { + return nil, err + } + if item == nil { + return nil, currency.ErrNotFound + } + + ts, err := itemTime(item) + if err != nil { + return nil, err + } + res := ¤cy.MultiRateRecord{ + Time: ts, + Rates: make(map[string]float64), + } + for symbol, av := range asM(item[attrRates]) { + rate, err := parseF(av) + if err != nil { + return nil, err + } + res.Rates[symbol] = rate + } + return res, nil +} + +func (s *store) GetExchangeRatesInRange(ctx context.Context, symbol string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ExchangeRateRecord, error) { + if interval > query.IntervalMonth { + return nil, currency.ErrInvalidInterval + } + if start.IsZero() || end.IsZero() { + return nil, currency.ErrInvalidRange + } + + actualStart, actualEnd := start, end + if start.Unix() > end.Unix() { + actualStart, actualEnd = end, start + } + + // Honor the requested interval directly by reading the matching stored + // resolution. The data is pre-aggregated, so no in-app downsampling is needed. + res := resolutionForInterval(interval) + projection, names := symbolProjection(symbol) + items, err := s.queryAll(ctx, &dynamodb.QueryInput{ + TableName: aws.String(s.table), + KeyConditionExpression: aws.String(fmt.Sprintf("%s = :pk AND %s BETWEEN :start AND :end", attrPK, attrSK)), + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":pk": avS(partition(res)), + // Lower-bound on the bucket containing actualStart so a bucket whose + // start precedes actualStart is still returned. + ":start": skN(bucketStart(actualStart, res)), + ":end": skN(actualEnd), + }, + ProjectionExpression: aws.String(projection), + ExpressionAttributeNames: names, + ScanIndexForward: aws.Bool(ordering != query.Descending), + }) + if err != nil { + return nil, err + } + + records := make([]*currency.ExchangeRateRecord, 0, len(items)) + for _, item := range items { + rate, ok, err := rateForSymbol(item, symbol) + if err != nil { + return nil, err + } + if ok { + records = append(records, rate) + } + } + + if len(records) == 0 { + return nil, currency.ErrNotFound + } + return records, nil +} + +// reset deletes every item from the table, for tests. +func (s *store) reset() { + if err := clearTable(context.Background(), s.client, s.table); err != nil { + panic(err) + } +} + +// latestItemAtOrBefore returns the most recent raw item at or before t, or nil +// if none exists. A non-empty projection (with its attribute-name aliases) +// limits which attributes are returned. +func (s *store) latestItemAtOrBefore(ctx context.Context, t time.Time, projection string, names map[string]string) (map[string]types.AttributeValue, error) { + input := &dynamodb.QueryInput{ + TableName: aws.String(s.table), + KeyConditionExpression: aws.String(fmt.Sprintf("%s = :pk AND %s <= :sk", attrPK, attrSK)), + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":pk": avS(rawPK), + ":sk": skN(t), + }, + ScanIndexForward: aws.Bool(false), + Limit: aws.Int32(1), + } + if projection != "" { + input.ProjectionExpression = aws.String(projection) + input.ExpressionAttributeNames = names + } + + out, err := s.client.Query(ctx, input) + if err != nil { + return nil, err + } + if len(out.Items) == 0 { + return nil, nil + } + return out.Items[0], nil +} + +// symbolProjection builds a ProjectionExpression (and its attribute-name +// aliases) that fetches only the given symbol's rate from the rates map, plus +// ts and optionally date — avoiding deserialization of the full rate map when +// only one symbol is needed. +func symbolProjection(symbol string) (string, map[string]string) { + // #sk is projected so the sample time can be recovered from the sort key for + // raw items, which omit #ts. + names := map[string]string{"#sk": attrSK, "#ts": attrTS, "#r": attrRates, "#s": symbol} + return "#sk, #ts, #r.#s", names +} + +// queryAll runs the query, following LastEvaluatedKey until the result set is +// drained, and returns every matched item. +func (s *store) queryAll(ctx context.Context, input *dynamodb.QueryInput) ([]map[string]types.AttributeValue, error) { + var items []map[string]types.AttributeValue + for { + out, err := s.client.Query(ctx, input) + if err != nil { + return nil, err + } + items = append(items, out.Items...) + if len(out.LastEvaluatedKey) == 0 { + break + } + input.ExclusiveStartKey = out.LastEvaluatedKey + } + return items, nil +} + +// resolutionForInterval maps a requested interval to the stored resolution that +// serves it. Sub-hour intervals are served from raw, the finest stored +// resolution. The caller is responsible for choosing an interval appropriate to +// the range; this store honors it directly rather than re-deriving one. +func resolutionForInterval(interval query.Interval) string { + switch interval { + case query.IntervalHour: + return resHour + case query.IntervalDay: + return resDay + case query.IntervalWeek: + return resWeek + case query.IntervalMonth: + return resMonth + default: // raw, second, minute + return resRaw + } +} + +// rateForSymbol builds the per-symbol record from an item's rate map, reporting +// whether the symbol is present. +func rateForSymbol(item map[string]types.AttributeValue, symbol string) (*currency.ExchangeRateRecord, bool, error) { + av, ok := asM(item[attrRates])[symbol] + if !ok { + return nil, false, nil + } + rate, err := parseF(av) + if err != nil { + return nil, false, err + } + ts, err := itemTime(item) + if err != nil { + return nil, false, err + } + return ¤cy.ExchangeRateRecord{ + Time: ts, + Symbol: symbol, + Rate: rate, + }, true, nil +} + +// itemTime returns the point's timestamp: the close sample time from `ts` for +// rollup items, or the sample time from the sort key for raw items (which omit +// `ts`). +func itemTime(item map[string]types.AttributeValue) (time.Time, error) { + av, ok := item[attrTS] + if !ok { + av = item[attrSK] + } + nanos, err := parseN(av) + if err != nil { + return time.Time{}, err + } + return time.Unix(0, nanos).UTC(), nil +} + +// bucketStart truncates t to the start of its bucket for the given resolution, +// in UTC. Weeks start on Sunday. +func bucketStart(t time.Time, res string) time.Time { + t = t.UTC() + switch res { + case resHour: + return t.Truncate(time.Hour) + case resDay: + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + case resWeek: + day := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + return day.AddDate(0, 0, -int(day.Weekday())) // time.Weekday: Sunday=0 + case resMonth: + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) + default: // resRaw + return t + } +} + +func partition(res string) string { return partitionPrefix + res } + +// skN encodes t's unix-nanos as the numeric sort key, which sorts in +// chronological order. +func skN(t time.Time) types.AttributeValue { return avN(t.UTC().UnixNano()) } + +func avS(v string) types.AttributeValue { return &types.AttributeValueMemberS{Value: v} } +func avN(v int64) types.AttributeValue { + return &types.AttributeValueMemberN{Value: strconv.FormatInt(v, 10)} +} +func avF(v float64) types.AttributeValue { + return &types.AttributeValueMemberN{Value: strconv.FormatFloat(v, 'g', -1, 64)} +} + +func asM(av types.AttributeValue) map[string]types.AttributeValue { + if m, ok := av.(*types.AttributeValueMemberM); ok { + return m.Value + } + return nil +} + +func parseN(av types.AttributeValue) (int64, error) { + n, ok := av.(*types.AttributeValueMemberN) + if !ok { + return 0, fmt.Errorf("expected number attribute, got %T", av) + } + return strconv.ParseInt(n.Value, 10, 64) +} + +func parseF(av types.AttributeValue) (float64, error) { + n, ok := av.(*types.AttributeValueMemberN) + if !ok { + return 0, fmt.Errorf("expected number attribute, got %T", av) + } + return strconv.ParseFloat(n.Value, 64) +} diff --git a/ocp/data/currency/exchange/dynamodb/store_test.go b/ocp/data/currency/exchange/dynamodb/store_test.go new file mode 100644 index 0000000..717a4f0 --- /dev/null +++ b/ocp/data/currency/exchange/dynamodb/store_test.go @@ -0,0 +1,151 @@ +package dynamodb + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + dynamotest "github.com/code-payments/ocp-server/database/dynamodb/test" + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange/tests" +) + +const exchangeRateTable = "exchange_rate_test" + +var testEnv *dynamotest.TestEnv + +func TestMain(m *testing.M) { + log := zap.Must(zap.NewDevelopment()) + + env, err := dynamotest.NewTestEnv() + if err != nil { + log.With(zap.Error(err)).Error("Error creating dynamodb test environment") + os.Exit(1) + } + + testEnv = env + + os.Exit(m.Run()) +} + +func newTestStore(t *testing.T) *store { + require.NoError(t, CreateTables(context.Background(), testEnv.Client, exchangeRateTable)) + s := New(testEnv.Client, exchangeRateTable).(*store) + s.reset() + return s +} + +func TestExchange_DynamoDBStore(t *testing.T) { + testStore := newTestStore(t) + teardown := func() { + testStore.reset() + } + tests.RunStoreTests(t, testStore, teardown) +} + +// TestExchange_DynamoDBRollups verifies that a coarse interval reads from the +// matching rollup partition and that each bucket holds the close (the most +// recent sample within the bucket). +func TestExchange_DynamoDBRollups(t *testing.T) { + s := newTestStore(t) + defer s.reset() + ctx := context.Background() + + // Two samples in hour 10 (close = 2.0 at 10:45), one in hour 11. + samples := []struct { + at time.Time + rate float64 + }{ + {time.Date(2021, 06, 01, 10, 05, 0, 0, time.UTC), 1.0}, + {time.Date(2021, 06, 01, 10, 45, 0, 0, time.UTC), 2.0}, + {time.Date(2021, 06, 01, 11, 30, 0, 0, time.UTC), 3.0}, + } + for _, sample := range samples { + require.NoError(t, s.PutExchangeRates(ctx, ¤cy.MultiRateRecord{ + Time: sample.at, + Rates: map[string]float64{"usd": sample.rate}, + })) + } + + start := time.Date(2021, 06, 01, 10, 0, 0, 0, time.UTC) + end := time.Date(2021, 06, 01, 12, 0, 0, 0, time.UTC) + + // Hourly buckets: hour 10 -> close 2.0 @ 10:45, hour 11 -> 3.0 @ 11:30. + hourly, err := s.GetExchangeRatesInRange(ctx, "usd", query.IntervalHour, start, end, query.Ascending) + require.NoError(t, err) + require.Len(t, hourly, 2) + assert.Equal(t, samples[1].at.Unix(), hourly[0].Time.Unix()) + assert.EqualValues(t, 2.0, hourly[0].Rate) + assert.Equal(t, samples[2].at.Unix(), hourly[1].Time.Unix()) + assert.EqualValues(t, 3.0, hourly[1].Rate) + + // The raw partition still holds every sample. + raw, err := s.GetExchangeRatesInRange(ctx, "usd", query.IntervalRaw, start, end, query.Ascending) + require.NoError(t, err) + assert.Len(t, raw, len(samples)) +} + +// TestExchange_DynamoDBResolutionForInterval verifies the requested interval is +// honored directly, with sub-hour intervals served from raw. +func TestExchange_DynamoDBResolutionForInterval(t *testing.T) { + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalRaw)) + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalSecond)) + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalMinute)) + assert.Equal(t, resHour, resolutionForInterval(query.IntervalHour)) + assert.Equal(t, resDay, resolutionForInterval(query.IntervalDay)) + assert.Equal(t, resWeek, resolutionForInterval(query.IntervalWeek)) + assert.Equal(t, resMonth, resolutionForInterval(query.IntervalMonth)) +} + +// TestExchange_DynamoDBSymbolProjection verifies single-symbol reads return the +// requested symbol's rate and treat an absent symbol as not found. +func TestExchange_DynamoDBSymbolProjection(t *testing.T) { + s := newTestStore(t) + defer s.reset() + ctx := context.Background() + + now := time.Date(2022, 01, 02, 12, 0, 0, 0, time.UTC) + require.NoError(t, s.PutExchangeRates(ctx, ¤cy.MultiRateRecord{ + Time: now, + Rates: map[string]float64{"usd": 1.5, "cad": 2.5}, + })) + + // A present symbol returns its own rate. + rec, err := s.GetExchangeRate(ctx, "cad", now) + require.NoError(t, err) + assert.EqualValues(t, 2.5, rec.Rate) + assert.Equal(t, now.Unix(), rec.Time.Unix()) + + // A symbol absent from the record is not found. + _, err = s.GetExchangeRate(ctx, "eur", now) + assert.Equal(t, currency.ErrNotFound, err) + + // The range query returns only the requested symbol. + got, err := s.GetExchangeRatesInRange(ctx, "usd", query.IntervalRaw, now.Add(-time.Hour), now.Add(time.Hour), query.Ascending) + require.NoError(t, err) + require.Len(t, got, 1) + assert.Equal(t, "usd", got[0].Symbol) + assert.EqualValues(t, 1.5, got[0].Rate) + + // A range query for an absent symbol finds nothing. + _, err = s.GetExchangeRatesInRange(ctx, "eur", query.IntervalRaw, now.Add(-time.Hour), now.Add(time.Hour), query.Ascending) + assert.Equal(t, currency.ErrNotFound, err) +} + +// TestExchange_BucketStartWeek verifies weekly buckets start on Sunday. +func TestExchange_BucketStartWeek(t *testing.T) { + sunday := time.Date(2021, 05, 30, 0, 0, 0, 0, time.UTC) + + // 2021-06-01 is a Tuesday; its week starts Sunday 2021-05-30. + tuesday := time.Date(2021, 06, 01, 15, 30, 0, 0, time.UTC) + assert.Equal(t, sunday, bucketStart(tuesday, resWeek)) + + // A Sunday maps to itself (midnight). + assert.Equal(t, sunday, bucketStart(sunday.Add(9*time.Hour), resWeek)) +} diff --git a/ocp/data/currency/exchange/dynamodb/table.go b/ocp/data/currency/exchange/dynamodb/table.go new file mode 100644 index 0000000..4af4ed9 --- /dev/null +++ b/ocp/data/currency/exchange/dynamodb/table.go @@ -0,0 +1,87 @@ +package dynamodb + +import ( + "context" + "errors" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +// CreateTables provisions the exchange rate table with on-demand billing. The +// table is keyed by (pk, sk) with no secondary indexes. It is idempotent: a +// table that already exists is left as-is. The call blocks until the table is +// ACTIVE. +func CreateTables(ctx context.Context, client *dynamodb.Client, table string) error { + _, err := client.CreateTable(ctx, &dynamodb.CreateTableInput{ + TableName: aws.String(table), + BillingMode: types.BillingModePayPerRequest, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String(attrPK), AttributeType: types.ScalarAttributeTypeS}, + {AttributeName: aws.String(attrSK), AttributeType: types.ScalarAttributeTypeN}, + }, + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String(attrPK), KeyType: types.KeyTypeHash}, + {AttributeName: aws.String(attrSK), KeyType: types.KeyTypeRange}, + }, + }) + if err != nil { + var inUse *types.ResourceInUseException + if !errors.As(err, &inUse) { + return err + } + // Already exists; still ensure it is ACTIVE before returning. + } + + return dynamodb.NewTableExistsWaiter(client).Wait(ctx, &dynamodb.DescribeTableInput{ + TableName: aws.String(table), + }, 2*time.Minute) +} + +// maxBatchWriteItems is DynamoDB's per-call BatchWriteItem limit. +const maxBatchWriteItems = 25 + +// clearTable deletes every item from the table, for tests. It scans the key +// attributes and issues batched deletes. +func clearTable(ctx context.Context, client *dynamodb.Client, table string) error { + var startKey map[string]types.AttributeValue + for { + out, err := client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(table), + ProjectionExpression: aws.String(attrPK + ", " + attrSK), + ExclusiveStartKey: startKey, + }) + if err != nil { + return err + } + + for start := 0; start < len(out.Items); start += maxBatchWriteItems { + end := start + maxBatchWriteItems + if end > len(out.Items) { + end = len(out.Items) + } + requests := make([]types.WriteRequest, 0, end-start) + for _, item := range out.Items[start:end] { + requests = append(requests, types.WriteRequest{ + DeleteRequest: &types.DeleteRequest{Key: map[string]types.AttributeValue{ + attrPK: item[attrPK], + attrSK: item[attrSK], + }}, + }) + } + if _, err := client.BatchWriteItem(ctx, &dynamodb.BatchWriteItemInput{ + RequestItems: map[string][]types.WriteRequest{table: requests}, + }); err != nil { + return err + } + } + + if len(out.LastEvaluatedKey) == 0 { + break + } + startKey = out.LastEvaluatedKey + } + return nil +} diff --git a/ocp/data/currency/exchange/memory/store.go b/ocp/data/currency/exchange/memory/store.go new file mode 100644 index 0000000..6b11e18 --- /dev/null +++ b/ocp/data/currency/exchange/memory/store.go @@ -0,0 +1,152 @@ +// Package memory provides an in-memory exchange.Store implementation for fast +// unit tests. It mirrors the exchange-rate behavior of the in-memory currency +// store. +package memory + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" +) + +type store struct { + mu sync.Mutex + records []*currency.ExchangeRateRecord + lastID uint64 +} + +type rateByTime []*currency.ExchangeRateRecord + +func (a rateByTime) Len() int { return len(a) } +func (a rateByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a rateByTime) Less(i, j int) bool { + // DESC order (most recent first) + return a[i].Time.Unix() > a[j].Time.Unix() +} + +func New() exchange.Store { + return &store{ + records: make([]*currency.ExchangeRateRecord, 0), + lastID: 1, + } +} + +func (s *store) reset() { + s.mu.Lock() + s.records = make([]*currency.ExchangeRateRecord, 0) + s.lastID = 1 + s.mu.Unlock() +} + +func (s *store) PutExchangeRates(ctx context.Context, data *currency.MultiRateRecord) error { + s.mu.Lock() + defer s.mu.Unlock() + + for _, item := range s.records { + if item.Time.Unix() == data.Time.Unix() { + return currency.ErrExists + } + } + + for symbol, rate := range data.Rates { + s.records = append(s.records, ¤cy.ExchangeRateRecord{ + Id: s.lastID, + Rate: rate, + Time: data.Time, + Symbol: symbol, + }) + s.lastID++ + } + + return nil +} + +func (s *store) GetExchangeRate(ctx context.Context, symbol string, t time.Time) (*currency.ExchangeRateRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var results []*currency.ExchangeRateRecord + for _, item := range s.records { + if item.Symbol == symbol && item.Time.Unix() <= t.Unix() { + results = append(results, item) + } + } + + if len(results) == 0 { + return nil, currency.ErrNotFound + } + + sort.Sort(rateByTime(results)) + + return results[0], nil +} + +func (s *store) GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + sort.Sort(rateByTime(s.records)) // most recent first + + result := currency.MultiRateRecord{ + Rates: make(map[string]float64), + } + for _, item := range s.records { + if item.Time.Unix() > t.Unix() { + continue + } + // The first record at or before t (descending) is the most recent; + // collect every symbol recorded at that same instant, then stop. + if len(result.Rates) == 0 { + result.Time = item.Time + } else if !item.Time.Equal(result.Time) { + break + } + result.Rates[item.Symbol] = item.Rate + } + + if len(result.Rates) == 0 { + return nil, currency.ErrNotFound + } + + return &result, nil +} + +func (s *store) GetExchangeRatesInRange(ctx context.Context, symbol string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ExchangeRateRecord, error) { + if interval > query.IntervalMonth { + return nil, currency.ErrInvalidInterval + } + if start.IsZero() || end.IsZero() { + return nil, currency.ErrInvalidRange + } + + s.mu.Lock() + defer s.mu.Unlock() + + sort.Sort(rateByTime(s.records)) + + var all []*currency.ExchangeRateRecord + for _, item := range s.records { + if item.Symbol == symbol && item.Time.Unix() >= start.Unix() && item.Time.Unix() <= end.Unix() { + all = append(all, item) + } + } + + // TODO: handle the interval + + if len(all) == 0 { + return nil, currency.ErrNotFound + } + + if ordering == query.Ascending { + for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 { + all[i], all[j] = all[j], all[i] + } + } + + return all, nil +} diff --git a/ocp/data/currency/exchange/memory/store_test.go b/ocp/data/currency/exchange/memory/store_test.go new file mode 100644 index 0000000..b788292 --- /dev/null +++ b/ocp/data/currency/exchange/memory/store_test.go @@ -0,0 +1,15 @@ +package memory + +import ( + "testing" + + "github.com/code-payments/ocp-server/ocp/data/currency/exchange/tests" +) + +func TestExchange_MemoryStore(t *testing.T) { + testStore := New() + teardown := func() { + testStore.(*store).reset() + } + tests.RunStoreTests(t, testStore, teardown) +} diff --git a/ocp/data/currency/exchange/store.go b/ocp/data/currency/exchange/store.go new file mode 100644 index 0000000..4d2811b --- /dev/null +++ b/ocp/data/currency/exchange/store.go @@ -0,0 +1,46 @@ +// Package exchange defines a focused store for core-mint exchange rate records. +// +// It mirrors the exchange-rate portion of the larger ocp/data/currency store, +// reusing that package's record types (currency.ExchangeRateRecord, +// currency.MultiRateRecord) and error sentinels (currency.ErrNotFound, +// currency.ErrExists, currency.ErrInvalidRange, currency.ErrInvalidInterval) so +// callers see identical semantics. A DynamoDB-backed implementation lives in the +// dynamodb subpackage and an in-memory implementation lives in memory. +package exchange + +import ( + "context" + "time" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" +) + +type Store interface { + // PutExchangeRates puts exchange rate records for the core mint into the store. + // + // currency.ErrExists is returned if records already exist for the provided time. + PutExchangeRates(ctx context.Context, record *currency.MultiRateRecord) error + + // GetExchangeRate gets price information given a certain time and currency symbol + // for the core mint. The most recent record at or before the requested time is + // returned. + // + // currency.ErrNotFound is returned if no price data exists at or before the provided time. + GetExchangeRate(ctx context.Context, symbol string, t time.Time) (*currency.ExchangeRateRecord, error) + + // GetAllExchangeRates gets price information given a certain time for the core mint. + // The most recent record at or before the requested time is returned. + // + // currency.ErrNotFound is returned if no price data exists at or before the provided time. + GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) + + // GetExchangeRatesInRange gets the price information for a range of time given a currency + // symbol and interval for the core mint. The start and end timestamps are provided along + // with the interval. + // + // currency.ErrNotFound is returned if the symbol or the exchange rates for the symbol cannot be found + // currency.ErrInvalidRange is returned if the range is not valid + // currency.ErrInvalidInterval is returned if the interval is not valid + GetExchangeRatesInRange(ctx context.Context, symbol string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ExchangeRateRecord, error) +} diff --git a/ocp/data/currency/exchange/tests/tests.go b/ocp/data/currency/exchange/tests/tests.go new file mode 100644 index 0000000..0ac2e80 --- /dev/null +++ b/ocp/data/currency/exchange/tests/tests.go @@ -0,0 +1,143 @@ +// Package tests holds the shared conformance suite run against every +// exchange.Store implementation. +package tests + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" +) + +func RunStoreTests(t *testing.T, s exchange.Store, teardown func()) { + for _, tf := range []func(t *testing.T, s exchange.Store){ + testExchangeRateRoundTrip, + testGetExchangeRatesInRange, + } { + tf(t, s) + teardown() + } +} + +func testExchangeRateRoundTrip(t *testing.T, s exchange.Store) { + now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) + + record, err := s.GetAllExchangeRates(context.Background(), now) + assert.Nil(t, record) + assert.Equal(t, currency.ErrNotFound, err) + + rates := map[string]float64{ + "usd": 0.000055, + "cad": 0.00007, + } + require.NoError(t, s.PutExchangeRates(context.Background(), ¤cy.MultiRateRecord{ + Time: now, + Rates: rates, + })) + + // Overwrite should fail + assert.Equal(t, currency.ErrExists, s.PutExchangeRates(context.Background(), ¤cy.MultiRateRecord{ + Time: now, + Rates: rates, + })) + + // Test GetExchangeRate(), it should return the USD record + single, err := s.GetExchangeRate(context.Background(), "usd", now) + require.NoError(t, err) + assert.Equal(t, now.Unix(), single.Time.Unix()) + assert.EqualValues(t, rates["usd"], single.Rate) + + // Test GetAllExchangeRates(), it should return all recent rates + record, err = s.GetAllExchangeRates(context.Background(), now) + require.NoError(t, err) + + assert.Equal(t, now.Unix(), record.Time.Unix()) + assert.EqualValues(t, rates, record.Rates) + + // a later time the same day returns the most recent entry + record, err = s.GetAllExchangeRates(context.Background(), time.Date(2021, 01, 29, 14, 0, 5, 0, time.UTC)) + require.NoError(t, err) + + assert.Equal(t, now.Unix(), record.Time.Unix()) + assert.EqualValues(t, rates, record.Rates) + + // a later day still returns the most recent entry at or before the timestamp + tomorrow := time.Date(2021, 01, 30, 0, 0, 0, 0, time.UTC) + record, err = s.GetAllExchangeRates(context.Background(), tomorrow) + require.NoError(t, err) + assert.Equal(t, now.Unix(), record.Time.Unix()) + assert.EqualValues(t, rates, record.Rates) + + // a time before any record exists is not found + before := time.Date(2021, 01, 28, 0, 0, 0, 0, time.UTC) + record, err = s.GetAllExchangeRates(context.Background(), before) + assert.Nil(t, record) + assert.Equal(t, currency.ErrNotFound, err) +} + +func testGetExchangeRatesInRange(t *testing.T, s exchange.Store) { + var rates []currency.MultiRateRecord + + now := time.Now().UTC() + + for i := 0; i < 100; i++ { + rates = append(rates, currency.MultiRateRecord{ + Time: now.Add(time.Duration(i) * time.Hour), + Rates: map[string]float64{ + "usd": (0.000058 + float64(i/10000)), + "cad": (0.00008 + float64(i/10000)), + }, + }) + } + + record, err := s.GetAllExchangeRates(context.Background(), rates[0].Time) + assert.Nil(t, record) + assert.Equal(t, currency.ErrNotFound, err) + + for _, item := range rates { + require.NoError(t, s.PutExchangeRates(context.Background(), &item)) + } + + result, err := s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalRaw, rates[0].Time, rates[99].Time, query.Ascending) + require.NoError(t, err) + assert.Equal(t, len(result), 100) + for i, item := range result { + assert.Equal(t, rates[i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, rates[i].Rates["usd"], item.Rate) + } + + result, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalRaw, rates[0].Time, rates[49].Time, query.Ascending) + require.NoError(t, err) + assert.Equal(t, len(result), 50) + for i, item := range result { + assert.Equal(t, rates[i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, rates[i].Rates["usd"], item.Rate) + } + + result, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalRaw, rates[0].Time, rates[99].Time, query.Descending) + require.NoError(t, err) + assert.Equal(t, len(result), 100) + for i, item := range result { + assert.Equal(t, rates[99-i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, rates[99-i].Rates["usd"], item.Rate) + } + + _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalSecond, rates[0].Time, rates[99].Time, query.Ascending) + require.NoError(t, err) + _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalMinute, rates[0].Time, rates[99].Time, query.Ascending) + require.NoError(t, err) + _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalHour, rates[0].Time, rates[99].Time, query.Ascending) + require.NoError(t, err) + _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalDay, rates[0].Time, rates[99].Time, query.Ascending) + require.NoError(t, err) + _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalWeek, rates[0].Time, rates[99].Time, query.Ascending) + require.NoError(t, err) + _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalMonth, rates[0].Time, rates[99].Time, query.Ascending) + require.NoError(t, err) +} diff --git a/ocp/data/currency/holder/dynamodb/store.go b/ocp/data/currency/holder/dynamodb/store.go new file mode 100644 index 0000000..932d447 --- /dev/null +++ b/ocp/data/currency/holder/dynamodb/store.go @@ -0,0 +1,514 @@ +// Package dynamodb implements the holder.Store interface on top of DynamoDB. +// +// Two tables, both keyed per mint (the number of mints is unbounded, so each +// record is its own item — never a map-of-all-mints row): +// +// History table — per-mint time series, one partition per resolution: +// +// pk = "#raw" sk = count (every sample) +// pk = "#hour" sk = count, ts (close per hour) +// pk = "#day" sk = count, ts (close per day) +// pk = "#week" sk = count, ts (close per week, Sunday-start) +// pk = "#month" sk = count, ts (close per month) +// +// Writes fan out to all resolutions in one transaction: the raw point is a +// conditional Put enforcing once-per-timestamp, each rollup a monotonic +// last-write-wins Update holding the bucket close. Rollup items store `ts` (the +// close sample's time, which also drives the guard); raw items omit it because +// their sort key already is the sample time. Reads serve the resolution matching +// the requested interval; point lookups read the raw resolution at or before t. +// +// Live table — latest snapshot per mint, keyed by pk = "" (no sort key): +// +// pk = "" count, ts +// +// Live writes spread across mints (each mint its own partition); the +// timestamp-monotonic condition keeps the most recent sample. GetAllLiveHolderCounts +// scans this table — it holds one item per mint, so the scan touches only live state. +package dynamodb + +import ( + "context" + "errors" + "fmt" + "strconv" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" +) + +const ( + attrPK = "pk" + attrSK = "sk" + attrCount = "count" + attrTS = "ts" + + resRaw = "raw" + resHour = "hour" + resDay = "day" + resWeek = "week" + resMonth = "month" + + // codeConditionalCheckFailed is the DynamoDB cancellation reason code for a + // transaction item whose ConditionExpression evaluated false. + codeConditionalCheckFailed = "ConditionalCheckFailed" + + // maxBatchGetItems is DynamoDB's per-call BatchGetItem key limit. + maxBatchGetItems = 100 +) + +// rollupResolutions are the coarse resolutions maintained alongside the raw +// points on every historical write. +var rollupResolutions = []string{resHour, resDay, resWeek, resMonth} + +type store struct { + client *dynamodb.Client + historyTable string + liveTable string +} + +// New returns a holder.Store backed by the given DynamoDB tables. Use +// CreateTables to provision them. +func New(client *dynamodb.Client, historyTable, liveTable string) holder.Store { + return &store{ + client: client, + historyTable: historyTable, + liveTable: liveTable, + } +} + +// PutHistoricalHolderCount writes the raw sample and refreshes every rollup +// bucket that contains it, atomically. The raw item's attribute_not_exists +// condition enforces once-per-timestamp: a duplicate cancels the transaction and +// returns currency.ErrExists. Rollup guards assume monotonically increasing write +// timestamps per mint (the holder worker's behavior). +func (s *store) PutHistoricalHolderCount(ctx context.Context, record *currency.HolderCountRecord) error { + if err := record.Validate(); err != nil { + return err + } + + count := avNU(record.HolderCount) + tsVal := skN(record.Time) + + transactItems := []types.TransactWriteItem{ + {Put: &types.Put{ + TableName: aws.String(s.historyTable), + Item: map[string]types.AttributeValue{ + attrPK: avS(historyPK(record.Mint, resRaw)), + attrSK: skN(record.Time), + attrCount: count, + }, + ConditionExpression: aws.String(fmt.Sprintf("attribute_not_exists(%s)", attrPK)), + }}, + } + for _, res := range rollupResolutions { + transactItems = append(transactItems, types.TransactWriteItem{ + Update: &types.Update{ + TableName: aws.String(s.historyTable), + Key: map[string]types.AttributeValue{ + attrPK: avS(historyPK(record.Mint, res)), + attrSK: skN(bucketStart(record.Time, res)), + }, + UpdateExpression: aws.String("SET #c = :c, #ts = :ts"), + ConditionExpression: aws.String("attribute_not_exists(#pk) OR #ts < :ts"), + ExpressionAttributeNames: map[string]string{ + "#pk": attrPK, + "#c": attrCount, + "#ts": attrTS, + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":c": count, + ":ts": tsVal, + }, + }, + }) + } + + _, err := s.client.TransactWriteItems(ctx, &dynamodb.TransactWriteItemsInput{ + TransactItems: transactItems, + }) + if err != nil { + var tce *types.TransactionCanceledException + if errors.As(err, &tce) && len(tce.CancellationReasons) > 0 && + aws.ToString(tce.CancellationReasons[0].Code) == codeConditionalCheckFailed { + return currency.ErrExists + } + return err + } + return nil +} + +func (s *store) GetHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*currency.HolderCountRecord, error) { + out, err := s.client.Query(ctx, &dynamodb.QueryInput{ + TableName: aws.String(s.historyTable), + KeyConditionExpression: aws.String(fmt.Sprintf("%s = :pk AND %s <= :sk", attrPK, attrSK)), + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":pk": avS(historyPK(mint, resRaw)), + ":sk": skN(t), + }, + ScanIndexForward: aws.Bool(false), + Limit: aws.Int32(1), + }) + if err != nil { + return nil, err + } + if len(out.Items) == 0 { + return nil, currency.ErrNotFound + } + return historyRecord(mint, out.Items[0]) +} + +// GetHolderCountsForDay returns each mint's holder count as of the UTC day of t — +// the close of that mint's day rollup bucket. Because a rollup bucket has a +// deterministic key (mint#day, bucketStart), this is a single batched key get +// rather than a per-mint query. Mints with no record on that day are omitted. +func (s *store) GetHolderCountsForDay(ctx context.Context, mints []string, t time.Time) (map[string]*currency.HolderCountRecord, error) { + bucket := skN(bucketStart(t, resDay)) + + // Dedup so a repeated mint isn't fetched twice. + seen := make(map[string]struct{}, len(mints)) + keys := make([]map[string]types.AttributeValue, 0, len(mints)) + for _, mint := range mints { + if _, ok := seen[mint]; ok { + continue + } + seen[mint] = struct{}{} + keys = append(keys, map[string]types.AttributeValue{ + attrPK: avS(historyPK(mint, resDay)), + attrSK: bucket, + }) + } + + res := make(map[string]*currency.HolderCountRecord, len(keys)) + for start := 0; start < len(keys); start += maxBatchGetItems { + end := start + maxBatchGetItems + if end > len(keys) { + end = len(keys) + } + + req := map[string]types.KeysAndAttributes{ + s.historyTable: {Keys: keys[start:end]}, + } + // Drain UnprocessedKeys (DynamoDB may return a partial batch under load). + for len(req[s.historyTable].Keys) > 0 { + out, err := s.client.BatchGetItem(ctx, &dynamodb.BatchGetItemInput{RequestItems: req}) + if err != nil { + return nil, err + } + for _, item := range out.Responses[s.historyTable] { + mint, err := mintFromDayPK(item) + if err != nil { + return nil, err + } + rec, err := historyRecord(mint, item) + if err != nil { + return nil, err + } + res[mint] = rec + } + if unprocessed, ok := out.UnprocessedKeys[s.historyTable]; ok && len(unprocessed.Keys) > 0 { + req = map[string]types.KeysAndAttributes{s.historyTable: unprocessed} + } else { + break + } + } + } + return res, nil +} + +func (s *store) GetHolderCountsInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.HolderCountRecord, error) { + if interval > query.IntervalMonth { + return nil, currency.ErrInvalidInterval + } + if start.IsZero() || end.IsZero() { + return nil, currency.ErrInvalidRange + } + + actualStart, actualEnd := start, end + if start.Unix() > end.Unix() { + actualStart, actualEnd = end, start + } + + // Honor the requested interval directly by reading the matching stored + // resolution. The data is pre-aggregated, so no in-app downsampling is needed. + res := resolutionForInterval(interval) + items, err := s.queryAll(ctx, &dynamodb.QueryInput{ + TableName: aws.String(s.historyTable), + KeyConditionExpression: aws.String(fmt.Sprintf("%s = :pk AND %s BETWEEN :start AND :end", attrPK, attrSK)), + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":pk": avS(historyPK(mint, res)), + // Lower-bound on the bucket containing actualStart so a bucket whose + // start precedes actualStart is still returned. + ":start": skN(bucketStart(actualStart, res)), + ":end": skN(actualEnd), + }, + ScanIndexForward: aws.Bool(ordering != query.Descending), + }) + if err != nil { + return nil, err + } + + records := make([]*currency.HolderCountRecord, 0, len(items)) + for _, item := range items { + rec, err := historyRecord(mint, item) + if err != nil { + return nil, err + } + records = append(records, rec) + } + + if len(records) == 0 { + return nil, currency.ErrNotFound + } + return records, nil +} + +// PutLiveHolderCount upserts the mint's latest holder count, keeping the most +// recent sample. The timestamp-monotonic condition makes a stale (older or equal +// time) write return currency.ErrStaleHolderState. +func (s *store) PutLiveHolderCount(ctx context.Context, record *currency.HolderCountRecord) error { + if err := record.Validate(); err != nil { + return err + } + + _, err := s.client.PutItem(ctx, &dynamodb.PutItemInput{ + TableName: aws.String(s.liveTable), + Item: map[string]types.AttributeValue{ + attrPK: avS(record.Mint), + attrCount: avNU(record.HolderCount), + attrTS: avN(record.Time.UTC().UnixNano()), + }, + ConditionExpression: aws.String("attribute_not_exists(#pk) OR #ts < :ts"), + ExpressionAttributeNames: map[string]string{ + "#pk": attrPK, + "#ts": attrTS, + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":ts": avN(record.Time.UTC().UnixNano()), + }, + }) + if err != nil { + var ccf *types.ConditionalCheckFailedException + if errors.As(err, &ccf) { + return currency.ErrStaleHolderState + } + return err + } + return nil +} + +func (s *store) GetLiveHolderCount(ctx context.Context, mint string) (*currency.HolderCountRecord, error) { + out, err := s.client.GetItem(ctx, &dynamodb.GetItemInput{ + TableName: aws.String(s.liveTable), + Key: map[string]types.AttributeValue{attrPK: avS(mint)}, + }) + if err != nil { + return nil, err + } + if len(out.Item) == 0 { + return nil, currency.ErrNotFound + } + return liveRecord(out.Item) +} + +func (s *store) GetAllLiveHolderCounts(ctx context.Context) (map[string]*currency.HolderCountRecord, error) { + res := make(map[string]*currency.HolderCountRecord) + var startKey map[string]types.AttributeValue + for { + out, err := s.client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(s.liveTable), + ExclusiveStartKey: startKey, + }) + if err != nil { + return nil, err + } + for _, item := range out.Items { + rec, err := liveRecord(item) + if err != nil { + return nil, err + } + res[rec.Mint] = rec + } + if len(out.LastEvaluatedKey) == 0 { + break + } + startKey = out.LastEvaluatedKey + } + + if len(res) == 0 { + return nil, currency.ErrNotFound + } + return res, nil +} + +// reset deletes every item from both tables, for tests. +func (s *store) reset() { + if err := clearTable(context.Background(), s.client, s.historyTable, []string{attrPK, attrSK}); err != nil { + panic(err) + } + if err := clearTable(context.Background(), s.client, s.liveTable, []string{attrPK}); err != nil { + panic(err) + } +} + +// queryAll runs the query, following LastEvaluatedKey until the result set is +// drained, and returns every matched item. +func (s *store) queryAll(ctx context.Context, input *dynamodb.QueryInput) ([]map[string]types.AttributeValue, error) { + var items []map[string]types.AttributeValue + for { + out, err := s.client.Query(ctx, input) + if err != nil { + return nil, err + } + items = append(items, out.Items...) + if len(out.LastEvaluatedKey) == 0 { + break + } + input.ExclusiveStartKey = out.LastEvaluatedKey + } + return items, nil +} + +// historyRecord builds a holder count record from a history item. The mint is +// the query parameter (it is encoded in the pk alongside the resolution, so it +// is not duplicated onto the item). +func historyRecord(mint string, item map[string]types.AttributeValue) (*currency.HolderCountRecord, error) { + count, err := parseNU(item[attrCount]) + if err != nil { + return nil, err + } + ts, err := itemTime(item) + if err != nil { + return nil, err + } + return ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: count, + Time: ts, + }, nil +} + +func liveRecord(item map[string]types.AttributeValue) (*currency.HolderCountRecord, error) { + count, err := parseNU(item[attrCount]) + if err != nil { + return nil, err + } + nanos, err := parseN(item[attrTS]) + if err != nil { + return nil, err + } + return ¤cy.HolderCountRecord{ + Mint: asS(item[attrPK]), + HolderCount: count, + Time: time.Unix(0, nanos).UTC(), + }, nil +} + +// itemTime returns the point's timestamp: the close sample time from `ts` for +// rollup items, or the sample time from the sort key for raw items (which omit +// `ts`). +func itemTime(item map[string]types.AttributeValue) (time.Time, error) { + av, ok := item[attrTS] + if !ok { + av = item[attrSK] + } + nanos, err := parseN(av) + if err != nil { + return time.Time{}, err + } + return time.Unix(0, nanos).UTC(), nil +} + +// resolutionForInterval maps a requested interval to the stored resolution that +// serves it. Sub-hour intervals are served from raw, the finest stored +// resolution. The caller chooses an interval appropriate to the range; this +// store honors it directly rather than re-deriving one. +func resolutionForInterval(interval query.Interval) string { + switch interval { + case query.IntervalHour: + return resHour + case query.IntervalDay: + return resDay + case query.IntervalWeek: + return resWeek + case query.IntervalMonth: + return resMonth + default: // raw, second, minute + return resRaw + } +} + +// bucketStart truncates t to the start of its bucket for the given resolution, +// in UTC. Weeks start on Sunday. +func bucketStart(t time.Time, res string) time.Time { + t = t.UTC() + switch res { + case resHour: + return t.Truncate(time.Hour) + case resDay: + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + case resWeek: + day := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + return day.AddDate(0, 0, -int(day.Weekday())) // time.Weekday: Sunday=0 + case resMonth: + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) + default: // resRaw + return t + } +} + +func historyPK(mint, res string) string { return mint + "#" + res } + +// mintFromDayPK recovers the mint from a day-rollup item's pk ("#day"). +// Mints are base58, which never contains '#', so the resolution suffix is +// unambiguous. +func mintFromDayPK(item map[string]types.AttributeValue) (string, error) { + pk := asS(item[attrPK]) + suffix := "#" + resDay + if !strings.HasSuffix(pk, suffix) { + return "", fmt.Errorf("unexpected day-rollup pk %q", pk) + } + return strings.TrimSuffix(pk, suffix), nil +} + +// skN encodes t's unix-nanos as the numeric sort key, which sorts in +// chronological order. +func skN(t time.Time) types.AttributeValue { return avN(t.UTC().UnixNano()) } + +func avS(v string) types.AttributeValue { return &types.AttributeValueMemberS{Value: v} } +func avN(v int64) types.AttributeValue { + return &types.AttributeValueMemberN{Value: strconv.FormatInt(v, 10)} +} +func avNU(v uint64) types.AttributeValue { + return &types.AttributeValueMemberN{Value: strconv.FormatUint(v, 10)} +} + +func asS(av types.AttributeValue) string { + if s, ok := av.(*types.AttributeValueMemberS); ok { + return s.Value + } + return "" +} + +func parseN(av types.AttributeValue) (int64, error) { + n, ok := av.(*types.AttributeValueMemberN) + if !ok { + return 0, fmt.Errorf("expected number attribute, got %T", av) + } + return strconv.ParseInt(n.Value, 10, 64) +} + +func parseNU(av types.AttributeValue) (uint64, error) { + n, ok := av.(*types.AttributeValueMemberN) + if !ok { + return 0, fmt.Errorf("expected number attribute, got %T", av) + } + return strconv.ParseUint(n.Value, 10, 64) +} diff --git a/ocp/data/currency/holder/dynamodb/store_test.go b/ocp/data/currency/holder/dynamodb/store_test.go new file mode 100644 index 0000000..ceac14b --- /dev/null +++ b/ocp/data/currency/holder/dynamodb/store_test.go @@ -0,0 +1,106 @@ +package dynamodb + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + dynamotest "github.com/code-payments/ocp-server/database/dynamodb/test" + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/holder/tests" +) + +const ( + historyTable = "holder_history_test" + liveTable = "holder_live_test" +) + +var testEnv *dynamotest.TestEnv + +func TestMain(m *testing.M) { + log := zap.Must(zap.NewDevelopment()) + + env, err := dynamotest.NewTestEnv() + if err != nil { + log.With(zap.Error(err)).Error("Error creating dynamodb test environment") + os.Exit(1) + } + + testEnv = env + + os.Exit(m.Run()) +} + +func newTestStore(t *testing.T) *store { + require.NoError(t, CreateTables(context.Background(), testEnv.Client, historyTable, liveTable)) + s := New(testEnv.Client, historyTable, liveTable).(*store) + s.reset() + return s +} + +func TestHolder_DynamoDBStore(t *testing.T) { + testStore := newTestStore(t) + teardown := func() { + testStore.reset() + } + tests.RunStoreTests(t, testStore, teardown) +} + +// TestHolder_DynamoDBRollups verifies that a coarse interval reads from the +// matching rollup partition and that each bucket holds the close (the most +// recent sample within the bucket). +func TestHolder_DynamoDBRollups(t *testing.T) { + s := newTestStore(t) + defer s.reset() + ctx := context.Background() + mint := "rollup-mint" + + // Two samples in hour 10 (close = 20 at 10:45), one in hour 11. + samples := []struct { + at time.Time + count uint64 + }{ + {time.Date(2021, 06, 01, 10, 05, 0, 0, time.UTC), 10}, + {time.Date(2021, 06, 01, 10, 45, 0, 0, time.UTC), 20}, + {time.Date(2021, 06, 01, 11, 30, 0, 0, time.UTC), 30}, + } + for _, sample := range samples { + require.NoError(t, s.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: sample.count, + Time: sample.at, + })) + } + + start := time.Date(2021, 06, 01, 10, 0, 0, 0, time.UTC) + end := time.Date(2021, 06, 01, 12, 0, 0, 0, time.UTC) + + hourly, err := s.GetHolderCountsInRange(ctx, mint, query.IntervalHour, start, end, query.Ascending) + require.NoError(t, err) + require.Len(t, hourly, 2) + assert.Equal(t, samples[1].at.Unix(), hourly[0].Time.Unix()) + assert.EqualValues(t, 20, hourly[0].HolderCount) + assert.Equal(t, samples[2].at.Unix(), hourly[1].Time.Unix()) + assert.EqualValues(t, 30, hourly[1].HolderCount) + + // Raw still has every sample. + raw, err := s.GetHolderCountsInRange(ctx, mint, query.IntervalRaw, start, end, query.Ascending) + require.NoError(t, err) + assert.Len(t, raw, len(samples)) +} + +func TestHolder_DynamoDBResolutionForInterval(t *testing.T) { + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalRaw)) + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalSecond)) + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalMinute)) + assert.Equal(t, resHour, resolutionForInterval(query.IntervalHour)) + assert.Equal(t, resDay, resolutionForInterval(query.IntervalDay)) + assert.Equal(t, resWeek, resolutionForInterval(query.IntervalWeek)) + assert.Equal(t, resMonth, resolutionForInterval(query.IntervalMonth)) +} diff --git a/ocp/data/currency/holder/dynamodb/table.go b/ocp/data/currency/holder/dynamodb/table.go new file mode 100644 index 0000000..dedd05b --- /dev/null +++ b/ocp/data/currency/holder/dynamodb/table.go @@ -0,0 +1,106 @@ +package dynamodb + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +// maxBatchWriteItems is DynamoDB's per-call BatchWriteItem limit. +const maxBatchWriteItems = 25 + +// CreateTables provisions the history and live holder-count tables with +// on-demand billing. The history table is keyed by (pk, sk); the live table is +// keyed by pk only (one item per mint) so it can be scanned cheaply for all +// mints. It is idempotent and blocks until both tables are ACTIVE. +func CreateTables(ctx context.Context, client *dynamodb.Client, historyTable, liveTable string) error { + inputs := []*dynamodb.CreateTableInput{ + { + TableName: aws.String(historyTable), + BillingMode: types.BillingModePayPerRequest, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String(attrPK), AttributeType: types.ScalarAttributeTypeS}, + {AttributeName: aws.String(attrSK), AttributeType: types.ScalarAttributeTypeN}, + }, + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String(attrPK), KeyType: types.KeyTypeHash}, + {AttributeName: aws.String(attrSK), KeyType: types.KeyTypeRange}, + }, + }, + { + TableName: aws.String(liveTable), + BillingMode: types.BillingModePayPerRequest, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String(attrPK), AttributeType: types.ScalarAttributeTypeS}, + }, + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String(attrPK), KeyType: types.KeyTypeHash}, + }, + }, + } + + for _, input := range inputs { + if _, err := client.CreateTable(ctx, input); err != nil { + var inUse *types.ResourceInUseException + if !errors.As(err, &inUse) { + return err + } + // Already exists; still ensure it is ACTIVE before returning. + } + if err := dynamodb.NewTableExistsWaiter(client).Wait(ctx, &dynamodb.DescribeTableInput{ + TableName: input.TableName, + }, 2*time.Minute); err != nil { + return err + } + } + return nil +} + +// clearTable deletes every item from the table, for tests. keyAttrs are the +// table's key attribute names (one for a hash-only table, two for composite). +func clearTable(ctx context.Context, client *dynamodb.Client, table string, keyAttrs []string) error { + var startKey map[string]types.AttributeValue + for { + out, err := client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(table), + ProjectionExpression: aws.String(strings.Join(keyAttrs, ", ")), + ExclusiveStartKey: startKey, + }) + if err != nil { + return err + } + + for start := 0; start < len(out.Items); start += maxBatchWriteItems { + end := start + maxBatchWriteItems + if end > len(out.Items) { + end = len(out.Items) + } + requests := make([]types.WriteRequest, 0, end-start) + for _, item := range out.Items[start:end] { + key := make(map[string]types.AttributeValue, len(keyAttrs)) + for _, a := range keyAttrs { + key[a] = item[a] + } + requests = append(requests, types.WriteRequest{ + DeleteRequest: &types.DeleteRequest{Key: key}, + }) + } + if _, err := client.BatchWriteItem(ctx, &dynamodb.BatchWriteItemInput{ + RequestItems: map[string][]types.WriteRequest{table: requests}, + }); err != nil { + return err + } + } + + if len(out.LastEvaluatedKey) == 0 { + break + } + startKey = out.LastEvaluatedKey + } + return nil +} diff --git a/ocp/data/currency/holder/memory/store.go b/ocp/data/currency/holder/memory/store.go new file mode 100644 index 0000000..d0fc3a2 --- /dev/null +++ b/ocp/data/currency/holder/memory/store.go @@ -0,0 +1,198 @@ +// Package memory provides an in-memory holder.Store implementation for fast +// unit tests. +package memory + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" +) + +type store struct { + mu sync.Mutex + historical []*currency.HolderCountRecord + lastID uint64 + live map[string]*currency.HolderCountRecord +} + +type holderCountByTime []*currency.HolderCountRecord + +func (a holderCountByTime) Len() int { return len(a) } +func (a holderCountByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a holderCountByTime) Less(i, j int) bool { + // DESC order (most recent first) + return a[i].Time.Unix() > a[j].Time.Unix() +} + +func New() holder.Store { + return &store{ + historical: make([]*currency.HolderCountRecord, 0), + lastID: 1, + live: make(map[string]*currency.HolderCountRecord), + } +} + +func (s *store) reset() { + s.mu.Lock() + s.historical = make([]*currency.HolderCountRecord, 0) + s.lastID = 1 + s.live = make(map[string]*currency.HolderCountRecord) + s.mu.Unlock() +} + +func (s *store) PutHistoricalHolderCount(ctx context.Context, record *currency.HolderCountRecord) error { + if err := record.Validate(); err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + for _, item := range s.historical { + if item.Mint == record.Mint && item.Time.Unix() == record.Time.Unix() { + return currency.ErrExists + } + } + + cloned := record.Clone() + cloned.Id = s.lastID + s.historical = append(s.historical, cloned) + s.lastID++ + + return nil +} + +func (s *store) GetHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*currency.HolderCountRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var results []*currency.HolderCountRecord + for _, item := range s.historical { + if item.Mint == mint && item.Time.Unix() <= t.Unix() { + results = append(results, item) + } + } + + if len(results) == 0 { + return nil, currency.ErrNotFound + } + + sort.Sort(holderCountByTime(results)) + + return results[0].Clone(), nil +} + +func (s *store) GetHolderCountsForDay(ctx context.Context, mints []string, t time.Time) (map[string]*currency.HolderCountRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + res := make(map[string]*currency.HolderCountRecord, len(mints)) + for _, mint := range mints { + // The close of t's UTC day for this mint: its most recent record that day. + var latest *currency.HolderCountRecord + for _, item := range s.historical { + if item.Mint != mint || !sameUTCDay(item.Time, t) { + continue + } + if latest == nil || item.Time.After(latest.Time) { + latest = item + } + } + if latest != nil { + res[mint] = latest.Clone() + } + } + return res, nil +} + +func sameUTCDay(a, b time.Time) bool { + ay, am, ad := a.UTC().Date() + by, bm, bd := b.UTC().Date() + return ay == by && am == bm && ad == bd +} + +func (s *store) GetHolderCountsInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.HolderCountRecord, error) { + if interval > query.IntervalMonth { + return nil, currency.ErrInvalidInterval + } + if start.IsZero() || end.IsZero() { + return nil, currency.ErrInvalidRange + } + + actualStart, actualEnd := start, end + if start.Unix() > end.Unix() { + actualStart, actualEnd = end, start + } + + s.mu.Lock() + defer s.mu.Unlock() + + var all []*currency.HolderCountRecord + for _, item := range s.historical { + if item.Mint == mint && item.Time.Unix() >= actualStart.Unix() && item.Time.Unix() <= actualEnd.Unix() { + all = append(all, item.Clone()) + } + } + + // TODO: handle the interval + + if len(all) == 0 { + return nil, currency.ErrNotFound + } + + sort.Sort(holderCountByTime(all)) // DESC + if ordering == query.Ascending { + for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 { + all[i], all[j] = all[j], all[i] + } + } + + return all, nil +} + +func (s *store) PutLiveHolderCount(ctx context.Context, record *currency.HolderCountRecord) error { + if err := record.Validate(); err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if existing, ok := s.live[record.Mint]; ok && !record.Time.After(existing.Time) { + return currency.ErrStaleHolderState + } + + s.live[record.Mint] = record.Clone() + return nil +} + +func (s *store) GetLiveHolderCount(ctx context.Context, mint string) (*currency.HolderCountRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + record, ok := s.live[mint] + if !ok { + return nil, currency.ErrNotFound + } + return record.Clone(), nil +} + +func (s *store) GetAllLiveHolderCounts(ctx context.Context) (map[string]*currency.HolderCountRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.live) == 0 { + return nil, currency.ErrNotFound + } + + res := make(map[string]*currency.HolderCountRecord, len(s.live)) + for mint, record := range s.live { + res[mint] = record.Clone() + } + return res, nil +} diff --git a/ocp/data/currency/holder/memory/store_test.go b/ocp/data/currency/holder/memory/store_test.go new file mode 100644 index 0000000..7e03d84 --- /dev/null +++ b/ocp/data/currency/holder/memory/store_test.go @@ -0,0 +1,15 @@ +package memory + +import ( + "testing" + + "github.com/code-payments/ocp-server/ocp/data/currency/holder/tests" +) + +func TestHolder_MemoryStore(t *testing.T) { + testStore := New() + teardown := func() { + testStore.(*store).reset() + } + tests.RunStoreTests(t, testStore, teardown) +} diff --git a/ocp/data/currency/holder/store.go b/ocp/data/currency/holder/store.go new file mode 100644 index 0000000..694cb51 --- /dev/null +++ b/ocp/data/currency/holder/store.go @@ -0,0 +1,76 @@ +// Package holder defines a focused store for currency creator mint holder +// counts. +// +// It mirrors the holder-count portion of the larger ocp/data/currency store, +// reusing that package's record type (currency.HolderCountRecord) and error +// sentinels (currency.ErrNotFound, currency.ErrExists, currency.ErrInvalidRange, +// currency.ErrInvalidInterval, currency.ErrStaleHolderState). A DynamoDB-backed +// implementation lives in the dynamodb subpackage and an in-memory +// implementation lives in memory. +// +// Records are keyed per mint (the number of mints is unbounded), so every record +// is a single item — there is no map-of-all-mints row. +package holder + +import ( + "context" + "time" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" +) + +type Store interface { + // PutHistoricalHolderCount puts a currency creator mint holder count record + // into the store. + // + // currency.ErrExists is returned if a record already exists for the mint at the + // provided time. + PutHistoricalHolderCount(ctx context.Context, record *currency.HolderCountRecord) error + + // GetHolderCountAtTime gets the holder count for a given currency creator mint + // at a point in time. The most recent record at or before the requested time is + // returned. + // + // currency.ErrNotFound is returned if no holder count data exists at or before + // the provided time. + GetHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*currency.HolderCountRecord, error) + + // GetHolderCountsForDay gets the holder count for each of the given currency + // creator mints as of the UTC day of t — the close of that day (the mint's most + // recent record within the day), keyed by mint. Mints with no record on that day + // are omitted from the result rather than reported as an error. + // + // Unlike GetHolderCountAtTime this is day-granularity: it does not fall back to an + // earlier day, and for a mid-day t the returned record may be later than t (the + // day's close). It is served as a single batched key get against the day rollups. + GetHolderCountsForDay(ctx context.Context, mints []string, t time.Time) (map[string]*currency.HolderCountRecord, error) + + // GetHolderCountsInRange gets the holder count records for a range of time given + // a currency creator mint and interval. + // + // currency.ErrNotFound is returned if the mint or the holder counts for the mint cannot be found + // currency.ErrInvalidRange is returned if the range is not valid + // currency.ErrInvalidInterval is returned if the interval is not valid + GetHolderCountsInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.HolderCountRecord, error) + + // PutLiveHolderCount upserts the latest holder count record for a currency + // creator mint. An upsert is only performed if the provided timestamp is greater + // than the timestamp currently stored. + // + // currency.ErrStaleHolderState is returned if the provided timestamp is not + // greater than the stored timestamp. + PutLiveHolderCount(ctx context.Context, record *currency.HolderCountRecord) error + + // GetLiveHolderCount gets the latest live holder count record for a currency + // creator mint. + // + // currency.ErrNotFound is returned if no live holder count record exists for the provided mint. + GetLiveHolderCount(ctx context.Context, mint string) (*currency.HolderCountRecord, error) + + // GetAllLiveHolderCounts gets the latest live holder count records for all + // currency creator mints. + // + // currency.ErrNotFound is returned if no live holder count records exist. + GetAllLiveHolderCounts(ctx context.Context) (map[string]*currency.HolderCountRecord, error) +} diff --git a/ocp/data/currency/holder/tests/tests.go b/ocp/data/currency/holder/tests/tests.go new file mode 100644 index 0000000..2f1fafa --- /dev/null +++ b/ocp/data/currency/holder/tests/tests.go @@ -0,0 +1,256 @@ +// Package tests holds the shared conformance suite run against every +// holder.Store implementation. +package tests + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" +) + +func RunStoreTests(t *testing.T, s holder.Store, teardown func()) { + for _, tf := range []func(t *testing.T, s holder.Store){ + testHolderCountRoundTrip, + testGetHolderCountsForDay, + testGetHolderCountsInRange, + testLiveHolderCountRoundTrip, + testGetAllLiveHolderCounts, + } { + tf(t, s) + teardown() + } +} + +func testHolderCountRoundTrip(t *testing.T, s holder.Store) { + now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) + mint := "mint" + + record, err := s.GetHolderCountAtTime(context.Background(), mint, now) + assert.Nil(t, record) + assert.Equal(t, currency.ErrNotFound, err) + + expected := ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: 42, + Time: now, + } + require.NoError(t, s.PutHistoricalHolderCount(context.Background(), expected)) + + // Duplicate timestamp for the mint fails. + assert.Equal(t, currency.ErrExists, s.PutHistoricalHolderCount(context.Background(), expected)) + + actual, err := s.GetHolderCountAtTime(context.Background(), mint, now) + require.NoError(t, err) + assert.Equal(t, now.Unix(), actual.Time.Unix()) + assert.EqualValues(t, expected.HolderCount, actual.HolderCount) + + // A later time returns the most recent record at or before it. + actual, err = s.GetHolderCountAtTime(context.Background(), mint, time.Date(2021, 01, 29, 14, 0, 5, 0, time.UTC)) + require.NoError(t, err) + assert.Equal(t, now.Unix(), actual.Time.Unix()) + assert.EqualValues(t, expected.HolderCount, actual.HolderCount) + + // A later day still returns it (no same-day requirement). + tomorrow := time.Date(2021, 01, 30, 0, 0, 0, 0, time.UTC) + actual, err = s.GetHolderCountAtTime(context.Background(), mint, tomorrow) + require.NoError(t, err) + assert.Equal(t, now.Unix(), actual.Time.Unix()) + + // A time before any record exists is not found. + before := time.Date(2021, 01, 28, 0, 0, 0, 0, time.UTC) + actual, err = s.GetHolderCountAtTime(context.Background(), mint, before) + assert.Nil(t, actual) + assert.Equal(t, currency.ErrNotFound, err) + + // A different mint is independent. + _, err = s.GetHolderCountAtTime(context.Background(), "other-mint", now) + assert.Equal(t, currency.ErrNotFound, err) +} + +func testGetHolderCountsForDay(t *testing.T, s holder.Store) { + ctx := context.Background() + day := time.Date(2022, 05, 10, 0, 0, 0, 0, time.UTC) + + // mintA: a prior-day record plus two on `day` (close = 8 at 20:00). + require.NoError(t, s.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{Mint: "mintA", HolderCount: 1, Time: day.Add(-15 * time.Hour)})) + require.NoError(t, s.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{Mint: "mintA", HolderCount: 5, Time: day.Add(10 * time.Hour)})) + require.NoError(t, s.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{Mint: "mintA", HolderCount: 8, Time: day.Add(20 * time.Hour)})) + // mintB: a single record on `day`. + require.NoError(t, s.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{Mint: "mintB", HolderCount: 100, Time: day.Add(12 * time.Hour)})) + // mintC: a record only on the next day — should be omitted for a `day` query. + require.NoError(t, s.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{Mint: "mintC", HolderCount: 50, Time: day.AddDate(0, 0, 1).Add(8 * time.Hour)})) + + queryT := time.Date(2022, 05, 10, 23, 59, 59, 0, time.UTC) + res, err := s.GetHolderCountsForDay(ctx, []string{"mintA", "mintB", "mintC", "mintD"}, queryT) + require.NoError(t, err) + require.Len(t, res, 2) + + // mintA: close of the day = 8 at 20:00. + require.Contains(t, res, "mintA") + assert.EqualValues(t, 8, res["mintA"].HolderCount) + assert.Equal(t, day.Add(20*time.Hour).Unix(), res["mintA"].Time.Unix()) + + // mintB: its single same-day record. + require.Contains(t, res, "mintB") + assert.EqualValues(t, 100, res["mintB"].HolderCount) + + // mintC has no record on `day`; mintD has none at all — both omitted. + assert.NotContains(t, res, "mintC") + assert.NotContains(t, res, "mintD") + + // Empty input yields an empty map, not an error. + empty, err := s.GetHolderCountsForDay(ctx, nil, queryT) + require.NoError(t, err) + assert.Empty(t, empty) +} + +func testGetHolderCountsInRange(t *testing.T, s holder.Store) { + var counts []currency.HolderCountRecord + + now := time.Now().UTC() + mint := "test-mint" + + for i := 0; i < 100; i++ { + counts = append(counts, currency.HolderCountRecord{ + Mint: mint, + HolderCount: uint64(1000 + i), + Time: now.Add(time.Duration(i) * time.Hour), + }) + } + + for _, item := range counts { + itemCopy := item + require.NoError(t, s.PutHistoricalHolderCount(context.Background(), &itemCopy)) + } + + result, err := s.GetHolderCountsInRange(context.Background(), mint, query.IntervalRaw, counts[0].Time, counts[99].Time, query.Ascending) + require.NoError(t, err) + assert.Equal(t, len(result), 100) + for i, item := range result { + assert.Equal(t, counts[i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, counts[i].HolderCount, item.HolderCount) + } + + result, err = s.GetHolderCountsInRange(context.Background(), mint, query.IntervalRaw, counts[0].Time, counts[49].Time, query.Ascending) + require.NoError(t, err) + assert.Equal(t, len(result), 50) + for i, item := range result { + assert.Equal(t, counts[i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, counts[i].HolderCount, item.HolderCount) + } + + result, err = s.GetHolderCountsInRange(context.Background(), mint, query.IntervalRaw, counts[0].Time, counts[99].Time, query.Descending) + require.NoError(t, err) + assert.Equal(t, len(result), 100) + for i, item := range result { + assert.Equal(t, counts[99-i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, counts[99-i].HolderCount, item.HolderCount) + } + + for _, interval := range query.AllIntervals { + _, err = s.GetHolderCountsInRange(context.Background(), mint, interval, counts[0].Time, counts[99].Time, query.Ascending) + require.NoError(t, err) + } +} + +func testLiveHolderCountRoundTrip(t *testing.T, s holder.Store) { + ctx := context.Background() + mint := "live-holder-mint" + + _, err := s.GetLiveHolderCount(ctx, mint) + assert.Equal(t, currency.ErrNotFound, err) + + t1 := time.Date(2022, 03, 01, 10, 0, 0, 0, time.UTC) + require.NoError(t, s.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: 10, + Time: t1, + })) + + actual, err := s.GetLiveHolderCount(ctx, mint) + require.NoError(t, err) + assert.Equal(t, mint, actual.Mint) + assert.EqualValues(t, 10, actual.HolderCount) + + // Later timestamp advances. + t2 := t1.Add(time.Hour) + require.NoError(t, s.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: 20, + Time: t2, + })) + + actual, err = s.GetLiveHolderCount(ctx, mint) + require.NoError(t, err) + assert.EqualValues(t, 20, actual.HolderCount) + + // Equal timestamp is stale. + assert.Equal(t, currency.ErrStaleHolderState, s.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: 30, + Time: t2, + })) + + // Earlier timestamp is stale. + assert.Equal(t, currency.ErrStaleHolderState, s.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: 30, + Time: t1, + })) + + // Unchanged after stale attempts. + actual, err = s.GetLiveHolderCount(ctx, mint) + require.NoError(t, err) + assert.EqualValues(t, 20, actual.HolderCount) +} + +func testGetAllLiveHolderCounts(t *testing.T, s holder.Store) { + ctx := context.Background() + + _, err := s.GetAllLiveHolderCounts(ctx) + assert.Equal(t, currency.ErrNotFound, err) + + now := time.Now().UTC() + require.NoError(t, s.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: "mint-all-live-1", + HolderCount: 100, + Time: now, + })) + + counts, err := s.GetAllLiveHolderCounts(ctx) + require.NoError(t, err) + assert.Len(t, counts, 1) + assert.EqualValues(t, 100, counts["mint-all-live-1"].HolderCount) + + require.NoError(t, s.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: "mint-all-live-2", + HolderCount: 200, + Time: now, + })) + + counts, err = s.GetAllLiveHolderCounts(ctx) + require.NoError(t, err) + assert.Len(t, counts, 2) + assert.EqualValues(t, 100, counts["mint-all-live-1"].HolderCount) + assert.EqualValues(t, 200, counts["mint-all-live-2"].HolderCount) + + // Updating one mint is reflected. + require.NoError(t, s.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: "mint-all-live-1", + HolderCount: 150, + Time: now.Add(time.Hour), + })) + + counts, err = s.GetAllLiveHolderCounts(ctx) + require.NoError(t, err) + assert.Len(t, counts, 2) + assert.EqualValues(t, 150, counts["mint-all-live-1"].HolderCount) + assert.EqualValues(t, 200, counts["mint-all-live-2"].HolderCount) +} diff --git a/ocp/data/currency/reserve/dynamodb/store.go b/ocp/data/currency/reserve/dynamodb/store.go new file mode 100644 index 0000000..d9c9c70 --- /dev/null +++ b/ocp/data/currency/reserve/dynamodb/store.go @@ -0,0 +1,448 @@ +// Package dynamodb implements the reserve.Store interface on top of DynamoDB. +// +// Two tables, both keyed per mint (the number of mints is unbounded, so each +// record is its own item — never a map-of-all-mints row): +// +// History table — per-mint time series, one partition per resolution: +// +// pk = "#raw" sk = supply (every sample) +// pk = "#hour" sk = supply, ts (close per hour) +// pk = "#day" sk = supply, ts (close per day) +// pk = "#week" sk = supply, ts (close per week, Sunday-start) +// pk = "#month" sk = supply, ts (close per month) +// +// Writes fan out to all resolutions in one transaction: the raw point is a +// conditional Put enforcing once-per-timestamp, each rollup a monotonic +// last-write-wins Update holding the bucket close. Rollup items store `ts` (the +// close sample's time, which also drives the guard); raw items omit it because +// their sort key already is the sample time. Reads serve the resolution matching +// the requested interval; point lookups read the raw resolution at or before t. +// +// Live table — latest snapshot per mint, keyed by pk = "" (no sort key): +// +// pk = "" supply, slot, ts +// +// Live writes spread across mints (each mint its own partition) since the live +// path is hot and event-driven; the slot-monotonic condition keeps the highest +// slot. GetAllLiveReserves scans this table — it holds one item per mint, so the +// scan touches only live state. +package dynamodb + +import ( + "context" + "errors" + "fmt" + "strconv" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" +) + +const ( + attrPK = "pk" + attrSK = "sk" + attrSupply = "supply" + attrSlot = "slot" + attrTS = "ts" + + resRaw = "raw" + resHour = "hour" + resDay = "day" + resWeek = "week" + resMonth = "month" + + // codeConditionalCheckFailed is the DynamoDB cancellation reason code for a + // transaction item whose ConditionExpression evaluated false. + codeConditionalCheckFailed = "ConditionalCheckFailed" +) + +// rollupResolutions are the coarse resolutions maintained alongside the raw +// points on every historical write. +var rollupResolutions = []string{resHour, resDay, resWeek, resMonth} + +type store struct { + client *dynamodb.Client + historyTable string + liveTable string +} + +// New returns a reserve.Store backed by the given DynamoDB tables. Use +// CreateTables to provision them. +func New(client *dynamodb.Client, historyTable, liveTable string) reserve.Store { + return &store{ + client: client, + historyTable: historyTable, + liveTable: liveTable, + } +} + +// PutHistoricalReserve writes the raw sample and refreshes every rollup bucket +// that contains it, atomically. The raw item's attribute_not_exists condition +// enforces once-per-timestamp: a duplicate cancels the transaction and returns +// currency.ErrExists. Rollup guards assume monotonically increasing write +// timestamps per mint (the reserve worker's behavior). +func (s *store) PutHistoricalReserve(ctx context.Context, record *currency.ReserveRecord) error { + if err := record.Validate(); err != nil { + return err + } + + supply := avNU(record.SupplyFromBonding) + tsVal := skN(record.Time) + + transactItems := []types.TransactWriteItem{ + {Put: &types.Put{ + TableName: aws.String(s.historyTable), + Item: map[string]types.AttributeValue{ + attrPK: avS(historyPK(record.Mint, resRaw)), + attrSK: skN(record.Time), + attrSupply: supply, + }, + ConditionExpression: aws.String(fmt.Sprintf("attribute_not_exists(%s)", attrPK)), + }}, + } + for _, res := range rollupResolutions { + transactItems = append(transactItems, types.TransactWriteItem{ + Update: &types.Update{ + TableName: aws.String(s.historyTable), + Key: map[string]types.AttributeValue{ + attrPK: avS(historyPK(record.Mint, res)), + attrSK: skN(bucketStart(record.Time, res)), + }, + UpdateExpression: aws.String("SET #sup = :sup, #ts = :ts"), + ConditionExpression: aws.String("attribute_not_exists(#pk) OR #ts < :ts"), + ExpressionAttributeNames: map[string]string{ + "#pk": attrPK, + "#sup": attrSupply, + "#ts": attrTS, + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":sup": supply, + ":ts": tsVal, + }, + }, + }) + } + + _, err := s.client.TransactWriteItems(ctx, &dynamodb.TransactWriteItemsInput{ + TransactItems: transactItems, + }) + if err != nil { + var tce *types.TransactionCanceledException + if errors.As(err, &tce) && len(tce.CancellationReasons) > 0 && + aws.ToString(tce.CancellationReasons[0].Code) == codeConditionalCheckFailed { + return currency.ErrExists + } + return err + } + return nil +} + +func (s *store) GetReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) { + out, err := s.client.Query(ctx, &dynamodb.QueryInput{ + TableName: aws.String(s.historyTable), + KeyConditionExpression: aws.String(fmt.Sprintf("%s = :pk AND %s <= :sk", attrPK, attrSK)), + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":pk": avS(historyPK(mint, resRaw)), + ":sk": skN(t), + }, + ScanIndexForward: aws.Bool(false), + Limit: aws.Int32(1), + }) + if err != nil { + return nil, err + } + if len(out.Items) == 0 { + return nil, currency.ErrNotFound + } + return historyRecord(mint, out.Items[0]) +} + +func (s *store) GetReservesInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ReserveRecord, error) { + if interval > query.IntervalMonth { + return nil, currency.ErrInvalidInterval + } + if start.IsZero() || end.IsZero() { + return nil, currency.ErrInvalidRange + } + + actualStart, actualEnd := start, end + if start.Unix() > end.Unix() { + actualStart, actualEnd = end, start + } + + // Honor the requested interval directly by reading the matching stored + // resolution. The data is pre-aggregated, so no in-app downsampling is needed. + res := resolutionForInterval(interval) + items, err := s.queryAll(ctx, &dynamodb.QueryInput{ + TableName: aws.String(s.historyTable), + KeyConditionExpression: aws.String(fmt.Sprintf("%s = :pk AND %s BETWEEN :start AND :end", attrPK, attrSK)), + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":pk": avS(historyPK(mint, res)), + // Lower-bound on the bucket containing actualStart so a bucket whose + // start precedes actualStart is still returned. + ":start": skN(bucketStart(actualStart, res)), + ":end": skN(actualEnd), + }, + ScanIndexForward: aws.Bool(ordering != query.Descending), + }) + if err != nil { + return nil, err + } + + records := make([]*currency.ReserveRecord, 0, len(items)) + for _, item := range items { + rec, err := historyRecord(mint, item) + if err != nil { + return nil, err + } + records = append(records, rec) + } + + if len(records) == 0 { + return nil, currency.ErrNotFound + } + return records, nil +} + +// PutLiveReserve upserts the mint's latest reserve, keeping the record with the +// highest slot. The slot-monotonic condition makes a stale (lower or equal slot) +// write return currency.ErrStaleReserveState. +func (s *store) PutLiveReserve(ctx context.Context, record *currency.ReserveRecord) error { + if err := record.Validate(); err != nil { + return err + } + + _, err := s.client.PutItem(ctx, &dynamodb.PutItemInput{ + TableName: aws.String(s.liveTable), + Item: map[string]types.AttributeValue{ + attrPK: avS(record.Mint), + attrSupply: avNU(record.SupplyFromBonding), + attrSlot: avNU(record.Slot), + attrTS: avN(record.Time.UTC().UnixNano()), + }, + ConditionExpression: aws.String("attribute_not_exists(#pk) OR #slot < :slot"), + ExpressionAttributeNames: map[string]string{ + "#pk": attrPK, + "#slot": attrSlot, + }, + ExpressionAttributeValues: map[string]types.AttributeValue{ + ":slot": avNU(record.Slot), + }, + }) + if err != nil { + var ccf *types.ConditionalCheckFailedException + if errors.As(err, &ccf) { + return currency.ErrStaleReserveState + } + return err + } + return nil +} + +func (s *store) GetLiveReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) { + out, err := s.client.GetItem(ctx, &dynamodb.GetItemInput{ + TableName: aws.String(s.liveTable), + Key: map[string]types.AttributeValue{attrPK: avS(mint)}, + }) + if err != nil { + return nil, err + } + if len(out.Item) == 0 { + return nil, currency.ErrNotFound + } + return liveRecord(out.Item) +} + +func (s *store) GetAllLiveReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) { + res := make(map[string]*currency.ReserveRecord) + var startKey map[string]types.AttributeValue + for { + out, err := s.client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(s.liveTable), + ExclusiveStartKey: startKey, + }) + if err != nil { + return nil, err + } + for _, item := range out.Items { + rec, err := liveRecord(item) + if err != nil { + return nil, err + } + res[rec.Mint] = rec + } + if len(out.LastEvaluatedKey) == 0 { + break + } + startKey = out.LastEvaluatedKey + } + + if len(res) == 0 { + return nil, currency.ErrNotFound + } + return res, nil +} + +// reset deletes every item from both tables, for tests. +func (s *store) reset() { + if err := clearTable(context.Background(), s.client, s.historyTable, []string{attrPK, attrSK}); err != nil { + panic(err) + } + if err := clearTable(context.Background(), s.client, s.liveTable, []string{attrPK}); err != nil { + panic(err) + } +} + +// queryAll runs the query, following LastEvaluatedKey until the result set is +// drained, and returns every matched item. +func (s *store) queryAll(ctx context.Context, input *dynamodb.QueryInput) ([]map[string]types.AttributeValue, error) { + var items []map[string]types.AttributeValue + for { + out, err := s.client.Query(ctx, input) + if err != nil { + return nil, err + } + items = append(items, out.Items...) + if len(out.LastEvaluatedKey) == 0 { + break + } + input.ExclusiveStartKey = out.LastEvaluatedKey + } + return items, nil +} + +// historyRecord builds a reserve record from a history item. The mint is the +// query parameter (it is encoded in the pk alongside the resolution, so it is +// not duplicated onto the item). Historical records carry no slot. +func historyRecord(mint string, item map[string]types.AttributeValue) (*currency.ReserveRecord, error) { + supply, err := parseNU(item[attrSupply]) + if err != nil { + return nil, err + } + ts, err := itemTime(item) + if err != nil { + return nil, err + } + return ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: supply, + Time: ts, + }, nil +} + +func liveRecord(item map[string]types.AttributeValue) (*currency.ReserveRecord, error) { + supply, err := parseNU(item[attrSupply]) + if err != nil { + return nil, err + } + slot, err := parseNU(item[attrSlot]) + if err != nil { + return nil, err + } + nanos, err := parseN(item[attrTS]) + if err != nil { + return nil, err + } + return ¤cy.ReserveRecord{ + Mint: asS(item[attrPK]), + SupplyFromBonding: supply, + Slot: slot, + Time: time.Unix(0, nanos).UTC(), + }, nil +} + +// itemTime returns the point's timestamp: the close sample time from `ts` for +// rollup items, or the sample time from the sort key for raw items (which omit +// `ts`). +func itemTime(item map[string]types.AttributeValue) (time.Time, error) { + av, ok := item[attrTS] + if !ok { + av = item[attrSK] + } + nanos, err := parseN(av) + if err != nil { + return time.Time{}, err + } + return time.Unix(0, nanos).UTC(), nil +} + +// resolutionForInterval maps a requested interval to the stored resolution that +// serves it. Sub-hour intervals are served from raw, the finest stored +// resolution. The caller chooses an interval appropriate to the range; this +// store honors it directly rather than re-deriving one. +func resolutionForInterval(interval query.Interval) string { + switch interval { + case query.IntervalHour: + return resHour + case query.IntervalDay: + return resDay + case query.IntervalWeek: + return resWeek + case query.IntervalMonth: + return resMonth + default: // raw, second, minute + return resRaw + } +} + +// bucketStart truncates t to the start of its bucket for the given resolution, +// in UTC. Weeks start on Sunday. +func bucketStart(t time.Time, res string) time.Time { + t = t.UTC() + switch res { + case resHour: + return t.Truncate(time.Hour) + case resDay: + return time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + case resWeek: + day := time.Date(t.Year(), t.Month(), t.Day(), 0, 0, 0, 0, time.UTC) + return day.AddDate(0, 0, -int(day.Weekday())) // time.Weekday: Sunday=0 + case resMonth: + return time.Date(t.Year(), t.Month(), 1, 0, 0, 0, 0, time.UTC) + default: // resRaw + return t + } +} + +func historyPK(mint, res string) string { return mint + "#" + res } + +// skN encodes t's unix-nanos as the numeric sort key, which sorts in +// chronological order. +func skN(t time.Time) types.AttributeValue { return avN(t.UTC().UnixNano()) } + +func avS(v string) types.AttributeValue { return &types.AttributeValueMemberS{Value: v} } +func avN(v int64) types.AttributeValue { + return &types.AttributeValueMemberN{Value: strconv.FormatInt(v, 10)} +} +func avNU(v uint64) types.AttributeValue { + return &types.AttributeValueMemberN{Value: strconv.FormatUint(v, 10)} +} + +func asS(av types.AttributeValue) string { + if s, ok := av.(*types.AttributeValueMemberS); ok { + return s.Value + } + return "" +} + +func parseN(av types.AttributeValue) (int64, error) { + n, ok := av.(*types.AttributeValueMemberN) + if !ok { + return 0, fmt.Errorf("expected number attribute, got %T", av) + } + return strconv.ParseInt(n.Value, 10, 64) +} + +func parseNU(av types.AttributeValue) (uint64, error) { + n, ok := av.(*types.AttributeValueMemberN) + if !ok { + return 0, fmt.Errorf("expected number attribute, got %T", av) + } + return strconv.ParseUint(n.Value, 10, 64) +} diff --git a/ocp/data/currency/reserve/dynamodb/store_test.go b/ocp/data/currency/reserve/dynamodb/store_test.go new file mode 100644 index 0000000..791b78f --- /dev/null +++ b/ocp/data/currency/reserve/dynamodb/store_test.go @@ -0,0 +1,106 @@ +package dynamodb + +import ( + "context" + "os" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "go.uber.org/zap" + + dynamotest "github.com/code-payments/ocp-server/database/dynamodb/test" + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve/tests" +) + +const ( + historyTable = "reserve_history_test" + liveTable = "reserve_live_test" +) + +var testEnv *dynamotest.TestEnv + +func TestMain(m *testing.M) { + log := zap.Must(zap.NewDevelopment()) + + env, err := dynamotest.NewTestEnv() + if err != nil { + log.With(zap.Error(err)).Error("Error creating dynamodb test environment") + os.Exit(1) + } + + testEnv = env + + os.Exit(m.Run()) +} + +func newTestStore(t *testing.T) *store { + require.NoError(t, CreateTables(context.Background(), testEnv.Client, historyTable, liveTable)) + s := New(testEnv.Client, historyTable, liveTable).(*store) + s.reset() + return s +} + +func TestReserve_DynamoDBStore(t *testing.T) { + testStore := newTestStore(t) + teardown := func() { + testStore.reset() + } + tests.RunStoreTests(t, testStore, teardown) +} + +// TestReserve_DynamoDBRollups verifies that a coarse interval reads from the +// matching rollup partition and that each bucket holds the close (the most +// recent sample within the bucket). +func TestReserve_DynamoDBRollups(t *testing.T) { + s := newTestStore(t) + defer s.reset() + ctx := context.Background() + mint := "rollup-mint" + + // Two samples in hour 10 (close = 200 at 10:45), one in hour 11. + samples := []struct { + at time.Time + supply uint64 + }{ + {time.Date(2021, 06, 01, 10, 05, 0, 0, time.UTC), 100}, + {time.Date(2021, 06, 01, 10, 45, 0, 0, time.UTC), 200}, + {time.Date(2021, 06, 01, 11, 30, 0, 0, time.UTC), 300}, + } + for _, sample := range samples { + require.NoError(t, s.PutHistoricalReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: sample.supply, + Time: sample.at, + })) + } + + start := time.Date(2021, 06, 01, 10, 0, 0, 0, time.UTC) + end := time.Date(2021, 06, 01, 12, 0, 0, 0, time.UTC) + + hourly, err := s.GetReservesInRange(ctx, mint, query.IntervalHour, start, end, query.Ascending) + require.NoError(t, err) + require.Len(t, hourly, 2) + assert.Equal(t, samples[1].at.Unix(), hourly[0].Time.Unix()) + assert.EqualValues(t, 200, hourly[0].SupplyFromBonding) + assert.Equal(t, samples[2].at.Unix(), hourly[1].Time.Unix()) + assert.EqualValues(t, 300, hourly[1].SupplyFromBonding) + + // Raw still has every sample. + raw, err := s.GetReservesInRange(ctx, mint, query.IntervalRaw, start, end, query.Ascending) + require.NoError(t, err) + assert.Len(t, raw, len(samples)) +} + +func TestReserve_DynamoDBResolutionForInterval(t *testing.T) { + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalRaw)) + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalSecond)) + assert.Equal(t, resRaw, resolutionForInterval(query.IntervalMinute)) + assert.Equal(t, resHour, resolutionForInterval(query.IntervalHour)) + assert.Equal(t, resDay, resolutionForInterval(query.IntervalDay)) + assert.Equal(t, resWeek, resolutionForInterval(query.IntervalWeek)) + assert.Equal(t, resMonth, resolutionForInterval(query.IntervalMonth)) +} diff --git a/ocp/data/currency/reserve/dynamodb/table.go b/ocp/data/currency/reserve/dynamodb/table.go new file mode 100644 index 0000000..72fb535 --- /dev/null +++ b/ocp/data/currency/reserve/dynamodb/table.go @@ -0,0 +1,106 @@ +package dynamodb + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/dynamodb" + "github.com/aws/aws-sdk-go-v2/service/dynamodb/types" +) + +// maxBatchWriteItems is DynamoDB's per-call BatchWriteItem limit. +const maxBatchWriteItems = 25 + +// CreateTables provisions the history and live reserve tables with on-demand +// billing. The history table is keyed by (pk, sk); the live table is keyed by pk +// only (one item per mint) so it can be scanned cheaply for all mints. It is +// idempotent and blocks until both tables are ACTIVE. +func CreateTables(ctx context.Context, client *dynamodb.Client, historyTable, liveTable string) error { + inputs := []*dynamodb.CreateTableInput{ + { + TableName: aws.String(historyTable), + BillingMode: types.BillingModePayPerRequest, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String(attrPK), AttributeType: types.ScalarAttributeTypeS}, + {AttributeName: aws.String(attrSK), AttributeType: types.ScalarAttributeTypeN}, + }, + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String(attrPK), KeyType: types.KeyTypeHash}, + {AttributeName: aws.String(attrSK), KeyType: types.KeyTypeRange}, + }, + }, + { + TableName: aws.String(liveTable), + BillingMode: types.BillingModePayPerRequest, + AttributeDefinitions: []types.AttributeDefinition{ + {AttributeName: aws.String(attrPK), AttributeType: types.ScalarAttributeTypeS}, + }, + KeySchema: []types.KeySchemaElement{ + {AttributeName: aws.String(attrPK), KeyType: types.KeyTypeHash}, + }, + }, + } + + for _, input := range inputs { + if _, err := client.CreateTable(ctx, input); err != nil { + var inUse *types.ResourceInUseException + if !errors.As(err, &inUse) { + return err + } + // Already exists; still ensure it is ACTIVE before returning. + } + if err := dynamodb.NewTableExistsWaiter(client).Wait(ctx, &dynamodb.DescribeTableInput{ + TableName: input.TableName, + }, 2*time.Minute); err != nil { + return err + } + } + return nil +} + +// clearTable deletes every item from the table, for tests. keyAttrs are the +// table's key attribute names (one for a hash-only table, two for composite). +func clearTable(ctx context.Context, client *dynamodb.Client, table string, keyAttrs []string) error { + var startKey map[string]types.AttributeValue + for { + out, err := client.Scan(ctx, &dynamodb.ScanInput{ + TableName: aws.String(table), + ProjectionExpression: aws.String(strings.Join(keyAttrs, ", ")), + ExclusiveStartKey: startKey, + }) + if err != nil { + return err + } + + for start := 0; start < len(out.Items); start += maxBatchWriteItems { + end := start + maxBatchWriteItems + if end > len(out.Items) { + end = len(out.Items) + } + requests := make([]types.WriteRequest, 0, end-start) + for _, item := range out.Items[start:end] { + key := make(map[string]types.AttributeValue, len(keyAttrs)) + for _, a := range keyAttrs { + key[a] = item[a] + } + requests = append(requests, types.WriteRequest{ + DeleteRequest: &types.DeleteRequest{Key: key}, + }) + } + if _, err := client.BatchWriteItem(ctx, &dynamodb.BatchWriteItemInput{ + RequestItems: map[string][]types.WriteRequest{table: requests}, + }); err != nil { + return err + } + } + + if len(out.LastEvaluatedKey) == 0 { + break + } + startKey = out.LastEvaluatedKey + } + return nil +} diff --git a/ocp/data/currency/reserve/memory/store.go b/ocp/data/currency/reserve/memory/store.go new file mode 100644 index 0000000..719840a --- /dev/null +++ b/ocp/data/currency/reserve/memory/store.go @@ -0,0 +1,169 @@ +// Package memory provides an in-memory reserve.Store implementation for fast +// unit tests. +package memory + +import ( + "context" + "sort" + "sync" + "time" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" +) + +type store struct { + mu sync.Mutex + historical []*currency.ReserveRecord + lastID uint64 + live map[string]*currency.ReserveRecord +} + +type reserveByTime []*currency.ReserveRecord + +func (a reserveByTime) Len() int { return len(a) } +func (a reserveByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a reserveByTime) Less(i, j int) bool { + // DESC order (most recent first) + return a[i].Time.Unix() > a[j].Time.Unix() +} + +func New() reserve.Store { + return &store{ + historical: make([]*currency.ReserveRecord, 0), + lastID: 1, + live: make(map[string]*currency.ReserveRecord), + } +} + +func (s *store) reset() { + s.mu.Lock() + s.historical = make([]*currency.ReserveRecord, 0) + s.lastID = 1 + s.live = make(map[string]*currency.ReserveRecord) + s.mu.Unlock() +} + +func (s *store) PutHistoricalReserve(ctx context.Context, record *currency.ReserveRecord) error { + if err := record.Validate(); err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + for _, item := range s.historical { + if item.Mint == record.Mint && item.Time.Unix() == record.Time.Unix() { + return currency.ErrExists + } + } + + cloned := record.Clone() + cloned.Id = s.lastID + s.historical = append(s.historical, cloned) + s.lastID++ + + return nil +} + +func (s *store) GetReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var results []*currency.ReserveRecord + for _, item := range s.historical { + if item.Mint == mint && item.Time.Unix() <= t.Unix() { + results = append(results, item) + } + } + + if len(results) == 0 { + return nil, currency.ErrNotFound + } + + sort.Sort(reserveByTime(results)) + + return results[0].Clone(), nil +} + +func (s *store) GetReservesInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ReserveRecord, error) { + if interval > query.IntervalMonth { + return nil, currency.ErrInvalidInterval + } + if start.IsZero() || end.IsZero() { + return nil, currency.ErrInvalidRange + } + + actualStart, actualEnd := start, end + if start.Unix() > end.Unix() { + actualStart, actualEnd = end, start + } + + s.mu.Lock() + defer s.mu.Unlock() + + var all []*currency.ReserveRecord + for _, item := range s.historical { + if item.Mint == mint && item.Time.Unix() >= actualStart.Unix() && item.Time.Unix() <= actualEnd.Unix() { + all = append(all, item.Clone()) + } + } + + // TODO: handle the interval + + if len(all) == 0 { + return nil, currency.ErrNotFound + } + + sort.Sort(reserveByTime(all)) // DESC + if ordering == query.Ascending { + for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 { + all[i], all[j] = all[j], all[i] + } + } + + return all, nil +} + +func (s *store) PutLiveReserve(ctx context.Context, record *currency.ReserveRecord) error { + if err := record.Validate(); err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + if existing, ok := s.live[record.Mint]; ok && record.Slot <= existing.Slot { + return currency.ErrStaleReserveState + } + + s.live[record.Mint] = record.Clone() + return nil +} + +func (s *store) GetLiveReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + record, ok := s.live[mint] + if !ok { + return nil, currency.ErrNotFound + } + return record.Clone(), nil +} + +func (s *store) GetAllLiveReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.live) == 0 { + return nil, currency.ErrNotFound + } + + res := make(map[string]*currency.ReserveRecord, len(s.live)) + for mint, record := range s.live { + res[mint] = record.Clone() + } + return res, nil +} diff --git a/ocp/data/currency/reserve/memory/store_test.go b/ocp/data/currency/reserve/memory/store_test.go new file mode 100644 index 0000000..9a19746 --- /dev/null +++ b/ocp/data/currency/reserve/memory/store_test.go @@ -0,0 +1,15 @@ +package memory + +import ( + "testing" + + "github.com/code-payments/ocp-server/ocp/data/currency/reserve/tests" +) + +func TestReserve_MemoryStore(t *testing.T) { + testStore := New() + teardown := func() { + testStore.(*store).reset() + } + tests.RunStoreTests(t, testStore, teardown) +} diff --git a/ocp/data/currency/reserve/store.go b/ocp/data/currency/reserve/store.go new file mode 100644 index 0000000..de29b62 --- /dev/null +++ b/ocp/data/currency/reserve/store.go @@ -0,0 +1,65 @@ +// Package reserve defines a focused store for currency creator mint reserve +// states. +// +// It mirrors the reserve portion of the larger ocp/data/currency store, reusing +// that package's record type (currency.ReserveRecord) and error sentinels +// (currency.ErrNotFound, currency.ErrExists, currency.ErrInvalidRange, +// currency.ErrInvalidInterval, currency.ErrStaleReserveState). A DynamoDB-backed +// implementation lives in the dynamodb subpackage and an in-memory +// implementation lives in memory. +// +// Records are keyed per mint (the number of mints is unbounded), so every record +// is a single item — there is no map-of-all-mints row. +package reserve + +import ( + "context" + "time" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" +) + +type Store interface { + // PutHistoricalReserve puts a currency creator mint reserve record into the + // store. + // + // currency.ErrExists is returned if a record already exists for the mint at the + // provided time. + PutHistoricalReserve(ctx context.Context, record *currency.ReserveRecord) error + + // GetReserveAtTime gets reserve state for a given currency creator mint at a + // point in time. The most recent record at or before the requested time is + // returned. + // + // currency.ErrNotFound is returned if no reserve data exists at or before the + // provided time. + GetReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) + + // GetReservesInRange gets the reserve records for a range of time given a + // currency creator mint and interval. + // + // currency.ErrNotFound is returned if the mint or the reserves for the mint cannot be found + // currency.ErrInvalidRange is returned if the range is not valid + // currency.ErrInvalidInterval is returned if the interval is not valid + GetReservesInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ReserveRecord, error) + + // PutLiveReserve upserts the latest reserve record for a currency creator mint. + // An upsert is only performed if the provided slot is greater than the slot + // currently stored. + // + // currency.ErrStaleReserveState is returned if the provided slot is not greater + // than the stored slot. + PutLiveReserve(ctx context.Context, record *currency.ReserveRecord) error + + // GetLiveReserve gets the latest live reserve record for a currency creator mint. + // + // currency.ErrNotFound is returned if no live reserve record exists for the provided mint. + GetLiveReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) + + // GetAllLiveReserves gets the latest live reserve records for all currency + // creator mints. + // + // currency.ErrNotFound is returned if no live reserve records exist. + GetAllLiveReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) +} diff --git a/ocp/data/currency/reserve/tests/tests.go b/ocp/data/currency/reserve/tests/tests.go new file mode 100644 index 0000000..99fcc9a --- /dev/null +++ b/ocp/data/currency/reserve/tests/tests.go @@ -0,0 +1,231 @@ +// Package tests holds the shared conformance suite run against every +// reserve.Store implementation. +package tests + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" +) + +func RunStoreTests(t *testing.T, s reserve.Store, teardown func()) { + for _, tf := range []func(t *testing.T, s reserve.Store){ + testReserveRoundTrip, + testGetReservesInRange, + testLiveReserveRoundTrip, + testGetAllLiveReserves, + } { + tf(t, s) + teardown() + } +} + +func testReserveRoundTrip(t *testing.T, s reserve.Store) { + now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) + mint := "mint" + + record, err := s.GetReserveAtTime(context.Background(), mint, now) + assert.Nil(t, record) + assert.Equal(t, currency.ErrNotFound, err) + + expected := ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 1, + Time: now, + } + require.NoError(t, s.PutHistoricalReserve(context.Background(), expected)) + + // Duplicate timestamp for the mint fails. + assert.Equal(t, currency.ErrExists, s.PutHistoricalReserve(context.Background(), expected)) + + actual, err := s.GetReserveAtTime(context.Background(), mint, now) + require.NoError(t, err) + assert.Equal(t, now.Unix(), actual.Time.Unix()) + assert.EqualValues(t, expected.SupplyFromBonding, actual.SupplyFromBonding) + + // A later time returns the most recent record at or before it. + actual, err = s.GetReserveAtTime(context.Background(), mint, time.Date(2021, 01, 29, 14, 0, 5, 0, time.UTC)) + require.NoError(t, err) + assert.Equal(t, now.Unix(), actual.Time.Unix()) + assert.EqualValues(t, expected.SupplyFromBonding, actual.SupplyFromBonding) + + // A later day still returns it (no same-day requirement). + tomorrow := time.Date(2021, 01, 30, 0, 0, 0, 0, time.UTC) + actual, err = s.GetReserveAtTime(context.Background(), mint, tomorrow) + require.NoError(t, err) + assert.Equal(t, now.Unix(), actual.Time.Unix()) + + // A time before any record exists is not found. + before := time.Date(2021, 01, 28, 0, 0, 0, 0, time.UTC) + actual, err = s.GetReserveAtTime(context.Background(), mint, before) + assert.Nil(t, actual) + assert.Equal(t, currency.ErrNotFound, err) + + // A different mint is independent. + _, err = s.GetReserveAtTime(context.Background(), "other-mint", now) + assert.Equal(t, currency.ErrNotFound, err) +} + +func testGetReservesInRange(t *testing.T, s reserve.Store) { + var reserves []currency.ReserveRecord + + now := time.Now().UTC() + mint := "test-mint" + + for i := 0; i < 100; i++ { + reserves = append(reserves, currency.ReserveRecord{ + Mint: mint, + SupplyFromBonding: uint64(1000 + i), + Time: now.Add(time.Duration(i) * time.Hour), + }) + } + + record, err := s.GetReserveAtTime(context.Background(), mint, reserves[0].Time) + assert.Nil(t, record) + assert.Equal(t, currency.ErrNotFound, err) + + for _, item := range reserves { + itemCopy := item + require.NoError(t, s.PutHistoricalReserve(context.Background(), &itemCopy)) + } + + result, err := s.GetReservesInRange(context.Background(), mint, query.IntervalRaw, reserves[0].Time, reserves[99].Time, query.Ascending) + require.NoError(t, err) + assert.Equal(t, len(result), 100) + for i, item := range result { + assert.Equal(t, reserves[i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, reserves[i].SupplyFromBonding, item.SupplyFromBonding) + } + + result, err = s.GetReservesInRange(context.Background(), mint, query.IntervalRaw, reserves[0].Time, reserves[49].Time, query.Ascending) + require.NoError(t, err) + assert.Equal(t, len(result), 50) + for i, item := range result { + assert.Equal(t, reserves[i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, reserves[i].SupplyFromBonding, item.SupplyFromBonding) + } + + result, err = s.GetReservesInRange(context.Background(), mint, query.IntervalRaw, reserves[0].Time, reserves[99].Time, query.Descending) + require.NoError(t, err) + assert.Equal(t, len(result), 100) + for i, item := range result { + assert.Equal(t, reserves[99-i].Time.Unix(), item.Time.Unix()) + assert.EqualValues(t, reserves[99-i].SupplyFromBonding, item.SupplyFromBonding) + } + + for _, interval := range query.AllIntervals { + _, err = s.GetReservesInRange(context.Background(), mint, interval, reserves[0].Time, reserves[99].Time, query.Ascending) + require.NoError(t, err) + } +} + +func testLiveReserveRoundTrip(t *testing.T, s reserve.Store) { + ctx := context.Background() + mint := "live-reserve-mint" + + _, err := s.GetLiveReserve(ctx, mint) + assert.Equal(t, currency.ErrNotFound, err) + + require.NoError(t, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 1000, + Slot: 100, + Time: time.Now(), + })) + + actual, err := s.GetLiveReserve(ctx, mint) + require.NoError(t, err) + assert.Equal(t, mint, actual.Mint) + assert.EqualValues(t, 1000, actual.SupplyFromBonding) + assert.EqualValues(t, 100, actual.Slot) + + // Higher slot advances. + require.NoError(t, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 2000, + Slot: 200, + Time: time.Now(), + })) + + actual, err = s.GetLiveReserve(ctx, mint) + require.NoError(t, err) + assert.EqualValues(t, 2000, actual.SupplyFromBonding) + assert.EqualValues(t, 200, actual.Slot) + + // Equal slot is stale. + assert.Equal(t, currency.ErrStaleReserveState, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 3000, + Slot: 200, + Time: time.Now(), + })) + + // Lower slot is stale. + assert.Equal(t, currency.ErrStaleReserveState, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 3000, + Slot: 50, + Time: time.Now(), + })) + + // Unchanged after stale attempts. + actual, err = s.GetLiveReserve(ctx, mint) + require.NoError(t, err) + assert.EqualValues(t, 2000, actual.SupplyFromBonding) + assert.EqualValues(t, 200, actual.Slot) +} + +func testGetAllLiveReserves(t *testing.T, s reserve.Store) { + ctx := context.Background() + + _, err := s.GetAllLiveReserves(ctx) + assert.Equal(t, currency.ErrNotFound, err) + + require.NoError(t, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: "mint-all-live-1", + SupplyFromBonding: 1000, + Slot: 100, + Time: time.Now(), + })) + + reserves, err := s.GetAllLiveReserves(ctx) + require.NoError(t, err) + assert.Len(t, reserves, 1) + assert.EqualValues(t, 1000, reserves["mint-all-live-1"].SupplyFromBonding) + assert.EqualValues(t, 100, reserves["mint-all-live-1"].Slot) + + require.NoError(t, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: "mint-all-live-2", + SupplyFromBonding: 2000, + Slot: 200, + Time: time.Now(), + })) + + reserves, err = s.GetAllLiveReserves(ctx) + require.NoError(t, err) + assert.Len(t, reserves, 2) + assert.EqualValues(t, 1000, reserves["mint-all-live-1"].SupplyFromBonding) + assert.EqualValues(t, 2000, reserves["mint-all-live-2"].SupplyFromBonding) + assert.EqualValues(t, 200, reserves["mint-all-live-2"].Slot) + + // Updating one mint is reflected. + require.NoError(t, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: "mint-all-live-1", + SupplyFromBonding: 1500, + Slot: 150, + Time: time.Now(), + })) + + reserves, err = s.GetAllLiveReserves(ctx) + require.NoError(t, err) + assert.Len(t, reserves, 2) + assert.EqualValues(t, 1500, reserves["mint-all-live-1"].SupplyFromBonding) + assert.EqualValues(t, 150, reserves["mint-all-live-1"].Slot) +} diff --git a/ocp/rpc/currency/historical_data.go b/ocp/rpc/currency/historical_data.go index ee0196b..64bcec6 100644 --- a/ocp/rpc/currency/historical_data.go +++ b/ocp/rpc/currency/historical_data.go @@ -292,8 +292,8 @@ func getTimeRangeForPredefinedRange(predefinedRange currencypb.PredefinedRange, } else if currencyAge < 2*7*24*time.Hour { interval = query.IntervalHour } - // For all time, go back 100 years - return now.Add(-100 * 365 * 24 * time.Hour), now, interval + // For all time, go back 20 years + return now.Add(-20 * 365 * 24 * time.Hour), now, interval } } From 79b1dbc88788a9b8c96bff662351cef9bcd8c901 Mon Sep 17 00:00:00 2001 From: jeffyanta Date: Wed, 17 Jun 2026 09:06:16 -0400 Subject: [PATCH 2/5] Add a caching exchange store --- ocp/data/currency/exchange/cache/store.go | 89 +++++++++++++++++++ .../currency/exchange/cache/store_test.go | 67 ++++++++++++++ 2 files changed, 156 insertions(+) create mode 100644 ocp/data/currency/exchange/cache/store.go create mode 100644 ocp/data/currency/exchange/cache/store_test.go diff --git a/ocp/data/currency/exchange/cache/store.go b/ocp/data/currency/exchange/cache/store.go new file mode 100644 index 0000000..ec71f96 --- /dev/null +++ b/ocp/data/currency/exchange/cache/store.go @@ -0,0 +1,89 @@ +// Package cache provides an exchange.Store decorator that caches single- and +// all-symbol rate lookups in front of a wrapped store. +// +// Reads are keyed by a coarse time bucket that doubles as the freshness window, +// and a full set of rates is weighted more heavily than a single-symbol entry. +// Range and history reads, and all writes, pass straight through to the wrapped +// store. +package cache + +import ( + "context" + "fmt" + "time" + + lrucache "github.com/code-payments/ocp-server/cache" + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" +) + +const ( + // maxCacheBudget bounds the weighted size of the rate cache before the + // least-recently-used entries are evicted. + maxCacheBudget = 100_000 + + // singleRateWeight and multiRateWeight weight cached entries against the + // budget. A full set of rates spans many symbols, so it costs + // proportionally more than a single-symbol entry. + singleRateWeight = 1 + multiRateWeight = 100 + + // cacheBucket is the time granularity used to build cache keys. Lookups that + // truncate to the same bucket share a cached result, which doubles as the + // effective freshness window for cached rates. + cacheBucket = 5 * time.Minute +) + +type store struct { + backing exchange.Store + cache lrucache.Cache +} + +// New returns an exchange.Store that caches reads in front of backing. +func New(backing exchange.Store) exchange.Store { + return &store{ + backing: backing, + cache: lrucache.NewCache(maxCacheBudget), + } +} + +func (s *store) PutExchangeRates(ctx context.Context, record *currency.MultiRateRecord) error { + return s.backing.PutExchangeRates(ctx, record) +} + +func (s *store) GetExchangeRate(ctx context.Context, symbol string, t time.Time) (*currency.ExchangeRateRecord, error) { + key := fmt.Sprintf("%s:%s", symbol, t.Truncate(cacheBucket).Format(time.RFC3339)) + if cached, ok := s.cache.Retrieve(key); ok { + return cached.(*currency.ExchangeRateRecord), nil + } + + rate, err := s.backing.GetExchangeRate(ctx, symbol, t) + if err != nil { + return nil, err + } + + s.cache.Insert(key, rate, singleRateWeight) + + return rate, nil +} + +func (s *store) GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) { + key := fmt.Sprintf("everything:%s", t.Truncate(cacheBucket).Format(time.RFC3339)) + if cached, ok := s.cache.Retrieve(key); ok { + return cached.(*currency.MultiRateRecord), nil + } + + rates, err := s.backing.GetAllExchangeRates(ctx, t) + if err != nil { + return nil, err + } + + s.cache.Insert(key, rates, multiRateWeight) + + return rates, nil +} + +func (s *store) GetExchangeRatesInRange(ctx context.Context, symbol string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ExchangeRateRecord, error) { + return s.backing.GetExchangeRatesInRange(ctx, symbol, interval, start, end, ordering) +} diff --git a/ocp/data/currency/exchange/cache/store_test.go b/ocp/data/currency/exchange/cache/store_test.go new file mode 100644 index 0000000..5090b60 --- /dev/null +++ b/ocp/data/currency/exchange/cache/store_test.go @@ -0,0 +1,67 @@ +package cache + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange/memory" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange/tests" +) + +func TestExchange_CachedStore(t *testing.T) { + testStore := New(memory.New()).(*store) + teardown := func() { + testStore.backing = memory.New() + testStore.cache.Clear() + } + tests.RunStoreTests(t, testStore, teardown) +} + +// TestExchange_CachedReadsServedFromCache verifies that once a rate is read, a +// later read in the same time bucket is served from the cache even after the +// backing store no longer holds it, while a read in a different bucket falls +// through to the (now empty) backing store. +func TestExchange_CachedReadsServedFromCache(t *testing.T) { + ctx := context.Background() + now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) + + s := New(memory.New()).(*store) + require.NoError(t, s.PutExchangeRates(ctx, ¤cy.MultiRateRecord{ + Time: now, + Rates: map[string]float64{"usd": 0.000055, "cad": 0.00007}, + })) + + // Prime the cache with a single- and all-symbol read. + single, err := s.GetExchangeRate(ctx, "usd", now) + require.NoError(t, err) + assert.EqualValues(t, 0.000055, single.Rate) + + all, err := s.GetAllExchangeRates(ctx, now) + require.NoError(t, err) + assert.Len(t, all.Rates, 2) + + // Drop the backing data. Reads in the same bucket are still served. + s.backing = memory.New() + + single, err = s.GetExchangeRate(ctx, "usd", now) + require.NoError(t, err) + assert.EqualValues(t, 0.000055, single.Rate) + + all, err = s.GetAllExchangeRates(ctx, now) + require.NoError(t, err) + assert.Len(t, all.Rates, 2) + + // A read truncating to a different bucket misses the cache and falls + // through to the empty backing store. + otherBucket := now.Add(cacheBucket) + _, err = s.GetExchangeRate(ctx, "usd", otherBucket) + assert.Equal(t, currency.ErrNotFound, err) + + _, err = s.GetAllExchangeRates(ctx, otherBucket) + assert.Equal(t, currency.ErrNotFound, err) +} From eb4782f8ddb1dcaa6fc95fa0e465e3ce1de3d027 Mon Sep 17 00:00:00 2001 From: jeffyanta Date: Wed, 17 Jun 2026 09:30:05 -0400 Subject: [PATCH 3/5] Complete refactor of currency stores --- ocp/data/currency/exchange/store.go | 7 - ocp/data/currency/holder/store.go | 7 - ocp/data/currency/memory/store.go | 594 -------------- ocp/data/currency/metadata/memory/store.go | 197 +++++ .../{ => metadata}/memory/store_test.go | 4 +- ocp/data/currency/metadata/postgres/model.go | 306 ++++++++ ocp/data/currency/metadata/postgres/store.go | 75 ++ .../{ => metadata}/postgres/store_test.go | 59 +- ocp/data/currency/metadata/store.go | 41 + .../currency/{ => metadata}/tests/tests.go | 574 +------------- ocp/data/currency/postgres/model.go | 739 ------------------ ocp/data/currency/postgres/store.go | 318 -------- ocp/data/currency/reserve/store.go | 7 - ocp/data/currency/store.go | 143 ---- ocp/data/internal.go | 136 +--- ocp/data/provider.go | 4 - 16 files changed, 645 insertions(+), 2566 deletions(-) delete mode 100644 ocp/data/currency/memory/store.go create mode 100644 ocp/data/currency/metadata/memory/store.go rename ocp/data/currency/{ => metadata}/memory/store_test.go (56%) create mode 100644 ocp/data/currency/metadata/postgres/model.go create mode 100644 ocp/data/currency/metadata/postgres/store.go rename ocp/data/currency/{ => metadata}/postgres/store_test.go (60%) create mode 100644 ocp/data/currency/metadata/store.go rename ocp/data/currency/{ => metadata}/tests/tests.go (58%) delete mode 100644 ocp/data/currency/postgres/model.go delete mode 100644 ocp/data/currency/postgres/store.go diff --git a/ocp/data/currency/exchange/store.go b/ocp/data/currency/exchange/store.go index 4d2811b..c3b495f 100644 --- a/ocp/data/currency/exchange/store.go +++ b/ocp/data/currency/exchange/store.go @@ -1,11 +1,4 @@ // Package exchange defines a focused store for core-mint exchange rate records. -// -// It mirrors the exchange-rate portion of the larger ocp/data/currency store, -// reusing that package's record types (currency.ExchangeRateRecord, -// currency.MultiRateRecord) and error sentinels (currency.ErrNotFound, -// currency.ErrExists, currency.ErrInvalidRange, currency.ErrInvalidInterval) so -// callers see identical semantics. A DynamoDB-backed implementation lives in the -// dynamodb subpackage and an in-memory implementation lives in memory. package exchange import ( diff --git a/ocp/data/currency/holder/store.go b/ocp/data/currency/holder/store.go index 694cb51..804f31d 100644 --- a/ocp/data/currency/holder/store.go +++ b/ocp/data/currency/holder/store.go @@ -1,13 +1,6 @@ // Package holder defines a focused store for currency creator mint holder // counts. // -// It mirrors the holder-count portion of the larger ocp/data/currency store, -// reusing that package's record type (currency.HolderCountRecord) and error -// sentinels (currency.ErrNotFound, currency.ErrExists, currency.ErrInvalidRange, -// currency.ErrInvalidInterval, currency.ErrStaleHolderState). A DynamoDB-backed -// implementation lives in the dynamodb subpackage and an in-memory -// implementation lives in memory. -// // Records are keyed per mint (the number of mints is unbounded), so every record // is a single item — there is no map-of-all-mints row. package holder diff --git a/ocp/data/currency/memory/store.go b/ocp/data/currency/memory/store.go deleted file mode 100644 index 0a5e234..0000000 --- a/ocp/data/currency/memory/store.go +++ /dev/null @@ -1,594 +0,0 @@ -package memory - -import ( - "context" - "sort" - "strings" - "sync" - "time" - - "github.com/code-payments/ocp-server/database/query" - "github.com/code-payments/ocp-server/ocp/data/currency" -) - -const ( - dateFormat = "2006-01-02" -) - -type store struct { - mu sync.Mutex - exchangeRateRecords []*currency.ExchangeRateRecord - lastExchangeRateIndex uint64 - metadataRecords []*currency.MetadataRecord - lastMetadataIndex uint64 - historicalReserveRecords []*currency.ReserveRecord - lastHistoricalReserveIndex uint64 - liveReserveRecords map[string]*currency.ReserveRecord - lastLiveReserveIndex uint64 - historicalHolderCountRecords []*currency.HolderCountRecord - lastHistoricalHolderCountIndex uint64 - liveHolderCountRecords map[string]*currency.HolderCountRecord - lastLiveHolderCountIndex uint64 -} - -type RateByTime []*currency.ExchangeRateRecord - -func (a RateByTime) Len() int { return len(a) } -func (a RateByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a RateByTime) Less(i, j int) bool { - // DESC order (most recent first) - return a[i].Time.Unix() > a[j].Time.Unix() -} - -type ReserveByTime []*currency.ReserveRecord - -func (a ReserveByTime) Len() int { return len(a) } -func (a ReserveByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a ReserveByTime) Less(i, j int) bool { - // DESC order (most recent first) - return a[i].Time.Unix() > a[j].Time.Unix() -} - -type HolderCountByTime []*currency.HolderCountRecord - -func (a HolderCountByTime) Len() int { return len(a) } -func (a HolderCountByTime) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a HolderCountByTime) Less(i, j int) bool { - // DESC order (most recent first) - return a[i].Time.Unix() > a[j].Time.Unix() -} - -func New() currency.Store { - return &store{ - exchangeRateRecords: make([]*currency.ExchangeRateRecord, 0), - lastExchangeRateIndex: 1, - liveReserveRecords: make(map[string]*currency.ReserveRecord), - lastLiveReserveIndex: 1, - liveHolderCountRecords: make(map[string]*currency.HolderCountRecord), - lastLiveHolderCountIndex: 1, - } -} - -func (s *store) reset() { - s.mu.Lock() - s.exchangeRateRecords = make([]*currency.ExchangeRateRecord, 0) - s.lastExchangeRateIndex = 1 - s.metadataRecords = make([]*currency.MetadataRecord, 0) - s.lastMetadataIndex = 1 - s.historicalReserveRecords = make([]*currency.ReserveRecord, 0) - s.lastHistoricalReserveIndex = 1 - s.liveReserveRecords = make(map[string]*currency.ReserveRecord) - s.lastLiveReserveIndex = 1 - s.historicalHolderCountRecords = make([]*currency.HolderCountRecord, 0) - s.lastHistoricalHolderCountIndex = 1 - s.liveHolderCountRecords = make(map[string]*currency.HolderCountRecord) - s.lastLiveHolderCountIndex = 1 - s.mu.Unlock() -} - -func (s *store) PutExchangeRates(ctx context.Context, data *currency.MultiRateRecord) error { - s.mu.Lock() - defer s.mu.Unlock() - - // Not ideal but fine for testing the currency store - for _, item := range s.exchangeRateRecords { - if item.Time.Unix() == data.Time.Unix() { - return currency.ErrExists - } - } - - for symbol, item := range data.Rates { - s.exchangeRateRecords = append(s.exchangeRateRecords, ¤cy.ExchangeRateRecord{ - Id: s.lastExchangeRateIndex, - Rate: item, - Time: data.Time, - Symbol: symbol, - }) - s.lastExchangeRateIndex = s.lastExchangeRateIndex + 1 - } - - return nil -} - -func (s *store) GetExchangeRate(ctx context.Context, symbol string, t time.Time) (*currency.ExchangeRateRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - // Not ideal but fine for testing the currency store - var results []*currency.ExchangeRateRecord - for _, item := range s.exchangeRateRecords { - if item.Symbol == symbol && item.Time.Unix() <= t.Unix() && item.Time.Format(dateFormat) == t.Format(dateFormat) { - results = append(results, item) - } - } - - if len(results) == 0 { - return nil, currency.ErrNotFound - } - - sort.Sort(RateByTime(results)) - - return results[0], nil -} - -func (s *store) GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - // Not ideal but fine for testing the currency store - sort.Sort(RateByTime(s.exchangeRateRecords)) - - result := currency.MultiRateRecord{ - Rates: make(map[string]float64), - } - for _, item := range s.exchangeRateRecords { - if item.Time.Unix() <= t.Unix() && item.Time.Format(dateFormat) == t.Format(dateFormat) { - result.Rates[item.Symbol] = item.Rate - result.Time = item.Time - } - } - - if len(result.Rates) == 0 { - return nil, currency.ErrNotFound - } - - return &result, nil -} - -func (s *store) GetExchangeRatesInRange(ctx context.Context, symbol string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ExchangeRateRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - sort.Sort(RateByTime(s.exchangeRateRecords)) - - // Not ideal but fine for testing the currency store - var all []*currency.ExchangeRateRecord - for _, item := range s.exchangeRateRecords { - if item.Symbol == symbol && item.Time.Unix() >= start.Unix() && item.Time.Unix() <= end.Unix() { - all = append(all, item) - } - } - - // TODO: handle the interval - - if len(all) == 0 { - return nil, currency.ErrNotFound - } - - if ordering == query.Ascending { - for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 { - all[i], all[j] = all[j], all[i] - } - } - - return all, nil -} - -func (s *store) SaveMetadata(ctx context.Context, data *currency.MetadataRecord) error { - if err := data.Validate(); err != nil { - return err - } - - s.mu.Lock() - defer s.mu.Unlock() - - for i, item := range s.metadataRecords { - if item.Mint == data.Mint { - if item.Version != data.Version { - return currency.ErrStaleMetadataVersion - } - - cloned := item.Clone() - cloned.Description = data.Description - cloned.ImageUrl = data.ImageUrl - cloned.BillColors = append([]string(nil), data.BillColors...) - cloned.SocialLinks = append([]currency.SocialLink(nil), data.SocialLinks...) - cloned.Alt = data.Alt - cloned.State = data.State - cloned.Version = item.Version + 1 - - s.metadataRecords[i] = cloned - cloned.CopyTo(data) - return nil - } - } - - for _, item := range s.metadataRecords { - if strings.EqualFold(item.Name, data.Name) && item.State != currency.MetadataStateAbandoned { - return currency.ErrDuplicateCurrency - } - } - - data.Version = 1 - data.Id = s.lastMetadataIndex - s.metadataRecords = append(s.metadataRecords, data.Clone()) - s.lastMetadataIndex = s.lastMetadataIndex + 1 - - return nil -} - -func (s *store) GetMetadata(ctx context.Context, mint string) (*currency.MetadataRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - for _, item := range s.metadataRecords { - if item.Mint == mint { - return item.Clone(), nil - } - } - - return nil, currency.ErrNotFound -} - -type metadataById []*currency.MetadataRecord - -func (a metadataById) Len() int { return len(a) } -func (a metadataById) Swap(i, j int) { a[i], a[j] = a[j], a[i] } -func (a metadataById) Less(i, j int) bool { return a[i].Id < a[j].Id } - -func (s *store) GetAllMetadataByState(_ context.Context, state currency.MetadataState, cursor query.Cursor, limit uint64, direction query.Ordering) ([]*currency.MetadataRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - var items []*currency.MetadataRecord - for _, item := range s.metadataRecords { - if item.State == state { - items = append(items, item) - } - } - - if len(items) == 0 { - return nil, currency.ErrNotFound - } - - var start uint64 - start = 0 - if direction == query.Descending { - start = s.lastMetadataIndex + 1 - } - if len(cursor) > 0 { - start = cursor.ToUint64() - } - - var res []*currency.MetadataRecord - for _, item := range items { - if item.Id > start && direction == query.Ascending { - res = append(res, item.Clone()) - } - if item.Id < start && direction == query.Descending { - res = append(res, item.Clone()) - } - } - - if len(res) == 0 { - return nil, currency.ErrNotFound - } - - if direction == query.Descending { - sort.Sort(sort.Reverse(metadataById(res))) - } - - if uint64(len(res)) > limit { - res = res[:limit] - } - - return res, nil -} - -func (s *store) GetAllMints(ctx context.Context) ([]string, error) { - s.mu.Lock() - defer s.mu.Unlock() - - if len(s.metadataRecords) == 0 { - return nil, currency.ErrNotFound - } - - var mints []string - for _, item := range s.metadataRecords { - mints = append(mints, item.Mint) - } - - return mints, nil -} - -func (s *store) CountMetadataByState(_ context.Context, state currency.MetadataState) (uint64, error) { - s.mu.Lock() - defer s.mu.Unlock() - - var count uint64 - for _, item := range s.metadataRecords { - if item.State == state { - count++ - } - } - return count, nil -} - -func (s *store) CountMints(ctx context.Context) (uint64, error) { - s.mu.Lock() - defer s.mu.Unlock() - - var count uint64 - for _, item := range s.metadataRecords { - if item.State == currency.MetadataStateAbandoned { - continue - } - count++ - } - return count, nil -} - -func (s *store) IsNameAvailable(_ context.Context, name string) (bool, error) { - s.mu.Lock() - defer s.mu.Unlock() - - for _, item := range s.metadataRecords { - if strings.EqualFold(item.Name, name) && item.State != currency.MetadataStateAbandoned { - return false, nil - } - } - return true, nil -} - -func (s *store) PutHistoricalReserveRecord(ctx context.Context, data *currency.ReserveRecord) error { - if err := data.Validate(); err != nil { - return err - } - - s.mu.Lock() - defer s.mu.Unlock() - - // Not ideal but fine for testing the currency store - for _, item := range s.historicalReserveRecords { - if item.Mint == data.Mint && item.Time.Unix() == data.Time.Unix() { - return currency.ErrExists - } - } - - data.Id = s.lastHistoricalReserveIndex - s.historicalReserveRecords = append(s.historicalReserveRecords, data.Clone()) - s.lastHistoricalReserveIndex = s.lastHistoricalReserveIndex + 1 - - return nil -} - -func (s *store) GetReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - // Not ideal but fine for testing the currency store - var results []*currency.ReserveRecord - for _, item := range s.historicalReserveRecords { - if item.Mint == mint && item.Time.Unix() <= t.Unix() && item.Time.Format(dateFormat) == t.Format(dateFormat) { - results = append(results, item) - } - } - - if len(results) == 0 { - return nil, currency.ErrNotFound - } - - sort.Sort(ReserveByTime(results)) - - return results[0].Clone(), nil -} - -func (s *store) GetReservesInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ReserveRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - sort.Sort(ReserveByTime(s.historicalReserveRecords)) - - // Not ideal but fine for testing the currency store - var all []*currency.ReserveRecord - for _, item := range s.historicalReserveRecords { - if item.Mint == mint && item.Time.Unix() >= start.Unix() && item.Time.Unix() <= end.Unix() { - all = append(all, item.Clone()) - } - } - - // TODO: handle the interval - - if len(all) == 0 { - return nil, currency.ErrNotFound - } - - if ordering == query.Ascending { - for i, j := 0, len(all)-1; i < j; i, j = i+1, j-1 { - all[i], all[j] = all[j], all[i] - } - } - - return all, nil -} - -func (s *store) PutLiveReserveRecord(ctx context.Context, data *currency.ReserveRecord) error { - if err := data.Validate(); err != nil { - return err - } - - s.mu.Lock() - defer s.mu.Unlock() - - if existing, ok := s.liveReserveRecords[data.Mint]; ok { - if data.Slot <= existing.Slot { - return currency.ErrStaleReserveState - } - - cloned := data.Clone() - cloned.Id = existing.Id - s.liveReserveRecords[data.Mint] = cloned - cloned.CopyTo(data) - return nil - } - - data.Id = s.lastLiveReserveIndex - s.liveReserveRecords[data.Mint] = data.Clone() - s.lastLiveReserveIndex++ - - return nil -} - -func (s *store) GetLiveReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - record, ok := s.liveReserveRecords[mint] - if !ok { - return nil, currency.ErrNotFound - } - - return record.Clone(), nil -} - -func (s *store) GetAllLiveReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - if len(s.liveReserveRecords) == 0 { - return nil, currency.ErrNotFound - } - - res := make(map[string]*currency.ReserveRecord, len(s.liveReserveRecords)) - for mint, record := range s.liveReserveRecords { - res[mint] = record.Clone() - } - return res, nil -} - -func (s *store) PutHistoricalHolderCountRecord(ctx context.Context, data *currency.HolderCountRecord) error { - if err := data.Validate(); err != nil { - return err - } - - s.mu.Lock() - defer s.mu.Unlock() - - for _, item := range s.historicalHolderCountRecords { - if item.Mint == data.Mint && item.Time.Unix() == data.Time.Unix() { - return currency.ErrExists - } - } - - data.Id = s.lastHistoricalHolderCountIndex - s.historicalHolderCountRecords = append(s.historicalHolderCountRecords, data.Clone()) - s.lastHistoricalHolderCountIndex = s.lastHistoricalHolderCountIndex + 1 - - return nil -} - -func (s *store) GetHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*currency.HolderCountRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - var results []*currency.HolderCountRecord - for _, item := range s.historicalHolderCountRecords { - if item.Mint == mint && item.Time.Unix() <= t.Unix() && item.Time.Format(dateFormat) == t.Format(dateFormat) { - results = append(results, item) - } - } - - if len(results) == 0 { - return nil, currency.ErrNotFound - } - - sort.Sort(HolderCountByTime(results)) - - return results[0].Clone(), nil -} - -func (s *store) GetAllHolderCountsAtTime(ctx context.Context, t time.Time) (map[string]*currency.HolderCountRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - sort.Sort(HolderCountByTime(s.historicalHolderCountRecords)) - - result := make(map[string]*currency.HolderCountRecord) - for _, item := range s.historicalHolderCountRecords { - if item.Time.Unix() <= t.Unix() && item.Time.Format(dateFormat) == t.Format(dateFormat) { - if _, exists := result[item.Mint]; !exists { - result[item.Mint] = item.Clone() - } - } - } - - if len(result) == 0 { - return nil, currency.ErrNotFound - } - - return result, nil -} - -func (s *store) PutLiveHolderCountRecord(ctx context.Context, data *currency.HolderCountRecord) error { - if err := data.Validate(); err != nil { - return err - } - - s.mu.Lock() - defer s.mu.Unlock() - - if existing, ok := s.liveHolderCountRecords[data.Mint]; ok { - if !data.Time.After(existing.Time) { - return currency.ErrStaleHolderState - } - - cloned := data.Clone() - cloned.Id = existing.Id - s.liveHolderCountRecords[data.Mint] = cloned - cloned.CopyTo(data) - return nil - } - - data.Id = s.lastLiveHolderCountIndex - s.liveHolderCountRecords[data.Mint] = data.Clone() - s.lastLiveHolderCountIndex++ - - return nil -} - -func (s *store) GetLiveHolderCount(ctx context.Context, mint string) (*currency.HolderCountRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - record, ok := s.liveHolderCountRecords[mint] - if !ok { - return nil, currency.ErrNotFound - } - - return record.Clone(), nil -} - -func (s *store) GetAllLiveHolderCounts(ctx context.Context) (map[string]*currency.HolderCountRecord, error) { - s.mu.Lock() - defer s.mu.Unlock() - - if len(s.liveHolderCountRecords) == 0 { - return nil, currency.ErrNotFound - } - - res := make(map[string]*currency.HolderCountRecord, len(s.liveHolderCountRecords)) - for mint, record := range s.liveHolderCountRecords { - res[mint] = record.Clone() - } - return res, nil -} diff --git a/ocp/data/currency/metadata/memory/store.go b/ocp/data/currency/metadata/memory/store.go new file mode 100644 index 0000000..96b4284 --- /dev/null +++ b/ocp/data/currency/metadata/memory/store.go @@ -0,0 +1,197 @@ +package memory + +import ( + "context" + "sort" + "strings" + "sync" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/metadata" +) + +type store struct { + mu sync.Mutex + metadataRecords []*currency.MetadataRecord + lastMetadataIndex uint64 +} + +func New() metadata.Store { + return &store{ + lastMetadataIndex: 1, + } +} + +func (s *store) reset() { + s.mu.Lock() + s.metadataRecords = make([]*currency.MetadataRecord, 0) + s.lastMetadataIndex = 1 + s.mu.Unlock() +} + +func (s *store) SaveMetadata(ctx context.Context, data *currency.MetadataRecord) error { + if err := data.Validate(); err != nil { + return err + } + + s.mu.Lock() + defer s.mu.Unlock() + + for i, item := range s.metadataRecords { + if item.Mint == data.Mint { + if item.Version != data.Version { + return currency.ErrStaleMetadataVersion + } + + cloned := item.Clone() + cloned.Description = data.Description + cloned.ImageUrl = data.ImageUrl + cloned.BillColors = append([]string(nil), data.BillColors...) + cloned.SocialLinks = append([]currency.SocialLink(nil), data.SocialLinks...) + cloned.Alt = data.Alt + cloned.State = data.State + cloned.Version = item.Version + 1 + + s.metadataRecords[i] = cloned + cloned.CopyTo(data) + return nil + } + } + + for _, item := range s.metadataRecords { + if strings.EqualFold(item.Name, data.Name) && item.State != currency.MetadataStateAbandoned { + return currency.ErrDuplicateCurrency + } + } + + data.Version = 1 + data.Id = s.lastMetadataIndex + s.metadataRecords = append(s.metadataRecords, data.Clone()) + s.lastMetadataIndex = s.lastMetadataIndex + 1 + + return nil +} + +func (s *store) GetMetadata(ctx context.Context, mint string) (*currency.MetadataRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + for _, item := range s.metadataRecords { + if item.Mint == mint { + return item.Clone(), nil + } + } + + return nil, currency.ErrNotFound +} + +type metadataById []*currency.MetadataRecord + +func (a metadataById) Len() int { return len(a) } +func (a metadataById) Swap(i, j int) { a[i], a[j] = a[j], a[i] } +func (a metadataById) Less(i, j int) bool { return a[i].Id < a[j].Id } + +func (s *store) GetAllMetadataByState(_ context.Context, state currency.MetadataState, cursor query.Cursor, limit uint64, direction query.Ordering) ([]*currency.MetadataRecord, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var items []*currency.MetadataRecord + for _, item := range s.metadataRecords { + if item.State == state { + items = append(items, item) + } + } + + if len(items) == 0 { + return nil, currency.ErrNotFound + } + + var start uint64 + start = 0 + if direction == query.Descending { + start = s.lastMetadataIndex + 1 + } + if len(cursor) > 0 { + start = cursor.ToUint64() + } + + var res []*currency.MetadataRecord + for _, item := range items { + if item.Id > start && direction == query.Ascending { + res = append(res, item.Clone()) + } + if item.Id < start && direction == query.Descending { + res = append(res, item.Clone()) + } + } + + if len(res) == 0 { + return nil, currency.ErrNotFound + } + + if direction == query.Descending { + sort.Sort(sort.Reverse(metadataById(res))) + } + + if uint64(len(res)) > limit { + res = res[:limit] + } + + return res, nil +} + +func (s *store) GetAllMints(ctx context.Context) ([]string, error) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.metadataRecords) == 0 { + return nil, currency.ErrNotFound + } + + var mints []string + for _, item := range s.metadataRecords { + mints = append(mints, item.Mint) + } + + return mints, nil +} + +func (s *store) CountMetadataByState(_ context.Context, state currency.MetadataState) (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var count uint64 + for _, item := range s.metadataRecords { + if item.State == state { + count++ + } + } + return count, nil +} + +func (s *store) CountMints(ctx context.Context) (uint64, error) { + s.mu.Lock() + defer s.mu.Unlock() + + var count uint64 + for _, item := range s.metadataRecords { + if item.State == currency.MetadataStateAbandoned { + continue + } + count++ + } + return count, nil +} + +func (s *store) IsNameAvailable(_ context.Context, name string) (bool, error) { + s.mu.Lock() + defer s.mu.Unlock() + + for _, item := range s.metadataRecords { + if strings.EqualFold(item.Name, name) && item.State != currency.MetadataStateAbandoned { + return false, nil + } + } + return true, nil +} diff --git a/ocp/data/currency/memory/store_test.go b/ocp/data/currency/metadata/memory/store_test.go similarity index 56% rename from ocp/data/currency/memory/store_test.go rename to ocp/data/currency/metadata/memory/store_test.go index b011b1d..4eb79dc 100644 --- a/ocp/data/currency/memory/store_test.go +++ b/ocp/data/currency/metadata/memory/store_test.go @@ -3,10 +3,10 @@ package memory import ( "testing" - "github.com/code-payments/ocp-server/ocp/data/currency/tests" + "github.com/code-payments/ocp-server/ocp/data/currency/metadata/tests" ) -func TestCurrencyMemoryStore(t *testing.T) { +func TestMetadata_MemoryStore(t *testing.T) { testStore := New() teardown := func() { testStore.(*store).reset() diff --git a/ocp/data/currency/metadata/postgres/model.go b/ocp/data/currency/metadata/postgres/model.go new file mode 100644 index 0000000..11d63a1 --- /dev/null +++ b/ocp/data/currency/metadata/postgres/model.go @@ -0,0 +1,306 @@ +package postgres + +import ( + "context" + "database/sql" + "encoding/json" + "strings" + "time" + + "github.com/jmoiron/sqlx" + + q "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + + pgutil "github.com/code-payments/ocp-server/database/postgres" +) + +const ( + tableName = "ocp__core_currencymetadata" +) + +type model struct { + Id sql.NullInt64 `db:"id"` + + Name string `db:"name"` + Symbol string `db:"symbol"` + Description string `db:"description"` + ImageUrl string `db:"image_url"` + BillColors string `db:"bill_colors"` + SocialLinks string `db:"social_links"` + + Seed string `db:"seed"` + + Authority string `db:"authority"` + + Mint string `db:"mint"` + MintBump uint8 `db:"mint_bump"` + Decimals uint8 `db:"decimals"` + + CurrencyConfig string `db:"currency_config"` + CurrencyConfigBump uint8 `db:"currency_config_bump"` + + LiquidityPool string `db:"liquidity_pool"` + LiquidityPoolBump uint8 `db:"liquidity_pool_bump"` + + VaultMint string `db:"vault_mint"` + VaultMintBump uint8 `db:"vault_mint_bump"` + + VaultCore string `db:"vault_core"` + VaultCoreBump uint8 `db:"vault_core_bump"` + + SellFeeBps uint16 `db:"sell_fee_bps"` + + Alt string `db:"alt"` + + State uint8 `db:"state"` + Version uint64 `db:"version"` + + CreatedBy string `db:"created_by"` + CreatedAt time.Time `db:"created_at"` +} + +func toModel(obj *currency.MetadataRecord) (*model, error) { + if err := obj.Validate(); err != nil { + return nil, err + } + + return &model{ + Id: sql.NullInt64{Int64: int64(obj.Id), Valid: obj.Id > 0}, + + Name: obj.Name, + Symbol: obj.Symbol, + Description: obj.Description, + ImageUrl: obj.ImageUrl, + BillColors: strings.Join(obj.BillColors, ","), + SocialLinks: marshalSocialLinks(obj.SocialLinks), + + Seed: obj.Seed, + + Authority: obj.Authority, + + Mint: obj.Mint, + MintBump: obj.MintBump, + Decimals: obj.Decimals, + + CurrencyConfig: obj.CurrencyConfig, + CurrencyConfigBump: obj.CurrencyConfigBump, + + LiquidityPool: obj.LiquidityPool, + LiquidityPoolBump: obj.LiquidityPoolBump, + + VaultMint: obj.VaultMint, + VaultMintBump: obj.VaultMintBump, + + VaultCore: obj.VaultCore, + VaultCoreBump: obj.VaultCoreBump, + + SellFeeBps: obj.SellFeeBps, + + Alt: obj.Alt, + + State: uint8(obj.State), + Version: obj.Version, + + CreatedBy: obj.CreatedBy, + CreatedAt: obj.CreatedAt, + }, nil +} + +func fromModel(obj *model) *currency.MetadataRecord { + var billColors []string + if obj.BillColors != "" { + billColors = strings.Split(obj.BillColors, ",") + } + + return ¤cy.MetadataRecord{ + Id: uint64(obj.Id.Int64), + + Name: obj.Name, + Symbol: obj.Symbol, + Description: obj.Description, + ImageUrl: obj.ImageUrl, + BillColors: billColors, + SocialLinks: unmarshalSocialLinks(obj.SocialLinks), + + Seed: obj.Seed, + + Authority: obj.Authority, + + Mint: obj.Mint, + MintBump: obj.MintBump, + Decimals: obj.Decimals, + + CurrencyConfig: obj.CurrencyConfig, + CurrencyConfigBump: obj.CurrencyConfigBump, + + LiquidityPool: obj.LiquidityPool, + LiquidityPoolBump: obj.LiquidityPoolBump, + + VaultMint: obj.VaultMint, + VaultMintBump: obj.VaultMintBump, + + VaultCore: obj.VaultCore, + VaultCoreBump: obj.VaultCoreBump, + + SellFeeBps: obj.SellFeeBps, + + Alt: obj.Alt, + + State: currency.MetadataState(obj.State), + Version: obj.Version, + + CreatedBy: obj.CreatedBy, + CreatedAt: obj.CreatedAt, + } +} + +func marshalSocialLinks(links []currency.SocialLink) string { + if len(links) == 0 { + return "[]" + } + data, _ := json.Marshal(links) + return string(data) +} + +func unmarshalSocialLinks(data string) []currency.SocialLink { + if data == "" || data == "[]" { + return nil + } + var links []currency.SocialLink + _ = json.Unmarshal([]byte(data), &links) + return links +} + +func (m *model) dbSave(ctx context.Context, db *sqlx.DB) error { + return pgutil.ExecuteInTx(ctx, db, sql.LevelDefault, func(tx *sqlx.Tx) error { + err := tx.QueryRowxContext(ctx, + `INSERT INTO `+tableName+` + (name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23 + 1, $24, $25) + + ON CONFLICT (mint) + DO UPDATE + SET description = $3, image_url = $4, bill_colors = $5, social_links = $6, alt = $21, state = $22, version = `+tableName+`.version + 1 + WHERE `+tableName+`.mint = $9 AND `+tableName+`.version = $23 + + RETURNING id, name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at`, + m.Name, + m.Symbol, + m.Description, + m.ImageUrl, + m.BillColors, + m.SocialLinks, + m.Seed, + m.Authority, + m.Mint, + m.MintBump, + m.Decimals, + m.CurrencyConfig, + m.CurrencyConfigBump, + m.LiquidityPool, + m.LiquidityPoolBump, + m.VaultMint, + m.VaultMintBump, + m.VaultCore, + m.VaultCoreBump, + m.SellFeeBps, + m.Alt, + m.State, + m.Version, + m.CreatedBy, + m.CreatedAt, + ).StructScan(m) + + err = pgutil.CheckUniqueViolation(err, currency.ErrDuplicateCurrency) + if err == currency.ErrDuplicateCurrency { + return err + } + + return pgutil.CheckNoRows(err, currency.ErrStaleMetadataVersion) + }) +} + +func dbGetAllMetadataByState(ctx context.Context, db *sqlx.DB, state currency.MetadataState, cursor q.Cursor, limit uint64, direction q.Ordering) ([]*model, error) { + res := []*model{} + + query := `SELECT + id, name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at + FROM ` + tableName + ` + WHERE state = $1` + + opts := []interface{}{state} + query, opts = q.PaginateQuery(query, opts, cursor, limit, direction) + + err := db.SelectContext(ctx, &res, query, opts...) + if err != nil { + return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) + } + + if len(res) == 0 { + return nil, currency.ErrNotFound + } + return res, nil +} + +func dbGetMetadataByMint(ctx context.Context, db *sqlx.DB, mint string) (*model, error) { + res := &model{} + err := db.GetContext(ctx, res, + `SELECT id, name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at + FROM `+tableName+` + WHERE mint = $1`, + mint, + ) + return res, pgutil.CheckNoRows(err, currency.ErrNotFound) +} + +func dbGetAllMints(ctx context.Context, db *sqlx.DB) ([]string, error) { + var res []string + err := db.SelectContext(ctx, &res, + `SELECT mint FROM `+tableName, + ) + + if err != nil { + return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) + } + if len(res) == 0 { + return nil, currency.ErrNotFound + } + + return res, nil +} + +func dbCountMints(ctx context.Context, db *sqlx.DB) (uint64, error) { + var count uint64 + err := db.GetContext(ctx, &count, + `SELECT COUNT(*) FROM `+tableName+` WHERE state != $1`, + currency.MetadataStateAbandoned, + ) + if err != nil { + return 0, err + } + return count, nil +} + +func dbCountMetadataByState(ctx context.Context, db *sqlx.DB, state currency.MetadataState) (uint64, error) { + var res uint64 + query := `SELECT COUNT(*) FROM ` + tableName + ` WHERE state = $1` + err := db.GetContext(ctx, &res, query, state) + if err != nil { + return 0, err + } + return res, nil +} + +func dbIsNameAvailable(ctx context.Context, db *sqlx.DB, name string) (bool, error) { + var count uint64 + err := db.GetContext(ctx, &count, + `SELECT COUNT(*) FROM `+tableName+` WHERE LOWER(name) = LOWER($1) AND state != $2`, + name, + currency.MetadataStateAbandoned, + ) + if err != nil { + return false, err + } + return count == 0, nil +} diff --git a/ocp/data/currency/metadata/postgres/store.go b/ocp/data/currency/metadata/postgres/store.go new file mode 100644 index 0000000..e05de24 --- /dev/null +++ b/ocp/data/currency/metadata/postgres/store.go @@ -0,0 +1,75 @@ +package postgres + +import ( + "context" + "database/sql" + + "github.com/jmoiron/sqlx" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/metadata" +) + +type store struct { + db *sqlx.DB +} + +func New(db *sql.DB) metadata.Store { + return &store{ + db: sqlx.NewDb(db, "pgx"), + } +} + +func (s *store) SaveMetadata(ctx context.Context, record *currency.MetadataRecord) error { + model, err := toModel(record) + if err != nil { + return err + } + + err = model.dbSave(ctx, s.db) + if err != nil { + return err + } + + fromModel(model).CopyTo(record) + + return nil +} + +func (s *store) GetMetadata(ctx context.Context, mint string) (*currency.MetadataRecord, error) { + model, err := dbGetMetadataByMint(ctx, s.db, mint) + if err != nil { + return nil, err + } + return fromModel(model), nil +} + +func (s *store) GetAllMetadataByState(ctx context.Context, state currency.MetadataState, cursor query.Cursor, limit uint64, direction query.Ordering) ([]*currency.MetadataRecord, error) { + models, err := dbGetAllMetadataByState(ctx, s.db, state, cursor, limit, direction) + if err != nil { + return nil, err + } + + res := make([]*currency.MetadataRecord, len(models)) + for i, model := range models { + res[i] = fromModel(model) + } + return res, nil +} + +func (s *store) GetAllMints(ctx context.Context) ([]string, error) { + return dbGetAllMints(ctx, s.db) +} + +func (s *store) CountMints(ctx context.Context) (uint64, error) { + return dbCountMints(ctx, s.db) +} + +func (s *store) CountMetadataByState(ctx context.Context, state currency.MetadataState) (uint64, error) { + return dbCountMetadataByState(ctx, s.db, state) +} + +func (s *store) IsNameAvailable(ctx context.Context, name string) (bool, error) { + return dbIsNameAvailable(ctx, s.db, name) +} diff --git a/ocp/data/currency/postgres/store_test.go b/ocp/data/currency/metadata/postgres/store_test.go similarity index 60% rename from ocp/data/currency/postgres/store_test.go rename to ocp/data/currency/metadata/postgres/store_test.go index f97c27c..6cf4c7d 100644 --- a/ocp/data/currency/postgres/store_test.go +++ b/ocp/data/currency/metadata/postgres/store_test.go @@ -8,8 +8,8 @@ import ( "github.com/ory/dockertest/v3" "go.uber.org/zap" - "github.com/code-payments/ocp-server/ocp/data/currency" - "github.com/code-payments/ocp-server/ocp/data/currency/tests" + "github.com/code-payments/ocp-server/ocp/data/currency/metadata" + "github.com/code-payments/ocp-server/ocp/data/currency/metadata/tests" postgrestest "github.com/code-payments/ocp-server/database/postgres/test" @@ -19,17 +19,6 @@ import ( const ( // Used for testing ONLY, the table and migrations are external to this repository tableCreate = ` - CREATE TABLE ocp__core_exchangerate ( - id serial NOT NULL PRIMARY KEY, - - for_date VARCHAR(10) NOT NULL, - for_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - currency_code VARCHAR(3) NOT NULL, - currency_rate NUMERIC(18, 9) NOT NULL, - - CONSTRAINT ocp__core_exchangerate__uniq__timestamp__and__code UNIQUE (for_timestamp, currency_code), - CONSTRAINT ocp__core_exchangerate__currency_code CHECK (currency_code::text ~ '^[a-z]{3}$') - ); CREATE TABLE ocp__core_currencymetadata ( id serial NOT NULL PRIMARY KEY, @@ -71,56 +60,16 @@ const ( created_at TIMESTAMP WITH TIME ZONE NOT NULL ); CREATE UNIQUE INDEX ocp__core_currencymetadata__name__idx ON ocp__core_currencymetadata (LOWER(name)) WHERE state != 8; - CREATE TABLE ocp__core_currencyreserve ( - id serial NOT NULL PRIMARY KEY, - - for_date VARCHAR(10) NOT NULL, - for_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - mint TEXT NOT NULL, - supply_from_bonding BIGINT NOT NULL, - - CONSTRAINT ocp__core_currencyreserve__uniq__timestamp__and__mint UNIQUE (for_timestamp, mint) - ); - CREATE TABLE ocp__core_currencyreserve2 ( - id serial NOT NULL PRIMARY KEY, - - mint TEXT UNIQUE NOT NULL, - supply_from_bonding BIGINT NOT NULL, - slot BIGINT NOT NULL, - last_updated_at TIMESTAMP WITH TIME ZONE NOT NULL - ); - CREATE TABLE ocp__core_currencyholdercount ( - id serial NOT NULL PRIMARY KEY, - - for_date VARCHAR(10) NOT NULL, - for_timestamp TIMESTAMP WITH TIME ZONE NOT NULL, - mint TEXT NOT NULL, - holder_count BIGINT NOT NULL, - - CONSTRAINT ocp__core_currencyholdercount__uniq__timestamp__and__mint UNIQUE (for_timestamp, mint) - ); - CREATE TABLE ocp__core_currencyholdercount2 ( - id serial NOT NULL PRIMARY KEY, - - mint TEXT UNIQUE NOT NULL, - holder_count BIGINT NOT NULL, - last_updated_at TIMESTAMP WITH TIME ZONE NOT NULL - ); ` // Used for testing ONLY, the table and migrations are external to this repository tableDestroy = ` - DROP TABLE ocp__core_exchangerate; DROP TABLE ocp__core_currencymetadata; - DROP TABLE ocp__core_currencyreserve; - DROP TABLE ocp__core_currencyreserve2; - DROP TABLE ocp__core_currencyholdercount; - DROP TABLE ocp__core_currencyholdercount2; ` ) var ( - testStore currency.Store + testStore metadata.Store teardown func() ) @@ -166,7 +115,7 @@ func TestMain(m *testing.M) { os.Exit(code) } -func TestCurrencyPostgresStore(t *testing.T) { +func TestMetadata_PostgresStore(t *testing.T) { tests.RunTests(t, testStore, teardown) } diff --git a/ocp/data/currency/metadata/store.go b/ocp/data/currency/metadata/store.go new file mode 100644 index 0000000..08f89bc --- /dev/null +++ b/ocp/data/currency/metadata/store.go @@ -0,0 +1,41 @@ +// Package metadata defines a focused store for currency creator mint metadata. +package metadata + +import ( + "context" + + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" +) + +type Store interface { + // SaveMetadata creates or updates currency creator metadata in the store. + // On insert, Version is set to 1. On update, only mutable fields (Description, + // ImageUrl, BillColors, SocialLinks, Alt, State) are updated and Version is incremented. + // ErrStaleMetadataVersion is returned when the provided version doesn't match. + SaveMetadata(ctx context.Context, record *currency.MetadataRecord) error + + // GetMetadata gets currency creator mint metadata by the mint address + GetMetadata(ctx context.Context, mint string) (*currency.MetadataRecord, error) + + // GetAllMetadataByState returns all currency metadata records in a given state + // + // ErrNotFound is returned if no metadata records exist for the given state + GetAllMetadataByState(ctx context.Context, state currency.MetadataState, cursor query.Cursor, limit uint64, direction query.Ordering) ([]*currency.MetadataRecord, error) + + // CountMetadataByState returns the count of currency metadata records in the requested state + CountMetadataByState(ctx context.Context, state currency.MetadataState) (uint64, error) + + // GetAllMints returns the public keys of all currency creator mints + // + // ErrNotFound is returned if no mints exist + GetAllMints(ctx context.Context) ([]string, error) + + // CountMints returns the total number of currency creator mints, + // excluding those in the abandoned state. + CountMints(ctx context.Context) (uint64, error) + + // IsNameAvailable checks whether a currency name is available for use. + // The check is case-insensitive. + IsNameAvailable(ctx context.Context, name string) (bool, error) +} diff --git a/ocp/data/currency/tests/tests.go b/ocp/data/currency/metadata/tests/tests.go similarity index 58% rename from ocp/data/currency/tests/tests.go rename to ocp/data/currency/metadata/tests/tests.go index fc0e4e5..e9e0831 100644 --- a/ocp/data/currency/tests/tests.go +++ b/ocp/data/currency/metadata/tests/tests.go @@ -10,13 +10,12 @@ import ( "github.com/code-payments/ocp-server/database/query" "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/metadata" "github.com/code-payments/ocp-server/solana/currencycreator" ) -func RunTests(t *testing.T, s currency.Store, teardown func()) { - for _, tf := range []func(t *testing.T, s currency.Store){ - testExchangeRateRoundTrip, - testGetExchangeRatesInRange, +func RunTests(t *testing.T, s metadata.Store, teardown func()) { + for _, tf := range []func(t *testing.T, s metadata.Store){ testMetadataRoundTrip, testMetadataSaveWithVersioning, testMetadataUniqueNameConstraint, @@ -26,131 +25,13 @@ func RunTests(t *testing.T, s currency.Store, teardown func()) { testGetAllMints, testCountMints, testCountMetadataByState, - testReserveRoundTrip, - testGetReservesInRange, - testLiveReserveRoundTrip, - testGetAllLiveReserves, - testHolderCountRoundTrip, - testGetAllHolderCountsAtTime, - testLiveHolderCountRoundTrip, - testGetAllLiveHolderCounts, } { tf(t, s) teardown() } } -func testExchangeRateRoundTrip(t *testing.T, s currency.Store) { - now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) - - record, err := s.GetAllExchangeRates(context.Background(), now) - assert.Nil(t, record) - assert.Equal(t, currency.ErrNotFound, err) - - rates := map[string]float64{ - "usd": 0.000055, - "cad": 0.00007, - } - require.NoError(t, s.PutExchangeRates(context.Background(), ¤cy.MultiRateRecord{ - Time: now, - Rates: rates, - })) - - // Overwrite should fail - assert.Equal(t, currency.ErrExists, s.PutExchangeRates(context.Background(), ¤cy.MultiRateRecord{ - Time: now, - Rates: rates, - })) - - // Test GetExchangeRate(), it should return the USD record - single, err := s.GetExchangeRate(context.Background(), "usd", now) - require.NoError(t, err) - assert.Equal(t, now.Unix(), single.Time.Unix()) - assert.EqualValues(t, rates["usd"], single.Rate) - - // Test GetAllExchangeRates(), it should return all recent rates - record, err = s.GetAllExchangeRates(context.Background(), now) - require.NoError(t, err) - - assert.Equal(t, now.Unix(), record.Time.Unix()) - assert.EqualValues(t, rates, record.Rates) - - // within same day, should return entry - record, err = s.GetAllExchangeRates(context.Background(), time.Date(2021, 01, 29, 14, 0, 5, 0, time.UTC)) - require.NoError(t, err) - - assert.Equal(t, now.Unix(), record.Time.Unix()) - assert.EqualValues(t, rates, record.Rates) - - // day after, should be empty - tomorrow := time.Date(2021, 01, 30, 0, 0, 0, 0, time.UTC) - record, err = s.GetAllExchangeRates(context.Background(), tomorrow) - assert.Nil(t, record) - assert.Equal(t, currency.ErrNotFound, err) -} - -func testGetExchangeRatesInRange(t *testing.T, s currency.Store) { - var rates []currency.MultiRateRecord - - now := time.Now().UTC() - - for i := 0; i < 100; i++ { - rates = append(rates, currency.MultiRateRecord{ - Time: now.Add(time.Duration(i) * time.Hour), - Rates: map[string]float64{ - "usd": (0.000058 + float64(i/10000)), - "cad": (0.00008 + float64(i/10000)), - }, - }) - } - - record, err := s.GetAllExchangeRates(context.Background(), rates[0].Time) - assert.Nil(t, record) - assert.Equal(t, currency.ErrNotFound, err) - - for _, item := range rates { - require.NoError(t, s.PutExchangeRates(context.Background(), &item)) - } - - result, err := s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalRaw, rates[0].Time, rates[99].Time, query.Ascending) - require.NoError(t, err) - assert.Equal(t, len(result), 100) - for i, item := range result { - assert.Equal(t, rates[i].Time.Unix(), item.Time.Unix()) - assert.EqualValues(t, rates[i].Rates["usd"], item.Rate) - } - - result, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalRaw, rates[0].Time, rates[49].Time, query.Ascending) - require.NoError(t, err) - assert.Equal(t, len(result), 50) - for i, item := range result { - assert.Equal(t, rates[i].Time.Unix(), item.Time.Unix()) - assert.EqualValues(t, rates[i].Rates["usd"], item.Rate) - } - - result, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalRaw, rates[0].Time, rates[99].Time, query.Descending) - require.NoError(t, err) - assert.Equal(t, len(result), 100) - for i, item := range result { - assert.Equal(t, rates[99-i].Time.Unix(), item.Time.Unix()) - assert.EqualValues(t, rates[99-i].Rates["usd"], item.Rate) - } - - _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalSecond, rates[0].Time, rates[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalMinute, rates[0].Time, rates[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalHour, rates[0].Time, rates[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalDay, rates[0].Time, rates[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalWeek, rates[0].Time, rates[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetExchangeRatesInRange(context.Background(), "usd", query.IntervalMonth, rates[0].Time, rates[99].Time, query.Ascending) - require.NoError(t, err) -} - -func testMetadataRoundTrip(t *testing.T, s currency.Store) { +func testMetadataRoundTrip(t *testing.T, s metadata.Store) { expected := ¤cy.MetadataRecord{ Name: "Jeffy", Symbol: "JFY", @@ -202,12 +83,12 @@ func testMetadataRoundTrip(t *testing.T, s currency.Store) { actual, err := s.GetMetadata(context.Background(), expected.Mint) require.NoError(t, err) - assertEquivalentMetadataRecords(t, cloned, actual) + assertEquivalentRecords(t, cloned, actual) assert.EqualValues(t, currency.MetadataStateUnknown, actual.State) assert.EqualValues(t, 1, actual.Version) } -func testMetadataUniqueNameConstraint(t *testing.T, s currency.Store) { +func testMetadataUniqueNameConstraint(t *testing.T, s metadata.Store) { record1 := ¤cy.MetadataRecord{ Name: "UniqueName", Symbol: "UN1", @@ -284,7 +165,7 @@ func testMetadataUniqueNameConstraint(t *testing.T, s currency.Store) { assert.Equal(t, currency.ErrDuplicateCurrency, s.SaveMetadata(context.Background(), record2)) } -func testIsNameAvailable(t *testing.T, s currency.Store) { +func testIsNameAvailable(t *testing.T, s metadata.Store) { ctx := context.Background() // Name should be available when no records exist @@ -350,7 +231,7 @@ func testIsNameAvailable(t *testing.T, s currency.Store) { assert.True(t, available) } -func testAbandonedCurrencyNameReuse(t *testing.T, s currency.Store) { +func testAbandonedCurrencyNameReuse(t *testing.T, s metadata.Store) { ctx := context.Background() record := ¤cy.MetadataRecord{ @@ -458,7 +339,7 @@ func testAbandonedCurrencyNameReuse(t *testing.T, s currency.Store) { assert.False(t, available) } -func testGetAllMetadataByState(t *testing.T, s currency.Store) { +func testGetAllMetadataByState(t *testing.T, s metadata.Store) { t.Run("testGetAllMetadataByState", func(t *testing.T) { ctx := context.Background() @@ -570,7 +451,7 @@ func testGetAllMetadataByState(t *testing.T, s currency.Store) { }) } -func testGetAllMints(t *testing.T, s currency.Store) { +func testGetAllMints(t *testing.T, s metadata.Store) { // No mints should exist initially mints, err := s.GetAllMints(context.Background()) assert.Nil(t, mints) @@ -634,7 +515,7 @@ func testGetAllMints(t *testing.T, s currency.Store) { assert.Contains(t, mints, record2.Mint) } -func testCountMints(t *testing.T, s currency.Store) { +func testCountMints(t *testing.T, s metadata.Store) { // No mints should exist initially count, err := s.CountMints(context.Background()) require.NoError(t, err) @@ -709,7 +590,7 @@ func testCountMints(t *testing.T, s currency.Store) { assert.EqualValues(t, 1, count) } -func testCountMetadataByState(t *testing.T, s currency.Store) { +func testCountMetadataByState(t *testing.T, s metadata.Store) { ctx := context.Background() // No records should exist initially @@ -815,101 +696,7 @@ func testCountMetadataByState(t *testing.T, s currency.Store) { assert.EqualValues(t, 1, count) } -func testReserveRoundTrip(t *testing.T, s currency.Store) { - now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) - - record, err := s.GetReserveAtTime(context.Background(), "mint", now) - assert.Nil(t, record) - assert.Equal(t, currency.ErrNotFound, err) - - expected := ¤cy.ReserveRecord{ - Mint: "mint", - SupplyFromBonding: 1, - Time: now, - } - require.NoError(t, s.PutHistoricalReserveRecord(context.Background(), expected)) - - assert.Equal(t, currency.ErrExists, s.PutHistoricalReserveRecord(context.Background(), expected)) - - actual, err := s.GetReserveAtTime(context.Background(), "mint", now) - require.NoError(t, err) - assert.Equal(t, now.Unix(), actual.Time.Unix()) - assert.Equal(t, actual.SupplyFromBonding, expected.SupplyFromBonding) - - actual, err = s.GetReserveAtTime(context.Background(), "mint", time.Date(2021, 01, 29, 14, 0, 5, 0, time.UTC)) - require.NoError(t, err) - - assert.Equal(t, now.Unix(), actual.Time.Unix()) - assert.Equal(t, actual.SupplyFromBonding, expected.SupplyFromBonding) - - tomorrow := time.Date(2021, 01, 30, 0, 0, 0, 0, time.UTC) - actual, err = s.GetReserveAtTime(context.Background(), "mint", tomorrow) - assert.Nil(t, actual) - assert.Equal(t, currency.ErrNotFound, err) -} - -func testGetReservesInRange(t *testing.T, s currency.Store) { - var reserves []currency.ReserveRecord - - now := time.Now().UTC() - mint := "test-mint" - - for i := 0; i < 100; i++ { - reserves = append(reserves, currency.ReserveRecord{ - Mint: mint, - SupplyFromBonding: uint64(1000 + i), - Time: now.Add(time.Duration(i) * time.Hour), - }) - } - - record, err := s.GetReserveAtTime(context.Background(), mint, reserves[0].Time) - assert.Nil(t, record) - assert.Equal(t, currency.ErrNotFound, err) - - for _, item := range reserves { - itemCopy := item - require.NoError(t, s.PutHistoricalReserveRecord(context.Background(), &itemCopy)) - } - - result, err := s.GetReservesInRange(context.Background(), mint, query.IntervalRaw, reserves[0].Time, reserves[99].Time, query.Ascending) - require.NoError(t, err) - assert.Equal(t, len(result), 100) - for i, item := range result { - assert.Equal(t, reserves[i].Time.Unix(), item.Time.Unix()) - assert.EqualValues(t, reserves[i].SupplyFromBonding, item.SupplyFromBonding) - } - - result, err = s.GetReservesInRange(context.Background(), mint, query.IntervalRaw, reserves[0].Time, reserves[49].Time, query.Ascending) - require.NoError(t, err) - assert.Equal(t, len(result), 50) - for i, item := range result { - assert.Equal(t, reserves[i].Time.Unix(), item.Time.Unix()) - assert.EqualValues(t, reserves[i].SupplyFromBonding, item.SupplyFromBonding) - } - - result, err = s.GetReservesInRange(context.Background(), mint, query.IntervalRaw, reserves[0].Time, reserves[99].Time, query.Descending) - require.NoError(t, err) - assert.Equal(t, len(result), 100) - for i, item := range result { - assert.Equal(t, reserves[99-i].Time.Unix(), item.Time.Unix()) - assert.EqualValues(t, reserves[99-i].SupplyFromBonding, item.SupplyFromBonding) - } - - _, err = s.GetReservesInRange(context.Background(), mint, query.IntervalSecond, reserves[0].Time, reserves[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetReservesInRange(context.Background(), mint, query.IntervalMinute, reserves[0].Time, reserves[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetReservesInRange(context.Background(), mint, query.IntervalHour, reserves[0].Time, reserves[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetReservesInRange(context.Background(), mint, query.IntervalDay, reserves[0].Time, reserves[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetReservesInRange(context.Background(), mint, query.IntervalWeek, reserves[0].Time, reserves[99].Time, query.Ascending) - require.NoError(t, err) - _, err = s.GetReservesInRange(context.Background(), mint, query.IntervalMonth, reserves[0].Time, reserves[99].Time, query.Ascending) - require.NoError(t, err) -} - -func testMetadataSaveWithVersioning(t *testing.T, s currency.Store) { +func testMetadataSaveWithVersioning(t *testing.T, s metadata.Store) { record := ¤cy.MetadataRecord{ Name: "Versioned", Symbol: "VER", @@ -1018,340 +805,7 @@ func testMetadataSaveWithVersioning(t *testing.T, s currency.Store) { assert.Equal(t, "updatedalt1111111111111111111111111111111111111", actual.Alt) } -func testLiveReserveRoundTrip(t *testing.T, s currency.Store) { - ctx := context.Background() - mint := "live-reserve-mint" - - // No record should exist initially - _, err := s.GetLiveReserve(ctx, mint) - assert.Equal(t, currency.ErrNotFound, err) - - // Insert the first live reserve record - record := ¤cy.ReserveRecord{ - Mint: mint, - SupplyFromBonding: 1000, - Slot: 100, - Time: time.Now(), - } - require.NoError(t, s.PutLiveReserveRecord(ctx, record)) - - // Verify the record was stored - actual, err := s.GetLiveReserve(ctx, mint) - require.NoError(t, err) - assert.Equal(t, mint, actual.Mint) - assert.EqualValues(t, 1000, actual.SupplyFromBonding) - assert.EqualValues(t, 100, actual.Slot) - - // Update with a higher slot should succeed - record = ¤cy.ReserveRecord{ - Mint: mint, - SupplyFromBonding: 2000, - Slot: 200, - Time: time.Now(), - } - require.NoError(t, s.PutLiveReserveRecord(ctx, record)) - - actual, err = s.GetLiveReserve(ctx, mint) - require.NoError(t, err) - assert.EqualValues(t, 2000, actual.SupplyFromBonding) - assert.EqualValues(t, 200, actual.Slot) - - // Update with same slot should return stale error - record = ¤cy.ReserveRecord{ - Mint: mint, - SupplyFromBonding: 3000, - Slot: 200, - Time: time.Now(), - } - assert.Equal(t, currency.ErrStaleReserveState, s.PutLiveReserveRecord(ctx, record)) - - // Update with lower slot should return stale error - record = ¤cy.ReserveRecord{ - Mint: mint, - SupplyFromBonding: 3000, - Slot: 50, - Time: time.Now(), - } - assert.Equal(t, currency.ErrStaleReserveState, s.PutLiveReserveRecord(ctx, record)) - - // Verify original record unchanged after stale attempts - actual, err = s.GetLiveReserve(ctx, mint) - require.NoError(t, err) - assert.EqualValues(t, 2000, actual.SupplyFromBonding) - assert.EqualValues(t, 200, actual.Slot) - - // Different mint should work independently - otherMint := "other-live-mint" - record = ¤cy.ReserveRecord{ - Mint: otherMint, - SupplyFromBonding: 5000, - Slot: 50, - Time: time.Now(), - } - require.NoError(t, s.PutLiveReserveRecord(ctx, record)) - - actual, err = s.GetLiveReserve(ctx, otherMint) - require.NoError(t, err) - assert.EqualValues(t, 5000, actual.SupplyFromBonding) - assert.EqualValues(t, 50, actual.Slot) -} - -func testGetAllLiveReserves(t *testing.T, s currency.Store) { - ctx := context.Background() - - // No records should exist initially - _, err := s.GetAllLiveReserves(ctx) - assert.Equal(t, currency.ErrNotFound, err) - - // Insert live reserves for two mints - record1 := ¤cy.ReserveRecord{ - Mint: "mint-all-live-1", - SupplyFromBonding: 1000, - Slot: 100, - Time: time.Now(), - } - require.NoError(t, s.PutLiveReserveRecord(ctx, record1)) - - // Should return one record - reserves, err := s.GetAllLiveReserves(ctx) - require.NoError(t, err) - assert.Len(t, reserves, 1) - assert.EqualValues(t, 1000, reserves["mint-all-live-1"].SupplyFromBonding) - assert.EqualValues(t, 100, reserves["mint-all-live-1"].Slot) - - record2 := ¤cy.ReserveRecord{ - Mint: "mint-all-live-2", - SupplyFromBonding: 2000, - Slot: 200, - Time: time.Now(), - } - require.NoError(t, s.PutLiveReserveRecord(ctx, record2)) - - // Should return both records - reserves, err = s.GetAllLiveReserves(ctx) - require.NoError(t, err) - assert.Len(t, reserves, 2) - assert.EqualValues(t, 1000, reserves["mint-all-live-1"].SupplyFromBonding) - assert.EqualValues(t, 100, reserves["mint-all-live-1"].Slot) - assert.EqualValues(t, 2000, reserves["mint-all-live-2"].SupplyFromBonding) - assert.EqualValues(t, 200, reserves["mint-all-live-2"].Slot) - - // Update one mint and verify the change is reflected - record1Updated := ¤cy.ReserveRecord{ - Mint: "mint-all-live-1", - SupplyFromBonding: 1500, - Slot: 150, - Time: time.Now(), - } - require.NoError(t, s.PutLiveReserveRecord(ctx, record1Updated)) - - reserves, err = s.GetAllLiveReserves(ctx) - require.NoError(t, err) - assert.Len(t, reserves, 2) - assert.EqualValues(t, 1500, reserves["mint-all-live-1"].SupplyFromBonding) - assert.EqualValues(t, 150, reserves["mint-all-live-1"].Slot) - assert.EqualValues(t, 2000, reserves["mint-all-live-2"].SupplyFromBonding) - assert.EqualValues(t, 200, reserves["mint-all-live-2"].Slot) -} - -func testHolderCountRoundTrip(t *testing.T, s currency.Store) { - now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) - - record, err := s.GetHolderCountAtTime(context.Background(), "mint", now) - assert.Nil(t, record) - assert.Equal(t, currency.ErrNotFound, err) - - expected := ¤cy.HolderCountRecord{ - Mint: "mint", - HolderCount: 42, - Time: now, - } - require.NoError(t, s.PutHistoricalHolderCountRecord(context.Background(), expected)) - - assert.Equal(t, currency.ErrExists, s.PutHistoricalHolderCountRecord(context.Background(), expected)) - - actual, err := s.GetHolderCountAtTime(context.Background(), "mint", now) - require.NoError(t, err) - assert.Equal(t, now.Unix(), actual.Time.Unix()) - assert.EqualValues(t, expected.HolderCount, actual.HolderCount) - - actual, err = s.GetHolderCountAtTime(context.Background(), "mint", time.Date(2021, 01, 29, 14, 0, 5, 0, time.UTC)) - require.NoError(t, err) - - assert.Equal(t, now.Unix(), actual.Time.Unix()) - assert.EqualValues(t, expected.HolderCount, actual.HolderCount) - - tomorrow := time.Date(2021, 01, 30, 0, 0, 0, 0, time.UTC) - actual, err = s.GetHolderCountAtTime(context.Background(), "mint", tomorrow) - assert.Nil(t, actual) - assert.Equal(t, currency.ErrNotFound, err) -} - -func testGetAllHolderCountsAtTime(t *testing.T, s currency.Store) { - now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) - - // No records should exist initially - _, err := s.GetAllHolderCountsAtTime(context.Background(), now) - assert.Equal(t, currency.ErrNotFound, err) - - // Insert holder counts for two mints - record1 := ¤cy.HolderCountRecord{ - Mint: "mint1", - HolderCount: 42, - Time: now, - } - require.NoError(t, s.PutHistoricalHolderCountRecord(context.Background(), record1)) - - record2 := ¤cy.HolderCountRecord{ - Mint: "mint2", - HolderCount: 100, - Time: now, - } - require.NoError(t, s.PutHistoricalHolderCountRecord(context.Background(), record2)) - - // Should return both records at exact time - result, err := s.GetAllHolderCountsAtTime(context.Background(), now) - require.NoError(t, err) - assert.Len(t, result, 2) - assert.EqualValues(t, 42, result["mint1"].HolderCount) - assert.EqualValues(t, 100, result["mint2"].HolderCount) - - // Should return both records within same day - result, err = s.GetAllHolderCountsAtTime(context.Background(), time.Date(2021, 01, 29, 14, 0, 5, 0, time.UTC)) - require.NoError(t, err) - assert.Len(t, result, 2) - assert.EqualValues(t, 42, result["mint1"].HolderCount) - assert.EqualValues(t, 100, result["mint2"].HolderCount) - - // Day after should be empty - tomorrow := time.Date(2021, 01, 30, 0, 0, 0, 0, time.UTC) - _, err = s.GetAllHolderCountsAtTime(context.Background(), tomorrow) - assert.Equal(t, currency.ErrNotFound, err) -} - -func testLiveHolderCountRoundTrip(t *testing.T, s currency.Store) { - ctx := context.Background() - mint := "live-holder-mint" - - // No record should exist initially - _, err := s.GetLiveHolderCount(ctx, mint) - assert.Equal(t, currency.ErrNotFound, err) - - // Insert the first live holder count record - now := time.Now().UTC().Truncate(time.Second) - record := ¤cy.HolderCountRecord{ - Mint: mint, - HolderCount: 10, - Time: now, - } - require.NoError(t, s.PutLiveHolderCountRecord(ctx, record)) - - // Verify the record was stored - actual, err := s.GetLiveHolderCount(ctx, mint) - require.NoError(t, err) - assert.Equal(t, mint, actual.Mint) - assert.EqualValues(t, 10, actual.HolderCount) - - // Update with a later timestamp should succeed - record = ¤cy.HolderCountRecord{ - Mint: mint, - HolderCount: 20, - Time: now.Add(time.Second), - } - require.NoError(t, s.PutLiveHolderCountRecord(ctx, record)) - - actual, err = s.GetLiveHolderCount(ctx, mint) - require.NoError(t, err) - assert.EqualValues(t, 20, actual.HolderCount) - - // Update with same timestamp should return stale error - record = ¤cy.HolderCountRecord{ - Mint: mint, - HolderCount: 30, - Time: now.Add(time.Second), - } - assert.Equal(t, currency.ErrStaleHolderState, s.PutLiveHolderCountRecord(ctx, record)) - - // Update with earlier timestamp should return stale error - record = ¤cy.HolderCountRecord{ - Mint: mint, - HolderCount: 30, - Time: now, - } - assert.Equal(t, currency.ErrStaleHolderState, s.PutLiveHolderCountRecord(ctx, record)) - - // Verify original record unchanged after stale attempts - actual, err = s.GetLiveHolderCount(ctx, mint) - require.NoError(t, err) - assert.EqualValues(t, 20, actual.HolderCount) - - // Different mint should work independently - otherMint := "other-live-holder-mint" - record = ¤cy.HolderCountRecord{ - Mint: otherMint, - HolderCount: 50, - Time: now, - } - require.NoError(t, s.PutLiveHolderCountRecord(ctx, record)) - - actual, err = s.GetLiveHolderCount(ctx, otherMint) - require.NoError(t, err) - assert.EqualValues(t, 50, actual.HolderCount) -} - -func testGetAllLiveHolderCounts(t *testing.T, s currency.Store) { - ctx := context.Background() - - // No records should exist initially - _, err := s.GetAllLiveHolderCounts(ctx) - assert.Equal(t, currency.ErrNotFound, err) - - now := time.Now().UTC().Truncate(time.Second) - - // Insert holder counts for two mints - record1 := ¤cy.HolderCountRecord{ - Mint: "mint-all-live-holder-1", - HolderCount: 10, - Time: now, - } - require.NoError(t, s.PutLiveHolderCountRecord(ctx, record1)) - - // Should return one record - counts, err := s.GetAllLiveHolderCounts(ctx) - require.NoError(t, err) - assert.Len(t, counts, 1) - assert.EqualValues(t, 10, counts["mint-all-live-holder-1"].HolderCount) - - record2 := ¤cy.HolderCountRecord{ - Mint: "mint-all-live-holder-2", - HolderCount: 20, - Time: now, - } - require.NoError(t, s.PutLiveHolderCountRecord(ctx, record2)) - - // Should return both records - counts, err = s.GetAllLiveHolderCounts(ctx) - require.NoError(t, err) - assert.Len(t, counts, 2) - assert.EqualValues(t, 10, counts["mint-all-live-holder-1"].HolderCount) - assert.EqualValues(t, 20, counts["mint-all-live-holder-2"].HolderCount) - - // Update one mint and verify the change is reflected - record1Updated := ¤cy.HolderCountRecord{ - Mint: "mint-all-live-holder-1", - HolderCount: 15, - Time: now.Add(time.Second), - } - require.NoError(t, s.PutLiveHolderCountRecord(ctx, record1Updated)) - - counts, err = s.GetAllLiveHolderCounts(ctx) - require.NoError(t, err) - assert.Len(t, counts, 2) - assert.EqualValues(t, 15, counts["mint-all-live-holder-1"].HolderCount) - assert.EqualValues(t, 20, counts["mint-all-live-holder-2"].HolderCount) -} - -func assertEquivalentMetadataRecords(t *testing.T, obj1, obj2 *currency.MetadataRecord) { +func assertEquivalentRecords(t *testing.T, obj1, obj2 *currency.MetadataRecord) { assert.Equal(t, obj1.Name, obj2.Name) assert.Equal(t, obj1.Symbol, obj2.Symbol) assert.Equal(t, obj1.Description, obj2.Description) diff --git a/ocp/data/currency/postgres/model.go b/ocp/data/currency/postgres/model.go deleted file mode 100644 index 65acbcd..0000000 --- a/ocp/data/currency/postgres/model.go +++ /dev/null @@ -1,739 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "encoding/json" - "strings" - "time" - - "github.com/jmoiron/sqlx" - - q "github.com/code-payments/ocp-server/database/query" - "github.com/code-payments/ocp-server/ocp/data/currency" - - pgutil "github.com/code-payments/ocp-server/database/postgres" -) - -const ( - exchangeRateTableName = "ocp__core_exchangerate" - metadataTableName = "ocp__core_currencymetadata" - reserveTableName = "ocp__core_currencyreserve" - liveReserveTableName = "ocp__core_currencyreserve2" - holderCountTableName = "ocp__core_currencyholdercount" - liveHolderCountTableName = "ocp__core_currencyholdercount2" - - dateFormat = "2006-01-02" -) - -type exchangeRateModel struct { - Id sql.NullInt64 `db:"id"` - ForDate string `db:"for_date"` - ForTimestamp time.Time `db:"for_timestamp"` - CurrencyCode string `db:"currency_code"` - CurrencyRate float64 `db:"currency_rate"` -} - -func toExchangeRateModel(obj *currency.ExchangeRateRecord) *exchangeRateModel { - return &exchangeRateModel{ - Id: sql.NullInt64{Int64: int64(obj.Id), Valid: obj.Id > 0}, - ForDate: obj.Time.UTC().Format(dateFormat), - ForTimestamp: obj.Time.UTC(), - CurrencyCode: obj.Symbol, - CurrencyRate: obj.Rate, - } -} - -func fromExchangeRateModel(obj *exchangeRateModel) *currency.ExchangeRateRecord { - return ¤cy.ExchangeRateRecord{ - Id: uint64(obj.Id.Int64), - Time: obj.ForTimestamp.UTC(), - Symbol: obj.CurrencyCode, - Rate: obj.CurrencyRate, - } -} - -type metadataModel struct { - Id sql.NullInt64 `db:"id"` - - Name string `db:"name"` - Symbol string `db:"symbol"` - Description string `db:"description"` - ImageUrl string `db:"image_url"` - BillColors string `db:"bill_colors"` - SocialLinks string `db:"social_links"` - - Seed string `db:"seed"` - - Authority string `db:"authority"` - - Mint string `db:"mint"` - MintBump uint8 `db:"mint_bump"` - Decimals uint8 `db:"decimals"` - - CurrencyConfig string `db:"currency_config"` - CurrencyConfigBump uint8 `db:"currency_config_bump"` - - LiquidityPool string `db:"liquidity_pool"` - LiquidityPoolBump uint8 `db:"liquidity_pool_bump"` - - VaultMint string `db:"vault_mint"` - VaultMintBump uint8 `db:"vault_mint_bump"` - - VaultCore string `db:"vault_core"` - VaultCoreBump uint8 `db:"vault_core_bump"` - - SellFeeBps uint16 `db:"sell_fee_bps"` - - Alt string `db:"alt"` - - State uint8 `db:"state"` - Version uint64 `db:"version"` - - CreatedBy string `db:"created_by"` - CreatedAt time.Time `db:"created_at"` -} - -func toMetadataModel(obj *currency.MetadataRecord) (*metadataModel, error) { - if err := obj.Validate(); err != nil { - return nil, err - } - - return &metadataModel{ - Id: sql.NullInt64{Int64: int64(obj.Id), Valid: obj.Id > 0}, - - Name: obj.Name, - Symbol: obj.Symbol, - Description: obj.Description, - ImageUrl: obj.ImageUrl, - BillColors: strings.Join(obj.BillColors, ","), - SocialLinks: marshalSocialLinks(obj.SocialLinks), - - Seed: obj.Seed, - - Authority: obj.Authority, - - Mint: obj.Mint, - MintBump: obj.MintBump, - Decimals: obj.Decimals, - - CurrencyConfig: obj.CurrencyConfig, - CurrencyConfigBump: obj.CurrencyConfigBump, - - LiquidityPool: obj.LiquidityPool, - LiquidityPoolBump: obj.LiquidityPoolBump, - - VaultMint: obj.VaultMint, - VaultMintBump: obj.VaultMintBump, - - VaultCore: obj.VaultCore, - VaultCoreBump: obj.VaultCoreBump, - - SellFeeBps: obj.SellFeeBps, - - Alt: obj.Alt, - - State: uint8(obj.State), - Version: obj.Version, - - CreatedBy: obj.CreatedBy, - CreatedAt: obj.CreatedAt, - }, nil -} - -func fromMetadataModel(obj *metadataModel) *currency.MetadataRecord { - var billColors []string - if obj.BillColors != "" { - billColors = strings.Split(obj.BillColors, ",") - } - - return ¤cy.MetadataRecord{ - Id: uint64(obj.Id.Int64), - - Name: obj.Name, - Symbol: obj.Symbol, - Description: obj.Description, - ImageUrl: obj.ImageUrl, - BillColors: billColors, - SocialLinks: unmarshalSocialLinks(obj.SocialLinks), - - Seed: obj.Seed, - - Authority: obj.Authority, - - Mint: obj.Mint, - MintBump: obj.MintBump, - Decimals: obj.Decimals, - - CurrencyConfig: obj.CurrencyConfig, - CurrencyConfigBump: obj.CurrencyConfigBump, - - LiquidityPool: obj.LiquidityPool, - LiquidityPoolBump: obj.LiquidityPoolBump, - - VaultMint: obj.VaultMint, - VaultMintBump: obj.VaultMintBump, - - VaultCore: obj.VaultCore, - VaultCoreBump: obj.VaultCoreBump, - - SellFeeBps: obj.SellFeeBps, - - Alt: obj.Alt, - - State: currency.MetadataState(obj.State), - Version: obj.Version, - - CreatedBy: obj.CreatedBy, - CreatedAt: obj.CreatedAt, - } -} - -type historicalReserveModel struct { - Id sql.NullInt64 `db:"id"` - ForDate string `db:"for_date"` - ForTimestamp time.Time `db:"for_timestamp"` - Mint string `db:"mint"` - SupplyFromBonding uint64 `db:"supply_from_bonding"` -} - -func toHistoricalReserveModel(obj *currency.ReserveRecord) (*historicalReserveModel, error) { - if err := obj.Validate(); err != nil { - return nil, err - } - - return &historicalReserveModel{ - Id: sql.NullInt64{Int64: int64(obj.Id), Valid: obj.Id > 0}, - ForDate: obj.Time.UTC().Format(dateFormat), - ForTimestamp: obj.Time.UTC(), - Mint: obj.Mint, - SupplyFromBonding: obj.SupplyFromBonding, - }, nil -} - -func fromHistoricalReserveModel(obj *historicalReserveModel) *currency.ReserveRecord { - return ¤cy.ReserveRecord{ - Id: uint64(obj.Id.Int64), - Time: obj.ForTimestamp.UTC(), - Mint: obj.Mint, - SupplyFromBonding: obj.SupplyFromBonding, - } -} - -type liveReserveModel struct { - Id sql.NullInt64 `db:"id"` - Mint string `db:"mint"` - SupplyFromBonding uint64 `db:"supply_from_bonding"` - Slot uint64 `db:"slot"` - LastUpdatedAt time.Time `db:"last_updated_at"` -} - -func toLiveReserveModel(obj *currency.ReserveRecord) *liveReserveModel { - return &liveReserveModel{ - Id: sql.NullInt64{Int64: int64(obj.Id), Valid: obj.Id > 0}, - Mint: obj.Mint, - SupplyFromBonding: obj.SupplyFromBonding, - Slot: obj.Slot, - LastUpdatedAt: obj.Time.UTC(), - } -} - -func fromLiveReserveModel(obj *liveReserveModel) *currency.ReserveRecord { - return ¤cy.ReserveRecord{ - Id: uint64(obj.Id.Int64), - Mint: obj.Mint, - SupplyFromBonding: obj.SupplyFromBonding, - Slot: obj.Slot, - Time: obj.LastUpdatedAt.UTC(), - } -} - -type historicalHolderCountModel struct { - Id sql.NullInt64 `db:"id"` - ForDate string `db:"for_date"` - ForTimestamp time.Time `db:"for_timestamp"` - Mint string `db:"mint"` - HolderCount uint64 `db:"holder_count"` -} - -func toHistoricalHolderCountModel(obj *currency.HolderCountRecord) (*historicalHolderCountModel, error) { - if err := obj.Validate(); err != nil { - return nil, err - } - - return &historicalHolderCountModel{ - Id: sql.NullInt64{Int64: int64(obj.Id), Valid: obj.Id > 0}, - ForDate: obj.Time.UTC().Format(dateFormat), - ForTimestamp: obj.Time.UTC(), - Mint: obj.Mint, - HolderCount: obj.HolderCount, - }, nil -} - -func fromHistoricalHolderCountModel(obj *historicalHolderCountModel) *currency.HolderCountRecord { - return ¤cy.HolderCountRecord{ - Id: uint64(obj.Id.Int64), - Time: obj.ForTimestamp.UTC(), - Mint: obj.Mint, - HolderCount: obj.HolderCount, - } -} - -type liveHolderCountModel struct { - Id sql.NullInt64 `db:"id"` - Mint string `db:"mint"` - HolderCount uint64 `db:"holder_count"` - LastUpdatedAt time.Time `db:"last_updated_at"` -} - -func toLiveHolderCountModel(obj *currency.HolderCountRecord) *liveHolderCountModel { - return &liveHolderCountModel{ - Id: sql.NullInt64{Int64: int64(obj.Id), Valid: obj.Id > 0}, - Mint: obj.Mint, - HolderCount: obj.HolderCount, - LastUpdatedAt: obj.Time.UTC(), - } -} - -func fromLiveHolderCountModel(obj *liveHolderCountModel) *currency.HolderCountRecord { - return ¤cy.HolderCountRecord{ - Id: uint64(obj.Id.Int64), - Mint: obj.Mint, - HolderCount: obj.HolderCount, - Time: obj.LastUpdatedAt.UTC(), - } -} - -func marshalSocialLinks(links []currency.SocialLink) string { - if len(links) == 0 { - return "[]" - } - data, _ := json.Marshal(links) - return string(data) -} - -func unmarshalSocialLinks(data string) []currency.SocialLink { - if data == "" || data == "[]" { - return nil - } - var links []currency.SocialLink - _ = json.Unmarshal([]byte(data), &links) - return links -} - -func makeTimeBasedSelectQuery(table, condition string, ordering q.Ordering) string { - return `SELECT * FROM ` + table + ` WHERE ` + condition + ` ORDER BY for_timestamp ` + q.FromOrderingWithFallback(ordering, "asc") -} - -func makeTimeBasedGetQuery(table, condition string, ordering q.Ordering) string { - return makeTimeBasedSelectQuery(table, condition, ordering) + ` LIMIT 1` -} - -func makeTimeBasedRangeQuery(table, condition string, ordering q.Ordering, interval q.Interval) string { - var query, bucket string - - if interval == q.IntervalRaw { - query = `SELECT *` - } else { - bucket = `date_trunc('` + q.FromIntervalWithFallback(interval, "hour") + `', for_timestamp)` - query = `SELECT DISTINCT ON (` + bucket + `) *` - } - - query = query + ` FROM ` + table + ` WHERE ` + condition - - if interval == q.IntervalRaw { - query = query + ` ORDER BY for_timestamp ` + q.FromOrderingWithFallback(ordering, "asc") - } else { - query = query + ` ORDER BY ` + bucket + `, for_timestamp DESC` // keep only the latest record for each bucket - } - - return query -} - -func (m *exchangeRateModel) txSave(ctx context.Context, tx *sqlx.Tx) error { - err := tx.QueryRowxContext(ctx, - `INSERT INTO `+exchangeRateTableName+` - (for_date, for_timestamp, currency_code, currency_rate) - VALUES ($1, $2, $3, $4) - RETURNING id, for_date, for_timestamp, currency_code, currency_rate`, - m.ForDate, - m.ForTimestamp, - m.CurrencyCode, - m.CurrencyRate, - ).StructScan(m) - - return pgutil.CheckUniqueViolation(err, currency.ErrExists) -} - -func (m *metadataModel) dbSave(ctx context.Context, db *sqlx.DB) error { - return pgutil.ExecuteInTx(ctx, db, sql.LevelDefault, func(tx *sqlx.Tx) error { - err := tx.QueryRowxContext(ctx, - `INSERT INTO `+metadataTableName+` - (name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at) - VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16, $17, $18, $19, $20, $21, $22, $23 + 1, $24, $25) - - ON CONFLICT (mint) - DO UPDATE - SET description = $3, image_url = $4, bill_colors = $5, social_links = $6, alt = $21, state = $22, version = `+metadataTableName+`.version + 1 - WHERE `+metadataTableName+`.mint = $9 AND `+metadataTableName+`.version = $23 - - RETURNING id, name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at`, - m.Name, - m.Symbol, - m.Description, - m.ImageUrl, - m.BillColors, - m.SocialLinks, - m.Seed, - m.Authority, - m.Mint, - m.MintBump, - m.Decimals, - m.CurrencyConfig, - m.CurrencyConfigBump, - m.LiquidityPool, - m.LiquidityPoolBump, - m.VaultMint, - m.VaultMintBump, - m.VaultCore, - m.VaultCoreBump, - m.SellFeeBps, - m.Alt, - m.State, - m.Version, - m.CreatedBy, - m.CreatedAt, - ).StructScan(m) - - err = pgutil.CheckUniqueViolation(err, currency.ErrDuplicateCurrency) - if err == currency.ErrDuplicateCurrency { - return err - } - - return pgutil.CheckNoRows(err, currency.ErrStaleMetadataVersion) - }) -} - -func (m *historicalReserveModel) dbSave(ctx context.Context, db *sqlx.DB) error { - return pgutil.ExecuteInTx(ctx, db, sql.LevelDefault, func(tx *sqlx.Tx) error { - err := tx.QueryRowxContext(ctx, - `INSERT INTO `+reserveTableName+` - (for_date, for_timestamp, mint, supply_from_bonding) - VALUES ($1, $2, $3, $4) - RETURNING id, for_date, for_timestamp, mint, supply_from_bonding`, - m.ForDate, - m.ForTimestamp, - m.Mint, - m.SupplyFromBonding, - ).StructScan(m) - - return pgutil.CheckUniqueViolation(err, currency.ErrExists) - }) -} - -func dbGetExchangeRateBySymbolAndTime(ctx context.Context, db *sqlx.DB, symbol string, t time.Time, ordering q.Ordering) (*exchangeRateModel, error) { - res := &exchangeRateModel{} - err := db.GetContext(ctx, res, - makeTimeBasedGetQuery(exchangeRateTableName, "currency_code = $1 AND for_date = $2 AND for_timestamp <= $3", ordering), - symbol, - t.UTC().Format(dateFormat), - t.UTC(), - ) - return res, pgutil.CheckNoRows(err, currency.ErrNotFound) -} - -func dbGetAllExchangeRatesByTime(ctx context.Context, db *sqlx.DB, t time.Time, ordering q.Ordering) ([]*exchangeRateModel, error) { - query := `SELECT DISTINCT ON (currency_code) * - FROM ` + exchangeRateTableName + ` - WHERE for_date = $1 AND for_timestamp <= $2 - ORDER BY currency_code, for_timestamp ` + q.FromOrderingWithFallback(ordering, "asc") - - res := []*exchangeRateModel{} - err := db.SelectContext(ctx, &res, query, t.UTC().Format(dateFormat), t.UTC()) - - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - if res == nil { - return nil, currency.ErrNotFound - } - if len(res) == 0 { - return nil, currency.ErrNotFound - } - - return res, nil -} - -func dbGetAllExchangeRatesForRange(ctx context.Context, db *sqlx.DB, symbol string, interval q.Interval, start time.Time, end time.Time, ordering q.Ordering) ([]*exchangeRateModel, error) { - res := []*exchangeRateModel{} - err := db.SelectContext(ctx, &res, - makeTimeBasedRangeQuery(exchangeRateTableName, "currency_code = $1 AND for_timestamp >= $2 AND for_timestamp <= $3", ordering, interval), - symbol, start.UTC(), end.UTC(), - ) - - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - if len(res) == 0 { - return nil, currency.ErrNotFound - } - - return res, nil -} - -func dbGetAllMetadataByState(ctx context.Context, db *sqlx.DB, state currency.MetadataState, cursor q.Cursor, limit uint64, direction q.Ordering) ([]*metadataModel, error) { - res := []*metadataModel{} - - query := `SELECT - id, name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at - FROM ` + metadataTableName + ` - WHERE state = $1` - - opts := []interface{}{state} - query, opts = q.PaginateQuery(query, opts, cursor, limit, direction) - - err := db.SelectContext(ctx, &res, query, opts...) - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - - if len(res) == 0 { - return nil, currency.ErrNotFound - } - return res, nil -} - -func dbGetMetadataByMint(ctx context.Context, db *sqlx.DB, mint string) (*metadataModel, error) { - res := &metadataModel{} - err := db.GetContext(ctx, res, - `SELECT id, name, symbol, description, image_url, bill_colors, social_links, seed, authority, mint, mint_bump, decimals, currency_config, currency_config_bump, liquidity_pool, liquidity_pool_bump, vault_mint, vault_mint_bump, vault_core, vault_core_bump, sell_fee_bps, alt, state, version, created_by, created_at - FROM `+metadataTableName+` - WHERE mint = $1`, - mint, - ) - return res, pgutil.CheckNoRows(err, currency.ErrNotFound) -} - -func dbGetAllMints(ctx context.Context, db *sqlx.DB) ([]string, error) { - var res []string - err := db.SelectContext(ctx, &res, - `SELECT mint FROM `+metadataTableName, - ) - - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - if len(res) == 0 { - return nil, currency.ErrNotFound - } - - return res, nil -} - -func dbCountMints(ctx context.Context, db *sqlx.DB) (uint64, error) { - var count uint64 - err := db.GetContext(ctx, &count, - `SELECT COUNT(*) FROM `+metadataTableName+` WHERE state != $1`, - currency.MetadataStateAbandoned, - ) - if err != nil { - return 0, err - } - return count, nil -} - -func dbCountMetadataByState(ctx context.Context, db *sqlx.DB, state currency.MetadataState) (uint64, error) { - var res uint64 - query := `SELECT COUNT(*) FROM ` + metadataTableName + ` WHERE state = $1` - err := db.GetContext(ctx, &res, query, state) - if err != nil { - return 0, err - } - return res, nil -} - -func dbIsNameAvailable(ctx context.Context, db *sqlx.DB, name string) (bool, error) { - var count uint64 - err := db.GetContext(ctx, &count, - `SELECT COUNT(*) FROM `+metadataTableName+` WHERE LOWER(name) = LOWER($1) AND state != $2`, - name, - currency.MetadataStateAbandoned, - ) - if err != nil { - return false, err - } - return count == 0, nil -} - -func dbGetReserveByMintAndTime(ctx context.Context, db *sqlx.DB, mint string, t time.Time, ordering q.Ordering) (*historicalReserveModel, error) { - res := &historicalReserveModel{} - err := db.GetContext(ctx, res, - makeTimeBasedGetQuery(reserveTableName, "mint = $1 AND for_date = $2 AND for_timestamp <= $3", ordering), - mint, - t.UTC().Format(dateFormat), - t.UTC(), - ) - return res, pgutil.CheckNoRows(err, currency.ErrNotFound) -} - -func dbGetAllReservesForRange(ctx context.Context, db *sqlx.DB, mint string, interval q.Interval, start time.Time, end time.Time, ordering q.Ordering) ([]*historicalReserveModel, error) { - res := []*historicalReserveModel{} - err := db.SelectContext(ctx, &res, - makeTimeBasedRangeQuery(reserveTableName, "mint = $1 AND for_timestamp >= $2 AND for_timestamp <= $3", ordering, interval), - mint, start.UTC(), end.UTC(), - ) - - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - if len(res) == 0 { - return nil, currency.ErrNotFound - } - - return res, nil -} - -func (m *liveReserveModel) dbSave(ctx context.Context, db *sqlx.DB) error { - return pgutil.ExecuteInTx(ctx, db, sql.LevelDefault, func(tx *sqlx.Tx) error { - err := tx.QueryRowxContext(ctx, - `INSERT INTO `+liveReserveTableName+` - (mint, supply_from_bonding, slot, last_updated_at) - VALUES ($1, $2, $3, $4) - - ON CONFLICT (mint) - DO UPDATE SET supply_from_bonding = $2, slot = $3, last_updated_at = $4 - WHERE `+liveReserveTableName+`.slot < $3 - - RETURNING id, mint, supply_from_bonding, slot, last_updated_at`, - m.Mint, - m.SupplyFromBonding, - m.Slot, - m.LastUpdatedAt, - ).StructScan(m) - - return pgutil.CheckNoRows(err, currency.ErrStaleReserveState) - }) -} - -func dbGetLiveReserveByMint(ctx context.Context, db *sqlx.DB, mint string) (*liveReserveModel, error) { - res := &liveReserveModel{} - err := db.GetContext(ctx, res, - `SELECT id, mint, supply_from_bonding, slot, last_updated_at - FROM `+liveReserveTableName+` - WHERE mint = $1`, - mint, - ) - return res, pgutil.CheckNoRows(err, currency.ErrNotFound) -} - -func dbGetAllLiveReserves(ctx context.Context, db *sqlx.DB) ([]*liveReserveModel, error) { - var res []*liveReserveModel - err := db.SelectContext(ctx, &res, - `SELECT id, mint, supply_from_bonding, slot, last_updated_at - FROM `+liveReserveTableName, - ) - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - if len(res) == 0 { - return nil, currency.ErrNotFound - } - return res, nil -} - -func (m *historicalHolderCountModel) dbSave(ctx context.Context, db *sqlx.DB) error { - return pgutil.ExecuteInTx(ctx, db, sql.LevelDefault, func(tx *sqlx.Tx) error { - err := tx.QueryRowxContext(ctx, - `INSERT INTO `+holderCountTableName+` - (for_date, for_timestamp, mint, holder_count) - VALUES ($1, $2, $3, $4) - RETURNING id, for_date, for_timestamp, mint, holder_count`, - m.ForDate, - m.ForTimestamp, - m.Mint, - m.HolderCount, - ).StructScan(m) - - return pgutil.CheckUniqueViolation(err, currency.ErrExists) - }) -} - -func dbGetHolderCountByMintAndTime(ctx context.Context, db *sqlx.DB, mint string, t time.Time, ordering q.Ordering) (*historicalHolderCountModel, error) { - res := &historicalHolderCountModel{} - err := db.GetContext(ctx, res, - makeTimeBasedGetQuery(holderCountTableName, "mint = $1 AND for_date = $2 AND for_timestamp <= $3", ordering), - mint, - t.UTC().Format(dateFormat), - t.UTC(), - ) - return res, pgutil.CheckNoRows(err, currency.ErrNotFound) -} - -func dbGetAllHolderCountsByTime(ctx context.Context, db *sqlx.DB, t time.Time, ordering q.Ordering) ([]*historicalHolderCountModel, error) { - query := `SELECT DISTINCT ON (mint) * - FROM ` + holderCountTableName + ` - WHERE for_date = $1 AND for_timestamp <= $2 - ORDER BY mint, for_timestamp ` + q.FromOrderingWithFallback(ordering, "asc") - - res := []*historicalHolderCountModel{} - err := db.SelectContext(ctx, &res, query, t.UTC().Format(dateFormat), t.UTC()) - - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - if res == nil { - return nil, currency.ErrNotFound - } - if len(res) == 0 { - return nil, currency.ErrNotFound - } - - return res, nil -} - -func (m *liveHolderCountModel) dbSave(ctx context.Context, db *sqlx.DB) error { - return pgutil.ExecuteInTx(ctx, db, sql.LevelDefault, func(tx *sqlx.Tx) error { - err := tx.QueryRowxContext(ctx, - `INSERT INTO `+liveHolderCountTableName+` - (mint, holder_count, last_updated_at) - VALUES ($1, $2, $3) - - ON CONFLICT (mint) - DO UPDATE SET holder_count = $2, last_updated_at = $3 - WHERE `+liveHolderCountTableName+`.last_updated_at < $3 - - RETURNING id, mint, holder_count, last_updated_at`, - m.Mint, - m.HolderCount, - m.LastUpdatedAt, - ).StructScan(m) - - return pgutil.CheckNoRows(err, currency.ErrStaleHolderState) - }) -} - -func dbGetLiveHolderCountByMint(ctx context.Context, db *sqlx.DB, mint string) (*liveHolderCountModel, error) { - res := &liveHolderCountModel{} - err := db.GetContext(ctx, res, - `SELECT id, mint, holder_count, last_updated_at - FROM `+liveHolderCountTableName+` - WHERE mint = $1`, - mint, - ) - return res, pgutil.CheckNoRows(err, currency.ErrNotFound) -} - -func dbGetAllLiveHolderCounts(ctx context.Context, db *sqlx.DB) ([]*liveHolderCountModel, error) { - var res []*liveHolderCountModel - err := db.SelectContext(ctx, &res, - `SELECT id, mint, holder_count, last_updated_at - FROM `+liveHolderCountTableName, - ) - if err != nil { - return nil, pgutil.CheckNoRows(err, currency.ErrNotFound) - } - if len(res) == 0 { - return nil, currency.ErrNotFound - } - return res, nil -} diff --git a/ocp/data/currency/postgres/store.go b/ocp/data/currency/postgres/store.go deleted file mode 100644 index e0cd801..0000000 --- a/ocp/data/currency/postgres/store.go +++ /dev/null @@ -1,318 +0,0 @@ -package postgres - -import ( - "context" - "database/sql" - "time" - - "github.com/jmoiron/sqlx" - - "github.com/code-payments/ocp-server/database/query" - "github.com/code-payments/ocp-server/ocp/data/currency" - - pg "github.com/code-payments/ocp-server/database/postgres" -) - -type store struct { - db *sqlx.DB -} - -func New(db *sql.DB) currency.Store { - return &store{ - db: sqlx.NewDb(db, "pgx"), - } -} - -func (s *store) PutExchangeRates(ctx context.Context, obj *currency.MultiRateRecord) error { - return pg.ExecuteInTx(ctx, s.db, sql.LevelDefault, func(tx *sqlx.Tx) error { - // Loop through all rates and save individual records (within a transaction) - for symbol, item := range obj.Rates { - err := toExchangeRateModel(¤cy.ExchangeRateRecord{ - Time: obj.Time, - Rate: item, - Symbol: symbol, - }).txSave(ctx, tx) - - if err != nil { - return err - } - } - return nil - }) -} - -func (s *store) GetExchangeRate(ctx context.Context, symbol string, t time.Time) (*currency.ExchangeRateRecord, error) { - obj, err := dbGetExchangeRateBySymbolAndTime(ctx, s.db, symbol, t, query.Descending) - if err != nil { - return nil, err - } - - return fromExchangeRateModel(obj), nil -} - -func (s *store) GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) { - list, err := dbGetAllExchangeRatesByTime(ctx, s.db, t, query.Descending) - if err != nil { - return nil, err - } - - res := ¤cy.MultiRateRecord{ - Time: list[0].ForTimestamp, - Rates: map[string]float64{}, - } - for _, item := range list { - res.Rates[item.CurrencyCode] = item.CurrencyRate - } - - return res, nil -} - -func (s *store) GetExchangeRatesInRange(ctx context.Context, symbol string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ExchangeRateRecord, error) { - if interval > query.IntervalMonth { - return nil, currency.ErrInvalidInterval - } - - if start.IsZero() || end.IsZero() { - return nil, currency.ErrInvalidRange - } - - var actualStart, actualEnd time.Time - if start.Unix() > end.Unix() { - actualStart = end - actualEnd = start - } else { - actualStart = start - actualEnd = end - } - - // TODO: check that the range is reasonable - - list, err := dbGetAllExchangeRatesForRange(ctx, s.db, symbol, interval, actualStart, actualEnd, ordering) - if err != nil { - return nil, err - } - - res := []*currency.ExchangeRateRecord{} - for _, item := range list { - res = append(res, fromExchangeRateModel(item)) - } - - return res, nil -} - -func (s *store) SaveMetadata(ctx context.Context, record *currency.MetadataRecord) error { - model, err := toMetadataModel(record) - if err != nil { - return err - } - - err = model.dbSave(ctx, s.db) - if err != nil { - return err - } - - fromMetadataModel(model).CopyTo(record) - - return nil -} - -func (s *store) GetMetadata(ctx context.Context, mint string) (*currency.MetadataRecord, error) { - model, err := dbGetMetadataByMint(ctx, s.db, mint) - if err != nil { - return nil, err - } - return fromMetadataModel(model), nil -} - -func (s *store) GetAllMetadataByState(ctx context.Context, state currency.MetadataState, cursor query.Cursor, limit uint64, direction query.Ordering) ([]*currency.MetadataRecord, error) { - models, err := dbGetAllMetadataByState(ctx, s.db, state, cursor, limit, direction) - if err != nil { - return nil, err - } - - res := make([]*currency.MetadataRecord, len(models)) - for i, model := range models { - res[i] = fromMetadataModel(model) - } - return res, nil -} - -func (s *store) GetAllMints(ctx context.Context) ([]string, error) { - return dbGetAllMints(ctx, s.db) -} - -func (s *store) CountMints(ctx context.Context) (uint64, error) { - return dbCountMints(ctx, s.db) -} - -func (s *store) CountMetadataByState(ctx context.Context, state currency.MetadataState) (uint64, error) { - return dbCountMetadataByState(ctx, s.db, state) -} - -func (s *store) IsNameAvailable(ctx context.Context, name string) (bool, error) { - return dbIsNameAvailable(ctx, s.db, name) -} - -func (s *store) PutHistoricalReserveRecord(ctx context.Context, record *currency.ReserveRecord) error { - model, err := toHistoricalReserveModel(record) - if err != nil { - return err - } - - err = model.dbSave(ctx, s.db) - if err != nil { - return err - } - - fromHistoricalReserveModel(model).CopyTo(record) - - return nil -} - -func (s *store) GetReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) { - model, err := dbGetReserveByMintAndTime(ctx, s.db, mint, t, query.Descending) - if err != nil { - return nil, err - } - return fromHistoricalReserveModel(model), nil -} - -func (s *store) GetReservesInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ReserveRecord, error) { - if interval > query.IntervalMonth { - return nil, currency.ErrInvalidInterval - } - - if start.IsZero() || end.IsZero() { - return nil, currency.ErrInvalidRange - } - - var actualStart, actualEnd time.Time - if start.Unix() > end.Unix() { - actualStart = end - actualEnd = start - } else { - actualStart = start - actualEnd = end - } - - list, err := dbGetAllReservesForRange(ctx, s.db, mint, interval, actualStart, actualEnd, ordering) - if err != nil { - return nil, err - } - - res := []*currency.ReserveRecord{} - for _, item := range list { - res = append(res, fromHistoricalReserveModel(item)) - } - - return res, nil -} - -func (s *store) PutLiveReserveRecord(ctx context.Context, record *currency.ReserveRecord) error { - model := toLiveReserveModel(record) - - err := model.dbSave(ctx, s.db) - if err != nil { - return err - } - - fromLiveReserveModel(model).CopyTo(record) - - return nil -} - -func (s *store) GetLiveReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) { - model, err := dbGetLiveReserveByMint(ctx, s.db, mint) - if err != nil { - return nil, err - } - return fromLiveReserveModel(model), nil -} - -func (s *store) GetAllLiveReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) { - models, err := dbGetAllLiveReserves(ctx, s.db) - if err != nil { - return nil, err - } - - res := make(map[string]*currency.ReserveRecord, len(models)) - for _, model := range models { - record := fromLiveReserveModel(model) - res[record.Mint] = record - } - return res, nil -} - -func (s *store) PutHistoricalHolderCountRecord(ctx context.Context, record *currency.HolderCountRecord) error { - model, err := toHistoricalHolderCountModel(record) - if err != nil { - return err - } - - err = model.dbSave(ctx, s.db) - if err != nil { - return err - } - - fromHistoricalHolderCountModel(model).CopyTo(record) - - return nil -} - -func (s *store) GetHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*currency.HolderCountRecord, error) { - model, err := dbGetHolderCountByMintAndTime(ctx, s.db, mint, t, query.Descending) - if err != nil { - return nil, err - } - return fromHistoricalHolderCountModel(model), nil -} - -func (s *store) GetAllHolderCountsAtTime(ctx context.Context, t time.Time) (map[string]*currency.HolderCountRecord, error) { - list, err := dbGetAllHolderCountsByTime(ctx, s.db, t, query.Descending) - if err != nil { - return nil, err - } - - res := make(map[string]*currency.HolderCountRecord, len(list)) - for _, item := range list { - record := fromHistoricalHolderCountModel(item) - res[record.Mint] = record - } - - return res, nil -} - -func (s *store) PutLiveHolderCountRecord(ctx context.Context, record *currency.HolderCountRecord) error { - model := toLiveHolderCountModel(record) - - err := model.dbSave(ctx, s.db) - if err != nil { - return err - } - - fromLiveHolderCountModel(model).CopyTo(record) - - return nil -} - -func (s *store) GetLiveHolderCount(ctx context.Context, mint string) (*currency.HolderCountRecord, error) { - model, err := dbGetLiveHolderCountByMint(ctx, s.db, mint) - if err != nil { - return nil, err - } - return fromLiveHolderCountModel(model), nil -} - -func (s *store) GetAllLiveHolderCounts(ctx context.Context) (map[string]*currency.HolderCountRecord, error) { - models, err := dbGetAllLiveHolderCounts(ctx, s.db) - if err != nil { - return nil, err - } - - res := make(map[string]*currency.HolderCountRecord, len(models)) - for _, model := range models { - record := fromLiveHolderCountModel(model) - res[record.Mint] = record - } - return res, nil -} diff --git a/ocp/data/currency/reserve/store.go b/ocp/data/currency/reserve/store.go index de29b62..33fb821 100644 --- a/ocp/data/currency/reserve/store.go +++ b/ocp/data/currency/reserve/store.go @@ -1,13 +1,6 @@ // Package reserve defines a focused store for currency creator mint reserve // states. // -// It mirrors the reserve portion of the larger ocp/data/currency store, reusing -// that package's record type (currency.ReserveRecord) and error sentinels -// (currency.ErrNotFound, currency.ErrExists, currency.ErrInvalidRange, -// currency.ErrInvalidInterval, currency.ErrStaleReserveState). A DynamoDB-backed -// implementation lives in the dynamodb subpackage and an in-memory -// implementation lives in memory. -// // Records are keyed per mint (the number of mints is unbounded), so every record // is a single item — there is no map-of-all-mints row. package reserve diff --git a/ocp/data/currency/store.go b/ocp/data/currency/store.go index bd82db3..2d30f89 100644 --- a/ocp/data/currency/store.go +++ b/ocp/data/currency/store.go @@ -1,11 +1,7 @@ package currency import ( - "context" "errors" - "time" - - "github.com/code-payments/ocp-server/database/query" ) var ( @@ -18,142 +14,3 @@ var ( ErrStaleHolderState = errors.New("holder count state is stale") ErrDuplicateCurrency = errors.New("duplicate currency detected") ) - -type Store interface { - // PutExchangeRates puts exchange rate records for the core mint into the store. - PutExchangeRates(ctx context.Context, record *MultiRateRecord) error - - // GetExchangeRate gets price information given a certain time and currency symbol - // for the core mint. If the exact time is not available, the most recent data prior - // to the requested date within the same day will get returned, if available. - // - // ErrNotFound is returned if no price data was found for the provided Timestamp. - GetExchangeRate(ctx context.Context, symbol string, t time.Time) (*ExchangeRateRecord, error) - - // GetAllExchangeRates gets price information given a certain time for the core mint. - // If the exact time is not available, the most recent data prior to the requested date - // within the same day will get returned, if available. - // - // ErrNotFound is returned if no price data was found for the provided Timestamp. - GetAllExchangeRates(ctx context.Context, t time.Time) (*MultiRateRecord, error) - - // GetExchangeRatesInRange gets the price information for a range of time given a currency - // symbol and interval for the core mint. The start and end timestamps are provided along - // with the interval. If the raw data is not available at the sampling frequency requested, - // it will be linearly interpolated between available points. - // - // ErrNotFound is returned if the symbol or the exchange rates for the symbol cannot be found - // ErrInvalidRange is returned if the range is not valid - // ErrInvalidInterval is returned if the interval is not valid - GetExchangeRatesInRange(ctx context.Context, symbol string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*ExchangeRateRecord, error) - - // SaveMetadata creates or updates currency creator metadata in the store. - // On insert, Version is set to 1. On update, only mutable fields (Description, - // ImageUrl, BillColors, SocialLinks, Alt, State) are updated and Version is incremented. - // ErrStaleMetadataVersion is returned when the provided version doesn't match. - SaveMetadata(ctx context.Context, record *MetadataRecord) error - - // GetMetadata gets currency creator mint metadata by the mint address - GetMetadata(ctx context.Context, mint string) (*MetadataRecord, error) - - // GetAllMetadataByState returns all currency metadata records in a given state - // - // ErrNotFound is returned if no metadata records exist for the given state - GetAllMetadataByState(ctx context.Context, state MetadataState, cursor query.Cursor, limit uint64, direction query.Ordering) ([]*MetadataRecord, error) - - // CountMetadataByState returns the count of currency metadata records in the requested state - CountMetadataByState(ctx context.Context, state MetadataState) (uint64, error) - - // GetAllMints returns the public keys of all currency creator mints - // - // ErrNotFound is returned if no mints exist - GetAllMints(ctx context.Context) ([]string, error) - - // CountMints returns the total number of currency creator mints, - // excluding those in the abandoned state. - CountMints(ctx context.Context) (uint64, error) - - // IsNameAvailable checks whether a currency name is available for use. - // The check is case-insensitive. - IsNameAvailable(ctx context.Context, name string) (bool, error) - - // PutHistoricalReserveRecord puts a currency creator mint reserve records into the store. - PutHistoricalReserveRecord(ctx context.Context, record *ReserveRecord) error - - // GetReserveAtTime gets reserve state for a given currency creator mint at a point - // in time. If the exact time is not available, the most recent data prior to the - // requested date within the same day will get returned, if available. - // - // ErrNotFound is returned if no reserve data was found for the provided Timestamp. - GetReserveAtTime(ctx context.Context, mint string, t time.Time) (*ReserveRecord, error) - - // GetReservesInRange gets the reserve records for a range of time given a currency - // creator mint and interval. The start and end timestamps are provided along with - // the interval. - // - // ErrNotFound is returned if the mint or the reserves for the mint cannot be found - // ErrInvalidRange is returned if the range is not valid - // ErrInvalidInterval is returned if the interval is not valid - GetReservesInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*ReserveRecord, error) - - // PutLiveReserveRecord upserts the latest reserve record for a currency creator - // mint. An upsert is only performed if the provided slot is greater than the slot - // currently stored. - // - // ErrStaleReserveState is returned if the provided slot is not greater than the - // stored slot. - PutLiveReserveRecord(ctx context.Context, record *ReserveRecord) error - - // GetLiveReserve gets the latest live reserve record for a currency creator mint. - // - // ErrNotFound is returned if no live reserve record exists for the provided mint. - GetLiveReserve(ctx context.Context, mint string) (*ReserveRecord, error) - - // GetAllLiveReserves gets the latest live reserve records for all currency - // creator mints. - // - // ErrNotFound is returned if no live reserve records exist. - GetAllLiveReserves(ctx context.Context) (map[string]*ReserveRecord, error) - - // PutHistoricalHolderCountRecord puts a currency creator mint holder count - // record into the store. - PutHistoricalHolderCountRecord(ctx context.Context, record *HolderCountRecord) error - - // GetHolderCountAtTime gets the holder count for a given currency creator - // mint at a point in time. If the exact time is not available, the most - // recent data prior to the requested date within the same day will get - // returned, if available. - // - // ErrNotFound is returned if no holder count data was found for the - // provided Timestamp. - GetHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*HolderCountRecord, error) - - // GetAllHolderCountsAtTime gets all holder counts at a specific point in - // time. For each mint, the most recent record prior to the requested time - // within the same day is returned. - // - // ErrNotFound is returned if no holder count data was found for the - // provided Timestamp. - GetAllHolderCountsAtTime(ctx context.Context, t time.Time) (map[string]*HolderCountRecord, error) - - // PutLiveHolderCountRecord upserts the latest holder count record for a - // currency creator mint. An upsert is only performed if the provided - // timestamp is greater than the timestamp currently stored. - // - // ErrStaleHolderState is returned if the provided timestamp is not greater - // than the stored timestamp. - PutLiveHolderCountRecord(ctx context.Context, record *HolderCountRecord) error - - // GetLiveHolderCount gets the latest live holder count record for a - // currency creator mint. - // - // ErrNotFound is returned if no live holder count record exists for the - // provided mint. - GetLiveHolderCount(ctx context.Context, mint string) (*HolderCountRecord, error) - - // GetAllLiveHolderCounts gets the latest live holder count records for all - // currency creator mints. - // - // ErrNotFound is returned if no live holder count records exist. - GetAllLiveHolderCounts(ctx context.Context) (map[string]*HolderCountRecord, error) -} diff --git a/ocp/data/internal.go b/ocp/data/internal.go index b846cf0..3a21613 100644 --- a/ocp/data/internal.go +++ b/ocp/data/internal.go @@ -10,7 +10,6 @@ import ( "github.com/jmoiron/sqlx" "github.com/code-payments/ocp-server/cache" - currency_lib "github.com/code-payments/ocp-server/currency" pg "github.com/code-payments/ocp-server/database/postgres" "github.com/code-payments/ocp-server/database/query" timelock_token "github.com/code-payments/ocp-server/solana/timelock/v1" @@ -23,6 +22,7 @@ import ( "github.com/code-payments/ocp-server/ocp/data/action" "github.com/code-payments/ocp-server/ocp/data/balance" "github.com/code-payments/ocp-server/ocp/data/currency" + currency_metadata "github.com/code-payments/ocp-server/ocp/data/currency/metadata" "github.com/code-payments/ocp-server/ocp/data/deposit" "github.com/code-payments/ocp-server/ocp/data/fulfillment" "github.com/code-payments/ocp-server/ocp/data/intent" @@ -40,7 +40,7 @@ import ( account_memory_client "github.com/code-payments/ocp-server/ocp/data/account/memory" action_memory_client "github.com/code-payments/ocp-server/ocp/data/action/memory" balance_memory_client "github.com/code-payments/ocp-server/ocp/data/balance/memory" - currency_memory_client "github.com/code-payments/ocp-server/ocp/data/currency/memory" + currency_metadata_memory_client "github.com/code-payments/ocp-server/ocp/data/currency/metadata/memory" deposit_memory_client "github.com/code-payments/ocp-server/ocp/data/deposit/memory" fulfillment_memory_client "github.com/code-payments/ocp-server/ocp/data/fulfillment/memory" intent_memory_client "github.com/code-payments/ocp-server/ocp/data/intent/memory" @@ -58,7 +58,7 @@ import ( account_postgres_client "github.com/code-payments/ocp-server/ocp/data/account/postgres" action_postgres_client "github.com/code-payments/ocp-server/ocp/data/action/postgres" balance_postgres_client "github.com/code-payments/ocp-server/ocp/data/balance/postgres" - currency_postgres_client "github.com/code-payments/ocp-server/ocp/data/currency/postgres" + currency_metadata_postgres_client "github.com/code-payments/ocp-server/ocp/data/currency/metadata/postgres" deposit_postgres_client "github.com/code-payments/ocp-server/ocp/data/deposit/postgres" fulfillment_postgres_client "github.com/code-payments/ocp-server/ocp/data/fulfillment/postgres" intent_postgres_client "github.com/code-payments/ocp-server/ocp/data/intent/postgres" @@ -76,10 +76,6 @@ import ( // Cache Constants const ( - maxExchangeRateCacheBudget = 1000000 // 1 million - singleExchangeRateCacheWeight = 1 - multiExchangeRateCacheWeight = 60 // usually we get 60 exchange rates from CoinGecko for a single time interval - maxTimelockCacheBudget = 100000 timelockCacheTTL = 5 * time.Second // Keep this relatively small ) @@ -131,10 +127,6 @@ type DatabaseData interface { // Currency // -------------------------------------------------------------------------------- - GetExchangeRate(ctx context.Context, code currency_lib.Code, t time.Time) (*currency.ExchangeRateRecord, error) - GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) - GetExchangeRateHistory(ctx context.Context, code currency_lib.Code, opts ...query.Option) ([]*currency.ExchangeRateRecord, error) - ImportExchangeRates(ctx context.Context, record *currency.MultiRateRecord) error SaveCurrencyMetadata(ctx context.Context, record *currency.MetadataRecord) error GetCurrencyMetadata(ctx context.Context, mint string) (*currency.MetadataRecord, error) GetAllCurrencyMetadataByState(ctx context.Context, state currency.MetadataState, opts ...query.Option) ([]*currency.MetadataRecord, error) @@ -142,18 +134,6 @@ type DatabaseData interface { GetAllCurrencyMints(ctx context.Context) ([]string, error) CountCurrencyMints(ctx context.Context) (uint64, error) IsCurrencyNameAvailable(ctx context.Context, name string) (bool, error) - PutHistoricalCurrencyReserve(ctx context.Context, record *currency.ReserveRecord) error - GetCurrencyReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) - GetCurrencyReserveHistory(ctx context.Context, mint string, opts ...query.Option) ([]*currency.ReserveRecord, error) - PutLiveCurrencyReserve(ctx context.Context, record *currency.ReserveRecord) error - GetLiveCurrencyReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) - GetAllLiveCurrencyReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) - PutHistoricalCurrencyHolderCount(ctx context.Context, record *currency.HolderCountRecord) error - GetCurrencyHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*currency.HolderCountRecord, error) - GetAllCurrencyHolderCountsAtTime(ctx context.Context, t time.Time) (map[string]*currency.HolderCountRecord, error) - PutLiveCurrencyHolderCount(ctx context.Context, record *currency.HolderCountRecord) error - GetLiveCurrencyHolderCount(ctx context.Context, mint string) (*currency.HolderCountRecord, error) - GetAllLiveCurrencyHolderCounts(ctx context.Context) (map[string]*currency.HolderCountRecord, error) // Deposits // -------------------------------------------------------------------------------- @@ -286,7 +266,7 @@ type DatabaseProvider struct { accounts account.Store actions action.Store balance balance.Store - currencies currency.Store + currencies currency_metadata.Store deposits deposit.Store fulfillments fulfillment.Store intents intent.Store @@ -301,7 +281,6 @@ type DatabaseProvider struct { vmRam vm_ram.Store vmStorage vm_storage.Store - exchangeCache cache.Cache timelockCache cache.Cache db *sqlx.DB @@ -332,7 +311,7 @@ func NewDatabaseProvider(dbConfig *pg.Config) (DatabaseData, error) { accounts: account_postgres_client.New(db), actions: action_postgres_client.New(db), balance: balance_postgres_client.New(db), - currencies: currency_postgres_client.New(db), + currencies: currency_metadata_postgres_client.New(db), deposits: deposit_postgres_client.New(db), fulfillments: fulfillment_postgres_client.New(db), intents: intent_postgres_client.New(db), @@ -347,7 +326,6 @@ func NewDatabaseProvider(dbConfig *pg.Config) (DatabaseData, error) { vmRam: vm_ram_postgres_client.New(db), vmStorage: vm_storage_postgres_client.New(db), - exchangeCache: cache.NewCache(maxExchangeRateCacheBudget), timelockCache: cache.NewCache(maxTimelockCacheBudget), db: sqlx.NewDb(db, "pgx"), @@ -359,7 +337,7 @@ func NewTestDatabaseProvider() DatabaseData { accounts: account_memory_client.New(), actions: action_memory_client.New(), balance: balance_memory_client.New(), - currencies: currency_memory_client.New(), + currencies: currency_metadata_memory_client.New(), deposits: deposit_memory_client.New(), fulfillments: fulfillment_memory_client.New(), intents: intent_memory_client.New(), @@ -374,7 +352,6 @@ func NewTestDatabaseProvider() DatabaseData { vmRam: vm_ram_memory_client.New(), vmStorage: vm_storage_memory_client.New(), - exchangeCache: cache.NewCache(maxExchangeRateCacheBudget), timelockCache: nil, // Shouldn't be used for tests } } @@ -489,56 +466,6 @@ func (dp *DatabaseProvider) GetExternalBalanceCheckpoint(ctx context.Context, ac // Currencies // -------------------------------------------------------------------------------- -func (dp *DatabaseProvider) GetExchangeRate(ctx context.Context, code currency_lib.Code, t time.Time) (*currency.ExchangeRateRecord, error) { - key := fmt.Sprintf("%s:%s", code, t.Truncate(5*time.Minute).Format(time.RFC3339)) - if rate, ok := dp.exchangeCache.Retrieve(key); ok { - return rate.(*currency.ExchangeRateRecord), nil - } - - rate, err := dp.currencies.GetExchangeRate(ctx, string(code), t) - if err != nil { - return nil, err - } - - dp.exchangeCache.Insert(key, rate, singleExchangeRateCacheWeight) - - return rate, nil -} -func (dp *DatabaseProvider) GetAllExchangeRates(ctx context.Context, t time.Time) (*currency.MultiRateRecord, error) { - key := fmt.Sprintf("everything:%s", t.Truncate(5*time.Minute).Format(time.RFC3339)) - if rates, ok := dp.exchangeCache.Retrieve(key); ok { - return rates.(*currency.MultiRateRecord), nil - } - - rates, err := dp.currencies.GetAllExchangeRates(ctx, t) - if err != nil { - return nil, err - } - dp.exchangeCache.Insert(key, rates, multiExchangeRateCacheWeight) - - return rates, nil -} -func (dp *DatabaseProvider) GetExchangeRateHistory(ctx context.Context, code currency_lib.Code, opts ...query.Option) ([]*currency.ExchangeRateRecord, error) { - req := query.QueryOptions{ - Limit: maxCurrencyHistoryReqSize, - End: time.Now(), - SortBy: query.Ascending, - Supported: query.CanLimitResults | query.CanSortBy | query.CanBucketBy | query.CanQueryByStartTime | query.CanQueryByEndTime, - } - req.Apply(opts...) - - if req.Start.IsZero() { - return nil, query.ErrQueryNotSupported - } - if req.Limit > maxCurrencyHistoryReqSize { - return nil, query.ErrQueryNotSupported - } - - return dp.currencies.GetExchangeRatesInRange(ctx, string(code), req.Interval, req.Start, req.End, req.SortBy) -} -func (dp *DatabaseProvider) ImportExchangeRates(ctx context.Context, data *currency.MultiRateRecord) error { - return dp.currencies.PutExchangeRates(ctx, data) -} func (dp *DatabaseProvider) SaveCurrencyMetadata(ctx context.Context, record *currency.MetadataRecord) error { return dp.currencies.SaveMetadata(ctx, record) } @@ -564,57 +491,6 @@ func (dp *DatabaseProvider) CountCurrencyMints(ctx context.Context) (uint64, err func (dp *DatabaseProvider) IsCurrencyNameAvailable(ctx context.Context, name string) (bool, error) { return dp.currencies.IsNameAvailable(ctx, name) } -func (dp *DatabaseProvider) PutHistoricalCurrencyReserve(ctx context.Context, record *currency.ReserveRecord) error { - return dp.currencies.PutHistoricalReserveRecord(ctx, record) -} -func (dp *DatabaseProvider) GetCurrencyReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) { - return dp.currencies.GetReserveAtTime(ctx, mint, t) -} -func (dp *DatabaseProvider) GetCurrencyReserveHistory(ctx context.Context, mint string, opts ...query.Option) ([]*currency.ReserveRecord, error) { - req := query.QueryOptions{ - Limit: maxCurrencyHistoryReqSize, - End: time.Now(), - SortBy: query.Ascending, - Supported: query.CanLimitResults | query.CanSortBy | query.CanBucketBy | query.CanQueryByStartTime | query.CanQueryByEndTime, - } - req.Apply(opts...) - - if req.Start.IsZero() { - return nil, query.ErrQueryNotSupported - } - if req.Limit > maxCurrencyHistoryReqSize { - return nil, query.ErrQueryNotSupported - } - - return dp.currencies.GetReservesInRange(ctx, mint, req.Interval, req.Start, req.End, req.SortBy) -} -func (dp *DatabaseProvider) PutLiveCurrencyReserve(ctx context.Context, record *currency.ReserveRecord) error { - return dp.currencies.PutLiveReserveRecord(ctx, record) -} -func (dp *DatabaseProvider) GetLiveCurrencyReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) { - return dp.currencies.GetLiveReserve(ctx, mint) -} -func (dp *DatabaseProvider) GetAllLiveCurrencyReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) { - return dp.currencies.GetAllLiveReserves(ctx) -} -func (dp *DatabaseProvider) PutHistoricalCurrencyHolderCount(ctx context.Context, record *currency.HolderCountRecord) error { - return dp.currencies.PutHistoricalHolderCountRecord(ctx, record) -} -func (dp *DatabaseProvider) GetCurrencyHolderCountAtTime(ctx context.Context, mint string, t time.Time) (*currency.HolderCountRecord, error) { - return dp.currencies.GetHolderCountAtTime(ctx, mint, t) -} -func (dp *DatabaseProvider) GetAllCurrencyHolderCountsAtTime(ctx context.Context, t time.Time) (map[string]*currency.HolderCountRecord, error) { - return dp.currencies.GetAllHolderCountsAtTime(ctx, t) -} -func (dp *DatabaseProvider) PutLiveCurrencyHolderCount(ctx context.Context, record *currency.HolderCountRecord) error { - return dp.currencies.PutLiveHolderCountRecord(ctx, record) -} -func (dp *DatabaseProvider) GetLiveCurrencyHolderCount(ctx context.Context, mint string) (*currency.HolderCountRecord, error) { - return dp.currencies.GetLiveHolderCount(ctx, mint) -} -func (dp *DatabaseProvider) GetAllLiveCurrencyHolderCounts(ctx context.Context) (map[string]*currency.HolderCountRecord, error) { - return dp.currencies.GetAllLiveHolderCounts(ctx) -} // Deposits // -------------------------------------------------------------------------------- diff --git a/ocp/data/provider.go b/ocp/data/provider.go index a1212a0..89a2039 100644 --- a/ocp/data/provider.go +++ b/ocp/data/provider.go @@ -4,10 +4,6 @@ import ( pg "github.com/code-payments/ocp-server/database/postgres" ) -const ( - maxCurrencyHistoryReqSize = 1024 -) - type Provider interface { BlockchainData DatabaseData From 9133a6d4e4c92406f325f63e4d839a617320ebe2 Mon Sep 17 00:00:00 2001 From: jeffyanta Date: Wed, 17 Jun 2026 10:46:37 -0400 Subject: [PATCH 4/5] Update currency store usage sites --- go.mod | 6 +++ go.sum | 12 +++++ ocp/aml/guard_test.go | 8 ---- ocp/currency/data_provider.go | 43 ++++++++++++++--- ocp/currency/usd_market_value.go | 10 ++-- ocp/rpc/account/server_test.go | 26 +++++++---- ocp/rpc/currency/historical_data.go | 22 ++++----- ocp/rpc/messaging/testutil.go | 21 +++++++-- ocp/rpc/transaction/limits.go | 7 ++- ocp/worker/account/testutil.go | 8 ---- ocp/worker/currency/exchangerate/runtime.go | 17 ++++--- ocp/worker/currency/holder/runtime.go | 52 ++++++++------------- ocp/worker/currency/launcher/runtime.go | 22 +++++---- ocp/worker/currency/launcher/util.go | 12 ++--- ocp/worker/currency/reserve/runtime.go | 23 +++++---- ocp/worker/geyser/backup.go | 2 +- ocp/worker/geyser/currency_reserve.go | 9 ++-- ocp/worker/geyser/external_deposit.go | 16 ++++--- ocp/worker/geyser/handler.go | 28 ++++++----- ocp/worker/geyser/runtime.go | 16 +++++-- ocp/worker/swap/runtime.go | 36 ++++++++------ ocp/worker/swap/util.go | 8 ++-- testutil/currency.go | 8 ++-- 23 files changed, 248 insertions(+), 164 deletions(-) diff --git a/go.mod b/go.mod index c8d93d9..493cad8 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( filippo.io/edwards25519 v1.1.0 github.com/anyproto/go-slip10 v1.0.1 github.com/aws/aws-sdk-go-v2 v1.42.0 + github.com/aws/aws-sdk-go-v2/config v1.32.25 github.com/aws/aws-sdk-go-v2/credentials v1.19.24 github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.25 github.com/aws/aws-sdk-go-v2/service/dynamodb v1.59.0 @@ -46,6 +47,7 @@ require ( github.com/Microsoft/go-winio v0.4.14 // indirect github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 // indirect github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 // indirect github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 // indirect github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.7.29 // indirect github.com/aws/aws-sdk-go-v2/internal/v4a v1.4.30 // indirect @@ -54,6 +56,10 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.12.6 // indirect github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.13.29 // indirect github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 // indirect + github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 // indirect + github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 // indirect github.com/aws/smithy-go v1.27.1 // indirect github.com/cenkalti/backoff/v4 v4.1.0 // indirect github.com/containerd/continuity v0.0.0-20190827140505-75bee3e2ccb6 // indirect diff --git a/go.sum b/go.sum index 655fd4a..ed5105d 100644 --- a/go.sum +++ b/go.sum @@ -37,8 +37,12 @@ github.com/aws/aws-sdk-go-v2 v1.42.0 h1:XvXMJTkFQtpBKIWZnmr9ZEOc2InWM2yldjXEJ/by github.com/aws/aws-sdk-go-v2 v1.42.0/go.mod h1:27+ACypSLljLAEKsCYOmrjKh83vuTRkuAe9Uv/3A4bg= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11 h1:h5+3VT69KUBK24grGuuA5saDJTj2IIjLb9au668Fo5I= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.7.11/go.mod h1:dnakxebH6UwFvcvujL0LVggYQ8nEvBGjU4G/V79Nv94= +github.com/aws/aws-sdk-go-v2/config v1.32.25 h1:ACCejvStYoilgwrfegSt5ZntCbPrk52qfwyNcnl3omM= +github.com/aws/aws-sdk-go-v2/config v1.32.25/go.mod h1:LJyU8sDRbXUxFn8xMJIGP+v9QYYwveNLI8a/giAOiAs= github.com/aws/aws-sdk-go-v2/credentials v1.19.24 h1:2hQqYCV9yqyePQ9o6dCrZc/zO8U3TwPr9mIKlZnPu/I= github.com/aws/aws-sdk-go-v2/credentials v1.19.24/go.mod h1:IDwpACtwqHLISdzfwUUNq4P9DsB/h5BLg4FwJPNfqFY= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29 h1:r6qZHbT+wxgWO/e9vYNUEtg7lv5+UN3pRqKhLXvnArg= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.18.29/go.mod h1:QRnaRcTVGKPGRy8w78HMQtKUGRYcnMZAANATkeVA6Mo= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.25 h1:EeK30mZmhopHcNmKykyGF0LwmFB1ZwQNr+FeyRjcN0U= github.com/aws/aws-sdk-go-v2/feature/rds/auth v1.6.25/go.mod h1:tudVnwAJyXgCh4N6ABYdzUM+i+PXsx8700FOyhMn+4k= github.com/aws/aws-sdk-go-v2/internal/configsources v1.4.29 h1:f3vKqSo13fhTYb+JEcXwXefZQE26I1FB5eTSniU67ko= @@ -61,6 +65,14 @@ github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25 h1:2pQEbwf+/6EDb github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.19.25/go.mod h1:KvT6NCcQ0EZ+ZkVRrlBMt04Po3ok23YELEp7WimhLhM= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2 h1:ie4ElCmUKS26pzrZcIk/lmt4yWjAqLLcawstyQCh298= github.com/aws/aws-sdk-go-v2/service/s3 v1.102.2/go.mod h1:zjsomFeX5duj+4PlMB+o4JoWTIx+G0XMyzjYrUbQkN0= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0 h1:3nXpRcFwRCW8n7HgO2QGy0Dc20eQNfBuUemGQhpF8m8= +github.com/aws/aws-sdk-go-v2/service/signin v1.2.0/go.mod h1:LxYujSTLPRlp2vTtcUO/+1ilrew8ytt6SvQyOgejzFQ= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3 h1:ey1XLTYXb9PcLt4535632o5kCGXNXEhNb620Dqwuylo= +github.com/aws/aws-sdk-go-v2/service/sso v1.31.3/go.mod h1:Lk7PlmoTYryQmyBG0EXqj5BcUbj3whXdU2s3yGI3EAc= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6 h1:yLr03zQE/5Eu5l3QU0Si+xMbLMbSDF2YXsigqXngs6g= +github.com/aws/aws-sdk-go-v2/service/ssooidc v1.36.6/go.mod h1:Q5N6icH+KJZDLh+ESNwzdv6cZ6vLFF/egy3IOxWhmz4= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3 h1:VrIhKRCSK1umelSgB9RghvA9RTUYeQffyAS5ApXehNI= +github.com/aws/aws-sdk-go-v2/service/sts v1.43.3/go.mod h1:r8wkDOuLaaMFqFiYAb8dGY2A3gJCOujMc6CFOVC4Zhc= github.com/aws/smithy-go v1.27.1 h1:4T340VFndXtADGF52gYa1POyL7s9E4Z1OeZ1hCscIw8= github.com/aws/smithy-go v1.27.1/go.mod h1:YE2RhdIuDbA5E5bTdciG9KrW3+TiEONeUWCqxX9i1Fc= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= diff --git a/ocp/aml/guard_test.go b/ocp/aml/guard_test.go index bb81663..5e13178 100644 --- a/ocp/aml/guard_test.go +++ b/ocp/aml/guard_test.go @@ -13,7 +13,6 @@ import ( "github.com/code-payments/ocp-server/ocp/common" currency_util "github.com/code-payments/ocp-server/ocp/currency" ocp_data "github.com/code-payments/ocp-server/ocp/data" - "github.com/code-payments/ocp-server/ocp/data/currency" "github.com/code-payments/ocp-server/ocp/data/intent" "github.com/code-payments/ocp-server/testutil" ) @@ -151,13 +150,6 @@ func setupAmlTest(t *testing.T) (env amlTestEnv) { testutil.SetupRandomSubsidizer(t, env.data) - env.data.ImportExchangeRates(env.ctx, ¤cy.MultiRateRecord{ - Time: time.Now(), - Rates: map[string]float64{ - string(currency_lib.USD): 0.1, - }, - }) - return env } diff --git a/ocp/currency/data_provider.go b/ocp/currency/data_provider.go index 7d30600..ce9dad2 100644 --- a/ocp/currency/data_provider.go +++ b/ocp/currency/data_provider.go @@ -16,11 +16,15 @@ import ( commonpb "github.com/code-payments/ocp-protobuf-api/generated/go/common/v1" currencypb "github.com/code-payments/ocp-protobuf-api/generated/go/currency/v1" + "github.com/code-payments/ocp-server/database/query" "github.com/code-payments/ocp-server/ocp/auth" "github.com/code-payments/ocp-server/ocp/common" "github.com/code-payments/ocp-server/ocp/config" ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/solana/currencycreator" timelock_token "github.com/code-payments/ocp-server/solana/timelock/v1" "github.com/code-payments/ocp-server/usdc" @@ -52,8 +56,12 @@ type cachedProtoMint struct { } type MintDataProvider struct { - log *zap.Logger - data ocp_data.Provider + log *zap.Logger + + data ocp_data.Provider + exchangeRateStore exchange.Store + reserveStore reserve.Store + holderStore holder.Store protoMintCacheTTL time.Duration exchangeRatePollInterval time.Duration @@ -102,6 +110,9 @@ type MintDataProvider struct { func NewMintDataProvider( log *zap.Logger, data ocp_data.Provider, + exchangeRateStore exchange.Store, + reserveStore reserve.Store, + holderStore holder.Store, protoMintCacheTTL, exchangeRatePollInterval, launchpadCurrencyPollInterval time.Duration, @@ -110,6 +121,9 @@ func NewMintDataProvider( return &MintDataProvider{ log: log, data: data, + exchangeRateStore: exchangeRateStore, + reserveStore: reserveStore, + holderStore: holderStore, protoMintCacheTTL: protoMintCacheTTL, exchangeRatePollInterval: exchangeRatePollInterval, launchpadCurrencyPollInterval: launchpadCurrencyPollInterval, @@ -476,6 +490,18 @@ func (m *MintDataProvider) GetLiveExchangeRates(ctx context.Context) (*LiveExcha return m.exchangeRates, nil } +// GetExchangeRateHistory returns historical exchange rate records for the core +// mint, passing straight through to the underlying exchange rate store. +func (m *MintDataProvider) GetExchangeRateHistory(ctx context.Context, symbol string, interval query.Interval, start, end time.Time, ordering query.Ordering) ([]*currency.ExchangeRateRecord, error) { + return m.exchangeRateStore.GetExchangeRatesInRange(ctx, symbol, interval, start, end, ordering) +} + +// GetReserveHistory returns historical reserve records for a currency creator +// mint, passing straight through to the underlying reserve store. +func (m *MintDataProvider) GetReserveHistory(ctx context.Context, mint string, interval query.Interval, start, end time.Time, ordering query.Ordering) ([]*currency.ReserveRecord, error) { + return m.reserveStore.GetReservesInRange(ctx, mint, interval, start, end, ordering) +} + // GetAllCachedReserveStates returns a snapshot of all currently cached reserve // states keyed by mint address. It blocks until the first successful poll has // completed. @@ -856,7 +882,7 @@ func (m *MintDataProvider) pollExchangeRates(ctx context.Context) { } func (m *MintDataProvider) fetchAndUpdateExchangeRates(ctx context.Context, log *zap.Logger) { - rates, err := m.data.GetAllExchangeRates(ctx, time.Now()) + rates, err := m.exchangeRateStore.GetAllExchangeRates(ctx, time.Now()) if err != nil { log.With(zap.Error(err)).Warn("failed to fetch exchange rates") return @@ -902,7 +928,7 @@ func (m *MintDataProvider) pollReserveState(ctx context.Context) { } func (m *MintDataProvider) fetchAndUpdateReserveStates(ctx context.Context) { - liveReserves, err := m.data.GetAllLiveCurrencyReserves(ctx) + liveReserves, err := m.reserveStore.GetAllLiveReserves(ctx) if err == currency.ErrNotFound { return } @@ -974,7 +1000,7 @@ func (m *MintDataProvider) pollHolderCounts(ctx context.Context) { } func (m *MintDataProvider) fetchAndUpdateHolderCounts(ctx context.Context) { - liveCounts, err := m.data.GetAllLiveCurrencyHolderCounts(ctx) + liveCounts, err := m.holderStore.GetAllLiveHolderCounts(ctx) if err == currency.ErrNotFound { return } @@ -983,10 +1009,15 @@ func (m *MintDataProvider) fetchAndUpdateHolderCounts(ctx context.Context) { return } + mints := make([]string, 0, len(liveCounts)) + for mintAddr := range liveCounts { + mints = append(mints, mintAddr) + } + var includeWeeklyDeltas bool oneWeekAgo := time.Now().Add(-7 * 24 * time.Hour) endOfWeekAgoDay := time.Date(oneWeekAgo.Year(), oneWeekAgo.Month(), oneWeekAgo.Day(), 23, 59, 59, 0, time.UTC) - historicalCounts, err := m.data.GetAllCurrencyHolderCountsAtTime(ctx, endOfWeekAgoDay) + historicalCounts, err := m.holderStore.GetHolderCountsForDay(ctx, mints, endOfWeekAgoDay) if err != nil && err != currency.ErrNotFound { m.log.With(zap.Error(err)).Warn("failed to fetch historical holder counts for weekly delta") } else { diff --git a/ocp/currency/usd_market_value.go b/ocp/currency/usd_market_value.go index eb3a497..6695872 100644 --- a/ocp/currency/usd_market_value.go +++ b/ocp/currency/usd_market_value.go @@ -10,6 +10,8 @@ import ( "github.com/code-payments/ocp-server/ocp/common" ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/solana/currencycreator" ) @@ -28,7 +30,7 @@ func CalculateUsdMarketValueFromFiatAmount(fiatAmount, fiatToUsdRate float64) (f // CalculateUsdMarketValueFromTokenAmount calculates the current USD market value // of a crypto amount in quarks. -func CalculateUsdMarketValueFromTokenAmount(ctx context.Context, data ocp_data.Provider, mint *common.Account, quarks uint64, at time.Time) (float64, error) { +func CalculateUsdMarketValueFromTokenAmount(ctx context.Context, data ocp_data.Provider, exchangeRateStore exchange.Store, reserveStore reserve.Store, mint *common.Account, quarks uint64, at time.Time) (float64, error) { isLive := time.Since(at) < 5*time.Second isSupportedMint, err := common.IsSupportedMint(ctx, data, mint) @@ -48,7 +50,7 @@ func CalculateUsdMarketValueFromTokenAmount(ctx context.Context, data ocp_data.P Time: at, } } else { - exchangeRateRecord, err = data.GetExchangeRate(ctx, currency_lib.USD, at) + exchangeRateRecord, err = exchangeRateStore.GetExchangeRate(ctx, string(currency_lib.USD), at) if err != nil { return 0, err } @@ -64,12 +66,12 @@ func CalculateUsdMarketValueFromTokenAmount(ctx context.Context, data ocp_data.P var reserveRecord *currency.ReserveRecord if isLive { - reserveRecord, err = data.GetLiveCurrencyReserve(ctx, mint.PublicKey().ToBase58()) + reserveRecord, err = reserveStore.GetLiveReserve(ctx, mint.PublicKey().ToBase58()) if err != nil { return 0, err } } else { - reserveRecord, err = data.GetCurrencyReserveAtTime(ctx, mint.PublicKey().ToBase58(), at) + reserveRecord, err = reserveStore.GetReserveAtTime(ctx, mint.PublicKey().ToBase58(), at) if err != nil { return 0, err } diff --git a/ocp/rpc/account/server_test.go b/ocp/rpc/account/server_test.go index f86a7d3..98eba93 100644 --- a/ocp/rpc/account/server_test.go +++ b/ocp/rpc/account/server_test.go @@ -24,6 +24,11 @@ import ( ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/account" "github.com/code-payments/ocp-server/ocp/data/action" + exchange_memory "github.com/code-payments/ocp-server/ocp/data/currency/exchange/memory" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" + holder_memory "github.com/code-payments/ocp-server/ocp/data/currency/holder/memory" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" + reserve_memory "github.com/code-payments/ocp-server/ocp/data/currency/reserve/memory" "github.com/code-payments/ocp-server/ocp/data/deposit" "github.com/code-payments/ocp-server/ocp/data/intent" "github.com/code-payments/ocp-server/ocp/data/timelock" @@ -36,11 +41,13 @@ import ( ) type testEnv struct { - ctx context.Context - client accountpb.AccountClient - server *server - data ocp_data.Provider - subsidizer *common.Account + ctx context.Context + client accountpb.AccountClient + server *server + data ocp_data.Provider + reserveStore reserve.Store + holderStore holder.Store + subsidizer *common.Account } func setup(t *testing.T) (env testEnv, cleanup func()) { @@ -52,9 +59,12 @@ func setup(t *testing.T) (env testEnv, cleanup func()) { env.ctx = context.Background() env.client = accountpb.NewAccountClient(conn) env.data = ocp_data.NewTestDataProvider() + env.reserveStore = reserve_memory.New() + env.holderStore = holder_memory.New() env.subsidizer = testutil.SetupRandomSubsidizer(t, env.data) - mintDataProvider := currency_util.NewMintDataProvider(log, env.data, 0, time.Second, time.Second) + exchangeRateStore := exchange_memory.New() + mintDataProvider := currency_util.NewMintDataProvider(log, env.data, exchangeRateStore, env.reserveStore, env.holderStore, 0, time.Second, time.Second) s := NewAccountServer(log, env.data, mintDataProvider) env.server = s.(*server) @@ -155,7 +165,7 @@ func TestGetTokenAccountInfos_UserAccounts_HappyPath(t *testing.T) { coreVmConfig := testutil.NewRandomVmConfig(t, true) swapVmConfig := testutil.NewRandomVmConfig(t, false) - jeffyMint := testutil.SetupLaunchpadCurrency(t, env.data) + jeffyMint := testutil.SetupLaunchpadCurrency(t, env.data, env.reserveStore, env.holderStore) jeffyVmConfig, err := common.GetVmConfigForMint(env.ctx, env.data, jeffyMint) require.NoError(t, err) @@ -312,7 +322,7 @@ func TestGetTokenAccountInfos_RemoteSendGiftCard_HappyPath(t *testing.T) { env, cleanup := setup(t) defer cleanup() - mint := testutil.SetupLaunchpadCurrency(t, env.data) + mint := testutil.SetupLaunchpadCurrency(t, env.data, env.reserveStore, env.holderStore) vmConfig, err := common.GetVmConfigForMint(env.ctx, env.data, mint) require.NoError(t, err) diff --git a/ocp/rpc/currency/historical_data.go b/ocp/rpc/currency/historical_data.go index 64bcec6..8d4f54a 100644 --- a/ocp/rpc/currency/historical_data.go +++ b/ocp/rpc/currency/historical_data.go @@ -188,13 +188,13 @@ func (s *currencyServer) getCachedReserveHistory( return cached.([]*currency.ReserveRecord), nil } - reserveHistory, err := s.data.GetCurrencyReserveHistory( + reserveHistory, err := s.mintDataProvider.GetReserveHistory( ctx, mint, - query.WithStartTime(startTime), - query.WithEndTime(endTime), - query.WithInterval(interval), - query.WithDirection(query.Ascending), + interval, + startTime, + endTime, + query.Ascending, ) if err != nil { return nil, err @@ -217,13 +217,13 @@ func (s *currencyServer) getCachedExchangeRateHistory( return cached.([]*currency.ExchangeRateRecord), nil } - exchangeRateHistory, err := s.data.GetExchangeRateHistory( + exchangeRateHistory, err := s.mintDataProvider.GetExchangeRateHistory( ctx, - currencyCode, - query.WithStartTime(startTime), - query.WithEndTime(endTime), - query.WithInterval(interval), - query.WithDirection(query.Ascending), + string(currencyCode), + interval, + startTime, + endTime, + query.Ascending, ) if err != nil { return nil, err diff --git a/ocp/rpc/messaging/testutil.go b/ocp/rpc/messaging/testutil.go index f8a8843..523c0b2 100644 --- a/ocp/rpc/messaging/testutil.go +++ b/ocp/rpc/messaging/testutil.go @@ -28,6 +28,11 @@ import ( ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/account" "github.com/code-payments/ocp-server/ocp/data/currency" + exchange_memory "github.com/code-payments/ocp-server/ocp/data/currency/exchange/memory" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" + holder_memory "github.com/code-payments/ocp-server/ocp/data/currency/holder/memory" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" + reserve_memory "github.com/code-payments/ocp-server/ocp/data/currency/reserve/memory" "github.com/code-payments/ocp-server/ocp/data/messaging" messaging_memory "github.com/code-payments/ocp-server/ocp/data/messaging/memory" "github.com/code-payments/ocp-server/ocp/data/rendezvous" @@ -53,12 +58,18 @@ func setup(t *testing.T, enableMultiServer bool) (env testEnv, cleanup func()) { data := ocp_data.NewTestDataProvider() messages := messaging_memory.New() + exchangeRateStore := exchange_memory.New() + reserveStore := reserve_memory.New() + holderStore := holder_memory.New() + env.client1 = &clientEnv{ ctx: context.Background(), client: messagingpb.NewMessagingClient(conn1), conf: &clientConf{}, streams: make(map[string][]*cancellableStream), directDataAccess: data, + reserveStore: reserveStore, + holderStore: holderStore, } env.client2 = &clientEnv{ ctx: context.Background(), @@ -66,6 +77,8 @@ func setup(t *testing.T, enableMultiServer bool) (env testEnv, cleanup func()) { conf: &clientConf{}, streams: make(map[string][]*cancellableStream), directDataAccess: data, + reserveStore: reserveStore, + holderStore: holderStore, } if enableMultiServer { env.client2.client = messagingpb.NewMessagingClient(conn2) @@ -73,14 +86,14 @@ func setup(t *testing.T, enableMultiServer bool) (env testEnv, cleanup func()) { subsidizer := testutil.SetupRandomSubsidizer(t, data) - require.NoError(t, data.ImportExchangeRates(context.Background(), ¤cy.MultiRateRecord{ + require.NoError(t, exchangeRateStore.PutExchangeRates(context.Background(), ¤cy.MultiRateRecord{ Time: time.Now(), Rates: map[string]float64{ "usd": 0.1, }, })) - mintDataProvider := currency_util.NewMintDataProvider(log, data, 0, time.Second, time.Second) + mintDataProvider := currency_util.NewMintDataProvider(log, data, exchangeRateStore, reserveStore, holderStore, 0, time.Second, time.Second) require.NoError(t, mintDataProvider.Start(context.Background())) s1 := NewMessagingClientAndServer(log, data, messages, mintDataProvider, auth.NewRPCSignatureVerifier(log, data), conn1.Target(), withManualTestOverrides(&testOverrides{})) @@ -211,6 +224,8 @@ type clientEnv struct { // Direct data access to help test/pass validation checks directDataAccess ocp_data.Provider + reserveStore reserve.Store + holderStore holder.Store } func (c *clientEnv) openMessageStream(t *testing.T, rendezvousKey *common.Account, enableKeepAlive bool) { @@ -471,7 +486,7 @@ func (c *clientEnv) sendRequestToGiveBillMessage(t *testing.T, rendezvousKey *co if c.conf.simulateUnsupportedMint { mintAccount = testutil.NewRandomAccount(t) } else if c.conf.simulateLaunchpadMint { - mintAccount = testutil.SetupLaunchpadCurrency(t, c.directDataAccess) + mintAccount = testutil.SetupLaunchpadCurrency(t, c.directDataAccess, c.reserveStore, c.holderStore) } requestToGiveBill := &messagingpb.RequestToGiveBill{ diff --git a/ocp/rpc/transaction/limits.go b/ocp/rpc/transaction/limits.go index ac8d751..0c14685 100644 --- a/ocp/rpc/transaction/limits.go +++ b/ocp/rpc/transaction/limits.go @@ -4,7 +4,6 @@ import ( "context" "fmt" "math" - "time" "go.uber.org/zap" "google.golang.org/grpc/codes" @@ -36,13 +35,13 @@ func (s *transactionServer) GetLimits(ctx context.Context, req *transactionpb.Ge return nil, err } - multiRateRecord, err := s.data.GetAllExchangeRates(ctx, time.Now()) + liveExchangeData, err := s.mintDataProvider.GetLiveExchangeRates(ctx) if err != nil { log.With(zap.Error(err)).Warn("failure getting current exchange rates") return nil, status.Error(codes.Internal, "") } - usdRate, ok := multiRateRecord.Rates[string(currency_lib.USD)] + usdRate, ok := liveExchangeData.Rates[string(currency_lib.USD)] if !ok { log.With(zap.Error(err)).Warn("usd rate is missing") return nil, status.Error(codes.Internal, "") @@ -61,7 +60,7 @@ func (s *transactionServer) GetLimits(ctx context.Context, req *transactionpb.Ge } sendLimits := make(map[string]*transactionpb.SendLimit) for currency, sendLimit := range currency_util.SendLimits { - otherRate, ok := multiRateRecord.Rates[string(currency)] + otherRate, ok := liveExchangeData.Rates[string(currency)] if !ok { log.Debug(fmt.Sprintf("%s rate is missing", currency)) continue diff --git a/ocp/worker/account/testutil.go b/ocp/worker/account/testutil.go index ece8876..af6aa33 100644 --- a/ocp/worker/account/testutil.go +++ b/ocp/worker/account/testutil.go @@ -17,7 +17,6 @@ import ( ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/account" "github.com/code-payments/ocp-server/ocp/data/action" - "github.com/code-payments/ocp-server/ocp/data/currency" "github.com/code-payments/ocp-server/ocp/data/fulfillment" "github.com/code-payments/ocp-server/ocp/data/intent" "github.com/code-payments/ocp-server/pointer" @@ -46,13 +45,6 @@ func setup(t *testing.T) *testEnv { require.NoError(t, common.InjectTestSubsidizer(context.Background(), data, testutil.NewRandomAccount(t))) - require.NoError(t, data.ImportExchangeRates(context.Background(), ¤cy.MultiRateRecord{ - Time: time.Now(), - Rates: map[string]float64{ - "usd": 0.1, - }, - })) - return &testEnv{ ctx: context.Background(), data: data, diff --git a/ocp/worker/currency/exchangerate/runtime.go b/ocp/worker/currency/exchangerate/runtime.go index e06c445..c0ca499 100644 --- a/ocp/worker/currency/exchangerate/runtime.go +++ b/ocp/worker/currency/exchangerate/runtime.go @@ -14,20 +14,23 @@ import ( currency_lib "github.com/code-payments/ocp-server/currency" ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" "github.com/code-payments/ocp-server/ocp/worker" ) type exchangeRateRuntime struct { - log *zap.Logger - data ocp_data.Provider + log *zap.Logger + data ocp_data.Provider + exchangeRateStore exchange.Store lastRates *currency.MultiRateRecord } -func New(log *zap.Logger, data ocp_data.Provider) worker.Runtime { +func New(log *zap.Logger, data ocp_data.Provider, exchangeRateStore exchange.Store) worker.Runtime { return &exchangeRateRuntime{ - log: log, - data: data, + log: log, + data: data, + exchangeRateStore: exchangeRateStore, } } @@ -77,7 +80,7 @@ func (p *exchangeRateRuntime) GetCurrentExchangeRates(ctx context.Context) error lastRates := p.lastRates if lastRates == nil { - lastRates, _ = p.data.GetAllExchangeRates(ctx, time.Now()) + lastRates, _ = p.exchangeRateStore.GetAllExchangeRates(ctx, time.Now()) } if lastRates == nil { @@ -110,7 +113,7 @@ func (p *exchangeRateRuntime) GetCurrentExchangeRates(ctx context.Context) error delete(data.Rates, string(currency_lib.ZMK)) delete(data.Rates, string(currency_lib.ZWL)) - if err = p.data.ImportExchangeRates(ctx, data); err != nil { + if err = p.exchangeRateStore.PutExchangeRates(ctx, data); err != nil { return errors.Wrap(err, "failed to store rate data") } diff --git a/ocp/worker/currency/holder/runtime.go b/ocp/worker/currency/holder/runtime.go index 625f4dc..bf6f84b 100644 --- a/ocp/worker/currency/holder/runtime.go +++ b/ocp/worker/currency/holder/runtime.go @@ -15,27 +15,29 @@ import ( ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/account" "github.com/code-payments/ocp-server/ocp/data/currency" + currency_holder "github.com/code-payments/ocp-server/ocp/data/currency/holder" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/ocp/worker" "github.com/code-payments/ocp-server/solana/currencycreator" ) -const ( - historicalUpdateTimeInterval = time.Hour -) - var ( minHoldingValue = common.ToCoreMintQuarks(10) // $10 ) type holderRuntime struct { - log *zap.Logger - data ocp_data.Provider + log *zap.Logger + data ocp_data.Provider + reserveStore reserve.Store + holderStore currency_holder.Store } -func New(log *zap.Logger, data ocp_data.Provider) worker.Runtime { +func New(log *zap.Logger, data ocp_data.Provider, reserveStore reserve.Store, holderStore currency_holder.Store) worker.Runtime { return &holderRuntime{ - log: log, - data: data, + log: log, + data: data, + reserveStore: reserveStore, + holderStore: holderStore, } } @@ -64,7 +66,7 @@ func (p *holderRuntime) Start(runtimeCtx context.Context, interval time.Duration } func (p *holderRuntime) UpdateAllLaunchpadCurrencyHolderCounts(ctx context.Context) { - liveReserveRecordsByMint, err := p.data.GetAllLiveCurrencyReserves(ctx) + liveReserveRecordsByMint, err := p.reserveStore.GetAllLiveReserves(ctx) if err != nil { p.log.With(zap.Error(err)).Warn("failed getting all available currencies") return @@ -81,7 +83,7 @@ func (p *holderRuntime) UpdateAllLaunchpadCurrencyHolderCounts(ctx context.Conte now := time.Now() - err = p.data.PutLiveCurrencyHolderCount(ctx, ¤cy.HolderCountRecord{ + err = p.holderStore.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ Mint: mint, HolderCount: holderCount, Time: now, @@ -91,29 +93,15 @@ func (p *holderRuntime) UpdateAllLaunchpadCurrencyHolderCounts(ctx context.Conte continue } - var shouldCreateHistoricalRecord bool - historicalRecord, err := p.data.GetCurrencyHolderCountAtTime(ctx, mint, now) - switch err { - case nil: - shouldCreateHistoricalRecord = time.Since(historicalRecord.Time) >= historicalUpdateTimeInterval - case currency.ErrNotFound: - shouldCreateHistoricalRecord = true - default: - log.With(zap.Error(err)).Warn("failed getting historical record") + err = p.holderStore.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{ + Mint: mint, + HolderCount: holderCount, + Time: now, + }) + if err != nil { + log.With(zap.Error(err)).Warn("failed creating historical holder count") continue } - - if shouldCreateHistoricalRecord { - err = p.data.PutHistoricalCurrencyHolderCount(ctx, ¤cy.HolderCountRecord{ - Mint: mint, - HolderCount: holderCount, - Time: now, - }) - if err != nil { - log.With(zap.Error(err)).Warn("failed creating historical holder count") - continue - } - } } } diff --git a/ocp/worker/currency/launcher/runtime.go b/ocp/worker/currency/launcher/runtime.go index 100994d..893d0ac 100644 --- a/ocp/worker/currency/launcher/runtime.go +++ b/ocp/worker/currency/launcher/runtime.go @@ -11,21 +11,27 @@ import ( "github.com/code-payments/ocp-server/ocp/common" ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/ocp/worker" ) type runtime struct { - log *zap.Logger - conf *conf - data ocp_data.Provider - subsidizer *common.Account + log *zap.Logger + conf *conf + data ocp_data.Provider + reserveStore reserve.Store + holderStore holder.Store + subsidizer *common.Account } -func New(log *zap.Logger, data ocp_data.Provider, configProvider ConfigProvider) (worker.Runtime, error) { +func New(log *zap.Logger, data ocp_data.Provider, reserveStore reserve.Store, holderStore holder.Store, configProvider ConfigProvider) (worker.Runtime, error) { p := &runtime{ - log: log, - conf: configProvider(), - data: data, + log: log, + conf: configProvider(), + data: data, + reserveStore: reserveStore, + holderStore: holderStore, } err := p.loadSubsidizer() diff --git a/ocp/worker/currency/launcher/util.go b/ocp/worker/currency/launcher/util.go index 3c13bf8..6829d1e 100644 --- a/ocp/worker/currency/launcher/util.go +++ b/ocp/worker/currency/launcher/util.go @@ -126,12 +126,12 @@ func (p *runtime) markCurrencyMetadataAbandoned(ctx context.Context, record *cur func (p *runtime) putInitialReserveState(ctx context.Context, record *currency.MetadataRecord) error { // Note: The live reserve state is initialized by the swap worker on initial purchase - err := p.data.PutHistoricalCurrencyReserve(ctx, ¤cy.ReserveRecord{ + err := p.reserveStore.PutHistoricalReserve(ctx, ¤cy.ReserveRecord{ Mint: record.Mint, SupplyFromBonding: 0, Time: record.CreatedAt, }) - if err != nil { + if err != nil && err != currency.ErrExists { return errors.Wrap(err, "error putting initial historical reserve state") } @@ -139,21 +139,21 @@ func (p *runtime) putInitialReserveState(ctx context.Context, record *currency.M } func (p *runtime) putInitialHolderCount(ctx context.Context, record *currency.MetadataRecord) error { - err := p.data.PutLiveCurrencyHolderCount(ctx, ¤cy.HolderCountRecord{ + err := p.holderStore.PutLiveHolderCount(ctx, ¤cy.HolderCountRecord{ Mint: record.Mint, HolderCount: 1, Time: time.Now(), }) - if err != nil { + if err != nil && err != currency.ErrStaleHolderState { return errors.Wrap(err, "error putting initial live holder count") } - err = p.data.PutHistoricalCurrencyHolderCount(ctx, ¤cy.HolderCountRecord{ + err = p.holderStore.PutHistoricalHolderCount(ctx, ¤cy.HolderCountRecord{ Mint: record.Mint, HolderCount: 0, Time: record.CreatedAt, }) - if err != nil { + if err != nil && err != currency.ErrExists { return errors.Wrap(err, "error putting initial historical holder count") } diff --git a/ocp/worker/currency/reserve/runtime.go b/ocp/worker/currency/reserve/runtime.go index 7b4ef82..f29c109 100644 --- a/ocp/worker/currency/reserve/runtime.go +++ b/ocp/worker/currency/reserve/runtime.go @@ -11,20 +11,23 @@ import ( currency_util "github.com/code-payments/ocp-server/ocp/currency" ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/currency" + currency_reserve "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/ocp/worker" ) type reserveRuntime struct { - log *zap.Logger - conf *conf - data ocp_data.Provider + log *zap.Logger + conf *conf + data ocp_data.Provider + reserveStore currency_reserve.Store } -func New(log *zap.Logger, data ocp_data.Provider, configProvider ConfigProvider) worker.Runtime { +func New(log *zap.Logger, data ocp_data.Provider, reserveStore currency_reserve.Store, configProvider ConfigProvider) worker.Runtime { return &reserveRuntime{ - log: log, - conf: configProvider(), - data: data, + log: log, + conf: configProvider(), + data: data, + reserveStore: reserveStore, } } @@ -55,7 +58,7 @@ func (p *reserveRuntime) Start(runtimeCtx context.Context, interval time.Duratio func (p *reserveRuntime) UpdateAllLaunchpadCurrencyReserves(ctx context.Context) { staleThreshold := p.conf.staleThreshold.Get(ctx) - liveReserveStatesByMint, err := p.data.GetAllLiveCurrencyReserves(ctx) + liveReserveStatesByMint, err := p.reserveStore.GetAllLiveReserves(ctx) if err != nil { p.log.With(zap.Error(err)).Warn("failed getting all live reserve states") return @@ -72,7 +75,7 @@ func (p *reserveRuntime) UpdateAllLaunchpadCurrencyReserves(ctx context.Context) } } - err = p.data.PutHistoricalCurrencyReserve(ctx, ¤cy.ReserveRecord{ + err = p.reserveStore.PutHistoricalReserve(ctx, ¤cy.ReserveRecord{ Mint: mint, SupplyFromBonding: liveReserveRecord.SupplyFromBonding, Time: now, @@ -105,7 +108,7 @@ func (p *reserveRuntime) refreshLiveReserveState(ctx context.Context, log *zap.L Slot: slot, Time: time.Now(), } - err = p.data.PutLiveCurrencyReserve(ctx, record) + err = p.reserveStore.PutLiveReserve(ctx, record) if err != nil && err != currency.ErrStaleReserveState { log.With(zap.Error(err)).Warn("failed to update live reserve state") return nil, err diff --git a/ocp/worker/geyser/backup.go b/ocp/worker/geyser/backup.go index 65a6a85..61a9b10 100644 --- a/ocp/worker/geyser/backup.go +++ b/ocp/worker/geyser/backup.go @@ -166,7 +166,7 @@ func (p *runtime) backupExternalDepositWorker(runtimeCtx context.Context, interv zap.String("mint", mintAccount.PublicKey().ToBase58()), ) - err = fixMissingExternalDeposits(tracedCtx, p.data, p.integration, authorityAccount, mintAccount) + err = fixMissingExternalDeposits(tracedCtx, p.data, p.exchangeRateStore, p.reserveStore, p.integration, authorityAccount, mintAccount) if err != nil { log.With(zap.Error(err)).Warn("failed to fix missing external deposits") } diff --git a/ocp/worker/geyser/currency_reserve.go b/ocp/worker/geyser/currency_reserve.go index 6cbcf54..ede120c 100644 --- a/ocp/worker/geyser/currency_reserve.go +++ b/ocp/worker/geyser/currency_reserve.go @@ -8,6 +8,7 @@ import ( "github.com/code-payments/ocp-server/ocp/common" ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/solana/currencycreator" ) @@ -15,7 +16,7 @@ var ( liquidityPoolVaultCache = cache.NewCache(10_000) ) -func processPotentialCirculatingSupplyUpdate(ctx context.Context, data ocp_data.Provider, tokenAccount, mintAccount *common.Account, amount uint64, slot uint64) error { +func processPotentialCirculatingSupplyUpdate(ctx context.Context, data ocp_data.Provider, reserveStore reserve.Store, tokenAccount, mintAccount *common.Account, amount uint64, slot uint64) error { cachedVault, ok := liquidityPoolVaultCache.Retrieve(mintAccount.PublicKey().ToBase58()) if !ok { metadataRecord, err := data.GetCurrencyMetadata(ctx, mintAccount.PublicKey().ToBase58()) @@ -33,11 +34,11 @@ func processPotentialCirculatingSupplyUpdate(ctx context.Context, data ocp_data. return nil } - return updateLiveReserveState(ctx, data, mintAccount.PublicKey().ToBase58(), amount, slot) + return updateLiveReserveState(ctx, reserveStore, mintAccount.PublicKey().ToBase58(), amount, slot) } -func updateLiveReserveState(ctx context.Context, data ocp_data.Provider, mint string, vaultAmount uint64, slot uint64) error { - err := data.PutLiveCurrencyReserve(ctx, ¤cy.ReserveRecord{ +func updateLiveReserveState(ctx context.Context, reserveStore reserve.Store, mint string, vaultAmount uint64, slot uint64) error { + err := reserveStore.PutLiveReserve(ctx, ¤cy.ReserveRecord{ Mint: mint, SupplyFromBonding: currencycreator.DefaultMintMaxQuarkSupply - vaultAmount, Slot: slot, diff --git a/ocp/worker/geyser/external_deposit.go b/ocp/worker/geyser/external_deposit.go index 7df5820..073c31c 100644 --- a/ocp/worker/geyser/external_deposit.go +++ b/ocp/worker/geyser/external_deposit.go @@ -20,6 +20,8 @@ import ( "github.com/code-payments/ocp-server/ocp/currency" currency_util "github.com/code-payments/ocp-server/ocp/currency" ocp_data "github.com/code-payments/ocp-server/ocp/data" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/ocp/data/deposit" "github.com/code-payments/ocp-server/ocp/data/intent" "github.com/code-payments/ocp-server/ocp/data/swap" @@ -44,7 +46,7 @@ var ( syncedDepositCache = cache.NewCache(1_000_000) ) -func fixMissingExternalDeposits(ctx context.Context, data ocp_data.Provider, integration integration.Geyser, userAuthority, mint *common.Account) error { +func fixMissingExternalDeposits(ctx context.Context, data ocp_data.Provider, exchangeRateStore exchange.Store, reserveStore reserve.Store, integration integration.Geyser, userAuthority, mint *common.Account) error { err := maybeInitiateExternalDepositIntoVm(ctx, data, userAuthority, mint) if err != nil { return errors.Wrap(err, "error depositing into the vm") @@ -57,7 +59,7 @@ func fixMissingExternalDeposits(ctx context.Context, data ocp_data.Provider, int var anyError error for _, signature := range signatures { - err := processPotentialExternalDepositIntoVm(ctx, data, integration, signature, userAuthority, mint) + err := processPotentialExternalDepositIntoVm(ctx, data, exchangeRateStore, reserveStore, integration, signature, userAuthority, mint) if err != nil { anyError = errors.Wrap(err, "error processing signature for external deposit into vm") } @@ -229,7 +231,7 @@ func findPotentialExternalDepositsIntoVm(ctx context.Context, data ocp_data.Prov } } -func processPotentialExternalDepositIntoVm(ctx context.Context, data ocp_data.Provider, integration integration.Geyser, signature string, userAuthority, mint *common.Account) error { +func processPotentialExternalDepositIntoVm(ctx context.Context, data ocp_data.Provider, exchangeRateStore exchange.Store, reserveStore reserve.Store, integration integration.Geyser, signature string, userAuthority, mint *common.Account) error { vmConfig, err := common.GetVmConfigForMint(ctx, data, mint) if err != nil { return err @@ -319,13 +321,13 @@ func processPotentialExternalDepositIntoVm(ctx context.Context, data ocp_data.Pr return errors.Wrap(err, "invalid owner account") } - isInitialPurchase, usdMarketValue, err := isInitialCurrencyCreatorDeposit(ctx, data, ownerAccount, mint, userVirtualTimelockVaultAccount, uint64(deltaQuarksIntoOmnibus)) + isInitialPurchase, usdMarketValue, err := isInitialCurrencyCreatorDeposit(ctx, data, exchangeRateStore, reserveStore, ownerAccount, mint, userVirtualTimelockVaultAccount, uint64(deltaQuarksIntoOmnibus)) if err != nil { return errors.Wrap(err, "error checking for initial currency creator deposit") } if !isInitialPurchase { - usdMarketValue, err = currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, data, mint, uint64(deltaQuarksIntoOmnibus), time.Now()) + usdMarketValue, err = currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, data, exchangeRateStore, reserveStore, mint, uint64(deltaQuarksIntoOmnibus), time.Now()) if err != nil { return errors.Wrap(err, "error calculating usd market value") } @@ -486,7 +488,7 @@ func getSyncedVmDepositCacheKey(signature string, vmDepositAta *common.Account) // 2. The depositor is the currency creator // 3. This is their first deposit for this currency // 4. The deposit amount matches the expected output of the first swap within 1 quark error tolerance -func isInitialCurrencyCreatorDeposit(ctx context.Context, data ocp_data.Provider, owner, mint, destination *common.Account, depositQuarks uint64) (bool, float64, error) { +func isInitialCurrencyCreatorDeposit(ctx context.Context, data ocp_data.Provider, exchangeRateStore exchange.Store, reserveStore reserve.Store, owner, mint, destination *common.Account, depositQuarks uint64) (bool, float64, error) { if common.IsCoreMint(mint) { return false, 0, nil } @@ -520,7 +522,7 @@ func isInitialCurrencyCreatorDeposit(ctx context.Context, data ocp_data.Provider initialSwap := swapRecords[0] - usdMarketValue, err := currency.CalculateUsdMarketValueFromTokenAmount(ctx, data, common.CoreMintAccount, initialSwap.SwapAmount, initialSwap.CreatedAt) + usdMarketValue, err := currency.CalculateUsdMarketValueFromTokenAmount(ctx, data, exchangeRateStore, reserveStore, common.CoreMintAccount, initialSwap.SwapAmount, initialSwap.CreatedAt) if err != nil { return false, 0, errors.Wrap(err, "error calculating usd market value") } diff --git a/ocp/worker/geyser/handler.go b/ocp/worker/geyser/handler.go index bb6e9b9..d54fd54 100644 --- a/ocp/worker/geyser/handler.go +++ b/ocp/worker/geyser/handler.go @@ -12,6 +12,8 @@ import ( "github.com/code-payments/ocp-server/ocp/common" ocp_data "github.com/code-payments/ocp-server/ocp/data" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/solana/token" ) @@ -28,16 +30,20 @@ type ProgramAccountUpdateHandler interface { } type TokenProgramAccountHandler struct { - conf *conf - data ocp_data.Provider - integration integration.Geyser + conf *conf + data ocp_data.Provider + exchangeRateStore exchange.Store + reserveStore reserve.Store + integration integration.Geyser } -func NewTokenProgramAccountHandler(conf *conf, data ocp_data.Provider, integration integration.Geyser) ProgramAccountUpdateHandler { +func NewTokenProgramAccountHandler(conf *conf, data ocp_data.Provider, exchangeRateStore exchange.Store, reserveStore reserve.Store, integration integration.Geyser) ProgramAccountUpdateHandler { return &TokenProgramAccountHandler{ - conf: conf, - data: data, - integration: integration, + conf: conf, + data: data, + exchangeRateStore: exchangeRateStore, + reserveStore: reserveStore, + integration: integration, } } @@ -89,7 +95,7 @@ func (h *TokenProgramAccountHandler) Handle(ctx context.Context, update *geyserp return nil } - err = processPotentialCirculatingSupplyUpdate(ctx, h.data, tokenAccount, mintAccount, unmarshalled.Amount, update.Slot) + err = processPotentialCirculatingSupplyUpdate(ctx, h.data, h.reserveStore, tokenAccount, mintAccount, unmarshalled.Amount, update.Slot) if err != nil { return errors.Wrap(err, "error processing potential currency circulating supply update") } @@ -106,7 +112,7 @@ func (h *TokenProgramAccountHandler) Handle(ctx context.Context, update *geyserp return nil } - err = processPotentialExternalDepositIntoVm(ctx, h.data, h.integration, signature, userAuthorityAccount, mintAccount) + err = processPotentialExternalDepositIntoVm(ctx, h.data, h.exchangeRateStore, h.reserveStore, h.integration, signature, userAuthorityAccount, mintAccount) if err != nil { return errors.Wrap(err, "error processing signature for external deposit into vm") } @@ -126,8 +132,8 @@ func (h *TokenProgramAccountHandler) Handle(ctx context.Context, update *geyserp return nil } -func initializeProgramAccountUpdateHandlers(conf *conf, data ocp_data.Provider, integration integration.Geyser) map[string]ProgramAccountUpdateHandler { +func initializeProgramAccountUpdateHandlers(conf *conf, data ocp_data.Provider, exchangeRateStore exchange.Store, reserveStore reserve.Store, integration integration.Geyser) map[string]ProgramAccountUpdateHandler { return map[string]ProgramAccountUpdateHandler{ - base58.Encode(token.ProgramKey): NewTokenProgramAccountHandler(conf, data, integration), + base58.Encode(token.ProgramKey): NewTokenProgramAccountHandler(conf, data, exchangeRateStore, reserveStore, integration), } } diff --git a/ocp/worker/geyser/runtime.go b/ocp/worker/geyser/runtime.go index f7b7d4e..c0604ca 100644 --- a/ocp/worker/geyser/runtime.go +++ b/ocp/worker/geyser/runtime.go @@ -12,6 +12,8 @@ import ( geyserpb "github.com/code-payments/ocp-server/ocp/worker/geyser/api/gen" ocp_data "github.com/code-payments/ocp-server/ocp/data" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" ) type eventWorkerMetrics struct { @@ -21,9 +23,11 @@ type eventWorkerMetrics struct { // todo: we can consolidate the various subscription streams into one type runtime struct { - log *zap.Logger - data ocp_data.Provider - conf *conf + log *zap.Logger + data ocp_data.Provider + exchangeRateStore exchange.Store + reserveStore reserve.Store + conf *conf integration integration.Geyser @@ -45,15 +49,17 @@ type runtime struct { backupExternalDepositWorkerStatus bool } -func New(log *zap.Logger, data ocp_data.Provider, integration integration.Geyser, configProvider ConfigProvider) worker.Runtime { +func New(log *zap.Logger, data ocp_data.Provider, exchangeRateStore exchange.Store, reserveStore reserve.Store, integration integration.Geyser, configProvider ConfigProvider) worker.Runtime { conf := configProvider() return &runtime{ log: log, data: data, + exchangeRateStore: exchangeRateStore, + reserveStore: reserveStore, conf: configProvider(), integration: integration, programUpdatesChan: make(chan *geyserpb.SubscribeUpdateAccount, conf.programUpdateQueueSize.Get(context.Background())), - programUpdateHandlers: initializeProgramAccountUpdateHandlers(conf, data, integration), + programUpdateHandlers: initializeProgramAccountUpdateHandlers(conf, data, exchangeRateStore, reserveStore, integration), programUpdateWorkerMetrics: make(map[int]*eventWorkerMetrics), } } diff --git a/ocp/worker/swap/runtime.go b/ocp/worker/swap/runtime.go index 3e2fd91..344cb65 100644 --- a/ocp/worker/swap/runtime.go +++ b/ocp/worker/swap/runtime.go @@ -11,6 +11,8 @@ import ( "github.com/code-payments/ocp-server/coinbase" ocp_data "github.com/code-payments/ocp-server/ocp/data" + "github.com/code-payments/ocp-server/ocp/data/currency/exchange" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/ocp/data/nonce" "github.com/code-payments/ocp-server/ocp/data/swap" "github.com/code-payments/ocp-server/ocp/integration" @@ -19,18 +21,22 @@ import ( ) type runtime struct { - log *zap.Logger - conf *conf - data ocp_data.Provider - vmIndexerClient indexerpb.IndexerClient - integration integration.Swap - solanaNoncePool *transaction.LocalNoncePool - coinbaseClient *coinbase.Client + log *zap.Logger + conf *conf + data ocp_data.Provider + exchangeRateStore exchange.Store + reserveStore reserve.Store + vmIndexerClient indexerpb.IndexerClient + integration integration.Swap + solanaNoncePool *transaction.LocalNoncePool + coinbaseClient *coinbase.Client } func New( log *zap.Logger, data ocp_data.Provider, + exchangeRateStore exchange.Store, + reserveStore reserve.Store, vmIndexerClient indexerpb.IndexerClient, integration integration.Swap, solanaNoncePool *transaction.LocalNoncePool, @@ -42,13 +48,15 @@ func New( } return &runtime{ - log: log, - conf: configProvider(), - data: data, - vmIndexerClient: vmIndexerClient, - integration: integration, - solanaNoncePool: solanaNoncePool, - coinbaseClient: coinbaseClient, + log: log, + conf: configProvider(), + data: data, + exchangeRateStore: exchangeRateStore, + reserveStore: reserveStore, + vmIndexerClient: vmIndexerClient, + integration: integration, + solanaNoncePool: solanaNoncePool, + coinbaseClient: coinbaseClient, }, nil } diff --git a/ocp/worker/swap/util.go b/ocp/worker/swap/util.go index af7dd40..0589548 100644 --- a/ocp/worker/swap/util.go +++ b/ocp/worker/swap/util.go @@ -428,7 +428,7 @@ func (p *runtime) buildRefundRecordsForCancelledSwap(ctx context.Context, swapRe return nil, nil, errors.New("unexpected source mint") } exchangeCurrency = currency_lib.USD - usdMarketValue, err = currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, p.data, common.CoreMintAccount, quantity, time.Now()) + usdMarketValue, err = currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, p.data, p.exchangeRateStore, p.reserveStore, common.CoreMintAccount, quantity, time.Now()) if err != nil { return nil, nil, err } @@ -600,7 +600,7 @@ func (p *runtime) maybeUpdateBalancesForFinalizedReserveSwap(ctx context.Context usdMarketValueWithoutFees = fundingIntentRecord.SendPublicPaymentMetadata.UsdMarketValue if common.IsCoreMint(toMint) { - usdMarketValue, err := currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, p.data, common.CoreMintAccount, uint64(deltaQuarksIntoOmnibus), time.Now()) + usdMarketValue, err := currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, p.data, p.exchangeRateStore, p.reserveStore, common.CoreMintAccount, uint64(deltaQuarksIntoOmnibus), time.Now()) if err != nil { return 0, false, err } @@ -627,7 +627,7 @@ func (p *runtime) maybeUpdateBalancesForFinalizedReserveSwap(ctx context.Context } exchangeCurrency = currency_lib.USD - usdMarketValueWithoutFees, err = currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, p.data, common.CoreMintAccount, swapRecord.SwapAmount, time.Now()) + usdMarketValueWithoutFees, err = currency_util.CalculateUsdMarketValueFromTokenAmount(ctx, p.data, p.exchangeRateStore, p.reserveStore, common.CoreMintAccount, swapRecord.SwapAmount, time.Now()) if err != nil { return 0, false, err } @@ -1126,7 +1126,7 @@ func (p *runtime) updateLiveReserveStateForFinalizedSwap(ctx context.Context, sw continue } - err = p.data.PutLiveCurrencyReserve(ctx, ¤cy.ReserveRecord{ + err = p.reserveStore.PutLiveReserve(ctx, ¤cy.ReserveRecord{ Mint: mint.PublicKey().ToBase58(), SupplyFromBonding: currencycreator.DefaultMintMaxQuarkSupply - postBalance, Slot: tokenBalances.Slot, diff --git a/testutil/currency.go b/testutil/currency.go index afc7d36..a3c8034 100644 --- a/testutil/currency.go +++ b/testutil/currency.go @@ -10,6 +10,8 @@ import ( "github.com/code-payments/ocp-server/ocp/common" ocp_data "github.com/code-payments/ocp-server/ocp/data" "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/holder" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" "github.com/code-payments/ocp-server/ocp/data/vault" vm_metadata "github.com/code-payments/ocp-server/ocp/data/vm/metadata" "github.com/code-payments/ocp-server/solana/currencycreator" @@ -19,7 +21,7 @@ import ( // records needed for a launchpad currency to be fully functional: currency // metadata (Available), VM metadata (Available), authority private key in // vault, and a live reserve record. -func SetupLaunchpadCurrency(t *testing.T, data ocp_data.Provider) *common.Account { +func SetupLaunchpadCurrency(t *testing.T, data ocp_data.Provider, reserveStore reserve.Store, holderStore holder.Store) *common.Account { vmConfig := NewRandomVmConfig(t, false) metadataRecord := ¤cy.MetadataRecord{ @@ -83,14 +85,14 @@ func SetupLaunchpadCurrency(t *testing.T, data ocp_data.Provider) *common.Accoun Slot: 1, Time: time.Now(), } - require.NoError(t, data.PutLiveCurrencyReserve(t.Context(), reserveRecord)) + require.NoError(t, reserveStore.PutLiveReserve(t.Context(), reserveRecord)) holderCountRecord := ¤cy.HolderCountRecord{ Mint: vmConfig.Mint.PublicKey().ToBase58(), HolderCount: 32, Time: time.Now(), } - require.NoError(t, data.PutLiveCurrencyHolderCount(t.Context(), holderCountRecord)) + require.NoError(t, holderStore.PutLiveHolderCount(t.Context(), holderCountRecord)) return vmConfig.Mint } From 4a6e9d579be18cf24dbe4ef90f7a371f38eb4f12 Mon Sep 17 00:00:00 2001 From: jeffyanta Date: Wed, 17 Jun 2026 10:56:11 -0400 Subject: [PATCH 5/5] Add a caching reserve store --- ocp/data/currency/reserve/cache/store.go | 113 ++++++++++++++++++ ocp/data/currency/reserve/cache/store_test.go | 111 +++++++++++++++++ 2 files changed, 224 insertions(+) create mode 100644 ocp/data/currency/reserve/cache/store.go create mode 100644 ocp/data/currency/reserve/cache/store_test.go diff --git a/ocp/data/currency/reserve/cache/store.go b/ocp/data/currency/reserve/cache/store.go new file mode 100644 index 0000000..5b64844 --- /dev/null +++ b/ocp/data/currency/reserve/cache/store.go @@ -0,0 +1,113 @@ +// Package cache provides a reserve.Store decorator that caches point-in-time +// reserve lookups in front of a wrapped store. +// +// Point-in-time reads are keyed by mint and a coarse time bucket that doubles as +// the freshness window. Range and live reads pass straight through. Live writes +// are guarded against the last successfully saved slot per mint: a write whose +// slot is not greater is rejected with currency.ErrStaleReserveState without a +// round-trip to the backing store. Everything else passes straight through. +package cache + +import ( + "context" + "fmt" + "sync" + "time" + + lrucache "github.com/code-payments/ocp-server/cache" + "github.com/code-payments/ocp-server/database/query" + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve" +) + +const ( + // maxCacheBudget bounds the weighted size of the reserve cache before the + // least-recently-used entries are evicted. + maxCacheBudget = 100_000 + + // reserveWeight weights each cached entry against the budget. A cached entry + // is a single per-mint, point-in-time record. + reserveWeight = 1 + + // cacheBucket is the time granularity used to build cache keys. Lookups that + // truncate to the same bucket share a cached result, which doubles as the + // effective freshness window for cached reserves. + cacheBucket = 5 * time.Minute +) + +type store struct { + backing reserve.Store + cache lrucache.Cache + + // liveSlots tracks the slot of the last live reserve successfully saved per + // mint, so stale writes can be rejected without a round-trip to the backing + // store. + liveSlotsMu sync.Mutex + liveSlots map[string]uint64 +} + +// New returns a reserve.Store that caches reads in front of backing. +func New(backing reserve.Store) reserve.Store { + return &store{ + backing: backing, + cache: lrucache.NewCache(maxCacheBudget), + liveSlots: make(map[string]uint64), + } +} + +func (s *store) PutHistoricalReserve(ctx context.Context, record *currency.ReserveRecord) error { + return s.backing.PutHistoricalReserve(ctx, record) +} + +func (s *store) GetReserveAtTime(ctx context.Context, mint string, t time.Time) (*currency.ReserveRecord, error) { + key := fmt.Sprintf("%s:%s", mint, t.Truncate(cacheBucket).Format(time.RFC3339)) + if cached, ok := s.cache.Retrieve(key); ok { + return cached.(*currency.ReserveRecord), nil + } + + record, err := s.backing.GetReserveAtTime(ctx, mint, t) + if err != nil { + return nil, err + } + + s.cache.Insert(key, record, reserveWeight) + + return record, nil +} + +func (s *store) GetReservesInRange(ctx context.Context, mint string, interval query.Interval, start time.Time, end time.Time, ordering query.Ordering) ([]*currency.ReserveRecord, error) { + return s.backing.GetReservesInRange(ctx, mint, interval, start, end, ordering) +} + +func (s *store) PutLiveReserve(ctx context.Context, record *currency.ReserveRecord) error { + s.liveSlotsMu.Lock() + lastSlot, tracked := s.liveSlots[record.Mint] + s.liveSlotsMu.Unlock() + + // Reject stale writes locally. The backing store remains the source of truth + // and performs the same check atomically, so a race here at worst lets a + // stale write through to be rejected there. + if tracked && record.Slot <= lastSlot { + return currency.ErrStaleReserveState + } + + if err := s.backing.PutLiveReserve(ctx, record); err != nil { + return err + } + + s.liveSlotsMu.Lock() + if cur, ok := s.liveSlots[record.Mint]; !ok || record.Slot > cur { + s.liveSlots[record.Mint] = record.Slot + } + s.liveSlotsMu.Unlock() + + return nil +} + +func (s *store) GetLiveReserve(ctx context.Context, mint string) (*currency.ReserveRecord, error) { + return s.backing.GetLiveReserve(ctx, mint) +} + +func (s *store) GetAllLiveReserves(ctx context.Context) (map[string]*currency.ReserveRecord, error) { + return s.backing.GetAllLiveReserves(ctx) +} diff --git a/ocp/data/currency/reserve/cache/store_test.go b/ocp/data/currency/reserve/cache/store_test.go new file mode 100644 index 0000000..f9862ec --- /dev/null +++ b/ocp/data/currency/reserve/cache/store_test.go @@ -0,0 +1,111 @@ +package cache + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/code-payments/ocp-server/ocp/data/currency" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve/memory" + "github.com/code-payments/ocp-server/ocp/data/currency/reserve/tests" +) + +func TestReserve_CachedStore(t *testing.T) { + testStore := New(memory.New()).(*store) + teardown := func() { + testStore.backing = memory.New() + testStore.cache.Clear() + testStore.liveSlots = make(map[string]uint64) + } + tests.RunStoreTests(t, testStore, teardown) +} + +// TestReserve_CachedReadsServedFromCache verifies that once a reserve is read at +// a point in time, a later read in the same time bucket is served from the cache +// even after the backing store no longer holds it, while a read in a different +// bucket falls through to the (now empty) backing store. +func TestReserve_CachedReadsServedFromCache(t *testing.T) { + ctx := context.Background() + mint := "test-mint" + now := time.Date(2021, 01, 29, 13, 0, 5, 0, time.UTC) + + s := New(memory.New()).(*store) + require.NoError(t, s.PutHistoricalReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 1000, + Time: now, + })) + + // Prime the cache with a point-in-time read. + record, err := s.GetReserveAtTime(ctx, mint, now) + require.NoError(t, err) + assert.EqualValues(t, 1000, record.SupplyFromBonding) + + // Drop the backing data. A read in the same bucket is still served. + s.backing = memory.New() + + record, err = s.GetReserveAtTime(ctx, mint, now) + require.NoError(t, err) + assert.EqualValues(t, 1000, record.SupplyFromBonding) + + // A read truncating to a different bucket misses the cache and falls through + // to the empty backing store. + otherBucket := now.Add(cacheBucket) + _, err = s.GetReserveAtTime(ctx, mint, otherBucket) + assert.Equal(t, currency.ErrNotFound, err) +} + +// TestReserve_StaleLiveSlotRejectedLocally verifies that once a live reserve is +// saved, a write with an older or equal slot is rejected with a stale error +// without reaching the backing store, while a newer slot passes through. +func TestReserve_StaleLiveSlotRejectedLocally(t *testing.T) { + ctx := context.Background() + mint := "test-mint" + + s := New(memory.New()).(*store) + require.NoError(t, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 1000, + Slot: 200, + Time: time.Now(), + })) + + // Swap in a fresh, empty backing. If the stale check were delegated to the + // backing it would accept the older slot; the cache must reject it itself. + s.backing = memory.New() + + // An older slot is rejected locally and never reaches the backing. + err := s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 999, + Slot: 100, + Time: time.Now(), + }) + assert.Equal(t, currency.ErrStaleReserveState, err) + + _, err = s.backing.GetLiveReserve(ctx, mint) + assert.Equal(t, currency.ErrNotFound, err) + + // An equal slot is also rejected. + err = s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 999, + Slot: 200, + Time: time.Now(), + }) + assert.Equal(t, currency.ErrStaleReserveState, err) + + // A newer slot passes through to the backing and advances the tracked slot. + require.NoError(t, s.PutLiveReserve(ctx, ¤cy.ReserveRecord{ + Mint: mint, + SupplyFromBonding: 1500, + Slot: 300, + Time: time.Now(), + })) + rec, err := s.backing.GetLiveReserve(ctx, mint) + require.NoError(t, err) + assert.EqualValues(t, 300, rec.Slot) +}