support downsampling in search, aggregation, histogram

api/dashboards/v1/dashboards.proto

@@ -82,7 +82,7 @@ message DeleteRequest {
 
 message DeleteResponse {}
 
-message SearchRequest{
+message SearchRequest {
   message Filter {
     optional string owner_name = 1;
   }
@@ -93,12 +93,12 @@ message SearchRequest{
   optional Filter filter = 4;
 }
 
-message SearchResponse{
+message SearchResponse {
   message Dashboard {
     string uuid = 1;
     string name = 2;
     string owner_name = 3;
   }
 
   repeated Dashboard dashboards = 1;
-}
+}

api/massexport/v1/massexport.proto

@@ -2,8 +2,8 @@ syntax = "proto3";
 
 package massexport.v1;
 
-import "google/protobuf/timestamp.proto";
 import "google/protobuf/duration.proto";
+import "google/protobuf/timestamp.proto";
 
 option go_package = "github.com/ozontech/seq-ui/pkg/massexport/v1;massexport";
 

api/seqapi/v1/seq_api.proto

@@ -128,6 +128,7 @@ message SearchRequest {
   repeated AggregationQuery aggregations = 8;
   Order order = 9;
   string offset_id = 10;
+  uint32 downsample = 11;
 }
 
 message SearchResponse {
@@ -152,6 +153,7 @@ message GetHistogramRequest {
   string interval = 2;
   google.protobuf.Timestamp from = 3;
   google.protobuf.Timestamp to = 4;
+  uint32 downsample = 5;
 }
 
 message GetHistogramResponse {
@@ -166,6 +168,7 @@ message GetAggregationRequest {
   google.protobuf.Timestamp to = 3;
   string agg_field = 4 [deprecated = true];
   repeated AggregationQuery aggregations = 5;
+  uint32 downsample = 6;
 }
 
 message GetAggregationResponse {
@@ -207,6 +210,7 @@ message ExportRequest {
   int32 offset = 5;
   ExportFormat format = 6;
   repeated string fields = 7;
+  uint32 downsample = 8;
 }
 
 message GetLimitsRequest {}
@@ -257,6 +261,7 @@ message StartAsyncSearchRequest {
   bool with_docs = 7;
   int32 size = 8;
   string meta = 9;
+  uint32 downsample = 10;
 }
 
 message StartAsyncSearchResponse {

api/userprofile/v1/userprofile.proto

@@ -36,7 +36,6 @@ message UpdateUserProfileRequest {
 
 message UpdateUserProfileResponse {}
 
-
 message GetFavoriteQueriesRequest {}
 
 message GetFavoriteQueriesResponse {
@@ -66,7 +65,6 @@ message DeleteFavoriteQueryRequest {
 
 message DeleteFavoriteQueryResponse {}
 
-
 message GetDashboardsRequest {}
 
 message GetDashboardsResponse {
@@ -109,4 +107,4 @@ message DeleteDashboardRequest {
   string uuid = 1;
 }
 
-message DeleteDashboardResponse {}
+message DeleteDashboardResponse {}

docs/en/03-seq-api.md

@@ -72,6 +72,8 @@ Returns a list of events that satisfy the [search query](https://github.com/ozon
 - `offset` (*int*, *optional*): Search offset.
 - `withTotal` (*bool*, *optional*): If set, returns the total number of events found.
 - `order` (*enum*, *optional*): Search order. One of `"desc"|"asc"` (`"desc"` by default).
+- `offset_id` (*string*, *optional*): Pagination cursor. The response contains events that come after specified identifier. Mutually exclusive with offset: only one of these fields can be set per request.
+- `downsample` (*uint32*, *optional*): Random sampling coefficient. Higher values yield a smaller fraction of returned events.
 
 #### Request
 
@@ -178,6 +180,7 @@ Downloads events to file in the specified format.
 - `limit` (*int*, *required*): Export limit.
 - `offset` (*int*, *optional*): Export offset.
 - `fields` (*[]string*, *optional*): List of fields to export (only for `format:csv`, must be non-empty in this case).
+- `downsample` (*uint32*, *optional*): Random sampling coefficient. Higher values yield a smaller fraction of returned events.
 
 #### Request
 
@@ -252,6 +255,7 @@ Calculates aggregations based on events that satisfy the search query.
   - `field` (*string*, *required*): Aggregation calculation field.
   - `group_by` (*string*, *optional*): Field for grouping the aggregation results.
   - `quantiles` (*[]int*, *optional*): List of quantiles (only for `agg_func:quantile`, must be non-empty in this case).
+- `downsample` (*uint32*, *optional*): Random sampling coefficient. Higher values yield a smaller fraction of returned events.
 
 #### Request
 
@@ -414,6 +418,7 @@ Calculates histogram based on events that satisfy the search query.
 - `from` (*string*, *required*): Timestamp of the start of search in `date-time` format.
 - `to` (*string*, *required*): Timestamp of the end of search in `date-time` format.
 - `interval` (*string*, *required*): Interval for calculating the bucket in `duration` format.
+- `downsample` (*uint32*, *optional*): Random sampling coefficient. Higher values yield a smaller fraction of returned events.
 
 #### Request
 

docs/ru/03-seq-api.md

@@ -72,6 +72,9 @@ curl -X GET \
 - `offset` (*int*, *optional*): Смещение поиска.
 - `withTotal` (*bool*, *optional*): Если задано, то возвращает общее количество найденных событий.
 - `order` (*enum*, *optional*): Порядок поиска. Одно из `"desc"|"asc"` (по умолчанию `"desc"`).
+- `offset_id` (*string*, *optional*): Курсор пагинации. Ответ запроса содержит события, следующие за указанным идентификатором. 
+Альтернатива `offset`: в одном запросе можно задать только одно из этих полей.
+- `downsample` (*uint32*, *optional*): коэффициент случайного семплирования результата. Чем больше значение, тем меньше доля возвращаемых событий.
 
 #### Запрос
 
@@ -178,6 +181,7 @@ curl -X GET \
 - `limit` (*int*, *required*): Ограничение поиска.
 - `offset` (*int*, *optional*): Смещение поиска.
 - `fields` (*[]string*, *optional*): Список полей для экспорта (только для `format:csv`, в этом случае список должен быть непустым).
+- `downsample` (*uint32*, *optional*): коэффициент случайного семплирования результата. Чем больше значение, тем меньше доля возвращаемых событий.
 
 #### Запрос
 
@@ -252,6 +256,7 @@ level,message
   - `field` (*string*, *required*): Поле для расчета агрегации.
   - `group_by` (*string*, *optional*): Поле для группировки результатов агрегирования.
   - `quantiles` (*[]int*, *optional*): Список квантилей (только для `agg_func:quantile`, в этом случае список должен быть непустым).
+- `downsample` (*uint32*, *optional*): коэффициент случайного семплирования результата. Чем больше значение, тем меньше доля возвращаемых событий.
 
 #### Запрос
 
@@ -414,6 +419,7 @@ curl -X POST \
 - `from` (*string*, *required*): Временная метка начала поиска в `date-time` формате.
 - `to` (*string*, *required*): Временная метка окончания поиска в `date-time` формате.
 - `interval` (*string*, *required*): Интервал гистограммы в `duration` формате.
+- `downsample` (*uint32*, *optional*): коэффициент случайного семплирования результата. Чем больше значение, тем меньше доля возвращаемых событий.
 
 #### Запрос
 

internal/api/seqapi/v1/grpc/aggregation.go

@@ -42,6 +42,10 @@ func (a *API) GetAggregation(ctx context.Context, req *seqapi.GetAggregationRequ
 			Key:   "aggregations",
 			Value: attribute.StringValue(string(aggregations)),
 		},
+		{
+			Key:   "downsample",
+			Value: attribute.IntValue(int(req.GetDownsample())),
+		},
 	}
 
 	if env != "" {

internal/api/seqapi/v1/grpc/histogram.go

@@ -34,6 +34,10 @@ func (a *API) GetHistogram(ctx context.Context, req *seqapi.GetHistogramRequest)
 			Key:   "interval",
 			Value: attribute.StringValue(req.GetInterval()),
 		},
+		{
+			Key:   "downsample",
+			Value: attribute.IntValue(int(req.GetDownsample())),
+		},
 	}
 
 	if env != "" {

internal/api/seqapi/v1/grpc/search.go

@@ -53,6 +53,10 @@ func (a *API) Search(ctx context.Context, req *seqapi.SearchRequest) (*seqapi.Se
 			Key:   "offset_id",
 			Value: attribute.StringValue(req.GetOffsetId()),
 		},
+		{
+			Key:   "downsample",
+			Value: attribute.IntValue(int(req.GetDownsample())),
+		},
 	}
 
 	if env != "" {

internal/api/seqapi/v1/grpc/start_async_search.go

@@ -50,6 +50,10 @@ func (a *API) StartAsyncSearch(
 			Key:   "size",
 			Value: attribute.Int64Value(int64(req.GetSize())),
 		},
+		{
+			Key:   "downsample",
+			Value: attribute.IntValue(int(req.GetDownsample())),
+		},
 	}
 	if req.Hist != nil && req.Hist.Interval != "" {
 		spanAttributes = append(spanAttributes, attribute.KeyValue{

internal/api/seqapi/v1/http/aggregation.go

@@ -63,6 +63,10 @@ func (a *API) serveGetAggregation(w http.ResponseWriter, r *http.Request) {
 			Key:   "aggregations",
 			Value: attribute.StringValue(string(aggsRaw)),
 		},
+		{
+			Key:   "downsample",
+			Value: attribute.IntValue(int(httpReq.Downsample)),
+		},
 	}
 
 	if env != "" {
@@ -202,6 +206,7 @@ type getAggregationRequest struct {
 	To           time.Time          `json:"to" format:"date-time"`
 	AggField     string             `json:"aggField"`
 	Aggregations aggregationQueries `json:"aggregations"`
+	Downsample   uint32             `json:"downsample"`
 } //	@name	seqapi.v1.GetAggregationRequest
 
 func (r getAggregationRequest) toProto() *seqapi.GetAggregationRequest {
@@ -211,6 +216,7 @@ func (r getAggregationRequest) toProto() *seqapi.GetAggregationRequest {
 		To:           timestamppb.New(r.To),
 		AggField:     r.AggField,
 		Aggregations: r.Aggregations.toProto(),
+		Downsample:   r.Downsample,
 	}
 }
 

internal/api/seqapi/v1/http/export.go

@@ -79,6 +79,10 @@ func (a *API) serveExport(w http.ResponseWriter, r *http.Request) {
 			Key:   "fields",
 			Value: attribute.StringSliceValue(httpReq.Fields),
 		},
+		{
+			Key:   "downsample",
+			Value: attribute.IntValue(int(httpReq.Downsample)),
+		},
 	}
 
 	if env != "" {
@@ -144,24 +148,26 @@ func (f exportFormat) toProto() seqapi.ExportFormat {
 }
 
 type exportRequest struct {
-	Query  string       `json:"query"`
-	From   time.Time    `json:"from" format:"date-time"`
-	To     time.Time    `json:"to" format:"date-time"`
-	Limit  int32        `json:"limit" format:"int32"`
-	Offset int32        `json:"offset" format:"int32"`
-	Format exportFormat `json:"format" default:"jsonl"`
-	Fields []string     `json:"fields,omitempty"`
+	Query      string       `json:"query"`
+	From       time.Time    `json:"from" format:"date-time"`
+	To         time.Time    `json:"to" format:"date-time"`
+	Limit      int32        `json:"limit" format:"int32"`
+	Offset     int32        `json:"offset" format:"int32"`
+	Format     exportFormat `json:"format" default:"jsonl"`
+	Fields     []string     `json:"fields,omitempty"`
+	Downsample uint32       `json:"downsample"`
 } //	@name	seqapi.v1.ExportRequest
 
 func (r exportRequest) toProto() *seqapi.ExportRequest {
 	return &seqapi.ExportRequest{
-		Query:  r.Query,
-		From:   timestamppb.New(r.From),
-		To:     timestamppb.New(r.To),
-		Limit:  r.Limit,
-		Offset: r.Offset,
-		Format: r.Format.toProto(),
-		Fields: r.Fields,
+		Query:      r.Query,
+		From:       timestamppb.New(r.From),
+		To:         timestamppb.New(r.To),
+		Limit:      r.Limit,
+		Offset:     r.Offset,
+		Format:     r.Format.toProto(),
+		Fields:     r.Fields,
+		Downsample: r.Downsample,
 	}
 }
 

internal/api/seqapi/v1/http/fetch_async_search_result_test.go

@@ -174,7 +174,7 @@ func TestServeFetchAsyncSearchResult(t *testing.T) {
 					Meta:     meta,
 				},
 			},
-			wantRespBody: `{"status":"done","request":{"retention":"seconds:60","query":"message:error","from":"2025-08-06T17:37:12.000000123Z","to":"2025-08-06T17:52:12.000000123Z","aggregations":[{"field":"x","group_by":"level","agg_func":"avg","quantiles":[0.9,0.5]},{"field":"y","group_by":"level","agg_func":"sum","interval":"30s"}],"histogram":{"interval":"1s"},"with_docs":true,"size":100},"response":{"events":[{"id":"017a854298010000-850287cfa326a7fc","data":{"level":"3","message":"some error","x":"2"},"time":"2025-08-06T17:51:12.000000123Z"},{"id":"017a854298010000-8502fe7f2aa33df3","data":{"level":"2","message":"some error 2","x":"8"},"time":"2025-08-06T17:50:12.000000123Z"}],"histogram":{"buckets":[{"key":"1","docCount":"7"},{"key":"2","docCount":"9"}]},"aggregations":[{"buckets":[{"key":"3","value":2,"quantiles":[2,1]},{"key":"2","value":8,"not_exists":1,"quantiles":[7,4]}]}],"aggregations_ts":[{"data":{"result":[{"metric":{"level":"33"},"values":[{"timestamp":1754502702,"value":2},{"timestamp":1754502732,"value":5}]},{"metric":{"level":"22"},"values":[{"timestamp":1754502672,"value":8}]}]}}],"total":"2","error":{"code":"ERROR_CODE_UNSPECIFIED","message":"some error"}},"started_at":"2025-08-06T17:51:42.000000123Z","expires_at":"2025-08-06T17:52:42.000000123Z","progress":1,"disk_usage":"512","meta":"{\"some\":\"meta\"}","error":{"code":"ERROR_CODE_NO"}}`,
+			wantRespBody: `{"status":"done","request":{"retention":"seconds:60","query":"message:error","from":"2025-08-06T17:37:12.000000123Z","to":"2025-08-06T17:52:12.000000123Z","aggregations":[{"field":"x","group_by":"level","agg_func":"avg","quantiles":[0.9,0.5]},{"field":"y","group_by":"level","agg_func":"sum","interval":"30s"}],"histogram":{"interval":"1s"},"with_docs":true,"size":100,"downsample":0},"response":{"events":[{"id":"017a854298010000-850287cfa326a7fc","data":{"level":"3","message":"some error","x":"2"},"time":"2025-08-06T17:51:12.000000123Z"},{"id":"017a854298010000-8502fe7f2aa33df3","data":{"level":"2","message":"some error 2","x":"8"},"time":"2025-08-06T17:50:12.000000123Z"}],"histogram":{"buckets":[{"key":"1","docCount":"7"},{"key":"2","docCount":"9"}]},"aggregations":[{"buckets":[{"key":"3","value":2,"quantiles":[2,1]},{"key":"2","value":8,"not_exists":1,"quantiles":[7,4]}]}],"aggregations_ts":[{"data":{"result":[{"metric":{"level":"33"},"values":[{"timestamp":1754502702,"value":2},{"timestamp":1754502732,"value":5}]},{"metric":{"level":"22"},"values":[{"timestamp":1754502672,"value":8}]}]}}],"total":"2","error":{"code":"ERROR_CODE_UNSPECIFIED","message":"some error"}},"started_at":"2025-08-06T17:51:42.000000123Z","expires_at":"2025-08-06T17:52:42.000000123Z","progress":1,"disk_usage":"512","meta":"{\"some\":\"meta\"}","error":{"code":"ERROR_CODE_NO"}}`,
 			wantStatus:   http.StatusOK,
 		},
 		{
@@ -238,7 +238,7 @@ func TestServeFetchAsyncSearchResult(t *testing.T) {
 					Meta:     meta,
 				},
 			},
-			wantRespBody: `{"status":"done","request":{"retention":"seconds:60","query":"message:error","from":"2025-08-06T17:37:12.000000123Z","to":"2025-08-06T17:52:12.000000123Z","with_docs":true,"size":100},"response":{"events":[{"id":"017a854298010000-850287cfa326a7fc","data":{"level":"3","message":"some error","x":"2"},"time":"2025-08-06T17:51:12.000000123Z"},{"id":"017a854298010000-8502fe7f2aa33df3","data":{"level":"2","message":"some error 2","x":"8"},"time":"2025-08-06T17:50:12.000000123Z"}],"total":"2","error":{"code":"ERROR_CODE_UNSPECIFIED","message":"some error"}},"started_at":"2025-08-06T17:51:42.000000123Z","expires_at":"2025-08-06T17:52:42.000000123Z","progress":1,"disk_usage":"512","meta":"{\"some\":\"meta\"}","error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`,
+			wantRespBody: `{"status":"done","request":{"retention":"seconds:60","query":"message:error","from":"2025-08-06T17:37:12.000000123Z","to":"2025-08-06T17:52:12.000000123Z","with_docs":true,"size":100,"downsample":0},"response":{"events":[{"id":"017a854298010000-850287cfa326a7fc","data":{"level":"3","message":"some error","x":"2"},"time":"2025-08-06T17:51:12.000000123Z"},{"id":"017a854298010000-8502fe7f2aa33df3","data":{"level":"2","message":"some error 2","x":"8"},"time":"2025-08-06T17:50:12.000000123Z"}],"total":"2","error":{"code":"ERROR_CODE_UNSPECIFIED","message":"some error"}},"started_at":"2025-08-06T17:51:42.000000123Z","expires_at":"2025-08-06T17:52:42.000000123Z","progress":1,"disk_usage":"512","meta":"{\"some\":\"meta\"}","error":{"code":"ERROR_CODE_PARTIAL_RESPONSE","message":"partial response"}}`,
 			wantStatus:   http.StatusOK,
 		},
 		{

internal/api/seqapi/v1/http/get_async_searches_list.go

@@ -176,6 +176,7 @@ func startAsyncSearchRequestFromProto(r *seqapi.StartAsyncSearchRequest) startAs
 		Histogram:    hist,
 		WithDocs:     r.WithDocs,
 		Size:         r.Size,
+		Downsample:   r.Downsample,
 	}
 }