Skip to content

Commit 641b0b2

Browse files
aniruddha2000aniruddha2000
authored
add generic metrics for external API (#1228) (#1353)
* add genetric metrics for external API * make fmt * add tests for the new http error metrics * remove loadbalancer prefix from metrics names * add tests for HTTPRequestDurationHistogram and HTTPRequestCount * add api label from client creation * make mod tidy * add metrics for csi * call metric server from csi/main.go * create prometheus metrics exporter * remove api name required check and update tests * remove error from func * make status_code * make add more labels in new metrics * make lint * fix test * optimize code
1 parent 1f3f0b1 commit 641b0b2

6 files changed

Lines changed: 225 additions & 40 deletions

File tree

‎cmd/stackit-csi-plugin/main.go‎

Lines changed: 22 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,13 @@
11
package main
22

33
import (
4+
"context"
45
"fmt"
56
"os"
7+
"os/signal"
8+
"syscall"
69

10+
"github.com/prometheus/client_golang/prometheus"
711
"github.com/spf13/cobra"
812
"github.com/spf13/pflag"
913
"github.com/stackitcloud/cloud-provider-stackit/pkg/csi"
@@ -22,7 +26,7 @@ var (
2226
endpoint string
2327
cloudConfig string
2428
cluster string
25-
httpEndpoint string
29+
metricsAddress string
2630
provideControllerService bool
2731
provideNodeService bool
2832
legacyStorageMode bool
@@ -34,7 +38,10 @@ func main() {
3438
Use: "stackit-csi-plugin",
3539
Short: "STACKIT block-storage CSI plugin",
3640
Run: func(_ *cobra.Command, _ []string) {
37-
handle()
41+
ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGTERM, syscall.SIGINT)
42+
defer cancel()
43+
44+
handle(ctx)
3845
},
3946
PersistentPreRunE: func(cmd *cobra.Command, _ []string) error {
4047
f := cmd.Flags()
@@ -67,7 +74,7 @@ func main() {
6774
cmd.Flags().StringVar(&cloudConfig, "cloud-config", "", "CSI driver cloud config. This option can be given multiple times")
6875

6976
cmd.PersistentFlags().StringVar(&cluster, "cluster", "", "The identifier of the cluster that the plugin is running in.")
70-
cmd.PersistentFlags().StringVar(&httpEndpoint, "http-endpoint", "",
77+
cmd.PersistentFlags().StringVar(&metricsAddress, "metrics-address", "",
7178
"The TCP network address where the HTTP server for providing metrics for diagnostics, will listen (example: `:8080`)."+
7279
"The default is empty string, which means the server is disabled.")
7380

@@ -85,7 +92,16 @@ func main() {
8592
os.Exit(code)
8693
}
8794

88-
func handle() {
95+
func handle(ctx context.Context) {
96+
if metricsAddress != "" {
97+
metricsExporter := metrics.NewExporter()
98+
prometheus.MustRegister(metricsExporter)
99+
go func() {
100+
if err := metrics.Run(ctx, metricsAddress); err != nil {
101+
klog.Fatalf("Run metrics returned an error: %v", err)
102+
}
103+
}()
104+
}
89105
// Initialize cloud
90106
driverOpts := &blockstorage.DriverOpts{
91107
Endpoint: endpoint,
@@ -110,8 +126,9 @@ func handle() {
110126
klog.Fatal(err)
111127
}
112128

129+
iaasHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameIaaS)
113130
iaasOpts := []sdkconfig.ConfigurationOption{
114-
sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient()),
131+
sdkconfig.WithHTTPClient(iaasHTTPClient),
115132
}
116133

117134
if cfg.Global.APIEndpoints.IaasAPI != "" {

‎go.mod‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ require (
1111
github.com/onsi/ginkgo/v2 v2.32.0
1212
github.com/onsi/gomega v1.42.1
1313
github.com/prometheus/client_golang v1.23.2
14+
github.com/prometheus/client_model v0.6.2
1415
github.com/spf13/cobra v1.10.2
1516
github.com/spf13/pflag v1.0.10
1617
github.com/stackitcloud/stackit-sdk-go/core v0.26.0
@@ -87,7 +88,6 @@ require (
8788
github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect
8889
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
8990
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
90-
github.com/prometheus/client_model v0.6.2 // indirect
9191
github.com/prometheus/common v0.66.1 // indirect
9292
github.com/prometheus/procfs v0.16.1 // indirect
9393
github.com/stackitcloud/stackit-sdk-go/services/resourcemanager v0.24.0 // indirect

‎pkg/ccm/stackit.go‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -117,8 +117,9 @@ func BuildObservability() (*MetricsRemoteWrite, error) {
117117

118118
// NewCloudControllerManager creates a new instance of the stackit struct from a stackitconfig struct
119119
func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteWrite) (*CloudControllerManager, error) {
120+
lbHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameLoadBalancer)
120121
lbOpts := []sdkconfig.ConfigurationOption{
121-
sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient()),
122+
sdkconfig.WithHTTPClient(lbHTTPClient),
122123
}
123124

124125
if cfg.Global.APIEndpoints.LoadBalancerAPI != "" {
@@ -138,8 +139,9 @@ func NewCloudControllerManager(cfg *stackitconfig.CCMConfig, obs *MetricsRemoteW
138139
return nil, fmt.Errorf("failed to create lb client: %v", err)
139140
}
140141

142+
iaasHTTPClient := metrics.NewInstrumentedHTTPClient(metrics.APINameIaaS)
141143
iaasOpts := []sdkconfig.ConfigurationOption{
142-
sdkconfig.WithHTTPClient(metrics.NewInstrumentedHTTPClient()),
144+
sdkconfig.WithHTTPClient(iaasHTTPClient),
143145
}
144146

145147
if cfg.Global.APIEndpoints.IaasAPI != "" {

‎pkg/metrics/http.go‎

Lines changed: 26 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -3,19 +3,24 @@ package metrics
33
import (
44
"fmt"
55
"net/http"
6+
"strconv"
67
"strings"
78
"time"
89

910
"github.com/prometheus/client_golang/prometheus"
1011
)
1112

12-
func NewInstrumentedHTTPClient() *http.Client {
13+
func NewInstrumentedHTTPClient(api string) *http.Client {
1314
return &http.Client{
14-
Transport: &InstrumentedRoundTripper{http.DefaultTransport},
15+
Transport: &InstrumentedRoundTripper{
16+
api: api,
17+
base: http.DefaultTransport,
18+
},
1519
}
1620
}
1721

1822
type InstrumentedRoundTripper struct {
23+
api string
1924
base http.RoundTripper
2025
}
2126

@@ -26,15 +31,26 @@ func (rt *InstrumentedRoundTripper) RoundTrip(request *http.Request) (*http.Resp
2631
response, err := rt.base.RoundTrip(request)
2732
duration := time.Since(startTime)
2833

29-
LoadBalancerResponseTimeHistogram.
30-
With(prometheus.Labels{operationLabel: operation}).
31-
Observe(float64(duration.Seconds()))
32-
LoadBalancerRequestCount.
33-
With(prometheus.Labels{operationLabel: operation}).
34-
Inc()
34+
statusCode := "network_error"
35+
if response != nil {
36+
statusCode = strconv.Itoa(response.StatusCode)
37+
}
38+
39+
labels := prometheus.Labels{
40+
apiLabel: rt.api,
41+
methodLabel: request.Method,
42+
operationLabel: operation,
43+
codeLabel: statusCode,
44+
}
45+
46+
HTTPRequestDurationHistogram.With(labels).Observe(duration.Seconds())
47+
HTTPRequestCount.With(labels).Inc()
48+
49+
isHTTPError := response != nil && response.StatusCode >= 400
50+
isNetworkError := err != nil
3551

36-
if response != nil && response.StatusCode >= http.StatusInternalServerError {
37-
LoadBalancerErrorCount.Inc()
52+
if isHTTPError || isNetworkError {
53+
HTTPErrorCount.With(labels).Inc()
3854
}
3955

4056
return response, err

‎pkg/metrics/http_test.go‎

Lines changed: 148 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,10 +2,14 @@ package metrics
22

33
import (
44
"net/http"
5+
"net/http/httptest"
56
"net/url"
67

78
. "github.com/onsi/ginkgo/v2"
89
. "github.com/onsi/gomega"
10+
"github.com/prometheus/client_golang/prometheus"
11+
"github.com/prometheus/client_golang/prometheus/testutil"
12+
dto "github.com/prometheus/client_model/go"
913
)
1014

1115
var _ = Describe("Metrics", func() {
@@ -22,4 +26,148 @@ var _ = Describe("Metrics", func() {
2226
Entry("get load-balancers", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers", "get_load-balancers"),
2327
Entry("get load-balancers instance", "GET", "/v2/projects/6-a-4-8-c/regions/eu01/load-balancers/id", "get_load-balancers_instance"),
2428
)
29+
30+
Describe("InstrumentedRoundTripper", func() {
31+
It("increments HTTPRequestCount for responses", func() {
32+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
33+
w.WriteHeader(http.StatusOK)
34+
}))
35+
defer server.Close()
36+
37+
labels := prometheus.Labels{
38+
apiLabel: "test",
39+
methodLabel: "GET",
40+
operationLabel: "get_request-count-test",
41+
codeLabel: "200",
42+
}
43+
before := testutil.ToFloat64(HTTPRequestCount.With(labels))
44+
45+
client := NewInstrumentedHTTPClient("test")
46+
47+
response, err := client.Get(server.URL + "/request-count-test")
48+
Expect(err).NotTo(HaveOccurred())
49+
defer response.Body.Close()
50+
51+
after := testutil.ToFloat64(HTTPRequestCount.With(labels))
52+
Expect(after - before).To(Equal(float64(1)))
53+
})
54+
55+
It("records HTTPRequestDurationHistogram observations for responses", func() {
56+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
57+
w.WriteHeader(http.StatusOK)
58+
}))
59+
defer server.Close()
60+
61+
labels := prometheus.Labels{
62+
apiLabel: "test",
63+
methodLabel: "GET",
64+
operationLabel: "get_request-duration-test",
65+
codeLabel: "200",
66+
}
67+
before := histogramSampleCount(HTTPRequestDurationHistogram.With(labels))
68+
69+
client := NewInstrumentedHTTPClient("test")
70+
71+
response, err := client.Get(server.URL + "/request-duration-test")
72+
Expect(err).NotTo(HaveOccurred())
73+
defer response.Body.Close()
74+
75+
after := histogramSampleCount(HTTPRequestDurationHistogram.With(labels))
76+
Expect(after - before).To(Equal(uint64(1)))
77+
})
78+
79+
It("increments HTTPErrorCount for error responses (400, 404, 500)", func() {
80+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
81+
if r.Method == http.MethodPost {
82+
w.WriteHeader(http.StatusInternalServerError)
83+
return
84+
}
85+
if r.URL.Path == "/404" {
86+
w.WriteHeader(http.StatusNotFound)
87+
return
88+
}
89+
w.WriteHeader(http.StatusBadRequest)
90+
}))
91+
defer server.Close()
92+
93+
labels400 := prometheus.Labels{
94+
apiLabel: "test",
95+
methodLabel: http.MethodGet,
96+
operationLabel: "get_",
97+
codeLabel: "400",
98+
}
99+
labels404 := prometheus.Labels{
100+
apiLabel: "test",
101+
methodLabel: http.MethodGet,
102+
operationLabel: "get_404",
103+
codeLabel: "404",
104+
}
105+
labels500 := prometheus.Labels{
106+
apiLabel: "test",
107+
methodLabel: http.MethodPost,
108+
operationLabel: "post_",
109+
codeLabel: "500",
110+
}
111+
before400 := testutil.ToFloat64(HTTPErrorCount.With(labels400))
112+
before404 := testutil.ToFloat64(HTTPErrorCount.With(labels404))
113+
before500 := testutil.ToFloat64(HTTPErrorCount.With(labels500))
114+
115+
client := NewInstrumentedHTTPClient("test")
116+
117+
response1, err := client.Get(server.URL)
118+
Expect(err).NotTo(HaveOccurred())
119+
defer response1.Body.Close()
120+
121+
response2, err := client.Get(server.URL + "/404")
122+
Expect(err).NotTo(HaveOccurred())
123+
defer response2.Body.Close()
124+
125+
response3, err := client.Post(server.URL, "application/json", nil)
126+
Expect(err).NotTo(HaveOccurred())
127+
defer response3.Body.Close()
128+
129+
after400 := testutil.ToFloat64(HTTPErrorCount.With(labels400))
130+
after404 := testutil.ToFloat64(HTTPErrorCount.With(labels404))
131+
after500 := testutil.ToFloat64(HTTPErrorCount.With(labels500))
132+
133+
Expect(after400 - before400).To(Equal(float64(1)))
134+
Expect(after404 - before404).To(Equal(float64(1)))
135+
Expect(after500 - before500).To(Equal(float64(1)))
136+
Expect((after400 - before400) + (after404 - before404) + (after500 - before500)).To(Equal(float64(3)))
137+
})
138+
139+
It("does not increment HTTPErrorCount for successful responses", func() {
140+
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, _ *http.Request) {
141+
w.WriteHeader(http.StatusOK)
142+
}))
143+
defer server.Close()
144+
145+
labels := prometheus.Labels{
146+
apiLabel: "test",
147+
methodLabel: http.MethodGet,
148+
operationLabel: "get_",
149+
codeLabel: "200",
150+
}
151+
before := testutil.ToFloat64(HTTPErrorCount.With(labels))
152+
153+
client := NewInstrumentedHTTPClient("test")
154+
155+
response, err := client.Get(server.URL)
156+
Expect(err).NotTo(HaveOccurred())
157+
defer response.Body.Close()
158+
159+
after := testutil.ToFloat64(HTTPErrorCount.With(labels))
160+
Expect(after - before).To(Equal(float64(0)))
161+
})
162+
})
25163
})
164+
165+
func histogramSampleCount(observer prometheus.Observer) uint64 {
166+
metric, ok := observer.(prometheus.Metric)
167+
Expect(ok).To(BeTrue())
168+
169+
dtoMetric := &dto.Metric{}
170+
Expect(metric.Write(dtoMetric)).To(Succeed())
171+
172+
return dtoMetric.GetHistogram().GetSampleCount()
173+
}

0 commit comments

Comments
 (0)