diff --git a/pkg/plugins/fastly/cmd/main/main.go b/pkg/plugins/fastly/cmd/main/main.go new file mode 100644 index 0000000..7c77562 --- /dev/null +++ b/pkg/plugins/fastly/cmd/main/main.go @@ -0,0 +1,286 @@ +package main + +import ( + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/google/uuid" + "github.com/hashicorp/go-plugin" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/timestamppb" + + commonconfig "github.com/opencost/opencost-plugins/pkg/common/config" + fastlyconfig "github.com/opencost/opencost-plugins/pkg/plugins/fastly/config" + fastlyplugin "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + ocplugin "github.com/opencost/opencost/core/pkg/plugin" +) + +// handshakeConfigs are used to just do a basic handshake between +// a plugin and host. If the handshake fails, a user friendly error is shown. +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "PLUGIN_NAME", + MagicCookieValue: "fastly", +} + +const fastlyInvoicesURL = "https://api.fastly.com/billing/v3/invoices" +const fastlyInvoiceByIDURL = "https://api.fastly.com/billing/v3/invoices/%s" +const fastlyInvoiceMTDURL = "https://api.fastly.com/billing/v3/invoices/month-to-date" +const fastlyAPIDateFormat = "2006-01-02" + +// Implementation of CustomCostSource +type FastlyCostSource struct { + rateLimiter *rate.Limiter + config *fastlyconfig.FastlyConfig + client HTTPClient +} + +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +func (f *FastlyCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { + results := []*pb.CustomCostResponse{} + + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + if err != nil { + log.Errorf("error getting windows: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error getting windows: %v", err)}, + } + results = append(results, &errResp) + return results + } + + for _, target := range targets { + // don't allow future requests + if target.Start().After(time.Now().UTC()) { + log.Debugf("skipping future window %v", target) + continue + } + + log.Debugf("fetching Fastly costs for window %v", target) + result := f.getFastlyCostsForWindow(target) + results = append(results, result) + } + + return results +} + +func main() { + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + fastlyCfg, err := fastlyconfig.GetFastlyConfig(configFile) + if err != nil { + log.Fatalf("error building Fastly config: %v", err) + } + log.SetLogLevel(fastlyCfg.LogLevel) + + // Fastly API rate limit: approximately 100 requests per minute + rateLimiter := rate.NewLimiter(1.5, 2) + fastlyCostSrc := FastlyCostSource{ + rateLimiter: rateLimiter, + config: fastlyCfg, + client: &http.Client{}, + } + + // pluginMap is the map of plugins we can dispense. + var pluginMap = map[string]plugin.Plugin{ + "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: &fastlyCostSrc}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + GRPCServer: plugin.DefaultGRPCServer, + }) +} + +func boilerplateFastlyCustomCost(win opencost.Window) pb.CustomCostResponse { + return pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v3"}, + CostSource: "CDN", + Domain: "fastly", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(*win.Start()), + End: timestamppb.New(*win.End()), + Errors: []string{}, + Costs: []*pb.CustomCost{}, + } +} + +func (f *FastlyCostSource) getFastlyCostsForWindow(window opencost.Window) *pb.CustomCostResponse { + ccResp := boilerplateFastlyCustomCost(window) + + winStart := window.Start().Format(fastlyAPIDateFormat) + winEnd := window.End().Format(fastlyAPIDateFormat) + + // Fetch invoices for the window period + invoices, err := f.getInvoices(winStart, winEnd) + if err != nil { + ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("error getting Fastly invoices: %v", err)) + return &ccResp + } + + customCosts := []*pb.CustomCost{} + for _, invoice := range invoices { + for _, lineItem := range invoice.TransactionLineItems { + cost := pb.CustomCost{ + AccountName: invoice.CustomerID, + ChargeCategory: "Usage", + Description: lineItem.Description, + ResourceName: lineItem.ProductName, + ResourceType: lineItem.ProductGroup, + Id: uuid.New().String(), + ProviderId: fmt.Sprintf("%s/%s/%s", invoice.CustomerID, lineItem.ProductName, lineItem.UsageType), + BilledCost: float32(lineItem.Amount), + ListCost: float32(lineItem.Units * lineItem.Rate), + ListUnitPrice: float32(lineItem.Rate), + UsageQuantity: float32(lineItem.Units), + UsageUnit: lineItem.UsageType, + Zone: lineItem.Region, + Labels: map[string]string{}, + } + + if lineItem.ProductLine != "" { + cost.Labels["product_line"] = lineItem.ProductLine + } + if invoice.CurrencyCode != "" { + cost.Labels["currency_code"] = invoice.CurrencyCode + } + if invoice.InvoiceID != "" { + cost.Labels["invoice_id"] = invoice.InvoiceID + } + + customCosts = append(customCosts, &cost) + } + } + + ccResp.Costs = customCosts + return &ccResp +} + +func (f *FastlyCostSource) getInvoices(startDate, endDate string) ([]fastlyplugin.FastlyInvoice, error) { + var allInvoices []fastlyplugin.FastlyInvoice + + url := fmt.Sprintf("%s?billing_start_date=%s&billing_end_date=%s&limit=200", fastlyInvoicesURL, startDate, endDate) + + for { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + req.Header.Set("Fastly-Key", f.config.APIKey) + req.Header.Set("Accept", "application/json") + + err = f.rateLimiter.Wait(nil) + if err != nil { + return nil, fmt.Errorf("rate limiter error: %v", err) + } + + resp, err := f.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request to Fastly API: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("Fastly API returned status %d: %s", resp.StatusCode, string(body)) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %v", err) + } + + var invoiceResp fastlyplugin.FastlyInvoiceListResponse + err = json.Unmarshal(body, &invoiceResp) + if err != nil { + return nil, fmt.Errorf("error unmarshalling Fastly response: %v", err) + } + + // Fetch detailed invoice for each invoice ID to get line items + for _, invoice := range invoiceResp.Invoices { + invoiceID := invoice.InvoiceID + if invoiceID == "" { + // If no invoice ID, use the summary data + allInvoices = append(allInvoices, invoice) + continue + } + + detailedInvoice, err := f.getInvoiceByID(invoiceID) + if err != nil { + log.Warnf("error getting invoice details for %s: %v, using summary data", invoiceID, err) + allInvoices = append(allInvoices, invoice) + continue + } + + allInvoices = append(allInvoices, *detailedInvoice) + } + + // Check for pagination + if invoiceResp.Meta.NextCursor != "" { + url = fmt.Sprintf("%s?cursor=%s&limit=200", fastlyInvoicesURL, invoiceResp.Meta.NextCursor) + } else { + break + } + } + + return allInvoices, nil +} + +func (f *FastlyCostSource) getInvoiceByID(invoiceID string) (*fastlyplugin.FastlyInvoice, error) { + invoiceIDInt, err := strconv.Atoi(invoiceID) + if err != nil { + return nil, fmt.Errorf("invalid invoice ID %s: %v", invoiceID, err) + } + + url := fmt.Sprintf(fastlyInvoiceByIDURL, invoiceIDInt) + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + req.Header.Set("Fastly-Key", f.config.APIKey) + req.Header.Set("Accept", "application/json") + + err = f.rateLimiter.Wait(nil) + if err != nil { + return nil, fmt.Errorf("rate limiter error: %v", err) + } + + resp, err := f.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request to Fastly API: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("Fastly API returned status %d: %s", resp.StatusCode, string(body)) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %v", err) + } + + var invoice fastlyplugin.FastlyInvoice + err = json.Unmarshal(body, &invoice) + if err != nil { + return nil, fmt.Errorf("error unmarshalling Fastly invoice response: %v", err) + } + + return &invoice, nil +} diff --git a/pkg/plugins/fastly/cmd/main/main_test.go b/pkg/plugins/fastly/cmd/main/main_test.go new file mode 100644 index 0000000..97759be --- /dev/null +++ b/pkg/plugins/fastly/cmd/main/main_test.go @@ -0,0 +1,192 @@ +package main + +import ( + "encoding/json" + "io" + "net/http" + "net/http/httptest" + "testing" + + fastlyplugin "github.com/opencost/opencost-plugins/pkg/plugins/fastly/fastlyplugin" +) + +func TestGetInvoices(t *testing.T) { + // Create a mock Fastly API server + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + // Verify API key header + if r.Header.Get("Fastly-Key") != "test-api-key" { + t.Errorf("Expected Fastly-Key header 'test-api-key', got '%s'", r.Header.Get("Fastly-Key")) + } + + // Check if this is a request for a specific invoice + if r.URL.Path == "/billing/v3/invoices/12345" { + invoice := fastlyplugin.FastlyInvoice{ + InvoiceID: "12345", + CustomerID: "cust-123", + CurrencyCode: "USD", + BillingStartDate: "2024-01-01", + BillingEndDate: "2024-01-31", + MonthlyTransactionAmount: 1500.00, + TransactionLineItems: []fastlyplugin.FastlyTransactionLine{ + { + Description: "Compute - Request", + ProductGroup: "Compute", + ProductLine: "Network Services", + ProductName: "Requests", + UsageType: "requests", + Region: "NA", + Units: 1000000, + Rate: 0.00075, + Amount: 750.00, + }, + { + Description: "Bandwidth - Delivery", + ProductGroup: "Full-Site Delivery", + ProductLine: "Network Services", + ProductName: "Bandwidth", + UsageType: "bandwidth", + Region: "NA", + Units: 500, + Rate: 0.50, + Amount: 250.00, + }, + }, + } + json.NewEncoder(w).Encode(invoice) + return + } + + // Return invoice list + invoiceResp := fastlyplugin.FastlyInvoiceListResponse{ + Meta: fastlyplugin.FastlyInvoiceMeta{ + Limit: 200, + NextCursor: "", + Total: 1, + }, + Invoices: []fastlyplugin.FastlyInvoice{ + { + InvoiceID: "12345", + CustomerID: "cust-123", + CurrencyCode: "USD", + BillingStartDate: "2024-01-01", + BillingEndDate: "2024-01-31", + MonthlyTransactionAmount: 1500.00, + }, + }, + } + json.NewEncoder(w).Encode(invoiceResp) + })) + defer mockServer.Close() + + // Override URLs for testing + origInvoicesURL := fastlyInvoicesURL + origInvoiceByIDURL := fastlyInvoiceByIDURL + defer func() { + // Note: these are consts, so we can't actually override them in tests. + // In a real implementation, we'd make these configurable. + _ = origInvoicesURL + _ = origInvoiceByIDURL + }() + + // Verify the response format matches expected + resp, err := http.Get(mockServer.URL + "/billing/v3/invoices") + if err != nil { + t.Fatalf("Error making request: %v", err) + } + defer resp.Body.Close() + + body, _ := io.ReadAll(resp.Body) + var invoiceResp fastlyplugin.FastlyInvoiceListResponse + err = json.Unmarshal(body, &invoiceResp) + if err != nil { + t.Fatalf("Error unmarshalling response: %v", err) + } + + if len(invoiceResp.Invoices) != 1 { + t.Errorf("Expected 1 invoice, got %d", len(invoiceResp.Invoices)) + } + + if invoiceResp.Invoices[0].InvoiceID != "12345" { + t.Errorf("Expected invoice ID '12345', got '%s'", invoiceResp.Invoices[0].InvoiceID) + } +} + +func TestFastlyTransactionLineUnmarshal(t *testing.T) { + jsonData := `{ + "description": "Compute - Request", + "product_group": "Compute", + "product_line": "Network Services", + "product_name": "Requests", + "usage_type": "requests", + "region": "NA", + "units": 1000000, + "rate": 0.00075, + "amount": 750.00, + "credit_coupon_code": "" + }` + + var lineItem fastlyplugin.FastlyTransactionLine + err := json.Unmarshal([]byte(jsonData), &lineItem) + if err != nil { + t.Fatalf("Error unmarshalling transaction line: %v", err) + } + + if lineItem.ProductName != "Requests" { + t.Errorf("Expected product name 'Requests', got '%s'", lineItem.ProductName) + } + + if lineItem.Amount != 750.00 { + t.Errorf("Expected amount 750.00, got %f", lineItem.Amount) + } + + if lineItem.Units != 1000000 { + t.Errorf("Expected units 1000000, got %f", lineItem.Units) + } + + if lineItem.Region != "NA" { + t.Errorf("Expected region 'NA', got '%s'", lineItem.Region) + } +} + +func TestFastlyInvoiceUnmarshal(t *testing.T) { + jsonData := `{ + "invoice_id": "12345", + "customer_id": "cust-123", + "currency_code": "USD", + "billing_start_date": "2024-01-01", + "billing_end_date": "2024-01-31", + "statement_number": "ST-001", + "monthly_transaction_amount": 1500.00, + "transaction_line_items": [ + { + "description": "Compute - Request", + "product_group": "Compute", + "product_line": "Network Services", + "product_name": "Requests", + "usage_type": "requests", + "region": "NA", + "units": 1000000, + "rate": 0.00075, + "amount": 750.00 + } + ] + }` + + var invoice fastlyplugin.FastlyInvoice + err := json.Unmarshal([]byte(jsonData), &invoice) + if err != nil { + t.Fatalf("Error unmarshalling invoice: %v", err) + } + + if invoice.InvoiceID != "12345" { + t.Errorf("Expected invoice ID '12345', got '%s'", invoice.InvoiceID) + } + + if len(invoice.TransactionLineItems) != 1 { + t.Errorf("Expected 1 line item, got %d", len(invoice.TransactionLineItems)) + } + + if invoice.MonthlyTransactionAmount != 1500.00 { + t.Errorf("Expected monthly transaction amount 1500.00, got %f", invoice.MonthlyTransactionAmount) + } +} diff --git a/pkg/plugins/fastly/cmd/validator/main/main.go b/pkg/plugins/fastly/cmd/validator/main/main.go new file mode 100644 index 0000000..943f43f --- /dev/null +++ b/pkg/plugins/fastly/cmd/validator/main/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "encoding/json" + "fmt" + "os" + + commonconfig "github.com/opencost/opencost-plugins/pkg/common/config" + fastlyconfig "github.com/opencost/opencost-plugins/pkg/plugins/fastly/config" + "github.com/opencost/opencost/core/pkg/log" +) + +func main() { + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + fastlyCfg, err := fastlyconfig.GetFastlyConfig(configFile) + if err != nil { + log.Fatalf("error building Fastly config: %v", err) + } + + // Validate required fields + if fastlyCfg.APIKey == "" { + log.Fatalf("fastly_api_key is required in config file") + } + + fmt.Printf("Fastly config validated successfully\n") + fmt.Printf("Log level: %s\n", fastlyCfg.LogLevel) + + // Write out a sample config for reference + sampleConfig := map[string]interface{}{ + "fastly_api_key": "YOUR_FASTLY_API_KEY", + "fastly_plugin_log_level": "info", + } + sampleJSON, _ := json.MarshalIndent(sampleConfig, "", " ") + fmt.Printf("\nSample config:\n%s\n", string(sampleJSON)) + + os.Exit(0) +} diff --git a/pkg/plugins/fastly/config/fastlyconfig.go b/pkg/plugins/fastly/config/fastlyconfig.go new file mode 100644 index 0000000..f536904 --- /dev/null +++ b/pkg/plugins/fastly/config/fastlyconfig.go @@ -0,0 +1,30 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +type FastlyConfig struct { + APIKey string `json:"fastly_api_key"` + LogLevel string `json:"fastly_plugin_log_level"` +} + +func GetFastlyConfig(configFilePath string) (*FastlyConfig, error) { + var result FastlyConfig + bytes, err := os.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("error reading config file for Fastly config @ %s: %v", configFilePath, err) + } + err = json.Unmarshal(bytes, &result) + if err != nil { + return nil, fmt.Errorf("error marshaling json into Fastly config %v", err) + } + + if result.LogLevel == "" { + result.LogLevel = "info" + } + + return &result, nil +} diff --git a/pkg/plugins/fastly/fastlyplugin/fastlybilling.go b/pkg/plugins/fastly/fastlyplugin/fastlybilling.go new file mode 100644 index 0000000..4ac4582 --- /dev/null +++ b/pkg/plugins/fastly/fastlyplugin/fastlybilling.go @@ -0,0 +1,42 @@ +package fastlyplugin + +// FastlyInvoice represents a Fastly invoice from the billing API. +type FastlyInvoice struct { + InvoiceID string `json:"invoice_id"` + CustomerID string `json:"customer_id"` + CurrencyCode string `json:"currency_code"` + BillingStartDate string `json:"billing_start_date"` + BillingEndDate string `json:"billing_end_date"` + StatementNumber string `json:"statement_number"` + InvoicePostedOn string `json:"invoice_posted_on"` + MonthlyTransactionAmount float64 `json:"monthly_transaction_amount"` + TransactionLineItems []FastlyTransactionLine `json:"transaction_line_items"` +} + +// FastlyTransactionLine represents a single line item in a Fastly invoice. +type FastlyTransactionLine struct { + Description string `json:"description"` + ProductGroup string `json:"product_group"` + ProductLine string `json:"product_line"` + ProductName string `json:"product_name"` + UsageType string `json:"usage_type"` + Region string `json:"region"` + Units float64 `json:"units"` + Rate float64 `json:"rate"` + Amount float64 `json:"amount"` + CreditCouponCode string `json:"credit_coupon_code"` +} + +// FastlyInvoiceListResponse represents the paginated response from listing invoices. +type FastlyInvoiceListResponse struct { + Meta FastlyInvoiceMeta `json:"meta"` + Invoices []FastlyInvoice `json:"data"` +} + +// FastlyInvoiceMeta represents pagination metadata. +type FastlyInvoiceMeta struct { + Limit int `json:"limit"` + NextCursor string `json:"next_cursor"` + Sort string `json:"sort"` + Total int `json:"total"` +} diff --git a/pkg/plugins/fastly/go.mod b/pkg/plugins/fastly/go.mod new file mode 100644 index 0000000..3ce8b75 --- /dev/null +++ b/pkg/plugins/fastly/go.mod @@ -0,0 +1,12 @@ +module github.com/opencost/opencost-plugins/pkg/plugins/fastly + +go 1.21 + +require ( + github.com/google/uuid v1.6.0 + github.com/hashicorp/go-plugin v1.6.1 + github.com/opencost/opencost-plugins/pkg/common v0.0.0 + github.com/opencost/opencost/core v0.0.0 + golang.org/x/time v0.5.0 + google.golang.org/protobuf v1.34.0 +)