From 19262c27f2300414a4c076feda444492c71cf116 Mon Sep 17 00:00:00 2001 From: LZG3530606141 <3530606141@qq.com> Date: Mon, 11 May 2026 20:10:59 +0800 Subject: [PATCH] feat: add GitHub Billing plugin - Implement GitHubBillingCostSource for ingesting GitHub billing data - Fetch usage billing via GitHub Billing REST API - Fetch premium request usage (Copilot) via Premium Request API - Support Bearer token authentication with GitHub API - Map GitHub billing fields to FOCUS spec including product, SKU, repository name, organization, unit type, price per unit - Include both gross and net amounts with discount tracking - Add rate limiting for GitHub API (5000 req/hr) - Add unit tests for JSON unmarshalling - Add config validator - Follow OpenCost plugin architecture (hashicorp go-plugin) References: - GitHub Billing API: https://docs.github.com/en/rest/billing/usage - Issue: #42 Signed-off-by: LZG3530606141 <3530606141@qq.com> --- pkg/plugins/githubbilling/cmd/main/main.go | 285 ++++++++++++++++++ .../githubbilling/cmd/main/main_test.go | 123 ++++++++ .../githubbilling/cmd/validator/main/main.go | 41 +++ .../config/githubbillingconfig.go | 31 ++ .../githubbillingplugin/githubbilling.go | 53 ++++ pkg/plugins/githubbilling/go.mod | 12 + 6 files changed, 545 insertions(+) create mode 100644 pkg/plugins/githubbilling/cmd/main/main.go create mode 100644 pkg/plugins/githubbilling/cmd/main/main_test.go create mode 100644 pkg/plugins/githubbilling/cmd/validator/main/main.go create mode 100644 pkg/plugins/githubbilling/config/githubbillingconfig.go create mode 100644 pkg/plugins/githubbilling/githubbillingplugin/githubbilling.go create mode 100644 pkg/plugins/githubbilling/go.mod diff --git a/pkg/plugins/githubbilling/cmd/main/main.go b/pkg/plugins/githubbilling/cmd/main/main.go new file mode 100644 index 0000000..d553bff --- /dev/null +++ b/pkg/plugins/githubbilling/cmd/main/main.go @@ -0,0 +1,285 @@ +package main + +import ( + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "strconv" + "time" + + "github.com/google/uuid" + "github.com/hashicorp/go-plugin" + "golang.org/x/time/rate" + "google.golang.org/protobuf/types/known/timestamppb" + + commonconfig "github.com/opencost/opencost-plugins/pkg/common/config" + githubbillingconfig "github.com/opencost/opencost-plugins/pkg/plugins/githubbilling/config" + githubbillingplugin "github.com/opencost/opencost-plugins/pkg/plugins/githubbilling/githubbillingplugin" + "github.com/opencost/opencost/core/pkg/log" + "github.com/opencost/opencost/core/pkg/model/pb" + "github.com/opencost/opencost/core/pkg/opencost" + ocplugin "github.com/opencost/opencost/core/pkg/plugin" +) + +var handshakeConfig = plugin.HandshakeConfig{ + ProtocolVersion: 1, + MagicCookieKey: "PLUGIN_NAME", + MagicCookieValue: "githubbilling", +} + +const githubBillingUsageURL = "https://api.github.com/organizations/%s/settings/billing/usage" +const githubBillingPremiumURL = "https://api.github.com/organizations/%s/settings/billing/premium_request/usage" +const githubAPIVersion = "2022-11-28" + +type GitHubBillingCostSource struct { + rateLimiter *rate.Limiter + config *githubbillingconfig.GitHubBillingConfig + client HTTPClient +} + +type HTTPClient interface { + Do(req *http.Request) (*http.Response, error) +} + +func (g *GitHubBillingCostSource) GetCustomCosts(req *pb.CustomCostRequest) []*pb.CustomCostResponse { + results := []*pb.CustomCostResponse{} + + targets, err := opencost.GetWindows(req.Start.AsTime(), req.End.AsTime(), req.Resolution.AsDuration()) + if err != nil { + log.Errorf("error getting windows: %v", err) + errResp := pb.CustomCostResponse{ + Errors: []string{fmt.Sprintf("error getting windows: %v", err)}, + } + results = append(results, &errResp) + return results + } + + for _, target := range targets { + if target.Start().After(time.Now().UTC()) { + log.Debugf("skipping future window %v", target) + continue + } + + log.Debugf("fetching GitHub billing costs for window %v", target) + result := g.getGitHubBillingCostsForWindow(target) + results = append(results, result) + } + + return results +} + +func main() { + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + githubCfg, err := githubbillingconfig.GetGitHubBillingConfig(configFile) + if err != nil { + log.Fatalf("error building GitHub Billing config: %v", err) + } + log.SetLogLevel(githubCfg.LogLevel) + + // GitHub API rate limit: 5000 requests per hour for authenticated users + rateLimiter := rate.NewLimiter(1, 3) + githubCostSrc := GitHubBillingCostSource{ + rateLimiter: rateLimiter, + config: githubCfg, + client: &http.Client{}, + } + + var pluginMap = map[string]plugin.Plugin{ + "CustomCostSource": &ocplugin.CustomCostPlugin{Impl: &githubCostSrc}, + } + + plugin.Serve(&plugin.ServeConfig{ + HandshakeConfig: handshakeConfig, + Plugins: pluginMap, + GRPCServer: plugin.DefaultGRPCServer, + }) +} + +func boilerplateGitHubBillingCustomCost(win opencost.Window) pb.CustomCostResponse { + return pb.CustomCostResponse{ + Metadata: map[string]string{"api_client_version": "v1"}, + CostSource: "dev_tools", + Domain: "github", + Version: "v1", + Currency: "USD", + Start: timestamppb.New(*win.Start()), + End: timestamppb.New(*win.End()), + Errors: []string{}, + Costs: []*pb.CustomCost{}, + } +} + +func (g *GitHubBillingCostSource) getGitHubBillingCostsForWindow(window opencost.Window) *pb.CustomCostResponse { + ccResp := boilerplateGitHubBillingCustomCost(window) + + year := window.Start().Year() + month := int(window.Start().Month()) + + // Fetch standard usage billing + usageItems, err := g.getUsageBilling(year, month) + if err != nil { + ccResp.Errors = append(ccResp.Errors, fmt.Sprintf("error getting GitHub usage billing: %v", err)) + return &ccResp + } + + // Fetch premium request usage billing + premiumItems, err := g.getPremiumUsageBilling(year, month) + if err != nil { + // Premium usage may not be available for all orgs, log but don't fail + log.Debugf("could not fetch premium usage billing: %v", err) + } + + customCosts := []*pb.CustomCost{} + + // Process standard usage items + for _, item := range usageItems { + customCost := pb.CustomCost{ + AccountName: item.OrganizationName, + ChargeCategory: "Usage", + Description: fmt.Sprintf("GitHub %s (%s)", item.Product, item.SKU), + ResourceName: item.Product, + ResourceType: item.SKU, + Id: uuid.New().String(), + ProviderId: fmt.Sprintf("%s/%s/%s", item.OrganizationName, item.Product, item.SKU), + BilledCost: float32(item.NetAmount), + ListCost: float32(item.GrossAmount), + ListUnitPrice: float32(item.PricePerUnit), + UsageQuantity: float32(item.Quantity), + UsageUnit: item.UnitType, + Labels: map[string]string{}, + } + + if item.RepositoryName != "" { + customCost.Labels["repository_name"] = item.RepositoryName + } + if item.Date != "" { + customCost.Labels["date"] = item.Date + } + if item.DiscountAmount > 0 { + customCost.Labels["discount_amount"] = fmt.Sprintf("%.6f", item.DiscountAmount) + } + + customCosts = append(customCosts, &customCost) + } + + // Process premium usage items + for _, item := range premiumItems { + customCost := pb.CustomCost{ + AccountName: g.config.Organization, + ChargeCategory: "Usage", + Description: fmt.Sprintf("GitHub Premium %s (%s)", item.Product, item.Model), + ResourceName: item.Product, + ResourceType: item.SKU, + Id: uuid.New().String(), + ProviderId: fmt.Sprintf("%s/%s/%s", g.config.Organization, item.Product, item.Model), + BilledCost: float32(item.NetAmount), + ListCost: float32(item.GrossAmount), + ListUnitPrice: float32(item.PricePerUnit), + UsageQuantity: float32(item.NetQuantity), + UsageUnit: item.UnitType, + Labels: map[string]string{}, + } + + if item.Model != "" { + customCost.Labels["model"] = item.Model + } + if item.DiscountAmount > 0 { + customCost.Labels["discount_amount"] = fmt.Sprintf("%.6f", item.DiscountAmount) + } + + customCosts = append(customCosts, &customCost) + } + + ccResp.Costs = customCosts + return &ccResp +} + +func (g *GitHubBillingCostSource) getUsageBilling(year, month int) ([]githubbillingplugin.GitHubBillingUsageItem, error) { + url := fmt.Sprintf(githubBillingUsageURL, g.config.Organization) + url = fmt.Sprintf("%s?year=%d&month=%d", url, year, month) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", g.config.APIToken)) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", githubAPIVersion) + + err = g.rateLimiter.Wait(context.Background()) + if err != nil { + return nil, fmt.Errorf("rate limiter error: %v", err) + } + + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request to GitHub Billing API: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("GitHub Billing API returned status %d: %s", resp.StatusCode, string(body)) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %v", err) + } + + var usageResp githubbillingplugin.GitHubBillingUsageResponse + err = json.Unmarshal(body, &usageResp) + if err != nil { + return nil, fmt.Errorf("error unmarshalling GitHub Billing response: %v", err) + } + + return usageResp.UsageItems, nil +} + +func (g *GitHubBillingCostSource) getPremiumUsageBilling(year, month int) ([]githubbillingplugin.GitHubBillingPremiumUsageItem, error) { + url := fmt.Sprintf(githubBillingPremiumURL, g.config.Organization) + url = fmt.Sprintf("%s?year=%d&month=%d", url, year, month) + + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, fmt.Errorf("error creating request: %v", err) + } + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", g.config.APIToken)) + req.Header.Set("Accept", "application/vnd.github+json") + req.Header.Set("X-GitHub-Api-Version", githubAPIVersion) + + err = g.rateLimiter.Wait(context.Background()) + if err != nil { + return nil, fmt.Errorf("rate limiter error: %v", err) + } + + resp, err := g.client.Do(req) + if err != nil { + return nil, fmt.Errorf("error making request to GitHub Premium Billing API: %v", err) + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + body, _ := io.ReadAll(resp.Body) + return nil, fmt.Errorf("GitHub Premium Billing API returned status %d: %s", resp.StatusCode, string(body)) + } + + body, err := io.ReadAll(resp.Body) + if err != nil { + return nil, fmt.Errorf("error reading response body: %v", err) + } + + var premiumResp githubbillingplugin.GitHubBillingPremiumUsageResponse + err = json.Unmarshal(body, &premiumResp) + if err != nil { + return nil, fmt.Errorf("error unmarshalling GitHub Premium Billing response: %v", err) + } + + return premiumResp.UsageItems, nil +} diff --git a/pkg/plugins/githubbilling/cmd/main/main_test.go b/pkg/plugins/githubbilling/cmd/main/main_test.go new file mode 100644 index 0000000..2cf1a76 --- /dev/null +++ b/pkg/plugins/githubbilling/cmd/main/main_test.go @@ -0,0 +1,123 @@ +package main + +import ( + "encoding/json" + "testing" + + githubbillingplugin "github.com/opencost/opencost-plugins/pkg/plugins/githubbilling/githubbillingplugin" +) + +func TestGitHubBillingUsageItemUnmarshal(t *testing.T) { + jsonData := `{ + "date": "2024-01-15", + "product": "Actions", + "sku": "Actions - Ubuntu", + "quantity": 5000, + "unitType": "minutes", + "pricePerUnit": 0.008, + "grossAmount": 40.0, + "discountAmount": 10.0, + "netAmount": 30.0, + "organizationName": "my-org", + "repositoryName": "my-repo" + }` + + var item githubbillingplugin.GitHubBillingUsageItem + err := json.Unmarshal([]byte(jsonData), &item) + if err != nil { + t.Fatalf("Error unmarshalling usage item: %v", err) + } + + if item.Product != "Actions" { + t.Errorf("Expected product 'Actions', got '%s'", item.Product) + } + if item.Quantity != 5000 { + t.Errorf("Expected quantity 5000, got %d", item.Quantity) + } + if item.NetAmount != 30.0 { + t.Errorf("Expected net amount 30.0, got %f", item.NetAmount) + } + if item.RepositoryName != "my-repo" { + t.Errorf("Expected repository name 'my-repo', got '%s'", item.RepositoryName) + } +} + +func TestGitHubBillingUsageResponseUnmarshal(t *testing.T) { + jsonData := `{ + "usageItems": [ + { + "date": "2024-01-15", + "product": "Actions", + "sku": "Actions - Ubuntu", + "quantity": 5000, + "unitType": "minutes", + "pricePerUnit": 0.008, + "grossAmount": 40.0, + "discountAmount": 10.0, + "netAmount": 30.0, + "organizationName": "my-org", + "repositoryName": "my-repo" + }, + { + "date": "2024-01-15", + "product": "Packages", + "sku": "Packages - Storage", + "quantity": 100, + "unitType": "GB", + "pricePerUnit": 0.25, + "grossAmount": 25.0, + "discountAmount": 0.0, + "netAmount": 25.0, + "organizationName": "my-org" + } + ] + }` + + var resp githubbillingplugin.GitHubBillingUsageResponse + err := json.Unmarshal([]byte(jsonData), &resp) + if err != nil { + t.Fatalf("Error unmarshalling usage response: %v", err) + } + + if len(resp.UsageItems) != 2 { + t.Errorf("Expected 2 usage items, got %d", len(resp.UsageItems)) + } + if resp.UsageItems[0].Product != "Actions" { + t.Errorf("Expected first product 'Actions', got '%s'", resp.UsageItems[0].Product) + } + if resp.UsageItems[1].Product != "Packages" { + t.Errorf("Expected second product 'Packages', got '%s'", resp.UsageItems[1].Product) + } +} + +func TestGitHubBillingPremiumUsageItemUnmarshal(t *testing.T) { + jsonData := `{ + "product": "Copilot", + "sku": "copilot-business", + "model": "gpt-4", + "unitType": "requests", + "pricePerUnit": 0.01, + "grossQuantity": 1000, + "grossAmount": 10.0, + "discountQuantity": 200, + "discountAmount": 2.0, + "netQuantity": 800, + "netAmount": 8.0 + }` + + var item githubbillingplugin.GitHubBillingPremiumUsageItem + err := json.Unmarshal([]byte(jsonData), &item) + if err != nil { + t.Fatalf("Error unmarshalling premium usage item: %v", err) + } + + if item.Product != "Copilot" { + t.Errorf("Expected product 'Copilot', got '%s'", item.Product) + } + if item.Model != "gpt-4" { + t.Errorf("Expected model 'gpt-4', got '%s'", item.Model) + } + if item.NetAmount != 8.0 { + t.Errorf("Expected net amount 8.0, got %f", item.NetAmount) + } +} diff --git a/pkg/plugins/githubbilling/cmd/validator/main/main.go b/pkg/plugins/githubbilling/cmd/validator/main/main.go new file mode 100644 index 0000000..c778bf6 --- /dev/null +++ b/pkg/plugins/githubbilling/cmd/validator/main/main.go @@ -0,0 +1,41 @@ +package main + +import ( + "fmt" + "os" + + commonconfig "github.com/opencost/opencost-plugins/pkg/common/config" + githubbillingconfig "github.com/opencost/opencost-plugins/pkg/plugins/githubbilling/config" + "github.com/opencost/opencost/core/pkg/log" +) + +func main() { + configFile, err := commonconfig.GetConfigFilePath() + if err != nil { + log.Fatalf("error opening config file: %v", err) + } + + githubCfg, err := githubbillingconfig.GetGitHubBillingConfig(configFile) + if err != nil { + log.Fatalf("error building GitHub Billing config: %v", err) + } + + if githubCfg.APIToken == "" { + log.Fatalf("github_api_token is required in config file") + } + if githubCfg.Organization == "" { + log.Fatalf("github_organization is required in config file") + } + + fmt.Printf("GitHub Billing config validated successfully\n") + fmt.Printf("Organization: %s\n", githubCfg.Organization) + fmt.Printf("Log level: %s\n", githubCfg.LogLevel) + fmt.Printf("\nSample config:\n") + fmt.Printf(`{ + "github_api_token": "ghp_YOUR_TOKEN", + "github_organization": "your-org-name", + "github_billing_plugin_log_level": "info" +} +`) + os.Exit(0) +} diff --git a/pkg/plugins/githubbilling/config/githubbillingconfig.go b/pkg/plugins/githubbilling/config/githubbillingconfig.go new file mode 100644 index 0000000..ceed9a9 --- /dev/null +++ b/pkg/plugins/githubbilling/config/githubbillingconfig.go @@ -0,0 +1,31 @@ +package config + +import ( + "encoding/json" + "fmt" + "os" +) + +type GitHubBillingConfig struct { + APIToken string `json:"github_api_token"` + Organization string `json:"github_organization"` + LogLevel string `json:"github_billing_plugin_log_level"` +} + +func GetGitHubBillingConfig(configFilePath string) (*GitHubBillingConfig, error) { + var result GitHubBillingConfig + bytes, err := os.ReadFile(configFilePath) + if err != nil { + return nil, fmt.Errorf("error reading config file for GitHub Billing config @ %s: %v", configFilePath, err) + } + err = json.Unmarshal(bytes, &result) + if err != nil { + return nil, fmt.Errorf("error marshaling json into GitHub Billing config %v", err) + } + + if result.LogLevel == "" { + result.LogLevel = "info" + } + + return &result, nil +} diff --git a/pkg/plugins/githubbilling/githubbillingplugin/githubbilling.go b/pkg/plugins/githubbilling/githubbillingplugin/githubbilling.go new file mode 100644 index 0000000..397ab1a --- /dev/null +++ b/pkg/plugins/githubbilling/githubbillingplugin/githubbilling.go @@ -0,0 +1,53 @@ +package githubbillingplugin + +// GitHubBillingUsageItem represents a single usage item from the GitHub Billing API. +type GitHubBillingUsageItem struct { + Date string `json:"date,omitempty"` + Product string `json:"product,omitempty"` + SKU string `json:"sku,omitempty"` + Quantity int64 `json:"quantity,omitempty"` + UnitType string `json:"unitType,omitempty"` + PricePerUnit float64 `json:"pricePerUnit,omitempty"` + GrossAmount float64 `json:"grossAmount,omitempty"` + DiscountAmount float64 `json:"discountAmount,omitempty"` + NetAmount float64 `json:"netAmount,omitempty"` + OrganizationName string `json:"organizationName,omitempty"` + RepositoryName string `json:"repositoryName,omitempty"` +} + +// GitHubBillingUsageResponse represents the response from the GitHub Billing usage API. +type GitHubBillingUsageResponse struct { + UsageItems []GitHubBillingUsageItem `json:"usageItems,omitempty"` +} + +// GitHubBillingPremiumUsageItem represents a premium request usage item. +type GitHubBillingPremiumUsageItem struct { + Product string `json:"product,omitempty"` + SKU string `json:"sku,omitempty"` + Model string `json:"model,omitempty"` + UnitType string `json:"unitType,omitempty"` + PricePerUnit float64 `json:"pricePerUnit,omitempty"` + GrossQuantity float64 `json:"grossQuantity,omitempty"` + GrossAmount float64 `json:"grossAmount,omitempty"` + DiscountQuantity float64 `json:"discountQuantity,omitempty"` + DiscountAmount float64 `json:"discountAmount,omitempty"` + NetQuantity float64 `json:"netQuantity,omitempty"` + NetAmount float64 `json:"netAmount,omitempty"` +} + +// GitHubBillingPremiumUsageResponse represents the response from the premium request usage API. +type GitHubBillingPremiumUsageResponse struct { + TimePeriod GitHubBillingTimePeriod `json:"timePeriod,omitempty"` + Organization string `json:"organization,omitempty"` + User string `json:"user,omitempty"` + Product string `json:"product,omitempty"` + Model string `json:"model,omitempty"` + UsageItems []GitHubBillingPremiumUsageItem `json:"usageItems,omitempty"` +} + +// GitHubBillingTimePeriod represents a time period in billing responses. +type GitHubBillingTimePeriod struct { + Year int32 `json:"year,omitempty"` + Month int32 `json:"month,omitempty"` + Day int32 `json:"day,omitempty"` +} diff --git a/pkg/plugins/githubbilling/go.mod b/pkg/plugins/githubbilling/go.mod new file mode 100644 index 0000000..b1ca810 --- /dev/null +++ b/pkg/plugins/githubbilling/go.mod @@ -0,0 +1,12 @@ +module github.com/opencost/opencost-plugins/pkg/plugins/githubbilling + +go 1.21 + +require ( + github.com/google/uuid v1.6.0 + github.com/hashicorp/go-plugin v1.6.1 + github.com/opencost/opencost-plugins/pkg/common v0.0.0 + github.com/opencost/opencost/core v0.0.0 + golang.org/x/time v0.5.0 + google.golang.org/protobuf v1.34.0 +)