Skip to content

Commit e49adb2

Browse files
KbayeroKbayero
committed
fix(installer,ci): fix OpenSearch memory starvation, dev CM misrouting, port v12 pipeline changes to v11
1 parent 8705a6b commit e49adb2

6 files changed

Lines changed: 218 additions & 61 deletions

File tree

‎.github/workflows/v11-deployment-pipeline.yml‎

Lines changed: 118 additions & 45 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@ jobs:
1616
environment: ${{ steps.set-env.outputs.environment }}
1717
cm_url: ${{ steps.set-env.outputs.cm_url }}
1818
event_processor_tag: ${{ steps.set-env.outputs.event_processor_tag }}
19+
gh_environment: ${{ steps.set-env.outputs.gh_environment }}
1920
steps:
2021
- name: Determine Build Environment
2122
id: set-env
@@ -34,7 +35,7 @@ jobs:
3435
BRANCH_VERSION=$(echo "${{ github.ref }}" | sed 's|refs/heads/release/||')
3536
echo "Branch version: $BRANCH_VERSION"
3637
37-
RESPONSE=$(curl -s "${CM_URL}/api/v1/versions/latest")
38+
RESPONSE=$(curl -s "${CM_URL}/api/v1/versions/latest?product=utmstack-v11-dev")
3839
LATEST_VERSION=$(echo "$RESPONSE" | jq -r '.version // empty')
3940
echo "Latest version from CM: $LATEST_VERSION"
4041
@@ -58,6 +59,7 @@ jobs:
5859
echo "environment=$ENVIRONMENT" >> $GITHUB_OUTPUT
5960
echo "cm_url=$CM_URL" >> $GITHUB_OUTPUT
6061
echo "event_processor_tag=${{ vars.TW_EVENT_PROCESSOR_VERSION_DEV }}" >> $GITHUB_OUTPUT
62+
echo "gh_environment=v11-dev" >> $GITHUB_OUTPUT
6163
6264
# =====================================================================
6365
# PRODUCTION — release.published (prerelease or not)
@@ -82,6 +84,7 @@ jobs:
8284
echo "environment=production" >> $GITHUB_OUTPUT
8385
echo "cm_url=$CM_URL" >> $GITHUB_OUTPUT
8486
echo "event_processor_tag=${{ vars.TW_EVENT_PROCESSOR_VERSION_PROD }}" >> $GITHUB_OUTPUT
87+
echo "gh_environment=v11-prod" >> $GITHUB_OUTPUT
8588
fi
8689
8790
build_agent:
@@ -512,6 +515,7 @@ jobs:
512515
name: Publish New Version to Customer Manager
513516
needs: [all_builds_complete, generate_changelog, setup_deployment]
514517
if: ${{ always() && needs.all_builds_complete.result == 'success' && needs.setup_deployment.outputs.tag != '' }}
518+
environment: ${{ needs.setup_deployment.outputs.gh_environment }}
515519
runs-on: ubuntu-24.04
516520
steps:
517521
- name: Check out code
@@ -523,6 +527,7 @@ jobs:
523527
ENVIRONMENT: ${{ needs.setup_deployment.outputs.environment }}
524528
TAG: ${{ needs.setup_deployment.outputs.tag }}
525529
CM_URL: ${{ needs.setup_deployment.outputs.cm_url }}
530+
SCHEDULE_INSTANCES: ${{ vars.SCHEDULE_INSTANCES }}
526531
run: |
527532
# Use AI changelog for production releases, generic for dev.
528533
if [ "$ENVIRONMENT" != "dev" ] && [ -n "$CHANGELOG_CONTENT" ]; then
@@ -535,87 +540,155 @@ jobs:
535540
echo "CM URL: $CM_URL"
536541
echo "Tag: $TAG"
537542
538-
# Select CM_SERVICE_ACCOUNT based on environment
539543
if [ "$ENVIRONMENT" = "dev" ]; then
540-
cmAuth=$(echo '${{ secrets.CM_SERVICE_ACCOUNT_DEV }}' | jq -r '.')
544+
cmAuth='${{ secrets.CM_SERVICE_ACCOUNT_DEV }}'
541545
else
542-
cmAuth=$(echo '${{ secrets.CM_SERVICE_ACCOUNT_PROD }}' | jq -r '.')
546+
cmAuth='${{ secrets.CM_SERVICE_ACCOUNT_PROD }}'
543547
fi
544-
545548
id=$(echo "$cmAuth" | jq -r '.id')
546549
key=$(echo "$cmAuth" | jq -r '.key')
547550
548-
body=$(jq -n \
549-
--arg version "$TAG" \
550-
--arg changelog "$changelog" \
551-
'{version: $version, changelog: $changelog}'
552-
)
551+
# CM-dev already has the products schema (same as v12). CM-prod
552+
# doesn't yet, so production keeps registering one version with no
553+
# product_id, exactly as before -- only dev goes through the
554+
# per-product loop.
555+
if [ "$ENVIRONMENT" = "dev" ]; then
556+
if ! echo "$SCHEDULE_INSTANCES" | jq empty > /dev/null 2>&1; then
557+
echo "❌ SCHEDULE_INSTANCES is not valid JSON: $SCHEDULE_INSTANCES"
558+
exit 1
559+
fi
560+
561+
for product_id in $(echo "$SCHEDULE_INSTANCES" | jq -r 'keys[]'); do
562+
echo "Registering $TAG for product: $product_id"
563+
564+
body=$(jq -n \
565+
--arg version "$TAG" \
566+
--arg changelog "$changelog" \
567+
--arg product_id "$product_id" \
568+
'{version: $version, changelog: $changelog, product_id: $product_id}'
569+
)
570+
571+
response=$(curl -s -X POST "${CM_URL}/api/v1/versions/register" \
572+
-H "Content-Type: application/json" \
573+
-H "id: $id" \
574+
-H "key: $key" \
575+
-d "$body")
553576
554-
response=$(curl -s -X POST "${CM_URL}/api/v1/versions/register" \
555-
-H "Content-Type: application/json" \
556-
-H "id: $id" \
557-
-H "key: $key" \
558-
-d "$body")
577+
echo "Response: $response"
578+
done
579+
else
580+
body=$(jq -n \
581+
--arg version "$TAG" \
582+
--arg changelog "$changelog" \
583+
'{version: $version, changelog: $changelog}'
584+
)
585+
586+
response=$(curl -s -X POST "${CM_URL}/api/v1/versions/register" \
587+
-H "Content-Type: application/json" \
588+
-H "id: $id" \
589+
-H "key: $key" \
590+
-d "$body")
559591
560-
echo "Response: $response"
592+
echo "Response: $response"
593+
fi
561594
562595
schedule:
563596
name: Schedule release to our instances
564597
needs: [publish_new_version, setup_deployment]
565598
if: ${{ always() && needs.publish_new_version.result == 'success' && needs.setup_deployment.outputs.tag != '' }}
599+
environment: ${{ needs.setup_deployment.outputs.gh_environment }}
566600
runs-on: ubuntu-24.04
567601
env:
568602
ENVIRONMENT: ${{ needs.setup_deployment.outputs.environment }}
569603
TAG: ${{ needs.setup_deployment.outputs.tag }}
570604
CM_URL: ${{ needs.setup_deployment.outputs.cm_url }}
605+
SCHEDULE_INSTANCES: ${{ vars.SCHEDULE_INSTANCES }}
606+
CM_AUTH_DEV: ${{ secrets.CM_SERVICE_ACCOUNT_DEV }}
607+
CM_AUTH_PROD: ${{ secrets.CM_SERVICE_ACCOUNT_PROD }}
571608
steps:
572609
- name: Schedule updates
573610
run: |
574611
echo "🔍 Environment: $ENVIRONMENT"
575612
echo "🔍 Version: $TAG"
576613
echo "🔍 CM URL: $CM_URL"
577614
578-
# Select instance IDs and auth based on environment
579615
if [ "$ENVIRONMENT" = "dev" ]; then
580-
instance_ids="${{ vars.SCHEDULE_INSTANCES_DEV }}"
581-
auth_json='${{ secrets.CM_SERVICE_ACCOUNT_DEV }}'
616+
auth_json="$CM_AUTH_DEV"
582617
else
583-
instance_ids="${{ vars.SCHEDULE_INSTANCES_PROD }}"
584-
auth_json='${{ secrets.CM_SERVICE_ACCOUNT_PROD }}'
618+
auth_json="$CM_AUTH_PROD"
585619
fi
586-
587-
# Extract id and key from auth JSON
588620
auth_id=$(echo "$auth_json" | jq -r '.id')
589621
auth_key=$(echo "$auth_json" | jq -r '.key')
590622
591-
# Parse IDs (handle single ID or comma-separated IDs)
592-
IFS=',' read -ra ID_ARRAY <<< "$instance_ids"
623+
scheduled=0
593624
594-
# Iterate over each instance ID
595-
for instance_id in "${ID_ARRAY[@]}"; do
596-
instance_id=$(echo "$instance_id" | xargs)
625+
# CM-dev has the products schema (same as v12). CM-prod doesn't
626+
# yet, so production keeps scheduling against the flat instance
627+
# list with no product_id, exactly as before -- only dev goes
628+
# through the per-product loop.
629+
if [ "$ENVIRONMENT" = "dev" ]; then
630+
if ! echo "$SCHEDULE_INSTANCES" | jq empty > /dev/null 2>&1; then
631+
echo "❌ SCHEDULE_INSTANCES is not valid JSON: $SCHEDULE_INSTANCES"
632+
exit 1
633+
fi
597634
598-
echo "📅 Scheduling release for instance: $instance_id"
635+
for product_id in $(echo "$SCHEDULE_INSTANCES" | jq -r 'keys[]'); do
636+
for instance_id in $(echo "$SCHEDULE_INSTANCES" | jq -r --arg p "$product_id" '.[$p][]'); do
637+
echo "📅 Scheduling release for instance: $instance_id (product: $product_id)"
638+
639+
response=$(curl -s -w "\n%{http_code}" -X POST "${CM_URL}/api/v1/updates" \
640+
-H "Content-Type: application/json" \
641+
-H "id: $auth_id" \
642+
-H "key: $auth_key" \
643+
-d "{\"instances_ids\": [\"$instance_id\"], \"version\": \"$TAG\", \"product_id\": \"$product_id\"}")
644+
645+
http_code=$(echo "$response" | tail -n1)
646+
body=$(echo "$response" | sed '$d')
647+
648+
if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
649+
echo "✅ Successfully scheduled for instance: $instance_id"
650+
scheduled=$((scheduled + 1))
651+
else
652+
echo "❌ Failed to schedule for instance: $instance_id (HTTP $http_code)"
653+
echo "Response: $body"
654+
exit 1
655+
fi
656+
done
657+
done
658+
else
659+
IFS=',' read -ra ID_ARRAY <<< "$SCHEDULE_INSTANCES"
599660
600-
response=$(curl -s -w "\n%{http_code}" -X POST "${CM_URL}/api/v1/updates" \
601-
-H "Content-Type: application/json" \
602-
-H "id: $auth_id" \
603-
-H "key: $auth_key" \
604-
-d "{\"instances_ids\": [\"$instance_id\"], \"version\": \"$TAG\"}")
661+
for instance_id in "${ID_ARRAY[@]}"; do
662+
instance_id=$(echo "$instance_id" | xargs)
605663
606-
http_code=$(echo "$response" | tail -n1)
607-
body=$(echo "$response" | sed '$d')
664+
echo "📅 Scheduling release for instance: $instance_id"
608665
609-
if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
610-
echo "✅ Successfully scheduled for instance: $instance_id"
611-
else
612-
echo "❌ Failed to schedule for instance: $instance_id (HTTP $http_code)"
613-
echo "Response: $body"
614-
exit 1
615-
fi
616-
done
666+
response=$(curl -s -w "\n%{http_code}" -X POST "${CM_URL}/api/v1/updates" \
667+
-H "Content-Type: application/json" \
668+
-H "id: $auth_id" \
669+
-H "key: $auth_key" \
670+
-d "{\"instances_ids\": [\"$instance_id\"], \"version\": \"$TAG\"}")
671+
672+
http_code=$(echo "$response" | tail -n1)
673+
body=$(echo "$response" | sed '$d')
674+
675+
if [ "$http_code" -ge 200 ] && [ "$http_code" -lt 300 ]; then
676+
echo "✅ Successfully scheduled for instance: $instance_id"
677+
scheduled=$((scheduled + 1))
678+
else
679+
echo "❌ Failed to schedule for instance: $instance_id (HTTP $http_code)"
680+
echo "Response: $body"
681+
exit 1
682+
fi
683+
done
684+
fi
685+
686+
if [ "$scheduled" -eq 0 ]; then
687+
echo "❌ No instances were scheduled — check SCHEDULE_INSTANCES for this environment"
688+
exit 1
689+
fi
617690
618-
echo "✅ Scheduled release for all instances with version $TAG"
691+
echo "✅ Scheduled release for all $scheduled instance(s) with version $TAG"
619692
620693
621694

‎installer/config/const.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,7 @@ var (
4040
VersionFilePath = filepath.Join(GetConfig().UpdatesFolder, "version.json")
4141
LicenseFilePath = filepath.Join(GetConfig().UpdatesFolder, "LICENSE")
4242
PendingUpdatesPath = filepath.Join(GetConfig().UpdatesFolder, "pending-updates.json")
43+
MemoryAllocationPath = filepath.Join(GetConfig().UpdatesFolder, "memory-allocation.json")
4344
LastAdminEmailPath = filepath.Join(GetConfig().UpdatesFolder, "last-admin-email.txt")
4445
EventProcessorLogsPath = filepath.Join(GetConfig().DataDir, "events-engine-workdir", "logs")
4546
CheckUpdatesEvery = 5 * time.Minute
@@ -49,7 +50,7 @@ var (
4950

5051
func GetCMServer() string {
5152
cnf := GetConfig()
52-
if cnf.Branch == "alpha" {
53+
if cnf.Branch == "dev" {
5354
return "https://cm.dev.utmstack.com"
5455
}
5556
return "https://cm.utmstack.com"

‎installer/docker/stack.go‎

Lines changed: 55 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,6 @@ func GetStackConfig() *StackConfig {
4545
os.Exit(1)
4646
}
4747

48-
mem := sigar.Mem{}
49-
err = mem.Get()
50-
if err != nil {
51-
fmt.Printf("error getting memory: %v\n", err)
52-
os.Exit(1)
53-
}
54-
5548
stackConfig = &StackConfig{}
5649
stackConfig.Threads = cores
5750
stackConfig.Cert = utils.MakeDir(0777, cnf.DataDir, "cert")
@@ -64,30 +57,78 @@ func GetStackConfig() *StackConfig {
6457
stackConfig.ShmFolder = utils.MakeDir(0777, cnf.DataDir, "tmpfs")
6558

6659
Services = []system.ServiceConfig{
67-
{Name: "event-processor", Priority: 1, MinMemory: 4 * 1024, MaxMemory: 60 * 1024},
68-
{Name: "opensearch", Priority: 1, MinMemory: 4350, MaxMemory: 60 * 1024},
69-
{Name: "backend", Priority: 2, MinMemory: 700, MaxMemory: 2 * 1024},
70-
{Name: "web-pdf", Priority: 2, MinMemory: 1024, MaxMemory: 2 * 1024},
60+
{Name: "event-processor", Priority: 1, MinMemory: 5120, MaxMemory: 60 * 1024},
61+
{Name: "opensearch", Priority: 1, MinMemory: 5120, MaxMemory: 60 * 1024},
62+
{Name: "backend", Priority: 3, MinMemory: 700, MaxMemory: 2 * 1024},
63+
{Name: "web-pdf", Priority: 3, MinMemory: 1024, MaxMemory: 2 * 1024},
7164
{Name: "postgres", Priority: 2, MinMemory: 500, MaxMemory: 2 * 1024},
7265
{Name: "user-auditor", Priority: 3, MinMemory: 200, MaxMemory: 1024},
7366
{Name: "agentmanager", Priority: 3, MinMemory: 200, MaxMemory: 1024},
7467
{Name: "frontend", Priority: 3, MinMemory: 80, MaxMemory: 1024},
7568
}
7669

77-
total := int(mem.Total/1024/1024) - system.SYSTEM_RESERVED_MEMORY
70+
if rsrcs, ok := loadPersistedMemoryAllocation(); ok {
71+
stackConfig.ServiceResources = rsrcs
72+
return
73+
}
7874

79-
rsrcs, err := system.BalanceMemory(Services, total)
75+
rsrcs, err := balanceMemoryNow()
8076
if err != nil {
8177
fmt.Printf("error balancing memory: %v\n", err)
8278
os.Exit(1)
8379
}
84-
8580
stackConfig.ServiceResources = rsrcs
8681
})
8782

8883
return stackConfig
8984
}
9085

86+
func balanceMemoryNow() (map[string]*system.ServiceConfig, error) {
87+
mem := sigar.Mem{}
88+
if err := mem.Get(); err != nil {
89+
return nil, fmt.Errorf("error getting memory: %v", err)
90+
}
91+
92+
total := int(mem.Total / 1024 / 1024)
93+
94+
rsrcs, err := system.BalanceMemory(Services, total)
95+
if err != nil {
96+
return nil, fmt.Errorf("error balancing memory: %v", err)
97+
}
98+
99+
if err := utils.WriteJSON(config.MemoryAllocationPath, rsrcs); err != nil {
100+
fmt.Printf("warning: could not persist memory allocation: %v\n", err)
101+
}
102+
103+
return rsrcs, nil
104+
}
105+
106+
func loadPersistedMemoryAllocation() (map[string]*system.ServiceConfig, bool) {
107+
if !utils.CheckIfPathExist(config.MemoryAllocationPath) {
108+
return nil, false
109+
}
110+
111+
var rsrcs map[string]*system.ServiceConfig
112+
if err := utils.ReadJson(config.MemoryAllocationPath, &rsrcs); err != nil {
113+
fmt.Printf("warning: could not read persisted memory allocation, recalculating: %v\n", err)
114+
return nil, false
115+
}
116+
117+
return rsrcs, true
118+
}
119+
120+
func RecalculateMemory() error {
121+
stack := GetStackConfig() // ensures Services and directories are set up
122+
123+
rsrcs, err := balanceMemoryNow()
124+
if err != nil {
125+
return err
126+
}
127+
128+
stack.ServiceResources = rsrcs
129+
return nil
130+
}
131+
91132
func StackUP(tag string) error {
92133
var compose = new(Compose)
93134
err := compose.Populate(config.GetConfig(), GetStackConfig())

‎installer/main.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,6 +40,13 @@ func main() {
4040
os.Exit(1)
4141
}
4242

43+
case "--recalculate-memory":
44+
err := RecalculateMemory()
45+
if err != nil {
46+
fmt.Printf("\nerror recalculating memory: %v", err)
47+
os.Exit(1)
48+
}
49+
4350
default:
4451
help()
4552
}
@@ -60,4 +67,5 @@ func help() {
6067
fmt.Println(" --install, -i Install UTMStack")
6168
fmt.Println(" --uninstall, -u Uninstall UTMStack")
6269
fmt.Println(" --version, -v Show UTMStack version")
70+
fmt.Println(" --recalculate-memory Recalculate container memory limits from current VM resources and redeploy")
6371
}

0 commit comments

Comments
 (0)