From da340163ba3ca1c4caf35e13d70f89ff06a45ce6 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Wed, 12 Nov 2025 09:17:54 +0100 Subject: [PATCH 01/33] Deploy coop maps --- .../scripts/deploy-coop-maps.py | 381 ++++++++++++++++++ .../templates/deploy-coop-maps.yaml | 56 +++ 2 files changed, 437 insertions(+) create mode 100644 apps/faf-legacy-deployment/scripts/deploy-coop-maps.py create mode 100644 apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml diff --git a/apps/faf-legacy-deployment/scripts/deploy-coop-maps.py b/apps/faf-legacy-deployment/scripts/deploy-coop-maps.py new file mode 100644 index 00000000..b28badef --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/deploy-coop-maps.py @@ -0,0 +1,381 @@ +#!/usr/bin/python3 +# -*- coding: utf-8 -*- + +""" +clone: https://github.com/FAForever/faf-coop-maps + +FAF coop maps updater + +All default settings are setup for FAF production! +Override the directory settings for local testing. +To get more help run + $ pipenv run patch-coop-maps -h + +Default usage: + $ pipenv run patch-coop-maps -s +""" +import argparse +import hashlib +import logging +import os +import shutil +import subprocess +import sys +import zipfile +from tempfile import TemporaryDirectory +from typing import NamedTuple, List + +import mysql.connector + +logger: logging.Logger = logging.getLogger() +logger.setLevel(logging.DEBUG) + +fixed_file_timestamp = 1078100502 # 2004-03-01T00:21:42Z + + +db_config = { + "host": os.getenv("DATABASE_HOST", "localhost"), + "user": os.getenv("DATABASE_USERNAME", "root"), + "password": os.getenv("DATABASE_PASSWORD", "banana"), + "database": os.getenv("DATABASE_NAME", "faf_lobby"), +} + + +def get_db_connection(): + """Create and return a MySQL connection.""" + try: + conn = mysql.connector.connect(**db_config) + if conn.is_connected(): + logger.debug(f"Connected to MySQL at {db_config['host']}") + return conn + except Error as e: + logger.error(f"MySQL connection failed: {e}") + sys.exit(1) + + +def run_sql(conn, sql: str) -> str: + """ + Run an SQL query directly on the MySQL database instead of via Docker. + Returns output in a string format similar to the old implementation. + """ + logger.debug(f"Executing SQL query:\n{sql}") + try: + with conn.cursor() as cursor: + cursor.execute(sql) + + # If it's a SELECT query, fetch and format results + if sql.strip().lower().startswith("select"): + rows = cursor.fetchall() + column_names = [desc[0] for desc in cursor.description] + # Simulate the Docker mysql CLI tabular text output + lines = ["\t".join(column_names)] + for row in rows: + lines.append("\t".join(str(x) for x in row)) + result = "\n".join(lines) + else: + conn.commit() + result = "Query OK" + + logger.debug(f"SQL result:\n{result}") + return result + + except Error as e: + logger.error(f"SQL execution failed: {e}") + sys.exit(1) + + +class CoopMap(NamedTuple): + folder_name: str + map_id: int + map_type: int + + def build_zip_filename(self, version: int) -> str: + return f"{self.folder_name.lower()}.v{version:04d}.zip" + + def build_folder_name(self, version: int) -> str: + return f"{self.folder_name.lower()}.v{version:04d}" + + +# Coop maps are in db table `coop_map` +coop_maps: List[CoopMap] = [ + # Forged Alliance missions + CoopMap("X1CA_Coop_001", 1, 0), + CoopMap("X1CA_Coop_002", 3, 0), + CoopMap("X1CA_Coop_003", 4, 0), + CoopMap("X1CA_Coop_004", 5, 0), + CoopMap("X1CA_Coop_005", 6, 0), + CoopMap("X1CA_Coop_006", 7, 0), + + # Vanilla Aeon missions + CoopMap("SCCA_Coop_A01", 8, 1), + CoopMap("SCCA_Coop_A02", 9, 1), + CoopMap("SCCA_Coop_A03", 10, 1), + CoopMap("SCCA_Coop_A04", 11, 1), + CoopMap("SCCA_Coop_A05", 12, 1), + CoopMap("SCCA_Coop_A06", 13, 1), + + # Vanilla Cybran missions + CoopMap("SCCA_Coop_R01", 20, 2), + CoopMap("SCCA_Coop_R02", 21, 2), + CoopMap("SCCA_Coop_R03", 22, 2), + CoopMap("SCCA_Coop_R04", 23, 2), + CoopMap("SCCA_Coop_R05", 24, 2), + CoopMap("SCCA_Coop_R06", 25, 2), + + # Vanilla UEF missions + CoopMap("SCCA_Coop_E01", 14, 3), + CoopMap("SCCA_Coop_E02", 15, 3), + CoopMap("SCCA_Coop_E03", 16, 3), + CoopMap("SCCA_Coop_E04", 17, 3), + CoopMap("SCCA_Coop_E05", 18, 3), + CoopMap("SCCA_Coop_E06", 19, 3), + + # Custom missions + CoopMap("FAF_Coop_Prothyon_16", 26, 4), + CoopMap("FAF_Coop_Fort_Clarke_Assault", 27, 4), + CoopMap("FAF_Coop_Theta_Civilian_Rescue", 28, 4), + CoopMap("FAF_Coop_Novax_Station_Assault", 31, 4), + CoopMap("FAF_Coop_Operation_Tha_Atha_Aez", 32, 4), + CoopMap("FAF_Coop_Havens_Invasion", 33, 4), + CoopMap("FAF_Coop_Operation_Rescue", 35, 4), + CoopMap("FAF_Coop_Operation_Uhthe_Thuum_QAI", 36, 4), + CoopMap("FAF_Coop_Operation_Yath_Aez", 37, 4), + CoopMap("FAF_Coop_Operation_Ioz_Shavoh_Kael", 38, 4), + CoopMap("FAF_Coop_Operation_Trident", 39, 4), + CoopMap("FAF_Coop_Operation_Blockade", 40, 4), + CoopMap("FAF_Coop_Operation_Golden_Crystals", 41, 4), + CoopMap("FAF_Coop_Operation_Holy_Raid", 42, 4), + CoopMap("FAF_Coop_Operation_Tight_Spot", 45, 4), + CoopMap("FAF_Coop_Operation_Overlord_Surth_Velsok", 47, 4), + CoopMap("FAF_Coop_Operation_Rebel's_Rest", 48, 4), + CoopMap("FAF_Coop_Operation_Red_Revenge", 49, 4), +] + +def fix_file_timestamps(files: List[str]) -> None: + for file in files: + logger.debug(f"Fixing timestamp in {file}") + os.utime(file, (fixed_file_timestamp, fixed_file_timestamp)) + + +def fix_folder_paths(folder_name: str, files: List[str], new_version: int) -> None: + old_maps_lua_path = f"/maps/{folder_name}/" + new_maps_lua_path = f"/maps/{folder_name.lower()}.v{new_version:04d}/" + + for file in files: + logger.debug(f"Fixing lua folder path in {file}: '{old_maps_lua_path}' -> '{new_maps_lua_path}'") + + with open(file, "rb") as file_handler: + data = file_handler.read() + data = data.replace(old_maps_lua_path.encode(), new_maps_lua_path.encode()) + + with open(file, "wb") as file_handler: + file_handler.seek(0) + file_handler.write(data) + + +def get_latest_map_version(coop_map: CoopMap) -> int: + logger.debug(f"Fetching latest map version for coop map {coop_map}") + + query = f""" + SELECT version FROM coop_map WHERE id = {coop_map.map_id}; + """ + result = run_sql(query).split("\n") + assert len(result) == 3, f"Mysql returned wrong result! Either map id {coop_map.map_id} is not in table coop_map" \ + f" or the where clause is wrong. Result: " + "\n".join(result) + return int(result[1]) + + +def new_file_is_different(old_file_name: str, new_file_name: str) -> bool: + old_file_md5 = calc_md5(old_file_name) + new_file_md5 = calc_md5(new_file_name) + + logger.debug(f"MD5 hash of {old_file_name} is: {old_file_md5}") + logger.debug(f"MD5 hash of {new_file_name} is: {new_file_md5}") + + return old_file_md5 != new_file_md5 + + +def update_database(coop_map: CoopMap, new_version: int) -> None: + logger.debug(f"Updating coop map {coop_map} in database to version {new_version}") + + query = f""" + UPDATE coop_map + SET version = {new_version}, filename = "maps/{coop_map.build_zip_filename(new_version)}" + WHERE id = {coop_map.map_id} + """ + run_sql(query) + + +def copytree(src, dst, symlinks=False, ignore=None): + """ + Reason for that method is because shutil.copytree will raise exception on existing + temporary directory + """ + + for item in os.listdir(src): + s = os.path.join(src, item) + d = os.path.join(dst, item) + if os.path.isdir(s): + shutil.copytree(s, d, symlinks, ignore) + else: + shutil.copy2(s, d) + + +def create_zip_package(coop_map: CoopMap, version: int, files: List[str], tmp_folder_path: str, zip_file_path: str): + fix_folder_paths(coop_map.folder_name, files, version) + fix_file_timestamps(files) + with zipfile.ZipFile(zip_file_path, 'w', zipfile.ZIP_BZIP2) as zip_file: + for path in files: + zip_file.write(path, arcname=f"/{coop_map.build_folder_name(version)}/{os.path.relpath(path, tmp_folder_path)}") + + +def process_coop_map(coop_map: CoopMap, simulate: bool, git_directory:str, coop_maps_path: str): + logger.info(f"Processing: {coop_map}") + + temp_dir = TemporaryDirectory() + copytree(os.path.join(git_directory, coop_map.folder_name), temp_dir.name) + processing_files = [] + for root, dirs, files in os.walk(temp_dir.name): + for f in files: + processing_files.append(os.path.relpath(os.path.join(root, f), temp_dir.name)) + + logger.debug(f"Files to process in {coop_map}: {processing_files}") + current_version = get_latest_map_version(coop_map) + current_file_path = os.path.join(coop_maps_path, coop_map.build_zip_filename(current_version)) + zip_file_path = os.path.join(temp_dir.name, coop_map.build_zip_filename(current_version)) + create_zip_package(coop_map, current_version, processing_files, temp_dir.name, zip_file_path) + if current_version == 0 or new_file_is_different(current_file_path, zip_file_path): + new_version = current_version + 1 + + if current_version == 0: + logger.info(f"{coop_map} first upload. New version: {new_version}") + else: + logger.info(f"{coop_map} has changed. New version: {new_version}") + + if not simulate: + temp_dir.cleanup() + temp_dir = TemporaryDirectory() + copytree(os.path.join(git_directory, coop_map.folder_name), temp_dir.name) + + zip_file_path = os.path.join(coop_maps_path, coop_map.build_zip_filename(new_version)) + create_zip_package(coop_map, new_version, processing_files, temp_dir.name, zip_file_path) + + update_database(coop_map, new_version) + else: + logger.info(f"Updating database skipped due to simulation") + else: + logger.info(f"{coop_map} remains unchanged") + temp_dir.cleanup() + + +def calc_md5(filename: str) -> str: + """ + Calculate the MD5 hash of a file + """ + hash_md5 = hashlib.md5() + with open(filename, "rb") as f: + for chunk in iter(lambda: f.read(4096), b""): + hash_md5.update(chunk) + return hash_md5.hexdigest() + + +def run_checked_shell(cmd: List[str]) -> subprocess.CompletedProcess: + """ + Runs a command as a shell process and checks for success + Output is captured in the result object + :param cmd: command to run + :return: CompletedProcess of the execution + """ + logger.debug("Run shell command: {cmd}".format(cmd=cmd)) + return subprocess.run(cmd, check=True, stdout=subprocess.PIPE) + + +def run_sql(sql: str, container: str = "faf-db", database: str = "faf_lobby") -> str: + + """ + Run a sql-query against the faf-db in the docker container + :param database: name of the database where to run the query + :param container: name of the docker container where to run the query + :param sql: the sql-query to run + :return: the query output as string + """ + try: + sql_text_result = run_checked_shell( + ["docker", "exec", "-u", "root", container, "mysql", database, "-e", sql] + ).stdout.decode() # type: str + logger.debug(f"SQL output >>> \n{sql_text_result}<<<") + return sql_text_result + except subprocess.CalledProcessError as e: + logger.error(f"""Executing sql query failed: {sql}\n\t\tError message: {str(e)}""") + exit(1) + + +def git_checkout(path: str, tag: str) -> None: + """ + Checkout a git tag of the git repository. This requires the repo to be checked out in the path folder! + + :param path: the path of the git repository to checkout + :param tag: version of the git tag (full name) + :return: nothing + """ + cwd = os.getcwd() + os.chdir(path) + logger.debug(f"Git checkout from path {path}") + + try: + run_checked_shell(["git", "fetch"]) + run_checked_shell(["git", "checkout", tag]) + except subprocess.CalledProcessError as e: + logger.error(f"git checkout failed - please check the error message: {e.stderr}") + exit(1) + finally: + os.chdir(cwd) + + +def create_zip(content: List[str], relative_to: str, output_file: str) -> None: + logger.debug(f"Zipping files to file `{output_file}`: {content}") + + with zipfile.ZipFile(output_file, 'w', zipfile.ZIP_DEFLATED) as zip_file: + for path in content: + if os.path.isdir(path): + cwd = os.getcwd() + os.chdir(path) + + for root, dirs, files in os.walk(path): + for next_file in files: + file_path = os.path.join(root, next_file) + zip_file.write(file_path, os.path.relpath(file_path, relative_to)) + + os.chdir(cwd) + else: + zip_file.write(path, os.path.relpath(path, relative_to)) + + +if __name__ == "__main__": + # Setting up logger + stream_handler = logging.StreamHandler(sys.stdout) + stream_handler.setFormatter(logging.Formatter('%(levelname)-5s - %(message)s')) + logger.addHandler(stream_handler) + + # Setting up CLI arguments + parser = argparse.ArgumentParser(description=__doc__) + + parser.add_argument("version", help="the git tag name of the version") + parser.add_argument("-s", "--simulate", dest="simulate", action="store_true", default=False, + help="only runs a simulation without updating the database") + parser.add_argument("--git-directory", dest="git_directory", action="store", + default="/opt/featured-mods/faf-coop-maps", + help="base directory of the faf-coop-maps repository") + parser.add_argument("--maps-directory", dest="coop_maps_path", action="store", + default="/opt/faf/data/maps", + help="directory of the coop map files (content server)") + + args = parser.parse_args() + + git_checkout(args.git_directory, args.version) + + for coop_map in coop_maps: + try: + process_coop_map(coop_map, args.simulate, args.git_directory, args.coop_maps_path) + except Exception as error: + logger.warning(f"Unable to parse {coop_map}", exc_info=True) diff --git a/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml b/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml new file mode 100644 index 00000000..37ee5092 --- /dev/null +++ b/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml @@ -0,0 +1,56 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: faf-deploy-coop-maps + labels: + app: faf-deploy-coop-maps +data: + PATCH_VERSION: "65" + DATABASE_HOST: "mariadb" + DATABASE_NAME: "faf_lobby" + "deploy-coop.py": |- +{{ tpl ( .Files.Get "scripts/deploy-coop-maps.py" ) . | indent 4 }} + +--- + +kind: CronJob +apiVersion: batch/v1 +metadata: + name: faf-deploy-coop-maps + namespace: faf-apps + labels: + app: faf-deploy-coop-maps +spec: + # Disabled because triggered manually + schedule: "0 0 31 2 *" + suspend: true + concurrencyPolicy: Forbid + jobTemplate: + metadata: + labels: + app: faf-deploy-coop-maps + annotations: + prometheus.io/scrape: 'false' + spec: + template: + spec: + containers: + - image: python:3.13 + imagePullPolicy: Always + name: faf-coop-deployment + envFrom: + - configMapRef: + name: faf-deploy-coop-maps + - secretRef: + name: faf-legacy-deployment + command: [ "sh" ] + args: [ "-c", "pip install mysql-connector-python && python3 /tmp/deploy-coop.py" ] + volumeMounts: + - mountPath: /tmp/deploy-coop.py + name: faf-deploy-coop-maps + subPath: "deploy-coop.py" + restartPolicy: Never + volumes: + - name: faf-deploy-coop-maps + configMap: + name: "faf-deploy-coop-maps" From 5cac9fa7ee54a7847adf52c492490b638caa9048 Mon Sep 17 00:00:00 2001 From: Pablo <42.pablo.ms@gmail.com> Date: Sun, 21 Sep 2025 17:33:20 +0200 Subject: [PATCH 02/33] Drop promtail --- ops/monitoring/Chart.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ops/monitoring/Chart.yaml b/ops/monitoring/Chart.yaml index a787d872..b48a5be9 100644 --- a/ops/monitoring/Chart.yaml +++ b/ops/monitoring/Chart.yaml @@ -11,3 +11,6 @@ dependencies: - name: k8s-monitoring version: 3.5.1 repository: https://grafana.github.io/helm-charts +- name: k8s-monitoring + version: 3.5.1 + repository: https://grafana.github.io/helm-charts From 9d56f008642c3e27d993e907363a772a8eb98af4 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Sun, 23 Nov 2025 10:30:24 +0100 Subject: [PATCH 03/33] Enable Cloudflare --- apps/faf-icebreaker/templates/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/faf-icebreaker/templates/config.yaml b/apps/faf-icebreaker/templates/config.yaml index 08bbfc4a..8b485953 100644 --- a/apps/faf-icebreaker/templates/config.yaml +++ b/apps/faf-icebreaker/templates/config.yaml @@ -13,7 +13,7 @@ data: RABBITMQ_HOST: "rabbitmq" RABBITMQ_USER: "faf-icebreaker" RABBITMQ_PORT: "5672" - CLOUDFLARE_ENABLED: "false" + CLOUDFLARE_ENABLED: "true" XIRSYS_ENABLED: "true" XIRSYS_TURN_ENABLED: "true" GEOIPUPDATE_EDITION_IDS: "GeoLite2-City" From 74c96db3616a80009cfb1bd87eb86c9b92d63477 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Tue, 9 Dec 2025 21:56:33 +0100 Subject: [PATCH 04/33] ChatGPT failed, lets see Gemini Flash at work --- .../scripts/deploy-coop.py | 31 ++++++++++++++----- 1 file changed, 23 insertions(+), 8 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/deploy-coop.py b/apps/faf-legacy-deployment/scripts/deploy-coop.py index 293d7568..1c1a42f2 100644 --- a/apps/faf-legacy-deployment/scripts/deploy-coop.py +++ b/apps/faf-legacy-deployment/scripts/deploy-coop.py @@ -113,31 +113,42 @@ def calc_md5(fname): return hash_md5.hexdigest() def zipdir(path, ziph): + """ + Adds a path (folder or file) to the zip file handle `ziph`. + Preserves the directory name in the archive. + """ if not os.path.exists(path): print(f"Warning: {path} does not exist, skipping") return - if os.path.isdir(path): - for root, dirs, files in os.walk(path): + # Remove trailing slash if present, otherwise dirname might return the path itself + clean_path = path.rstrip(os.sep) + + # Calculate the base directory (parent of the target path) + # This ensures that if we zip '/repo/lua', the archive entry starts with 'lua/...' + base_dir = os.path.dirname(clean_path) + + if os.path.isdir(clean_path): + for root, dirs, files in os.walk(clean_path): files.sort() # deterministic order dirs.sort() # deterministic order for file in files: full_path = os.path.join(root, file) - arcname = os.path.relpath(full_path, start=path) # preserve folder structure + # Relpath relative to base_dir (parent) ensures folder structure is kept + arcname = os.path.relpath(full_path, start=base_dir) info = zipfile.ZipInfo(arcname, FIXED_ZIP_TIMESTAMP) with open(full_path, "rb") as f: data = f.read() ziph.writestr(info, data, compress_type=zipfile.ZIP_DEFLATED) else: # single file outside a directory - arcname = os.path.basename(path) + arcname = os.path.relpath(clean_path, start=base_dir) info = zipfile.ZipInfo(arcname, FIXED_ZIP_TIMESTAMP) - with open(path, "rb") as f: + with open(clean_path, "rb") as f: data = f.read() ziph.writestr(info, data, compress_type=zipfile.ZIP_DEFLATED) - def create_file(conn, mod, fileId, version, name, source, target_dir, old_md5, dryrun): """Pack or copy files, compare MD5, update DB if changed.""" target_dir = os.path.join(target_dir, f"updates_{mod}_files") @@ -238,8 +249,12 @@ def download_vo_assets(version, target_dir): # 1. Get latest release JSON from GitHub api_url = f"https://api.github.com/repos/FAForever/fa-coop/releases/tags/v{version}" - with urllib.request.urlopen(api_url) as response: - release_info = json.load(response) + try: + with urllib.request.urlopen(api_url) as response: + release_info = json.load(response) + except urllib.error.HTTPError as e: + print(f"Failed to fetch release info: {e}") + return # 2. Filter assets ending with .nx2 nx2_urls = [ From c00f924a6d24e4e70575c9a15e9175a716c0e80c Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Wed, 10 Dec 2025 22:51:19 +0100 Subject: [PATCH 05/33] Go Kotlin --- .../scripts/deploy-coop.main.kts | 424 ++++++++++++++++++ 1 file changed, 424 insertions(+) create mode 100755 apps/faf-legacy-deployment/scripts/deploy-coop.main.kts diff --git a/apps/faf-legacy-deployment/scripts/deploy-coop.main.kts b/apps/faf-legacy-deployment/scripts/deploy-coop.main.kts new file mode 100755 index 00000000..6ff8ef6a --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/deploy-coop.main.kts @@ -0,0 +1,424 @@ +#!/usr/bin/env kotlin + +@file:DependsOn("com.mysql:mysql-connector-j:9.5.0") +@file:DependsOn("org.eclipse.jgit:org.eclipse.jgit:7.4.0.202509020913-r") +@file:DependsOn("com.squareup.okio:okio:3.16.4") + +import org.eclipse.jgit.api.Git +import java.io.IOException +import java.net.HttpURLConnection +import java.net.URL +import java.nio.file.Files +import java.nio.file.Path +import java.nio.file.Paths +import java.nio.file.StandardCopyOption +import java.nio.file.StandardOpenOption +import java.nio.file.attribute.PosixFilePermission +import java.security.MessageDigest +import java.sql.DriverManager +import java.util.zip.ZipEntry +import java.util.zip.ZipOutputStream +import kotlin.io.path.inputStream + +// --------------------------- CONFIG --------------------------- + +val PATCH_VERSION = System.getenv("PATCH_VERSION") ?: error("PATCH_VERSION required") +val REPO_URL = System.getenv("GIT_REPO_URL") ?: "https://github.com/FAForever/fa-coop.git" +val GIT_REF = System.getenv("GIT_REF") ?: "v$PATCH_VERSION" +val WORKDIR = System.getenv("GIT_WORKDIR") ?: "/tmp/fa-coop-kt" +val DRYRUN = (System.getenv("DRY_RUN") ?: "false").lowercase() in listOf("1", "true", "yes") + +val DB_HOST = System.getenv("DATABASE_HOST") ?: "localhost" +val DB_NAME = System.getenv("DATABASE_NAME") ?: "faf" +val DB_USER = System.getenv("DATABASE_USERNAME") ?: "root" +val DB_PASS = System.getenv("DATABASE_PASSWORD") ?: "banana" + +val TARGET_DIR = Paths.get("./legacy-featured-mod-files") +val VO_DOWNLOAD_TMP = Paths.get("/tmp", "vo_download_$PATCH_VERSION") + +// --------------------------- UTILS --------------------------- + +fun md5(path: Path): String { + val md = MessageDigest.getInstance("MD5") + path.inputStream().use { input -> + val buf = ByteArray(4096) + var r: Int + while (input.read(buf).also { r = it } != -1) { + md.update(buf, 0, r) + } + } + return md.digest().joinToString("") { "%02x".format(it) } +} + +fun Path.commonPath(other: Path): Path { + val a = this.toAbsolutePath().normalize() + val b = other.toAbsolutePath().normalize() + val min = minOf(a.nameCount, b.nameCount) + var i = 0 + while (i < min && a.getName(i) == b.getName(i)) i++ + return if (i == 0) a.root else a.root.resolve(a.subpath(0, i)) +} + +fun setPerm664(path: Path) { + try { + val perms = mutableSetOf().apply { + add(PosixFilePermission.OWNER_READ); add(PosixFilePermission.OWNER_WRITE) + add(PosixFilePermission.GROUP_READ); add(PosixFilePermission.GROUP_WRITE) + add(PosixFilePermission.OTHERS_READ) + } + Files.setPosixFilePermissions(path, perms) + } catch (e: Exception) { + println("Warning: couldn't set perms on $path: ${e.message}") + } +} + +// --------------------------- ZIP with preserved hierarchy --------------------------- + +fun zipPreserveStructure(sources: List, outputFile: Path, base: Path) { + Files.createDirectories(outputFile.parent) + ZipOutputStream(Files.newOutputStream(outputFile)).use { zos -> + for (src in sources) { + if (!Files.exists(src)) { + // skip + continue + } + if (Files.isDirectory(src)) { + Files.walk(src).use { stream -> + stream.filter { Files.isRegularFile(it) }.forEach { file -> + val arcname = base.relativize(file).toString().replace("\\", "/") + val entry = ZipEntry(arcname) + // fix timestamp for determinism (not strictly necessary) + entry.time = 315532800000L // 1980-01-01 00:00:00 UTC in ms + zos.putNextEntry(entry) + Files.newInputStream(file).use { inp -> inp.copyTo(zos) } + zos.closeEntry() + } + } + } else { + val arcname = base.relativize(src).toString().replace("\\", "/") + val entry = ZipEntry(arcname) + entry.time = 315532800000L + zos.putNextEntry(entry) + Files.newInputStream(src).use { inp -> inp.copyTo(zos) } + zos.closeEntry() + } + } + } +} + +// --------------------------- GIT --------------------------- + +fun prepareRepo(): Path { + val dir = Paths.get(WORKDIR) + if (Files.exists(dir.resolve(".git"))) { + println("Repo exists — fetching and checking out $GIT_REF...") + Git.open(dir.toFile()).use { git -> + git.fetch().call() + git.checkout().setName(GIT_REF).call() + } + } else { + println("Cloning $REPO_URL -> $dir ...") + Git.cloneRepository() + .setURI(REPO_URL) + .setDirectory(dir.toFile()) + .call() + Git.open(dir.toFile()).use { git -> + git.checkout().setName(GIT_REF).call() + } + } + return dir +} + +// --------------------------- GITHUB RELEASE ASSET DOWNLOAD --------------------------- + +fun httpGet(url: String): String { + val conn = URL(url).openConnection() as HttpURLConnection + conn.requestMethod = "GET" + conn.setRequestProperty("Accept", "application/vnd.github.v3+json") + conn.connectTimeout = 15000 + conn.readTimeout = 15000 + val code = conn.responseCode + val stream = if (code in 200..299) conn.inputStream else conn.errorStream + return stream.bufferedReader().use { it.readText() } +} + +fun downloadFile(url: String, dest: Path) { + val u = URL(url) + val conn = u.openConnection() as HttpURLConnection + conn.requestMethod = "GET" + conn.connectTimeout = 20000 + conn.readTimeout = 20000 + conn.instanceFollowRedirects = true + val code = conn.responseCode + if (code !in 200..299) { + throw IOException("Failed to download $url -> HTTP $code") + } + Files.createDirectories(dest.parent) + conn.inputStream.use { input -> + Files.newOutputStream(dest, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING).use { out -> + input.copyTo(out) + } + } +} + +fun findNx2UrlsFromReleaseJson(json: String): List { + // crude but effective: find all "browser_download_url": "..." and filter .nx2 + val regex = Regex("\"browser_download_url\"\\s*:\\s*\"([^\"]+)\"") + return regex.findAll(json).mapNotNull { it.groupValues.getOrNull(1) }.filter { it.endsWith(".nx2") }.toList() +} + +fun downloadVoAssets(version: String, targetDir: Path): List { + println("Downloading VO assets for version $version from GitHub releases...") + Files.createDirectories(VO_DOWNLOAD_TMP) + val apiUrl = "https://api.github.com/repos/FAForever/fa-coop/releases/tags/v$version" + val json = try { + httpGet(apiUrl) + } catch (e: Exception) { + println("Warning: failed to fetch release JSON: ${e.message}") + return emptyList() + } + + val urls = findNx2UrlsFromReleaseJson(json) + if (urls.isEmpty()) { + println("No .nx2 assets found in release v$version") + return emptyList() + } + + val downloaded = mutableListOf() + for (u in urls) { + val filename = Paths.get(URL(u).path).fileName.toString() + val dst = VO_DOWNLOAD_TMP.resolve(filename) + try { + println("Downloading $u -> $dst") + downloadFile(u, dst) + // rename to include .v{version}.nx2 before .nx2 + val newName = filename.replace(Regex("\\.nx2$"), ".v$version.nx2") + val newPath = VO_DOWNLOAD_TMP.resolve(newName) + Files.move(dst, newPath, StandardCopyOption.REPLACE_EXISTING) + downloaded.add(newPath) + } catch (e: Exception) { + println("Warning: failed to download $u: ${e.message}") + } + } + + // copy to target updates dir + val outDir = targetDir.resolve("updates_coop_files") + Files.createDirectories(outDir) + for (p in downloaded) { + val dest = outDir.resolve(p.fileName.toString()) + println("Copying VO $p -> $dest") + try { + if (!DRYRUN) { + Files.copy(p, dest, StandardCopyOption.REPLACE_EXISTING) + setPerm664(dest) + } else { + println("[DRYRUN] Would copy $p -> $dest") + } + } catch (e: Exception) { + println("Warning: copying failed: ${e.message}") + } + } + + return downloaded.map { outDir.resolve(it.fileName) } +} + +// --------------------------- DATABASE --------------------------- + +fun dbConnection() = + DriverManager.getConnection("jdbc:mysql://$DB_HOST/$DB_NAME?useSSL=false&serverTimezone=UTC", DB_USER, DB_PASS) + +fun readExisting(conn: java.sql.Connection, mod: String): Map> { + val sql = """ + SELECT uf.fileId, uf.name, uf.md5 + FROM ( + SELECT fileId, MAX(version) AS v + FROM updates_${mod}_files + GROUP BY fileId + ) t + JOIN updates_${mod}_files uf ON uf.fileId = t.fileId AND uf.version = t.v + """.trimIndent() + val out = mutableMapOf>() + conn.prepareStatement(sql).use { stmt -> + val rs = stmt.executeQuery() + while (rs.next()) { + out[rs.getInt(1)] = rs.getString(2) to rs.getString(3) + } + } + return out +} + +fun updateDb(conn: java.sql.Connection, mod: String, fileId: Int, version: Int, name: String, md5: String) { + println("Updating DB: $name (fileId=$fileId, version=$version) md5=$md5") + if (DRYRUN) { + println("[DRYRUN] DB: would delete/insert for $fileId,$version") + return + } + val del = "DELETE FROM updates_${mod}_files WHERE fileId=? AND version=?" + val ins = "INSERT INTO updates_${mod}_files (fileId, version, name, md5, obselete) VALUES (?, ?, ?, ?, 0)" + conn.prepareStatement(del).use { + it.setInt(1, fileId) + it.setInt(2, version) + it.executeUpdate() + } + conn.prepareStatement(ins).use { + it.setInt(1, fileId) + it.setInt(2, version) + it.setString(3, name) + it.setString(4, md5) + it.executeUpdate() + } +} + +// --------------------------- PROCESS FILES --------------------------- + +fun processItem( + conn: java.sql.Connection, + mod: String, + version: Int, + fileId: Int, + nameFmt: String, + sources: List?, + voDownloadedMap: Map +) { + val name = nameFmt.format(version) + val outDir = TARGET_DIR.resolve("updates_${mod}_files") + Files.createDirectories(outDir) + val target = outDir.resolve(name) + + println("Processing $name (fileId $fileId)") + + if (sources == null) { + // VO file: look for downloaded file in target dir (name formatted) + val expectedName = name // matches nameFmt.format(version) + val candidate = outDir.resolve(expectedName) + if (!Files.exists(candidate)) { + println("Warning: VO file $expectedName not found in $outDir, skipping") + return + } + val newMd5 = md5(candidate) + val oldMd5 = readExisting(conn, mod)[fileId]?.second + if (newMd5 != oldMd5) { + updateDb(conn, mod, fileId, version, expectedName, newMd5) + } else { + println("VO $expectedName unchanged") + } + return + } + + // sources present -> create zip or copy single file + val existing = sources.filter { p -> Files.exists(p) } + if (existing.isEmpty()) { + println("Warning: no existing sources for $name, skipping") + return + } + + // if single source and it's a file, copy it directly (like init_coop.lua) + if (existing.size == 1 && Files.isRegularFile(existing[0])) { + val src = existing[0] + println("Single file source for $name: copying $src -> $target") + if (!DRYRUN) { + Files.copy(src, target, StandardCopyOption.REPLACE_EXISTING) + setPerm664(target) + } else { + println("[DRYRUN] Would copy $src -> $target") + } + val newMd5 = md5(target) + val oldMd5 = readExisting(conn, mod)[fileId]?.second + if (newMd5 != oldMd5) { + updateDb(conn, mod, fileId, version, name, newMd5) + } else { + println("$name unchanged") + } + return + } + + // multiple sources -> zip them; determine base as common parent of the directories (so top-level folders like 'mods'/'units' remain) + // compute base as common path of all existing sources + var base = existing[0].toAbsolutePath().normalize() + for (i in 1 until existing.size) { + base = base.commonPath(existing[i].toAbsolutePath().normalize()) + } + + val tmp = Files.createTempFile("coop", ".zip") + println("Zipping sources with base=$base -> $tmp") + zipPreserveStructure(existing, tmp, base) + println("Moving zip to $target") + if (!DRYRUN) { + Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING) + setPerm664(target) + } else { + println("[DRYRUN] Would move $tmp -> $target") + } + + val newMd5 = md5(target) + val oldMd5 = readExisting(conn, mod)[fileId]?.second + if (newMd5 != oldMd5) { + updateDb(conn, mod, fileId, version, name, newMd5) + } else { + println("$name unchanged") + } +} + +// --------------------------- MAIN --------------------------- + +println("=== Kotlin Coop Deployer v$PATCH_VERSION ===") +val repo = prepareRepo() +println("Repo ready at $repo") + +// Download VO assets first +val voFiles = downloadVoAssets(PATCH_VERSION, TARGET_DIR) // returns list of paths in target dir +val voMap = voFiles.associateBy { it.fileName.toString() } // name -> Path + +val conn = dbConnection() +try { + val existing = readExisting(conn, "coop") + + data class PatchFile(val id: Int, val fileTemplate: String, val includes: List?) + + val filesList = listOf( + PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), + PatchFile( + 2, "lobby_coop_v%d.cop", listOf( + repo.resolve("mods"), + repo.resolve("units"), + repo.resolve("mod_info.lua"), + repo.resolve("readme.md"), + repo.resolve("changelog.md") + ) + ), + + // all VO files (no sources → already downloaded externally) + PatchFile(3, "A01_VO.v%d.nx2", null), + PatchFile(4, "A02_VO.v%d.nx2", null), + PatchFile(5, "A03_VO.v%d.nx2", null), + PatchFile(6, "A04_VO.v%d.nx2", null), + PatchFile(7, "A05_VO.v%d.nx2", null), + PatchFile(8, "A06_VO.v%d.nx2", null), + PatchFile(9, "C01_VO.v%d.nx2", null), + PatchFile(10, "C02_VO.v%d.nx2", null), + PatchFile(11, "C03_VO.v%d.nx2", null), + PatchFile(12, "C04_VO.v%d.nx2", null), + PatchFile(13, "C05_VO.v%d.nx2", null), + PatchFile(14, "C06_VO.v%d.nx2", null), + PatchFile(15, "E01_VO.v%d.nx2", null), + PatchFile(16, "E02_VO.v%d.nx2", null), + PatchFile(17, "E03_VO.v%d.nx2", null), + PatchFile(18, "E04_VO.v%d.nx2", null), + PatchFile(19, "E05_VO.v%d.nx2", null), + PatchFile(20, "E06_VO.v%d.nx2", null), + PatchFile(21, "Prothyon16_VO.v%d.nx2", null), + PatchFile(22, "A03_VO.v%d.nx2", null), + PatchFile(23, "A03_VO.v%d.nx2", null), + PatchFile(24, "A03_VO.v%d.nx2", null), + PatchFile(25, "A03_VO.v%d.nx2", null), + // … add the rest + ) + + for ((fileId, fmt, srcs) in filesList) { + processItem(conn, "coop", PATCH_VERSION.toInt(), fileId, fmt, srcs?.map { it }, voMap) + } +} finally { + conn.close() +} + +println("=== Done ===") From cf9d8635c5cb5e640edbd05f294906b92a12b15c Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Sat, 20 Dec 2025 14:46:54 +0100 Subject: [PATCH 06/33] Update faf-icebreaker to 1.2.0-RC2 --- apps/faf-icebreaker/templates/config.yaml | 1 + apps/faf-icebreaker/templates/deployment.yaml | 2 +- apps/faf-icebreaker/templates/local-secret.yaml | 1 + 3 files changed, 3 insertions(+), 1 deletion(-) diff --git a/apps/faf-icebreaker/templates/config.yaml b/apps/faf-icebreaker/templates/config.yaml index 8b485953..3186f54c 100644 --- a/apps/faf-icebreaker/templates/config.yaml +++ b/apps/faf-icebreaker/templates/config.yaml @@ -19,4 +19,5 @@ data: GEOIPUPDATE_EDITION_IDS: "GeoLite2-City" LOKI_BASE_URL: "http://monitoring-loki-gateway.faf-ops.svc" FORCE_RELAY: "true" + REAL_IP_HEADER: "Cf-Connecting-Ip" QUARKUS_LOG_CATEGORY__COM_FAFOREVER__LEVEL: "TRACE" \ No newline at end of file diff --git a/apps/faf-icebreaker/templates/deployment.yaml b/apps/faf-icebreaker/templates/deployment.yaml index f38ed8ef..b9accae9 100644 --- a/apps/faf-icebreaker/templates/deployment.yaml +++ b/apps/faf-icebreaker/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: - name: geolite-db mountPath: /usr/share/GeoIP containers: - - image: faforever/faf-icebreaker:1.1.9 + - image: faforever/faf-icebreaker:1.2.0-RC2 imagePullPolicy: Always name: faf-icebreaker envFrom: diff --git a/apps/faf-icebreaker/templates/local-secret.yaml b/apps/faf-icebreaker/templates/local-secret.yaml index 31d072e5..481deae6 100644 --- a/apps/faf-icebreaker/templates/local-secret.yaml +++ b/apps/faf-icebreaker/templates/local-secret.yaml @@ -13,6 +13,7 @@ stringData: RABBITMQ_PASSWORD: "banana" XIRSYS_IDENT: "banana" XIRSYS_SECRET: "banana" + HETZNER_API_KEY: "banana" JWT_PRIVATE_KEY_PATH: |- -----BEGIN PRIVATE KEY----- MIIEvAIBADANBgkqhkiG9w0BAQEFAASCBKYwggSiAgEAAoIBAQDXsCsl9W0vnW2k From e5a0bd9c899dcf23d57ae35dbead7e728ccf1fd5 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Wed, 24 Dec 2025 21:27:12 +0100 Subject: [PATCH 07/33] gradle --- .../{deploy-coop.main.kts => CoopDeployer.kt} | 185 +++++++++--------- .../scripts/build.gradle.kts | 26 +++ 2 files changed, 118 insertions(+), 93 deletions(-) rename apps/faf-legacy-deployment/scripts/{deploy-coop.main.kts => CoopDeployer.kt} (72%) create mode 100644 apps/faf-legacy-deployment/scripts/build.gradle.kts diff --git a/apps/faf-legacy-deployment/scripts/deploy-coop.main.kts b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt similarity index 72% rename from apps/faf-legacy-deployment/scripts/deploy-coop.main.kts rename to apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 6ff8ef6a..8ee75234 100755 --- a/apps/faf-legacy-deployment/scripts/deploy-coop.main.kts +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -1,13 +1,8 @@ -#!/usr/bin/env kotlin - -@file:DependsOn("com.mysql:mysql-connector-j:9.5.0") -@file:DependsOn("org.eclipse.jgit:org.eclipse.jgit:7.4.0.202509020913-r") -@file:DependsOn("com.squareup.okio:okio:3.16.4") - import org.eclipse.jgit.api.Git +import org.slf4j.LoggerFactory import java.io.IOException import java.net.HttpURLConnection -import java.net.URL +import java.net.URI import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths @@ -20,6 +15,9 @@ import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream import kotlin.io.path.inputStream +private val log = LoggerFactory.getLogger("CoopDeployer") + + // --------------------------- CONFIG --------------------------- val PATCH_VERSION = System.getenv("PATCH_VERSION") ?: error("PATCH_VERSION required") @@ -68,7 +66,7 @@ fun setPerm664(path: Path) { } Files.setPosixFilePermissions(path, perms) } catch (e: Exception) { - println("Warning: couldn't set perms on $path: ${e.message}") + log.warn("Warning: couldn't set perms on {}", path, e) } } @@ -111,13 +109,13 @@ fun zipPreserveStructure(sources: List, outputFile: Path, base: Path) { fun prepareRepo(): Path { val dir = Paths.get(WORKDIR) if (Files.exists(dir.resolve(".git"))) { - println("Repo exists — fetching and checking out $GIT_REF...") + log.info("Repo exists — fetching and checking out $GIT_REF...") Git.open(dir.toFile()).use { git -> git.fetch().call() git.checkout().setName(GIT_REF).call() } } else { - println("Cloning $REPO_URL -> $dir ...") + log.info("Cloning $REPO_URL -> $dir ...") Git.cloneRepository() .setURI(REPO_URL) .setDirectory(dir.toFile()) @@ -132,7 +130,7 @@ fun prepareRepo(): Path { // --------------------------- GITHUB RELEASE ASSET DOWNLOAD --------------------------- fun httpGet(url: String): String { - val conn = URL(url).openConnection() as HttpURLConnection + val conn = URI.create(url).toURL().openConnection() as HttpURLConnection conn.requestMethod = "GET" conn.setRequestProperty("Accept", "application/vnd.github.v3+json") conn.connectTimeout = 15000 @@ -143,7 +141,7 @@ fun httpGet(url: String): String { } fun downloadFile(url: String, dest: Path) { - val u = URL(url) + val u = URI.create(url).toURL() val conn = u.openConnection() as HttpURLConnection conn.requestMethod = "GET" conn.connectTimeout = 20000 @@ -168,28 +166,28 @@ fun findNx2UrlsFromReleaseJson(json: String): List { } fun downloadVoAssets(version: String, targetDir: Path): List { - println("Downloading VO assets for version $version from GitHub releases...") + log.info("Downloading VO assets for version $version from GitHub releases...") Files.createDirectories(VO_DOWNLOAD_TMP) val apiUrl = "https://api.github.com/repos/FAForever/fa-coop/releases/tags/v$version" val json = try { httpGet(apiUrl) } catch (e: Exception) { - println("Warning: failed to fetch release JSON: ${e.message}") + log.warn("Warning: failed to fetch release JSON", e) return emptyList() } val urls = findNx2UrlsFromReleaseJson(json) if (urls.isEmpty()) { - println("No .nx2 assets found in release v$version") + log.info("No .nx2 assets found in release v{}", version) return emptyList() } val downloaded = mutableListOf() for (u in urls) { - val filename = Paths.get(URL(u).path).fileName.toString() + val filename = Paths.get(URI.create(u).toURL().path).fileName.toString() val dst = VO_DOWNLOAD_TMP.resolve(filename) try { - println("Downloading $u -> $dst") + log.info("Downloading {} -> {}",u, dst) downloadFile(u, dst) // rename to include .v{version}.nx2 before .nx2 val newName = filename.replace(Regex("\\.nx2$"), ".v$version.nx2") @@ -197,7 +195,7 @@ fun downloadVoAssets(version: String, targetDir: Path): List { Files.move(dst, newPath, StandardCopyOption.REPLACE_EXISTING) downloaded.add(newPath) } catch (e: Exception) { - println("Warning: failed to download $u: ${e.message}") + log.warn("Warning: failed to download {}", u, e) } } @@ -206,16 +204,16 @@ fun downloadVoAssets(version: String, targetDir: Path): List { Files.createDirectories(outDir) for (p in downloaded) { val dest = outDir.resolve(p.fileName.toString()) - println("Copying VO $p -> $dest") + log.info("Copying VO {} -> {}", p, dest) try { if (!DRYRUN) { Files.copy(p, dest, StandardCopyOption.REPLACE_EXISTING) setPerm664(dest) } else { - println("[DRYRUN] Would copy $p -> $dest") + log.info("[DRYRUN] Would copy {} -> {}", p, dest) } } catch (e: Exception) { - println("Warning: copying failed: ${e.message}") + log.warn("Warning: copying failed", e) } } @@ -225,7 +223,7 @@ fun downloadVoAssets(version: String, targetDir: Path): List { // --------------------------- DATABASE --------------------------- fun dbConnection() = - DriverManager.getConnection("jdbc:mysql://$DB_HOST/$DB_NAME?useSSL=false&serverTimezone=UTC", DB_USER, DB_PASS) + DriverManager.getConnection("jdbc:mariadb://$DB_HOST/$DB_NAME?useSSL=false&serverTimezone=UTC", DB_USER, DB_PASS) fun readExisting(conn: java.sql.Connection, mod: String): Map> { val sql = """ @@ -248,9 +246,9 @@ fun readExisting(conn: java.sql.Connection, mod: String): Map create zip or copy single file val existing = sources.filter { p -> Files.exists(p) } if (existing.isEmpty()) { - println("Warning: no existing sources for $name, skipping") + log.info("Warning: no existing sources for {}, skipping", name) return } // if single source and it's a file, copy it directly (like init_coop.lua) if (existing.size == 1 && Files.isRegularFile(existing[0])) { val src = existing[0] - println("Single file source for $name: copying $src -> $target") + log.info("Single file source for {}: copying {} -> {}", name, src, target) if (!DRYRUN) { Files.copy(src, target, StandardCopyOption.REPLACE_EXISTING) setPerm664(target) } else { - println("[DRYRUN] Would copy $src -> $target") + log.info("[DRYRUN] Would copy {} -> {}", src, target) } val newMd5 = md5(target) val oldMd5 = readExisting(conn, mod)[fileId]?.second if (newMd5 != oldMd5) { updateDb(conn, mod, fileId, version, name, newMd5) } else { - println("$name unchanged") + log.info("{} unchanged", name) } return } @@ -340,14 +338,14 @@ fun processItem( } val tmp = Files.createTempFile("coop", ".zip") - println("Zipping sources with base=$base -> $tmp") + log.info("Zipping sources with base={} -> {}", base, tmp) zipPreserveStructure(existing, tmp, base) - println("Moving zip to $target") + log.info("Moving zip to {}", target) if (!DRYRUN) { Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING) setPerm664(target) } else { - println("[DRYRUN] Would move $tmp -> $target") + log.info("[DRYRUN] Would move {} -> {}", tmp, target) } val newMd5 = md5(target) @@ -355,70 +353,71 @@ fun processItem( if (newMd5 != oldMd5) { updateDb(conn, mod, fileId, version, name, newMd5) } else { - println("$name unchanged") + log.info("{} unchanged", name) } } // --------------------------- MAIN --------------------------- - -println("=== Kotlin Coop Deployer v$PATCH_VERSION ===") -val repo = prepareRepo() -println("Repo ready at $repo") +fun main() { + log.info("=== Kotlin Coop Deployer v{} ===", PATCH_VERSION) + val repo = prepareRepo() + log.info("Repo ready at {}", repo) // Download VO assets first -val voFiles = downloadVoAssets(PATCH_VERSION, TARGET_DIR) // returns list of paths in target dir -val voMap = voFiles.associateBy { it.fileName.toString() } // name -> Path - -val conn = dbConnection() -try { - val existing = readExisting(conn, "coop") - - data class PatchFile(val id: Int, val fileTemplate: String, val includes: List?) - - val filesList = listOf( - PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), - PatchFile( - 2, "lobby_coop_v%d.cop", listOf( - repo.resolve("mods"), - repo.resolve("units"), - repo.resolve("mod_info.lua"), - repo.resolve("readme.md"), - repo.resolve("changelog.md") - ) - ), - - // all VO files (no sources → already downloaded externally) - PatchFile(3, "A01_VO.v%d.nx2", null), - PatchFile(4, "A02_VO.v%d.nx2", null), - PatchFile(5, "A03_VO.v%d.nx2", null), - PatchFile(6, "A04_VO.v%d.nx2", null), - PatchFile(7, "A05_VO.v%d.nx2", null), - PatchFile(8, "A06_VO.v%d.nx2", null), - PatchFile(9, "C01_VO.v%d.nx2", null), - PatchFile(10, "C02_VO.v%d.nx2", null), - PatchFile(11, "C03_VO.v%d.nx2", null), - PatchFile(12, "C04_VO.v%d.nx2", null), - PatchFile(13, "C05_VO.v%d.nx2", null), - PatchFile(14, "C06_VO.v%d.nx2", null), - PatchFile(15, "E01_VO.v%d.nx2", null), - PatchFile(16, "E02_VO.v%d.nx2", null), - PatchFile(17, "E03_VO.v%d.nx2", null), - PatchFile(18, "E04_VO.v%d.nx2", null), - PatchFile(19, "E05_VO.v%d.nx2", null), - PatchFile(20, "E06_VO.v%d.nx2", null), - PatchFile(21, "Prothyon16_VO.v%d.nx2", null), - PatchFile(22, "A03_VO.v%d.nx2", null), - PatchFile(23, "A03_VO.v%d.nx2", null), - PatchFile(24, "A03_VO.v%d.nx2", null), - PatchFile(25, "A03_VO.v%d.nx2", null), - // … add the rest - ) - - for ((fileId, fmt, srcs) in filesList) { - processItem(conn, "coop", PATCH_VERSION.toInt(), fileId, fmt, srcs?.map { it }, voMap) + val voFiles = downloadVoAssets(PATCH_VERSION, TARGET_DIR) // returns list of paths in target dir + val voMap = voFiles.associateBy { it.fileName.toString() } // name -> Path + + val conn = dbConnection() + try { + val existing = readExisting(conn, "coop") + + data class PatchFile(val id: Int, val fileTemplate: String, val includes: List?) + + val filesList = listOf( + PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), + PatchFile( + 2, "lobby_coop_v%d.cop", listOf( + repo.resolve("mods"), + repo.resolve("units"), + repo.resolve("mod_info.lua"), + repo.resolve("readme.md"), + repo.resolve("changelog.md") + ) + ), + + // all VO files (no sources → already downloaded externally) + PatchFile(3, "A01_VO.v%d.nx2", null), + PatchFile(4, "A02_VO.v%d.nx2", null), + PatchFile(5, "A03_VO.v%d.nx2", null), + PatchFile(6, "A04_VO.v%d.nx2", null), + PatchFile(7, "A05_VO.v%d.nx2", null), + PatchFile(8, "A06_VO.v%d.nx2", null), + PatchFile(9, "C01_VO.v%d.nx2", null), + PatchFile(10, "C02_VO.v%d.nx2", null), + PatchFile(11, "C03_VO.v%d.nx2", null), + PatchFile(12, "C04_VO.v%d.nx2", null), + PatchFile(13, "C05_VO.v%d.nx2", null), + PatchFile(14, "C06_VO.v%d.nx2", null), + PatchFile(15, "E01_VO.v%d.nx2", null), + PatchFile(16, "E02_VO.v%d.nx2", null), + PatchFile(17, "E03_VO.v%d.nx2", null), + PatchFile(18, "E04_VO.v%d.nx2", null), + PatchFile(19, "E05_VO.v%d.nx2", null), + PatchFile(20, "E06_VO.v%d.nx2", null), + PatchFile(21, "Prothyon16_VO.v%d.nx2", null), + PatchFile(22, "A03_VO.v%d.nx2", null), + PatchFile(23, "A03_VO.v%d.nx2", null), + PatchFile(24, "A03_VO.v%d.nx2", null), + PatchFile(25, "A03_VO.v%d.nx2", null), + // … add the rest + ) + + for ((fileId, fmt, srcs) in filesList) { + processItem(conn, "coop", PATCH_VERSION.toInt(), fileId, fmt, srcs?.map { it }, voMap) + } + } finally { + conn.close() } -} finally { - conn.close() -} -println("=== Done ===") + log.info("=== Done ===") +} diff --git a/apps/faf-legacy-deployment/scripts/build.gradle.kts b/apps/faf-legacy-deployment/scripts/build.gradle.kts new file mode 100644 index 00000000..392e922f --- /dev/null +++ b/apps/faf-legacy-deployment/scripts/build.gradle.kts @@ -0,0 +1,26 @@ +plugins { + kotlin("jvm") version "1.9.24" + application +} + +repositories { + mavenCentral() +} + +dependencies { + implementation("org.mariadb.jdbc:mariadb-java-client:3.5.7") + implementation("org.eclipse.jgit:org.eclipse.jgit:7.5.0.202512021534-r") + implementation("org.slf4j:slf4j-api:2.0.13") + runtimeOnly("ch.qos.logback:logback-classic:1.5.23") +} + +application { + mainClass.set("CoopDeployerKt") // filename + Kt +} + +// Use the root level for files +sourceSets { + main { + kotlin.srcDirs(".") + } +} \ No newline at end of file From 8e517d512bf68a0ff6bbeb319b78879e0b914651 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Wed, 24 Dec 2025 22:30:48 +0100 Subject: [PATCH 08/33] more coop fixes --- .../scripts/CoopDeployer.kt | 66 +++++++++++-------- 1 file changed, 38 insertions(+), 28 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 8ee75234..678c1dcc 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -187,7 +187,7 @@ fun downloadVoAssets(version: String, targetDir: Path): List { val filename = Paths.get(URI.create(u).toURL().path).fileName.toString() val dst = VO_DOWNLOAD_TMP.resolve(filename) try { - log.info("Downloading {} -> {}",u, dst) + log.info("Downloading {} -> {}", u, dst) downloadFile(u, dst) // rename to include .v{version}.nx2 before .nx2 val newName = filename.replace(Regex("\\.nx2$"), ".v$version.nx2") @@ -225,9 +225,11 @@ fun downloadVoAssets(version: String, targetDir: Path): List { fun dbConnection() = DriverManager.getConnection("jdbc:mariadb://$DB_HOST/$DB_NAME?useSSL=false&serverTimezone=UTC", DB_USER, DB_PASS) -fun readExisting(conn: java.sql.Connection, mod: String): Map> { +data class ExistingFile(val fileId: Int, val name: String, val md5: String, val version: Int) + +fun readExisting(conn: java.sql.Connection, mod: String): Map { val sql = """ - SELECT uf.fileId, uf.name, uf.md5 + SELECT uf.fileId, uf.name, uf.md5, t.v FROM ( SELECT fileId, MAX(version) AS v FROM updates_${mod}_files @@ -235,11 +237,15 @@ fun readExisting(conn: java.sql.Connection, mod: String): Map>() + val out = mutableMapOf() conn.prepareStatement(sql).use { stmt -> val rs = stmt.executeQuery() while (rs.next()) { - out[rs.getInt(1)] = rs.getString(2) to rs.getString(3) + val fileId = rs.getInt(1) + val name = rs.getString(2) + val md5 = rs.getString(3) + val version = rs.getInt(4) + out[fileId] = ExistingFile(fileId = fileId, name = name, md5 = md5, version = version) } } return out @@ -275,8 +281,7 @@ fun processItem( version: Int, fileId: Int, nameFmt: String, - sources: List?, - voDownloadedMap: Map + sources: List? ) { val name = nameFmt.format(version) val outDir = TARGET_DIR.resolve("updates_${mod}_files") @@ -294,17 +299,17 @@ fun processItem( return } val newMd5 = md5(candidate) - val oldMd5 = readExisting(conn, mod)[fileId]?.second - if (newMd5 != oldMd5) { + val oldFile = readExisting(conn, mod)[fileId] + if (newMd5 != oldFile?.md5) { updateDb(conn, mod, fileId, version, expectedName, newMd5) } else { - log.info("VO {} unchanged", expectedName) + log.info("VO {} unchanged from version {}", expectedName, oldFile.version) } return } // sources present -> create zip or copy single file - val existing = sources.filter { p -> Files.exists(p) } + val existing = sources.filter(Files::exists) if (existing.isEmpty()) { log.info("Warning: no existing sources for {}, skipping", name) return @@ -314,20 +319,21 @@ fun processItem( if (existing.size == 1 && Files.isRegularFile(existing[0])) { val src = existing[0] log.info("Single file source for {}: copying {} -> {}", name, src, target) + + val newMd5 = md5(src) + val oldFile = readExisting(conn, mod)[fileId] + if (newMd5 == oldFile?.md5) { + log.info("{} unchanged from version {}, skipping", name, oldFile.version) + return + } + if (!DRYRUN) { Files.copy(src, target, StandardCopyOption.REPLACE_EXISTING) setPerm664(target) - } else { - log.info("[DRYRUN] Would copy {} -> {}", src, target) - } - val newMd5 = md5(target) - val oldMd5 = readExisting(conn, mod)[fileId]?.second - if (newMd5 != oldMd5) { updateDb(conn, mod, fileId, version, name, newMd5) } else { - log.info("{} unchanged", name) + log.info("[DRYRUN] Would copy {} -> {}", src, target) } - return } // multiple sources -> zip them; determine base as common parent of the directories (so top-level folders like 'mods'/'units' remain) @@ -340,20 +346,24 @@ fun processItem( val tmp = Files.createTempFile("coop", ".zip") log.info("Zipping sources with base={} -> {}", base, tmp) zipPreserveStructure(existing, tmp, base) - log.info("Moving zip to {}", target) + + val newMd5 = md5(tmp) + val oldFile = readExisting(conn, mod)[fileId] + + if (newMd5 == oldFile?.md5) { + log.info("{} unchanged from version {}, skipping", name, oldFile.version) + return + } + if (!DRYRUN) { + log.info("Moving zip to {}", target) Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING) setPerm664(target) - } else { - log.info("[DRYRUN] Would move {} -> {}", tmp, target) - } - val newMd5 = md5(target) - val oldMd5 = readExisting(conn, mod)[fileId]?.second - if (newMd5 != oldMd5) { + log.info("Writing fileId {} with version {} to database", fileId, version) updateDb(conn, mod, fileId, version, name, newMd5) } else { - log.info("{} unchanged", name) + log.info("[DRYRUN] Would move {} -> {}", tmp, target) } } @@ -413,7 +423,7 @@ fun main() { ) for ((fileId, fmt, srcs) in filesList) { - processItem(conn, "coop", PATCH_VERSION.toInt(), fileId, fmt, srcs?.map { it }, voMap) + processItem(conn, "coop", PATCH_VERSION.toInt(), fileId, fmt, srcs?.map { it }) } } finally { conn.close() From d943a045accf4652b88cb90e5c5945bdc2ee8327 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Wed, 24 Dec 2025 23:49:16 +0100 Subject: [PATCH 09/33] improve deploy script --- .../scripts/CoopDeployer.kt | 381 +++++++++--------- 1 file changed, 194 insertions(+), 187 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 678c1dcc..db61dc5f 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -1,16 +1,19 @@ import org.eclipse.jgit.api.Git import org.slf4j.LoggerFactory import java.io.IOException -import java.net.HttpURLConnection import java.net.URI +import java.net.http.HttpClient +import java.net.http.HttpRequest +import java.net.http.HttpResponse import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.nio.file.StandardCopyOption -import java.nio.file.StandardOpenOption import java.nio.file.attribute.PosixFilePermission import java.security.MessageDigest +import java.sql.Connection import java.sql.DriverManager +import java.time.Duration import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream import kotlin.io.path.inputStream @@ -18,6 +21,11 @@ import kotlin.io.path.inputStream private val log = LoggerFactory.getLogger("CoopDeployer") +/** + * Definition of a file to create a patch for + */ +data class PatchFile(val id: Int, val fileTemplate: String, val includes: List?, val mod: String = "coop") + // --------------------------- CONFIG --------------------------- val PATCH_VERSION = System.getenv("PATCH_VERSION") ?: error("PATCH_VERSION required") @@ -36,9 +44,9 @@ val VO_DOWNLOAD_TMP = Paths.get("/tmp", "vo_download_$PATCH_VERSION") // --------------------------- UTILS --------------------------- -fun md5(path: Path): String { +fun Path.md5(): String { val md = MessageDigest.getInstance("MD5") - path.inputStream().use { input -> + this.inputStream().use { input -> val buf = ByteArray(4096) var r: Int while (input.read(buf).also { r = it } != -1) { @@ -57,16 +65,16 @@ fun Path.commonPath(other: Path): Path { return if (i == 0) a.root else a.root.resolve(a.subpath(0, i)) } -fun setPerm664(path: Path) { +fun Path.setPerm664() { try { val perms = mutableSetOf().apply { add(PosixFilePermission.OWNER_READ); add(PosixFilePermission.OWNER_WRITE) add(PosixFilePermission.GROUP_READ); add(PosixFilePermission.GROUP_WRITE) add(PosixFilePermission.OTHERS_READ) } - Files.setPosixFilePermissions(path, perms) + Files.setPosixFilePermissions(this, perms) } catch (e: Exception) { - log.warn("Warning: couldn't set perms on {}", path, e) + log.warn("Warning: couldn't set perms on {}", this, e) } } @@ -127,108 +135,109 @@ fun prepareRepo(): Path { return dir } -// --------------------------- GITHUB RELEASE ASSET DOWNLOAD --------------------------- - -fun httpGet(url: String): String { - val conn = URI.create(url).toURL().openConnection() as HttpURLConnection - conn.requestMethod = "GET" - conn.setRequestProperty("Accept", "application/vnd.github.v3+json") - conn.connectTimeout = 15000 - conn.readTimeout = 15000 - val code = conn.responseCode - val stream = if (code in 200..299) conn.inputStream else conn.errorStream - return stream.bufferedReader().use { it.readText() } -} +object GithubAssetDownloader { + private const val API_URL = "https://api.github.com/repos/FAForever/fa-coop/releases/tags/v%s" + private val DOWNLOAD_URL_REGEX = Regex(""""browser_download_url"\s*:\s*"(?[^"]+)"""") -fun downloadFile(url: String, dest: Path) { - val u = URI.create(url).toURL() - val conn = u.openConnection() as HttpURLConnection - conn.requestMethod = "GET" - conn.connectTimeout = 20000 - conn.readTimeout = 20000 - conn.instanceFollowRedirects = true - val code = conn.responseCode - if (code !in 200..299) { - throw IOException("Failed to download $url -> HTTP $code") - } - Files.createDirectories(dest.parent) - conn.inputStream.use { input -> - Files.newOutputStream(dest, StandardOpenOption.CREATE, StandardOpenOption.TRUNCATE_EXISTING).use { out -> - input.copyTo(out) - } - } -} + private val httpClient = HttpClient.newBuilder() + .connectTimeout(Duration.ofSeconds(10)) + .followRedirects(HttpClient.Redirect.NORMAL) + .build() -fun findNx2UrlsFromReleaseJson(json: String): List { - // crude but effective: find all "browser_download_url": "..." and filter .nx2 - val regex = Regex("\"browser_download_url\"\\s*:\\s*\"([^\"]+)\"") - return regex.findAll(json).mapNotNull { it.groupValues.getOrNull(1) }.filter { it.endsWith(".nx2") }.toList() -} + fun downloadVoiceOverAssets(version: String, targetDir: Path): List { + log.info("Downloading VO assets for version $version from GitHub releases...") -fun downloadVoAssets(version: String, targetDir: Path): List { - log.info("Downloading VO assets for version $version from GitHub releases...") - Files.createDirectories(VO_DOWNLOAD_TMP) - val apiUrl = "https://api.github.com/repos/FAForever/fa-coop/releases/tags/v$version" - val json = try { - httpGet(apiUrl) - } catch (e: Exception) { - log.warn("Warning: failed to fetch release JSON", e) - return emptyList() - } + val urls = getAssetUrisBySuffix(version, ".nx2") + if (urls.isEmpty()) { + log.info("No .nx2 assets found in release v{}", version) + return emptyList() + } - val urls = findNx2UrlsFromReleaseJson(json) - if (urls.isEmpty()) { - log.info("No .nx2 assets found in release v{}", version) - return emptyList() - } + val downloaded = mutableListOf() + Files.createDirectories(VO_DOWNLOAD_TMP) + for (u in urls) { + val filename = Paths.get(u.toURL().path).fileName.toString() + val dst = VO_DOWNLOAD_TMP.resolve(filename) - val downloaded = mutableListOf() - for (u in urls) { - val filename = Paths.get(URI.create(u).toURL().path).fileName.toString() - val dst = VO_DOWNLOAD_TMP.resolve(filename) - try { - log.info("Downloading {} -> {}", u, dst) downloadFile(u, dst) // rename to include .v{version}.nx2 before .nx2 val newName = filename.replace(Regex("\\.nx2$"), ".v$version.nx2") val newPath = VO_DOWNLOAD_TMP.resolve(newName) Files.move(dst, newPath, StandardCopyOption.REPLACE_EXISTING) downloaded.add(newPath) - } catch (e: Exception) { - log.warn("Warning: failed to download {}", u, e) } - } - // copy to target updates dir - val outDir = targetDir.resolve("updates_coop_files") - Files.createDirectories(outDir) - for (p in downloaded) { - val dest = outDir.resolve(p.fileName.toString()) - log.info("Copying VO {} -> {}", p, dest) - try { + // copy to target updates dir + val outDir = targetDir.resolve("updates_coop_files") + Files.createDirectories(outDir) + for (p in downloaded) { + val dest = outDir.resolve(p.fileName.toString()) + log.info("Copying VO {} -> {}", p, dest) + if (!DRYRUN) { Files.copy(p, dest, StandardCopyOption.REPLACE_EXISTING) - setPerm664(dest) + dest.setPerm664() } else { log.info("[DRYRUN] Would copy {} -> {}", p, dest) } - } catch (e: Exception) { - log.warn("Warning: copying failed", e) + } + + return downloaded.map { outDir.resolve(it.fileName) } + } + + private fun getAssetUrisBySuffix(version: String, suffix: String): List { + val request = HttpRequest.newBuilder() + .uri(URI.create(API_URL.format(version))) + .timeout(Duration.ofSeconds(10)) + .header("Accept", "application/vnd.github.v3+json") + .GET() + .build() + val apiResponse = httpClient.send(request, HttpResponse.BodyHandlers.ofString()).body() + + return DOWNLOAD_URL_REGEX.findAll(apiResponse) + .mapNotNull { it.groups["url"]?.value } + .filter { it.endsWith(suffix, ignoreCase = true) } + .map { URI.create(it) } + .toList() + } + + private fun downloadFile(source: URI, dest: Path) { + log.debug("Downloading {} -> {}", source, dest) + + val request = HttpRequest.newBuilder() + .uri(source) + .GET() + .build() + + Files.createDirectories(dest.parent) + + val response = httpClient.send( + request, + HttpResponse.BodyHandlers.ofFile(dest) + ) + + if (response.statusCode() !in 200..299) { + Files.deleteIfExists(dest) + throw IOException("Failed to download $source -> HTTP ${response.statusCode()}") } } - return downloaded.map { outDir.resolve(it.fileName) } } + // --------------------------- DATABASE --------------------------- -fun dbConnection() = - DriverManager.getConnection("jdbc:mariadb://$DB_HOST/$DB_NAME?useSSL=false&serverTimezone=UTC", DB_USER, DB_PASS) +class FafDatabase(val host: String, val database: String, val dryRun: Boolean) : AutoCloseable { + /** + * Definition of an existing file in the database + */ + data class PatchFile(val mod: String, val fileId: Int, val name: String, val md5: String, val version: Int) -data class ExistingFile(val fileId: Int, val name: String, val md5: String, val version: Int) + private val connection: Connection = + DriverManager.getConnection("jdbc:mariadb://$host/$database?useSSL=false&serverTimezone=UTC", DB_USER, DB_PASS) -fun readExisting(conn: java.sql.Connection, mod: String): Map { - val sql = """ + fun getCurrentPatchFile(mod: String, fileId: Int): PatchFile? { + val sql = """ SELECT uf.fileId, uf.name, uf.md5, t.v FROM ( SELECT fileId, MAX(version) AS v @@ -236,72 +245,81 @@ fun readExisting(conn: java.sql.Connection, mod: String): Map GROUP BY fileId ) t JOIN updates_${mod}_files uf ON uf.fileId = t.fileId AND uf.version = t.v + WHERE uf.fileId = ? """.trimIndent() - val out = mutableMapOf() - conn.prepareStatement(sql).use { stmt -> - val rs = stmt.executeQuery() - while (rs.next()) { - val fileId = rs.getInt(1) - val name = rs.getString(2) - val md5 = rs.getString(3) - val version = rs.getInt(4) - out[fileId] = ExistingFile(fileId = fileId, name = name, md5 = md5, version = version) + + connection.prepareStatement(sql).use { stmt -> + stmt.setInt(1, fileId) + val rs = stmt.executeQuery() + while (rs.next()) { + val fileId = rs.getInt(1) + val name = rs.getString(2) + val md5 = rs.getString(3) + val version = rs.getInt(4) + return PatchFile(mod = mod, fileId = fileId, name = name, md5 = md5, version = version) + } } + return null } - return out -} -fun updateDb(conn: java.sql.Connection, mod: String, fileId: Int, version: Int, name: String, md5: String) { - log.info("Updating DB: {} (fileId={}, version={}) md5={}", name, fileId, version, md5) - if (DRYRUN) { - log.info("[DRYRUN] DB: would delete/insert for {},{}", fileId, version) - return - } - val del = "DELETE FROM updates_${mod}_files WHERE fileId=? AND version=?" - val ins = "INSERT INTO updates_${mod}_files (fileId, version, name, md5, obselete) VALUES (?, ?, ?, ?, 0)" - conn.prepareStatement(del).use { - it.setInt(1, fileId) - it.setInt(2, version) - it.executeUpdate() + fun insertOrReplace(mod: String, fileId: Int, version: Int, name: String, md5: String) { + log.info("Updating DB: {} (fileId={}, version={}) md5={}", name, fileId, version, md5) + if (dryRun) { + log.info("[DRYRUN] DB: would delete/insert for {},{}", fileId, version) + return + } + val del = "DELETE FROM updates_${mod}_files WHERE fileId=? AND version=?" + val ins = "INSERT INTO updates_${mod}_files (fileId, version, name, md5, obselete) VALUES (?, ?, ?, ?, 0)" + connection.prepareStatement(del).use { + it.setInt(1, fileId) + it.setInt(2, version) + it.executeUpdate() + } + connection.prepareStatement(ins).use { + it.setInt(1, fileId) + it.setInt(2, version) + it.setString(3, name) + it.setString(4, md5) + it.executeUpdate() + } } - conn.prepareStatement(ins).use { - it.setInt(1, fileId) - it.setInt(2, version) - it.setString(3, name) - it.setString(4, md5) - it.executeUpdate() + + override fun close() { + connection.close() } } + // --------------------------- PROCESS FILES --------------------------- fun processItem( - conn: java.sql.Connection, - mod: String, - version: Int, - fileId: Int, - nameFmt: String, - sources: List? + db: FafDatabase, + patchFile: PatchFile, ) { - val name = nameFmt.format(version) + val mod: String = patchFile.mod + val version: Int = PATCH_VERSION.toInt() + val fileId: Int = patchFile.id + + val name = patchFile.fileTemplate.format(version) val outDir = TARGET_DIR.resolve("updates_${mod}_files") Files.createDirectories(outDir) val target = outDir.resolve(name) log.info("Processing {} (fileId {})", name, fileId) - if (sources == null) { + if (patchFile.includes == null) { // VO file: look for downloaded file in target dir (name formatted) val expectedName = name // matches nameFmt.format(version) val candidate = outDir.resolve(expectedName) if (!Files.exists(candidate)) { - log.info("Warning: VO file {} not found in {}, skipping", expectedName, outDir) + log.debug("VO file {} not found in {}, skipping", expectedName, outDir) return } - val newMd5 = md5(candidate) - val oldFile = readExisting(conn, mod)[fileId] + + val newMd5 = candidate.md5() + val oldFile = db.getCurrentPatchFile(mod, fileId) if (newMd5 != oldFile?.md5) { - updateDb(conn, mod, fileId, version, expectedName, newMd5) + db.insertOrReplace(mod, fileId, version, expectedName, newMd5) } else { log.info("VO {} unchanged from version {}", expectedName, oldFile.version) } @@ -309,7 +327,7 @@ fun processItem( } // sources present -> create zip or copy single file - val existing = sources.filter(Files::exists) + val existing = patchFile.includes.filter(Files::exists) if (existing.isEmpty()) { log.info("Warning: no existing sources for {}, skipping", name) return @@ -320,8 +338,8 @@ fun processItem( val src = existing[0] log.info("Single file source for {}: copying {} -> {}", name, src, target) - val newMd5 = md5(src) - val oldFile = readExisting(conn, mod)[fileId] + val newMd5 = src.md5() + val oldFile = db.getCurrentPatchFile(mod, fileId) if (newMd5 == oldFile?.md5) { log.info("{} unchanged from version {}, skipping", name, oldFile.version) return @@ -329,8 +347,8 @@ fun processItem( if (!DRYRUN) { Files.copy(src, target, StandardCopyOption.REPLACE_EXISTING) - setPerm664(target) - updateDb(conn, mod, fileId, version, name, newMd5) + target.setPerm664() + db.insertOrReplace(mod, fileId, version, name, newMd5) } else { log.info("[DRYRUN] Would copy {} -> {}", src, target) } @@ -347,8 +365,8 @@ fun processItem( log.info("Zipping sources with base={} -> {}", base, tmp) zipPreserveStructure(existing, tmp, base) - val newMd5 = md5(tmp) - val oldFile = readExisting(conn, mod)[fileId] + val newMd5 = tmp.md5() + val oldFile = db.getCurrentPatchFile(mod, fileId) if (newMd5 == oldFile?.md5) { log.info("{} unchanged from version {}, skipping", name, oldFile.version) @@ -358,75 +376,64 @@ fun processItem( if (!DRYRUN) { log.info("Moving zip to {}", target) Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING) - setPerm664(target) + target.setPerm664() log.info("Writing fileId {} with version {} to database", fileId, version) - updateDb(conn, mod, fileId, version, name, newMd5) + db.insertOrReplace(mod, fileId, version, name, newMd5) } else { log.info("[DRYRUN] Would move {} -> {}", tmp, target) } } -// --------------------------- MAIN --------------------------- fun main() { log.info("=== Kotlin Coop Deployer v{} ===", PATCH_VERSION) val repo = prepareRepo() log.info("Repo ready at {}", repo) -// Download VO assets first - val voFiles = downloadVoAssets(PATCH_VERSION, TARGET_DIR) // returns list of paths in target dir - val voMap = voFiles.associateBy { it.fileName.toString() } // name -> Path - - val conn = dbConnection() - try { - val existing = readExisting(conn, "coop") - - data class PatchFile(val id: Int, val fileTemplate: String, val includes: List?) - - val filesList = listOf( - PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), - PatchFile( - 2, "lobby_coop_v%d.cop", listOf( - repo.resolve("mods"), - repo.resolve("units"), - repo.resolve("mod_info.lua"), - repo.resolve("readme.md"), - repo.resolve("changelog.md") - ) - ), - - // all VO files (no sources → already downloaded externally) - PatchFile(3, "A01_VO.v%d.nx2", null), - PatchFile(4, "A02_VO.v%d.nx2", null), - PatchFile(5, "A03_VO.v%d.nx2", null), - PatchFile(6, "A04_VO.v%d.nx2", null), - PatchFile(7, "A05_VO.v%d.nx2", null), - PatchFile(8, "A06_VO.v%d.nx2", null), - PatchFile(9, "C01_VO.v%d.nx2", null), - PatchFile(10, "C02_VO.v%d.nx2", null), - PatchFile(11, "C03_VO.v%d.nx2", null), - PatchFile(12, "C04_VO.v%d.nx2", null), - PatchFile(13, "C05_VO.v%d.nx2", null), - PatchFile(14, "C06_VO.v%d.nx2", null), - PatchFile(15, "E01_VO.v%d.nx2", null), - PatchFile(16, "E02_VO.v%d.nx2", null), - PatchFile(17, "E03_VO.v%d.nx2", null), - PatchFile(18, "E04_VO.v%d.nx2", null), - PatchFile(19, "E05_VO.v%d.nx2", null), - PatchFile(20, "E06_VO.v%d.nx2", null), - PatchFile(21, "Prothyon16_VO.v%d.nx2", null), - PatchFile(22, "A03_VO.v%d.nx2", null), - PatchFile(23, "A03_VO.v%d.nx2", null), - PatchFile(24, "A03_VO.v%d.nx2", null), - PatchFile(25, "A03_VO.v%d.nx2", null), - // … add the rest - ) - - for ((fileId, fmt, srcs) in filesList) { - processItem(conn, "coop", PATCH_VERSION.toInt(), fileId, fmt, srcs?.map { it }) - } - } finally { - conn.close() + // Download VO assets first + GithubAssetDownloader.downloadVoiceOverAssets(PATCH_VERSION, TARGET_DIR) // returns list of paths in target dir + + val filesList = listOf( + PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), + PatchFile( + 2, "lobby_coop_v%d.cop", listOf( + repo.resolve("mods"), + repo.resolve("units"), + repo.resolve("mod_info.lua"), + repo.resolve("readme.md"), + repo.resolve("changelog.md") + ) + ), + + // all VO files (no sources → already downloaded externally) + PatchFile(3, "A01_VO.v%d.nx2", null), + PatchFile(4, "A02_VO.v%d.nx2", null), + PatchFile(5, "A03_VO.v%d.nx2", null), + PatchFile(6, "A04_VO.v%d.nx2", null), + PatchFile(7, "A05_VO.v%d.nx2", null), + PatchFile(8, "A06_VO.v%d.nx2", null), + PatchFile(9, "C01_VO.v%d.nx2", null), + PatchFile(10, "C02_VO.v%d.nx2", null), + PatchFile(11, "C03_VO.v%d.nx2", null), + PatchFile(12, "C04_VO.v%d.nx2", null), + PatchFile(13, "C05_VO.v%d.nx2", null), + PatchFile(14, "C06_VO.v%d.nx2", null), + PatchFile(15, "E01_VO.v%d.nx2", null), + PatchFile(16, "E02_VO.v%d.nx2", null), + PatchFile(17, "E03_VO.v%d.nx2", null), + PatchFile(18, "E04_VO.v%d.nx2", null), + PatchFile(19, "E05_VO.v%d.nx2", null), + PatchFile(20, "E06_VO.v%d.nx2", null), + PatchFile(21, "Prothyon16_VO.v%d.nx2", null), + PatchFile(22, "A03_VO.v%d.nx2", null), + PatchFile(23, "A03_VO.v%d.nx2", null), + PatchFile(24, "A03_VO.v%d.nx2", null), + PatchFile(25, "A03_VO.v%d.nx2", null), + // … add the rest + ) + + FafDatabase(DB_HOST, DB_NAME, DRYRUN).use { db -> + filesList.forEach { processItem(db, it) } } log.info("=== Done ===") From 8efe91139034fc83a61268f1344271c8d1f499ce Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Thu, 25 Dec 2025 01:09:52 +0100 Subject: [PATCH 10/33] FeaturedModGitRepo --- .../scripts/CoopDeployer.kt | 102 +++++++++++------- .../scripts/build.gradle.kts | 2 +- 2 files changed, 66 insertions(+), 38 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index db61dc5f..3da638f5 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -112,42 +112,54 @@ fun zipPreserveStructure(sources: List, outputFile: Path, base: Path) { } } -// --------------------------- GIT --------------------------- - -fun prepareRepo(): Path { - val dir = Paths.get(WORKDIR) - if (Files.exists(dir.resolve(".git"))) { - log.info("Repo exists — fetching and checking out $GIT_REF...") - Git.open(dir.toFile()).use { git -> - git.fetch().call() - git.checkout().setName(GIT_REF).call() - } - } else { - log.info("Cloning $REPO_URL -> $dir ...") - Git.cloneRepository() - .setURI(REPO_URL) - .setDirectory(dir.toFile()) - .call() - Git.open(dir.toFile()).use { git -> - git.checkout().setName(GIT_REF).call() +data class FeatureModGitRepo( + val workDir: Path, + val repoUrl: String, + val gitRef: String, +) { + fun checkout(): Path { + if (Files.exists(workDir.resolve(".git"))) { + log.info("Repo exists — fetching and checking out $gitRef...") + Git.open(workDir.toFile()).use { git -> + git.fetch().call() + git.checkout().setName(gitRef).call() + } + } else { + log.info("Cloning repository $repoUrl") + Git.cloneRepository() + .setURI(repoUrl) + .setDirectory(workDir.toFile()) + .call() + log.info("Checking out $gitRef") + Git.open(workDir.toFile()).use { git -> + git.checkout().setName(gitRef).call() + } } + + return workDir } - return dir } -object GithubAssetDownloader { - private const val API_URL = "https://api.github.com/repos/FAForever/fa-coop/releases/tags/v%s" - private val DOWNLOAD_URL_REGEX = Regex(""""browser_download_url"\s*:\s*"(?[^"]+)"""") +data class GithubAssetDownloader( + val repoOwner: String = "FAForever", + val repoName: String, + val suffix: String, + val version: String, +) { + companion object { + private const val API_URL = "https://api.github.com/repos/%s/%s/releases/tags/v%s" + private val DOWNLOAD_URL_REGEX = Regex(""""browser_download_url"\s*:\s*"(?[^"]+)"""") + } private val httpClient = HttpClient.newBuilder() .connectTimeout(Duration.ofSeconds(10)) .followRedirects(HttpClient.Redirect.NORMAL) .build() - fun downloadVoiceOverAssets(version: String, targetDir: Path): List { + fun downloadAssets(targetDir: Path): List { log.info("Downloading VO assets for version $version from GitHub releases...") - val urls = getAssetUrisBySuffix(version, ".nx2") + val urls = getAssetUrisBySuffix() if (urls.isEmpty()) { log.info("No .nx2 assets found in release v{}", version) return emptyList() @@ -185,9 +197,9 @@ object GithubAssetDownloader { return downloaded.map { outDir.resolve(it.fileName) } } - private fun getAssetUrisBySuffix(version: String, suffix: String): List { + private fun getAssetUrisBySuffix(): List { val request = HttpRequest.newBuilder() - .uri(URI.create(API_URL.format(version))) + .uri(URI.create(API_URL.format(repoOwner, repoName, version))) .timeout(Duration.ofSeconds(10)) .header("Accept", "application/vnd.github.v3+json") .GET() @@ -224,17 +236,20 @@ object GithubAssetDownloader { } - -// --------------------------- DATABASE --------------------------- - -class FafDatabase(val host: String, val database: String, val dryRun: Boolean) : AutoCloseable { +data class FafDatabase( + val host: String, + val database: String, + val username: String, + val password: String, + val dryRun: Boolean +) : AutoCloseable { /** * Definition of an existing file in the database */ data class PatchFile(val mod: String, val fileId: Int, val name: String, val md5: String, val version: Int) private val connection: Connection = - DriverManager.getConnection("jdbc:mariadb://$host/$database?useSSL=false&serverTimezone=UTC", DB_USER, DB_PASS) + DriverManager.getConnection("jdbc:mariadb://$host/$database?useSSL=false&serverTimezone=UTC", username, password) fun getCurrentPatchFile(mod: String, fileId: Int): PatchFile? { val sql = """ @@ -289,7 +304,6 @@ class FafDatabase(val host: String, val database: String, val dryRun: Boolean) : } } - // --------------------------- PROCESS FILES --------------------------- fun processItem( @@ -387,11 +401,19 @@ fun processItem( fun main() { log.info("=== Kotlin Coop Deployer v{} ===", PATCH_VERSION) - val repo = prepareRepo() - log.info("Repo ready at {}", repo) + + val repo = FeatureModGitRepo( + workDir = Paths.get(WORKDIR), + repoUrl = REPO_URL, + gitRef = GIT_REF + ).checkout() // Download VO assets first - GithubAssetDownloader.downloadVoiceOverAssets(PATCH_VERSION, TARGET_DIR) // returns list of paths in target dir + GithubAssetDownloader( + repoName = "fa-coop", + suffix = ".nx2", + version = PATCH_VERSION + ).downloadAssets(TARGET_DIR) val filesList = listOf( PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), @@ -432,9 +454,15 @@ fun main() { // … add the rest ) - FafDatabase(DB_HOST, DB_NAME, DRYRUN).use { db -> + FafDatabase( + host = DB_HOST, + database = DB_NAME, + username = DB_USER, + password = DB_PASS, + dryRun = DRYRUN + ).use { db -> filesList.forEach { processItem(db, it) } } - log.info("=== Done ===") + log.info("=== Deployment complete ===") } diff --git a/apps/faf-legacy-deployment/scripts/build.gradle.kts b/apps/faf-legacy-deployment/scripts/build.gradle.kts index 392e922f..ec9ea43c 100644 --- a/apps/faf-legacy-deployment/scripts/build.gradle.kts +++ b/apps/faf-legacy-deployment/scripts/build.gradle.kts @@ -1,5 +1,5 @@ plugins { - kotlin("jvm") version "1.9.24" + kotlin("jvm") version "2.3.0" application } From 10d9be6dc294a16b8c649399c5f6716908a1ef80 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Thu, 25 Dec 2025 10:53:20 +0100 Subject: [PATCH 11/33] Introduce Patcher class --- .../scripts/CoopDeployer.kt | 388 +++++++++--------- 1 file changed, 196 insertions(+), 192 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 3da638f5..6b8de425 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -20,96 +20,13 @@ import kotlin.io.path.inputStream private val log = LoggerFactory.getLogger("CoopDeployer") - -/** - * Definition of a file to create a patch for - */ -data class PatchFile(val id: Int, val fileTemplate: String, val includes: List?, val mod: String = "coop") - -// --------------------------- CONFIG --------------------------- - -val PATCH_VERSION = System.getenv("PATCH_VERSION") ?: error("PATCH_VERSION required") -val REPO_URL = System.getenv("GIT_REPO_URL") ?: "https://github.com/FAForever/fa-coop.git" -val GIT_REF = System.getenv("GIT_REF") ?: "v$PATCH_VERSION" -val WORKDIR = System.getenv("GIT_WORKDIR") ?: "/tmp/fa-coop-kt" -val DRYRUN = (System.getenv("DRY_RUN") ?: "false").lowercase() in listOf("1", "true", "yes") - -val DB_HOST = System.getenv("DATABASE_HOST") ?: "localhost" -val DB_NAME = System.getenv("DATABASE_NAME") ?: "faf" -val DB_USER = System.getenv("DATABASE_USERNAME") ?: "root" -val DB_PASS = System.getenv("DATABASE_PASSWORD") ?: "banana" - -val TARGET_DIR = Paths.get("./legacy-featured-mod-files") -val VO_DOWNLOAD_TMP = Paths.get("/tmp", "vo_download_$PATCH_VERSION") - -// --------------------------- UTILS --------------------------- - -fun Path.md5(): String { - val md = MessageDigest.getInstance("MD5") - this.inputStream().use { input -> - val buf = ByteArray(4096) - var r: Int - while (input.read(buf).also { r = it } != -1) { - md.update(buf, 0, r) - } - } - return md.digest().joinToString("") { "%02x".format(it) } -} - -fun Path.commonPath(other: Path): Path { - val a = this.toAbsolutePath().normalize() - val b = other.toAbsolutePath().normalize() - val min = minOf(a.nameCount, b.nameCount) - var i = 0 - while (i < min && a.getName(i) == b.getName(i)) i++ - return if (i == 0) a.root else a.root.resolve(a.subpath(0, i)) -} - fun Path.setPerm664() { - try { - val perms = mutableSetOf().apply { - add(PosixFilePermission.OWNER_READ); add(PosixFilePermission.OWNER_WRITE) - add(PosixFilePermission.GROUP_READ); add(PosixFilePermission.GROUP_WRITE) - add(PosixFilePermission.OTHERS_READ) - } - Files.setPosixFilePermissions(this, perms) - } catch (e: Exception) { - log.warn("Warning: couldn't set perms on {}", this, e) - } -} - -// --------------------------- ZIP with preserved hierarchy --------------------------- - -fun zipPreserveStructure(sources: List, outputFile: Path, base: Path) { - Files.createDirectories(outputFile.parent) - ZipOutputStream(Files.newOutputStream(outputFile)).use { zos -> - for (src in sources) { - if (!Files.exists(src)) { - // skip - continue - } - if (Files.isDirectory(src)) { - Files.walk(src).use { stream -> - stream.filter { Files.isRegularFile(it) }.forEach { file -> - val arcname = base.relativize(file).toString().replace("\\", "/") - val entry = ZipEntry(arcname) - // fix timestamp for determinism (not strictly necessary) - entry.time = 315532800000L // 1980-01-01 00:00:00 UTC in ms - zos.putNextEntry(entry) - Files.newInputStream(file).use { inp -> inp.copyTo(zos) } - zos.closeEntry() - } - } - } else { - val arcname = base.relativize(src).toString().replace("\\", "/") - val entry = ZipEntry(arcname) - entry.time = 315532800000L - zos.putNextEntry(entry) - Files.newInputStream(src).use { inp -> inp.copyTo(zos) } - zos.closeEntry() - } - } + val perms = mutableSetOf().apply { + add(PosixFilePermission.OWNER_READ); add(PosixFilePermission.OWNER_WRITE) + add(PosixFilePermission.GROUP_READ); add(PosixFilePermission.GROUP_WRITE) + add(PosixFilePermission.OTHERS_READ) } + Files.setPosixFilePermissions(this, perms) } data class FeatureModGitRepo( @@ -140,11 +57,12 @@ data class FeatureModGitRepo( } } -data class GithubAssetDownloader( +data class GithubReleaseAssetDownloader( val repoOwner: String = "FAForever", val repoName: String, val suffix: String, val version: String, + val dryRun: Boolean, ) { companion object { private const val API_URL = "https://api.github.com/repos/%s/%s/releases/tags/v%s" @@ -157,7 +75,8 @@ data class GithubAssetDownloader( .build() fun downloadAssets(targetDir: Path): List { - log.info("Downloading VO assets for version $version from GitHub releases...") + log.info("Downloading assets for version $version from GitHub releases...") + val tempDir = Paths.get("/tmp", "asset_download_$version") val urls = getAssetUrisBySuffix() if (urls.isEmpty()) { @@ -166,15 +85,15 @@ data class GithubAssetDownloader( } val downloaded = mutableListOf() - Files.createDirectories(VO_DOWNLOAD_TMP) + Files.createDirectories(tempDir) for (u in urls) { val filename = Paths.get(u.toURL().path).fileName.toString() - val dst = VO_DOWNLOAD_TMP.resolve(filename) + val dst = tempDir.resolve(filename) downloadFile(u, dst) // rename to include .v{version}.nx2 before .nx2 val newName = filename.replace(Regex("\\.nx2$"), ".v$version.nx2") - val newPath = VO_DOWNLOAD_TMP.resolve(newName) + val newPath = tempDir.resolve(newName) Files.move(dst, newPath, StandardCopyOption.REPLACE_EXISTING) downloaded.add(newPath) } @@ -186,7 +105,7 @@ data class GithubAssetDownloader( val dest = outDir.resolve(p.fileName.toString()) log.info("Copying VO {} -> {}", p, dest) - if (!DRYRUN) { + if (!dryRun) { Files.copy(p, dest, StandardCopyOption.REPLACE_EXISTING) dest.setPerm664() } else { @@ -249,7 +168,11 @@ data class FafDatabase( data class PatchFile(val mod: String, val fileId: Int, val name: String, val md5: String, val version: Int) private val connection: Connection = - DriverManager.getConnection("jdbc:mariadb://$host/$database?useSSL=false&serverTimezone=UTC", username, password) + DriverManager.getConnection( + "jdbc:mariadb://$host/$database?useSSL=false&serverTimezone=UTC", + username, + password + ) fun getCurrentPatchFile(mod: String, fileId: Int): PatchFile? { val sql = """ @@ -304,102 +227,176 @@ data class FafDatabase( } } -// --------------------------- PROCESS FILES --------------------------- - -fun processItem( - db: FafDatabase, - patchFile: PatchFile, +class Patcher( + val patchVersion: Int, + val targetDir: Path, + val db: FafDatabase, + val dryRun: Boolean, ) { - val mod: String = patchFile.mod - val version: Int = PATCH_VERSION.toInt() - val fileId: Int = patchFile.id - - val name = patchFile.fileTemplate.format(version) - val outDir = TARGET_DIR.resolve("updates_${mod}_files") - Files.createDirectories(outDir) - val target = outDir.resolve(name) - - log.info("Processing {} (fileId {})", name, fileId) - - if (patchFile.includes == null) { - // VO file: look for downloaded file in target dir (name formatted) - val expectedName = name // matches nameFmt.format(version) - val candidate = outDir.resolve(expectedName) - if (!Files.exists(candidate)) { - log.debug("VO file {} not found in {}, skipping", expectedName, outDir) + /** + * Definition of a file to create a patch for + */ + data class PatchFile(val id: Int, val fileTemplate: String, val includes: List?, val mod: String = "coop") + + fun process(patchFile: PatchFile) { + val mod: String = patchFile.mod + val fileId: Int = patchFile.id + val name = patchFile.fileTemplate.format(patchVersion) + val outDir = targetDir.resolve("updates_${mod}_files") + + Files.createDirectories(outDir) + val target = outDir.resolve(name) + + log.info("Processing {} (fileId {})", name, fileId) + + if (patchFile.includes == null) { + // VO file: look for downloaded file in target dir (name formatted) + val expectedName = name // matches nameFmt.format(version) + val candidate = outDir.resolve(expectedName) + if (!Files.exists(candidate)) { + log.debug("VO file {} not found in {}, skipping", expectedName, outDir) + return + } + + val newMd5 = candidate.md5() + val oldFile = db.getCurrentPatchFile(mod, fileId) + if (newMd5 != oldFile?.md5) { + db.insertOrReplace(mod, fileId, patchVersion, expectedName, newMd5) + } else { + log.info("VO {} unchanged from version {}", expectedName, oldFile.version) + } return } - val newMd5 = candidate.md5() - val oldFile = db.getCurrentPatchFile(mod, fileId) - if (newMd5 != oldFile?.md5) { - db.insertOrReplace(mod, fileId, version, expectedName, newMd5) - } else { - log.info("VO {} unchanged from version {}", expectedName, oldFile.version) + // sources present -> create zip or copy single file + val existing = patchFile.includes.filter(Files::exists) + if (existing.isEmpty()) { + log.info("Warning: no existing sources for {}, skipping", name) + return } - return - } - // sources present -> create zip or copy single file - val existing = patchFile.includes.filter(Files::exists) - if (existing.isEmpty()) { - log.info("Warning: no existing sources for {}, skipping", name) - return - } + // if single source and it's a file, copy it directly (like init_coop.lua) + if (existing.size == 1 && Files.isRegularFile(existing[0])) { + val src = existing[0] + log.info("Single file source for {}: copying {} -> {}", name, src, target) - // if single source and it's a file, copy it directly (like init_coop.lua) - if (existing.size == 1 && Files.isRegularFile(existing[0])) { - val src = existing[0] - log.info("Single file source for {}: copying {} -> {}", name, src, target) + val newMd5 = src.md5() + val oldFile = db.getCurrentPatchFile(mod, fileId) + if (newMd5 == oldFile?.md5) { + log.info("{} unchanged from version {}, skipping", name, oldFile.version) + return + } - val newMd5 = src.md5() + if (!dryRun) { + Files.copy(src, target, StandardCopyOption.REPLACE_EXISTING) + target.setPerm664() + db.insertOrReplace(mod, fileId, patchVersion, name, newMd5) + } else { + log.info("[DRYRUN] Would copy {} -> {}", src, target) + } + } + + // multiple sources -> zip them; determine base as common parent of the directories (so top-level folders like 'mods'/'units' remain) + // compute base as common path of all existing sources + var base = existing[0].toAbsolutePath().normalize() + for (i in 1 until existing.size) { + base = base.commonPath(existing[i].toAbsolutePath().normalize()) + } + + val tmp = Files.createTempFile("coop", ".zip") + log.info("Zipping sources with base={} -> {}", base, tmp) + zipPreserveStructure(existing, tmp, base) + + val newMd5 = tmp.md5() val oldFile = db.getCurrentPatchFile(mod, fileId) + if (newMd5 == oldFile?.md5) { log.info("{} unchanged from version {}, skipping", name, oldFile.version) return } - if (!DRYRUN) { - Files.copy(src, target, StandardCopyOption.REPLACE_EXISTING) + if (!dryRun) { + log.info("Moving zip to {}", target) + Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING) target.setPerm664() - db.insertOrReplace(mod, fileId, version, name, newMd5) + + log.info("Writing fileId {} with version {} to database", fileId, patchVersion) + db.insertOrReplace(mod, fileId, patchVersion, name, newMd5) } else { - log.info("[DRYRUN] Would copy {} -> {}", src, target) + log.info("[DRYRUN] Would move {} -> {}", tmp, target) } } - // multiple sources -> zip them; determine base as common parent of the directories (so top-level folders like 'mods'/'units' remain) - // compute base as common path of all existing sources - var base = existing[0].toAbsolutePath().normalize() - for (i in 1 until existing.size) { - base = base.commonPath(existing[i].toAbsolutePath().normalize()) + private fun Path.md5(): String { + val md = MessageDigest.getInstance("MD5") + this.inputStream().use { input -> + val buf = ByteArray(4096) + var r: Int + while (input.read(buf).also { r = it } != -1) { + md.update(buf, 0, r) + } + } + return md.digest().joinToString("") { "%02x".format(it) } } - val tmp = Files.createTempFile("coop", ".zip") - log.info("Zipping sources with base={} -> {}", base, tmp) - zipPreserveStructure(existing, tmp, base) - - val newMd5 = tmp.md5() - val oldFile = db.getCurrentPatchFile(mod, fileId) - - if (newMd5 == oldFile?.md5) { - log.info("{} unchanged from version {}, skipping", name, oldFile.version) - return + private fun Path.commonPath(other: Path): Path { + val a = this.toAbsolutePath().normalize() + val b = other.toAbsolutePath().normalize() + val min = minOf(a.nameCount, b.nameCount) + var i = 0 + while (i < min && a.getName(i) == b.getName(i)) i++ + return if (i == 0) a.root else a.root.resolve(a.subpath(0, i)) } - if (!DRYRUN) { - log.info("Moving zip to {}", target) - Files.move(tmp, target, StandardCopyOption.REPLACE_EXISTING) - target.setPerm664() - - log.info("Writing fileId {} with version {} to database", fileId, version) - db.insertOrReplace(mod, fileId, version, name, newMd5) - } else { - log.info("[DRYRUN] Would move {} -> {}", tmp, target) + private fun zipPreserveStructure(sources: List, outputFile: Path, base: Path) { + Files.createDirectories(outputFile.parent) + ZipOutputStream(Files.newOutputStream(outputFile)).use { zos -> + for (src in sources) { + if (!Files.exists(src)) { + // skip + continue + } + if (Files.isDirectory(src)) { + Files.walk(src).use { stream -> + stream.filter { Files.isRegularFile(it) }.forEach { file -> + val arcname = base.relativize(file).toString().replace("\\", "/") + val entry = ZipEntry(arcname) + // fix timestamp for determinism (not strictly necessary) + entry.time = 315532800000L // 1980-01-01 00:00:00 UTC in ms + zos.putNextEntry(entry) + Files.newInputStream(file).use { inp -> inp.copyTo(zos) } + zos.closeEntry() + } + } + } else { + val arcname = base.relativize(src).toString().replace("\\", "/") + val entry = ZipEntry(arcname) + // earliest timestamp allowed in zip file + entry.time = 315532800000L + zos.putNextEntry(entry) + Files.newInputStream(src).use { inp -> inp.copyTo(zos) } + zos.closeEntry() + } + } + } } + } fun main() { + val PATCH_VERSION = System.getenv("PATCH_VERSION") ?: error("PATCH_VERSION required") + val REPO_URL = System.getenv("GIT_REPO_URL") ?: "https://github.com/FAForever/fa-coop.git" + val GIT_REF = System.getenv("GIT_REF") ?: "v$PATCH_VERSION" + val WORKDIR = System.getenv("GIT_WORKDIR") ?: "/tmp/fa-coop-kt" + val DRYRUN = (System.getenv("DRY_RUN") ?: "false").lowercase() in listOf("1", "true", "yes") + + val DB_HOST = System.getenv("DATABASE_HOST") ?: "localhost" + val DB_NAME = System.getenv("DATABASE_NAME") ?: "faf" + val DB_USER = System.getenv("DATABASE_USERNAME") ?: "root" + val DB_PASS = System.getenv("DATABASE_PASSWORD") ?: "banana" + + val TARGET_DIR = Paths.get("./legacy-featured-mod-files") + log.info("=== Kotlin Coop Deployer v{} ===", PATCH_VERSION) val repo = FeatureModGitRepo( @@ -409,15 +406,16 @@ fun main() { ).checkout() // Download VO assets first - GithubAssetDownloader( + GithubReleaseAssetDownloader( repoName = "fa-coop", suffix = ".nx2", - version = PATCH_VERSION + version = PATCH_VERSION, + dryRun = DRYRUN ).downloadAssets(TARGET_DIR) val filesList = listOf( - PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), - PatchFile( + Patcher.PatchFile(1, "init_coop.v%d.lua", listOf(repo.resolve("init_coop.lua"))), + Patcher.PatchFile( 2, "lobby_coop_v%d.cop", listOf( repo.resolve("mods"), repo.resolve("units"), @@ -428,29 +426,29 @@ fun main() { ), // all VO files (no sources → already downloaded externally) - PatchFile(3, "A01_VO.v%d.nx2", null), - PatchFile(4, "A02_VO.v%d.nx2", null), - PatchFile(5, "A03_VO.v%d.nx2", null), - PatchFile(6, "A04_VO.v%d.nx2", null), - PatchFile(7, "A05_VO.v%d.nx2", null), - PatchFile(8, "A06_VO.v%d.nx2", null), - PatchFile(9, "C01_VO.v%d.nx2", null), - PatchFile(10, "C02_VO.v%d.nx2", null), - PatchFile(11, "C03_VO.v%d.nx2", null), - PatchFile(12, "C04_VO.v%d.nx2", null), - PatchFile(13, "C05_VO.v%d.nx2", null), - PatchFile(14, "C06_VO.v%d.nx2", null), - PatchFile(15, "E01_VO.v%d.nx2", null), - PatchFile(16, "E02_VO.v%d.nx2", null), - PatchFile(17, "E03_VO.v%d.nx2", null), - PatchFile(18, "E04_VO.v%d.nx2", null), - PatchFile(19, "E05_VO.v%d.nx2", null), - PatchFile(20, "E06_VO.v%d.nx2", null), - PatchFile(21, "Prothyon16_VO.v%d.nx2", null), - PatchFile(22, "A03_VO.v%d.nx2", null), - PatchFile(23, "A03_VO.v%d.nx2", null), - PatchFile(24, "A03_VO.v%d.nx2", null), - PatchFile(25, "A03_VO.v%d.nx2", null), + Patcher.PatchFile(3, "A01_VO.v%d.nx2", null), + Patcher.PatchFile(4, "A02_VO.v%d.nx2", null), + Patcher.PatchFile(5, "A03_VO.v%d.nx2", null), + Patcher.PatchFile(6, "A04_VO.v%d.nx2", null), + Patcher.PatchFile(7, "A05_VO.v%d.nx2", null), + Patcher.PatchFile(8, "A06_VO.v%d.nx2", null), + Patcher.PatchFile(9, "C01_VO.v%d.nx2", null), + Patcher.PatchFile(10, "C02_VO.v%d.nx2", null), + Patcher.PatchFile(11, "C03_VO.v%d.nx2", null), + Patcher.PatchFile(12, "C04_VO.v%d.nx2", null), + Patcher.PatchFile(13, "C05_VO.v%d.nx2", null), + Patcher.PatchFile(14, "C06_VO.v%d.nx2", null), + Patcher.PatchFile(15, "E01_VO.v%d.nx2", null), + Patcher.PatchFile(16, "E02_VO.v%d.nx2", null), + Patcher.PatchFile(17, "E03_VO.v%d.nx2", null), + Patcher.PatchFile(18, "E04_VO.v%d.nx2", null), + Patcher.PatchFile(19, "E05_VO.v%d.nx2", null), + Patcher.PatchFile(20, "E06_VO.v%d.nx2", null), + Patcher.PatchFile(21, "Prothyon16_VO.v%d.nx2", null), + Patcher.PatchFile(22, "A03_VO.v%d.nx2", null), + Patcher.PatchFile(23, "A03_VO.v%d.nx2", null), + Patcher.PatchFile(24, "A03_VO.v%d.nx2", null), + Patcher.PatchFile(25, "A03_VO.v%d.nx2", null), // … add the rest ) @@ -461,7 +459,13 @@ fun main() { password = DB_PASS, dryRun = DRYRUN ).use { db -> - filesList.forEach { processItem(db, it) } + val patcher = Patcher( + patchVersion = PATCH_VERSION.toInt(), + targetDir = TARGET_DIR, + db = db, + dryRun = DRYRUN, + ) + filesList.forEach(patcher::process) } log.info("=== Deployment complete ===") From 171427199be469e4296c51d44055424ecb499ae8 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Thu, 25 Dec 2025 11:53:05 +0100 Subject: [PATCH 12/33] more refactor --- .../scripts/CoopDeployer.kt | 55 ++++++++++--------- 1 file changed, 29 insertions(+), 26 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 6b8de425..46791d4e 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -340,12 +340,15 @@ class Patcher( } private fun Path.commonPath(other: Path): Path { - val a = this.toAbsolutePath().normalize() + val a = toAbsolutePath().normalize() val b = other.toAbsolutePath().normalize() - val min = minOf(a.nameCount, b.nameCount) - var i = 0 - while (i < min && a.getName(i) == b.getName(i)) i++ - return if (i == 0) a.root else a.root.resolve(a.subpath(0, i)) + + val commonCount = (0 until minOf(a.nameCount, b.nameCount)) + .takeWhile { a.getName(it) == b.getName(it) } + .count() + + return if (commonCount == 0) a.root + else a.root.resolve(a.subpath(0, commonCount)) } private fun zipPreserveStructure(sources: List, outputFile: Path, base: Path) { @@ -354,33 +357,34 @@ class Patcher( for (src in sources) { if (!Files.exists(src)) { // skip + log.warn("Could not find path {}", src) continue } if (Files.isDirectory(src)) { Files.walk(src).use { stream -> - stream.filter { Files.isRegularFile(it) }.forEach { file -> - val arcname = base.relativize(file).toString().replace("\\", "/") - val entry = ZipEntry(arcname) - // fix timestamp for determinism (not strictly necessary) - entry.time = 315532800000L // 1980-01-01 00:00:00 UTC in ms - zos.putNextEntry(entry) - Files.newInputStream(file).use { inp -> inp.copyTo(zos) } - zos.closeEntry() - } + stream + .filter { Files.isRegularFile(it) } + .forEach { zos.pushNormalizedFile(base, it) } } } else { - val arcname = base.relativize(src).toString().replace("\\", "/") - val entry = ZipEntry(arcname) - // earliest timestamp allowed in zip file - entry.time = 315532800000L - zos.putNextEntry(entry) - Files.newInputStream(src).use { inp -> inp.copyTo(zos) } - zos.closeEntry() + zos.pushNormalizedFile(base, src) } } } } + private fun ZipOutputStream.pushNormalizedFile(base: Path, path: Path) { + require(Files.isRegularFile(path)) { "Path $path is not a regular file" } + + val archiveName = base.relativize(path).toString().replace("\\", "/") + val entry = ZipEntry(archiveName) + // fix timestamp for determinism (not strictly necessary) + entry.time = 315532800000L + this.putNextEntry(entry) + Files.newInputStream(path).use { inp -> inp.copyTo(this) } + this.closeEntry() + } + } fun main() { @@ -445,11 +449,10 @@ fun main() { Patcher.PatchFile(19, "E05_VO.v%d.nx2", null), Patcher.PatchFile(20, "E06_VO.v%d.nx2", null), Patcher.PatchFile(21, "Prothyon16_VO.v%d.nx2", null), - Patcher.PatchFile(22, "A03_VO.v%d.nx2", null), - Patcher.PatchFile(23, "A03_VO.v%d.nx2", null), - Patcher.PatchFile(24, "A03_VO.v%d.nx2", null), - Patcher.PatchFile(25, "A03_VO.v%d.nx2", null), - // … add the rest + Patcher.PatchFile(22, "TCR_VO.v%d.nx2", null), + Patcher.PatchFile(23, "SCCA_Briefings.v%d.nx2", null), + Patcher.PatchFile(24, "SCCA_FMV.v%d.nx2", null), + Patcher.PatchFile(25, "FAF_Coop_Operation_Tight_Spot_VO.v%d.nx2", null), ) FafDatabase( From 2b87522381db6f8e898dfa6b5490e5571a2568b6 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Thu, 25 Dec 2025 22:16:43 +0100 Subject: [PATCH 13/33] deploy coop v66 test --- .../scripts/CoopDeployer.kt | 2 + .../scripts/deploy-coop.py | 372 ------------------ .../templates/deploy-coop-maps.yaml | 8 +- .../templates/deploy-coop.yaml | 21 +- 4 files changed, 19 insertions(+), 384 deletions(-) delete mode 100644 apps/faf-legacy-deployment/scripts/deploy-coop.py diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 46791d4e..28dd751d 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -294,6 +294,8 @@ class Patcher( } else { log.info("[DRYRUN] Would copy {} -> {}", src, target) } + + return } // multiple sources -> zip them; determine base as common parent of the directories (so top-level folders like 'mods'/'units' remain) diff --git a/apps/faf-legacy-deployment/scripts/deploy-coop.py b/apps/faf-legacy-deployment/scripts/deploy-coop.py deleted file mode 100644 index 1c1a42f2..00000000 --- a/apps/faf-legacy-deployment/scripts/deploy-coop.py +++ /dev/null @@ -1,372 +0,0 @@ -#!/usr/bin/env python3 - -""" - Mod updater script - - This script packs up the coop mod files, writes them to /opt/faf/data/content/legacy-featured-mod-files/.../, and updates the database. - - Code is mostly self-explanatory - haha, fat chance! Read it from bottom to top and don't blink. Blink and you're dead, no wait where were we? - To adapt this duct-tape based blob of shit for new mission voice overs, just change files array at the very bottom. - - Environment variables required: - PATCH_VERSION - DATABASE_HOST - DATABASE_NAME - DATABASE_USERNAME - DATABASE_PASSWORD -""" -import glob -import hashlib -import json -import os -import re -import shutil -import subprocess -import sys -import tempfile -import urllib.request -import urllib.error -import zipfile - -import mysql.connector - -FIXED_ZIP_TIMESTAMP = (1980, 1, 1, 0, 0, 0) # year, month, day, hour, min, sec - - -def get_db_connection(): - """Establish and return a MySQL connection using environment variables.""" - host = os.getenv("DATABASE_HOST", "localhost") - db = os.getenv("DATABASE_NAME", "faf") - user = os.getenv("DATABASE_USERNAME", "root") - password = os.getenv("DATABASE_PASSWORD", "banana") - - return mysql.connector.connect( - host=host, - user=user, - password=password, - database=db, - charset="utf8mb4", - collation="utf8mb4_unicode_ci", - ) - - -def read_db(conn, mod): - """ - Read latest versions and md5's from db - Returns dict {fileId: {version, name, md5}} - """ - query = f""" - SELECT uf.fileId, uf.version, uf.name, uf.md5 - FROM ( - SELECT fileId, MAX(version) AS version - FROM updates_{mod}_files - GROUP BY fileId - ) AS maxthings - INNER JOIN updates_{mod}_files AS uf - ON maxthings.fileId = uf.fileId AND maxthings.version = uf.version; - """ - - with conn.cursor() as cursor: - cursor.execute(query) - - oldfiles = {} - for (fileId, version, name, md5) in cursor.fetchall(): - oldfiles[int(fileId)] = { - "version": version, - "name": name, - "md5": md5, - } - - return oldfiles - - -def update_db(conn, mod, fileId, version, name, md5, dryrun): - """ - Delete and reinsert a file record in updates_{mod}_files - """ - delete_query = f"DELETE FROM updates_{mod}_files WHERE fileId=%s AND version=%s" - insert_query = f""" - INSERT INTO updates_{mod}_files (fileId, version, name, md5, obselete) - VALUES (%s, %s, %s, %s, 0) - """ - - print(f"Updating DB for {name} (fileId={fileId}, version={version})") - - if not dryrun: - try: - with conn.cursor() as cursor: - cursor.execute(delete_query, (fileId, version)) - cursor.execute(insert_query, (fileId, version, name, md5)) - conn.commit() - except mysql.connector.Error as err: - print(f"MySQL error while updating {name}: {err}") - conn.rollback() - exit(1) - else: - print(f"Dryrun: would run for {name}") - -def calc_md5(fname): - hash_md5 = hashlib.md5() - with open(fname, "rb") as f: - for chunk in iter(lambda: f.read(4096), b""): - hash_md5.update(chunk) - return hash_md5.hexdigest() - -def zipdir(path, ziph): - """ - Adds a path (folder or file) to the zip file handle `ziph`. - Preserves the directory name in the archive. - """ - if not os.path.exists(path): - print(f"Warning: {path} does not exist, skipping") - return - - # Remove trailing slash if present, otherwise dirname might return the path itself - clean_path = path.rstrip(os.sep) - - # Calculate the base directory (parent of the target path) - # This ensures that if we zip '/repo/lua', the archive entry starts with 'lua/...' - base_dir = os.path.dirname(clean_path) - - if os.path.isdir(clean_path): - for root, dirs, files in os.walk(clean_path): - files.sort() # deterministic order - dirs.sort() # deterministic order - for file in files: - full_path = os.path.join(root, file) - # Relpath relative to base_dir (parent) ensures folder structure is kept - arcname = os.path.relpath(full_path, start=base_dir) - info = zipfile.ZipInfo(arcname, FIXED_ZIP_TIMESTAMP) - with open(full_path, "rb") as f: - data = f.read() - ziph.writestr(info, data, compress_type=zipfile.ZIP_DEFLATED) - else: - # single file outside a directory - arcname = os.path.relpath(clean_path, start=base_dir) - info = zipfile.ZipInfo(arcname, FIXED_ZIP_TIMESTAMP) - with open(clean_path, "rb") as f: - data = f.read() - ziph.writestr(info, data, compress_type=zipfile.ZIP_DEFLATED) - - -def create_file(conn, mod, fileId, version, name, source, target_dir, old_md5, dryrun): - """Pack or copy files, compare MD5, update DB if changed.""" - target_dir = os.path.join(target_dir, f"updates_{mod}_files") - os.makedirs(target_dir, exist_ok=True) - - name = name.format(version) - target_name = os.path.join(target_dir, name) - - print(f"Processing {name} (fileId {fileId})") - - if isinstance(source, list): - print(f"Zipping {source} -> {target_name}") - fd, fname = tempfile.mkstemp("_" + name, "patcher_") - os.close(fd) - with zipfile.ZipFile(fname, "w", zipfile.ZIP_DEFLATED) as zf: - for sm in source: - zipdir(sm, zf) - rename = True - checksum = calc_md5(fname) - else: - rename = False - fname = source - if source is None: - checksum = calc_md5(target_name) if os.path.exists(target_name) else None - else: - checksum = calc_md5(fname) - - if checksum is None: - print(f"Skipping {name} (no source file and no existing file to checksum)") - return - - print(f"Compared checksums: Old {old_md5} New {checksum}") - - # Otherwise proceed with copy + DB update - print(f"Detected content change for {name}: old md5={old_md5}, new md5={checksum}") - if fname is not None: - print(f"Copying {fname} -> {target_name}") - if not dryrun: - shutil.copy(fname, target_name) - else: - print("No source file, not moving") - - if os.path.exists(target_name): - update_db(conn, mod, fileId, version, name, checksum, dryrun) - if not dryrun: - try: - os.chmod(target_name, 0o664) - except PermissionError: - print(f"Warning: Could not chmod {target_name}") - else: - print(f"Target file {target_name} does not exist, not updating db") - - -def do_files(conn, mod, version, files, target_dir, dryrun): - """Process all files for given mod/version.""" - current_files = read_db(conn, mod) - for name, fileId, source in files: - old_md5 = current_files.get(fileId, {}).get("md5") - create_file(conn, mod, fileId, version, name, source, target_dir, old_md5, dryrun) - - -def prepare_repo(): - """Clone or update the fa-coop repository and checkout the specified ref.""" - repo_url = os.getenv("GIT_REPO_URL", "https://github.com/FAForever/fa-coop.git") - git_ref = os.getenv("GIT_REF", "v" + os.getenv("PATCH_VERSION")) - workdir = os.getenv("GIT_WORKDIR", "/tmp/fa-coop") - - if not git_ref: - print("Error: GIT_REF or PATCH_VERSION must be specified.") - sys.exit(1) - - print(f"=== Preparing repository {repo_url} at ref {git_ref} in {workdir} ===") - - # Clone if not exists - if not os.path.isdir(os.path.join(workdir, ".git")): - print(f"Cloning repository into {workdir} ...") - subprocess.check_call(["git", "clone", repo_url, workdir]) - else: - print(f"Repository already exists in {workdir}, fetching latest changes...") - subprocess.check_call(["git", "-C", workdir, "fetch", "--all", "--tags"]) - - # Checkout the desired ref - print(f"Checking out {git_ref} ...") - subprocess.check_call(["git", "-C", workdir, "fetch", "--tags"]) - subprocess.check_call(["git", "-C", workdir, "checkout", git_ref]) - - print(f"=== Repository ready at {workdir} ===") - return workdir - - -def download_vo_assets(version, target_dir): - """ - Download VO .nx2 files from latest GitHub release of fa-coop, - rename them for the given patch version, and copy to target directory. - """ - os.makedirs(target_dir, exist_ok=True) - print(f"Fetching VO assets for patch version {version}...") - - # 1. Get latest release JSON from GitHub - api_url = f"https://api.github.com/repos/FAForever/fa-coop/releases/tags/v{version}" - try: - with urllib.request.urlopen(api_url) as response: - release_info = json.load(response) - except urllib.error.HTTPError as e: - print(f"Failed to fetch release info: {e}") - return - - # 2. Filter assets ending with .nx2 - nx2_urls = [ - asset["browser_download_url"] - for asset in release_info.get("assets", []) - if asset["browser_download_url"].endswith(".nx2") - ] - - if not nx2_urls: - print("No VO .nx2 assets found in the latest release.") - return - - temp_dir = os.path.join("/tmp", f"vo_download_{version}") - os.makedirs(temp_dir, exist_ok=True) - - # 3. Download each .nx2 file - for url in nx2_urls: - filename = os.path.basename(url) - dest_path = os.path.join(temp_dir, filename) - print(f"Downloading {url} -> {dest_path}") - urllib.request.urlretrieve(url, dest_path) - - # 4. Rename files to include patch version (e.g., A01_VO.v49.nx2) - for filepath in glob.glob(os.path.join(temp_dir, "*.nx2")): - base = os.path.basename(filepath) - # Insert .vXX. before the extension - new_name = re.sub(r"\.nx2$", f".v{version}.nx2", base) - new_path = os.path.join(temp_dir, new_name) - os.rename(filepath, new_path) - - # 5. Copy to target directory - for filepath in glob.glob(os.path.join(temp_dir, "*.nx2")): - target_path = os.path.join(target_dir, os.path.basename(filepath)) - print(f"Copying {filepath} -> {target_path}") - shutil.copy(filepath, target_path) - # Set permissions like in your script - os.chmod(target_path, 0o664) - try: - shutil.chown(target_path, group="www-data") - except Exception: - print(f"Warning: could not chown {target_path}, continue...") - - print("VO assets processed successfully.") - - -def main(): - mod = "coop" - dryrun = os.getenv("DRY_RUN", "false").lower() in ("1", "true", "yes") - version = os.getenv("PATCH_VERSION") - - if version is None: - print('Please pass patch version in environment variable PATCH_VERSION') - sys.exit(1) - - print(f"=== Starting mod updater for version {version}, dryrun={dryrun} ===") - - # /updater_{mod}_files will be appended by create_file - target_dir = '/tmp/legacy-featured-mod-files' - - # Prepare git repo - repo_dir = prepare_repo() - - # Download VO assets - vo_dir = os.path.join(target_dir, f"updates_{mod}_files") - download_vo_assets(version, vo_dir) - - # target filename / fileId in updates_{mod}_files table / source files with version placeholder - # if source files is single string, file is copied directly - # if source files is a list, files are zipped - files = [ - ('init_coop.v{}.lua', 1, os.path.join(repo_dir, 'init_coop.lua')), - ('lobby_coop_v{}.cop', 2, [ - os.path.join(repo_dir, 'lua'), - os.path.join(repo_dir, 'mods'), - os.path.join(repo_dir, 'units'), - os.path.join(repo_dir, 'mod_info.lua'), - os.path.join(repo_dir, 'readme.md'), - os.path.join(repo_dir, 'changelog.md'), - ]), - ('A01_VO.v{}.nx2', 3, None), - ('A02_VO.v{}.nx2', 4, None), - ('A03_VO.v{}.nx2', 5, None), - ('A04_VO.v{}.nx2', 6, None), - ('A05_VO.v{}.nx2', 7, None), - ('A06_VO.v{}.nx2', 8, None), - ('C01_VO.v{}.nx2', 9, None), - ('C02_VO.v{}.nx2', 10, None), - ('C03_VO.v{}.nx2', 11, None), - ('C04_VO.v{}.nx2', 12, None), - ('C05_VO.v{}.nx2', 13, None), - ('C06_VO.v{}.nx2', 14, None), - ('E01_VO.v{}.nx2', 15, None), - ('E02_VO.v{}.nx2', 16, None), - ('E03_VO.v{}.nx2', 17, None), - ('E04_VO.v{}.nx2', 18, None), - ('E05_VO.v{}.nx2', 19, None), - ('E06_VO.v{}.nx2', 20, None), - ('Prothyon16_VO.v{}.nx2', 21, None), - ('TCR_VO.v{}.nx2', 22, None), - ('SCCA_Briefings.v{}.nx2', 23, None), - ('SCCA_FMV.nx2.v{}.nx2', 24, None), - ('FAF_Coop_Operation_Tight_Spot_VO.v{}.nx2', 25, None), - ] - - conn = get_db_connection() - try: - do_files(conn, mod, version, files, target_dir, dryrun) - finally: - conn.close() - - print(f"=== Deployment finished for version {version}, dryrun={dryrun} ===") - - -if __name__ == "__main__": - main() \ No newline at end of file diff --git a/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml b/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml index 37ee5092..f1629a38 100644 --- a/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml +++ b/apps/faf-legacy-deployment/templates/deploy-coop-maps.yaml @@ -8,7 +8,7 @@ data: PATCH_VERSION: "65" DATABASE_HOST: "mariadb" DATABASE_NAME: "faf_lobby" - "deploy-coop.py": |- + "deploy-coop-maps.py": |- {{ tpl ( .Files.Get "scripts/deploy-coop-maps.py" ) . | indent 4 }} --- @@ -44,11 +44,11 @@ spec: - secretRef: name: faf-legacy-deployment command: [ "sh" ] - args: [ "-c", "pip install mysql-connector-python && python3 /tmp/deploy-coop.py" ] + args: [ "-c", "pip install mysql-connector-python && python3 /tmp/deploy-coop-maps.py" ] volumeMounts: - - mountPath: /tmp/deploy-coop.py + - mountPath: /tmp/deploy-coop-maps.py name: faf-deploy-coop-maps - subPath: "deploy-coop.py" + subPath: "deploy-coop-maps.py" restartPolicy: Never volumes: - name: faf-deploy-coop-maps diff --git a/apps/faf-legacy-deployment/templates/deploy-coop.yaml b/apps/faf-legacy-deployment/templates/deploy-coop.yaml index 3d6c676f..6cdbec34 100644 --- a/apps/faf-legacy-deployment/templates/deploy-coop.yaml +++ b/apps/faf-legacy-deployment/templates/deploy-coop.yaml @@ -8,8 +8,10 @@ data: PATCH_VERSION: "66" DATABASE_HOST: "mariadb" DATABASE_NAME: "faf_lobby" - "deploy-coop.py": |- -{{ tpl ( .Files.Get "scripts/deploy-coop.py" ) . | indent 4 }} + "build.gradle.kts": |- +{{ tpl ( .Files.Get "scripts/build.gradle.kts" ) . | indent 4 }} + "CoopDeployer.kt": |- +{{ tpl ( .Files.Get "scripts/CoopDeployer.kt" ) . | indent 4 }} --- @@ -35,21 +37,24 @@ spec: template: spec: containers: - - image: python:3.13 + - image: gradle:9.2-jdk21 imagePullPolicy: Always name: faf-coop-deployment + workingDir: /workspace envFrom: - configMapRef: name: faf-deploy-coop - secretRef: name: faf-legacy-deployment - command: [ "sh" ] - args: [ "-c", "pip install mysql-connector-python && python3 /tmp/deploy-coop.py" ] + command: [ "gradle", "run" ] volumeMounts: - - mountPath: /tmp/deploy-coop.py + - mountPath: /workspace/build.gradle.kts name: faf-deploy-coop - subPath: "deploy-coop.py" - - mountPath: /tmp/legacy-featured-mod-files + subPath: "build.gradle.kts" + - mountPath: /workspace/CoopDeployer.kt + name: faf-deploy-coop + subPath: "CoopDeployer.kt" + - mountPath: /workspace/legacy-featured-mod-files name: faf-featured-mods restartPolicy: Never volumes: From 2d138c1aceddf6af0deeb80bdb21513650829525 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Fri, 26 Dec 2025 21:12:23 +0100 Subject: [PATCH 14/33] fix fa zip handling --- .../scripts/CoopDeployer.kt | 20 +++++++++++++++---- 1 file changed, 16 insertions(+), 4 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index 28dd751d..e7f6f8ab 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -379,11 +379,23 @@ class Patcher( require(Files.isRegularFile(path)) { "Path $path is not a regular file" } val archiveName = base.relativize(path).toString().replace("\\", "/") - val entry = ZipEntry(archiveName) - // fix timestamp for determinism (not strictly necessary) - entry.time = 315532800000L + + + // Read file fully (FA requires sizes & CRC up front otherwise can't read the zip file) + val bytes = Files.readAllBytes(path) + val crc = java.util.zip.CRC32().apply { update(bytes) } + + val entry = ZipEntry(archiveName).apply { + method = ZipEntry.DEFLATED + size = bytes.size.toLong() + compressedSize = -1 // let deflater handle, still no descriptor + this.crc = crc.value + // fix timestamp for determinism (not strictly necessary) + time = 315532800000L // 1980-01-01 + } + this.putNextEntry(entry) - Files.newInputStream(path).use { inp -> inp.copyTo(this) } + this.write(bytes) this.closeEntry() } From c6b8f5209f6de7cb15bae490c40cbe9081101607 Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Fri, 26 Dec 2025 22:10:05 +0100 Subject: [PATCH 15/33] Try apache commons compress --- .../scripts/CoopDeployer.kt | 41 +++++++++++-------- .../scripts/build.gradle.kts | 1 + 2 files changed, 25 insertions(+), 17 deletions(-) diff --git a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt index e7f6f8ab..4e3859cb 100755 --- a/apps/faf-legacy-deployment/scripts/CoopDeployer.kt +++ b/apps/faf-legacy-deployment/scripts/CoopDeployer.kt @@ -1,3 +1,6 @@ +import org.apache.commons.compress.archivers.zip.Zip64Mode +import org.apache.commons.compress.archivers.zip.ZipArchiveEntry +import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream import org.eclipse.jgit.api.Git import org.slf4j.LoggerFactory import java.io.IOException @@ -9,11 +12,13 @@ import java.nio.file.Files import java.nio.file.Path import java.nio.file.Paths import java.nio.file.StandardCopyOption +import java.nio.file.attribute.FileTime import java.nio.file.attribute.PosixFilePermission import java.security.MessageDigest import java.sql.Connection import java.sql.DriverManager import java.time.Duration +import java.util.zip.CRC32 import java.util.zip.ZipEntry import java.util.zip.ZipOutputStream import kotlin.io.path.inputStream @@ -227,6 +232,9 @@ data class FafDatabase( } } +private const val MINIMUM_ZIP_DATE = 315532800000L // 1980-01-01 +private val MINIMUM_ZIP_FILE_TIME = FileTime.fromMillis(MINIMUM_ZIP_DATE) + class Patcher( val patchVersion: Int, val targetDir: Path, @@ -355,7 +363,11 @@ class Patcher( private fun zipPreserveStructure(sources: List, outputFile: Path, base: Path) { Files.createDirectories(outputFile.parent) - ZipOutputStream(Files.newOutputStream(outputFile)).use { zos -> + + // Never pass a stream here; this will cause extended local headers to be used, making it incompatible to FA! + ZipArchiveOutputStream(outputFile.toFile()).use { zos -> + zos.setMethod(ZipArchiveEntry.DEFLATED) + for (src in sources) { if (!Files.exists(src)) { // skip @@ -375,28 +387,23 @@ class Patcher( } } - private fun ZipOutputStream.pushNormalizedFile(base: Path, path: Path) { + private fun ZipArchiveOutputStream.pushNormalizedFile(base: Path, path: Path) { require(Files.isRegularFile(path)) { "Path $path is not a regular file" } val archiveName = base.relativize(path).toString().replace("\\", "/") - - // Read file fully (FA requires sizes & CRC up front otherwise can't read the zip file) - val bytes = Files.readAllBytes(path) - val crc = java.util.zip.CRC32().apply { update(bytes) } - - val entry = ZipEntry(archiveName).apply { - method = ZipEntry.DEFLATED - size = bytes.size.toLong() - compressedSize = -1 // let deflater handle, still no descriptor - this.crc = crc.value - // fix timestamp for determinism (not strictly necessary) - time = 315532800000L // 1980-01-01 + // Use the same constructor as the FAF API: + val entry = ZipArchiveEntry(path.toFile(), archiveName).apply { + // Ensure deterministic times + setTime(MINIMUM_ZIP_FILE_TIME) + setCreationTime(MINIMUM_ZIP_FILE_TIME) + setLastModifiedTime(MINIMUM_ZIP_FILE_TIME) + setLastAccessTime(MINIMUM_ZIP_FILE_TIME) } - this.putNextEntry(entry) - this.write(bytes) - this.closeEntry() + this.putArchiveEntry(entry) + Files.newInputStream(path).use { inp -> inp.copyTo(this) } + this.closeArchiveEntry() } } diff --git a/apps/faf-legacy-deployment/scripts/build.gradle.kts b/apps/faf-legacy-deployment/scripts/build.gradle.kts index ec9ea43c..10f0c44b 100644 --- a/apps/faf-legacy-deployment/scripts/build.gradle.kts +++ b/apps/faf-legacy-deployment/scripts/build.gradle.kts @@ -10,6 +10,7 @@ repositories { dependencies { implementation("org.mariadb.jdbc:mariadb-java-client:3.5.7") implementation("org.eclipse.jgit:org.eclipse.jgit:7.5.0.202512021534-r") + implementation("org.apache.commons:commons-compress:1.28.0") implementation("org.slf4j:slf4j-api:2.0.13") runtimeOnly("ch.qos.logback:logback-classic:1.5.23") } From 4abaeae320ea675ff628bed50b70a00c134c728d Mon Sep 17 00:00:00 2001 From: Brutus5000 Date: Fri, 26 Dec 2025 23:53:00 +0100 Subject: [PATCH 16/33] Improve helm script bundling --- .../templates/deploy-coop.yaml | 11 ++++++----- .../faf-legacy-deployment/templates/scripts-cm.yaml | 13 +++++++++++++ 2 files changed, 19 insertions(+), 5 deletions(-) create mode 100644 apps/faf-legacy-deployment/templates/scripts-cm.yaml diff --git a/apps/faf-legacy-deployment/templates/deploy-coop.yaml b/apps/faf-legacy-deployment/templates/deploy-coop.yaml index 6cdbec34..e181a37e 100644 --- a/apps/faf-legacy-deployment/templates/deploy-coop.yaml +++ b/apps/faf-legacy-deployment/templates/deploy-coop.yaml @@ -39,7 +39,7 @@ spec: containers: - image: gradle:9.2-jdk21 imagePullPolicy: Always - name: faf-coop-deployment + name: faf-deploy-coop workingDir: /workspace envFrom: - configMapRef: @@ -47,20 +47,21 @@ spec: - secretRef: name: faf-legacy-deployment command: [ "gradle", "run" ] + # We need to mount single files via subpath because Gradle breaks otherwise (symbolic link to read-only directory) volumeMounts: - mountPath: /workspace/build.gradle.kts - name: faf-deploy-coop + name: faf-deploy-scripts subPath: "build.gradle.kts" - mountPath: /workspace/CoopDeployer.kt - name: faf-deploy-coop + name: faf-deploy-scripts subPath: "CoopDeployer.kt" - mountPath: /workspace/legacy-featured-mod-files name: faf-featured-mods restartPolicy: Never volumes: - - name: faf-deploy-coop + - name: faf-deploy-scripts configMap: - name: "faf-deploy-coop" + name: "faf-deploy-scripts" - name: faf-featured-mods hostPath: path: /opt/faf/data/legacy-featured-mod-files diff --git a/apps/faf-legacy-deployment/templates/scripts-cm.yaml b/apps/faf-legacy-deployment/templates/scripts-cm.yaml new file mode 100644 index 00000000..caa91380 --- /dev/null +++ b/apps/faf-legacy-deployment/templates/scripts-cm.yaml @@ -0,0 +1,13 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: faf-deploy-scripts + labels: + app: faf-deploy-coop +data: + # Loop through all files in the 'scripts' directory + {{- range $path, $bytes := .Files.Glob "scripts/*" }} + {{- $file := base $path }} + {{ $file }}: |- +{{ tpl ($bytes | toString) $ | indent 4 }} + {{- end }} \ No newline at end of file From 36d4266b5a296ac644b6c035ae2f667fb3fdd729 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 10:39:30 -0700 Subject: [PATCH 17/33] Set up tilt CI for testing gitops-stack --- .github/worklows/checks.yml | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) create mode 100644 .github/worklows/checks.yml diff --git a/.github/worklows/checks.yml b/.github/worklows/checks.yml new file mode 100644 index 00000000..c7766c9b --- /dev/null +++ b/.github/worklows/checks.yml @@ -0,0 +1,24 @@ +name: Checks + +on: + push: + pull_request: + branches: [ develop ] + +jobs: + checks: + + runs-on: ubuntu-latest + container: + image: docker/tilt:latest + + steps: + - uses: actions/checkout@v4 + + - name: Create k8s Kind Cluster + run: ctlptl create cluster kind --registry=ctlptl-registry + + - name: Test Using Local Config + run: tilt ci + + From 54cf5865fedc0b1ed3b4f18048862608b531e5cf Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 10:44:40 -0700 Subject: [PATCH 18/33] Use array syntax --- .github/worklows/checks.yml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/worklows/checks.yml b/.github/worklows/checks.yml index c7766c9b..6b1f36cd 100644 --- a/.github/worklows/checks.yml +++ b/.github/worklows/checks.yml @@ -3,7 +3,8 @@ name: Checks on: push: pull_request: - branches: [ develop ] + branches: + - develop jobs: checks: From e19cdf7596e743431b8aaf493d98b7e8608a39d4 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 10:51:28 -0700 Subject: [PATCH 19/33] Correct folder name --- .github/{worklows => workflows}/checks.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{worklows => workflows}/checks.yml (100%) diff --git a/.github/worklows/checks.yml b/.github/workflows/checks.yml similarity index 100% rename from .github/worklows/checks.yml rename to .github/workflows/checks.yml From 36279936f621228093141cc4f049215f77385d4b Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 10:53:55 -0700 Subject: [PATCH 20/33] Make helm-with-cache.sh executable --- tilt/scripts/helm-with-cache.sh | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100644 => 100755 tilt/scripts/helm-with-cache.sh diff --git a/tilt/scripts/helm-with-cache.sh b/tilt/scripts/helm-with-cache.sh old mode 100644 new mode 100755 From be9779c00c0362e4a0761c9e712a3e80bd28d1ae Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 10:58:55 -0700 Subject: [PATCH 21/33] Add helm install --- .github/workflows/checks.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 6b1f36cd..ece603c6 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -19,6 +19,9 @@ jobs: - name: Create k8s Kind Cluster run: ctlptl create cluster kind --registry=ctlptl-registry + - name: Install Helm + run: snap install helm --classic + - name: Test Using Local Config run: tilt ci From ca899254604682a826c1c24140b90d6cb6ce1943 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 11:02:07 -0700 Subject: [PATCH 22/33] Use helm script for installing --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index ece603c6..fe032ead 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -20,8 +20,8 @@ jobs: run: ctlptl create cluster kind --registry=ctlptl-registry - name: Install Helm - run: snap install helm --classic - + run: curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 | bash + - name: Test Using Local Config run: tilt ci From f652ee134f405613ceaf6ac7e204e6f957a71273 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 11:11:30 -0700 Subject: [PATCH 23/33] Add traefik namespace to namespaces.yaml --- cluster/namespaces.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/cluster/namespaces.yaml b/cluster/namespaces.yaml index ac703faa..5b72ac64 100644 --- a/cluster/namespaces.yaml +++ b/cluster/namespaces.yaml @@ -13,4 +13,10 @@ metadata: apiVersion: v1 kind: Namespace metadata: - name: faf-ops \ No newline at end of file + name: faf-ops + +--- +apiVersion: v1 +kind: Namespace +metadata: + name: traefik \ No newline at end of file From 466ca7be5ebdb41d6f17c7ce329f81fbc4223d8a Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 11:12:44 -0700 Subject: [PATCH 24/33] add dependency of traefik on namespaces --- Tiltfile | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Tiltfile b/Tiltfile index a4ab8816..b1028880 100644 --- a/Tiltfile +++ b/Tiltfile @@ -156,7 +156,7 @@ def no_policy_server(yaml): k8s_yaml("cluster/namespaces.yaml") k8s_yaml(helm_with_build_cache("infra/clusterroles", namespace="faf-infra", values=["config/local.yaml"])) -k8s_resource(new_name="namespaces", objects=["faf-infra:namespace", "faf-apps:namespace", "faf-ops:namespace"], labels=["core"]) +k8s_resource(new_name="namespaces", objects=["faf-infra:namespace", "faf-apps:namespace", "faf-ops:namespace", "traefik:namespace"], labels=["core"]) k8s_resource(new_name="clusterroles", objects=["read-cm-secrets:clusterrole"], labels=["core"]) k8s_resource(new_name="init-apps", objects=["init-apps:serviceaccount:faf-infra", "init-apps:serviceaccount:faf-apps", "allow-init-apps-read-app-config-infra:rolebinding", "allow-init-apps-read-app-config-apps:rolebinding"], resource_deps=["clusterroles"], labels=["core"]) @@ -182,7 +182,7 @@ for object in decode_yaml_stream(traefik_yaml): if kind != "deployment" and kind != "service": traefik_identifiers.append(name + ":" + kind) -k8s_resource(new_name="traefik-setup", objects=traefik_identifiers, labels=["traefik"]) +k8s_resource(new_name="traefik-setup", objects=traefik_identifiers, resource_deps=["namespaces"], labels=["traefik"]) k8s_resource(workload="release-name-traefik", new_name="traefik", port_forwards=["443:8443"], resource_deps=["traefik-setup"], labels=["traefik"]) postgres_yaml = helm_with_build_cache("infra/postgres", namespace="faf-infra", values=["config/local.yaml"]) From 001c9fa723473960011a17f935090544f1061b3c Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 11:24:47 -0700 Subject: [PATCH 25/33] Add proper escape to hydra client init job --- apps/ory-hydra/templates/init-clients.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/ory-hydra/templates/init-clients.yaml b/apps/ory-hydra/templates/init-clients.yaml index 6e99a8b2..089ea9ef 100644 --- a/apps/ory-hydra/templates/init-clients.yaml +++ b/apps/ory-hydra/templates/init-clients.yaml @@ -66,7 +66,7 @@ spec: --policy-uri "{{ .policyUri }}" \ {{- end }} {{- if .tokenEndpointAuthMethod }} - --token-endpoint-auth-method "{{ .tokenEndpointAuthMethod }}" + --token-endpoint-auth-method "{{ .tokenEndpointAuthMethod }}" \ {{- end }} {{- if .owner }} --owner "{{ .owner }}" From 94983283df93b54a5933f94ad9c1fa59e87a2f76 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 11:47:39 -0700 Subject: [PATCH 26/33] Add icebreaker mariadb user creation --- apps/faf-icebreaker/templates/config.yaml | 1 + infra/mariadb/values.yaml | 7 +++++++ 2 files changed, 8 insertions(+) diff --git a/apps/faf-icebreaker/templates/config.yaml b/apps/faf-icebreaker/templates/config.yaml index 3186f54c..5890ab6e 100644 --- a/apps/faf-icebreaker/templates/config.yaml +++ b/apps/faf-icebreaker/templates/config.yaml @@ -9,6 +9,7 @@ data: HYDRA_URL: "https://hydra.{{.Values.baseDomain}}" SELF_URL: "https://ice.{{.Values.baseDomain}}" DB_USERNAME: "faf-icebreaker" + DB_NAME: "faf-icebreaker" DB_URL: "jdbc:mariadb://mariadb:3306/faf-icebreaker?ssl=false" RABBITMQ_HOST: "rabbitmq" RABBITMQ_USER: "faf-icebreaker" diff --git a/infra/mariadb/values.yaml b/infra/mariadb/values.yaml index 0845f456..cff135f2 100644 --- a/infra/mariadb/values.yaml +++ b/infra/mariadb/values.yaml @@ -50,6 +50,13 @@ databasesAndUsers: usernameKey: DB_LOGIN passwordKey: DB_PASSWORD + # Icebreaker database + - configMapRef: faf-icebreaker + secretRef: faf-icebreaker + databaseKey: DB_NAME + usernameKey: DB_USERNAME + passwordKey: DB_PASSWORD + # Others - configMapRef: wordpress secretRef: wordpress From 67efdb24dba2123f22eba87a60f1d4c36a7e9633 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 12:05:12 -0700 Subject: [PATCH 27/33] Update icebreak version to support default --- apps/faf-icebreaker/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/faf-icebreaker/templates/deployment.yaml b/apps/faf-icebreaker/templates/deployment.yaml index b9accae9..442101ee 100644 --- a/apps/faf-icebreaker/templates/deployment.yaml +++ b/apps/faf-icebreaker/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: - name: geolite-db mountPath: /usr/share/GeoIP containers: - - image: faforever/faf-icebreaker:1.2.0-RC2 + - image: faforever/faf-icebreaker:1.2.0-RC3 imagePullPolicy: Always name: faf-icebreaker envFrom: From 6f44cf379a7c84da56e0348b2b198c73364dd4c2 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 13:07:09 -0700 Subject: [PATCH 28/33] Set 5m timeout --- .github/workflows/checks.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index fe032ead..2825e145 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -21,8 +21,8 @@ jobs: - name: Install Helm run: curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 | bash - + - name: Test Using Local Config - run: tilt ci + run: tilt ci --timeout "5m" From 790a9f23feacb9b7b6507ad0607e672b19a58348 Mon Sep 17 00:00:00 2001 From: Sheikah45 <66929319+Sheikah45@users.noreply.github.com> Date: Tue, 23 Dec 2025 16:22:48 -0500 Subject: [PATCH 29/33] Add CPU and memory request to user service --- apps/faf-user-service/templates/deployment.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/apps/faf-user-service/templates/deployment.yaml b/apps/faf-user-service/templates/deployment.yaml index 616bb16e..debb1588 100644 --- a/apps/faf-user-service/templates/deployment.yaml +++ b/apps/faf-user-service/templates/deployment.yaml @@ -40,6 +40,9 @@ spec: limits: memory: 10Gi cpu: 3000m + requests: + memory: 2Gi + cpu: 1000m startupProbe: httpGet: port: 8080 From ed6de34d4c7c3b39df64a887afe53bfd27f74866 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 15:07:57 -0700 Subject: [PATCH 30/33] Use azure setup-helm action --- .github/workflows/checks.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/workflows/checks.yml b/.github/workflows/checks.yml index 2825e145..8e820801 100644 --- a/.github/workflows/checks.yml +++ b/.github/workflows/checks.yml @@ -16,12 +16,11 @@ jobs: steps: - uses: actions/checkout@v4 + - uses: azure/setup-helm@v4.3.0 + - name: Create k8s Kind Cluster run: ctlptl create cluster kind --registry=ctlptl-registry - - name: Install Helm - run: curl https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-4 | bash - - name: Test Using Local Config run: tilt ci --timeout "5m" From 0499e483e1acdbe60000e84594bcc0c84c5129b3 Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 16:18:59 -0700 Subject: [PATCH 31/33] Fix hydra init errors and website urls --- Tiltfile | 6 ++++-- apps/ory-hydra/templates/init-clients.yaml | 3 ++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/Tiltfile b/Tiltfile index b1028880..f7db4af4 100644 --- a/Tiltfile +++ b/Tiltfile @@ -237,9 +237,11 @@ k8s_resource(workload="populate-db", resource_deps=["faf-db-migrations"], labels k8s_yaml(keep_objects_of_kind(helm_with_build_cache("apps/faf-voting", namespace="faf-apps", values=["config/local.yaml"]), kinds=["ConfigMap", "Secret"])) k8s_resource(new_name="faf-voting-config", objects=["faf-voting:configmap", "faf-voting:secret"], labels=["voting"]) -k8s_yaml(helm_with_build_cache("apps/faf-website", namespace="faf-apps", values=["config/local.yaml", "apps/faf-website/values-prod.yaml"])) +website_yaml = helm_with_build_cache("apps/faf-website", namespace="faf-apps", values=["config/local.yaml", "apps/faf-website/values-prod.yaml"]) +website_yaml = patch_config(website_yaml, "faf-website", {"OAUTH_URL": "http://ory-hydra:4444", "OAUTH_PUBLIC_URL": "http://localhost:4444", "API_URL": "http://faf-api:8010", "WP_URL": "http://wordpress:80"}) +k8s_yaml(website_yaml) k8s_resource(new_name="faf-website-config", objects=["faf-website:configmap", "faf-website:secret"], labels=["website"]) -k8s_resource(workload="faf-website", objects=["faf-website:ingressroute"], resource_deps=["traefik"], labels=["website"], links=[link("https://www.localhost", "FAForever Website")]) +k8s_resource(workload="faf-website", objects=["faf-website:ingressroute"], resource_deps=["traefik", "wordpress"], labels=["website"], links=[link("https://www.localhost", "FAForever Website")]) # k8s_yaml(helm_with_build_cache("apps/faf-content", namespace="faf-apps", values=["config/local.yaml"])) # k8s_resource(new_name="faf-content-config", objects=["faf-content:configmap"], labels=["content"]) diff --git a/apps/ory-hydra/templates/init-clients.yaml b/apps/ory-hydra/templates/init-clients.yaml index 089ea9ef..bfb76466 100644 --- a/apps/ory-hydra/templates/init-clients.yaml +++ b/apps/ory-hydra/templates/init-clients.yaml @@ -69,8 +69,9 @@ spec: --token-endpoint-auth-method "{{ .tokenEndpointAuthMethod }}" \ {{- end }} {{- if .owner }} - --owner "{{ .owner }}" + --owner "{{ .owner }}" \ {{- end }} + ; else echo "Client {{ .id }} already exists, skipping." fi From c99954930a91f6affa8add7d1e3250af3d4add2a Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Tue, 23 Dec 2025 16:28:32 -0700 Subject: [PATCH 32/33] Update readme for test data --- README.MD | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.MD b/README.MD index 160d8014..210d979f 100644 --- a/README.MD +++ b/README.MD @@ -79,5 +79,5 @@ In the root directory of the repository run `tilt up`. This will start all the f To develop against the FAF infrastructure you should disable the service in tilt that you are actively developing. Once disabled you can start up your developed version. Some tweaks may need to be made to the default configuration parameters in the source code. The proper values can be found in the configMaps in each of the services kubernetes deploy yaml files. ## Test Data -The default test data that is loaded can be found in /sql/test-data.sql. This can be overridden by providing a new path with the tilt configuration key test-data-path when running tilt up or in the tilt_config.json file in the repository root directory. +The default test data that is loaded can be found in [faf-db](https://github.com/FAForever/db/blob/develop/test-data.sql) From 5f5a0cdc365f38554f6ccbd0800d41770c01d50a Mon Sep 17 00:00:00 2001 From: Sheikah45 Date: Fri, 26 Dec 2025 22:57:23 -0500 Subject: [PATCH 33/33] Update to RC4 --- apps/faf-icebreaker/templates/deployment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/apps/faf-icebreaker/templates/deployment.yaml b/apps/faf-icebreaker/templates/deployment.yaml index 442101ee..6c0e0c1d 100644 --- a/apps/faf-icebreaker/templates/deployment.yaml +++ b/apps/faf-icebreaker/templates/deployment.yaml @@ -32,7 +32,7 @@ spec: - name: geolite-db mountPath: /usr/share/GeoIP containers: - - image: faforever/faf-icebreaker:1.2.0-RC3 + - image: faforever/faf-icebreaker:1.2.0-RC4 imagePullPolicy: Always name: faf-icebreaker envFrom: