diff --git a/configuration/builders/definitions/connectors/README.md b/configuration/builders/definitions/connectors/README.md index e183a46bf..193103122 100644 --- a/configuration/builders/definitions/connectors/README.md +++ b/configuration/builders/definitions/connectors/README.md @@ -26,7 +26,7 @@ Under `CONNECTORS_PACKAGES_DIR/` on the worker hosts, served at `/` | -| A builder's packages | `//` | +| A builder's packages and `sha256sums.txt` | `//` | ## Maintenance diff --git a/configuration/builders/sequences/connectors/conc.py b/configuration/builders/sequences/connectors/conc.py index cff31d461..2e58c69b4 100644 --- a/configuration/builders/sequences/connectors/conc.py +++ b/configuration/builders/sequences/connectors/conc.py @@ -84,7 +84,6 @@ def tarball(config: DockerConfig): input_dir=PurePath("."), output_dir=PurePath("ci_source"), tarball_name="ci.tar.gz", - generate_sha256=True, ), options=StepOptions( description="Archive source code", @@ -124,9 +123,9 @@ def tarball(config: DockerConfig): packages=[ "ci_source/ci.tar.gz", "*src.tar.gz", - "ci_source/sha256sums.txt", ], destination="/packages/%(prop:buildnumber)s", + generate_sha256=True, ), url=URL( url=f"{os.environ['ARTIFACTS_URL']}/connector-c/%(prop:buildnumber)s", @@ -323,6 +322,7 @@ def save_packages(config: DockerConfig, packages: list[str], user: str = "buildb packages=packages, destination="/packages/%(prop:tarbuildnum)s/%(prop:buildername)s", user=user, + generate_sha256=True, ), options=StepOptions( description="Save packages", @@ -419,6 +419,20 @@ def windows(jobs: int, target_platform: str): ), ) + sequence.add_step( + ShellStep( + command=BashCommand( + name="MSI sha256sums", + cmd='sha256sum "%(prop:packages)s" > sha256sums.txt', + workdir=PurePath("win\\packaging"), + ), + options=StepOptions( + description="Write MSI sha256sums", + descriptionDone="MSI sha256sums written", + ), + ), + ) + sequence.add_step( FileUpload( workersrc="win\\packaging\\%(prop:packages)s", @@ -427,4 +441,13 @@ def windows(jobs: int, target_platform: str): url=f"{os.environ['ARTIFACTS_URL']}/connector-c/%(prop:tarbuildnum)s/%(prop:buildername)s/", ) ) + + sequence.add_step( + FileUpload( + workersrc="win\\packaging\\sha256sums.txt", + masterdest="/srv/buildbot/connectors/c/%(prop:tarbuildnum)s/%(prop:buildername)s/sha256sums.txt", + mode=0o644, + url=f"{os.environ['ARTIFACTS_URL']}/connector-c/%(prop:tarbuildnum)s/%(prop:buildername)s/sha256sums.txt", + ) + ) return sequence diff --git a/configuration/builders/sequences/connectors/concpp.py b/configuration/builders/sequences/connectors/concpp.py index 6a14732eb..7a7d0b828 100644 --- a/configuration/builders/sequences/connectors/concpp.py +++ b/configuration/builders/sequences/connectors/concpp.py @@ -92,7 +92,6 @@ def tarball(config: DockerConfig): input_dir=PurePath("."), output_dir=PurePath("ci_source"), tarball_name="ci.tar.gz", - generate_sha256=True, ), options=StepOptions( description="Archive source code", @@ -176,9 +175,9 @@ def tarball(config: DockerConfig): packages=[ "ci_source/ci.tar.gz", "*src.tar.gz", - "ci_source/sha256sums.txt", ], destination="/packages/%(prop:buildnumber)s", + generate_sha256=True, ), url=URL( url=f"{os.environ['ARTIFACTS_URL']}/connector-cpp/%(prop:buildnumber)s", @@ -798,6 +797,7 @@ def save_packages(config: DockerConfig, packages: list[str], user: str = "buildb packages=packages, destination="/packages/%(prop:tarbuildnum)s/%(prop:buildername)s", user=user, + generate_sha256=True, ), options=StepOptions( description="Save packages", @@ -893,6 +893,20 @@ def windows(jobs: int, target_platform: str): ), ) + sequence.add_step( + ShellStep( + command=BashCommand( + name="MSI sha256sums", + cmd='sha256sum "%(prop:packages)s" > sha256sums.txt', + workdir=PurePath("wininstall"), + ), + options=StepOptions( + description="Write MSI sha256sums", + descriptionDone="MSI sha256sums written", + ), + ), + ) + sequence.add_step( FileUpload( workersrc="wininstall\\%(prop:packages)s", @@ -901,4 +915,13 @@ def windows(jobs: int, target_platform: str): url=f"{os.environ['ARTIFACTS_URL']}/connector-cpp/%(prop:tarbuildnum)s/%(prop:buildername)s/", ) ) + + sequence.add_step( + FileUpload( + workersrc="wininstall\\sha256sums.txt", + masterdest="/srv/buildbot/connectors/cpp/%(prop:tarbuildnum)s/%(prop:buildername)s/sha256sums.txt", + mode=0o644, + url=f"{os.environ['ARTIFACTS_URL']}/connector-cpp/%(prop:tarbuildnum)s/%(prop:buildername)s/sha256sums.txt", + ) + ) return sequence diff --git a/configuration/builders/sequences/connectors/conodbc.py b/configuration/builders/sequences/connectors/conodbc.py index 490c94860..5c8affb2c 100644 --- a/configuration/builders/sequences/connectors/conodbc.py +++ b/configuration/builders/sequences/connectors/conodbc.py @@ -92,7 +92,6 @@ def tarball(config: DockerConfig): input_dir=PurePath("."), output_dir=PurePath("ci_source"), tarball_name="ci.tar.gz", - generate_sha256=True, ), options=StepOptions( description="Archive source code", @@ -177,9 +176,9 @@ def tarball(config: DockerConfig): packages=[ "ci_source/ci.tar.gz", "*src.tar.gz", - "ci_source/sha256sums.txt", ], destination="/packages/%(prop:buildnumber)s", + generate_sha256=True, ), url=URL( url=f"{os.environ['ARTIFACTS_URL']}/connector-odbc/%(prop:buildnumber)s", @@ -816,6 +815,7 @@ def save_packages(config: DockerConfig, packages: list[str], user: str = "buildb packages=packages, destination="/packages/%(prop:tarbuildnum)s/%(prop:buildername)s", user=user, + generate_sha256=True, ), options=StepOptions( description="Save packages", @@ -1012,6 +1012,20 @@ def windows(jobs: int, target_platform: str): ), ) + sequence.add_step( + ShellStep( + command=BashCommand( + name="MSI sha256sums", + cmd='sha256sum "%(prop:packages)s" > sha256sums.txt', + workdir=PurePath("packaging\\windows"), + ), + options=StepOptions( + description="Write MSI sha256sums", + descriptionDone="MSI sha256sums written", + ), + ), + ) + sequence.add_step( FileUpload( workersrc="packaging/windows/%(prop:packages)s", @@ -1021,6 +1035,15 @@ def windows(jobs: int, target_platform: str): ) ) + sequence.add_step( + FileUpload( + workersrc="packaging/windows/sha256sums.txt", + masterdest="/srv/buildbot/connectors/odbc/%(prop:tarbuildnum)s/%(prop:buildername)s/sha256sums.txt", + mode=0o644, + url=f"{os.environ['ARTIFACTS_URL']}/connector-odbc/%(prop:tarbuildnum)s/%(prop:buildername)s/sha256sums.txt", + ) + ) + sequence.add_step( ShellStep( command=PowerShellCommand( diff --git a/configuration/steps/commands/packages.py b/configuration/steps/commands/packages.py index 4906c3212..a431cd079 100644 --- a/configuration/steps/commands/packages.py +++ b/configuration/steps/commands/packages.py @@ -110,6 +110,8 @@ class SavePackages(Command): This class is used to recursively copy a list of files and dirs to CI, starting from the current working directory and assuming that /packages is bind mounted. + With generate_sha256, it also writes a sha256sums.txt listing every + file the build saved to the destination. """ def __init__( @@ -118,14 +120,34 @@ def __init__( workdir: PurePath = PurePath("."), destination: str = "/packages/%(prop:tarbuildnum)s/%(prop:buildername)s", user: str = "buildbot", + generate_sha256: bool = False, ): name = "Save packages" self.packages = packages self.destination = destination + self.generate_sha256 = generate_sha256 + assert not (generate_sha256 and workdir.is_absolute()) + # The sums are kept at the build's root, which the workdir is + # relative to: /home/buildbot in a container, build/ on a host. + root = "../" * len(workdir.parts) + self.sums_dir = f"{root}sha256sums/{destination.lstrip('/')}" super().__init__(name=name, workdir=workdir, user=user) def as_cmd_arg(self) -> list[str]: package_list = " ".join(self.packages) + sha256sums = f""" + mkdir -p "{self.sums_dir}" + for package in {package_list}; do + [[ -e "${{package}}" ]] || continue + ( + cd "$(dirname "${{package}}")" + find -L "$(basename "${{package}}")" -type f -exec sha256sum {{}} + + ) + done >> "{self.sums_dir}/sha256sums.txt" + if [[ -s "{self.sums_dir}/sha256sums.txt" ]]; then + cp -f "{self.sums_dir}/sha256sums.txt" "{self.destination}/sha256sums.txt" + fi + """ result = [ "bash", "-exc", @@ -139,6 +161,7 @@ def as_cmd_arg(self) -> list[str]: fi cp -r $package {self.destination} done + {sha256sums if self.generate_sha256 else ""} """ ), ] @@ -222,12 +245,10 @@ def __init__( input_dir: PurePath, output_dir: PurePath, tarball_name: str, - generate_sha256: bool = False, ): self.input_dir = input_dir self.output_dir = output_dir self.tarball_name = tarball_name - self.generate_sha256 = generate_sha256 super().__init__(name="Archive source code", workdir=input_dir) def as_cmd_arg(self) -> list[str]: @@ -237,7 +258,6 @@ def as_cmd_arg(self) -> list[str]: f""" mkdir -p {self.output_dir} tar --exclude-vcs --exclude {self.output_dir} -czf {self.output_dir}/{self.tarball_name} -C {self.input_dir} . - {f"sha256sum {self.output_dir}/{self.tarball_name} > {self.output_dir}/sha256sums.txt" if self.generate_sha256 else ""} """, ] return result diff --git a/script_templates/get_tarball.sh b/script_templates/get_tarball.sh index a92e907fe..8fcc7067f 100644 --- a/script_templates/get_tarball.sh +++ b/script_templates/get_tarball.sh @@ -26,7 +26,7 @@ f="${tarbuildnum}/$tarball" verify_checksum() { echo "verify checksum" - checksum=$(wget -qO- "$artifacts_url/$tarbuildnum/sha256sums.txt" | awk '{print $1}') + checksum=$(wget -qO- "$artifacts_url/$tarbuildnum/sha256sums.txt" | grep -F "${tarball}" | awk '{print $1}') checksum_file=$(sha256sum "$1" | awk '{print $1}') if [[ "$checksum_file" == "$checksum" ]]; then