Keep values tagged with !override when parsing compose files

core/src/main/java/org/testcontainers/containers/ParsedDockerComposeFile.java

@@ -1,6 +1,7 @@
 package org.testcontainers.containers;
 
 import com.google.common.annotations.VisibleForTesting;
+import com.google.common.collect.ImmutableSet;
 import com.google.common.collect.Sets;
 import lombok.EqualsAndHashCode;
 import lombok.Getter;
@@ -12,6 +13,8 @@
 import org.yaml.snakeyaml.Yaml;
 import org.yaml.snakeyaml.constructor.SafeConstructor;
 import org.yaml.snakeyaml.nodes.Node;
+import org.yaml.snakeyaml.nodes.NodeId;
+import org.yaml.snakeyaml.nodes.ScalarNode;
 import org.yaml.snakeyaml.nodes.Tag;
 import org.yaml.snakeyaml.representer.Representer;
 import org.yaml.snakeyaml.resolver.Resolver;
@@ -32,6 +35,19 @@
 @EqualsAndHashCode
 class ParsedDockerComposeFile {
 
+    /**
+     * Top-level elements of the Compose file format that are not services, e.g. in a file without a 'services' element
+     */
+    private static final Set<String> TOP_LEVEL_ELEMENTS = ImmutableSet.of(
+        "version",
+        "name",
+        "include",
+        "networks",
+        "volumes",
+        "configs",
+        "secrets"
+    );
+
     private final Map<String, Object> composeFileContent;
 
     private final String composeFileName;
@@ -46,17 +62,22 @@ class ParsedDockerComposeFile {
         LoaderOptions options = new LoaderOptions();
         options.setMaxAliasesForCollections(1_000);
         DumperOptions dumperOptions = new DumperOptions();
+        Resolver resolver = new Resolver();
 
         SafeConstructor constructor = new SafeConstructor(options) {
             @Override
             protected Object constructObject(Node node) {
-                if (node.getTag().equals(new Tag("!reset")) || node.getTag().equals(new Tag("!override"))) {
+                if (node.getTag().equals(new Tag("!reset"))) {
                     return null;
                 }
+                if (node.getTag().equals(new Tag("!override"))) {
+                    // !override replaces the value from previous compose files, so keep the value as if it was untagged
+                    node.setTag(resolveUntaggedTag(node, resolver));
+                }
                 return super.constructObject(node);
             }
         };
-        Yaml yaml = new Yaml(constructor, new Representer(dumperOptions), dumperOptions, options, new Resolver());
+        Yaml yaml = new Yaml(constructor, new Representer(dumperOptions), dumperOptions, options, resolver);
         try (FileInputStream fileInputStream = FileUtils.openInputStream(composeFile)) {
             composeFileContent = yaml.load(fileInputStream);
         } catch (Exception e) {
@@ -76,8 +97,21 @@ protected Object constructObject(Node node) {
         parseAndValidate();
     }
 
+    private static Tag resolveUntaggedTag(Node node, Resolver resolver) {
+        switch (node.getNodeId()) {
+            case mapping:
+                return Tag.MAP;
+            case sequence:
+                return Tag.SEQ;
+            default:
+                ScalarNode scalarNode = (ScalarNode) node;
+                return resolver.resolve(NodeId.scalar, scalarNode.getValue(), scalarNode.isPlain());
+        }
+    }
+
     private void parseAndValidate() {
         final Map<String, ?> servicesMap;
+        final boolean legacyFormat;
         if (composeFileContent.containsKey("version") && "2.0".equals(composeFileContent.get("version"))) {
             log.warn(
                 "Testcontainers may not be able to clean up networks spawned using Docker Compose v2.0 files. " +
@@ -104,20 +138,26 @@ private void parseAndValidate() {
             @SuppressWarnings("unchecked")
             Map<String, ?> temp = (Map<String, ?>) servicesElement;
             servicesMap = temp;
+            legacyFormat = false;
         } else {
+            // Compose file format v1: top-level elements are services
             servicesMap = composeFileContent;
+            legacyFormat = true;
         }
 
         for (Map.Entry<String, ?> entry : servicesMap.entrySet()) {
             String serviceName = entry.getKey();
+            if (legacyFormat && (TOP_LEVEL_ELEMENTS.contains(serviceName) || serviceName.startsWith("x-"))) {
+                continue;
+            }
             Object serviceDefinition = entry.getValue();
             if (!(serviceDefinition instanceof Map)) {
                 log.debug(
                     "Compose file {} has an unknown format: service '{}' is not Map",
                     composeFileName,
                     serviceName
                 );
-                break;
+                continue;
             }
 
             @SuppressWarnings("unchecked")

core/src/test/java/org/testcontainers/containers/DockerComposeFilesTest.java

@@ -29,4 +29,16 @@ void shouldGetDependencyImagesWhenOverriding() {
         assertThat(dockerComposeFiles.getDependencyImages())
             .containsExactlyInAnyOrder("alpine:3.17", "redis:b", "mysql:b", "aservice:latest");
     }
+
+    @Test
+    void shouldGetDependencyImagesWhenOverridingWithOverrideTag() {
+        DockerComposeFiles dockerComposeFiles = new DockerComposeFiles(
+            Lists.newArrayList(
+                new File("src/test/resources/docker-compose-imagename-overriding-tag-a.yml"),
+                new File("src/test/resources/docker-compose-imagename-overriding-tag-b.yml")
+            )
+        );
+        assertThat(dockerComposeFiles.getDependencyImages())
+            .containsExactlyInAnyOrder("postgres:16", "redis:7", "mysql:8.4");
+    }
 }

core/src/test/java/org/testcontainers/containers/ParsedDockerComposeFileValidationTest.java

@@ -150,6 +150,81 @@ void shouldObtainImageFromDockerfileBuildWithContext() {
             ); // redis, mysql from compose file, alpine:3.17 from Dockerfile build
     }
 
+    @Test
+    void shouldObtainImageNamesFromOverrideTag() throws Exception {
+        File file = temporaryFolder.resolve("override-docker-compose.yml").toFile();
+        try (PrintWriter writer = new PrintWriter(file)) {
+            writer.println("services:");
+            writer.println("  db: !override");
+            writer.println("    image: postgres:16");
+            writer.println("  redis:");
+            writer.println("    image: !override redis:7");
+        }
+        ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
+        assertThat(parsedFile.getServiceNameToImageNames())
+            .as("values tagged with !override are kept")
+            .containsOnly(entry("db", Sets.newHashSet("postgres:16")), entry("redis", Sets.newHashSet("redis:7")));
+    }
+
+    @Test
+    void shouldKeepQuotedValuesTaggedWithOverrideAsStrings() throws Exception {
+        File file = temporaryFolder.resolve("override-quoted-docker-compose.yml").toFile();
+        try (PrintWriter writer = new PrintWriter(file)) {
+            writer.println("services:");
+            writer.println("  db:");
+            writer.println("    image: !override \"1.0\"");
+        }
+        ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
+        assertThat(parsedFile.getServiceNameToImageNames())
+            .as("quoted values tagged with !override are not implicitly typed")
+            .containsOnly(entry("db", Sets.newHashSet("1.0")));
+    }
+
+    @Test
+    void shouldIgnoreImageNamesRemovedWithResetTag() throws Exception {
+        File file = temporaryFolder.resolve("reset-docker-compose.yml").toFile();
+        try (PrintWriter writer = new PrintWriter(file)) {
+            writer.println("services:");
+            writer.println("  db:");
+            writer.println("    image: !reset null");
+            writer.println("  redis:");
+            writer.println("    image: redis:7");
+        }
+        ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
+        assertThat(parsedFile.getServiceNameToImageNames())
+            .as("values tagged with !reset are removed")
+            .containsOnly(entry("redis", Sets.newHashSet("redis:7")));
+    }
+
+    @Test
+    void shouldContinueAfterServiceWithUnknownStructure() {
+        ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(
+            ImmutableMap.of(
+                "services",
+                ImmutableMap.of("unknown", "not a map", "redis", ImmutableMap.of("image", "redis:7"))
+            )
+        );
+        assertThat(parsedFile.getServiceNameToImageNames())
+            .as("services after one with an unknown structure are still parsed")
+            .containsOnly(entry("redis", Sets.newHashSet("redis:7")));
+    }
+
+    @Test
+    void shouldIgnoreTopLevelElementsWithoutServicesElement() throws Exception {
+        File file = temporaryFolder.resolve("no-services-docker-compose.yml").toFile();
+        try (PrintWriter writer = new PrintWriter(file)) {
+            writer.println("version: \"3.8\"");
+            writer.println("x-common:");
+            writer.println("  image: busybox:1.36");
+            writer.println("networks:");
+            writer.println("  backend: {}");
+        }
+        ParsedDockerComposeFile parsedFile = new ParsedDockerComposeFile(file);
+        assertThat(parsedFile.getServiceNameToImageNames())
+            .as("extensions and top-level elements are not services")
+            .isEmpty();
+    }
+
     @Test
     void shouldSupportALotOfAliases() throws Exception {
         File file = temporaryFolder.resolve("tmp-docker-compose.yml").toFile();

core/src/test/resources/docker-compose-imagename-overriding-tag-a.yml

@@ -0,0 +1,7 @@
+services:
+    db:
+        image: postgres:15
+    redis:
+        image: redis:6
+    mysql:
+        image: mysql:8.0

core/src/test/resources/docker-compose-imagename-overriding-tag-b.yml

@@ -0,0 +1,7 @@
+services:
+    db: !override
+        image: postgres:16
+    redis:
+        image: !override redis:7
+    mysql:
+        image: mysql:8.4