From cfca405021b07c0130e4fdb5597d1ad0c842bfe0 Mon Sep 17 00:00:00 2001 From: aemous Date: Thu, 20 Aug 2026 16:50:51 -0400 Subject: [PATCH 1/2] Add blackbox tests for s3 cp and s3 mv commands Port functional tests for s3 cp (133 tests) and s3 mv (47 tests) to blackbox tests that run against any CLI binary. Tests cover: uploads, downloads, s3-to-s3 copies, multipart transfers, SSE, checksums, --no-overwrite, --copy-props, --request-payer, --metadata-directive, --case-conflict, --validate-same-s3-paths, streaming, glacier handling, access points, MRAP, and S3 Express. --- tests/blackbox/s3_assertions.py | 94 + tests/blackbox/test_cp_command.py | 5483 +++++++++++++++++++++++++++++ tests/blackbox/test_mv_command.py | 1852 ++++++++++ tests/blackbox/utils.py | 140 + 4 files changed, 7569 insertions(+) create mode 100644 tests/blackbox/test_cp_command.py create mode 100644 tests/blackbox/test_mv_command.py diff --git a/tests/blackbox/s3_assertions.py b/tests/blackbox/s3_assertions.py index c2f937086eab..5fd79a565526 100644 --- a/tests/blackbox/s3_assertions.py +++ b/tests/blackbox/s3_assertions.py @@ -683,3 +683,97 @@ def assert_list_buckets(request, **params): ) _check_params(request, params, _LIST_BUCKETS_PARAMS, "ListBuckets") + +# ListObjectAnnotations: GET /{Bucket}/{Key+}?annotation +_LIST_OBJECT_ANNOTATIONS_PARAMS = { + "VersionId": ("querystring", "versionId"), + "MaxAnnotationResults": ("querystring", "max-annotation-results"), + "AnnotationPrefix": ("querystring", "annotation-prefix"), + "ContinuationToken": ("querystring", "continuation-token"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), +} + + +def assert_list_object_annotations(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a ListObjectAnnotations (GET /{Bucket}/{Key+}?annotation).""" + assert request.method == "GET", ( + f"ListObjectAnnotations: expected GET, got {request.method}" + ) + assert "annotation" in urlparse(request.path).query, ( + f"ListObjectAnnotations: expected ?annotation in {request.path}" + ) + _check_bucket(request, Bucket, addressing_style, "ListObjectAnnotations") + _check_key(request, Key, addressing_style, "ListObjectAnnotations") + _check_params(request, params, _LIST_OBJECT_ANNOTATIONS_PARAMS, "ListObjectAnnotations") + + +# GetObjectAnnotation: GET /{Bucket}/{Key+}?annotation +_GET_OBJECT_ANNOTATION_PARAMS = { + "AnnotationName": ("querystring", "annotationName"), + "VersionId": ("querystring", "versionId"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), + "ChecksumMode": ("header", "x-amz-checksum-mode"), +} + + +def assert_get_object_annotation(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a GetObjectAnnotation (GET /{Bucket}/{Key+}?annotation&annotationName=...).""" + assert request.method == "GET", ( + f"GetObjectAnnotation: expected GET, got {request.method}" + ) + _qs = parse_qs(urlparse(request.path).query) + assert "annotationName" in _qs, ( + f"GetObjectAnnotation: required query param annotationName is missing" + ) + _check_bucket(request, Bucket, addressing_style, "GetObjectAnnotation") + _check_key(request, Key, addressing_style, "GetObjectAnnotation") + _check_params(request, params, _GET_OBJECT_ANNOTATION_PARAMS, "GetObjectAnnotation") + + +# PutObjectAnnotation: PUT /{Bucket}/{Key+}?annotation +_PUT_OBJECT_ANNOTATION_PARAMS = { + "AnnotationName": ("querystring", "annotationName"), + "VersionId": ("querystring", "versionId"), + "ObjectIfMatch": ("header", "x-amz-object-if-match"), + "ChecksumAlgorithm": ("header", "x-amz-sdk-checksum-algorithm"), + "RequestPayer": ("header", "x-amz-request-payer"), + "ExpectedBucketOwner": ("header", "x-amz-expected-bucket-owner"), +} + + +def assert_put_object_annotation(request, Bucket: str, Key: str, addressing_style: str = "virtual", **params): + """Assert request is a PutObjectAnnotation (PUT /{Bucket}/{Key+}?annotation).""" + assert request.method == "PUT", ( + f"PutObjectAnnotation: expected PUT, got {request.method}" + ) + _qs = parse_qs(urlparse(request.path).query) + assert "annotationName" in _qs, ( + f"PutObjectAnnotation: required query param annotationName is missing" + ) + _check_bucket(request, Bucket, addressing_style, "PutObjectAnnotation") + _check_key(request, Key, addressing_style, "PutObjectAnnotation") + _check_params(request, params, _PUT_OBJECT_ANNOTATION_PARAMS, "PutObjectAnnotation") + + +# --------------------------------------------------------------------------- +# Non-S3 service helpers (used by mv --validate-same-s3-paths tests) +# --------------------------------------------------------------------------- + + +def assert_get_access_point(request): + """Assert request is an S3 Control GetAccessPoint (GET /v20180820/accesspoint/{name}).""" + assert request.method == "GET", ( + f"GetAccessPoint: expected GET, got {request.method}" + ) + assert "/v20180820/accesspoint/" in request.path, ( + f"GetAccessPoint: expected /v20180820/accesspoint/ in {request.path}" + ) + + +def assert_get_caller_identity(request): + """Assert request is an STS GetCallerIdentity (POST /).""" + assert request.method == "POST", ( + f"GetCallerIdentity: expected POST, got {request.method}" + ) diff --git a/tests/blackbox/test_cp_command.py b/tests/blackbox/test_cp_command.py new file mode 100644 index 000000000000..3dae7d9262f1 --- /dev/null +++ b/tests/blackbox/test_cp_command.py @@ -0,0 +1,5483 @@ +"""Blackbox tests for `aws s3 cp` command.""" + +from __future__ import annotations + +import os + +import pytest +from localstub.handlers import handle_expect_header +from localstub.server import HTTPResponse + +from tests.blackbox.s3_assertions import ( + assert_abort_multipart_upload, + assert_complete_multipart_upload, + assert_copy_object, + assert_create_multipart_upload, + assert_delete_object, + assert_get_object, + assert_get_object_annotation, + assert_get_object_tagging, + assert_head_object, + assert_list_object_annotations, + assert_list_objects_v2, + assert_put_object, + assert_put_object_annotation, + assert_put_object_tagging, + assert_upload_part_copy, +) +from tests.blackbox.utils import ( + abort_mpu_response, + cli_env, + complete_mpu_response, + copy_object_response, + create_mpu_response, + create_session_response, + empty_list_objects_response, + error_response, + format_requests, + get_object_response, + get_object_tagging_response, + head_object_response, + list_objects_xml, + put_object_response, + put_object_tagging_response, + run_cli, + mock_server, + setup_responses, + upload_part_copy_response, + upload_part_response, + xml_response, +) + + +def relative_path(filename): + """Cross platform relative path of a filename.""" + try: + dirname, basename = os.path.split(filename) + relative_dir = os.path.relpath(dirname) + return os.path.join(relative_dir, basename) + except ValueError: + return os.path.abspath(filename) + + +def list_object_annotations_response(names: list[str]) -> HTTPResponse: + entries = "" + for name in names: + entries += f"{name}" + body = ( + '' + "" + f"{entries}" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def get_object_annotation_response(payload: bytes) -> HTTPResponse: + return HTTPResponse.raw( + payload, + status=200, + headers={"Content-Type": "application/octet-stream"}, + ) + + +def put_object_annotation_response() -> HTTPResponse: + return HTTPResponse.raw(b"", status=200) + + +@pytest.mark.asyncio +class TestCPCommand: + async def test_operations_used_in_upload(self, aws_cli, tmp_path): + """cp local_file s3://bucket/key.txt makes exactly one PUT request.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/key.txt"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object(req, Bucket="bucket", Key="key.txt") + + async def test_key_name_added_when_only_bucket_provided( + self, aws_cli, tmp_path + ): + """cp foo.txt s3://bucket/ should use foo.txt as the key.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "cp", str(src), "s3://bucket/"], cli_env(proxy) + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object(req, Bucket="bucket", Key="foo.txt") + + async def test_trailing_slash_appended(self, aws_cli, tmp_path): + """cp foo.txt s3://bucket (no trailing slash) should still use foo.txt as key.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, ["s3", "cp", str(src), "s3://bucket"], cli_env(proxy) + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object(req, Bucket="bucket", Key="foo.txt") + + async def test_dryrun_upload(self, aws_cli, tmp_path): + """cp --dryrun should not make any requests.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/key.txt", "--dryrun"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 0, format_requests(server) + output = stdout.decode() + assert ( + f"(dryrun) upload: {relative_path(str(src))} to s3://bucket/key.txt" + in output + ) + + async def test_error_on_same_line_as_status(self, aws_cli, tmp_path): + """Upload failure should show error in stderr.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + error_response( + "BucketNotExists", "Bucket does not exist", status=400 + ) + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket-not-exist/key.txt"], + cli_env(proxy), + ) + + assert rc == 1 + err = stderr.decode() + assert ( + f"upload failed: {relative_path(str(src))} to s3://bucket-not-exist/key.txt An error" + in err + ) + + async def test_upload_grants(self, aws_cli, tmp_path): + """cp --grants should send grant headers.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--grants", + "read=id=foo", + "full=id=bar", + "readacl=id=biz", + "writeacl=id=baz", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + GrantRead="id=foo", + GrantFullControl="id=bar", + GrantReadACP="id=biz", + GrantWriteACP="id=baz", + ContentType="text/plain", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_upload_expires(self, aws_cli, tmp_path): + """cp --expires should send expires header.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--expires", + "90", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + Expires="Thu, 01 Jan 1970 00:01:30 GMT", + ) + + async def test_upload_standard_ia(self, aws_cli, tmp_path): + """cp --storage-class STANDARD_IA should send storage class header.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--storage-class", + "STANDARD_IA", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + StorageClass="STANDARD_IA", + ) + + async def test_upload_onezone_ia(self, aws_cli, tmp_path): + """cp --storage-class ONEZONE_IA should send storage class header.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--storage-class", + "ONEZONE_IA", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + StorageClass="ONEZONE_IA", + ) + + async def test_upload_intelligent_tiering(self, aws_cli, tmp_path): + """cp --storage-class INTELLIGENT_TIERING should send storage class header.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--storage-class", + "INTELLIGENT_TIERING", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + StorageClass="INTELLIGENT_TIERING", + ) + + async def test_upload_glacier(self, aws_cli, tmp_path): + """cp --storage-class GLACIER should send storage class header.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--storage-class", + "GLACIER", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + StorageClass="GLACIER", + ) + + async def test_upload_deep_archive(self, aws_cli, tmp_path): + """cp --storage-class DEEP_ARCHIVE should send storage class header.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--storage-class", + "DEEP_ARCHIVE", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + StorageClass="DEEP_ARCHIVE", + ) + + async def test_operations_used_in_download_file(self, aws_cli, tmp_path): + """cp s3://bucket/key.txt localdir uses HeadObject then GetObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/key.txt", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert_get_object( + server.requests[1], Bucket="bucket", Key="key.txt" + ) + + async def test_operations_used_in_recursive_download( + self, aws_cli, tmp_path + ): + """cp s3://bucket/key.txt localdir --recursive with no objects only calls ListObjectsV2.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [empty_list_objects_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key.txt", + str(tmp_path), + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2( + server.requests[0], Bucket="bucket" + ) + + async def test_no_overwrite_flag_when_object_not_exists_on_target( + self, aws_cli, tmp_path + ): + """cp --no-overwrite sends IfNoneMatch=* on PutObject.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, Bucket="bucket", Key="key.txt", IfNoneMatch="*" + ) + + async def test_no_overwrite_flag_when_object_exists_on_target( + self, aws_cli, tmp_path + ): + """cp --no-overwrite with 412 PreconditionFailed succeeds (file skipped).""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ) + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, Bucket="bucket", Key="key.txt", IfNoneMatch="*" + ) + + async def test_no_overwrite_flag_multipart_upload_when_object_not_exists_on_target( + self, aws_cli, tmp_path + ): + """cp --no-overwrite with large file sends IfNoneMatch=* on CompleteMultipartUpload.""" + src = tmp_path / "foo.txt" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("foo"), + upload_part_response("foo-1"), + upload_part_response("foo-2"), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + # CreateMultipartUpload + assert_create_multipart_upload( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ) + # CompleteMultipartUpload — should have if-none-match + assert_complete_multipart_upload( + server.requests[3], + Bucket="bucket", + Key="key.txt", + UploadId="foo", + IfNoneMatch="*", + ) + + async def test_no_overwrite_flag_multipart_upload_when_object_exists_on_target( + self, aws_cli, tmp_path + ): + """cp --no-overwrite large file with 412 on Complete succeeds (file skipped) and aborts.""" + src = tmp_path / "foo.txt" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("foo"), + upload_part_response("foo-1"), + upload_part_response("foo-2"), + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ), + abort_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + # Verify CompleteMultipartUpload had if-none-match + assert_complete_multipart_upload( + server.requests[3], + Bucket="bucket", + Key="key.txt", + UploadId="foo", + IfNoneMatch="*", + ) + # Verify AbortMultipartUpload was called + assert_abort_multipart_upload( + server.requests[4], + Bucket="bucket", + Key="key.txt", + UploadId="foo", + ) + + async def test_no_overwrite_flag_on_copy_when_small_object_does_not_exist_on_target( + self, aws_cli, tmp_path + ): + """cp s3://src s3://dst --no-overwrite sends IfNoneMatch=* on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=5), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket1/key.txt", + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket1", Key="key.txt" + ) + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key.txt", + CopySource="bucket1/key.txt", + IfNoneMatch="*", + ) + + async def test_no_overwrite_flag_on_copy_when_small_object_exists_on_target( + self, aws_cli, tmp_path + ): + """cp s3://src s3://dst --no-overwrite with 412 succeeds (file skipped).""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=5), + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket1/key.txt", + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket1", Key="key.txt" + ) + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key.txt", + CopySource="bucket1/key.txt", + IfNoneMatch="*", + ) + + async def test_dryrun_download(self, aws_cli, tmp_path): + """cp s3://bucket/key.txt local --dryrun only calls HeadObject.""" + target = tmp_path / "file.txt" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [head_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/key.txt", str(target), "--dryrun"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumMode="ENABLED", + ) + output = stdout.decode() + assert ( + f"(dryrun) download: s3://bucket/key.txt to {relative_path(str(target))}" + in output + ) + + async def test_website_redirect_ignore_paramfile(self, aws_cli, tmp_path): + """cp --website-redirect sends the URL as header, not as paramfile.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--website-redirect", + "http://someserver", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + WebsiteRedirectLocation="http://someserver", + ) + + async def test_dryrun_copy(self, aws_cli, tmp_path): + """cp s3://bucket/key.txt s3://bucket/key2.txt --dryrun only calls HeadObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [head_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key.txt", + "s3://bucket/key2.txt", + "--dryrun", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumMode="ENABLED", + ) + output = stdout.decode() + assert ( + "(dryrun) copy: s3://bucket/key.txt to s3://bucket/key2.txt" + in output + ) + + async def test_metadata_copy(self, aws_cli, tmp_path): + """cp s3://src s3://dst --metadata sends metadata on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key.txt", + "s3://bucket/key2.txt", + "--metadata", + "KeyName=Value", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key2.txt", + CopySource="bucket/key.txt", + Metadata={"keyname": "Value"}, + ) + + async def test_metadata_copy_with_put_object(self, aws_cli, tmp_path): + """cp local s3://dst --metadata sends metadata on PutObject.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key2.txt", + "--metadata", + "KeyName=Value", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key2.txt", + Metadata={"keyname": "Value"}, + ) + + async def test_metadata_copy_with_multipart_upload( + self, aws_cli, tmp_path + ): + """cp large_file s3://dst --metadata sends metadata on CreateMultipartUpload.""" + src = tmp_path / "foo.txt" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("foo"), + upload_part_response("foo-1"), + upload_part_response("foo-2"), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key2.txt", + "--metadata", + "KeyName=Value", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) >= 3, format_requests(server) + # CreateMultipartUpload should have the metadata header + assert_create_multipart_upload( + server.requests[0], + Bucket="bucket", + Key="key2.txt", + Metadata={"keyname": "Value"}, + ) + + async def test_metadata_directive_copy(self, aws_cli, tmp_path): + """cp s3://src s3://dst --metadata-directive REPLACE sends directive on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key.txt", + "s3://bucket/key2.txt", + "--metadata-directive", + "REPLACE", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key2.txt", + CopySource="bucket/key.txt", + MetadataDirective="REPLACE", + ) + + async def test_no_metadata_directive_for_non_copy(self, aws_cli, tmp_path): + """cp local s3://dst --metadata-directive REPLACE does not send directive on PutObject.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--metadata-directive", + "REPLACE", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object(req, Bucket="bucket", Key="key.txt") + assert req.headers.get("x-amz-metadata-directive") is None + + async def test_recursive_glacier_download_with_force_glacier( + self, aws_cli, tmp_path + ): + """cp --recursive --force-glacier-transfer downloads GLACIER objects.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "foo/bar.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + "StorageClass": "GLACIER", + "ETag": '"foo-1"', + } + ], + prefix="foo", + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/foo", + str(tmp_path), + "--recursive", + "--force-glacier-transfer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2( + server.requests[0], Bucket="bucket" + ) + assert_get_object( + server.requests[1], Bucket="bucket", Key="foo/bar.txt" + ) + + async def test_recursive_glacier_download_without_force_glacier( + self, aws_cli, tmp_path + ): + """cp --recursive skips GLACIER objects and warns.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "foo/bar.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + "StorageClass": "GLACIER", + "ETag": '"foo-1"', + } + ], + prefix="foo", + ) + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/foo", str(tmp_path), "--recursive"], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert_list_objects_v2( + server.requests[0], Bucket="bucket" + ) + assert "GLACIER" in stderr.decode() + + async def test_warns_on_glacier_incompatible_operation( + self, aws_cli, tmp_path + ): + """cp s3://bucket/key.txt . skips GLACIER objects and warns.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": "100", + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-storage-class": "GLACIER", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/key.txt", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert "GLACIER" in stderr.decode() + + async def test_warns_on_deep_archive_incompatible_operation( + self, aws_cli, tmp_path + ): + """cp s3://bucket/key.txt . skips DEEP_ARCHIVE objects and warns.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": "100", + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-storage-class": "DEEP_ARCHIVE", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/key.txt", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert "GLACIER" in stderr.decode() + + async def test_turn_off_glacier_warnings(self, aws_cli, tmp_path): + """cp --ignore-glacier-warnings skips GLACIER silently with rc=0.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": str(20 * (1024**2)), + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-storage-class": "GLACIER", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key.txt", + str(tmp_path), + "--ignore-glacier-warnings", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert stderr.decode() == "" + + async def test_turn_off_glacier_warnings_for_deep_archive( + self, aws_cli, tmp_path + ): + """cp --ignore-glacier-warnings skips DEEP_ARCHIVE silently with rc=0.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": str(20 * (1024**2)), + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-storage-class": "DEEP_ARCHIVE", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key.txt", + str(tmp_path), + "--ignore-glacier-warnings", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert stderr.decode() == "" + + async def test_cp_with_sse_flag(self, aws_cli, tmp_path): + """cp --sse sends ServerSideEncryption=AES256 header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), "s3://bucket/key.txt", "--sse"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + ContentType="text/plain", + ServerSideEncryption="AES256", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_cp_with_sse_c_flag(self, aws_cli, tmp_path): + """cp --sse-c --sse-c-key sends SSECustomerAlgorithm and SSECustomerKey headers.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--sse-c", + "--sse-c-key", + "foo", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + ContentType="text/plain", + SSECustomerAlgorithm="AES256", + ChecksumAlgorithm="CRC64NVME", + ) + assert ( + req.headers.get("x-amz-server-side-encryption-customer-key") + is not None + ) + + async def test_cp_upload_with_sse_kms_and_key_id(self, aws_cli, tmp_path): + """cp --sse aws:kms --sse-kms-key-id sends KMS headers.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--sse", + "aws:kms", + "--sse-kms-key-id", + "foo", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + ContentType="text/plain", + ServerSideEncryption="aws:kms", + SSEKMSKeyId="foo", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_cp_upload_large_file_with_sse_kms_and_key_id( + self, aws_cli, tmp_path + ): + """cp large file --sse aws:kms --sse-kms-key-id sends KMS on CreateMultipartUpload.""" + src = tmp_path / "foo.txt" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("foo"), + upload_part_response("foo-1"), + upload_part_response("foo-2"), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--sse", + "aws:kms", + "--sse-kms-key-id", + "foo", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + create_req = server.requests[0] + assert_create_multipart_upload( + create_req, + Bucket="bucket", + Key="key.txt", + ContentType="text/plain", + ServerSideEncryption="aws:kms", + SSEKMSKeyId="foo", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_cannot_use_recursive_with_stream(self, aws_cli, tmp_path): + """cp - s3://bucket/key.txt --recursive is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "-", "s3://bucket/key.txt", "--recursive"], + cli_env(proxy), + ) + + assert rc == 252 + assert ( + "Streaming currently is only compatible with non-recursive cp commands" + in stderr.decode() + ) + + async def test_upload_with_checksum_algorithm_crc32( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm CRC32 sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "CRC32", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="CRC32", + ) + + async def test_upload_with_checksum_algorithm_sha256( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm SHA256 sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "SHA256", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="SHA256", + ) + + async def test_upload_with_checksum_algorithm_crc32c( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm CRC32C sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "CRC32C", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="CRC32C", + ) + + async def test_upload_with_checksum_algorithm_crc64nvme( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm CRC64NVME sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "CRC64NVME", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_multipart_upload_with_checksum_algorithm_crc32( + self, aws_cli, tmp_path + ): + """cp large file --checksum-algorithm CRC32 sends algorithm on CreateMultipartUpload.""" + src = tmp_path / "foo.txt" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("foo"), + upload_part_response("foo-1"), + upload_part_response("foo-2"), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key2.txt", + "--checksum-algorithm", + "CRC32", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_create_multipart_upload( + server.requests[0], + Bucket="bucket", + Key="key2.txt", + ChecksumAlgorithm="CRC32", + ) + + async def test_cp_with_sse_c_fileb(self, aws_cli, tmp_path): + """cp --sse-c --sse-c-key fileb:// reads binary key from file.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + key_path = tmp_path / "foo.key" + key_contents = ( + b'K\xc9G\xe1\xf9&\xee\xd1\x03\xf3\xd4\x10\x18o9E\xc2\xaeD' + b'\x89(\x18\xea\xda\xf6\x81\xc3\xd2\x9d\\\xa8\xe6' + ) + key_path.write_bytes(key_contents) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--sse-c", + "--sse-c-key", + f"fileb://{key_path}", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="key.txt", + ContentType="text/plain", + SSECustomerAlgorithm="AES256", + ChecksumAlgorithm="CRC64NVME", + ) + assert ( + req.headers.get("x-amz-server-side-encryption-customer-key") + is not None + ) + + async def test_cp_with_sse_c_copy_source_fileb(self, aws_cli, tmp_path): + """cp s3->s3 --sse-c-copy-source --sse-c-copy-source-key fileb:// sends source SSE-C.""" + key_path = tmp_path / "foo.key" + key_contents = ( + b'K\xc9G\xe1\xf9&\xee\xd1\x03\xf3\xd4\x10\x18o9E\xc2\xaeD' + b'\x89(\x18\xea\xda\xf6\x81\xc3\xd2\x9d\\\xa8\xe6' + ) + key_path.write_bytes(key_contents) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=4), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket-one/key.txt", + "s3://bucket/key.txt", + "--sse-c-copy-source", + "--sse-c-copy-source-key", + f"fileb://{key_path}", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + # HeadObject should have SSE-C source params + assert_head_object( + server.requests[0], + Bucket="bucket-one", + Key="key.txt", + ChecksumMode="ENABLED", + SSECustomerAlgorithm="AES256", + ) + # CopyObject should have copy-source SSE-C params + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key.txt", + CopySource="bucket-one/key.txt", + CopySourceSSECustomerAlgorithm="AES256", + ) + + async def test_s3s3_cp_with_destination_sse_c(self, aws_cli, tmp_path): + """S3->S3 copy with encrypted destination sends SSE-C on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket-one/key.txt", + "s3://bucket/key.txt", + "--sse-c", + "--sse-c-key", + "destination-key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + # HeadObject should NOT have SSE-C params (source is unencrypted) + assert_head_object( + server.requests[0], + Bucket="bucket-one", + Key="key.txt", + ChecksumMode="ENABLED", + ) + assert ( + server.requests[0].headers.get( + "x-amz-server-side-encryption-customer-algorithm" + ) + is None + ) + # CopyObject should have destination SSE-C + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key.txt", + CopySource="bucket-one/key.txt", + SSECustomerAlgorithm="AES256", + AnnotationDirective="EXCLUDE", + ) + assert ( + server.requests[1].headers.get( + "x-amz-server-side-encryption-customer-key" + ) + is not None + ) + + async def test_s3s3_cp_with_different_sse_c_keys(self, aws_cli, tmp_path): + """S3->S3 copy with different SSE-C keys for source and destination.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket-one/key.txt", + "s3://bucket/key.txt", + "--sse-c-copy-source", + "--sse-c-copy-source-key", + "foo", + "--sse-c", + "--sse-c-key", + "bar", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + # HeadObject should have source SSE-C + assert_head_object( + server.requests[0], + Bucket="bucket-one", + Key="key.txt", + ChecksumMode="ENABLED", + SSECustomerAlgorithm="AES256", + ) + assert ( + server.requests[0].headers.get( + "x-amz-server-side-encryption-customer-key" + ) + is not None + ) + # CopyObject should have both source and destination SSE-C + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key.txt", + CopySource="bucket-one/key.txt", + SSECustomerAlgorithm="AES256", + CopySourceSSECustomerAlgorithm="AES256", + ) + + async def test_cp_copy_with_sse_kms_and_key_id(self, aws_cli, tmp_path): + """S3->S3 copy with --sse aws:kms --sse-kms-key-id sends KMS on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=5), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key1.txt", + "s3://bucket/key2.txt", + "--sse", + "aws:kms", + "--sse-kms-key-id", + "foo", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key2.txt", + CopySource="bucket/key1.txt", + ServerSideEncryption="aws:kms", + SSEKMSKeyId="foo", + AnnotationDirective="EXCLUDE", + ) + + async def test_upload_unicode_path(self, aws_cli, tmp_path): + """cp s3://bucket/\u2603 s3://bucket/\u2713 handles unicode paths.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/\u2603", "s3://bucket/\u2713"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + output = stdout.decode() + assert "copy: s3://bucket/\u2603 to s3://bucket/\u2713" in output + assert "Completed 10 Bytes" in output + + async def test_upload_with_checksum_algorithm_sha1( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm SHA1 sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "SHA1", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="SHA1", + ) + + async def test_upload_with_checksum_algorithm_sha512( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm SHA512 sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "SHA512", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="SHA512", + ) + + async def test_upload_with_checksum_algorithm_xxhash3( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm XXHASH3 sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "XXHASH3", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="XXHASH3", + ) + + async def test_upload_with_checksum_algorithm_xxhash64( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm XXHASH64 sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "XXHASH64", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="XXHASH64", + ) + + async def test_upload_with_checksum_algorithm_xxhash128( + self, aws_cli, tmp_path + ): + """cp --checksum-algorithm XXHASH128 sends the algorithm header.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "XXHASH128", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="XXHASH128", + ) + + async def test_copy_with_checksum_algorithm_crc32(self, aws_cli, tmp_path): + """cp s3->s3 --checksum-algorithm CRC32 sends algorithm on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket1/key.txt", + "s3://bucket2/key.txt", + "--checksum-algorithm", + "CRC32", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="bucket2", + Key="key.txt", + CopySource="bucket1/key.txt", + ChecksumAlgorithm="CRC32", + ) + + async def test_download_with_checksum_mode_crc32(self, aws_cli, tmp_path): + """cp s3://bucket/foo local --checksum-mode ENABLED sends ChecksumMode on GetObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response( + b"foo", **{"x-amz-checksum-crc32": "jHNlIQ=="} + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/foo", + str(tmp_path), + "--checksum-mode", + "ENABLED", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + get_req = server.requests[1] + assert_get_object( + get_req, + Bucket="bucket", + Key="foo", + ChecksumMode="ENABLED", + ) + + async def test_download_with_checksum_mode_crc32c(self, aws_cli, tmp_path): + """cp s3://bucket/foo local --checksum-mode ENABLED works with CRC32C objects.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response( + b"foo", **{"x-amz-checksum-crc32c": "z8SuHQ=="} + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/foo", + str(tmp_path), + "--checksum-mode", + "ENABLED", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_get_object( + server.requests[1], + Bucket="bucket", + Key="foo", + ChecksumMode="ENABLED", + ) + + async def test_no_overwrite_flag_on_copy_when_large_object_does_not_exist_on_target( + self, aws_cli, tmp_path + ): + """cp s3->s3 --no-overwrite large object multipart copy sends IfNoneMatch=* on Complete.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("foo"), + upload_part_copy_response(), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket1/key.txt", + "s3://bucket/key1.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 6, format_requests(server) + # CompleteMultipartUpload should have if-none-match + assert_complete_multipart_upload( + server.requests[5], + Bucket="bucket", + Key="key1.txt", + UploadId="foo", + MultipartUpload={ + "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] + }, + IfNoneMatch="*", + ) + + async def test_no_overwrite_flag_on_copy_when_large_object_exists_on_target( + self, aws_cli, tmp_path + ): + """cp s3->s3 --no-overwrite large object with 412 on Complete aborts.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("foo"), + upload_part_copy_response(), + upload_part_copy_response(), + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ), + abort_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket1/key.txt", + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 7, format_requests(server) + # CompleteMultipartUpload should have if-none-match + assert_complete_multipart_upload( + server.requests[5], + Bucket="bucket", + Key="key.txt", + UploadId="foo", + MultipartUpload={ + "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] + }, + IfNoneMatch="*", + ) + # AbortMultipartUpload should have been called + assert_abort_multipart_upload( + server.requests[6], + Bucket="bucket", + Key="key.txt", + UploadId="foo", + ) + + async def test_no_overwrite_flag_on_download_when_single_object_already_exists_at_target( + self, aws_cli, tmp_path + ): + """cp s3://bucket/foo.txt local --no-overwrite skips if file exists locally.""" + target = tmp_path / "foo.txt" + target.write_text("existing content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [head_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/foo.txt", + str(target), + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="foo.txt" + ) + # File should not have been overwritten + assert target.read_text() == "existing content" + + async def test_no_overwrite_flag_on_download_when_single_object_does_not_exist_at_target( + self, aws_cli, tmp_path + ): + """cp s3://bucket/foo.txt local --no-overwrite downloads if file doesn't exist.""" + target = tmp_path / "foo.txt" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/foo.txt", + str(target), + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="foo.txt" + ) + assert_get_object( + server.requests[1], Bucket="bucket", Key="foo.txt" + ) + assert target.read_text() == "foo" + + async def test_warns_on_deep_arhive_incompatible_operation( + self, aws_cli, tmp_path + ): + """cp s3://bucket/key.txt . skips DEEP_ARCHIVE and warns (original typo preserved).""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": "100", + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-storage-class": "DEEP_ARCHIVE", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/key.txt", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert "GLACIER" in stderr.decode() + + async def test_warns_on_glacier_incompatible_operation_for_multipart_file( + self, aws_cli, tmp_path + ): + """cp s3://bucket/key.txt . skips large GLACIER objects and warns.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": str(20 * (1024**2)), + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-storage-class": "GLACIER", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/key.txt", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert "GLACIER" in stderr.decode() + + async def test_warns_on_deep_archive_incompatible_op_for_multipart_file( + self, aws_cli, tmp_path + ): + """cp s3://bucket/key.txt . skips large DEEP_ARCHIVE objects and warns.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": str(20 * (1024**2)), + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-storage-class": "DEEP_ARCHIVE", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/key.txt", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 2 + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert "GLACIER" in stderr.decode() + + async def test_s3s3_cp_with_destination_sse_c_multipart( + self, aws_cli, tmp_path + ): + """S3->S3 multipart copy with encrypted destination sends SSE-C on all parts.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket-one/key.txt", + "s3://bucket/key.txt", + "--sse-c", + "--sse-c-key", + "destination-key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + # HeadObject should NOT have SSE-C (source is unencrypted) + assert_head_object( + server.requests[0], + Bucket="bucket-one", + Key="key.txt", + ChecksumMode="ENABLED", + ) + assert ( + server.requests[0].headers.get( + "x-amz-server-side-encryption-customer-algorithm" + ) + is None + ) + # CreateMultipartUpload should have destination SSE-C + assert_create_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="key.txt", + SSECustomerAlgorithm="AES256", + ) + + async def test_s3s3_cp_with_different_sse_c_keys_multipart( + self, aws_cli, tmp_path + ): + """S3->S3 multipart copy with different SSE-C keys for source and destination.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket-one/key.txt", + "s3://bucket/key.txt", + "--sse-c-copy-source", + "--sse-c-copy-source-key", + "source-key", + "--sse-c", + "--sse-c-key", + "destination-key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + # HeadObject should have source SSE-C + assert_head_object( + server.requests[0], + Bucket="bucket-one", + Key="key.txt", + ChecksumMode="ENABLED", + SSECustomerAlgorithm="AES256", + ) + assert ( + server.requests[0].headers.get( + "x-amz-server-side-encryption-customer-key" + ) + is not None + ) + # CreateMultipartUpload should have destination SSE-C + assert_create_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="key.txt", + SSECustomerAlgorithm="AES256", + ) + + async def test_cp_copy_large_file_with_sse_kms_and_key_id( + self, aws_cli, tmp_path + ): + """S3->S3 multipart copy with --sse aws:kms sends KMS on CreateMultipartUpload.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10 * (1024**2)), + create_mpu_response("foo"), + upload_part_copy_response(), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/key1.txt", + "s3://bucket/key2.txt", + "--copy-props", + "none", + "--sse", + "aws:kms", + "--sse-kms-key-id", + "foo", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + # CreateMultipartUpload should have KMS headers + assert_create_multipart_upload( + server.requests[1], + Bucket="bucket", + Key="key2.txt", + ServerSideEncryption="aws:kms", + SSEKMSKeyId="foo", + ) + + +@pytest.mark.asyncio +class TestStreamingCPCommand: + async def test_streaming_upload(self, aws_cli, tmp_path): + """cp - s3://bucket/streaming.txt uploads stdin content.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "-", "s3://bucket/streaming.txt"], + cli_env(proxy), + stdin=b"foo\n", + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + req = server.requests[0] + assert_put_object( + req, + Bucket="bucket", + Key="streaming.txt", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_streaming_upload_with_expected_size( + self, aws_cli, tmp_path + ): + """cp - s3://bucket/streaming.txt --expected-size 4 uploads stdin.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "-", + "s3://bucket/streaming.txt", + "--expected-size", + "4", + ], + cli_env(proxy), + stdin=b"foo\n", + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="streaming.txt", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_streaming_upload_error(self, aws_cli, tmp_path): + """cp - s3://bucket/streaming.txt with error shows error message.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + error_response( + "NoSuchBucket", + "The specified bucket does not exist", + status=404, + ) + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "-", "s3://bucket/streaming.txt"], + cli_env(proxy), + stdin=b"foo\n", + ) + + assert rc == 1 + err = stderr.decode() + error_message = ( + 'An error occurred (NoSuchBucket) when calling ' + 'the PutObject operation (reached max retries: 0): ' + 'The specified bucket does not exist' + ) + assert error_message in err + + async def test_streaming_download(self, aws_cli, tmp_path): + """cp s3://bucket/streaming.txt - outputs content to stdout.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response( + content_length=4, + **{ + "Accept-Ranges": "bytes", + "Content-Type": "binary/octet-stream", + }, + ), + get_object_response( + b"foo\n", + **{ + "Accept-Ranges": "bytes", + "Content-Type": "binary/octet-stream", + }, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket/streaming.txt", "-"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert stdout == b"foo\n" + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="bucket", + Key="streaming.txt", + ) + assert_get_object( + server.requests[1], + Bucket="bucket", + Key="streaming.txt", + ) + + async def test_no_overwrite_cannot_be_used_with_streaming_download( + self, aws_cli, tmp_path + ): + """cp s3://bucket/streaming.txt - --no-overwrite is rejected.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket/streaming.txt", + "-", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert ( + "--no-overwrite parameter is not supported for streaming downloads" + in stderr.decode() + ) + + +@pytest.mark.asyncio +class TestCpCommandWithRequesterPayer: + async def test_single_upload(self, aws_cli, tmp_path): + """cp --request-payer sends x-amz-request-payer on PutObject.""" + src = tmp_path / "myfile" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://mybucket/mykey", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_multipart_upload(self, aws_cli, tmp_path): + """cp --request-payer large file sends requester-payer on all MPU operations.""" + src = tmp_path / "myfile" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("myid"), + upload_part_response("myetag"), + upload_part_response("myetag"), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(src), + "s3://mybucket/mykey", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_create_multipart_upload( + server.requests[0], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ChecksumAlgorithm="CRC64NVME", + ) + # UploadPart requests should have requester-payer + for req in server.requests[1:3]: + assert req.headers.get("x-amz-request-payer") == "requester" + assert_complete_multipart_upload( + server.requests[3], + Bucket="mybucket", + Key="mykey", + UploadId="myid", + RequestPayer="requester", + ) + + async def test_recursive_upload(self, aws_cli, tmp_path): + """cp --recursive --request-payer sends requester-payer on PutObject.""" + src = tmp_path / "myfile" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(tmp_path), + "s3://mybucket/", + "--request-payer", + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="mybucket", + Key="myfile", + RequestPayer="requester", + ChecksumAlgorithm="CRC64NVME", + ) + + async def test_single_download(self, aws_cli, tmp_path): + """cp s3://bucket/key local --request-payer sends requester-payer on Head+Get.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://mybucket/mykey", + str(tmp_path), + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ChecksumMode="ENABLED", + ) + assert_get_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + + async def test_ranged_download(self, aws_cli, tmp_path): + """cp s3://bucket/key local --request-payer with large file sends requester-payer on ranged Gets.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10 * (1024**2)), + get_object_response(b"a" * 5 * (1024**2)), + get_object_response(b"a" * 5 * (1024**2)), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://mybucket/mykey", + str(tmp_path), + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ChecksumMode="ENABLED", + ) + # GetObject requests should have requester-payer and Range + assert_get_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + assert server.requests[1].headers.get("range") is not None + assert_get_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + assert server.requests[2].headers.get("range") is not None + + async def test_recursive_download(self, aws_cli, tmp_path): + """cp s3://bucket/ local --recursive --request-payer sends requester-payer on List+Get.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "mykey", + "Size": 4, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://mybucket/", + str(tmp_path), + "--request-payer", + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2( + server.requests[0], + Bucket="mybucket", + RequestPayer="requester", + ) + assert_get_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + + async def test_single_copy(self, aws_cli, tmp_path): + """cp s3://src s3://dst --request-payer sends requester-payer on Head+Copy.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://sourcebucket/sourcekey", + "s3://mybucket/mykey", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="sourcebucket", + Key="sourcekey", + RequestPayer="requester", + ChecksumMode="ENABLED", + ) + assert_copy_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + CopySource="sourcebucket/sourcekey", + RequestPayer="requester", + ) + + async def test_multipart_copy(self, aws_cli, tmp_path): + """cp s3://src s3://dst --request-payer large file sends requester-payer on all MPU ops.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10 * (1024**2)), + create_mpu_response("id"), + upload_part_copy_response(), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://sourcebucket/sourcekey", + "s3://mybucket/mykey", + "--copy-props", + "none", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="sourcebucket", + Key="sourcekey", + RequestPayer="requester", + ChecksumMode="ENABLED", + ) + assert_create_multipart_upload( + server.requests[1], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + # UploadPartCopy requests should have requester-payer + for req in server.requests[2:4]: + assert req.headers.get("x-amz-request-payer") == "requester" + assert_complete_multipart_upload( + server.requests[4], + Bucket="mybucket", + Key="mykey", + UploadId="id", + MultipartUpload={ + "Parts": [{"PartNumber": "1"}, {"PartNumber": "2"}] + }, + RequestPayer="requester", + ) + + async def test_recursive_copy(self, aws_cli, tmp_path): + """cp s3://src/ s3://dst/ --recursive --request-payer sends requester-payer on List+Copy.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "mykey", + "Size": 4, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://sourcebucket/", + "s3://mybucket/", + "--request-payer", + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2( + server.requests[0], + Bucket="sourcebucket", + RequestPayer="requester", + ) + assert_copy_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + CopySource="sourcebucket/mykey", + RequestPayer="requester", + ) + + async def test_mp_copy_object(self, aws_cli, tmp_path): + """cp s3->s3 --request-payer multipart copy sends requester-payer on GetObjectTagging.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://sourcebucket/mykey", + "s3://mybucket/mykey", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_get_object_tagging( + server.requests[1], + Bucket="sourcebucket", + Key="mykey", + RequestPayer="requester", + ) + assert_create_multipart_upload( + server.requests[2], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + + async def test_mp_copy_object_with_tags_exceed_2k(self, aws_cli, tmp_path): + """cp s3->s3 --request-payer multipart copy with large tags sends PutObjectTagging.""" + large_value = "value" * (2 * 1024) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response({"tag-key": large_value}), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + put_object_tagging_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://sourcebucket/mykey", + "s3://mybucket/mykey", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 6, format_requests(server) + assert_get_object_tagging( + server.requests[1], + Bucket="sourcebucket", + Key="mykey", + RequestPayer="requester", + ) + assert_create_multipart_upload( + server.requests[2], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + assert_put_object_tagging( + server.requests[5], + Bucket="mybucket", + Key="mykey", + Tagging={"TagSet": [{"Key": "tag-key", "Value": large_value}]}, + RequestPayer="requester", + ) + + +@pytest.mark.asyncio +class TestAccesspointCPCommand: + """Tests for cp with S3 Access Point ARNs.""" + + ARN = "arn:aws:s3:us-west-2:123456789012:accesspoint/endpoint" + HOST = "endpoint-123456789012.s3-accesspoint.us-west-2.amazonaws.com" + + async def test_upload(self, aws_cli, tmp_path): + """cp local s3:///key uploads to accesspoint.""" + src = tmp_path / "myfile" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), f"s3://{self.ARN}/mykey"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], Bucket=self.ARN, Key="mykey" + ) + + async def test_recursive_upload(self, aws_cli, tmp_path): + """cp local s3:/// --recursive uploads to accesspoint.""" + (tmp_path / "myfile").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + str(tmp_path), + f"s3://{self.ARN}/", + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], Bucket=self.ARN, Key="myfile" + ) + + async def test_download(self, aws_cli, tmp_path): + """cp s3:///key local downloads from accesspoint.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", f"s3://{self.ARN}/mykey", str(tmp_path)], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], + Bucket=self.ARN, + Key="mykey", + ChecksumMode="ENABLED", + ) + assert_get_object( + server.requests[1], Bucket=self.ARN, Key="mykey" + ) + + async def test_recursive_download(self, aws_cli, tmp_path): + """cp s3:// local --recursive downloads from accesspoint.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "mykey", + "Size": 4, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", f"s3://{self.ARN}", str(tmp_path), "--recursive"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2( + server.requests[0], + Bucket=self.ARN, + ) + assert_get_object( + server.requests[1], Bucket=self.ARN, Key="mykey" + ) + + async def test_copy(self, aws_cli, tmp_path): + """cp s3:///key s3:///key copies between accesspoints.""" + dest_arn = self.ARN + "-dest" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + f"s3://{self.ARN}/mykey", + f"s3://{dest_arn}/mykey", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], + Bucket=self.ARN, + Key="mykey", + ChecksumMode="ENABLED", + ) + assert_copy_object( + server.requests[1], Bucket=dest_arn, Key="mykey" + ) + + async def test_recursive_copy(self, aws_cli, tmp_path): + """cp s3:// s3:// --recursive copies between accesspoints.""" + dest_arn = self.ARN + "-dest" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "mykey", + "Size": 4, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + f"s3://{self.ARN}", + f"s3://{dest_arn}", + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_list_objects_v2( + server.requests[0], + Bucket=self.ARN, + ) + assert_copy_object( + server.requests[1], Bucket=dest_arn, Key="mykey" + ) + + async def test_accepts_mrap_arns(self, aws_cli, tmp_path): + """cp to MRAP ARN (colon separator) works.""" + mrap_arn = "arn:aws:s3::123456789012:accesspoint:mfzwi23gnjvgw.mrap" + src = tmp_path / "myfile" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), f"s3://{mrap_arn}/mykey"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], Bucket=mrap_arn, Key="mykey" + ) + + async def test_accepts_mrap_arns_with_slash(self, aws_cli, tmp_path): + """cp to MRAP ARN (slash separator) works.""" + mrap_arn = "arn:aws:s3::123456789012:accesspoint/mfzwi23gnjvgw.mrap" + src = tmp_path / "myfile" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", str(src), f"s3://{mrap_arn}/mykey"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], Bucket=mrap_arn, Key="mykey" + ) + + +@pytest.mark.asyncio +class TestCopyPropsNoneCpCommand: + """Tests for cp --copy-props none (S3->S3 copy).""" + + async def test_copy_object(self, aws_cli, tmp_path): + """--copy-props none sends MetadataDirective=REPLACE and TaggingDirective=REPLACE.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "none", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + MetadataDirective="REPLACE", + TaggingDirective="REPLACE", + ) + + async def test_mp_copy_object(self, aws_cli, tmp_path): + """--copy-props none multipart copy has no extra params on CreateMPU.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "none", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_create_multipart_upload( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + ) + # Should NOT have metadata/tagging headers on CreateMPU + req = server.requests[1] + assert req.headers.get("x-amz-tagging") is None + assert req.headers.get("cache-control") is None + + async def test_metadata_directive_disables_copy_props( + self, aws_cli, tmp_path + ): + """--copy-props none --metadata-directive COPY bypasses copy-props entirely.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "none", + "--metadata-directive", + "COPY", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + MetadataDirective="COPY", + ) + # No tagging or annotation directives when --metadata-directive is used + req = server.requests[1] + assert req.headers.get("x-amz-tagging-directive") is None + assert req.headers.get("x-amz-object-annotation-directive") is None + + +@pytest.mark.asyncio +class TestCopyPropsMetadataDirectiveCpCommand: + """Tests for cp --copy-props metadata-directive (S3->S3 copy).""" + + ALL_METADATA_HEADERS = { + "cache-control": "cache-control", + "content-disposition": "content-disposition", + "content-encoding": "content-encoding", + "content-language": "content-language", + "content-type": "content-type", + "expires": "Tue, 07 Jan 2020 20:40:03 GMT", + "x-amz-meta-key": "value", + } + + def _head_with_metadata(self, content_length: int = 100) -> HTTPResponse: + return head_object_response( + content_length=content_length, + **self.ALL_METADATA_HEADERS, + ) + + async def test_copy_object(self, aws_cli, tmp_path): + """--copy-props metadata-directive sends TaggingDirective=REPLACE on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "metadata-directive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + TaggingDirective="REPLACE", + ) + + async def test_copy_object_overrides_with_cmdline_props( + self, aws_cli, tmp_path + ): + """--copy-props metadata-directive with --content-type override.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + self._head_with_metadata(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "metadata-directive", + "--content-type", + "content-type-from-cmdline", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + CacheControl="cache-control", + ContentDisposition="content-disposition", + ContentEncoding="content-encoding", + ContentLanguage="content-language", + ContentType="content-type-from-cmdline", + Expires="Tue, 07 Jan 2020 20:40:03 GMT", + Metadata={"key": "value"}, + MetadataDirective="REPLACE", + TaggingDirective="REPLACE", + ) + + async def test_recursive_copy_object(self, aws_cli, tmp_path): + """--copy-props metadata-directive --recursive sends TaggingDirective=REPLACE.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "metadata-directive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="source-key", + CopySource="source-bucket/source-key", + TaggingDirective="REPLACE", + ) + + async def test_recursive_copy_object_overrides_with_cmdline_props( + self, aws_cli, tmp_path + ): + """--copy-props metadata-directive --recursive with --metadata override.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "metadata-directive", + "--metadata", + "key=val-from-cmdline", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_copy_object( + server.requests[2], + Bucket="target-bucket", + Key="source-key", + CopySource="source-bucket/source-key", + MetadataDirective="REPLACE", + TaggingDirective="REPLACE", + Metadata={"key": "val-from-cmdline"}, + ) + + async def test_recursive_copy_maps_additional_head_object_headers( + self, aws_cli, tmp_path + ): + """--copy-props metadata-directive --recursive --request-payer sends requester-payer on HeadObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "metadata-directive", + "--metadata", + "key=val-from-cmdline", + "--request-payer", + "requester", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3 + assert_head_object( + server.requests[1], + Bucket="source-bucket", + Key="source-key", + RequestPayer="requester", + ) + + async def test_mp_copy_object(self, aws_cli, tmp_path): + """--copy-props metadata-directive multipart copy sends metadata on CreateMPU.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + self._head_with_metadata(content_length=8 * (1024**2)), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "metadata-directive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_create_multipart_upload( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CacheControl="cache-control", + ContentDisposition="content-disposition", + ContentEncoding="content-encoding", + ContentLanguage="content-language", + ContentType="content-type", + Expires="Tue, 07 Jan 2020 20:40:03 GMT", + Metadata={"key": "value"}, + ) + + async def test_mp_copy_object_with_prop_overrides(self, aws_cli, tmp_path): + """--copy-props metadata-directive multipart copy with --content-type override.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + self._head_with_metadata(content_length=8 * (1024**2)), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "metadata-directive", + "--content-type", + "content-type-from-cmdline", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_create_multipart_upload( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CacheControl="cache-control", + ContentDisposition="content-disposition", + ContentEncoding="content-encoding", + ContentLanguage="content-language", + ContentType="content-type-from-cmdline", + Expires="Tue, 07 Jan 2020 20:40:03 GMT", + Metadata={"key": "value"}, + ) + + async def test_recursive_mp_copy(self, aws_cli, tmp_path): + """--copy-props metadata-directive --recursive multipart copy sends metadata on CreateMPU.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 8 * (1024**2), + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + self._head_with_metadata(content_length=8 * (1024**2)), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "metadata-directive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_create_multipart_upload( + server.requests[2], + Bucket="target-bucket", + Key="source-key", + ) + + async def test_recursive_mp_copy_object_with_prop_overrides( + self, aws_cli, tmp_path + ): + """--copy-props metadata-directive --recursive multipart with --content-type override.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 8 * (1024**2), + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + self._head_with_metadata(content_length=8 * (1024**2)), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "metadata-directive", + "--content-type", + "content-type-from-cmdline", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_create_multipart_upload( + server.requests[2], + Bucket="target-bucket", + Key="source-key", + ContentType="content-type-from-cmdline", + ) + + async def test_recursive_mp_copy_maps_additional_head_object_headers( + self, aws_cli, tmp_path + ): + """--copy-props metadata-directive --recursive multipart --request-payer on HeadObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 8 * (1024**2), + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + head_object_response(content_length=8 * (1024**2)), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "metadata-directive", + "--request-payer", + "requester", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_head_object( + server.requests[1], + Bucket="source-bucket", + Key="source-key", + RequestPayer="requester", + ) + + async def test_fails_when_head_object_fails(self, aws_cli, tmp_path): + """--copy-props metadata-directive --recursive fails when HeadObject returns 404.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 8 * (1024**2), + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + error_response( + "NoSuchKey", + "The specified key does not exist.", + status=404, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "metadata-directive", + ], + cli_env(proxy), + ) + + assert rc == 1 + assert "404" in stderr.decode() or "NoSuchKey" in stderr.decode() + + async def test_metadata_directive_disables_copy_props( + self, aws_cli, tmp_path + ): + """--copy-props metadata-directive --metadata-directive REPLACE bypasses copy-props.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "metadata-directive", + "--metadata-directive", + "REPLACE", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + MetadataDirective="REPLACE", + ) + # No tagging or annotation directives when --metadata-directive is used + req = server.requests[1] + assert req.headers.get("x-amz-tagging-directive") is None + assert req.headers.get("x-amz-object-annotation-directive") is None + + +@pytest.mark.asyncio +class TestCopyPropsDefaultCpCommand: + """Tests for cp --copy-props default (S3->S3 copy).""" + + ALL_METADATA_HEADERS = { + "cache-control": "cache-control", + "content-disposition": "content-disposition", + "content-encoding": "content-encoding", + "content-language": "content-language", + "content-type": "content-type", + "expires": "Tue, 07 Jan 2020 20:40:03 GMT", + "x-amz-meta-key": "value", + } + URLENCODED_TAGS = "tag-key=tag-value&tag-key2=tag-value2" + TAGS = {"tag-key": "tag-value", "tag-key2": "tag-value2"} + TAGS_OVER_2K = {"tag-key": "value" * (2 * 1024)} + + def _head_with_metadata(self, content_length: int = 100) -> HTTPResponse: + return head_object_response( + content_length=content_length, + **self.ALL_METADATA_HEADERS, + ) + + async def test_copy_object(self, aws_cli, tmp_path): + """--copy-props default single copy has no extra directives.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + ) + + async def test_is_default_value(self, aws_cli, tmp_path): + """No --copy-props flag behaves same as --copy-props default.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + ) + + async def test_copy_object_with_prop_overrides(self, aws_cli, tmp_path): + """--copy-props default with --content-language override sends REPLACE + all metadata.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + self._head_with_metadata(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + "--content-language", + "content-lang-from-cmdline", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + CacheControl="cache-control", + ContentDisposition="content-disposition", + ContentEncoding="content-encoding", + ContentLanguage="content-lang-from-cmdline", + ContentType="content-type", + Expires="Tue, 07 Jan 2020 20:40:03 GMT", + Metadata={"key": "value"}, + MetadataDirective="REPLACE", + ) + + async def test_recursive_copy_object(self, aws_cli, tmp_path): + """--copy-props default --recursive single copy has no extra directives.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="source-key", + CopySource="source-bucket/source-key", + ) + + async def test_recursive_copy_object_with_prop_overrides( + self, aws_cli, tmp_path + ): + """--copy-props default --recursive with --content-language override.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "source-key", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + self._head_with_metadata(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/", + "s3://target-bucket/", + "--recursive", + "--copy-props", + "default", + "--content-language", + "content-lang-from-cmdline", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_copy_object( + server.requests[2], + Bucket="target-bucket", + Key="source-key", + CopySource="source-bucket/source-key", + MetadataDirective="REPLACE", + ContentLanguage="content-lang-from-cmdline", + ) + + async def test_mp_copy_object(self, aws_cli, tmp_path): + """--copy-props default multipart copy sends metadata + tagging on CreateMPU.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + self._head_with_metadata(content_length=8 * (1024**2)), + get_object_tagging_response(self.TAGS), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_create_multipart_upload( + server.requests[2], + Bucket="target-bucket", + Key="target-key", + CacheControl="cache-control", + ContentDisposition="content-disposition", + ContentEncoding="content-encoding", + ContentLanguage="content-language", + ContentType="content-type", + Expires="Tue, 07 Jan 2020 20:40:03 GMT", + Metadata={"key": "value"}, + Tagging="tag-key=tag-value&tag-key2=tag-value2", + ) + + async def test_mp_copy_object_with_prop_overrides(self, aws_cli, tmp_path): + """--copy-props default multipart copy with --cache-control override.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + self._head_with_metadata(content_length=8 * (1024**2)), + get_object_tagging_response(self.TAGS), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + "--cache-control", + "cache-control-from-cmdline", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_create_multipart_upload( + server.requests[2], + Bucket="target-bucket", + Key="target-key", + CacheControl="cache-control-from-cmdline", + ContentDisposition="content-disposition", + ContentEncoding="content-encoding", + ContentLanguage="content-language", + ContentType="content-type", + Expires="Tue, 07 Jan 2020 20:40:03 GMT", + Metadata={"key": "value"}, + Tagging="tag-key=tag-value&tag-key2=tag-value2", + ) + + async def test_mp_copy_object_no_tags(self, aws_cli, tmp_path): + """--copy-props default multipart copy with no tags omits x-amz-tagging.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_create_multipart_upload( + server.requests[2], + Bucket="target-bucket", + Key="target-key", + ) + assert server.requests[2].headers.get("x-amz-tagging") is None + + async def test_mp_copy_object_tags_exceed_2k(self, aws_cli, tmp_path): + """--copy-props default multipart copy with >2k tags uses PutObjectTagging after copy.""" + large_value = "value" * (2 * 1024) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(self.TAGS_OVER_2K), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + put_object_tagging_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 6, format_requests(server) + assert_create_multipart_upload( + server.requests[2], + Bucket="target-bucket", + Key="target-key", + ) + assert server.requests[2].headers.get("x-amz-tagging") is None + assert_put_object_tagging( + server.requests[5], + Bucket="target-bucket", + Key="target-key", + Tagging={"TagSet": [{"Key": "tag-key", "Value": large_value}]}, + ) + + async def test_fails_when_get_tagging_object_fails( + self, aws_cli, tmp_path + ): + """--copy-props default fails when GetObjectTagging returns 403.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + error_response( + "AccessDenied", "Access Denied", status=403 + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 1 + assert "AccessDenied" in stderr.decode() + + async def test_fails_and_cleans_up_when_put_tagging_object_fails( + self, aws_cli, tmp_path + ): + """--copy-props default cleans up (DeleteObject) when PutObjectTagging fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(self.TAGS_OVER_2K), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + error_response( + "AccessDenied", "Access Denied", status=403 + ), + HTTPResponse.raw(b"", status=204), # DeleteObject + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 1 + assert "AccessDenied" in stderr.decode() + assert_delete_object( + server.requests[6], + Bucket="target-bucket", + Key="target-key", + ) + + async def test_clean_up_uses_requester_payer(self, aws_cli, tmp_path): + """--copy-props default cleanup DeleteObject uses --request-payer.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(self.TAGS_OVER_2K), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + error_response( + "AccessDenied", "Access Denied", status=403 + ), + HTTPResponse.raw(b"", status=204), # DeleteObject + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + "--request-payer", + "requester", + ], + cli_env(proxy), + ) + + assert rc == 1 + assert "AccessDenied" in stderr.decode() + assert_delete_object( + server.requests[6], + Bucket="target-bucket", + Key="target-key", + RequestPayer="requester", + ) + + async def test_metadata_directive_disables_copy_props( + self, aws_cli, tmp_path + ): + """--copy-props default --metadata-directive REPLACE bypasses copy-props.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "default", + "--metadata-directive", + "REPLACE", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + MetadataDirective="REPLACE", + ) + req = server.requests[1] + assert req.headers.get("x-amz-tagging-directive") is None + assert req.headers.get("x-amz-object-annotation-directive") is None + + +@pytest.mark.asyncio +class TestCopyPropsAllCpCommand: + """Tests for cp --copy-props all (S3->S3 copy with annotations).""" + + ANNOTATION_PAYLOAD = b"annotation-payload" + + def _complete_mpu_response_with_version(self) -> HTTPResponse: + body = ( + '' + '' + "http://bucket.s3.amazonaws.com/key" + "target-buckettarget-key" + '"dest-etag"' + "" + ) + return HTTPResponse.raw( + body.encode(), + status=200, + headers={ + "Content-Type": "application/xml", + "x-amz-version-id": "dest-version-id", + }, + ) + + async def test_copy_object_excludes_nothing_for_single_part( + self, aws_cli, tmp_path + ): + """--copy-props all single-part copy does NOT set AnnotationDirective=EXCLUDE.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "all", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_copy_object( + server.requests[1], + Bucket="target-bucket", + Key="target-key", + CopySource="source-bucket/source-key", + ) + # Should NOT have AnnotationDirective (relies on server default COPY) + assert ( + server.requests[1].headers.get("x-amz-object-annotation-directive") + is None + ) + + async def test_mp_copy_object_copies_annotations(self, aws_cli, tmp_path): + """--copy-props all multipart copy reads and writes annotations.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response({"tag-key": "tag-value"}), + list_object_annotations_response(["ann1", "ann2"]), + get_object_annotation_response(self.ANNOTATION_PAYLOAD), + get_object_annotation_response(self.ANNOTATION_PAYLOAD), + create_mpu_response("upload_id"), + upload_part_copy_response(), + self._complete_mpu_response_with_version(), + put_object_annotation_response(), + put_object_annotation_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "all", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 10, format_requests(server) + assert_list_object_annotations( + server.requests[2], + Bucket="source-bucket", + Key="source-key", + ) + assert_get_object_annotation( + server.requests[3], + Bucket="source-bucket", + Key="source-key", + AnnotationName="ann1", + ) + assert_get_object_annotation( + server.requests[4], + Bucket="source-bucket", + Key="source-key", + AnnotationName="ann2", + ) + assert_put_object_annotation( + server.requests[8], + Bucket="target-bucket", + Key="target-key", + AnnotationName="ann1", + ObjectIfMatch='"dest-etag"', + VersionId="dest-version-id", + ) + assert_put_object_annotation( + server.requests[9], + Bucket="target-bucket", + Key="target-key", + AnnotationName="ann2", + ObjectIfMatch='"dest-etag"', + VersionId="dest-version-id", + ) + + async def test_mp_copy_object_no_annotations(self, aws_cli, tmp_path): + """--copy-props all multipart copy with no annotations skips Get/PutObjectAnnotation.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(), + list_object_annotations_response([]), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "all", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 6, format_requests(server) + assert_list_object_annotations( + server.requests[2], + Bucket="source-bucket", + Key="source-key", + ) + # No GetObjectAnnotation or PutObjectAnnotation requests + for r in server.requests[3:]: + assert "annotationName" not in r.path + + async def test_mp_copy_object_partial_annotation_failure( + self, aws_cli, tmp_path + ): + """--copy-props all fails if PutObjectAnnotation fails, does NOT delete dest object.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response(), + list_object_annotations_response(["ann1", "ann2"]), + get_object_annotation_response(self.ANNOTATION_PAYLOAD), + get_object_annotation_response(self.ANNOTATION_PAYLOAD), + create_mpu_response("upload_id"), + upload_part_copy_response(), + self._complete_mpu_response_with_version(), + put_object_annotation_response(), # ann1 succeeds + error_response( + "AccessDenied", "Access Denied", status=403 + ), # ann2 fails + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "all", + ], + cli_env(proxy), + ) + + assert rc == 1 + err = stderr.decode() + assert "ann1" in err or "ann2" in err + # No DeleteObject — partial annotation failure does not clean up + for r in server.requests: + if r.method == "DELETE": + assert ( + "annotation" in r.path or "uploadId" in r.path + ), f"Unexpected DELETE: {r.path}" + + async def test_mp_copy_object_copies_annotations_with_source_version_id( + self, aws_cli, tmp_path + ): + """--copy-props all multipart copy passes source versionId to annotation reads.""" + source_version_id = "src-version-id" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + HTTPResponse.raw( + b"", + status=200, + headers={ + "Content-Length": str(8 * (1024**2)), + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + "x-amz-version-id": source_version_id, + }, + ), + get_object_tagging_response({"tag-key": "tag-value"}), + list_object_annotations_response(["ann1"]), + get_object_annotation_response(self.ANNOTATION_PAYLOAD), + create_mpu_response("upload_id"), + upload_part_copy_response(), + self._complete_mpu_response_with_version(), + put_object_annotation_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://source-bucket/source-key", + "s3://target-bucket/target-key", + "--copy-props", + "all", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 8, format_requests(server) + assert_list_object_annotations( + server.requests[2], + Bucket="source-bucket", + Key="source-key", + VersionId=source_version_id, + ) + assert_get_object_annotation( + server.requests[3], + Bucket="source-bucket", + Key="source-key", + AnnotationName="ann1", + VersionId=source_version_id, + ) + assert_put_object_annotation( + server.requests[7], + Bucket="target-bucket", + Key="target-key", + AnnotationName="ann1", + ObjectIfMatch='"dest-etag"', + VersionId="dest-version-id", + ) + + +@pytest.mark.asyncio +class TestCpSourceRegion: + """Tests for cp --source-region routing requests to correct endpoints.""" + + SOURCE_BUCKET = "sourcebucket" + SOURCE_REGION = "af-south-1" + TARGET_BUCKET = "bucket" + TARGET_REGION = "us-east-1" + SOURCE_HOST = f"{SOURCE_BUCKET}.s3.{SOURCE_REGION}.amazonaws.com" + TARGET_HOST = f"{TARGET_BUCKET}.s3.{TARGET_REGION}.amazonaws.com" + + async def test_respects_source_region_for_single_copy( + self, aws_cli, tmp_path + ): + """--source-region routes HeadObject to source region, CopyObject to target.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + f"s3://{self.SOURCE_BUCKET}/key", + f"s3://{self.TARGET_BUCKET}/", + "--source-region", + self.SOURCE_REGION, + "--region", + self.TARGET_REGION, + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], + Bucket=self.SOURCE_BUCKET, + Key="key", + ) + assert server.requests[0].headers.get("host") == self.SOURCE_HOST + assert_copy_object( + server.requests[1], + Bucket=self.TARGET_BUCKET, + Key="key", + ) + assert server.requests[1].headers.get("host") == self.TARGET_HOST + + async def test_respects_source_region_for_recursive_copy( + self, aws_cli, tmp_path + ): + """--source-region --recursive routes List to source, Copy to target.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "key", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + copy_object_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + f"s3://{self.SOURCE_BUCKET}/", + f"s3://{self.TARGET_BUCKET}/", + "--source-region", + self.SOURCE_REGION, + "--region", + self.TARGET_REGION, + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert server.requests[0].headers.get("host") == self.SOURCE_HOST + assert server.requests[1].headers.get("host") == self.TARGET_HOST + + async def test_respects_source_region_for_copying_mp_object_tags( + self, aws_cli, tmp_path + ): + """--source-region multipart copy routes tagging ops to correct regions.""" + large_tag_value = "value" * (2 * 1024) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=8 * (1024**2)), + get_object_tagging_response({"tag": large_tag_value}), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + put_object_tagging_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + f"s3://{self.SOURCE_BUCKET}/key", + f"s3://{self.TARGET_BUCKET}/", + "--source-region", + self.SOURCE_REGION, + "--region", + self.TARGET_REGION, + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 6, format_requests(server) + # HeadObject + GetObjectTagging → source region + assert server.requests[0].headers.get("host") == self.SOURCE_HOST + assert server.requests[1].headers.get("host") == self.SOURCE_HOST + # CreateMPU + UploadPartCopy + CompleteMPU + PutObjectTagging → target region + assert server.requests[2].headers.get("host") == self.TARGET_HOST + assert server.requests[3].headers.get("host") == self.TARGET_HOST + assert server.requests[4].headers.get("host") == self.TARGET_HOST + assert server.requests[5].headers.get("host") == self.TARGET_HOST + + async def test_respects_source_region_for_recursive_mp_copy( + self, aws_cli, tmp_path + ): + """--source-region --recursive multipart copy routes to correct regions.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "key", + "Size": 10 * (1024**2), + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + head_object_response(content_length=10 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("upload_id"), + upload_part_copy_response(), + upload_part_copy_response(), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + f"s3://{self.SOURCE_BUCKET}/", + f"s3://{self.TARGET_BUCKET}/", + "--source-region", + self.SOURCE_REGION, + "--region", + self.TARGET_REGION, + "--recursive", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 7, format_requests(server) + # Source region: ListObjectsV2 + assert_list_objects_v2( + server.requests[0], Bucket=self.SOURCE_BUCKET + ) + assert server.requests[0].headers.get("host") == self.SOURCE_HOST + # Source region: HeadObject (for metadata + size/etag) + assert_head_object( + server.requests[1], + Bucket=self.SOURCE_BUCKET, + Key="key", + ) + assert server.requests[1].headers.get("host") == self.SOURCE_HOST + # Source region: GetObjectTagging + assert_get_object_tagging( + server.requests[2], + Bucket=self.SOURCE_BUCKET, + Key="key", + ) + assert server.requests[2].headers.get("host") == self.SOURCE_HOST + # Target region: CreateMultipartUpload + assert_create_multipart_upload( + server.requests[3], + Bucket=self.TARGET_BUCKET, + Key="key", + ) + assert server.requests[3].headers.get("host") == self.TARGET_HOST + # Target region: UploadPartCopy x2 + assert_upload_part_copy( + server.requests[4], + Bucket=self.TARGET_BUCKET, + Key="key", + UploadId="upload_id", + ) + assert server.requests[4].headers.get("host") == self.TARGET_HOST + assert_upload_part_copy( + server.requests[5], + Bucket=self.TARGET_BUCKET, + Key="key", + UploadId="upload_id", + ) + assert server.requests[5].headers.get("host") == self.TARGET_HOST + # Target region: CompleteMultipartUpload + assert_complete_multipart_upload( + server.requests[6], + Bucket=self.TARGET_BUCKET, + Key="key", + UploadId="upload_id", + ) + assert server.requests[6].headers.get("host") == self.TARGET_HOST + + +@pytest.mark.asyncio +class TestCpRecursiveCaseConflict: + """Tests for cp --recursive case conflict handling.""" + + async def test_ignore_by_default(self, aws_cli, tmp_path): + """cp --recursive downloads case-conflicting keys without error by default.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + } + ], + ) + ), + head_object_response(), + get_object_response(b"foo"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "cp", "s3://bucket", str(tmp_path), "--recursive"], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + # No warnings in stderr + assert not stderr.decode().strip() + + +@pytest.mark.asyncio +class TestS3ExpressCpRecursive: + """Tests for cp --recursive with S3 Express directory buckets and --case-conflict.""" + + async def test_s3_express_error_raises_exception(self, aws_cli, tmp_path): + """--case-conflict error is not valid for S3 Express directory buckets.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket--usw2-az1--x-s3", + str(tmp_path), + "--recursive", + "--case-conflict", + "error", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert "`error` is not a valid value" in stderr.decode() + + async def test_s3_express_skip_raises_exception(self, aws_cli, tmp_path): + """--case-conflict skip is not valid for S3 Express directory buckets.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket--usw2-az1--x-s3", + str(tmp_path), + "--recursive", + "--case-conflict", + "skip", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert "`skip` is not a valid value" in stderr.decode() + + async def test_s3_express_warn_emits_warning(self, aws_cli, tmp_path): + """--case-conflict warn on S3 Express emits warning for case conflicts.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + # CreateSession for the source/list client + create_session_response(), + xml_response( + list_objects_xml( + contents=[ + { + "Key": "a.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + }, + { + "Key": "A.txt", + "Size": 100, + "LastModified": "2023-01-01T00:00:00.000Z", + }, + ], + ) + ), + # CreateSession for the transfer/download client + create_session_response(), + head_object_response(), + get_object_response(b"foo"), + head_object_response(), + get_object_response(b"bar"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "cp", + "s3://bucket--usw2-az1--x-s3", + str(tmp_path), + "--recursive", + "--case-conflict", + "warn", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert "warning: Recursive copies/moves" in stderr.decode() diff --git a/tests/blackbox/test_mv_command.py b/tests/blackbox/test_mv_command.py new file mode 100644 index 000000000000..428c678f2c72 --- /dev/null +++ b/tests/blackbox/test_mv_command.py @@ -0,0 +1,1852 @@ +"""Blackbox tests for `aws s3 mv` command.""" + +from __future__ import annotations + +import os + +import pytest +from localstub.handlers import handle_expect_header +from localstub.server import HTTPResponse + +from tests.blackbox.s3_assertions import ( + assert_abort_multipart_upload, + assert_complete_multipart_upload, + assert_copy_object, + assert_create_multipart_upload, + assert_delete_object, + assert_get_access_point, + assert_get_caller_identity, + assert_get_object, + assert_get_object_tagging, + assert_head_object, + assert_put_object, + assert_put_object_tagging, + assert_upload_part_copy, +) +from tests.blackbox.utils import ( + abort_mpu_response, + cli_env, + complete_mpu_response, + copy_object_response, + create_mpu_response, + delete_response, + error_response, + format_requests, + get_object_response, + get_object_tagging_response, + head_object_response, + list_objects_xml, + put_object_response, + put_object_tagging_response, + run_cli, + mock_server, + setup_responses, + upload_part_copy_response, + upload_part_response, + xml_response, +) + + +@pytest.mark.asyncio +class TestMvCommand: + async def test_cant_mv_object_onto_itself(self, aws_cli, tmp_path): + """mv s3://bucket/key s3://bucket/key should fail.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "mv", "s3://bucket/key", "s3://bucket/key"], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_cant_mv_object_with_implied_name(self, aws_cli, tmp_path): + """mv s3://bucket/key s3://bucket/ implies same key name.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + ["s3", "mv", "s3://bucket/key", "s3://bucket/"], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_dryrun_move(self, aws_cli, tmp_path): + """mv s3->s3 --dryrun only does HeadObject, no copy or delete.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [head_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key.txt", + "s3://bucket/key2.txt", + "--dryrun", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumMode="ENABLED", + ) + assert ( + b"(dryrun) move: s3://bucket/key.txt to s3://bucket/key2.txt" + in stdout + ) + + async def test_website_redirect_ignore_paramfile(self, aws_cli, tmp_path): + """mv local s3:// --website-redirect uses the URL, not its contents.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + str(src), + "s3://bucket/key.txt", + "--website-redirect", + "http://someserver", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + WebsiteRedirectLocation="http://someserver", + ) + # Source file should be deleted after successful move + assert not src.exists() + + async def test_metadata_directive_copy(self, aws_cli, tmp_path): + """mv s3->s3 --metadata-directive REPLACE sends the directive on CopyObject.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key.txt", + "s3://bucket/key2.txt", + "--metadata-directive", + "REPLACE", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + assert_copy_object( + server.requests[1], + Bucket="bucket", + Key="key2.txt", + MetadataDirective="REPLACE", + ) + assert_delete_object( + server.requests[2], Bucket="bucket", Key="key.txt" + ) + + async def test_no_metadata_directive_for_non_copy(self, aws_cli, tmp_path): + """mv local s3:// --metadata-directive REPLACE does not send directive on PutObject.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + str(src), + "s3://bucket/key.txt", + "--metadata-directive", + "REPLACE", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], Bucket="bucket", Key="key.txt" + ) + # MetadataDirective header should NOT be present for uploads + assert ( + server.requests[0].headers.get("x-amz-metadata-directive") is None + ) + + async def test_download_move_with_request_payer(self, aws_cli, tmp_path): + """mv s3://bucket/key local --request-payer sends RequestPayer on all ops.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response(b"foo"), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://mybucket/mykey", + str(tmp_path), + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="mybucket", + Key="mykey", + ChecksumMode="ENABLED", + RequestPayer="requester", + ) + assert_get_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + assert_delete_object( + server.requests[2], + Bucket="mybucket", + Key="mykey", + RequestPayer="requester", + ) + + async def test_copy_move_with_request_payer(self, aws_cli, tmp_path): + """mv s3->s3 --request-payer sends RequestPayer on all ops.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://sourcebucket/sourcekey", + "s3://mybucket/mykey", + "--request-payer", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="sourcebucket", + Key="sourcekey", + ChecksumMode="ENABLED", + RequestPayer="requester", + ) + assert_copy_object( + server.requests[1], + Bucket="mybucket", + Key="mykey", + CopySource="sourcebucket/sourcekey", + RequestPayer="requester", + ) + assert_delete_object( + server.requests[2], + Bucket="sourcebucket", + Key="sourcekey", + RequestPayer="requester", + ) + + async def test_with_copy_props(self, aws_cli, tmp_path): + """mv s3->s3 --copy-props default preserves tags via multipart copy.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + large_tag_set = {"tag-key": "val" * 3000} + setup_responses( + server, + [ + head_object_response(content_length=8 * 1024**2), + get_object_tagging_response(large_tag_set), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + put_object_tagging_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://sourcebucket/sourcekey", + "s3://bucket/key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 7, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="sourcebucket", + Key="sourcekey", + ChecksumMode="ENABLED", + ) + assert_get_object_tagging( + server.requests[1], + Bucket="sourcebucket", + Key="sourcekey", + ) + assert_create_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="key", + ) + assert_upload_part_copy( + server.requests[3], + Bucket="bucket", + Key="key", + CopySource="sourcebucket/sourcekey", + ) + assert_complete_multipart_upload( + server.requests[4], + Bucket="bucket", + Key="key", + UploadId="upload_id", + ) + assert_put_object_tagging( + server.requests[5], Bucket="bucket", Key="key" + ) + assert_delete_object( + server.requests[6], + Bucket="sourcebucket", + Key="sourcekey", + ) + + async def test_mv_does_not_delete_source_on_failed_put_tagging( + self, aws_cli, tmp_path + ): + """mv s3->s3 --copy-props default deletes dest (not source) on PutObjectTagging failure.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + large_tag_set = {"tag-key": "val" * 3000} + setup_responses( + server, + [ + head_object_response(content_length=8 * 1024**2), + get_object_tagging_response(large_tag_set), + create_mpu_response("upload_id"), + upload_part_copy_response(), + complete_mpu_response(), + error_response( + "AccessDenied", "Operation not allowed", status=403 + ), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://sourcebucket/sourcekey", + "s3://bucket/key", + "--copy-props", + "default", + ], + cli_env(proxy), + ) + + assert rc == 1 + assert len(server.requests) == 7, format_requests(server) + assert_head_object( + server.requests[0], + Bucket="sourcebucket", + Key="sourcekey", + ChecksumMode="ENABLED", + ) + assert_get_object_tagging( + server.requests[1], + Bucket="sourcebucket", + Key="sourcekey", + ) + assert_create_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="key", + ) + assert_upload_part_copy( + server.requests[3], + Bucket="bucket", + Key="key", + CopySource="sourcebucket/sourcekey", + ) + assert_complete_multipart_upload( + server.requests[4], + Bucket="bucket", + Key="key", + UploadId="upload_id", + ) + assert_put_object_tagging( + server.requests[5], Bucket="bucket", Key="key" + ) + # The delete should be for the destination (cleanup), not the source + assert_delete_object( + server.requests[6], Bucket="bucket", Key="key" + ) + + async def test_upload_with_checksum_algorithm_crc32( + self, aws_cli, tmp_path + ): + """mv local s3:// --checksum-algorithm CRC32 sends the algorithm.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + str(src), + "s3://bucket/key.txt", + "--checksum-algorithm", + "CRC32", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="key.txt", + ChecksumAlgorithm="CRC32", + ) + assert not src.exists() + + async def test_download_with_checksum_mode_crc32(self, aws_cli, tmp_path): + """mv s3://bucket/foo local --checksum-mode ENABLED sends ChecksumMode.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response( + b"foo", **{"x-amz-checksum-crc32": "jHNlIQ=="} + ), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/foo", + str(tmp_path), + "--checksum-mode", + "ENABLED", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_get_object( + server.requests[1], + Bucket="bucket", + Key="foo", + ChecksumMode="ENABLED", + ) + assert_delete_object( + server.requests[2], Bucket="bucket", Key="foo" + ) + + async def test_mv_no_overwrite_flag_when_object_not_exists_on_target( + self, aws_cli, tmp_path + ): + """mv local s3:// --no-overwrite sends IfNoneMatch=* and deletes source on success.""" + src = tmp_path / "foo.txt" + src.write_text("contents") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [put_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + str(src), + "s3://bucket/foo.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="foo.txt", + IfNoneMatch="*", + ) + assert not src.exists() + + async def test_mv_no_overwrite_flag_when_object_exists_on_target( + self, aws_cli, tmp_path + ): + """mv local s3:// --no-overwrite with 412 keeps source file.""" + src = tmp_path / "foo.txt" + src.write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + str(src), + "s3://bucket/foo.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_put_object( + server.requests[0], + Bucket="bucket", + Key="foo.txt", + IfNoneMatch="*", + ) + # Source file should NOT be deleted + assert src.exists() + + async def test_mv_no_overwrite_flag_multipart_upload_when_object_not_exists_on_target( + self, aws_cli, tmp_path + ): + """mv local s3:// --no-overwrite large file multipart sends IfNoneMatch on Complete.""" + src = tmp_path / "foo.txt" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("foo"), + upload_part_response("etag1"), + upload_part_response("etag2"), + complete_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + str(src), + "s3://bucket/foo.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_create_multipart_upload( + server.requests[0], + Bucket="bucket", + Key="foo.txt", + ) + assert_complete_multipart_upload( + server.requests[3], + Bucket="bucket", + Key="foo.txt", + IfNoneMatch="*", + ) + assert not src.exists() + + async def test_mv_no_overwrite_flag_multipart_upload_when_object_exists_on_target( + self, aws_cli, tmp_path + ): + """mv local s3:// --no-overwrite large file with 412 on Complete aborts and keeps source.""" + src = tmp_path / "foo.txt" + src.write_bytes(b"a" * 10 * (1024**2)) + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + create_mpu_response("foo"), + upload_part_response("etag1"), + upload_part_response("etag2"), + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ), + abort_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + str(src), + "s3://bucket/foo.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_create_multipart_upload( + server.requests[0], + Bucket="bucket", + Key="foo.txt", + ) + assert_complete_multipart_upload( + server.requests[3], + Bucket="bucket", + Key="foo.txt", + IfNoneMatch="*", + ) + assert_abort_multipart_upload( + server.requests[4], + Bucket="bucket", + Key="foo.txt", + UploadId="foo", + ) + assert src.exists() + + async def test_mv_no_overwrite_flag_on_copy_when_small_object_does_not_exist_on_target( + self, aws_cli, tmp_path + ): + """mv s3->s3 --no-overwrite small object sends IfNoneMatch on CopyObject and deletes source.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket1/key.txt", + "s3://bucket2/key1.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket1", Key="key.txt" + ) + assert_copy_object( + server.requests[1], + Bucket="bucket2", + Key="key1.txt", + IfNoneMatch="*", + ) + assert_delete_object( + server.requests[2], Bucket="bucket1", Key="key.txt" + ) + + async def test_mv_no_overwrite_flag_on_copy_when_small_object_exists_on_target( + self, aws_cli, tmp_path + ): + """mv s3->s3 --no-overwrite small object with 412 does not delete source.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket1/key.txt", + "s3://bucket2/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 2, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket1", Key="key.txt" + ) + assert_copy_object( + server.requests[1], + Bucket="bucket2", + Key="key.txt", + IfNoneMatch="*", + ) + + async def test_mv_no_overwrite_flag_when_large_object_does_not_exist_on_target( + self, aws_cli, tmp_path + ): + """mv s3->s3 --no-overwrite large object multipart copy succeeds and deletes source.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("foo"), + upload_part_copy_response(), + upload_part_copy_response(), + complete_mpu_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket1/key1.txt", + "s3://bucket/key.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 7, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket1", Key="key1.txt" + ) + assert_get_object_tagging( + server.requests[1], + Bucket="bucket1", + Key="key1.txt", + ) + assert_create_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="key.txt", + ) + assert_complete_multipart_upload( + server.requests[5], + Bucket="bucket", + Key="key.txt", + IfNoneMatch="*", + ) + assert_delete_object( + server.requests[6], + Bucket="bucket1", + Key="key1.txt", + ) + + async def test_mv_no_overwrite_flag_when_large_object_exists_on_target( + self, aws_cli, tmp_path + ): + """mv s3->s3 --no-overwrite large object with 412 on Complete aborts.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(content_length=10 * (1024**2)), + get_object_tagging_response(), + create_mpu_response("foo"), + upload_part_copy_response(), + upload_part_copy_response(), + error_response( + "PreconditionFailed", + "At least one of the pre-conditions you specified did not hold", + status=412, + ), + abort_mpu_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket1/key1.txt", + "s3://bucket/key1.txt", + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 7, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket1", Key="key1.txt" + ) + assert_get_object_tagging( + server.requests[1], + Bucket="bucket1", + Key="key1.txt", + ) + assert_create_multipart_upload( + server.requests[2], + Bucket="bucket", + Key="key1.txt", + ) + assert_complete_multipart_upload( + server.requests[5], + Bucket="bucket", + Key="key1.txt", + IfNoneMatch="*", + ) + assert_abort_multipart_upload( + server.requests[6], + Bucket="bucket", + Key="key1.txt", + UploadId="foo", + ) + + async def test_no_overwrite_flag_on_mv_download_when_single_object_exists_at_target( + self, aws_cli, tmp_path + ): + """mv s3://bucket/foo.txt local --no-overwrite skips if file exists locally.""" + target = tmp_path / "foo.txt" + target.write_text("existing content") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [head_object_response()]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/foo.txt", + str(target), + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 1, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="foo.txt" + ) + # File should retain original content (not overwritten) + assert target.read_text() == "existing content" + + async def test_no_overwrite_flag_on_mv_download_when_single_object_does_not_exist_at_target( + self, aws_cli, tmp_path + ): + """mv s3://bucket/foo.txt local --no-overwrite downloads and deletes source if no local file.""" + target = tmp_path / "foo.txt" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + get_object_response(b"foo"), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/foo.txt", + str(target), + "--no-overwrite", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + assert_head_object( + server.requests[0], Bucket="bucket", Key="foo.txt" + ) + assert_get_object( + server.requests[1], Bucket="bucket", Key="foo.txt" + ) + assert_delete_object( + server.requests[2], Bucket="bucket", Key="foo.txt" + ) + assert target.read_text() == "foo" + + +def get_access_point_response(bucket: str) -> HTTPResponse: + body = ( + '' + "" + f"{bucket}" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def get_caller_identity_response( + account: str = "123456789012", +) -> HTTPResponse: + body = ( + '' + "" + f"{account}" + "arn:aws:iam::123456789012:user/test" + "AIDATEST" + "" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "text/xml"} + ) + + +def list_multi_region_access_points_response( + alias: str, buckets: list[str] +) -> HTTPResponse: + regions = "".join( + f"{b}" for b in buckets + ) + body = ( + '' + "" + "" + f"{alias}{regions}" + "" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +@pytest.mark.asyncio +class TestMvCommandWithValidateSameS3Paths: + async def test_cant_mv_object_onto_itself_access_point_arn( + self, aws_cli, tmp_path + ): + """mv with access point ARN resolving to same bucket fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [get_access_point_response("bucket")]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_cant_mv_object_onto_itself_access_point_arn_as_source( + self, aws_cli, tmp_path + ): + """mv with access point ARN as source resolving to same bucket fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [get_access_point_response("bucket")]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/key", + "s3://bucket/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_cant_mv_object_onto_itself_access_point_arn_with_env_var( + self, aws_cli, tmp_path + ): + """mv with access point ARN uses env var to enable validation.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [get_access_point_response("bucket")]) + env = cli_env(proxy) + env["AWS_CLI_S3_MV_VALIDATE_SAME_S3_PATHS"] = "true" + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/key", + ], + env, + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_cant_mv_object_onto_itself_access_point_arn_base_key( + self, aws_cli, tmp_path + ): + """mv with access point ARN and implied key resolving to same bucket fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [get_access_point_response("bucket")]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_cant_mv_object_onto_itself_access_point_arn_base_prefix( + self, aws_cli, tmp_path + ): + """mv with access point ARN and matching prefix resolving to same bucket fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [get_access_point_response("bucket")]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/prefix/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/prefix/", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_cant_mv_object_onto_itself_access_point_alias( + self, aws_cli, tmp_path + ): + """mv with access point alias resolving to same bucket fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + get_caller_identity_response("123456789012"), + get_access_point_response("bucket"), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://myaccesspoint-foobar-s3alias/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_cant_mv_object_onto_itself_outpost_access_point_arn( + self, aws_cli, tmp_path + ): + """mv with outpost access point ARN resolving to same bucket fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, [get_access_point_response("bucket")]) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3-outposts:us-east-1:123456789012:outpost/op-foobar/accesspoint/myaccesspoint/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_outpost_access_point_alias_raises_error( + self, aws_cli, tmp_path + ): + """mv with outpost access point alias raises unresolvable error.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://myaccesspoint-foobar--op-s3/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Can't resolve underlying bucket name" in stderr + + async def test_cant_mv_object_onto_itself_mrap_arn( + self, aws_cli, tmp_path + ): + """mv with MRAP ARN resolving to same bucket fails.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + list_multi_region_access_points_response( + "foobar.mrap", ["differentbucket", "bucket"] + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3::123456789012:accesspoint/foobar.mrap/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"Cannot mv a file onto itself" in stderr + + async def test_get_mrap_buckets_raises_if_alias_not_found( + self, aws_cli, tmp_path + ): + """mv with MRAP ARN where alias not found raises error.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + list_multi_region_access_points_response( + "baz.mrap", ["differentbucket", "bucket"] + ), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3::123456789012:accesspoint/foobar.mrap/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert ( + b"Couldn't find multi-region access point with alias foobar.mrap" + in stderr + ) + + async def test_mv_works_if_access_point_arn_resolves_to_different_bucket( + self, aws_cli, tmp_path + ): + """mv with access point ARN resolving to different bucket succeeds.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + get_access_point_response("differentbucket"), + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_get_access_point(server.requests[0]) + assert_head_object( + server.requests[1], Bucket="bucket", Key="key" + ) + assert_copy_object(server.requests[2]) + assert_delete_object( + server.requests[3], Bucket="bucket", Key="key" + ) + + async def test_mv_works_if_access_point_alias_resolves_to_different_bucket( + self, aws_cli, tmp_path + ): + """mv with access point alias resolving to different bucket succeeds.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + get_caller_identity_response("123456789012"), + get_access_point_response("differentbucket"), + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://myaccesspoint-foobar-s3alias/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 5, format_requests(server) + assert_get_caller_identity(server.requests[0]) + assert_get_access_point(server.requests[1]) + assert_head_object( + server.requests[2], Bucket="bucket", Key="key" + ) + assert_copy_object(server.requests[3]) + assert_delete_object( + server.requests[4], Bucket="bucket", Key="key" + ) + + async def test_mv_works_if_outpost_access_point_arn_resolves_to_different_bucket( + self, aws_cli, tmp_path + ): + """mv with outpost access point ARN resolving to different bucket succeeds.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + get_access_point_response("differentbucket"), + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3-outposts:us-east-1:123456789012:outpost/op-foobar/accesspoint/myaccesspoint/key", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 4, format_requests(server) + assert_get_access_point(server.requests[0]) + assert_head_object( + server.requests[1], Bucket="bucket", Key="key" + ) + assert_copy_object(server.requests[2]) + assert_delete_object( + server.requests[3], Bucket="bucket", Key="key" + ) + + async def test_skips_validation_if_keys_are_different_accesspoint_arn( + self, aws_cli, tmp_path + ): + """mv with access point ARN and different key skips validation.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/key2", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + + async def test_skips_validation_if_prefixes_are_different_accesspoint_arn( + self, aws_cli, tmp_path + ): + """mv with access point ARN and different prefix skips validation.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/prefix/", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + + async def test_skips_validation_if_keys_are_different_accesspoint_alias( + self, aws_cli, tmp_path + ): + """mv with access point alias and different key skips validation.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://myaccesspoint-foobar-s3alias/key2", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + + async def test_skips_validation_if_keys_are_different_outpost_arn( + self, aws_cli, tmp_path + ): + """mv with outpost ARN and different key skips validation.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3-outposts:us-east-1:123456789012:outpost/op-foobar/accesspoint/myaccesspoint/key2", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + + async def test_skips_validation_if_keys_are_different_outpost_alias( + self, aws_cli, tmp_path + ): + """mv with outpost alias and different key skips validation.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://myaccesspoint-foobar--op-s3/key2", + "--validate-same-s3-paths", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert len(server.requests) == 3, format_requests(server) + + async def test_raises_warning_if_validation_not_set( + self, aws_cli, tmp_path + ): + """mv with access point ARN without --validate-same-s3-paths raises warning.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://bucket/key", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Provided s3 paths may resolve" in stderr + + async def test_raises_warning_if_validation_not_set_source( + self, aws_cli, tmp_path + ): + """mv with access point ARN as source without --validate-same-s3-paths raises warning.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + head_object_response(), + copy_object_response(), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "s3://arn:aws:s3:us-west-2:123456789012:accesspoint/myaccesspoint/key", + "s3://bucket/key", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert b"warning: Provided s3 paths may resolve" in stderr + + +def _is_case_insensitive() -> bool: + """Check if the filesystem is case-insensitive.""" + import tempfile + + with tempfile.TemporaryDirectory() as d: + upper = os.path.join(d, "A") + open(upper, "w").close() + return os.path.exists(os.path.join(d, "a")) + + +@pytest.mark.asyncio +class TestMvRecursiveCaseConflict: + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_warn_with_existing_file(self, aws_cli, tmp_path): + """mv --recursive s3->local --case-conflict warn warns on case conflict with existing file.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 3, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + get_object_response(b"foo"), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "--recursive", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "warn", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert "warning: Downloading bucket/A.txt" in stderr.decode() + + async def test_skip_with_case_conflicts_in_s3(self, aws_cli, tmp_path): + """mv --recursive s3->local --case-conflict skip skips conflicting keys.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 3, + "LastModified": "2023-01-01T00:00:00Z", + }, + { + "Key": "a.txt", + "Size": 3, + "LastModified": "2023-01-01T00:00:00Z", + }, + ], + ) + ), + get_object_response(b"foo"), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "--recursive", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "skip", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + assert "warning: Skipping bucket/a.txt" in stderr.decode() + + @pytest.mark.skipif( + not _is_case_insensitive(), + reason="Requires case-insensitive filesystem", + ) + async def test_ignore_with_existing_file(self, aws_cli, tmp_path): + """mv --recursive s3->local --case-conflict ignore proceeds without warning.""" + (tmp_path / "a.txt").write_text("mycontent") + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses( + server, + [ + xml_response( + list_objects_xml( + contents=[ + { + "Key": "A.txt", + "Size": 3, + "LastModified": "2023-01-01T00:00:00Z", + } + ], + ) + ), + get_object_response(b"foo"), + delete_response(), + ], + ) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "--recursive", + "s3://bucket", + str(tmp_path), + "--case-conflict", + "ignore", + ], + cli_env(proxy), + ) + + assert rc == 0, stderr.decode() + + +@pytest.mark.asyncio +class TestS3ExpressMvRecursive: + async def test_s3_express_error_raises_exception(self, aws_cli, tmp_path): + """mv --recursive with S3 Express bucket rejects --case-conflict error.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "--recursive", + "s3://bucket--usw2-az1--x-s3", + str(tmp_path), + "--case-conflict", + "error", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"`error` is not a valid value" in stderr + + async def test_s3_express_skip_raises_exception(self, aws_cli, tmp_path): + """mv --recursive with S3 Express bucket rejects --case-conflict skip.""" + async with mock_server(on_headers_received=handle_expect_header) as ( + server, + proxy, + ): + setup_responses(server, []) + stdout, stderr, rc = await run_cli( + aws_cli, + [ + "s3", + "mv", + "--recursive", + "s3://bucket--usw2-az1--x-s3", + str(tmp_path), + "--case-conflict", + "skip", + ], + cli_env(proxy), + ) + + assert rc == 252 + assert b"`skip` is not a valid value" in stderr diff --git a/tests/blackbox/utils.py b/tests/blackbox/utils.py index 72873b0aee34..be8bb20fde73 100644 --- a/tests/blackbox/utils.py +++ b/tests/blackbox/utils.py @@ -128,6 +128,146 @@ def list_objects_xml( return xml +def put_object_response() -> HTTPResponse: + return HTTPResponse.raw( + b"", status=200, headers={"ETag": '"c8afdb36c52cf4727836669019e69222"'} + ) + + +def head_object_response( + content_length: int = 100, **extra_headers +) -> HTTPResponse: + headers = { + "Content-Length": str(content_length), + "Last-Modified": "Thu, 01 Jan 1970 00:00:00 GMT", + "ETag": '"foo-1"', + } + headers.update(extra_headers) + return HTTPResponse.raw(b"", status=200, headers=headers) + + +def get_object_response(body: bytes = b"foo", **extra_headers) -> HTTPResponse: + headers = { + "Content-Length": str(len(body)), + "ETag": '"foo-1"', + } + headers.update(extra_headers) + return HTTPResponse.raw(body, status=200, headers=headers) + + +def empty_list_objects_response() -> HTTPResponse: + body = ( + '' + '' + "bucketfalse" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def create_mpu_response(upload_id: str) -> HTTPResponse: + body = ( + '' + '' + f"bucketkey.txt{upload_id}" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def upload_part_response(etag: str) -> HTTPResponse: + return HTTPResponse.raw(b"", status=200, headers={"ETag": f'"{etag}"'}) + + +def complete_mpu_response() -> HTTPResponse: + body = ( + '' + '' + "http://bucket.s3.amazonaws.com/key.txt" + "bucketkey.txt" + '"etag"' + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def abort_mpu_response() -> HTTPResponse: + return HTTPResponse.raw(b"", status=204) + + +def copy_object_response() -> HTTPResponse: + body = ( + '' + '' + '"etag"' + "2023-01-01T00:00:00.000Z" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def get_object_tagging_response(tags: dict | None = None) -> HTTPResponse: + tag_xml = "" + if tags: + for k, v in tags.items(): + tag_xml += f"{k}{v}" + body = ( + '' + '' + f"{tag_xml}" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def put_object_tagging_response() -> HTTPResponse: + return HTTPResponse.raw(b"", status=200) + + +def upload_part_copy_response() -> HTTPResponse: + body = ( + '' + '' + '"etag"' + "2023-01-01T00:00:00.000Z" + "" + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def create_session_response() -> HTTPResponse: + body = ( + '' + '' + '' + 'ASIATESTSESSIONKEY' + 'testsessionsecret' + 'testsessiontoken' + '2099-01-01T00:00:00Z' + '' + '' + ) + return HTTPResponse.raw( + body.encode(), status=200, headers={"Content-Type": "application/xml"} + ) + + +def get_path(request) -> str: + """Get the path portion without query string.""" + return urlparse(request.path).path + + def delete_response() -> HTTPResponse: return HTTPResponse.raw(b"", status=204) From beb314969e2117fa42e6952b6701f782da5a95e0 Mon Sep 17 00:00:00 2001 From: aemous Date: Mon, 24 Aug 2026 09:25:13 -0400 Subject: [PATCH 2/2] Remove unnecessary comment. --- tests/blackbox/s3_assertions.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/tests/blackbox/s3_assertions.py b/tests/blackbox/s3_assertions.py index 5fd79a565526..3a022f566255 100644 --- a/tests/blackbox/s3_assertions.py +++ b/tests/blackbox/s3_assertions.py @@ -757,11 +757,6 @@ def assert_put_object_annotation(request, Bucket: str, Key: str, addressing_styl _check_params(request, params, _PUT_OBJECT_ANNOTATION_PARAMS, "PutObjectAnnotation") -# --------------------------------------------------------------------------- -# Non-S3 service helpers (used by mv --validate-same-s3-paths tests) -# --------------------------------------------------------------------------- - - def assert_get_access_point(request): """Assert request is an S3 Control GetAccessPoint (GET /v20180820/accesspoint/{name}).""" assert request.method == "GET", (