diff --git a/cpp/src/arrow/util/compression_test.cc b/cpp/src/arrow/util/compression_test.cc
index 90d07ea63234..17232abf567d 100644
--- a/cpp/src/arrow/util/compression_test.cc
+++ b/cpp/src/arrow/util/compression_test.cc
@@ -558,9 +558,11 @@ TEST(TestCodecMisc, SpecifyCodecOptionsGZip) {
   };
   const std::pair<arrow::util::GZipCodecOptions, bool> options[]{
       {make_option(5, GZipFormat::GZIP, 15), true},
+      {make_option(0, GZipFormat::GZIP, 15), true},
       {make_option(9, GZipFormat::ZLIB, 12), true},
       {make_option(-1, GZipFormat::DEFLATE, 10), true},
-      {make_option(10, GZipFormat::GZIP, 25), false},
+      {make_option(10, GZipFormat::GZIP, 15), false},
+      {make_option(5, GZipFormat::GZIP, 25), false},
       {make_option(-992, GZipFormat::GZIP, 15), false},
   };
   CheckSpecifyCodecOptions<arrow::util::GZipCodecOptions>(Compression::GZIP, options);
diff --git a/cpp/src/arrow/util/compression_zlib.cc b/cpp/src/arrow/util/compression_zlib.cc
index b06cf2d22430..96f2e0c02685 100644
--- a/cpp/src/arrow/util/compression_zlib.cc
+++ b/cpp/src/arrow/util/compression_zlib.cc
@@ -494,11 +494,11 @@ class GZipCodec : public Codec {
       return Status::Invalid("GZip window_bits should be between ", kGZipMinWindowBits,
                              " and ", kGZipMaxWindowBits);
     }
-    const Status init_compressor_status = InitCompressor();
-    if (!init_compressor_status.ok()) {
-      return init_compressor_status;
+    if (compression_level_ < Z_DEFAULT_COMPRESSION ||
+        compression_level_ > Z_BEST_COMPRESSION) {
+      return Status::Invalid("Invalid GZip compression level: ", compression_level_);
     }
-    return InitDecompressor();
+    return Status::OK();
   }
 
   Compression::type compression_type() const override { return Compression::GZIP; }