diff --git a/cpp/src/arrow/filesystem/hdfs_test.cc b/cpp/src/arrow/filesystem/hdfs_test.cc
index db5cefef3748..eab536c77c96 100644
--- a/cpp/src/arrow/filesystem/hdfs_test.cc
+++ b/cpp/src/arrow/filesystem/hdfs_test.cc
@@ -346,7 +346,6 @@ class TestHadoopFileSystemGeneric : public ::testing::Test,
   bool allow_write_file_over_dir() const override { return true; }
   bool allow_move_dir_over_non_empty_dir() const override { return true; }
   bool have_implicit_directories() const override { return true; }
-  bool allow_append_to_new_file() const override { return false; }
 
   std::shared_ptr<FileSystem> GetEmptyFileSystem() override {
     // Since the HDFS contents are kept persistently between test runs,
diff --git a/cpp/src/arrow/io/hdfs.cc b/cpp/src/arrow/io/hdfs.cc
index 73d8e1575721..dc1878d368ef 100644
--- a/cpp/src/arrow/io/hdfs.cc
+++ b/cpp/src/arrow/io/hdfs.cc
@@ -529,7 +529,10 @@ class HadoopFileSystem::HadoopFileSystemImpl {
                       int16_t replication, int64_t default_block_size,
                       std::shared_ptr<HdfsOutputStream>* file) {
     int flags = O_WRONLY;
-    if (append) flags |= O_APPEND;
+    // Hadoop's FileSystem::append (unlike a POSIX O_CREAT|O_APPEND open) requires
+    // the target file to already exist, so only request append semantics if the
+    // file is actually there.
+    if (append && Exists(path)) flags |= O_APPEND;
 
     errno = 0;
     hdfsFile handle =