Fix Cuttlefish multi-instance restart deadlocks, socket hangs, and UWB/bootconfig configuration mismatches

base/cvd/cuttlefish/common/libs/utils/files.cpp

@@ -734,4 +734,35 @@ Result<std::string> Search(const std::vector<std::string>& path,
   return CF_ERR("Not found: ") << name << ", path " << absl::StrJoin(path, ":");
 }
 
+Result<SharedFD> CreateOrReuseAndDrainFifo(const std::string& path, mode_t mode) {
+  struct stat st {};
+  bool existed = false;
+  if (TEMP_FAILURE_RETRY(stat(path.c_str(), &st)) != 0) {
+    CF_EXPECTF(TEMP_FAILURE_RETRY(mkfifo(path.c_str(), mode)) == 0,
+               "Failed to mkfifo('{}', {:o}): {}", path, mode,
+               ::cuttlefish::StrError(errno));
+  } else {
+    CF_EXPECTF(S_ISFIFO(st.st_mode),
+               "File at '{}' exists but is not a FIFO", path);
+    existed = true;
+  }
+
+  auto ret = SharedFD::Open(path, O_RDWR);
+  CF_EXPECTF(ret->IsOpen(), "Failed to open '{}': '{}'", path, ret->StrError());
+
+  if (existed) {
+    int flags = ret->Fcntl(F_GETFL, 0);
+    if (flags >= 0) {
+      ret->Fcntl(F_SETFL, flags | O_NONBLOCK);
+      char buf[4096];
+      while (ret->Read(buf, sizeof(buf)) > 0) {
+        // Reading while there is data to read
+      }
+      ret->Fcntl(F_SETFL, flags);
+    }
+  }
+
+  return ret;
+}
+
 }  // namespace cuttlefish

base/cvd/cuttlefish/common/libs/utils/files.h

@@ -26,6 +26,7 @@
 #include <vector>
 
 #include "cuttlefish/result/result.h"
+#include "cuttlefish/common/libs/fs/shared_fd.h"
 
 namespace cuttlefish {
 bool FileExists(const std::string& path, bool follow_symlinks = true);
@@ -138,4 +139,6 @@ std::vector<std::string> Path(const std::string& env_name = "PATH");
 Result<std::string> Search(const std::vector<std::string>& path,
                            std::string_view name);
 
+Result<SharedFD> CreateOrReuseAndDrainFifo(const std::string& path, mode_t mode);
+
 }  // namespace cuttlefish

base/cvd/cuttlefish/host/commands/run_cvd/launch/BUILD.bazel

@@ -34,6 +34,7 @@ cf_cc_library(
     hdrs = ["bluetooth_connector.h"],
     deps = [
         "//cuttlefish/common/libs/fs",
+        "//cuttlefish/common/libs/utils:files",
         "//cuttlefish/host/libs/config:cuttlefish_config",
         "//cuttlefish/host/libs/config:known_paths",
         "//cuttlefish/host/libs/feature",
@@ -149,6 +150,7 @@ cf_cc_library(
     hdrs = ["gnss_grpc_proxy.h"],
     deps = [
         "//cuttlefish/common/libs/fs",
+        "//cuttlefish/common/libs/utils:files",
         "//cuttlefish/common/libs/utils:in_sandbox",
         "//cuttlefish/host/commands/run_cvd/launch:grpc_socket_creator",
         "//cuttlefish/host/libs/config:cuttlefish_config",
@@ -290,6 +292,7 @@ cf_cc_library(
     hdrs = ["nfc_connector.h"],
     deps = [
         "//cuttlefish/common/libs/fs",
+        "//cuttlefish/common/libs/utils:files",
         "//cuttlefish/common/libs/utils:subprocess",
         "//cuttlefish/host/libs/config:cuttlefish_config",
         "//cuttlefish/host/libs/config:known_paths",
@@ -383,6 +386,8 @@ cf_cc_library(
     srcs = ["secure_env.cpp"],
     hdrs = ["secure_env.h"],
     deps = [
+        "//cuttlefish/common/libs/fs",
+        "//cuttlefish/common/libs/utils:files",
         "//cuttlefish/common/libs/utils:subprocess",
         "//cuttlefish/host/commands/run_cvd/launch:snapshot_control_files",
         "//cuttlefish/host/libs/config:cuttlefish_config",
@@ -399,6 +404,7 @@ cf_cc_library(
     hdrs = ["sensors_simulator.h"],
     deps = [
         "//cuttlefish/common/libs/fs",
+        "//cuttlefish/common/libs/utils:files",
         "//cuttlefish/common/libs/utils:subprocess",
         "//cuttlefish/host/commands/run_cvd/launch:sensors_socket_pair",
         "//cuttlefish/host/libs/config:cuttlefish_config",
@@ -438,6 +444,7 @@ cf_cc_library(
     hdrs = ["streamer.h"],
     deps = [
         "//cuttlefish/common/libs/fs",
+        "//cuttlefish/common/libs/utils:files",
         "//cuttlefish/common/libs/utils:subprocess",
         "//cuttlefish/host/commands/run_cvd:reporting",
         "//cuttlefish/host/commands/run_cvd/launch:enable_multitouch",
@@ -506,6 +513,7 @@ cf_cc_library(
     hdrs = ["uwb_connector.h"],
     deps = [
         "//cuttlefish/common/libs/fs",
+        "//cuttlefish/common/libs/utils:files",
         "//cuttlefish/common/libs/utils:subprocess",
         "//cuttlefish/host/libs/config:config_utils",
         "//cuttlefish/host/libs/config:cuttlefish_config",

base/cvd/cuttlefish/host/commands/run_cvd/launch/bluetooth_connector.cpp

@@ -22,6 +22,7 @@
 #include <vector>
 
 #include "cuttlefish/common/libs/fs/shared_fd.h"
+#include "cuttlefish/common/libs/utils/files.h"
 #include "cuttlefish/host/libs/config/cuttlefish_config.h"
 #include "cuttlefish/host/libs/config/known_paths.h"
 #include "cuttlefish/host/libs/feature/command_source.h"
@@ -49,7 +50,7 @@ Result<std::optional<MonitorCommand>> BluetoothConnector(
   };
   std::vector<SharedFD> fifos;
   for (const auto& path : fifo_paths) {
-    fifos.emplace_back(CF_EXPECT(SharedFD::Fifo(path, 0660)));
+    fifos.emplace_back(CF_EXPECT(CreateOrReuseAndDrainFifo(path, 0660)));
   }
   return Command(TcpConnectorBinary())
       .AddParameter("-fifo_out=", fifos[0])

base/cvd/cuttlefish/host/commands/run_cvd/launch/gnss_grpc_proxy.cpp

@@ -20,6 +20,7 @@
 #include <vector>
 
 #include "cuttlefish/common/libs/fs/shared_fd.h"
+#include "cuttlefish/common/libs/utils/files.h"
 #include "cuttlefish/common/libs/utils/in_sandbox.h"
 #include "cuttlefish/host/commands/run_cvd/launch/grpc_socket_creator.h"
 #include "cuttlefish/host/libs/config/cuttlefish_config.h"
@@ -43,7 +44,7 @@ Result<std::optional<MonitorCommand>> GnssGrpcProxyServer(
       instance.PerInstanceInternalPath("locationhvc_fifo_vm.out"),
   };
   for (const auto& path : fifo_paths) {
-    fifos.emplace_back(CF_EXPECT(SharedFD::Fifo(path, 0660)));
+    fifos.emplace_back(CF_EXPECT(CreateOrReuseAndDrainFifo(path, 0660)));
   }
 
   auto gnss_grpc_proxy_cmd =

base/cvd/cuttlefish/host/commands/run_cvd/launch/netsim_server.cpp

@@ -209,7 +209,7 @@ class NetsimServer : public CommandSource {
   Result<SharedFD> MakeFifo(const CuttlefishConfig::InstanceSpecific& instance,
                             const char* relative_path) {
     auto path = instance.PerInstanceInternalPath(relative_path);
-    return CF_EXPECT(SharedFD::Fifo(path, 0660));
+    return CF_EXPECT(CreateOrReuseAndDrainFifo(path, 0660));
   }
 
  private:

base/cvd/cuttlefish/host/commands/run_cvd/launch/nfc_connector.cpp

@@ -21,6 +21,7 @@
 #include <vector>
 
 #include "cuttlefish/common/libs/fs/shared_fd.h"
+#include "cuttlefish/common/libs/utils/files.h"
 #include "cuttlefish/common/libs/utils/subprocess.h"
 #include "cuttlefish/host/libs/config/cuttlefish_config.h"
 #include "cuttlefish/host/libs/config/known_paths.h"
@@ -40,7 +41,7 @@ Result<MonitorCommand> NfcConnector(
   };
   std::vector<SharedFD> fifos;
   for (const auto& path : fifo_paths) {
-    fifos.emplace_back(CF_EXPECT(SharedFD::Fifo(path, 0660)));
+    fifos.emplace_back(CF_EXPECT(CreateOrReuseAndDrainFifo(path, 0660)));
   }
   return Command(TcpConnectorBinary())
       .AddParameter("-fifo_out=", fifos[0])

base/cvd/cuttlefish/host/commands/run_cvd/launch/secure_env.cpp

@@ -18,6 +18,7 @@
 #include <string>
 #include <vector>
 
+#include "cuttlefish/common/libs/utils/files.h"
 #include "cuttlefish/common/libs/utils/subprocess.h"
 #include "cuttlefish/host/commands/run_cvd/launch/snapshot_control_files.h"
 #include "cuttlefish/host/libs/config/cuttlefish_config.h"
@@ -54,7 +55,7 @@ Result<MonitorCommand> SecureEnv(
   };
   std::vector<SharedFD> fifos;
   for (const auto& path : fifo_paths) {
-    fifos.emplace_back(CF_EXPECT(SharedFD::Fifo(path, 0660)));
+    fifos.emplace_back(CF_EXPECT(CreateOrReuseAndDrainFifo(path, 0660)));
   }
   command.AddParameter("-keymaster_fd_out=", fifos[0]);
   command.AddParameter("-keymaster_fd_in=", fifos[1]);

base/cvd/cuttlefish/host/commands/run_cvd/launch/sensors_simulator.cpp

@@ -15,11 +15,10 @@
 
 #include "cuttlefish/host/commands/run_cvd/launch/sensors_simulator.h"
 
-#include <unistd.h>
-
 #include <string>
 
 #include "cuttlefish/common/libs/fs/shared_fd.h"
+#include "cuttlefish/common/libs/utils/files.h"
 #include "cuttlefish/common/libs/utils/subprocess.h"
 #include "cuttlefish/host/commands/run_cvd/launch/sensors_socket_pair.h"
 #include "cuttlefish/host/libs/config/cuttlefish_config.h"
@@ -32,8 +31,7 @@ namespace cuttlefish {
 namespace {
 
 Result<SharedFD> CreateFifo(const std::string& path) {
-  unlink(path.c_str());
-  return SharedFD::Fifo(path, 0660);
+  return CreateOrReuseAndDrainFifo(path, 0660);
 }
 
 }  // namespace

base/cvd/cuttlefish/host/commands/run_cvd/launch/streamer.cpp

@@ -17,7 +17,6 @@
 
 #include <errno.h>
 #include <sys/socket.h>
-#include <unistd.h>
 
 #include <sstream>
 #include <string>
@@ -31,6 +30,7 @@
 #include "absl/log/log.h"
 
 #include "cuttlefish/common/libs/fs/shared_fd.h"
+#include "cuttlefish/common/libs/utils/files.h"
 #include "cuttlefish/common/libs/utils/subprocess.h"
 #include "cuttlefish/host/commands/run_cvd/launch/enable_multitouch.h"
 #include "cuttlefish/host/commands/run_cvd/launch/input_connections_provider.h"
@@ -178,12 +178,9 @@ class StreamerSockets : public virtual SetupFeature {
         instance_.PerInstanceInternalPath("confui_fifo_vm.in"),
         instance_.PerInstanceInternalPath("confui_fifo_vm.out"),
     };
-    for (const auto& path : fifo_files) {
-      unlink(path.c_str());
-    }
     std::vector<SharedFD> fds;
     for (const auto& path : fifo_files) {
-      fds.emplace_back(CF_EXPECT(SharedFD::Fifo(path, 0660)));
+      fds.emplace_back(CF_EXPECT(CreateOrReuseAndDrainFifo(path, 0660)));
     }
     confui_in_fd_ = fds[0];
     confui_out_fd_ = fds[1];

base/cvd/cuttlefish/host/commands/run_cvd/launch/uwb_connector.cpp

@@ -22,6 +22,7 @@
 #include <vector>
 
 #include "cuttlefish/common/libs/fs/shared_fd.h"
+#include "cuttlefish/common/libs/utils/files.h"
 #include "cuttlefish/common/libs/utils/subprocess.h"
 #include "cuttlefish/host/libs/config/config_utils.h"
 #include "cuttlefish/host/libs/config/cuttlefish_config.h"
@@ -39,7 +40,7 @@ namespace cuttlefish {
 Result<std::optional<MonitorCommand>> UwbConnector(
     const CuttlefishConfig& config,
     const CuttlefishConfig::InstanceSpecific& instance) {
-  if (!instance.enable_host_uwb_connector()) {
+  if (!config.enable_host_uwb()) {
     return {};
   }
   std::vector<std::string> fifo_paths = {
@@ -48,7 +49,10 @@ Result<std::optional<MonitorCommand>> UwbConnector(
   };
   std::vector<SharedFD> fifos;
   for (const auto& path : fifo_paths) {
-    fifos.push_back(CF_EXPECT(SharedFD::Fifo(path, 0660)));
+    fifos.push_back(CF_EXPECT(CreateOrReuseAndDrainFifo(path, 0660)));
+  }
+  if (!instance.enable_host_uwb_connector()) {
+    return {};
   }
   return Command(HostBinaryPath("tcp_connector"))
       .AddParameter("-fifo_out=", fifos[0])

base/cvd/cuttlefish/host/commands/run_cvd/server_loop_impl.cpp

@@ -337,12 +337,6 @@ void ServerLoopImpl::DeleteFifos() {
       instance_.PerInstanceInternalPath("gatekeeper_fifo_vm.out"),
       instance_.PerInstanceInternalPath("oemlock_fifo_vm.in"),
       instance_.PerInstanceInternalPath("oemlock_fifo_vm.out"),
-      instance_.PerInstanceInternalPath("bt_fifo_vm.in"),
-      instance_.PerInstanceInternalPath("bt_fifo_vm.out"),
-      instance_.PerInstanceInternalPath("nfc_fifo_vm.in"),
-      instance_.PerInstanceInternalPath("nfc_fifo_vm.out"),
-      instance_.PerInstanceInternalPath("uwb_fifo_vm.in"),
-      instance_.PerInstanceInternalPath("uwb_fifo_vm.out"),
       instance_.PerInstanceInternalPath("gnsshvc_fifo_vm.in"),
       instance_.PerInstanceInternalPath("gnsshvc_fifo_vm.out"),
       instance_.PerInstanceInternalPath("locationhvc_fifo_vm.in"),

base/cvd/cuttlefish/host/libs/process_monitor/process_monitor.cc

@@ -278,7 +278,15 @@ Result<void> ProcessMonitor::StartSubprocesses(
 Result<void> ProcessMonitor::ReadMonitorSocketLoop(std::atomic_bool& running) {
   VLOG(0) << "Waiting for a `stop` message from the parent";
   while (running.load()) {
-    ManagedMessage message = CF_EXPECT(child_channel_->ReceiveMessage());
+    auto message_res = child_channel_->ReceiveMessage();
+    if (!message_res.ok()) {
+      if (!running.load()) {
+        // We were shut down intentionally, ignore error
+        return {};
+      }
+      CF_EXPECT(std::move(message_res));
+    }
+    auto message = std::move(*message_res);
     if (message->command == ParentToChildMessageType::kStop) {
       running.store(false);
       // Wake up the wait() loop by giving it an exited child process
@@ -406,6 +414,7 @@ Result<void> ProcessMonitor::StartAndMonitorProcesses() {
   if (monitor_ == 0) {
     pipe_read->Close();
     child_channel_ = transport::SharedFdChannel(child_sock, child_sock);
+    child_sock_ = std::move(child_sock);
     Result<void> monitor_result = MonitorRoutine();
     if (!monitor_result.ok()) {
       LOG(ERROR) << "Monitoring processes failed:\n" << monitor_result.error();
@@ -444,6 +453,9 @@ Result<void> ProcessMonitor::MonitorRoutine() {
   CF_EXPECT(MonitorLoop(running, properties_mutex_,
                         properties_.restart_subprocesses_,
                         properties_.entries_));
+  if (child_sock_->IsOpen()) {
+    child_sock_->Shutdown(SHUT_RDWR);
+  }
   CF_EXPECT(parent_comms.get(), "Should have exited if monitoring stopped");
 
   CF_EXPECT(StopSubprocesses(properties_.entries_));

base/cvd/cuttlefish/host/libs/process_monitor/process_monitor.h

@@ -94,6 +94,7 @@ class ProcessMonitor {
   pid_t monitor_;
   std::optional<transport::SharedFdChannel> parent_channel_;
   std::optional<transport::SharedFdChannel> child_channel_;
+  SharedFD child_sock_;
 
   /*
    * The lock that should be acquired when multiple threads