diff --git a/config/config.go b/config/config.go index cf0cd38d..d4cf3fee 100644 --- a/config/config.go +++ b/config/config.go @@ -297,6 +297,17 @@ type Transfers struct { // // Defaults to 0 (unlimited) DownloadLimit int `default:"0" yaml:"download_limit"` + + // StoragePool configures whether this node participates in a shared storage pool. + StoragePool StoragePoolConfiguration `yaml:"storage_pool"` +} + +type StoragePoolConfiguration struct { + // Enabled signals that this node shares a common data volume with other nodes. + Enabled bool `default:"false" yaml:"enabled"` + + // PoolName is a per-node identifier used to compare shared storage pool membership across nodes. + PoolName string `yaml:"pool_name"` } type ConsoleThrottles struct { diff --git a/router/router_server.go b/router/router_server.go index 19a1cb75..67cd03b8 100644 --- a/router/router_server.go +++ b/router/router_server.go @@ -275,15 +275,25 @@ func deleteServer(c *gin.Context) { // // In addition, servers with large amounts of files can take some time to finish deleting, // so we don't want to block the HTTP call while waiting on this. - go func(s *server.Server) { - fs := s.Filesystem() - p := fs.Path() - _ = fs.UnixFS().Close() - if err := os.RemoveAll(p); err != nil { - log.WithFields(log.Fields{"path": p, "error": err}). - Warn("failed to remove server files during deletion process") - } - }(s) + // + // Only skip file removal when: + // 1. shared storage pooling is explicitly enabled, + // 2. the pool has a name configured, and + // 3. this server is actively being transferred. + // + // This avoids preserving data for ordinary server deletions. + pool := config.Get().System.Transfers.StoragePool + skipFileRemoval := pool.Enabled && pool.PoolName != "" && s.IsTransferring() + if !skipFileRemoval { + go func(s *server.Server) { + fs := s.Filesystem() + p := fs.Path() + _ = fs.UnixFS().Close() + if err := os.RemoveAll(p); err != nil { + log.WithFields(log.Fields{"path": p, "error": err}).Warn("failed to remove server files during deletion process") + } + }(s) + } // remove hanging machine-id file for the server when removing go func(s *server.Server) { diff --git a/router/router_transfer.go b/router/router_transfer.go index f49ef709..ae6d3c7f 100644 --- a/router/router_transfer.go +++ b/router/router_transfer.go @@ -129,6 +129,20 @@ func postTransfers(c *gin.Context) { trnsfr.Server.Events().Publish(server.TransferStatusEvent, "success") }(ctx, trnsfr) + { + remotePool := config.Get().System.Transfers.StoragePool + sourcePool := c.GetHeader("X-Storage-Pool") + if remotePool.Enabled && remotePool.PoolName != "" && sourcePool != "" && strings.EqualFold(remotePool.PoolName, sourcePool) { + if err := trnsfr.Server.CreateEnvironment(); err != nil { + middleware.CaptureAndAbort(c, err) + return + } + successful = true + c.Status(http.StatusOK) + return + } + } + mediaType, params, err := mime.ParseMediaType(c.GetHeader("Content-Type")) if err != nil { trnsfr.Log().Debug("failed to parse content type header") diff --git a/server/transfer/source.go b/server/transfer/source.go index 9e6fef05..c8ffeded 100644 --- a/server/transfer/source.go +++ b/server/transfer/source.go @@ -10,6 +10,8 @@ import ( "mime/multipart" "net/http" "time" + + "github.com/pelican-dev/wings/config" ) // PushArchiveToTarget POSTs the archive to the target node and returns the @@ -21,6 +23,10 @@ func (t *Transfer) PushArchiveToTarget(url, token string, backups []string) ([]b t.SendMessage("Preparing to stream server data to destination...") t.SetStatus(StatusProcessing) + // Always include the configured storage pool identifier in the outgoing request headers. + // The destination can use this information to determine if it should skip copying files when both nodes share the same storage backend. + sp := config.Get().System.Transfers.StoragePool + a, err := t.Archive() if err != nil { t.Error(err, "Failed to get archive for transfer.") @@ -61,6 +67,9 @@ func (t *Transfer) PushArchiveToTarget(url, token string, backups []string) ([]b return nil, err } req.Header.Set("Authorization", token) + if sp.Enabled && sp.PoolName != "" { + req.Header.Set("X-Storage-Pool", sp.PoolName) + } // Create a new multipart writer that writes the archive to the pipe. mp := multipart.NewWriter(writer)