Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
865 changes: 865 additions & 0 deletions shortcuts/drive/drive_duplicate_remote_test.go

Large diffs are not rendered by default.

104 changes: 90 additions & 14 deletions shortcuts/drive/drive_pull.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,10 +28,17 @@
type drivePullItem struct {
RelPath string `json:"rel_path"`
FileToken string `json:"file_token,omitempty"`
SourceID string `json:"source_id,omitempty"`
Action string `json:"action"`
Error string `json:"error,omitempty"`
}

type drivePullTarget struct {
DownloadToken string
ItemFileToken string
ItemSourceID string
}

// DrivePull performs a one-way file-level mirror from a Drive folder onto
// a local directory: recursively lists --folder-token, downloads each
// type=file entry under --local-dir, and optionally deletes local files
Expand All @@ -54,12 +61,14 @@
{Name: "local-dir", Desc: "local root directory (relative to cwd)", Required: true},
{Name: "folder-token", Desc: "source Drive folder token", Required: true},
{Name: "if-exists", Desc: "policy when a local file already exists", Default: drivePullIfExistsOverwrite, Enum: []string{drivePullIfExistsOverwrite, drivePullIfExistsSkip}},
{Name: "on-duplicate-remote", Desc: "policy when multiple remote Drive entries map to the same rel_path", Default: driveDuplicateRemoteFail, Enum: []string{driveDuplicateRemoteFail, driveDuplicateRemoteRename, driveDuplicateRemoteNewest, driveDuplicateRemoteOldest}},
{Name: "delete-local", Type: "bool", Desc: "delete local regular files absent from Drive (file-level mirror; empty directories are NOT pruned); requires --yes"},
{Name: "yes", Type: "bool", Desc: "confirm --delete-local before deleting local files"},
},
Tips: []string{
"Only entries with type=file are downloaded; online docs (docx, sheet, bitable, mindnote, slides) and shortcuts are skipped.",
"Subfolders recurse and are reproduced as local directories under --local-dir; missing parents are created automatically.",
"Duplicate remote rel_path conflicts fail by default. Use --on-duplicate-remote=rename to download duplicate files with stable hashed suffixes.",
"--delete-local requires --yes; without --yes the command is rejected upfront so a stray flag never deletes anything.",
},
Validate: func(ctx context.Context, runtime *common.RuntimeContext) error {
Expand Down Expand Up @@ -102,6 +111,10 @@
if ifExists == "" {
ifExists = drivePullIfExistsOverwrite
}
duplicateRemote := strings.TrimSpace(runtime.Str("on-duplicate-remote"))
if duplicateRemote == "" {
duplicateRemote = driveDuplicateRemoteFail

Check warning on line 116 in shortcuts/drive/drive_pull.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_pull.go#L116

Added line #L116 was not covered by tests
}
deleteLocal := runtime.Bool("delete-local")

// Resolve --local-dir to its canonical absolute path before we
Expand Down Expand Up @@ -132,24 +145,23 @@
}

fmt.Fprintf(runtime.IO().ErrOut, "Listing Drive folder: %s\n", common.MaskToken(folderToken))
entries, err := listRemoteFolder(ctx, runtime, folderToken, "")
entries, err := listRemoteFolderEntries(ctx, runtime, folderToken, "")
if err != nil {
return err
}
if duplicates := blockingRemotePathConflicts(entries, duplicateRemote); len(duplicates) > 0 {
return duplicateRemotePathError(duplicates)
}
// Two views over the same listing:
// - remoteFiles drives the download/skip loop (only type=file
// has hashable bytes the local mirror can write back).
// - remotePaths is the --delete-local guard: it carries every
// rel_path Drive owns regardless of type, so a local file
// shadowed by a remote folder / online doc / shortcut is NOT
// treated as orphaned.
remoteFiles := make(map[string]string, len(entries))
remotePaths := make(map[string]struct{}, len(entries))
for rel, entry := range entries {
remotePaths[rel] = struct{}{}
if entry.Type == driveTypeFile {
remoteFiles[rel] = entry.FileToken
}
remoteFiles, remotePaths, err := drivePullRemoteViews(entries, duplicateRemote)
if err != nil {
return output.Errorf(output.ExitInternal, "internal", "%s", err)

Check warning on line 164 in shortcuts/drive/drive_pull.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_pull.go#L164

Added line #L164 was not covered by tests
}

var downloaded, skipped, failed, deletedLocal int
Expand All @@ -164,7 +176,10 @@
sort.Strings(downloadablePaths)

for _, rel := range downloadablePaths {
token := remoteFiles[rel]
targetFile := remoteFiles[rel]
downloadToken := targetFile.DownloadToken
itemFileToken := targetFile.ItemFileToken
itemSourceID := targetFile.ItemSourceID
target := filepath.Join(rootRelToCwd, rel)

if info, statErr := runtime.FileIO().Stat(target); statErr == nil {
Expand All @@ -178,7 +193,8 @@
if info.IsDir() {
items = append(items, drivePullItem{
RelPath: rel,
FileToken: token,
FileToken: itemFileToken,
SourceID: itemSourceID,
Action: "failed",
Error: fmt.Sprintf("local path is a directory, remote is a regular file: %s", target),
})
Expand All @@ -187,19 +203,19 @@
continue
}
if ifExists == drivePullIfExistsSkip {
items = append(items, drivePullItem{RelPath: rel, FileToken: token, Action: "skipped"})
items = append(items, drivePullItem{RelPath: rel, FileToken: itemFileToken, SourceID: itemSourceID, Action: "skipped"})
skipped++
continue
}
}

if err := drivePullDownload(ctx, runtime, token, target); err != nil {
items = append(items, drivePullItem{RelPath: rel, FileToken: token, Action: "failed", Error: err.Error()})
if err := drivePullDownload(ctx, runtime, downloadToken, target); err != nil {
items = append(items, drivePullItem{RelPath: rel, FileToken: itemFileToken, SourceID: itemSourceID, Action: "failed", Error: err.Error()})
failed++
downloadFailed++
continue
}
items = append(items, drivePullItem{RelPath: rel, FileToken: token, Action: "downloaded"})
items = append(items, drivePullItem{RelPath: rel, FileToken: itemFileToken, SourceID: itemSourceID, Action: "downloaded"})
downloaded++
}

Expand Down Expand Up @@ -307,6 +323,66 @@
return nil
}

func drivePullRemoteViews(entries []driveRemoteEntry, duplicateRemote string) (map[string]drivePullTarget, map[string]struct{}, error) {
remoteFiles := make(map[string]drivePullTarget, len(entries))
remotePaths := make(map[string]struct{}, len(entries))
fileGroups := make(map[string][]driveRemoteEntry)
occupied := occupiedRemotePaths(entries)

for _, entry := range entries {
if entry.Type == driveTypeFile {
fileGroups[entry.RelPath] = append(fileGroups[entry.RelPath], entry)
continue
}
remotePaths[entry.RelPath] = struct{}{}
}

relPaths := make([]string, 0, len(fileGroups))
for rel := range fileGroups {
relPaths = append(relPaths, rel)
}
sort.Strings(relPaths)

for _, rel := range relPaths {
files := fileGroups[rel]
if len(files) == 1 {
remoteFiles[rel] = drivePullTarget{DownloadToken: files[0].FileToken, ItemFileToken: files[0].FileToken}
remotePaths[rel] = struct{}{}
continue
}
switch duplicateRemote {
case driveDuplicateRemoteRename:
candidates := append([]driveRemoteEntry(nil), files...)
sortRemoteFiles(candidates, driveDuplicateRemoteOldest)
for idx, file := range candidates {
targetRel := rel
if idx > 0 {
var err error
targetRel, err = relPathWithUniqueFileTokenSuffix(rel, file.FileToken, occupied)
if err != nil {
return nil, nil, err

Check warning on line 363 in shortcuts/drive/drive_pull.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_pull.go#L363

Added line #L363 was not covered by tests
}
}
remoteFiles[targetRel] = drivePullTarget{
DownloadToken: file.FileToken,
ItemSourceID: stableTokenIdentifier(file.FileToken),
}
remotePaths[targetRel] = struct{}{}
}
case driveDuplicateRemoteNewest, driveDuplicateRemoteOldest:
chosen, err := chooseRemoteFile(files, duplicateRemote)
if err != nil {
return nil, nil, err

Check warning on line 375 in shortcuts/drive/drive_pull.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_pull.go#L375

Added line #L375 was not covered by tests
}
remoteFiles[rel] = drivePullTarget{DownloadToken: chosen.FileToken, ItemFileToken: chosen.FileToken}
remotePaths[rel] = struct{}{}
default:
return nil, nil, fmt.Errorf("unsupported duplicate remote strategy %q", duplicateRemote)
}
}
return remoteFiles, remotePaths, nil
}

// drivePullWalkLocal walks the canonical absolute root and returns the
// absolute paths of every regular file underneath it. The caller deletes
// some of these paths, so it is critical that they are produced by
Expand Down
43 changes: 43 additions & 0 deletions shortcuts/drive/drive_pull_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -293,6 +293,49 @@ func TestDrivePullPaginationHandlesPageTokenField(t *testing.T) {
reg.Verify(t)
}

func TestDrivePullRenameSummarizesDuplicateDownloadsAndAvoidsRawTokenInRelPath(t *testing.T) {
f, stdout, _, reg := cmdutil.TestFactory(t, driveTestConfig())

tmpDir := t.TempDir()
withDriveWorkingDir(t, tmpDir)
if err := os.MkdirAll("local", 0o755); err != nil {
t.Fatalf("MkdirAll: %v", err)
}

registerDuplicateRemoteFiles(reg)
registerDownload(reg, duplicateRemoteFileIDFirst, "FIRST")
registerDownload(reg, duplicateRemoteFileIDSecond, "SECOND")

err := mountAndRunDrive(t, DrivePull, []string{
"+pull",
"--local-dir", "local",
"--folder-token", "folder_root",
"--on-duplicate-remote", "rename",
"--as", "bot",
}, f, stdout)
if err != nil {
t.Fatalf("unexpected error: %v\nstdout: %s", err, stdout.String())
}

renamedRelPath := expectedRenamedRelPath("dup.txt", duplicateRemoteFileIDSecond, 12, 0)
payload := decodeDrivePullStdout(t, stdout.Bytes())
if got := payload.Data.Summary.Downloaded; got != 2 {
t.Fatalf("summary.downloaded = %d, want 2", got)
}
if out := stdout.String(); strings.Contains(out, duplicateRemoteFileIDSecond) {
t.Fatalf("stdout should not expose the raw duplicate file token in rename mode, got: %s", out)
}
if item := findPullItem(payload.Data.Items, renamedRelPath); item.SourceID == "" || item.FileToken != "" {
t.Fatalf("rename item should emit source_id without file_token, got: %#v", item)
}
mustReadFile(t, filepath.Join("local", "dup.txt"), "FIRST")
mustReadFile(t, filepath.Join("local", renamedRelPath), "SECOND")
assertPullItemAction(t, stdout.Bytes(), "dup.txt", "downloaded")
assertPullItemAction(t, stdout.Bytes(), renamedRelPath, "downloaded")

reg.Verify(t)
}

// TestDrivePullDeleteLocalRequiresYes verifies the upfront safety guard:
// --delete-local without --yes must be rejected before any API call.
func TestDrivePullDeleteLocalRequiresYes(t *testing.T) {
Expand Down
90 changes: 70 additions & 20 deletions shortcuts/drive/drive_push.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,12 +92,14 @@
{Name: "local-dir", Desc: "local root directory (relative to cwd)", Required: true},
{Name: "folder-token", Desc: "target Drive folder token", Required: true},
{Name: "if-exists", Desc: "policy when a Drive file already exists at the same rel_path (default: skip — safe; opt into overwrite explicitly while the backend version field is rolling out)", Default: drivePushIfExistsSkip, Enum: []string{drivePushIfExistsOverwrite, drivePushIfExistsSkip}},
{Name: "on-duplicate-remote", Desc: "policy when multiple remote Drive entries map to the same rel_path", Default: driveDuplicateRemoteFail, Enum: []string{driveDuplicateRemoteFail, driveDuplicateRemoteNewest, driveDuplicateRemoteOldest}},
{Name: "delete-remote", Type: "bool", Desc: "delete Drive files absent locally (file-level mirror; remote-only directories are not removed); requires --yes"},
{Name: "yes", Type: "bool", Desc: "confirm --delete-remote before deleting Drive files"},
},
Tips: []string{
"This is a file-level mirror: only type=file entries are uploaded, overwritten or deleted. Online docs (docx, sheet, bitable, mindnote, slides), shortcuts, and remote-only directories are never touched.",
"Local directory structure (including empty directories) is mirrored to Drive via create_folder; existing remote folders are reused.",
"Duplicate remote rel_path conflicts fail by default before upload, overwrite, or delete. Use --on-duplicate-remote=newest|oldest only when the conflict is duplicate files and you explicitly want to target one.",
"Default --if-exists=skip is the safe choice while the upload_all overwrite-version field is rolling out. Pass --if-exists=overwrite to replace remote bytes; on tenants without the field it surfaces a structured api_error and the run exits non-zero.",
"--delete-remote requires --yes; without --yes the command is rejected upfront so a stray flag never deletes anything.",
"--delete-remote --yes also requires the space:document:delete scope. Validate runs a dynamic pre-flight check when the flag is on, so a missing grant fails the run before any upload — preventing a half-synced state where files were uploaded but the cleanup pass cannot delete.",
Expand Down Expand Up @@ -164,6 +166,10 @@
// rolling-out upload_all `file_token`/`version` protocol field.
ifExists = drivePushIfExistsSkip
}
duplicateRemote := strings.TrimSpace(runtime.Str("on-duplicate-remote"))
if duplicateRemote == "" {
duplicateRemote = driveDuplicateRemoteFail

Check warning on line 171 in shortcuts/drive/drive_push.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_push.go#L171

Added line #L171 was not covered by tests
}
deleteRemote := runtime.Bool("delete-remote")

// Resolve --local-dir to its canonical absolute path before walking.
Expand All @@ -190,10 +196,13 @@
}

fmt.Fprintf(runtime.IO().ErrOut, "Listing Drive folder: %s\n", common.MaskToken(folderToken))
entries, err := listRemoteFolder(ctx, runtime, folderToken, "")
entries, err := listRemoteFolderEntries(ctx, runtime, folderToken, "")
if err != nil {
return err
}
if duplicates := blockingRemotePathConflicts(entries, duplicateRemote); len(duplicates) > 0 {
return duplicateRemotePathError(duplicates)
}
// Two views over the same listing:
// - remoteFiles drives upload / overwrite / orphan-delete
// decisions (only type=file entries are upload candidates;
Expand All @@ -203,15 +212,9 @@
// path skip create_folder when an intermediate folder already
// exists, and keeps directory recreation idempotent across
// reruns.
remoteFiles := make(map[string]driveRemoteEntry, len(entries))
remoteFolders := make(map[string]driveRemoteEntry, len(entries))
for rel, entry := range entries {
switch entry.Type {
case driveTypeFile:
remoteFiles[rel] = entry
case driveTypeFolder:
remoteFolders[rel] = entry
}
remoteFiles, remoteFolders, remoteFileGroups, err := drivePushRemoteViews(entries, duplicateRemote)
if err != nil {
return output.Errorf(output.ExitInternal, "internal", "%s", err)

Check warning on line 217 in shortcuts/drive/drive_push.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_push.go#L217

Added line #L217 was not covered by tests
}

var uploaded, skipped, failed, deletedRemote int
Expand Down Expand Up @@ -333,24 +336,31 @@
}
if deleteRemote && !uploadFailed {
// Stable iteration order so failures (and tests) are deterministic.
remoteRelPaths := make([]string, 0, len(remoteFiles))
for p := range remoteFiles {
remoteRelPaths := make([]string, 0, len(remoteFileGroups))
for p := range remoteFileGroups {
remoteRelPaths = append(remoteRelPaths, p)
}
sort.Strings(remoteRelPaths)

for _, rel := range remoteRelPaths {
keepToken := ""
if _, ok := localFiles[rel]; ok {
continue
if chosen, ok := remoteFiles[rel]; ok {
keepToken = chosen.FileToken
}
}
entry := remoteFiles[rel]
if err := drivePushDeleteFile(ctx, runtime, entry.FileToken); err != nil {
items = append(items, drivePushItem{RelPath: rel, FileToken: entry.FileToken, Action: "delete_failed", Error: err.Error()})
failed++
continue
for _, entry := range remoteFileGroups[rel] {
if entry.FileToken == keepToken {
continue
}
if err := drivePushDeleteFile(ctx, runtime, entry.FileToken); err != nil {
items = append(items, drivePushItem{RelPath: rel, FileToken: entry.FileToken, Action: "delete_failed", Error: err.Error()})
failed++
continue

Check warning on line 359 in shortcuts/drive/drive_push.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_push.go#L357-L359

Added lines #L357 - L359 were not covered by tests
}
items = append(items, drivePushItem{RelPath: rel, FileToken: entry.FileToken, Action: "deleted_remote"})
deletedRemote++
}
items = append(items, drivePushItem{RelPath: rel, FileToken: entry.FileToken, Action: "deleted_remote"})
deletedRemote++
}
}

Expand Down Expand Up @@ -463,6 +473,46 @@
return files, dirs, nil
}

func drivePushRemoteViews(entries []driveRemoteEntry, duplicateRemote string) (map[string]driveRemoteEntry, map[string]driveRemoteEntry, map[string][]driveRemoteEntry, error) {
remoteFiles := make(map[string]driveRemoteEntry, len(entries))
remoteFolders := make(map[string]driveRemoteEntry, len(entries))
fileGroups := make(map[string][]driveRemoteEntry)

for _, entry := range entries {
switch entry.Type {
case driveTypeFile:
fileGroups[entry.RelPath] = append(fileGroups[entry.RelPath], entry)
case driveTypeFolder:
remoteFolders[entry.RelPath] = entry
}
}

relPaths := make([]string, 0, len(fileGroups))
for rel := range fileGroups {
relPaths = append(relPaths, rel)
}
sort.Strings(relPaths)

for _, rel := range relPaths {
files := fileGroups[rel]
if len(files) == 1 {
remoteFiles[rel] = files[0]
continue
}
switch duplicateRemote {
case driveDuplicateRemoteNewest, driveDuplicateRemoteOldest:
chosen, err := chooseRemoteFile(files, duplicateRemote)
if err != nil {
return nil, nil, nil, err

Check warning on line 506 in shortcuts/drive/drive_push.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_push.go#L506

Added line #L506 was not covered by tests
}
remoteFiles[rel] = chosen
default:
return nil, nil, nil, fmt.Errorf("unsupported duplicate remote strategy %q", duplicateRemote)

Check warning on line 510 in shortcuts/drive/drive_push.go

View check run for this annotation

Codecov / codecov/patch

shortcuts/drive/drive_push.go#L509-L510

Added lines #L509 - L510 were not covered by tests
}
}
return remoteFiles, remoteFolders, fileGroups, nil
}

// drivePushEnsureFolder ensures a folder chain (rel_dir relative to the root
// folder identified by rootFolderToken) exists on Drive, creating any
// missing segments via /open-apis/drive/v1/files/create_folder. Returns the
Expand Down
Loading
Loading