Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
456 changes: 392 additions & 64 deletions README.md

Large diffs are not rendered by default.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"author": "Internxt <hello@internxt.com>",
"version": "1.6.2",
"version": "1.6.3",
"description": "Internxt CLI to manage your encrypted storage",
"scripts": {
"build": "yarn clean && tsc",
Expand Down
11 changes: 9 additions & 2 deletions src/commands/create-folder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import { ConfigService } from '../services/config.service';
import { ValidationService } from '../services/validation.service';
import { EmptyFolderNameError, MissingCredentialsError, NotValidFolderUuidError } from '../types/command.types';
import { AsyncUtils } from '../utils/async.utils';
import { AuthService } from '../services/auth.service';

export default class CreateFolder extends Command {
static readonly args = {};
Expand Down Expand Up @@ -56,8 +57,14 @@ export default class CreateFolder extends Command {
await AsyncUtils.sleep(500);

CLIUtils.done(flags['json']);
// eslint-disable-next-line max-len
const message = `Folder ${newFolder.plainName} created successfully, view it at ${ConfigService.instance.get('DRIVE_WEB_URL')}/folder/${newFolder.uuid}`;

const workspace = await AuthService.instance.getCurrentWorkspace();
const workspaceId = workspace?.workspaceData.workspace.id;

const message =
`Folder ${newFolder.plainName} created successfully, view it at ` +
`${ConfigService.instance.get('DRIVE_WEB_URL')}/folder/${newFolder.uuid}` +
`${workspaceId ? `?workspaceid=${workspaceId}` : ''}`;
CLIUtils.success(this.log.bind(this), message);
return { success: true, message, folder: newFolder };
};
Expand Down
31 changes: 20 additions & 11 deletions src/commands/upload-file.ts
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ import { BufferStream } from '../utils/stream.utils';
import { Readable } from 'node:stream';
import { ThumbnailUtils } from '../utils/thumbnail.utils';
import { ThumbnailService } from '../services/thumbnail.service';
import { AuthService } from '../services/auth.service';

export default class UploadFile extends Command {
static readonly args = {};
Expand Down Expand Up @@ -49,11 +50,13 @@ export default class UploadFile extends Command {
const fileInfo = path.parse(filePath);
const fileType = fileInfo.ext.replaceAll('.', '');

const reporter = this.log.bind(this);

const destinationFolderUuidFromFlag = await CLIUtils.getDestinationFolderUuid({
destinationFolderUuidFlag: flags['destination'],
destinationFlagName: UploadFile.flags['destination'].name,
nonInteractive,
reporter: this.log.bind(this),
reporter,
});
const destinationFolderUuid = await CLIUtils.fallbackToRootFolderIdIfEmpty(destinationFolderUuidFromFlag);

Expand Down Expand Up @@ -141,6 +144,7 @@ export default class UploadFile extends Command {
bucket,
fileUuid: createdDriveFile.uuid,
networkFacade,
size: fileSize,
});
}
timings.thumbnailUpload = thumbnailTimer.stop();
Expand All @@ -151,18 +155,23 @@ export default class UploadFile extends Command {
const totalTime = Object.values(timings).reduce((sum, time) => sum + time, 0);
const throughputMBps = CLIUtils.calculateThroughputMBps(stats.size, timings.networkUpload);

this.log('\n');
this.log(
'[PUT] Timing breakdown:\n' +
`Network upload: ${CLIUtils.formatDuration(timings.networkUpload)} (${throughputMBps.toFixed(2)} MB/s)\n` +
`Drive upload: ${CLIUtils.formatDuration(timings.driveUpload)}\n` +
`Thumbnail: ${CLIUtils.formatDuration(timings.thumbnailUpload)}\n`,
);
this.log('\n');
if (flags['debug']) {
CLIUtils.log(
reporter,
'[PUT] Timing breakdown:\n' +
`Network upload: ${CLIUtils.formatDuration(timings.networkUpload)} (${throughputMBps.toFixed(2)} MB/s)\n` +
`Drive upload: ${CLIUtils.formatDuration(timings.driveUpload)}\n` +
`Thumbnail: ${CLIUtils.formatDuration(timings.thumbnailUpload)}\n`,
);
}
const workspace = await AuthService.instance.getCurrentWorkspace();
const workspaceId = workspace?.workspaceData.workspace.id;

const message =
`File uploaded successfully in ${CLIUtils.formatDuration(totalTime)}, view it at ` +
`${ConfigService.instance.get('DRIVE_WEB_URL')}/file/${createdDriveFile.uuid}`;
CLIUtils.success(this.log.bind(this), message);
`${ConfigService.instance.get('DRIVE_WEB_URL')}/file/${createdDriveFile.uuid}` +
`${workspaceId ? `?workspaceid=${workspaceId}` : ''}`;
CLIUtils.success(reporter, message);
return {
success: true,
message,
Expand Down
6 changes: 5 additions & 1 deletion src/commands/upload-folder.ts
Original file line number Diff line number Diff line change
Expand Up @@ -34,11 +34,13 @@ export default class UploadFolder extends Command {

const localPath = await this.getFolderPath(flags['folder'], flags['non-interactive']);

const reporter = this.log.bind(this);

const destinationFolderUuidFromFlag = await CLIUtils.getDestinationFolderUuid({
destinationFolderUuidFlag: flags['destination'],
destinationFlagName: UploadFolder.flags['destination'].name,
nonInteractive: flags['non-interactive'],
reporter: this.log.bind(this),
reporter,
});
const destinationFolderUuid = await CLIUtils.fallbackToRootFolderIdIfEmpty(destinationFolderUuidFromFlag);

Expand All @@ -58,6 +60,8 @@ export default class UploadFolder extends Command {
onProgress: (progress) => {
progressBar?.update(progress.percentage);
},
debugMode: flags['debug'],
reporter,
});

progressBar?.update(100);
Expand Down
6 changes: 4 additions & 2 deletions src/commands/workspaces-unset.ts
Original file line number Diff line number Diff line change
@@ -1,8 +1,9 @@
import { Command } from '@oclif/core';
import { ConfigService } from '../services/config.service';
import { CLIUtils } from '../utils/cli.utils';
import { CLIUtils, LogReporter } from '../utils/cli.utils';
import { LoginCredentials, MissingCredentialsError } from '../types/command.types';
import { SdkManager } from '../services/sdk-manager.service';
import { DatabaseService } from '../services/database/database.service';

export default class WorkspacesUnset extends Command {
static readonly args = {};
Expand Down Expand Up @@ -37,9 +38,10 @@ export default class WorkspacesUnset extends Command {
this.exit(1);
};

static readonly unsetWorkspace = async (userCredentials: LoginCredentials, reporter: (message: string) => void) => {
static readonly unsetWorkspace = async (userCredentials: LoginCredentials, reporter: LogReporter) => {
SdkManager.init({ token: userCredentials.token });
await ConfigService.instance.saveUser({ ...userCredentials, workspace: undefined });
void DatabaseService.instance.clear();
CLIUtils.success(reporter, 'Personal drive space selected successfully.');
return { success: true, message: 'Personal drive space selected successfully.' };
};
Expand Down
3 changes: 3 additions & 0 deletions src/commands/workspaces-use.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import { FormatUtils } from '../utils/format.utils';
import { ValidationService } from '../services/validation.service';
import { SdkManager } from '../services/sdk-manager.service';
import WorkspacesUnset from './workspaces-unset';
import { DatabaseService } from '../services/database/database.service';

export default class WorkspacesUse extends Command {
static readonly args = {};
Expand Down Expand Up @@ -74,6 +75,8 @@ export default class WorkspacesUse extends Command {
},
});

void DatabaseService.instance.clear();

const message =
`Workspace ${workspaceUuid} selected successfully. Now WebDAV and all of the CLI commands ` +
'will operate within this workspace until it is changed or unset.';
Expand Down
16 changes: 12 additions & 4 deletions src/services/network/upload/upload-facade.service.ts
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
import { basename } from 'node:path';
import { CLIUtils } from '../../../utils/cli.utils';
import { logger } from '../../../utils/logger.utils';
import { LocalFilesystemService } from '../../local-filesystem/local-filesystem.service';
import { UploadFolderParams } from './upload.types';
import { UploadFolderService } from './upload-folder.service';
Expand All @@ -15,16 +14,21 @@ export class UploadFacade {
destinationFolderUuid,
loginUserDetails,
jsonFlag,
debugMode,
onProgress,
reporter,
}: UploadFolderParams) => {
const timer = CLIUtils.timer();
CLIUtils.doing('Preparing Network', jsonFlag);
const { networkFacade, bucket } = await CLIUtils.prepareNetwork(loginUserDetails);
CLIUtils.done(jsonFlag);
const scanResult = await LocalFilesystemService.instance.scanLocalDirectory(localPath);
logger.info(
`Scanned folder ${localPath}: found ${scanResult.totalItems} items, total size ${scanResult.totalBytes} bytes.`,
);
if (debugMode) {
CLIUtils.success(
reporter,
`Scanned folder ${localPath}: found ${scanResult.totalItems} items, total size ${scanResult.totalBytes} bytes.`,
);
}

const currentProgress = { itemsUploaded: 0, bytesUploaded: 0 };
const emitProgress = () => {
Expand All @@ -39,6 +43,8 @@ export class UploadFacade {
destinationFolderUuid,
currentProgress,
emitProgress,
reporter,
debugMode,
});

if (folderMap.size === 0) {
Expand All @@ -55,6 +61,8 @@ export class UploadFacade {
destinationFolderUuid,
currentProgress,
emitProgress,
debugMode,
reporter,
});

const rootFolderName = basename(localPath);
Expand Down
43 changes: 29 additions & 14 deletions src/services/network/upload/upload-file.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,8 @@ export class UploadFileService {
destinationFolderUuid,
currentProgress,
emitProgress,
debugMode,
reporter,
}: UploadFilesConcurrentlyParams): Promise<number> => {
let bytesUploaded = 0;

Expand All @@ -40,14 +42,18 @@ export class UploadFileService {
parentPath === '.' || parentPath === '' ? destinationFolderUuid : folderMap.get(parentPath);

if (!parentFolderUuid) {
logger.warn(`Parent folder not found for ${file.relativePath}, skipping...`);
if (debugMode) {
CLIUtils.warning(reporter, `Parent folder not found for ${file.relativePath}, skipping...`);
}
return null;
}
const createdFileUuid = await this.uploadFileWithRetry({
file,
network,
bucket,
parentFolderUuid,
debugMode,
reporter,
});
if (createdFileUuid) {
bytesUploaded += file.size;
Expand All @@ -66,6 +72,8 @@ export class UploadFileService {
network,
bucket,
parentFolderUuid,
debugMode,
reporter,
}: UploadFileWithRetryParams): Promise<DriveFileItem | null> => {
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
Expand Down Expand Up @@ -131,20 +139,25 @@ export class UploadFileService {
bucket,
fileUuid: createdDriveFile.uuid,
networkFacade: network,
size: fileSize,
});
}
timings.thumbnailUpload = thumbnailTimer.stop();

const totalTime = Object.values(timings).reduce((sum, time) => sum + time, 0);
const throughputMBps = CLIUtils.calculateThroughputMBps(stats.size, timings.networkUpload);
logger.info(`Uploaded '${file.name}' (${CLIUtils.formatBytesToString(stats.size)})`);
logger.info(
'Timing breakdown:\n' +
`Network upload: ${CLIUtils.formatDuration(timings.networkUpload)} (${throughputMBps.toFixed(2)} MB/s)\n` +
`Drive upload: ${CLIUtils.formatDuration(timings.driveUpload)}\n` +
`Thumbnail: ${CLIUtils.formatDuration(timings.thumbnailUpload)}\n` +
`Total: ${CLIUtils.formatDuration(totalTime)}\n`,
);
if (debugMode) {
const totalTime = Object.values(timings).reduce((sum, time) => sum + time, 0);
const throughputMBps = CLIUtils.calculateThroughputMBps(stats.size, timings.networkUpload);
CLIUtils.success(reporter, `Uploaded '${file.name}' (${CLIUtils.formatBytesToString(stats.size)})`);
CLIUtils.log(
reporter,
'Timing breakdown:\n' +
`Network upload: ${CLIUtils.formatDuration(timings.networkUpload)}` +
` (${throughputMBps.toFixed(2)} MB/s)\n` +
`Drive upload: ${CLIUtils.formatDuration(timings.driveUpload)}\n` +
`Thumbnail: ${CLIUtils.formatDuration(timings.thumbnailUpload)}\n` +
`Total: ${CLIUtils.formatDuration(totalTime)}\n`,
);
}

return createdDriveFile;
} catch (error: unknown) {
Expand All @@ -156,11 +169,13 @@ export class UploadFileService {

if (attempt < MAX_RETRIES) {
const delay = DELAYS_MS[attempt];
const retryMsg = `Failed to upload file ${file.name}, retrying in ${delay}ms...`;
logger.warn(`${retryMsg} (attempt ${attempt + 1}/${MAX_RETRIES + 1})`);
if (debugMode) {
const retryMsg = `Failed to upload file ${file.name}, retrying in ${delay}ms...`;
CLIUtils.warning(reporter, `${retryMsg} (attempt ${attempt + 1}/${MAX_RETRIES + 1})`);
}
await new Promise((resolve) => setTimeout(resolve, delay));
} else {
logger.error(`Failed to upload file ${file.name} after ${MAX_RETRIES + 1} attempts`);
CLIUtils.error(reporter, `Failed to upload file '${file.name}' after ${MAX_RETRIES + 1} attempts`);
return null;
}
}
Expand Down
23 changes: 17 additions & 6 deletions src/services/network/upload/upload-folder.service.ts
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ import { ErrorUtils } from '../../../utils/errors.utils';
import { logger } from '../../../utils/logger.utils';
import { DriveFolderService } from '../../drive/drive-folder.service';
import { CreateFoldersParams, CreateFolderWithRetryParams, DELAYS_MS, MAX_RETRIES } from './upload.types';
import { CLIUtils } from '../../../utils/cli.utils';

export class UploadFolderService {
static readonly instance = new UploadFolderService();
Expand All @@ -12,6 +13,8 @@ export class UploadFolderService {
destinationFolderUuid,
currentProgress,
emitProgress,
debugMode,
reporter,
}: CreateFoldersParams): Promise<Map<string, string>> => {
const folderMap = new Map<string, string>();
for (const folder of foldersToCreate) {
Expand All @@ -26,6 +29,8 @@ export class UploadFolderService {
const createdFolderUuid = await this.createFolderWithRetry({
folderName: folder.name,
parentFolderUuid: parentUuid,
debugMode,
reporter,
});

if (createdFolderUuid) {
Expand All @@ -40,6 +45,8 @@ export class UploadFolderService {
public createFolderWithRetry = async ({
folderName,
parentFolderUuid,
debugMode,
reporter,
}: CreateFolderWithRetryParams): Promise<string | null> => {
for (let attempt = 0; attempt <= MAX_RETRIES; attempt++) {
try {
Expand All @@ -52,18 +59,22 @@ export class UploadFolderService {
return createdFolder.uuid;
} catch (error: unknown) {
if (ErrorUtils.isAlreadyExistsError(error)) {
logger.info(`Folder ${folderName} already exists, skipping...`);
logger.warn(`Folder ${folderName} already exists, skipping...`);
return null;
}

if (attempt < MAX_RETRIES) {
const delay = DELAYS_MS[attempt];
logger.warn(
`Failed to create folder ${folderName},
retrying in ${delay}ms... (attempt ${attempt + 1}/${MAX_RETRIES + 1})`,
);
if (debugMode) {
CLIUtils.warning(
reporter,
`Failed to create folder '${folderName}', retrying in ${delay}ms... ` +
`(attempt ${attempt + 1}/${MAX_RETRIES + 1})`,
);
}
await new Promise((resolve) => setTimeout(resolve, delay));
} else {
logger.error(`Failed to create folder ${folderName} after ${MAX_RETRIES + 1} attempts`);
CLIUtils.error(reporter, `Failed to create folder '${folderName}' after ${MAX_RETRIES + 1} attempts`);
throw error;
}
}
Expand Down
Loading