Skip to content

fix(metadata-validation): complete SSRF protection #4213

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
kusssal wants to merge 1 commit into
base: develop
Choose a base branch
from
Open

fix(metadata-validation): complete SSRF protection #4213

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 14 additions & 4 deletions govtool/metadata-validation/package-lock.json
View file
Open in desktop

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

1 change: 1 addition & 0 deletions govtool/metadata-validation/package.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,7 @@
"blakejs": "^1.2.1",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.1",
"ipaddr.js": "^2.5.0",
"joi": "18.2.1",
"jsonld": "^9.0.0",
"reflect-metadata": "^0.2.0",
Expand Down
57 changes: 57 additions & 0 deletions govtool/metadata-validation/src/app.service.test.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -80,6 +80,8 @@ describe('AppService', () => {
expect.objectContaining({
httpAgent: expect.any(Object),
httpsAgent: expect.any(Object),
maxRedirects: 0,
proxy: false,
}),
);
});
Expand Down Expand Up @@ -154,6 +156,40 @@ describe('AppService', () => {
expect(httpService.get).not.toHaveBeenCalled();
});

it.each([
'http://[::]/metadata.json',
'http://[::1]/metadata.json',
'http://[fc00::1]/metadata.json',
'http://[fe90::1]/metadata.json',
'http://[febf::1]/metadata.json',
'http://[ff02::1]/metadata.json',
'http://[::ffff:7f00:1]/metadata.json',
])('should block special-use IPv6 URL %s before fetching', async (url) => {
const result = await service.validateMetadata({ hash: 'hash', url });

expect(result).toEqual({
status: MetadataValidationStatus.URL_BLOCKED,
valid: false,
metadata: undefined,
});
expect(httpService.get).not.toHaveBeenCalled();
});

it.each([
'http://192.0.2.1/metadata.json',
'http://198.51.100.1/metadata.json',
'http://203.0.113.1/metadata.json',
])('should block reserved IPv4 URL %s before fetching', async (url) => {
const result = await service.validateMetadata({ hash: 'hash', url });

expect(result).toEqual({
status: MetadataValidationStatus.URL_BLOCKED,
valid: false,
metadata: undefined,
});
expect(httpService.get).not.toHaveBeenCalled();
});

it('should block hostnames that resolve to private addresses', async () => {
(lookup as jest.Mock).mockResolvedValueOnce([{ address: '10.0.0.5' }]);

Expand Down Expand Up @@ -215,4 +251,25 @@ describe('AppService', () => {
}),
).rejects.toThrow(MetadataValidationStatus.URL_BLOCKED);
});

it('should preserve URL_BLOCKED from a connection-time lookup', async () => {
const blockedError = Object.assign(
new Error(MetadataValidationStatus.URL_BLOCKED),
{ code: MetadataValidationStatus.URL_BLOCKED },
);
jest
.spyOn(httpService, 'get')
.mockReturnValueOnce(throwError(() => blockedError));

const result = await service.validateMetadata({
hash: 'hash',
url: 'http://example.com',
});

expect(result).toEqual({
status: MetadataValidationStatus.URL_BLOCKED,
valid: false,
metadata: undefined,
});
});
});
91 changes: 42 additions & 49 deletions govtool/metadata-validation/src/app.service.ts
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,78 +3,68 @@ import { catchError, finalize, firstValueFrom } from 'rxjs';
import { HttpService } from '@nestjs/axios';
import * as blake from 'blakejs';
import { lookup } from 'node:dns/promises';
import { isIP, LookupFunction } from 'node:net';
import { LookupFunction } from 'node:net';
import { Agent as HttpAgent } from 'node:http';
import { Agent as HttpsAgent } from 'node:https';
import * as ipaddr from 'ipaddr.js';

import { ValidateMetadataDTO } from '@dto';
import { LoggerMessage, MetadataValidationStatus } from '@enums';
import { validateMetadataStandard, parseMetadata, getStandard } from '@utils';
import { /* MetadataStandard, */ ValidateMetadataResult } from '@types';

class UrlBlockedError extends Error {
readonly code = MetadataValidationStatus.URL_BLOCKED;

constructor() {
super(MetadataValidationStatus.URL_BLOCKED);
this.name = 'UrlBlockedError';
}
}

@Injectable()
export class AppService {
constructor(private readonly httpService: HttpService) {}

private readonly safeHttpAgent = new HttpAgent({
keepAlive: true,
lookup: this.createSafeLookup(),
});

private readonly safeHttpsAgent = new HttpsAgent({
keepAlive: true,
lookup: this.createSafeLookup(),
});

private isBlockedIPv4(address: string): boolean {
const parts = address.split('.').map(Number);
const [first, second] = parts;

return (
first === 0 ||
first === 10 ||
first === 127 ||
(first === 100 && second >= 64 && second <= 127) ||
(first === 169 && second === 254) ||
(first === 172 && second >= 16 && second <= 31) ||
(first === 192 && second === 0 && parts[2] === 0) ||
(first === 192 && second === 168) ||
(first === 198 && (second === 18 || second === 19)) ||
first >= 224
);
private stripIPv6Brackets(hostname: string): string {
return hostname.startsWith('[') && hostname.endsWith(']')
? hostname.slice(1, -1)
: hostname;
}

private isBlockedIPv6(address: string): boolean {
const normalized = address.toLowerCase();
const ipv4MappedPrefix = '::ffff:';

if (normalized.startsWith(ipv4MappedPrefix)) {
const mappedAddress = normalized.slice(ipv4MappedPrefix.length);
if (isIP(mappedAddress) === 4) {
return this.isBlockedIPv4(mappedAddress);
}
private isBlockedAddress(address: string): boolean {
const normalized = this.stripIPv6Brackets(address);
if (!ipaddr.isValid(normalized)) {
return false;
}

return (
normalized === '::' ||
normalized === '::1' ||
normalized.startsWith('fc') ||
normalized.startsWith('fd') ||
normalized.startsWith('fe80:') ||
normalized.startsWith('::ffff:0:')
);
return ipaddr.process(normalized).range() !== 'unicast';
}

private isBlockedAddress(address: string): boolean {
const version = isIP(address);

if (version === 4) {
return this.isBlockedIPv4(address);
private isUrlBlockedError(error: unknown): boolean {
if (!error || typeof error !== 'object') {
return false;
}

if (version === 6) {
return this.isBlockedIPv6(address);
}
const candidate = error as {
code?: unknown;
cause?: { code?: unknown };
};

return false;
return (
candidate.code === MetadataValidationStatus.URL_BLOCKED ||
candidate.cause?.code === MetadataValidationStatus.URL_BLOCKED
);
}

private async assertAllowedMetadataUrl(url: string): Promise<void> {
Expand All @@ -90,7 +80,7 @@ export class AppService {
throw MetadataValidationStatus.URL_BLOCKED;
}

const hostname = parsedUrl.hostname.toLowerCase();
const hostname = this.stripIPv6Brackets(parsedUrl.hostname.toLowerCase());
if (hostname === 'localhost' || hostname.endsWith('.localhost')) {
throw MetadataValidationStatus.URL_BLOCKED;
}
Expand Down Expand Up @@ -119,10 +109,8 @@ export class AppService {
.then((result) => {
const addresses = Array.isArray(result) ? result : [result];

if (
addresses.some(({ address }) => this.isBlockedAddress(address))
) {
callback(new Error(MetadataValidationStatus.URL_BLOCKED), '', 0);
if (addresses.some(({ address }) => this.isBlockedAddress(address))) {
callback(new UrlBlockedError(), '', 0);
return;
}

Expand Down Expand Up @@ -155,12 +143,13 @@ export class AppService {

try {
await this.assertAllowedMetadataUrl(url);

const { data: rawData } = await firstValueFrom(
this.httpService
.get(url, {
httpAgent: this.safeHttpAgent,
httpsAgent: this.safeHttpsAgent,
maxRedirects: 0,
proxy: false,
headers: {
// Required to not being blocked by APIs that require a User-Agent
'User-Agent': 'GovTool/Metadata-Validation-Tool',
Expand All @@ -175,6 +164,10 @@ export class AppService {
finalize(() => Logger.log(`Fetching ${url} completed`)),
catchError((error) => {
Logger.error(error, JSON.stringify(error));
if (this.isUrlBlockedError(error)) {
throw MetadataValidationStatus.URL_BLOCKED;
}

throw MetadataValidationStatus.URL_NOT_FOUND;
}),
),
Expand Down
Loading