diff --git a/govtool/metadata-validation/package-lock.json b/govtool/metadata-validation/package-lock.json index ef2cf171f..2689cb14d 100644 --- a/govtool/metadata-validation/package-lock.json +++ b/govtool/metadata-validation/package-lock.json @@ -20,6 +20,7 @@ "blakejs": "^1.2.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "ipaddr.js": "^2.5.0", "joi": "18.2.1", "jsonld": "^9.0.0", "reflect-metadata": "^0.2.0", @@ -5929,12 +5930,12 @@ "license": "ISC" }, "node_modules/ipaddr.js": { - "version": "1.9.1", - "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", - "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "version": "2.5.0", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-2.5.0.tgz", + "integrity": "sha512-aq+t5NAc+cS6rZQQVWC2x98CPqGtKKTMDd4Gaodv0wShnItdKg/51djkGJ1hqH+Oy0ivDftCbSLCQob8zso01w==", "license": "MIT", "engines": { - "node": ">= 0.10" + "node": ">= 10" } }, "node_modules/is-arrayish": { @@ -8151,6 +8152,15 @@ "node": ">= 0.10" } }, + "node_modules/proxy-addr/node_modules/ipaddr.js": { + "version": "1.9.1", + "resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.9.1.tgz", + "integrity": "sha512-0KI/607xoxSToH7GjN1FfSbLoU0+btTicjsQSWQlh/hZykN8KpmMf7uYwPW3R+akZ6R/w18ZlXSHBYXiYUPO3g==", + "license": "MIT", + "engines": { + "node": ">= 0.10" + } + }, "node_modules/proxy-from-env": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/proxy-from-env/-/proxy-from-env-2.1.0.tgz", diff --git a/govtool/metadata-validation/package.json b/govtool/metadata-validation/package.json index 977e4b109..90ef6a251 100644 --- a/govtool/metadata-validation/package.json +++ b/govtool/metadata-validation/package.json @@ -31,6 +31,7 @@ "blakejs": "^1.2.1", "class-transformer": "^0.5.1", "class-validator": "^0.14.1", + "ipaddr.js": "^2.5.0", "joi": "18.2.1", "jsonld": "^9.0.0", "reflect-metadata": "^0.2.0", diff --git a/govtool/metadata-validation/src/app.service.test.ts b/govtool/metadata-validation/src/app.service.test.ts index 981831319..5557e7b46 100644 --- a/govtool/metadata-validation/src/app.service.test.ts +++ b/govtool/metadata-validation/src/app.service.test.ts @@ -80,6 +80,8 @@ describe('AppService', () => { expect.objectContaining({ httpAgent: expect.any(Object), httpsAgent: expect.any(Object), + maxRedirects: 0, + proxy: false, }), ); }); @@ -154,6 +156,40 @@ describe('AppService', () => { expect(httpService.get).not.toHaveBeenCalled(); }); + it.each([ + 'http://[::]/metadata.json', + 'http://[::1]/metadata.json', + 'http://[fc00::1]/metadata.json', + 'http://[fe90::1]/metadata.json', + 'http://[febf::1]/metadata.json', + 'http://[ff02::1]/metadata.json', + 'http://[::ffff:7f00:1]/metadata.json', + ])('should block special-use IPv6 URL %s before fetching', async (url) => { + const result = await service.validateMetadata({ hash: 'hash', url }); + + expect(result).toEqual({ + status: MetadataValidationStatus.URL_BLOCKED, + valid: false, + metadata: undefined, + }); + expect(httpService.get).not.toHaveBeenCalled(); + }); + + it.each([ + 'http://192.0.2.1/metadata.json', + 'http://198.51.100.1/metadata.json', + 'http://203.0.113.1/metadata.json', + ])('should block reserved IPv4 URL %s before fetching', async (url) => { + const result = await service.validateMetadata({ hash: 'hash', url }); + + expect(result).toEqual({ + status: MetadataValidationStatus.URL_BLOCKED, + valid: false, + metadata: undefined, + }); + expect(httpService.get).not.toHaveBeenCalled(); + }); + it('should block hostnames that resolve to private addresses', async () => { (lookup as jest.Mock).mockResolvedValueOnce([{ address: '10.0.0.5' }]); @@ -215,4 +251,25 @@ describe('AppService', () => { }), ).rejects.toThrow(MetadataValidationStatus.URL_BLOCKED); }); + + it('should preserve URL_BLOCKED from a connection-time lookup', async () => { + const blockedError = Object.assign( + new Error(MetadataValidationStatus.URL_BLOCKED), + { code: MetadataValidationStatus.URL_BLOCKED }, + ); + jest + .spyOn(httpService, 'get') + .mockReturnValueOnce(throwError(() => blockedError)); + + const result = await service.validateMetadata({ + hash: 'hash', + url: 'http://example.com', + }); + + expect(result).toEqual({ + status: MetadataValidationStatus.URL_BLOCKED, + valid: false, + metadata: undefined, + }); + }); }); diff --git a/govtool/metadata-validation/src/app.service.ts b/govtool/metadata-validation/src/app.service.ts index a49b7502e..4e59667c0 100644 --- a/govtool/metadata-validation/src/app.service.ts +++ b/govtool/metadata-validation/src/app.service.ts @@ -3,78 +3,68 @@ import { catchError, finalize, firstValueFrom } from 'rxjs'; import { HttpService } from '@nestjs/axios'; import * as blake from 'blakejs'; import { lookup } from 'node:dns/promises'; -import { isIP, LookupFunction } from 'node:net'; +import { LookupFunction } from 'node:net'; import { Agent as HttpAgent } from 'node:http'; import { Agent as HttpsAgent } from 'node:https'; +import * as ipaddr from 'ipaddr.js'; import { ValidateMetadataDTO } from '@dto'; import { LoggerMessage, MetadataValidationStatus } from '@enums'; import { validateMetadataStandard, parseMetadata, getStandard } from '@utils'; import { /* MetadataStandard, */ ValidateMetadataResult } from '@types'; +class UrlBlockedError extends Error { + readonly code = MetadataValidationStatus.URL_BLOCKED; + + constructor() { + super(MetadataValidationStatus.URL_BLOCKED); + this.name = 'UrlBlockedError'; + } +} + @Injectable() export class AppService { constructor(private readonly httpService: HttpService) {} private readonly safeHttpAgent = new HttpAgent({ + keepAlive: true, lookup: this.createSafeLookup(), }); private readonly safeHttpsAgent = new HttpsAgent({ + keepAlive: true, lookup: this.createSafeLookup(), }); - private isBlockedIPv4(address: string): boolean { - const parts = address.split('.').map(Number); - const [first, second] = parts; - - return ( - first === 0 || - first === 10 || - first === 127 || - (first === 100 && second >= 64 && second <= 127) || - (first === 169 && second === 254) || - (first === 172 && second >= 16 && second <= 31) || - (first === 192 && second === 0 && parts[2] === 0) || - (first === 192 && second === 168) || - (first === 198 && (second === 18 || second === 19)) || - first >= 224 - ); + private stripIPv6Brackets(hostname: string): string { + return hostname.startsWith('[') && hostname.endsWith(']') + ? hostname.slice(1, -1) + : hostname; } - private isBlockedIPv6(address: string): boolean { - const normalized = address.toLowerCase(); - const ipv4MappedPrefix = '::ffff:'; - - if (normalized.startsWith(ipv4MappedPrefix)) { - const mappedAddress = normalized.slice(ipv4MappedPrefix.length); - if (isIP(mappedAddress) === 4) { - return this.isBlockedIPv4(mappedAddress); - } + private isBlockedAddress(address: string): boolean { + const normalized = this.stripIPv6Brackets(address); + if (!ipaddr.isValid(normalized)) { + return false; } - return ( - normalized === '::' || - normalized === '::1' || - normalized.startsWith('fc') || - normalized.startsWith('fd') || - normalized.startsWith('fe80:') || - normalized.startsWith('::ffff:0:') - ); + return ipaddr.process(normalized).range() !== 'unicast'; } - private isBlockedAddress(address: string): boolean { - const version = isIP(address); - - if (version === 4) { - return this.isBlockedIPv4(address); + private isUrlBlockedError(error: unknown): boolean { + if (!error || typeof error !== 'object') { + return false; } - if (version === 6) { - return this.isBlockedIPv6(address); - } + const candidate = error as { + code?: unknown; + cause?: { code?: unknown }; + }; - return false; + return ( + candidate.code === MetadataValidationStatus.URL_BLOCKED || + candidate.cause?.code === MetadataValidationStatus.URL_BLOCKED + ); } private async assertAllowedMetadataUrl(url: string): Promise { @@ -90,7 +80,7 @@ export class AppService { throw MetadataValidationStatus.URL_BLOCKED; } - const hostname = parsedUrl.hostname.toLowerCase(); + const hostname = this.stripIPv6Brackets(parsedUrl.hostname.toLowerCase()); if (hostname === 'localhost' || hostname.endsWith('.localhost')) { throw MetadataValidationStatus.URL_BLOCKED; } @@ -119,10 +109,8 @@ export class AppService { .then((result) => { const addresses = Array.isArray(result) ? result : [result]; - if ( - addresses.some(({ address }) => this.isBlockedAddress(address)) - ) { - callback(new Error(MetadataValidationStatus.URL_BLOCKED), '', 0); + if (addresses.some(({ address }) => this.isBlockedAddress(address))) { + callback(new UrlBlockedError(), '', 0); return; } @@ -155,12 +143,13 @@ export class AppService { try { await this.assertAllowedMetadataUrl(url); - const { data: rawData } = await firstValueFrom( this.httpService .get(url, { httpAgent: this.safeHttpAgent, httpsAgent: this.safeHttpsAgent, + maxRedirects: 0, + proxy: false, headers: { // Required to not being blocked by APIs that require a User-Agent 'User-Agent': 'GovTool/Metadata-Validation-Tool', @@ -175,6 +164,10 @@ export class AppService { finalize(() => Logger.log(`Fetching ${url} completed`)), catchError((error) => { Logger.error(error, JSON.stringify(error)); + if (this.isUrlBlockedError(error)) { + throw MetadataValidationStatus.URL_BLOCKED; + } + throw MetadataValidationStatus.URL_NOT_FOUND; }), ),