diff --git a/backend/internal/proxy-host.js b/backend/internal/proxy-host.js index 2c159d48ad..d606f53527 100644 --- a/backend/internal/proxy-host.js +++ b/backend/internal/proxy-host.js @@ -451,6 +451,31 @@ const internalProxyHost = { return rows; }, + /** + * @param {Access} access + * @param {Number} hostId + * @return {Promise} + */ + getHostForLogs: (access, hostId) => { + return access + .can("proxy_hosts:logs", hostId) + .then((access_data) => { + const query = proxyHostModel.query().where("is_deleted", 0).andWhere("id", hostId).first(); + + if (access_data.permission_visibility !== "all") { + query.andWhere("owner_user_id", access.token.getUserId(1)); + } + + return query; + }) + .then((row) => { + if (!row?.id) { + throw new errs.ItemNotFoundError(hostId); + } + return row; + }); + }, + /** * Report use * diff --git a/backend/lib/access/proxy_hosts-logs.json b/backend/lib/access/proxy_hosts-logs.json new file mode 100644 index 0000000000..d88e4cfff5 --- /dev/null +++ b/backend/lib/access/proxy_hosts-logs.json @@ -0,0 +1,23 @@ +{ + "anyOf": [ + { + "$ref": "roles#/definitions/admin" + }, + { + "type": "object", + "required": ["permission_proxy_hosts", "roles"], + "properties": { + "permission_proxy_hosts": { + "$ref": "perms#/definitions/view" + }, + "roles": { + "type": "array", + "items": { + "type": "string", + "enum": ["user"] + } + } + } + } + ] +} diff --git a/backend/routes/ci.js b/backend/routes/ci.js index a6f8be03a2..0fff3d0bbd 100644 --- a/backend/routes/ci.js +++ b/backend/routes/ci.js @@ -1,4 +1,5 @@ import express from "express"; +import fs from "node:fs/promises"; import dnsPlugins from "../certbot/dns-plugins.json" with { type: "json" }; import { installPlugin } from "../lib/certbot.js"; import { debug, express as logger } from "../logger.js"; @@ -56,4 +57,37 @@ router return; }); +/** + * /api/ci/mock-log + * + * Write mock log files in CI environment + */ +router + .route("/mock-log") + .options((_, res) => { + res.sendStatus(204); + }) + + .post(async (req, res, next) => { + try { + const { hostId, type, content } = req.body; + if (!hostId || !type || content === undefined) { + return res.status(400).send({ + error: "Missing required fields: hostId, type, or content", + }); + } + + const filePath = `/data/logs/proxy-host-${hostId}_${type}.log`; + const dirPath = "/data/logs"; + + await fs.mkdir(dirPath, { recursive: true }); + await fs.writeFile(filePath, content, "utf8"); + + res.status(200).send(true); + } catch (err) { + debug(logger, `${req.method.toUpperCase()} ${req.path}: ${err}`); + next(err); + } + }); + export default router; diff --git a/backend/routes/nginx/proxy_host_logs.js b/backend/routes/nginx/proxy_host_logs.js new file mode 100644 index 0000000000..49591e45c5 --- /dev/null +++ b/backend/routes/nginx/proxy_host_logs.js @@ -0,0 +1,353 @@ +import fs from "node:fs/promises"; +import express from "express"; +import moment from "moment"; +import jwtdecode from "../../lib/express/jwt-decode.js"; +import errs from "../../lib/error.js"; +import validator from "../../lib/validator/index.js"; +import { debug, express as logger } from "../../logger.js"; +import internalProxyHost from "../../internal/proxy-host.js"; + +const router = express.Router({ + caseSensitive: true, + strict: true, + mergeParams: true, +}); + +function parseLogLineTime(line, type) { + if (type === "access") { + const match = line.match(/^\[([^\]]+)\]/); + if (match) { + const m = moment(match[1], "DD/MMM/YYYY:HH:mm:ss ZZ"); + if (m.isValid()) { + return m; + } + } + } else if (type === "error") { + const match = line.match(/^(\d{4}\/\d{2}\/\d{2} \d{2}:\d{2}:\d{2})/); + if (match) { + const m = moment(match[1], "YYYY/MM/DD HH:mm:ss"); + if (m.isValid()) { + return m; + } + } + } + return null; +} + +async function readLastLines({ filePath, maxLines, search, since, type }) { + const chunkSize = 64 * 1024; // 64KB + const lines = []; + let fileHandle = null; + + try { + fileHandle = await fs.open(filePath, "r"); + const stat = await fileHandle.stat(); + const fileSize = stat.size; + + let position = fileSize; + const buffer = Buffer.alloc(chunkSize); + let leftover = ""; + + const sinceTime = since ? moment(since) : null; + while (position > 0 && lines.length < maxLines) { + const readSize = Math.min(chunkSize, position); + position -= readSize; + + const { bytesRead } = await fileHandle.read(buffer, 0, readSize, position); + const chunkStr = buffer.toString("utf8", 0, bytesRead) + leftover; + + const chunkLines = chunkStr.split(/\r?\n/); + // The first line of chunkLines is incomplete because it was cut in the middle of a line, + // unless position was 0. So we store it as leftover. + if (position > 0) { + leftover = chunkLines.shift(); + } else { + leftover = ""; + } + + // We process the chunkLines backwards + for (let i = chunkLines.length - 1; i >= 0; i--) { + const line = chunkLines[i]; + if (!line && i === chunkLines.length - 1 && position + readSize === fileSize) { + // Skip trailing empty line at the very end of the file + continue; + } + + // Check since timestamp if provided + if (sinceTime) { + const lineTime = parseLogLineTime(line, type); + if (lineTime) { + if (lineTime.isBefore(sinceTime)) { + // Since log lines are chronologically ordered, we can stop reading + // once we find a line with a timestamp before 'sinceTime'. + return lines; + } + } + } + + // Check search filter if provided + if (search && !line.includes(search)) { + continue; + } + + lines.unshift(line); + if (lines.length >= maxLines) { + break; + } + } + } + + // If we reached the beginning of the file and there's a leftover, process it + if (leftover && lines.length < maxLines) { + if (sinceTime) { + const lineTime = parseLogLineTime(leftover, type); + if (lineTime?.isBefore(sinceTime)) { + return lines; + } + } + if (!search || leftover.includes(search)) { + lines.unshift(leftover); + } + } + } finally { + if (fileHandle) { + await fileHandle.close(); + } + } + + return lines; +} + +/** + * /api/nginx/proxy-hosts/:host_id/logs + */ +router + .route("/:host_id/logs") + .options((_, res) => { + res.sendStatus(204); + }) + .all(jwtdecode()) + + /** + * GET /api/nginx/proxy-hosts/:host_id/logs + * + * Retrieve log lines for a specific proxy host + */ + .get(async (req, res, next) => { + try { + const payload = { + host_id: req.params.host_id, + type: req.query.type || "access", + lines: req.query.lines !== undefined ? Number.parseInt(req.query.lines, 10) : 100, + }; + if (req.query.search !== undefined) { + payload.search = req.query.search; + } + if (req.query.since !== undefined) { + payload.since = req.query.since; + } + + const data = await validator( + { + required: ["host_id"], + additionalProperties: false, + properties: { + host_id: { + $ref: "common#/properties/id", + }, + type: { + type: "string", + enum: ["access", "error"], + default: "access", + }, + lines: { + type: "integer", + minimum: 1, + maximum: 1000, + default: 100, + }, + search: { + type: "string", + }, + since: { + type: "string", + }, + }, + }, + payload, + ); + + if (data.since && !moment(data.since).isValid()) { + throw new errs.ValidationError("Invalid since timestamp format. Must be ISO 8601."); + } + + const hostId = Number.parseInt(data.host_id, 10); + // Call internalProxyHost helper to handle permissions and retrieve host + await internalProxyHost.getHostForLogs(res.locals.access, hostId); + + const filePath = `/data/logs/proxy-host-${hostId}_${data.type}.log`; + + // Check if file exists, if not throw 404 + try { + await fs.stat(filePath); + } catch (err) { + if (err.code === "ENOENT") { + throw new errs.ItemNotFoundError(`Log file not found for host ${hostId} (${data.type})`); + } + throw err; + } + + const matchedLines = await readLastLines({ + filePath, + maxLines: data.lines, + search: data.search, + since: data.since, + type: data.type, + }); + + res.status(200).send({ + host_id: hostId, + log_type: data.type, + returned_lines: matchedLines.length, + lines: matchedLines, + }); + } catch (err) { + debug(logger, `${req.method.toUpperCase()} ${req.path}: ${err}`); + next(err); + } + }); + +/** + * /api/nginx/proxy-hosts/:host_id/logs/summary + */ +router + .route("/:host_id/logs/summary") + .options((_, res) => { + res.sendStatus(204); + }) + .all(jwtdecode()) + + /** + * GET /api/nginx/proxy-hosts/:host_id/logs/summary + * + * Retrieve statistics summary for a specific proxy host's access log + */ + .get(async (req, res, next) => { + try { + const data = await validator( + { + required: ["host_id"], + additionalProperties: false, + properties: { + host_id: { + $ref: "common#/properties/id", + }, + }, + }, + { + host_id: req.params.host_id, + }, + ); + + const hostId = Number.parseInt(data.host_id, 10); + // Call internalProxyHost helper to handle permissions and retrieve host + await internalProxyHost.getHostForLogs(res.locals.access, hostId); + + const accessLogPath = `/data/logs/proxy-host-${hostId}_access.log`; + const errorLogPath = `/data/logs/proxy-host-${hostId}_error.log`; + + // Check access log exists, throw 404 if missing + let access_log_size_bytes = 0; + try { + const stat = await fs.stat(accessLogPath); + access_log_size_bytes = stat.size; + } catch (err) { + if (err.code === "ENOENT") { + throw new errs.ItemNotFoundError(`Access log file not found for host ${hostId}`); + } + throw err; + } + + // Get error log size, if missing default to 0 + let error_log_size_bytes = 0; + try { + const stat = await fs.stat(errorLogPath); + error_log_size_bytes = stat.size; + } catch (err) { + if (err.code !== "ENOENT") { + throw err; + } + } + + const accessLines = await readLastLines({ + filePath: accessLogPath, + maxLines: 1000, + type: "access", + }); + + const status_codes = {}; + const pathsMap = {}; + const clientsMap = {}; + let cacheHits = 0; + let cacheableCount = 0; + + const regex = /^\[([^\]]+)\] (\S+) (\S+) (\d{3}) - (\S+) (\S+) (\S+) "([^"]+)" \[Client ([^\]]+)\]/; + + for (const line of accessLines) { + const match = line.match(regex); + if (!match) continue; + + const cacheStatus = match[2]; + const status = match[4]; + const requestUri = match[8]; + const clientIp = match[9]; + + // Status codes + status_codes[status] = (status_codes[status] || 0) + 1; + + // Paths + const path = requestUri.split("?")[0]; + pathsMap[path] = (pathsMap[path] || 0) + 1; + + // Clients + clientsMap[clientIp] = (clientsMap[clientIp] || 0) + 1; + + // Cache hit rate + if (cacheStatus !== "-") { + cacheableCount++; + if (cacheStatus === "HIT" || cacheStatus === "REVALIDATED") { + cacheHits++; + } + } + } + + const cache_hit_rate = + cacheableCount > 0 ? Number.parseFloat((cacheHits / cacheableCount).toFixed(4)) : 0.0; + + const top_paths = Object.entries(pathsMap) + .map(([path, count]) => ({ path, count })) + .sort((a, b) => b.count - a.count) + .slice(0, 10); + + const top_clients = Object.entries(clientsMap) + .map(([client, count]) => ({ client, count })) + .sort((a, b) => b.count - a.count) + .slice(0, 10); + + res.status(200).send({ + host_id: hostId, + period: "last_1000_lines", + status_codes, + top_paths, + top_clients, + cache_hit_rate, + access_log_size_bytes, + error_log_size_bytes, + }); + } catch (err) { + debug(logger, `${req.method.toUpperCase()} ${req.path}: ${err}`); + next(err); + } + }); + +export default router; diff --git a/backend/routes/nginx/proxy_hosts.js b/backend/routes/nginx/proxy_hosts.js index 7045a195cc..dbd6464458 100644 --- a/backend/routes/nginx/proxy_hosts.js +++ b/backend/routes/nginx/proxy_hosts.js @@ -5,6 +5,7 @@ import apiValidator from "../../lib/validator/api.js"; import validator from "../../lib/validator/index.js"; import { debug, express as logger } from "../../logger.js"; import { getValidationSchema } from "../../schema/index.js"; +import proxyHostLogs from "./proxy_host_logs.js"; const router = express.Router({ caseSensitive: true, @@ -206,4 +207,6 @@ router } }); +router.use("/", proxyHostLogs); + export default router; diff --git a/test/cypress/e2e/api/ProxyHostLogs.cy.js b/test/cypress/e2e/api/ProxyHostLogs.cy.js new file mode 100644 index 0000000000..aa7f7b37b1 --- /dev/null +++ b/test/cypress/e2e/api/ProxyHostLogs.cy.js @@ -0,0 +1,208 @@ +/// + +describe('Proxy Host Logs endpoints', () => { + let token; + let hostId; + + before(() => { + cy.resetUsers(); + cy.getToken().then((tok) => { + token = tok; + + // Create a proxy host for testing logs + cy.task('backendApiPost', { + token: token, + path: '/api/nginx/proxy-hosts', + data: { + domain_names: ['logs-test.example.com'], + forward_scheme: 'http', + forward_host: '127.0.0.1', + forward_port: 80, + access_list_id: '0', + certificate_id: 0, + meta: { + dns_challenge: false + }, + advanced_config: '', + locations: [], + block_exploits: false, + caching_enabled: false, + allow_websocket_upgrade: false, + http2_support: false, + hsts_enabled: false, + hsts_subdomains: false, + ssl_forced: false + } + }).then((data) => { + hostId = data.id; + }); + }); + }); + + it('Should return 404 for nonexistent host', () => { + cy.task('backendApiGet', { + token: token, + path: '/api/nginx/proxy-hosts/999999/logs', + returnOnError: true + }).then((data) => { + expect(data).to.have.property('error'); + expect(data.error.code).to.be.equal(404); + }); + }); + + it('Should return 404 for unauthorized user', () => { + cy.task('backendApiGet', { + path: `/api/nginx/proxy-hosts/${hostId}/logs`, + returnOnError: true + }).then((data) => { + expect(data).to.have.property('error'); + // jwtdecode middleware returns 404 when no token is provided + expect(data.error.code).to.be.equal(404); + }); + }); + + it('Should handle empty or missing access log', () => { + cy.task('backendApiGet', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}/logs?type=access`, + returnOnError: true + }).then((data) => { + // nginx may or may not have created the log file yet. + // Accept both states: 404 if file doesn't exist, or 200 with 0 lines if file is empty. + if (data.error) { + expect(data.error.code).to.be.equal(404); + } else { + expect(data).to.have.property('host_id', hostId); + expect(data).to.have.property('returned_lines', 0); + expect(data.lines).to.be.an('array').that.is.empty; + } + }); + }); + + describe('With populated log files', () => { + const accessLogContent = [ + '[05/Jun/2026:15:00:00 +0000] HIT 200 200 - GET http logs-test.example.com "/api/v1/users" [Client 192.168.1.10] [Length 500] [Gzip 1.2] [Sent-to 127.0.0.1] "Mozilla" "referer"', + '[05/Jun/2026:15:01:00 +0000] MISS 200 200 - GET http logs-test.example.com "/api/v1/users" [Client 192.168.1.10] [Length 500] [Gzip 1.2] [Sent-to 127.0.0.1] "Mozilla" "referer"', + '[05/Jun/2026:15:02:00 +0000] HIT 304 304 - GET http logs-test.example.com "/assets/logo.png" [Client 192.168.1.11] [Length 120] [Gzip -] [Sent-to 127.0.0.1] "Mozilla" "referer"', + '[05/Jun/2026:15:03:00 +0000] - - 404 - GET http logs-test.example.com "/notfound" [Client 192.168.1.12] [Length 230] [Gzip -] [Sent-to 127.0.0.1] "Mozilla" "referer"' + ].join('\n'); + + const errorLogContent = [ + '2026/06/05 15:02:00 [warn] 123#123: *456 using cache key, client: 192.168.1.11', + '2026/06/05 15:03:00 [error] 123#123: *457 open() "/notfound" failed, client: 192.168.1.12' + ].join('\n'); + + before(() => { + // Populate the access logs + cy.task('writeMockLog', { + hostId: hostId, + type: 'access', + content: accessLogContent + }); + + // Populate the error logs + cy.task('writeMockLog', { + hostId: hostId, + type: 'error', + content: errorLogContent + }); + }); + + it('Should retrieve access logs with 200', () => { + cy.task('backendApiGet', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}/logs?type=access` + }).then((data) => { + expect(data).to.have.property('host_id', hostId); + expect(data).to.have.property('log_type', 'access'); + expect(data).to.not.have.property('file'); + expect(data).to.not.have.property('total_lines'); + expect(data.lines).to.be.an('array'); + expect(data.lines.length).to.be.equal(4); + expect(data.lines[0]).to.contain('15:00:00'); + expect(data.lines[3]).to.contain('15:03:00'); + }); + }); + + it('Should retrieve error logs with 200', () => { + cy.task('backendApiGet', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}/logs?type=error` + }).then((data) => { + expect(data).to.have.property('host_id', hostId); + expect(data).to.have.property('log_type', 'error'); + expect(data).to.not.have.property('file'); + expect(data).to.not.have.property('total_lines'); + expect(data.lines).to.be.an('array'); + expect(data.lines.length).to.be.equal(2); + expect(data.lines[0]).to.contain('[warn]'); + expect(data.lines[1]).to.contain('[error]'); + }); + }); + + it('Should filter logs with search parameter', () => { + cy.task('backendApiGet', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}/logs?type=access&search=logo.png` + }).then((data) => { + expect(data.lines.length).to.be.equal(1); + expect(data.lines[0]).to.contain('/assets/logo.png'); + }); + }); + + it('Should filter logs with since parameter', () => { + // ISO 8601 for 15:01:30 UTC on 2026-06-05 + const sinceTime = '2026-06-05T15:01:30Z'; + cy.task('backendApiGet', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}/logs?type=access&since=${sinceTime}` + }).then((data) => { + // Should return 15:02:00 and 15:03:00 lines (2 lines) + expect(data.lines.length).to.be.equal(2); + expect(data.lines[0]).to.contain('15:02:00'); + expect(data.lines[1]).to.contain('15:03:00'); + }); + }); + + it('Should limit lines with lines parameter', () => { + cy.task('backendApiGet', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}/logs?type=access&lines=2` + }).then((data) => { + expect(data.lines.length).to.be.equal(2); + // Since we read last 2 lines, it should be the ones from 15:02:00 and 15:03:00 + expect(data.lines[0]).to.contain('15:02:00'); + expect(data.lines[1]).to.contain('15:03:00'); + }); + }); + + it('Should return summary statistics for access log', () => { + cy.task('backendApiGet', { + token: token, + path: `/api/nginx/proxy-hosts/${hostId}/logs/summary` + }).then((data) => { + expect(data).to.have.property('host_id', hostId); + expect(data).to.have.property('period', 'last_1000_lines'); + expect(data.status_codes).to.deep.equal({ + '200': 2, + '304': 1, + '404': 1 + }); + expect(data.top_paths).to.deep.equal([ + { path: '/api/v1/users', count: 2 }, + { path: '/assets/logo.png', count: 1 }, + { path: '/notfound', count: 1 } + ]); + expect(data.top_clients).to.deep.equal([ + { client: '192.168.1.10', count: 2 }, + { client: '192.168.1.11', count: 1 }, + { client: '192.168.1.12', count: 1 } + ]); + // 3 cacheable requests: 2 HIT, 1 MISS. Hit rate = 2/3 = 0.6667 + expect(data.cache_hit_rate).to.be.equal(0.6667); + expect(data.access_log_size_bytes).to.be.greaterThan(0); + expect(data.error_log_size_bytes).to.be.greaterThan(0); + }); + }); + }); +}); diff --git a/test/cypress/plugins/backendApi/task.mjs b/test/cypress/plugins/backendApi/task.mjs index edc5015c69..818366aae5 100644 --- a/test/cypress/plugins/backendApi/task.mjs +++ b/test/cypress/plugins/backendApi/task.mjs @@ -96,5 +96,16 @@ export default (config) => { options.returnOnError || false, ); }, + + writeMockLog: (options) => { + const { hostId, type, content } = options; + const api = new Client(config); + return api.request( + "post", + "/api/ci/mock-log", + false, + { hostId, type, content } + ); + }, }; };