feat: chaitin waf response logging

apisix-master-0.rockspec

@@ -80,7 +80,7 @@ dependencies = {
     "xml2lua = 1.6-2",
     "lua-resty-mediador = 0.1.2-1",
     "lua-resty-ldap = 0.1.0-0",
-    "lua-resty-t1k = 1.1.6-0",
+    "lua-resty-t1k = 1.2.0-0",
     "brotli-ffi = 0.3-1",
     "lua-ffi-zlib = 0.6-0",
     "jsonpath = 1.0-1",

apisix/plugins/chaitin-waf.lua

@@ -88,6 +88,16 @@ local plugin_schema = {
                 },
                 real_client_ip = {
                     type = "boolean"
+                },
+                log_resp = {
+                    type = "boolean"
+                },
+                resp_body_size = {
+                    type = "integer",
+                    minimum = 0
+                },
+                extra_ignored_content_types = {
+                    type = "string"
                 }
             },
         },
@@ -152,6 +162,23 @@ local metadata_schema = {
                 real_client_ip = {
                     type = "boolean",
                     default = true
+                },
+                -- report the response to the WAF detection service
+                log_resp = {
+                    type = "boolean",
+                    default = false
+                },
+                -- amount of the response body to report, in KB,
+                -- 0 disables body buffering
+                resp_body_size = {
+                    type = "integer",
+                    minimum = 0,
+                    default = 4
+                },
+                -- extra response content types (comma separated) to skip
+                -- on top of the built-in ignored list
+                extra_ignored_content_types = {
+                    type = "string"
                 }
             },
             default = {},
@@ -273,30 +300,35 @@ local function get_conf(conf, metadata)
         real_client_ip = true,
     }
 
-    if metadata.config then
-        t.connect_timeout = metadata.config.connect_timeout
-        t.send_timeout = metadata.config.send_timeout
-        t.read_timeout = metadata.config.read_timeout
-        t.req_body_size = metadata.config.req_body_size
-        t.keepalive_size = metadata.config.keepalive_size
-        t.keepalive_timeout = metadata.config.keepalive_timeout
-        if metadata.config.real_client_ip ~= nil then
-            t.real_client_ip = metadata.config.real_client_ip
+    local function apply(config)
+        if not config then
+            return
         end
-    end
 
-    if conf.config then
-        t.connect_timeout = conf.config.connect_timeout
-        t.send_timeout = conf.config.send_timeout
-        t.read_timeout = conf.config.read_timeout
-        t.req_body_size = conf.config.req_body_size
-        t.keepalive_size = conf.config.keepalive_size
-        t.keepalive_timeout = conf.config.keepalive_timeout
-        if conf.config.real_client_ip ~= nil then
-            t.real_client_ip = conf.config.real_client_ip
+        t.connect_timeout = config.connect_timeout
+        t.send_timeout = config.send_timeout
+        t.read_timeout = config.read_timeout
+        t.req_body_size = config.req_body_size
+        t.keepalive_size = config.keepalive_size
+        t.keepalive_timeout = config.keepalive_timeout
+        if config.real_client_ip ~= nil then
+            t.real_client_ip = config.real_client_ip
+        end
+        if config.log_resp ~= nil then
+            t.log_resp = config.log_resp
+        end
+        if config.resp_body_size ~= nil then
+            t.resp_body_size = config.resp_body_size
+        end
+        if config.extra_ignored_content_types ~= nil then
+            t.extra_ignored_content_types = config.extra_ignored_content_types
         end
     end
 
+    -- metadata config first, then route/service level config overrides it
+    apply(metadata.config)
+    apply(conf.config)
+
     t.mode = conf.mode or metadata.mode or t.mode
 
     return t
@@ -421,4 +453,14 @@ function _M.header_filter(conf, ctx)
 end
 
 
+function _M.body_filter(conf, ctx)
+    t1k.do_body_filter()
+end
+
+
+function _M.log(conf, ctx)
+    t1k.do_log()
+end
+
+
 return _M

docs/en/latest/plugins/chaitin-waf.md

@@ -65,6 +65,9 @@ The Plugin can add the following response headers, depending on the configuratio
 | config.keepalive_size    | integer       | false    | 256     |                          | The maximum number of idle connections to the WAF detection service that can be maintained concurrently. |
 | config.keepalive_timeout | integer       | false    | 60000   |                          | The idle connection timeout for the WAF service, in milliseconds. |
 | config.real_client_ip    | boolean       | false    | true    |                          | If true, the Plugin sends APISIX's resolved client IP to the WAF service (the same value used elsewhere in APISIX, derived from the connection and `apisix.trusted_addresses`). If false, the Plugin sends the IP of the peer directly connected to APISIX. |
+| config.log_resp          | boolean       | false    | false   |                          | If true, the Plugin also reports the response to the WAF service. See [Response Logging](#response-logging). |
+| config.resp_body_size    | integer       | false    | 4       | >= 0                     | The amount of the response body to report, in KB. Set to `0` to report only the status line and response headers. Effective only when `config.log_resp` is `true`. |
+| config.extra_ignored_content_types | string | false  |         |                          | A comma separated list of response `Content-Type` values to skip, in addition to the built-in ignored list. Effective only when `config.log_resp` is `true`. |
 
 ## Plugin Metadata
 
@@ -82,6 +85,19 @@ The Plugin can add the following response headers, depending on the configuratio
 | config.keepalive_size    | integer       | False    | 256     |              | The maximum number of idle connections to the WAF detection service that can be maintained concurrently. |
 | config.keepalive_timeout | integer       | False    | 60000   |              | The idle connection timeout for the WAF service, in milliseconds. |
 | config.real_client_ip    | boolean       | False    | true    |              | If true, the Plugin sends APISIX's resolved client IP to the WAF service (the same value used elsewhere in APISIX, derived from the connection and `apisix.trusted_addresses`). If false, the Plugin sends the IP of the peer directly connected to APISIX. |
+| config.log_resp          | boolean       | False    | false   |              | If true, the Plugin also reports the response to the WAF service. See [Response Logging](#response-logging). |
+| config.resp_body_size    | integer       | False    | 4       | >= 0         | The amount of the response body to report, in KB. Set to `0` to report only the status line and response headers. Effective only when `config.log_resp` is `true`. |
+| config.extra_ignored_content_types | string | False  |         |              | A comma separated list of response `Content-Type` values to skip, in addition to the built-in ignored list. Effective only when `config.log_resp` is `true`. |
+
+## Response Logging
+
+By default the Plugin only reports the request to the WAF service. Setting `config.log_resp` to `true` makes it report the response as well: the status line, the response headers, and up to `config.resp_body_size` KB of the response body.
+
+The report is sent from an `ngx.timer` during the log phase, after the response has been handed back to the client, so it does not block the response or add latency to it. Detection results are visible in the SafeLine WAF console rather than acted on by APISIX, meaning a response is never blocked or modified based on this report.
+
+Responses are not reported when the request itself was already blocked, or when the response `Content-Type` matches an ignored type. Audio, video, font, image and other binary media types are ignored out of the box; use `config.extra_ignored_content_types` to skip more, for example `text/csv,application/pdf`.
+
+Note that buffering the response body costs memory per in-flight request, so raise `config.resp_body_size` with care on routes serving large responses.
 
 ## Examples
 

docs/zh/latest/plugins/chaitin-waf.md

@@ -65,6 +65,9 @@ description: chaitin-waf 插件与长亭雷池 WAF 集成，以检测和阻止
 | config.keepalive_size    | integer       | 否       | 256     |                          | 可同时维持的与 WAF 检测服务的空闲连接数上限。 |
 | config.keepalive_timeout | integer       | 否       | 60000   |                          | 与 WAF 服务的空闲连接超时时间，单位为毫秒。 |
 | config.real_client_ip    | boolean       | 否       | true    |                          | 若为 true，则插件将 APISIX 解析得到的客户端 IP（与 APISIX 其他地方使用的相同，由连接信息以及 `apisix.trusted_addresses` 共同决定）发送给 WAF 服务。若为 false，则插件发送与 APISIX 直接建立连接的对端 IP。 |
+| config.log_resp          | boolean       | 否       | false   |                          | 若为 true，则插件同时将响应上报给 WAF 服务。参见 [响应上报](#响应上报)。 |
+| config.resp_body_size    | integer       | 否       | 4       | >= 0                     | 上报的响应体大小，单位为 KB。设置为 `0` 表示仅上报状态行与响应头。仅当 `config.log_resp` 为 true 时生效。 |
+| config.extra_ignored_content_types | string | 否     |         |                          | 以逗号分隔的响应 `Content-Type` 列表，在内置忽略列表之外额外跳过这些类型。仅当 `config.log_resp` 为 true 时生效。 |
 
 ## 插件元数据
 
@@ -82,6 +85,19 @@ description: chaitin-waf 插件与长亭雷池 WAF 集成，以检测和阻止
 | config.keepalive_size    | integer       | 否       | 256     |              | 可同时维持的与 WAF 检测服务的空闲连接数上限。 |
 | config.keepalive_timeout | integer       | 否       | 60000   |              | 与 WAF 服务的空闲连接超时时间，单位为毫秒。 |
 | config.real_client_ip    | boolean       | 否       | true    |              | 若为 true，则插件将 APISIX 解析得到的客户端 IP（与 APISIX 其他地方使用的相同，由连接信息以及 `apisix.trusted_addresses` 共同决定）发送给 WAF 服务。若为 false，则插件发送与 APISIX 直接建立连接的对端 IP。 |
+| config.log_resp          | boolean       | 否       | false   |              | 若为 true，则插件同时将响应上报给 WAF 服务。参见 [响应上报](#响应上报)。 |
+| config.resp_body_size    | integer       | 否       | 4       | >= 0         | 上报的响应体大小，单位为 KB。设置为 `0` 表示仅上报状态行与响应头。仅当 `config.log_resp` 为 true 时生效。 |
+| config.extra_ignored_content_types | string | 否     |         |              | 以逗号分隔的响应 `Content-Type` 列表，在内置忽略列表之外额外跳过这些类型。仅当 `config.log_resp` 为 true 时生效。 |
+
+## 响应上报
+
+插件默认只将请求上报给 WAF 服务。将 `config.log_resp` 设置为 true 后，插件会同时上报响应：状态行、响应头，以及最多 `config.resp_body_size` KB 的响应体。
+
+上报在 log 阶段通过 `ngx.timer` 发出，此时响应已经返回给客户端，因此不会阻塞响应、也不会增加响应延迟。检测结果在雷池 WAF 控制台查看，APISIX 不会依据该上报结果拦截或修改响应。
+
+以下情况不会上报响应：请求本身已被拦截，或响应 `Content-Type` 属于被忽略的类型。音频、视频、字体、图片等二进制媒体类型默认被忽略，可通过 `config.extra_ignored_content_types` 追加，例如 `text/csv,application/pdf`。
+
+请注意，缓冲响应体会为每个进行中的请求占用内存，在返回大响应的路由上调高 `config.resp_body_size` 需谨慎。
 
 ## 示例
 

t/plugin/chaitin-waf.t

@@ -519,3 +519,184 @@ trigger: true
 chaitin-waf client_ip: 127.0.0.1
 --- no_error_log
 chaitin-waf client_ip: 192.0.2.10
+
+
+
+=== TEST 16: wrong schema: negative resp_body_size
+--- config
+    location /do {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/plugin_metadata/chaitin-waf',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "nodes": [
+                        {
+                            "host": "127.0.0.1",
+                            "port": 8088
+                        }
+                    ],
+                    "config": {
+                        "log_resp": true,
+                        "resp_body_size": -1
+                    }
+                 }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.print(body)
+        }
+    }
+--- error_code: 400
+--- response_body
+{"error_msg":"invalid configuration: property \"config\" validation failed: property \"resp_body_size\" validation failed: expected -1 to be at least 0"}
+
+
+
+=== TEST 17: wrong schema: extra_ignored_content_types wrong type
+--- config
+    location /do {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/plugin_metadata/chaitin-waf',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "nodes": [
+                        {
+                            "host": "127.0.0.1",
+                            "port": 8088
+                        }
+                    ],
+                    "config": {
+                        "log_resp": true,
+                        "extra_ignored_content_types": ["text/csv"]
+                    }
+                 }]]
+                )
+
+            if code >= 300 then
+                ngx.status = code
+            end
+            ngx.print(body)
+        }
+    }
+--- error_code: 400
+--- response_body
+{"error_msg":"invalid configuration: property \"config\" validation failed: property \"extra_ignored_content_types\" validation failed: wrong type: expected string, got table"}
+
+
+
+=== TEST 18: response logging enabled prepare
+--- config
+    location /do {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+
+            local code, body = t('/apisix/admin/plugin_metadata/chaitin-waf',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "nodes": [
+                        {
+                            "host": "127.0.0.1",
+                            "port": 8088
+                        }
+                    ]
+                 }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+                return ngx.print(body)
+            end
+
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "methods": ["GET"],
+                    "plugins": {
+                        "chaitin-waf": {
+                            "config": {
+                                "log_resp": true,
+                                "resp_body_size": 1,
+                                "extra_ignored_content_types": "text/csv"
+                            }
+                        }
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/*"
+                 }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+                return ngx.print(body)
+            end
+            ngx.say("passed")
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 19: response logging does not affect the response
+--- request
+GET /hello
+--- error_code: 200
+--- response_body
+hello world
+--- response_headers
+X-APISIX-CHAITIN-WAF: yes
+X-APISIX-CHAITIN-WAF-ACTION: pass
+X-APISIX-CHAITIN-WAF-STATUS: 200
+
+
+
+=== TEST 20: response logging off by default prepare
+--- config
+    location /do {
+        content_by_lua_block {
+            local t = require("lib.test_admin").test
+            local code, body = t('/apisix/admin/routes/1',
+                 ngx.HTTP_PUT,
+                 [[{
+                    "methods": ["GET"],
+                    "plugins": {
+                        "chaitin-waf": {}
+                    },
+                    "upstream": {
+                        "nodes": {
+                            "127.0.0.1:1980": 1
+                        },
+                        "type": "roundrobin"
+                    },
+                    "uri": "/*"
+                 }]]
+            )
+            if code >= 300 then
+                ngx.status = code
+                return ngx.print(body)
+            end
+            ngx.say("passed")
+        }
+    }
+--- response_body
+passed
+
+
+
+=== TEST 21: response logging off by default
+--- request
+GET /hello
+--- error_code: 200
+--- response_body
+hello world
+--- response_headers
+X-APISIX-CHAITIN-WAF: yes
+X-APISIX-CHAITIN-WAF-ACTION: pass
+X-APISIX-CHAITIN-WAF-STATUS: 200