feat: support configurable exclusion of JSON log fields

CHANGELOG.md

@@ -17,6 +17,9 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/).
 - fix: `init_sanic()` no longer raises `KeyError: 'format'` when Sanic 24.x and later omit
   formatter `format` keys; older Sanic logging configuration remains supported (PR #133).
 
+- feat: `json_logging.EXCLUDED_FIELDS` omits the named fields from every JSON log record. Empty by default, so
+  output is unchanged. Fixes #98
+
 - fix: FastAPI application logs and `json_logging.get_correlation_id()` now use the request's correlation id
   instead of `-` in synchronous (`def`) endpoints (fixes #124, PR #123). When request instrumentation is enabled with
   `init_request_instrument(app)` (Python 3.9+):

README.md

@@ -34,6 +34,7 @@ author.
    2.5 [Root logger](#25-root-logger)  
    2.6 [Custom log formatter](#26-custom-log-formatter)  
    2.7 [Exclude certain URL from request instrumentation](#27-exclude-certain-url-from-request-instrumentation)
+   2.8 [Exclude fields from the JSON output](#28-exclude-fields-from-the-json-output)
 3. [Configuration](#3-configuration)
 4. [Python References](#4-python-references)
 5. [Framework support plugin development](#5-framework-support-plugin-development)
@@ -288,6 +289,30 @@ method like below:
 json_logging.init_request_instrument(app, exclude_url_patterns=[r'/exclude_from_request_instrumentation'])
 ```
 
+## 2.8 Exclude fields from the JSON output
+
+Set `json_logging.EXCLUDED_FIELDS` to omit fields from every JSON record, including request logs:
+
+```python
+import json_logging
+
+json_logging.EXCLUDED_FIELDS = ["written_ts"]          # or a tuple/set/frozenset of field names
+json_logging.init_flask(enable_json=True)
+```
+
+- The default is `()`, so output is unchanged.
+- Set it **before** calling `init_*()`: formatters read it when they are constructed, the same way as `COMPONENT_*`.
+- A bare string, or a collection containing non-strings, raises `TypeError` from `init_*()`,
+  `init_request_instrument()` or a formatter constructor. The `init_*()` and `init_request_instrument()` calls
+  validate the setting before changing any state, so you can correct it and call them again.
+- Fields are removed after they are collected, so `extra` and `props` entries are removed too.
+- Exclusions apply to formatters that use `BaseJSONFormatter.format()`: the built-in JSON formatters and subclasses
+  that inherit or call it. A custom `logging.Formatter` that does not use it must filter fields itself.
+- Excluding a field changes the JSON schema. Downstream log consumers may depend on fields such as `written_ts`,
+  `correlation_id` or `level`; exclude a field only when you know nothing downstream reads it.
+- This is an output-formatting option, **not** a security redaction or data-loss-prevention control. A sensitive
+  value can still appear in the message, in another field, or through another handler or formatter.
+
 # 3. Configuration
 
 logging library can be configured by setting the value in json_logging, all configuration must be placed before

json_logging/__init__.py

@@ -6,7 +6,8 @@
 
 from json_logging import util
 from json_logging.dto import RequestResponseDTOBase, DefaultRequestResponseDTO
-from json_logging.formatters import JSONRequestLogFormatter, JSONLogFormatter, JSONLogWebFormatter
+from json_logging.formatters import JSONRequestLogFormatter, JSONLogFormatter, JSONLogWebFormatter, \
+    _validated_excluded_fields
 from json_logging.framework_base import BaseRequestInfoExtractor, BaseResponseInfoExtractor, \
     BaseAppRequestInstrumentationConfigurator, \
     BaseFrameworkConfigurator
@@ -23,6 +24,9 @@
 CREATE_CORRELATION_ID_IF_NOT_EXISTS = True
 JSON_SERIALIZER = lambda log: json.dumps(log, ensure_ascii=False, default=str)
 CORRELATION_ID_HEADERS = ['X-Correlation-ID', 'X-Request-ID']
+# Field names to omit from every JSON log record (e.g. 'written_ts'). Empty by default, so output is unchanged.
+# Read when a formatter is constructed, so set it before calling init_*().
+EXCLUDED_FIELDS = ()
 COMPONENT_ID = EMPTY_VALUE
 COMPONENT_NAME = EMPTY_VALUE
 COMPONENT_INSTANCE_INDEX = 0
@@ -100,6 +104,9 @@ def __init(framework_name=None, custom_formatter=None, enable_json=False):
         if not issubclass(custom_formatter, logging.Formatter):
             raise ValueError('custom_formatter is not subclass of logging.Formatter', custom_formatter)
 
+    # Validate before any global state changes, so a corrected setting can be retried in the same process
+    _validated_excluded_fields(EXCLUDED_FIELDS)
+
     if not enable_json and not ENABLE_JSON_LOGGING:
         _logger.warning(
             "JSON format is not enabled, normal log will be in plain text but request logging still in JSON format! "
@@ -160,6 +167,9 @@ def init_request_instrument(app=None, custom_formatter=None, exclude_url_pattern
         raise ValueError('request_response_dto_class is not subclass of json_logging.RequestInfoBase',
                          custom_formatter)
 
+    # Validate before hooks and handlers are installed, so a failed call leaves nothing behind
+    _validated_excluded_fields(EXCLUDED_FIELDS)
+
     configurator = _current_framework['app_request_instrumentation_configurator']()
     configurator.config(app, request_response_dto_class, exclude_url_patterns=exclude_url_patterns)
 

json_logging/formatters.py

@@ -38,14 +38,32 @@ def _sanitize_log_msg(record):
     return record.getMessage().replace('\n', '_').replace('\r', '_').replace('\t', '_')
 
 
+def _validated_excluded_fields(fields):
+    """
+    Check json_logging.EXCLUDED_FIELDS and return it as a frozenset of field names.
+    :raises TypeError: if it is not a list/tuple/set/frozenset of strings (a bare string is rejected
+    because iterating it would exclude single characters)
+    """
+    if isinstance(fields, (basestring, bytes)) or not isinstance(fields, (list, tuple, set, frozenset)):
+        raise TypeError("json_logging.EXCLUDED_FIELDS must be a list, tuple, set or frozenset of field names, "
+                        "got %r" % (fields,))
+    for field in fields:
+        # basestring also accepts unicode names on Python 2; on Python 3 it is str, so bytes are rejected
+        if not isinstance(field, basestring):
+            raise TypeError("json_logging.EXCLUDED_FIELDS entries must be strings, got %r" % (field,))
+    return frozenset(fields)
+
+
 class BaseJSONFormatter(logging.Formatter):
     """
        Base class for JSON formatters
     """
     base_object_common = {}
+    excluded_fields = frozenset()
 
     def __init__(self, *args, **kw):
         super(BaseJSONFormatter, self).__init__(*args, **kw)
+        self.excluded_fields = _validated_excluded_fields(json_logging.EXCLUDED_FIELDS)
         if json_logging.COMPONENT_ID and json_logging.COMPONENT_ID != json_logging.EMPTY_VALUE:
             self.base_object_common["component_id"] = json_logging.COMPONENT_ID
         if json_logging.COMPONENT_NAME and json_logging.COMPONENT_NAME != json_logging.EMPTY_VALUE:
@@ -58,6 +76,8 @@ def format(self, record):
             Format the specified record as text. Overriding default python logging implementation
         """
         log_object = self._format_log_object(record, request_util=json_logging._request_util)
+        for field in self.excluded_fields:
+            log_object.pop(field, None)
         return json_logging.JSON_SERIALIZER(log_object)
 
     def _format_log_object(self, record, request_util):