From 05c85e0189061019dc55888b33ddb30cc3e9d9ae Mon Sep 17 00:00:00 2001 From: Peter Corke Date: Fri, 14 Aug 2026 16:24:03 +1000 Subject: [PATCH] fix: suppress divide-by-zero RuntimeWarning in BagOfWords idf computation idf = np.log(N / ni) can legitimately divide by zero: after stopword removal (nstopwords > 0), a remaining word can end up with zero occurrences across every image (ni=0), giving idf=inf for that word. This is harmless -- every downstream use of idf multiplies it by that same word's occurrence count, which is also always 0 there, so the 0*inf->nan result gets replaced with 0 before any vector is returned (already the case at both existing call sites). The equivalent divide-by-zero one call site down (v = nid / nd * idf) was already wrapped in np.errstate(divide="ignore", invalid="ignore") -- this one just wasn't. Documented the inf possibility in __init__'s docstring, and added a regression test (existing test_retrieve already exercised the nstopwords=50 path that triggers this; wrapped it in warnings.simplefilter("error", RuntimeWarning) so it fails loudly if this regresses). Root-caused via RVC3-python's chap12.ipynb (BagOfWords(features, 2_000, nstopwords=50, seed=0)). Co-Authored-By: Claude Sonnet 5 --- src/machinevisiontoolbox/BagOfWords.py | 14 +++++++++++++- tests/test_bag_of_words.py | 11 ++++++++++- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/src/machinevisiontoolbox/BagOfWords.py b/src/machinevisiontoolbox/BagOfWords.py index 65b85885..33231f3e 100644 --- a/src/machinevisiontoolbox/BagOfWords.py +++ b/src/machinevisiontoolbox/BagOfWords.py @@ -58,6 +58,13 @@ def __init__( of image features is assigned labels from the ``.centroids`` any with a label greater that ``.nstopwords`` is a stop word and can be discarded. + .. note:: Removing stop words can leave a remaining word that never + occurs in any image, giving it an inverse document frequency (idf) + of ``inf`` (``log(N / 0)``). This is harmless -- every place idf is + used to compute a word-frequency vector, the corresponding word + count is also always zero there, so the ``0 * inf`` result is + ``nan`` and gets replaced with 0 before the vector is returned. + :reference: - Video Google: a text retrieval approach to object matching in videos J.Sivic and A.Zisserman, @@ -139,7 +146,12 @@ def __init__( # total number of occurrences of word i # multiple occurrences in the one image count only as one ni = (W > 0).sum(axis=1) - idf = np.log(N / ni) + # a word that never occurs in any image (ni=0, possible after + # stopword removal) gives idf=inf here; harmless since nid is + # always 0 at that same position below, so 0*inf->nan, already + # suppressed by the invalid="ignore" there. + with np.errstate(divide="ignore"): + idf = np.log(N / ni) M = [] diff --git a/tests/test_bag_of_words.py b/tests/test_bag_of_words.py index e8ab1f13..bd34e619 100644 --- a/tests/test_bag_of_words.py +++ b/tests/test_bag_of_words.py @@ -4,6 +4,7 @@ """ import unittest +import warnings from machinevisiontoolbox import FileCollection, BagOfWords import numpy as np @@ -26,7 +27,15 @@ def test_retrieve(self): bag = BagOfWords(features, 2_000, seed=0) # self.assertEqual(len(bag), 42_213) - bag = BagOfWords(features, 2_000, nstopwords=50, seed=0) + # Regression test: nstopwords>0 can leave a word with zero + # occurrences across every remaining image (ni=0), which used to + # raise "RuntimeWarning: divide by zero" from idf = np.log(N/ni) + # -- harmless (the inf it produces is always multiplied by 0 + # downstream) but not suppressed, unlike the equivalent case + # right below it in the same function. + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + bag = BagOfWords(features, 2_000, nstopwords=50, seed=0) # self.assertEqual(len(bag), 34_997) query = FileCollection("campus/holdout/*.png", mono=True)