Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion pythainlp/tokenize/core.py
Original file line number Diff line number Diff line change
Expand Up @@ -488,7 +488,7 @@ def sent_tokenize(
if isinstance(text, list):
try:
original_text = "".join(text)
except ValueError:
except TypeError:
return []
else:
original_text = str(text)
Expand Down
10 changes: 10 additions & 0 deletions tests/core/test_tokenize.py
Original file line number Diff line number Diff line change
Expand Up @@ -333,6 +333,16 @@ def test_sent_tokenize(self):
)
with self.assertRaises(ValueError):
sent_tokenize("ฉันไป กิน", engine="XX") # engine does not exist
# Reproduce: list with non-string items should return []
Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Not sure if this is an expected behavior.

@wannaphong thoughts?

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I agree.

# instead of raising TypeError (str.join raises TypeError, not ValueError)
self.assertEqual(
sent_tokenize(["สวัสดี", 123], engine="whitespace+newline"),
[],
)
self.assertEqual(
sent_tokenize(["สวัสดี", None], engine="whitespace+newline"),
[],
)

def test_subword_tokenize(self):
self.assertEqual(subword_tokenize(None), []) # type: ignore[arg-type]
Expand Down
Loading