diff --git a/cli/src/mpftp/rawpaste.py b/cli/src/mpftp/rawpaste.py index a229577..de39e64 100644 --- a/cli/src/mpftp/rawpaste.py +++ b/cli/src/mpftp/rawpaste.py @@ -185,6 +185,12 @@ def _repr_line(out: bytes) -> bytes: return line.rstrip(b"\x00 ") +#: Bytes the board reads per step while hashing, and how many steps between +#: the dots that tell the computer it's still at work (64 KB). +HASH_CHUNK = 2048 +HASH_DOT_EVERY = 32 + + def hash_file(transport: Any, path: str, algo: str = "sha256") -> bytes: """mpremote's ``fs_hashfile``, but the board does the hashing whenever it can. @@ -194,6 +200,11 @@ def hash_file(transport: Any, path: str, algo: str = "sha256") -> bytes: verified ``get`` over a 115200-baud UART took five times as long as the copy (mpftp#183). This asks for either form, and only a board with neither sends the file back, through the transport's own (faster) ``fs_readfile``. + + The board prints a dot every 64 KB while it hashes. mpremote gives up on a + reply after 10 s with no byte from the board, and an 8.5 MB file on an SD + card took longer than that to hash in silence, so a good copy was reported + as a failed one (mpftp#202). A board that really stops still times out. """ import hashlib @@ -208,13 +219,17 @@ def hash_file(transport: Any, path: str, algo: str = "sha256") -> bytes: return hashlib.new(algo, bytes(transport.fs_readfile(path))).digest() try: transport.exec( - "buf = memoryview(bytearray(512))\n" + "buf = memoryview(bytearray({c}))\n" + "k = 0\n" "with open({p!r}, 'rb') as f:\n" " while True:\n" " n = f.readinto(buf)\n" " if n == 0:\n" " break\n" - " h.update(buf if n == 512 else buf[:n])\n".format(p=path) + " h.update(buf if n == {c} else buf[:n])\n" + " k += 1\n" + " if k % {d} == 0:\n" + " print('.', end='')\n".format(p=path, c=HASH_CHUNK, d=HASH_DOT_EVERY) ) return transport.eval("h.digest()") except TransportExecError as e: diff --git a/cli/tests/test_hash_file.py b/cli/tests/test_hash_file.py index bd6e5d1..d6ccd1e 100644 --- a/cli/tests/test_hash_file.py +++ b/cli/tests/test_hash_file.py @@ -5,7 +5,9 @@ """Verification hashes on the board, CircuitPython included (mpftp#183).""" import hashlib +import io import unittest +from contextlib import redirect_stdout try: from mpremote.transport import TransportExecError @@ -36,7 +38,11 @@ def _open(path, mode): return io.BytesIO(files[path]) self.ns.update(hashlib=self.board_hashlib, open=_open) - exec(code, self.ns) + out = io.StringIO() + with redirect_stdout(out): + exec(code, self.ns) + self.output = out.getvalue() + return self.output.encode() def eval(self, expr): return eval(expr, self.ns) @@ -75,6 +81,17 @@ def test_circuitpython_hashes_on_the_board(self): def test_board_without_hashlib_reads_once(self): self.assertEqual(self._hash(None), 1) + def test_a_long_hash_keeps_the_board_talking(self): + # mpremote gives up after 10 s with no byte from the board; a big + # file on an SD card took longer than that to hash (mpftp#202). + from mpftp.rawpaste import hash_file + + size = 8_589_932 + data = bytes(range(256)) * (size // 256) + b"x" * (size % 256) + board = _Board({"/sd/f.wav": data}, hashlib) + self.assertEqual(hash_file(board, "/sd/f.wav", "sha256"), hashlib.sha256(data).digest()) + self.assertGreaterEqual(len(board.output), size // 65536) + if __name__ == "__main__": unittest.main()