diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index e8eaa94f..3a3c91b2 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -45,7 +45,7 @@ jobs: - name: Run Tests shell: bash -l {0} - run: pytest -vv s3fs + run: pytest -vv -s s3fs -k test_write_blocks pre-commit: diff --git a/s3fs/core.py b/s3fs/core.py index 0945384a..bfd10476 100644 --- a/s3fs/core.py +++ b/s3fs/core.py @@ -2395,7 +2395,9 @@ def upload_part(part_data: bytes): if len(part_data) == 0: return part = len(self.parts) + 1 - logger.debug("Upload chunk %s, %s" % (self, part)) + logger.debug( + "Upload chunk %s, %s; %s bytes" % (self, part, len(part_data)) + ) out = self._call_s3( "upload_part", diff --git a/s3fs/tests/test_s3fs.py b/s3fs/tests/test_s3fs.py index af426493..bc08c16f 100644 --- a/s3fs/tests/test_s3fs.py +++ b/s3fs/tests/test_s3fs.py @@ -1287,10 +1287,10 @@ def test_write_blocks(s3): assert f.mpu assert f.parts assert s3.info(test_bucket_name + "/temp")["size"] == 6 * 2**20 - with s3.open(test_bucket_name + "/temp", "wb", block_size=10 * 2**20) as f: + with s3.open(test_bucket_name + "/temp2", "wb", block_size=10 * 2**20) as f: f.write(b"a" * 15 * 2**20) assert f.buffer.tell() == 0 - assert s3.info(test_bucket_name + "/temp")["size"] == 15 * 2**20 + assert s3.info(test_bucket_name + "/temp2")["size"] == 15 * 2**20 def test_readline(s3): @@ -1406,7 +1406,8 @@ def test_append(s3): f.write(b"a" * 10 * 2**20) with s3.open(a, "ab") as f: pass # append, no write, big file - assert s3.cat(a) == b"a" * 10 * 2**20 + data = s3.cat(a) + assert len(data) == 10 * 2**20 and set(data) == set(b"a") with s3.open(a, "ab") as f: assert f.parts is None @@ -1414,13 +1415,18 @@ def test_append(s3): assert f.parts assert f.tell() == 10 * 2**20 f.write(b"extra") # append, small write, big file - assert s3.cat(a) == b"a" * 10 * 2**20 + b"extra" + data = s3.cat(a) + assert len(data) == 10 * 2**20 + len(b"extra") + assert data[-5:] == b"extra" with s3.open(a, "ab") as f: assert f.tell() == 10 * 2**20 + 5 f.write(b"b" * 10 * 2**20) # append, big write, big file assert f.tell() == 20 * 2**20 + 5 - assert s3.cat(a) == b"a" * 10 * 2**20 + b"extra" + b"b" * 10 * 2**20 + data = s3.cat(a) + assert len(data) == 10 * 2**20 + len(b"extra") + 10 * 2**20 + assert data[10 * 2**20 : 10 * 2**20 + 5] == b"extra" + assert set(data[-10 * 2**20 :]) == set(b"b") # Keep Head Metadata head = dict(