import io import pytest from ucc import * from ucc.s3 import * from .conftest import MULTIPART_UPLOAD_THRESHOLD, MULTIPART_PART_SIZE TEST_FOLDER = '/test_s3' @pytest.fixture(scope='module', autouse=True) def set_globals(path_client): global c c = path_client @pytest.fixture(scope='session', autouse=True) def test_folder(request, path_client): c = path_client c.mkdir(TEST_FOLDER) def fin(): c.rmdirs(TEST_FOLDER) request.addfinalizer(fin) return TEST_FOLDER def upload_file(fp: io.BytesIO, path: str = None) -> Metadata: if path is None: path = f'{TEST_FOLDER}/upload_{time.time()}' meta = c.upload(path, fp) fp.seek(0) return meta def compute_md5(fp): _hash = hashlib.md5() _compute_hash(fp, _hash) fp.seek(0) return _hash.hexdigest() def compute_awss3_multipart_etag(fp, part_size, part_num): _hash = AmazonS3MultipartHasher(part_size) _compute_hash(fp, _hash) local_etag_value = _hash.hexdigest() return f'"{local_etag_value}-{part_num}"' def _compute_hash(fp, hashobj, buf_size=65536): assert hasattr(fp, 'read') while True: data = fp.read(buf_size) if not data: break hashobj.update(data) fp.seek(0) def test_zero_size_etag(): fp = io.BytesIO(b"") meta = upload_file(fp) etag = meta.hash['etag'] expected_etag = f'"{compute_md5(fp)}"' assert expected_etag == etag, f'Expected ETag: {expected_etag}, Actual ETag: {etag}' def test_normal_upload_etag(): fp = io.BytesIO(b".") meta = upload_file(fp) etag = meta.hash['etag'] expected_etag = f'"{compute_md5(fp)}"' assert expected_etag == etag, f'Expected ETag: {expected_etag}, Actual ETag: {etag}' def test_multipart_upload_etag(): fp = io.BytesIO(b"."*(MULTIPART_UPLOAD_THRESHOLD+1)) meta = upload_file(fp) etag = meta.hash['etag'] assert '-' in etag, f'ETag: {etag} do not contains "-"' _, part_num = etag.strip('"').split('-') expected_etag = compute_awss3_multipart_etag(fp, MULTIPART_PART_SIZE, part_num) assert expected_etag == etag, f'Expected ETag: {expected_etag}, Actual ETag: {etag}' def test_normal_upload_etag_is_equal_to_list_etag(): fp = io.BytesIO(b".") meta = upload_file(fp) upload_etag = meta.hash['etag'] list_etag = get_etag_from_list(meta.path) assert upload_etag == list_etag, f'{upload_etag} is not equal to {list_etag}' def get_etag_from_list(path): for m in c.listdir(os.path.dirname(path)): if m.path == path: return m.hash['etag'] def test_multipart_upload_etag_is_equal_to_list_etag(): fp = io.BytesIO(b"."*(MULTIPART_UPLOAD_THRESHOLD+1)) meta = upload_file(fp) upload_etag = meta.hash['etag'] list_etag = get_etag_from_list(meta.path) assert upload_etag == list_etag, f'{upload_etag} is not equal to {list_etag}' def test_unicode_delete_multi(): path = '/Ã' result = c.delete_multi([path]) assert path == result['deleted'][0]