feat: Use part_size with fixed big size for better ETag

main.py

@@ -22,7 +22,9 @@ s3_config_data = json.loads(open(expanduser("~/.mc/config.json")).read())["alias
 def get_files(s3, bucket, prefix) -> Mapping[str, Tuple[int, str]]:
     res = {}
     prefix_len = len(prefix)
-    for obj in s3.list_objects(bucket, prefix=prefix, recursive=True):
+    for obj in tqdm.tqdm(
+        s3.list_objects(bucket, prefix=prefix, recursive=True), desc="S3 list objects"
+    ):
         if obj.is_dir:
             continue
         res[obj.object_name[prefix_len:].lstrip("/")] = (obj.size, obj.etag)
@@ -94,8 +96,10 @@ def put_object_data(s3: minio.Minio, bucket: str, object_name: str, data: bytes)
     s3.put_object(
         bucket,
         object_name,
-        io.BytesIO(data),
+        data=io.BytesIO(data),
-        len(data),
+        length=len(data),
+        num_parallel_uploads=1,
+        part_size=150 * 1024 * 1024,
     )
 
 
@@ -161,9 +165,9 @@ def main():
     print(f"Different {len(different_files)} files")
 
     for key in tqdm.tqdm(new_files.union(different_files)):
-        try:
         source_object = os.path.join(source_prefix, key)
         target_object = os.path.join(target_prefix, key)
+        try:
             logging.info(
                 f"Moving {source_bucket}:{source_object} to "
                 f"{target_bucket}:{target_object}"
@@ -178,8 +182,8 @@ def main():
         except RetryError:
             logging.warning(
                 "Retry on moving"
-                "{source_bucket}:{source_object} to "
+                f"{source_bucket}:{source_object} to "
-                "{target_bucket}:{target_object}"
+                f"{target_bucket}:{target_object}"
             )