结合之前一篇文章（阿里云oss图片压缩）
使用Python实现了阿里云oss的bucket迁移
迁移同时对图片进行压缩处理
脚本记录如下：
#!/usr/bin/env python2.7
# -*- coding: utf-8 -*-
from __future__ import print_function
import os
import signal
import sys
from datetime import datetime
from itertools import islice
from time import sleep
import oss2
from PIL import Image, ImageFile
def echo(msg):
time_str = datetime.now().strftime('%Y-%m-%d %H:%M:%S')
print('[%s]: %s' % (time_str, msg))
def exit(signum, frame):
echo('User exit')
sys.exit(0)
def loadMarker(file_path='.marker'):
"""
读取进度
"""
marker = ''
if not os.path.exists(file_path):
with open(file_path, 'w') as fh:
fh.write(marker)
else:
with open(file_path, 'r') as fh:
marker = fh.read()
return marker
def saveMarker(marker, file_path='.marker'):
"""
保存进度
"""
with open(file_path, 'w') as fh:
fh.write(marker)
def compressImage(file_path):
"""
压缩图片
"""
ok = False
backup_file_path = '%s.bak' % file_path
try:
os.rename(file_path, backup_file_path)
with open(backup_file_path, 'rb') as fp:
img = Image.open(fp)
file_format = str(img.format)
if file_format == 'PNG' or file_format == 'JPEG':
ImageFile.MAXBLOCK = img.size[0] * img.size[1]
img.save(file_path, quality=90, optimize=True)
osize = os.path.getsize(backup_file_path)
nsize = os.path.getsize(file_path)
if nsize >= osize:
result = 'Cannot further compress: %s' % file_path
os.remove(file_path)
else:
result = nsize
ok = True
else:
result = 'Ignoring file: %s format: %s' % (file_path, file_format)
except Exception as e:
# shutil.move(backup_file_path, file_path)
result = 'Compress %s failed, error: %s' % (file_path, e)
if os.path.exists(backup_file_path):
if os.path.exists(file_path):
os.remove(backup_file_path)
else:
os.rename(backup_file_path, file_path)
return ok, result
def do_compressOssImg(bucket, bucket2, opath, osize, is_compress):
"""
下载bucket图片压缩并上传
"""
ok = False
nsize = osize
msg = ''
exist = bucket2.object_exists(opath)
if exist:
ok = True
msg = 'object exist'
return ok, msg
file_dir, file_name = os.path.split(opath)
file_name = opath.replace('/', '_')
tmp_file_path = '%s%s' % (TMP_DIR, file_name)
s = bucket.get_object_to_file(opath, tmp_file_path)
if s.status == 200:
if is_compress:
ok, nsize = compressImage(tmp_file_path)
if ok:
msg = 'compress ok\t%s -> %s' % (osize, nsize)
else:
msg = 'compress failed\t%s' % nsize
s = bucket2.put_object_from_file(opath, tmp_file_path)
if s.status == 200:
ok = True
os.remove(tmp_file_path)
msg = 'transfer ok\t%s' % nsize
else:
msg = 'transfer failed\t%s' % nsize
else:
msg = 'transfer & compress failed'
return ok, msg
def main():
"""
main
"""
size = 50
marker = loadMarker()
# 源bucket认证
auth = oss2.Auth(OSS_ACCESS_ID, OSS_ACCESS_KEY)
# 目的bucket认证
auth2 = oss2.Auth(OSS_ACCESS_ID2, OSS_ACCESS_KEY2)
# 源bucket
bucket = oss2.Bucket(auth, OSS_ENDPOINT, OSS_BUCKET)
# 目的bucket
bucket2 = oss2.Bucket(auth2, OSS_ENDPOINT2, OSS_BUCKET2)
i = 0
iterable = oss2.ObjectIterator(bucket, marker=marker)
# 遍历bucket
while True:
# 从marker开始，获取bucket对象迭代器
iterable = oss2.ObjectIterator(bucket, marker=marker)
# 进行切片
iterator = islice(iterable, size)
opath = ''
for obj in iterator:
i += 1
opath = obj.key
# 防止老版本bucket兼容@符号进行图片裁剪处理导致
# 的可以遍历出而无法获取的bug，此处应注意避免文件名包含@符号
if '@' in opath:
echo('Failed:\t%s\tSkipped ...' % opath)
continue
osize = obj.size
otype = opath.split('.')[-1]
is_compress = False
if otype.lower() in IMG_TYPE and osize > 1024 * 500: # 大于500k的图片进行压缩
is_compress = True
ok, msg = do_compressOssImg(bucket, bucket2, opath, osize, is_compress)
if ok:
echo('OK:\t%s\t%s' % (opath, msg))
else:
echo('Failed:\t%s\t%s' % (opath, msg))
sys.exit(1)
if iterable.is_truncated:
marker = opath
else:
marker = ''
saveMarker(marker)
echo('End. Total: %s' % i)
break
saveMarker(marker)
sleep(0.01)
if __name__ == '__main__':
# 源bucket的access_id和access_key
OSS_ENDPOINT = 'oss-cn-beijing-internal.aliyuncs.com'
OSS_ACCESS_ID = 'xxxxxxxxxxxx'
OSS_ACCESS_KEY = 'xxxxxxxxxxxxxxxxxxxxxxxx'
OSS_BUCKET = 'test'
# 目的bucket的access_id和access_key
OSS_ENDPOINT2 = 'oss-cn-beijing-internal.aliyuncs.com'
OSS_ACCESS_ID2 = 'xxxxxxxxxxxxxx'
OSS_ACCESS_KEY2 = 'xxxxxxxxxxxxxxxxxxxxxxxxxxxx'
OSS_BUCKET2 = 'test-new'
TMP_DIR = '/dev/shm/'
IMG_TYPE = [
'jpg',
'jpeg',
'png'
]
# 捕获ctrl+c退出信号
signal.signal(signal.SIGINT, exit)
signal.signal(signal.SIGTERM, exit)
main()