s3 api接口的调用
最近公司使用s3做文件存储服务器，因此在程序中需要调用s3的api，目前程序中使用了python和java版本的s3的api，简单做下记录，方便以后使用。
一、s3 api使用python版
1.安装boto3的sdk
pip install boto3==1.4.7
2.创建s3的api客户端
import boto3
from botocore.client import Config
import os
import sys
import threading
#创建s3链接，如果s3服务器是第四代，则需要设置signature_version='s3v4'
s3_client = boto3.client('s3', endpoint_url='s3服务器地址',
aws_access_key_id='s3服务器的access_key_id',
aws_secret_access_key='s3服务器的secret_access_key',
region_name='s3服务器的时区，这里可以填写cn-north-1',
config=Config(signature_version='s3'))
3.获取s3中bucket列表
bucket_list = s3_client.list_buckets()
print bucket_list
4.创建bucket
bucket = s3_client.create_bucket(Bucket='bucket的名称')
print bucket
5.获取bucket信息
bucket_info = s3_client.head_bucket(Bucket='bucket的名称')
print bucket_info
6.删除bucket
#bucket_delete = s3_client.delete_bucket(Bucket='bucket的名称')
7.上传文件到s3服务器
# s3_client.upload_file("上传的源文件地址", "bucket名称", "上传文件在s3上对应的key名称", ExtraArgs={'ACL': 'public-read'})
或者
s3_client.upload_file("上传的源文件地址", "bucket名称", "上传文件在s3上对应的key名称",
ExtraArgs={'ACL': 'public-read'},
Callback=UploadProgressPercentage("上传的源文件地址"))
Callback属性对应的类方法如下，该类方法在控制台中打印了上传文件的进度
class UploadProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._size = float(os.path.getsize(filename))
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify we'll assume this is hooked up
# to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
percentage = (self._seen_so_far / self._size) * 100
sys.stdout.write(
"\r%s %s / %s (%.2f%%)" % (
self._filename, self._seen_so_far, self._size,
percentage))
sys.stdout.flush()
8.获取bucket下文件列表
object_list = s3_client.list_objects(Bucket='bucket名称')
print object_list
9.查看bucket下的某个文件信息
object_info = s3_client.get_object(Bucket='bucket名称', Key='文件对应的key名称')
print object_info
10.删除文件
object_delete = s3_client.delete_object(Bucket='bucket名称', Key='文件对应的key名称')
11.下载文件
s3_client.download_file("bucket名称", "文件对应的key名称", "文件下载到的地址")
或者
s3client.download_file("bucket名称", "文件对应的key名称", "文件下载到的地址",Callback=DownloadProgressPercentage("文件下载到的地址"))
Callback属性对应的类方法如下，该类方法在控制台中打印了下载文件的进度
class _DownloadProgressPercentage(object):
def __init__(self, filename):
self._filename = filename
self._seen_so_far = 0
self._lock = threading.Lock()
def __call__(self, bytes_amount):
# To simplify we'll assume this is hooked up
# to a single filename.
with self._lock:
self._seen_so_far += bytes_amount
sys.stdout.write(
"\r%s --> %s bytes transferred" % (
self._filename, self._seen_so_far))
sys.stdout.flush()
参考文档
boto3 api官方文档：http://boto3.readthedocs.io/en/latest/guide/quickstart.html
1.s3的client的api
2.s3的client的api上传下载示例
二、3 api使用java版
1.在pom.xml中增加依赖包
<dependency>
<groupId>com.amazonaws</groupId>
<artifactId>aws-java-sdk</artifactId>
<version>1.7.3</version>
</dependency>
2.java中使用s3的api的demo
package org.jenkinsci.plugins.s3_step;
import java.io.BufferedInputStream;
import java.io.BufferedOutputStream;
import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.URL;
import java.security.SecureRandom;
import java.util.ArrayList;
import java.util.List;
import com.amazonaws.ClientConfiguration;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.regions.Region;
import com.amazonaws.regions.Regions;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3Client;
import com.amazonaws.services.s3.model.AbortMultipartUploadRequest;
import com.amazonaws.services.s3.model.Bucket;
import com.amazonaws.services.s3.model.BucketPolicy;
import com.amazonaws.services.s3.model.CannedAccessControlList;
import com.amazonaws.services.s3.model.CompleteMultipartUploadRequest;
import com.amazonaws.services.s3.model.CompleteMultipartUploadResult;
import com.amazonaws.services.s3.model.CopyObjectResult;
import com.amazonaws.services.s3.model.GeneratePresignedUrlRequest;
import com.amazonaws.services.s3.model.GetBucketPolicyRequest;
import com.amazonaws.services.s3.model.GetObjectRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadRequest;
import com.amazonaws.services.s3.model.InitiateMultipartUploadResult;
import com.amazonaws.services.s3.model.ListMultipartUploadsRequest;
import com.amazonaws.services.s3.model.ListObjectsRequest;
import com.amazonaws.services.s3.model.ListPartsRequest;
import com.amazonaws.services.s3.model.MultipartUpload;
import com.amazonaws.services.s3.model.MultipartUploadListing;
import com.amazonaws.services.s3.model.ObjectListing;
import com.amazonaws.services.s3.model.ObjectMetadata;
import com.amazonaws.services.s3.model.PartETag;
import com.amazonaws.services.s3.model.PartListing;
import com.amazonaws.services.s3.model.PartSummary;
import com.amazonaws.services.s3.model.PutObjectRequest;
import com.amazonaws.services.s3.model.PutObjectResult;
import com.amazonaws.services.s3.model.S3Object;
import com.amazonaws.services.s3.model.S3ObjectInputStream;
import com.amazonaws.services.s3.model.S3ObjectSummary;
import com.amazonaws.services.s3.model.UploadPartRequest;
import com.amazonaws.services.s3.model.UploadPartResult;
public class Demo {
AWSCredentials credentials = null;
public AmazonS3 getS3Client(String access_key, String secret_key,
String endpoint) {
ClientConfiguration conf = null;
if (conf == null) {
conf = new ClientConfiguration();
credentials = new BasicAWSCredentials(access_key, secret_key);
}
AmazonS3 s3 = new AmazonS3Client(credentials, conf);
s3.setRegion(Region.getRegion(Regions.CN_NORTH_1));
if (endpoint != null)
s3.setEndpoint(endpoint);
return s3;
}
// ============================ bucket ================================
public void testCreateBucket(AmazonS3 s3Client, String bucketName) {
System.out.println("======================创建 Bucket==========================");
Bucket bucket = s3Client.createBucket(bucketName);
System.out.println(bucket);
System.out.println("创建 Bucket : " + bucket.getName());
System.out.println("======================创建 Bucket==========================");
}
public void testDeleteBucket(AmazonS3 s3Client, String bucketName) {
System.out.println("======================删除 Bucket==========================");
System.out.println("删除 Bucket : " + bucketName);
s3Client.deleteBucket(bucketName);
System.out.println("======================删除 Bucket==========================");
}
public void testDoesBucketExist(AmazonS3 s3Client, String bucketName) {
System.out.println("======================Bucket 是否存在==========================");
boolean bucketExist = s3Client.doesBucketExist(bucketName);
System.out.println(String.format("%s : %s", bucketName, bucketExist));
System.out.println("======================Bucket 是否存在==========================");
}
public void testListBuckets(AmazonS3 s3Client) {
System.out.println("======================枚举 Buckets==========================");
List<Bucket> buckets = s3Client.listBuckets();
for (Bucket bucket : buckets) {
System.out.println(String.format("%s - %s - %s", bucket.getName(), bucket.getOwner(), bucket.getCreationDate()));
}
System.out.println("======================枚举 Buckets==========================");
}
public void testListObjects(AmazonS3 s3Client, String bucketName) {
System.out.println("======================枚举 Objects==========================");
List<String> objectKeys = new ArrayList<String>();
String prefix = "/s3demo/test/";
// for (int i = 0; i < 5; i++) {
// objectKeys.add(prefix + i);
// }
// for (String objectKey : objectKeys) {
// testPutObject(s3Client, bucketName, objectKey);
// }
ObjectListing objects = s3Client.listObjects(bucketName);
List<S3ObjectSummary> s3Objects = objects.getObjectSummaries();
for (S3ObjectSummary s3os : s3Objects) {
System.out.println(String.format("%s - %s - %s", s3os.getKey(), s3os.getETag(), s3os.getSize()));
}
// prefix
ListObjectsRequest listObjReq = new ListObjectsRequest();
listObjReq.setBucketName(bucketName);
listObjReq.setDelimiter("/");
listObjReq.setPrefix(prefix);
ObjectListing preobjects = s3Client.listObjects(listObjReq);
List<String> preobjs = preobjects.getCommonPrefixes();
for (String preobj : preobjs) {
System.out.println(preobj + "\n");
}
System.out.println("======================枚举 Objects==========================");
}
// ============================ bucket ================================
// ============================ object ================================
public void testPutObject(AmazonS3 s3Client, String bucketName, String objectKey,String file_source_address) {
System.out.println("======================上传 object==========================");
s3Client.putObject(new PutObjectRequest(bucketName, objectKey, new File(file_source_address)).withCannedAcl(CannedAccessControlList.PublicRead));
System.out.println("======================上传 object==========================");
}
public void testGetObject(AmazonS3 s3Client, String bucketName,String objectKey) throws Exception {
System.out.println("======================下载 object==========================");
try {
S3Object s3Object = s3Client.getObject(new GetObjectRequest(bucketName,objectKey));
//下载文件到指定地方
InputStream reader = new BufferedInputStream(s3Object.getObjectContent());
File file = new File("localFilename");
FileOutputStream fos = new FileOutputStream(file);
OutputStream writer = new BufferedOutputStream(fos);
int read = -1;
while ( ( read = reader.read() ) != -1 ) {
writer.write(read);
}
writer.flush();
writer.close();
reader.close();
//获取一个request，生成一个下载的url地址，可以在浏览器上下载
GeneratePresignedUrlRequest urlRequest = new GeneratePresignedUrlRequest(bucketName, objectKey);
//生成公用的url
URL url = s3Client.generatePresignedUrl(urlRequest);
System.out.println("=========URL=================" + url + "============URL=============");
if (url == null) {
throw new Exception("can't get s3 file url!");
}
System.out.println("URL=" + url.toString());
} catch (Exception e) {
e.printStackTrace();
}
System.out.println("======================下载 object==========================");
}
public void testDeleteObject(AmazonS3 s3Client, String bucketName,String objectKey) {
System.out.println("======================删除 object==========================");
s3Client.deleteObject(bucketName, objectKey);
System.out.println("======================删除 object==========================");
}
public void testCopyObject(AmazonS3 s3Client, String srcBucketName, String srcObjectKey, String dstBucketName, String dstObjectKey) {
System.out.println("======================复制 object==========================");
CopyObjectResult cpres = s3Client.copyObject(srcBucketName, srcObjectKey, dstBucketName, dstObjectKey);
String text = String.format("test copy objcet %s from %s to %s.%s", srcObjectKey, srcBucketName, dstBucketName, dstObjectKey);
System.out.println(text);
System.out.println("object etag : " + cpres.getETag());
System.out.println("======================复制 objects==========================");
}
public void testMultiUpload(AmazonS3 s3Client, String bucketName, String objectKey) {
System.out.println("======================分块上传==========================");
System.out.println("初始化分块上传");
InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey);
InitiateMultipartUploadResult initMultiUpRes = s3Client.initiateMultipartUpload(initMultiUp);
String uploadId = initMultiUpRes.getUploadId();
System.out.println("分块上传 MultiUpload Id : " + uploadId);
System.out.println("上传块");
List<PartETag> partETags = new ArrayList<PartETag>();
for (int i = 1; i <= 3; i++) {
UploadPartRequest uploadPartReq = new UploadPartRequest();
uploadPartReq.setBucketName(bucketName);
uploadPartReq.setKey(objectKey);
uploadPartReq.setUploadId(uploadId);
uploadPartReq.setPartNumber(i);
byte[] part = new byte[1024 * 1024 * 6]; // 分块上传一个块必须大于等于5M
if (i == 3) {
uploadPartReq.setLastPart(true);
part = new byte[1024 * 1024 * 3]; // 最后一个块可以小于5M
}
SecureRandom random = new SecureRandom();
random.nextBytes(part);
System.out.println("块大小 = " + part.length);
uploadPartReq.setPartSize(part.length);
InputStream inputStream = new ByteArrayInputStream(part);
uploadPartReq.setInputStream(inputStream);
UploadPartResult uploadPartRes = s3Client.uploadPart(uploadPartReq);
PartETag partETag = uploadPartRes.getPartETag();
partETags.add(partETag);
System.out.println(String.format("上传块 %s - %s", partETag.getPartNumber(), partETag.getETag()));
}
CompleteMultipartUploadRequest comMultiReq = new CompleteMultipartUploadRequest(
bucketName, objectKey, uploadId, partETags);
CompleteMultipartUploadResult comMultires = s3Client.completeMultipartUpload(comMultiReq);
System.out.println("完成上传 etag : " + comMultires.getETag());
System.out.println("测试终止上传");
String objectKey1 = "test_abort_multi_upload";
InitiateMultipartUploadRequest initMultiUp1 = new InitiateMultipartUploadRequest(
bucketName, objectKey1);
InitiateMultipartUploadResult initMultiUpRes1 = s3Client
.initiateMultipartUpload(initMultiUp1);
String uploadId1 = initMultiUpRes1.getUploadId();
System.out.println("终止上传 upload id : " + uploadId1);
AbortMultipartUploadRequest abortMultiUp = new AbortMultipartUploadRequest(bucketName, objectKey1, uploadId1);
s3Client.abortMultipartUpload(abortMultiUp);
System.out.println("======================分块上传==========================");
}
public void testListUploadIds(AmazonS3 s3Client, String bucketName) {
System.out.println("======================枚举分块上传uploadId==========================");
String keyPrefix = "/demo/listupids";
for (int i = 1; i < 5; i++) {
String objectKey = keyPrefix + i;
InitiateMultipartUploadRequest initMultiUp = new InitiateMultipartUploadRequest(bucketName, objectKey);
s3Client.initiateMultipartUpload(initMultiUp);
}
ListMultipartUploadsRequest listMultiUpReq = new ListMultipartUploadsRequest(bucketName);
MultipartUploadListing multiUploads = s3Client.listMultipartUploads(listMultiUpReq);
List<MultipartUpload> uploads = multiUploads.getMultipartUploads();
for (MultipartUpload up : uploads) {
System.out.println(String.format("uploadId : %s - key : %s", up.getUploadId(), up.getKey()));
}
System.out.println("======================枚举分块上传uploadId==========================");
}
public void testListMultiParts(AmazonS3 s3Client, String bucketName, String objectkey, String uploadId) {
System.out.println("======================枚举分块上传part==========================");
ListPartsRequest listPartsReq = new ListPartsRequest(bucketName, objectkey, uploadId);
PartListing parts = s3Client.listParts(listPartsReq);
List<PartSummary> ps = parts.getParts();
for (PartSummary p : ps) {
System.out.println(String.format("partNum : %s - partEtag : %s", p.getPartNumber(), p.getETag()));
}
System.out.println("======================枚举分块上传part==========================");
}
public void testIsFileExists(AmazonS3 s3Client, String s3Path,String keyName){
System.out.println("======================判断文件是否已经存在==========================");
try{
boolean flag = false;
S3Object s3Object = s3Client.getObject(s3Path, keyName);
if(s3Object!=null){
flag = true;
}
if(flag){
System.out.println("File exists，filename:"+keyName);
}
}catch(Exception e){
e.printStackTrace();
}
}
// ============================ object ================================
public static void main(String[] args) throws IOException {
String access_key = "s3的access key";
String secret_key = "s3的secret key";
String endpoint = "s3的地址";
Demo d = new Demo();
AmazonS3 s3Client = d.getS3Client(access_key, secret_key, endpoint);
String bucketName = "demo";
String bucketName1 = "demo1";
//创建bucket
// d.testCreateBucket(s3Client, bucketName);
//查询bucket列表
// d.testListBuckets(s3Client);
//判断bucket是否存在
// d.testDoesBucketExist(s3Client, bucketName);
//删除bucket
// d.testDeleteBucket(s3Client, bucketName2);
//上传文件
// String objectKey = "var/lib/pom.xml";
// String objectKey1 = "var/lib/pom.xml";
// String file_source_address = "E:/demo/test1/pom.xml";
// d.testPutObject(s3Client, bucketName, objectKey,file_source_address);
//下载文件
// d.testGetObject(s3Client, bucketName, objectKey);
// //复制文件
// d.testCopyObject(s3Client, bucketName, objectKey, bucketName1, objectKey1);
//bucket下面文件列表
// d.testListObjects(s3Client, bucketName);
//判断文件是否存在
// d.testIsFileExists(s3Client, bucketName, objectKey);
}
}
