参考文档: https://docs.ceph.com/en/latest/radosgw/s3/java/#download-an-object-to-a-file

  1. Java环境:JDK6以上版本
  2. SpringBoot 2.2.5
  3. 存储服务的 accessKey、secreKey 以及 地址:端口

1. 引入依赖

在 pom.xml 文件中添加下列依赖:

<dependency>
    <groupId>com.amazonaws</groupId>
    <artifactId>aws-java-sdk</artifactId>
    <version>1.11.433</version>
</dependency>

2. 初始化

String accessKey = "your-accesskey";
String secretKey = "your-secretKey";
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 conn = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(credentials))
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("endpoint","region"))
                .build();
String accessKey = "TPDDEA5PCT9C8RUPKAWW";
String secretKey = "ryesPMSiSfOzaP1TkFe9TIOBnxpdrA2sw6isFwDZ";
AWSCredentials credentials = new BasicAWSCredentials(accessKey, secretKey);
AmazonS3 conn = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(credentials))
                .withEndpointConfiguration(new   AwsClientBuilder.EndpointConfiguration("http://10.200.100.37:7480","region"))
                .build();
import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
String accessKey = "your-accesskey";
String secretKey = "your-secretKey ";
AWSCredentials credentials = new BasicAWSCredentials(accessKey,secretKey);
ClientConfiguration conf = new ClientConfiguration();
// 设置AmazonS3使用的最大连接数
conf.setMaxConnections(200);
// 设置socket超时时间
conf.setSocketTimeout(10000);
// 设置失败请求重试次数
conf.setMaxErrorRetry(2);
// 如果要用https协议,请加上下面语句
conf.setProtocol(Protocol.HTTPS);
//AmazonS3 s3Client = new AmazonS3Client(credentials,clientConfiguration);
//s3Client.setEndpoint(endPoint);
AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
	            .withCredentials(new AWSStaticCredentialsProvider(credentials))
	            .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration("endpoint","region"))
            .withClientConfiguration(conf)
            .build();//endpoint,region请指定为NOS支持的
 

Amazon3 是线程安全的,因此可以将其注入到 Spring 容器中进行管理。

@AmazonS3Config.java

import com.amazonaws.ClientConfiguration;
import com.amazonaws.Protocol;
import com.amazonaws.auth.AWSCredentials;
import com.amazonaws.auth.AWSStaticCredentialsProvider;
import com.amazonaws.auth.BasicAWSCredentials;
import com.amazonaws.client.builder.AwsClientBuilder;
import com.amazonaws.services.s3.AmazonS3;
import com.amazonaws.services.s3.AmazonS3ClientBuilder;
import com.amazonaws.services.s3.model.CreateBucketRequest;
import com.amazonaws.services.s3.transfer.TransferManager;
import com.amazonaws.services.s3.transfer.TransferManagerBuilder;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
 * @Author: ChangXuan
 * @Decription: 初始化AmazonS3实例
 * @Date: 17:33 2020/6/3
@Configuration
@EnableConfigurationProperties(UploadConfig.class)
public class AmazonS3Config {
    @Autowired
    private UploadConfig uploadConfig;
    @Bean(name = "amazonS3")
    public AmazonS3 getAmazonS3(){
        String accessKey = uploadConfig.getCeph().getAccessKey();
        String secretKey = uploadConfig.getCeph().getSecretKey();
        AWSCredentials credentials = new BasicAWSCredentials(accessKey,secretKey);
        ClientConfiguration conf = new ClientConfiguration();
        // 设置AmazonS3使用的最大连接数
        conf.setMaxConnections(uploadConfig.getCeph().getAmazonS3MaxConnections());
        // 设置socket超时时间
        conf.setSocketTimeout(uploadConfig.getCeph().getAmazonS3SocketTimeout());
        // 设置失败请求重试次数
        conf.setMaxErrorRetry(uploadConfig.getCeph().getAmazonS3MaxErrorRetry());
        // 设置协议
        if (!"blank".equals(uploadConfig.getCeph().getAmazonS3Protocol())){
            switch (uploadConfig.getCeph().getAmazonS3Protocol()){
                case "https":
                    conf.setProtocol(Protocol.HTTPS);
                    break;
                case "http":
                    conf.setProtocol(Protocol.HTTP);
                    break;
                default:
                    break;
        AmazonS3 s3Client = AmazonS3ClientBuilder.standard()
                .withCredentials(new AWSStaticCredentialsProvider(credentials))
                .withEndpointConfiguration(new AwsClientBuilder.EndpointConfiguration(uploadConfig.getCeph().getHosts().get(0),uploadConfig.getCeph().getRegion()))
                .withClientConfiguration(conf)
                .build();
        checkAndCreateBucket(s3Client);
        return s3Client;
    @Bean(name = "transferManager")
    public TransferManager getTransferManager(){
        return TransferManagerBuilder.standard().withS3Client(getAmazonS3()).build();
     * 检查桶是否存在,不存在则创建创建
     * @param s3Client
    private void checkAndCreateBucket(AmazonS3 s3Client){
        boolean exists = s3Client.doesBucketExistV2(uploadConfig.getCeph().getBucketName());
        if (!exists){
            CreateBucketRequest request = new CreateBucketRequest(uploadConfig.getCeph().getBucketName());
            s3Client.createBucket(request);

@UploadConfig.java

import lombok.Data;
import org.springframework.boot.context.properties.ConfigurationProperties;
 * @Author: ChangXuan
 * @Decription: 上传 配置
 * @Date: 17:25 2020/6/3
@ConfigurationProperties(prefix = "upload")
@Data
public class UploadConfig {
    private CephConfig ceph;

@CephConfig.java

import com.google.common.collect.Lists;
import lombok.Data;
import java.util.List;
 * @Author: ChangXuan
 * @Decription:配置
 * @Date: 17:28 2020/6/3
@Data
public class CephConfig {
     * 集群ip:port
    private List<String> hosts = Lists.newArrayList();
    private String accessKey;
    private String secretKey;
    private String region;
     * 连接协议
    private String amazonS3Protocol;
     * 失败请求重试次数
    private Integer amazonS3MaxErrorRetry;
     * 超时时间
    private Integer amazonS3SocketTimeout;
     * 最大连接数
    private Integer amazonS3MaxConnections;
     * 桶名称
    private String bucketName;

@application.yml

upload:
  ceph:
    bucketName: ic-storage-dev
    region: region
    hosts:
      - "10.200.100.37:7480"
      - "10.200.100.38:7480"
      - "10.200.100.39:7480"
    accessKey: TPDDEA5PCT9C8RUPKAWW
    secretKey: ryesPMSiSfOzaP1TkFe9TIWQnxpdrA2sw6isFwDZ
    amazonS3MaxConnections: 200
    amazonS3SocketTimeout: 10000
    amazonS3MaxErrorRetry: 2
    amazonS3Protocol: http

3. 依赖注入

@Autowired
private AmazonS3 amazonS3;
@Autowired
private TransferManager transferManager;

4. 文件操作

1. 直接内容上传

//要上传文件内容
String content = "Object content";
try {
   amazonS3.putObject("your-bucketname","your-objectname",content);
}catch (Exception e){
   System.out.println(e.getMessage());

2. 本地文件普通上传

对于小对象可以使用putObject接口进行上传,putObject上传支持的最大文件大小为100M,如果上传大于100M的文件需要使用分块上传。本地文件普通上传的示例代码如下:

//要上传文件的路径
String filePath = "your-local-file-path";
try {
   amazonS3.putObject("your-bucketname","your-objectname", new File(filePath));
}catch (Exception e){
   System.out.println(e.getMessage());

3. 上传文件时设置文件元数据信息

String filePath = "your-local-file-path";
ObjectMetadata objectMetadata = new ObjectMetadata();
//设置Content-Type
objectMetadata.setContentType("application/xml");
//设置标准http消息头(元数据)
objectMetadata.setHeader("Cache-Control", "no-cache");
//设置用户自定义元数据信息
Map<String, String> userMeta = new HashMap<String, String>();
userMeta.put("ud", "test");
objectMetadata.setUserMetadata(userMeta);
PutObjectRequest putObjectRequest = new PutObjectRequest("your-bucketname","your-objectname", new File(filePath));
putObjectRequest.setMetadata(objectMetadata);
amazonS3.putObject(putObjectRequest);

4. 流式上传

try {
   ObjectMetadata objectMetadata = new ObjectMetadata();
   //设置流的长度,您还可以设置其他文件元数据信息
   objectMetadata.setContentLength(streamLength);
   amazonS3.putObject("your-bucketname","your-objectname", inputStream, objectMetadata)
}catch (Exception e){
   System.out.println(e.getMessage());

1. 流式下载

S3Object fileObject = amazonS3.getObject("your-bucketname","your-objectname");
//可以通过getObjectMetadata方法获取对象的ContentType等元数据信息
String contentType = fileObject.getObjectMetadata().getContentType();
//流式获取文件内容
InputStream in = fileObject.getObjectContent();
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
while (true) {
  String line;
  try {
    line = reader.readLine();
    if (line == null) break;
    System.out.println("\n" + line);
  } catch (IOException e) {
    e.printStackTrace();
try {
  reader.close();
} catch (IOException e) {
  e.printStackTrace();

2. 下载到本地文件

String destinationFile = "your-local-filepath";
GetObjectRequest getObjectRequest = new GetObjectRequest("your-bucketname","your-objectname");
ObjectMetadata objectMetadata = amazonS3.getObject(getObjectRequest, new File(destinationFile));

3. Range 下载

GetObjectRequest getObjectRequest = new GetObjectRequest("your-bucketname","your-objectname");
getObjectRequest.setRange(0, 100);
S3Object nosObject = amazonS3.getObject(getObjectRequest);
BufferedReader reader = new BufferedReader(new InputStreamReader(in));
while (true) {
  String line;
  try {
    line = reader.readLine();
    if (line == null) break;
      System.out.println("\n" + line);
  } catch (IOException e) {
    e.printStackTrace();
try {
  reader.close();
} catch (IOException e) {
  e.printStackTrace();

判断文件是否存在

您可以通过AmazonS3.doesObjectExist判断文件是否存在。

boolean isExist = amazonS3.doesObjectExist("your-bucketname","your-objectname");

您可以通过AmazonS3.deleteObject删除单个文件

amazonS3.deleteObject("your-bucketname","your-objectname");

您还可以通过AmazonS3.deleteObjects一次删除多个文件

try {
    DeleteObjectsResult result = amazonS3.deleteObjects(deleteObjectsRequest);
    List<DeletedObject>  deleteObjects = result.getDeletedObjects();
    //print the delete results
    for (DeletedObject items: deleteObjects){
      System.out.println(items.getKey());
// 部分对象删除失败
} catch (MultiObjectDeleteException e) { 
    List<DeleteError> deleteErrors = e.getErrors();
    for (DeleteError error : deleteErrors) {
        System.out.println(error.getKey());
} catch (AmazonServiceException  e) {
        //捕捉服务器异常错误
} catch (AmazonClientException ace) {
       //捕捉客户端错误

获取文件元数据信息

您可以通过AmazonS3.getObjectMetadata获取文件元数据信息

amazonS3.getObjectMetadata("your-bucketname","your-objectname");

文件复制(copy)

您可以通过AmazonS3.copyObject接口实现文件拷贝功能。

amazonS3.copyObject("source-bucket", "source-object", "dst-bucket", "dst-object");

列举桶内文件

您可以通过AmazonS3.listObjects列出桶里的文件。listObjects接口如果调用成功,会返回一个ObjectListing对象,列举的结果保持在该对象中。

ObjectListing的具体信息如下表所示:

方法含义
List getObjectSummaries()返回的文件列表(包含文件的名称、Etag的元数据信息)
String getPrefix()本次查询的文件名前缀
String getDelimiter()文件分界符
String getMarker()这次List Objects的起点
int getMaxKeys()响应请求内返回结果的最大数目
String getNextMarker()下一次List Object的起点
boolean isTruncated()是否截断,如果因为设置了limit导致不是所有的数据集都返回,则该值设置为true
List getCommonPrefixes()如果请求中指定了delimiter参数,则返回的包含CommonPrefixes元素。该元素标明以delimiter结尾,并有共同前缀的对象的集合

AmazonS3.listObjects接口提供两种调用方式:简单列举、通过ListObjectsRequest列举

简单列举只需指定需要列举的桶名,最多返回100条对象记录,建议桶内对象数较少时(小于100)使用。

ObjectListing objectListing = amazonS3.listObjects("your-bucketname");
List<S3ObjectSummary> sums = objectListing.getObjectSummaries();
for (S3ObjectSummary s : sums) {
  System.out.println("\t" + s.getKey());

通过ListObjectsRequest列举

您还可以通过设置ListObjectsReques参数实现各种灵活的查询功能。ListObjectsReques的可设置的参数如下:

设置方法作用
setPrefix(String prefix)限定返回的object key必须以prefix作为前缀
setDelimiter(String delimiter)是一个用于对Object名字进行分组的字符。所有名字包含指定的前缀且第一次出现delimiter字符之间的object作为一组元素——CommonPrefixes
setMarker(String marker)字典序的起始标记,只列出该标记之后的部分
setMaxKeys(Integer maxKeys)限定返回的数量,返回的结果小于或等于该值(默认值为100)

1、分页列举桶内的所有文件:

List<S3ObjectSummary> listResult = new ArrayList<S3ObjectSummary>();
ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName("your-bucketname");
listObjectsRequest.setMaxKeys(50);
ObjectListing listObjects = amazonS3.listObjects(listObjectsRequest);
  listResult.addAll(listObjects.getObjectSummaries());
  if (listObjects.isTruncated()) {
    ListObjectsRequest request = new ListObjectsRequest();
    request.setBucketName(listObjectsRequest.getBucketName());
    request.setMarker(listObjects.getNextMarker());
    listObjects =  amazonS3.listObjects(request);
  } else {
    break;
} while (listObjects != null);

2、使用Delimiter模拟文件夹功能

假设桶内有如下对象:a/1.jpg、a/2.jpg、a/b/1.txt、a/b/2.txt,列举a文件夹下的文件及子文件夹的示例代码如下:

ListObjectsRequest listObjectsRequest = new ListObjectsRequest();
listObjectsRequest.setBucketName("your-bucketname");
listObjectsRequest.setDelimiter("/");
listObjectsRequest.setPrefix("a/");
ObjectListing listing = amazonS3.listObjects(listObjectsRequest);
// 遍历所有Object
System.out.println("Objects:");
for (S3ObjectSummary objectSummary : listing.getObjectSummaries()) {
    System.out.println(objectSummary.getKey());
// 遍历所有CommonPrefix
System.out.println("CommonPrefixs:");
for (String commonPrefix : listing.getCommonPrefixes()) {
    System.out.println(commonPrefix);

示例代码的输出如下:

Objects:

a/1.jpg

a/2.jpg

CommonPrefixs:

生成私有对象可下载的URL链接

AWS Java SDK支持生成可下载私有对象的URL连接,您可以将该链接提供给第三方进行文件下载:

GeneratePresignedUrlRequest generatePresignedUrlRequest = new GeneratePresignedUrlRequest(bucketName, key);
// 设置可下载URL的过期时间为1天后
generatePresignedUrlRequest.setExpiration(new Date(System.currentTimeMillis()+3600*1000*24));
URL url = amazonS3.generatePresignedUrl(generatePresignedUrlRequest);//生成URL
System.out.println(url);//可以用这个url来下载文件

文件上传下载工具类 TransferManager

前文提到的是 Java SDK提供的基础接口,为方便用户进行文件上传下载,Java SDK提供了封装更好、使用更方便的工具类:TransferManager。

TransferManager的初始化

注:在 SpringBoot 项目中 TransferManager 的初始化 参考上述 [配置模式](#### 配置模式)

//先实例化一个AmazonS3
String accessKey = "your-accesskey";
String secretKey = "your-secretKey ";
Credentials credentials = new BasicCredentials(accessKey, secretKey);
AmazonS3 amazonS3 = new AmazonS3(credentials);
amazonS3.setEndpoint(endPoint);
//然后通过AmazonS3对象来初始化TransferManager
TransferManager transferManager = new TransferManager(amazonS3);
Download download = transferManager.download(TestConfig.bucketName,key,new File("localFilePath"));
try {
    download.waitForCompletion();
} catch (InterruptedException e) {
    e.printStackTrace();

使用TransferManager进行文件上传

TransferManager会根据文件大小,选择是否进行分块上传。当文件小于等于16M时,TransferManager会自动调用PutObject接口,否则TransferManager会自动对文件进行分块上传。

1、上传本地文件:

如果指定上传的本地文件大于16M,TransferManager会自动对文件进行分块,并发调用分块上传接口进行上传,大大提高上传文件的速度。

//上传文件
Upload upload = transferManager.upload("your-bucketname", "your-objectname", new File("your-file"));
try {
    upload.waitForUploadResult();
} catch (InterruptedException e) {
    e.printStackTrace();

2、流式上传:

您也可以使用TransferManager进行流式上传,但是相比本地文件上传,流式上传无法做到多个分块并发上传,只能一个分块一个分块顺序上传。

//流式上传文件
ObjectMetadata objectMetadata = new ObjectMetadata();
objectMetadata.setContentLength(file.length());
Upload upload = transferManager.upload("your-bucketname", "your-objectname", inputStream, objectMetadata);
UploadResult  result = upload.waitForUploadResult();

3、上传目录

您可以使用TransferManager将某个目录下的文件全部上传到NOS,对象名即文件名

  • 3.1 不支持多级目录
MultipleFileUpload result = transferManager.uploadDirectory("your-buckename", null, new File("dirPath"), false);
result.waitForCompletion();
  • 3.2 支持多级目录,会递归的上传目录下的所有文件
MultipleFileUpload result = transferManager.uploadDirectory("your-buckename", null, new File("dirPath"), true);
result.waitForCompletion();

4、下载文件

File file = new  File("your-destFile");
Download download = transferManager.download("your-bucketname", "your-objectname", file);
download.waitForCompletion();

[1] 网易数帆S3 Java SDK 手册

参考文档:https://docs.ceph.com/en/latest/radosgw/s3/java/#download-an-object-to-a-file环境准备Java环境:JDK6以上版本 SpringBoot 2.2.5 存储服务的 accessKey、secreKey 以及 地址:端口使用1. 引入依赖在pom.xml文件中添加下列依赖:&lt;dependency&gt; &lt;groupId&gt;com.amazonaws&lt;/group.. <groupId>com.amazonaws</groupId> <artifactId>aws-java-sdk-s3</artifactId> <version>1.11.625</version> </dependency&g... import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.a... import com.amazonaws.AmazonServiceException; import com.amazonaws.SdkClientException; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.client.builder.AwsClie... CREATE TABLE `brand` ( `id` int(11) NOT NULL AUTO_INCREMENT COMMENT '品牌id', `name` varchar(100) NOT NULL COMMENT '品牌名称', `image` varchar(1000) DEFAULT '' COMMENT '品牌图片地址', `initial` varchar(1) DEFAULT '' COMMENT '品牌的首字母', import com.amazonaws.ClientConfiguration; import com.amazonaws.Protocol; import com.amazonaws.auth.AWSCredentials; import com.amazonaws.auth.AWSStaticCredentialsProvider; import com.amazonaws.auth.BasicAWSCredentials; import com.amazonaws.servi. 一、安装ActiveMQ 具体的安装步骤,请参考我的另一篇博文: http://blog.csdn.net/liuchuanhong1/article/details/52057711 二、新建spring boot工程,并加入JMS(ActiveMQ)依赖 三、工程结构 pom依赖如下: [html] view pla...
Amazon S3:https://amazonaws-china.com/cn/s3/faqs/        Amazon S3 是一种对象存储,专为从 Internet 上的任意位置存储和检索任意数量的数据而构建。它是一种简单的存储服务,以非常低的成本提供极其持久、高度可用并可无限扩展的数据存储基础设施。        Amazon S3 提供一个简单的 Web 服务界面,使用该界面,您...
function uploadFile(file) { const CHUNK_SIZE = 1024 * 1024 * 5; // 5MB const bucketName = 'my-bucket'; const s3 = new AWS.S3(); const fileSize = file.size; const fileName = file.name; const uploadId = localStorage.getItem('uploadId') || null; const parts = []; let loaded = 0; if (!uploadId) { // 新建上传任务 s3.createMultipartUpload({ Bucket: bucketName, Key: fileName, }, (err, data) => { if (err) { console.error(err); return; uploadId = data.UploadId; localStorage.setItem('uploadId', uploadId); uploadPart(1, 0); } else { // 恢复上传任务 s3.listParts({ Bucket: bucketName, Key: fileName, UploadId: uploadId, }, (err, data) => { if (err) { console.error(err); return; data.Parts.forEach((part) => { parts[part.PartNumber - 1] = { ETag: part.ETag, PartNumber: part.PartNumber, loaded += part.Size; uploadPart(parts.length + 1, loaded); function uploadPart(partNumber, start) { const end = Math.min(start + CHUNK_SIZE, fileSize); const chunk = file.slice(start, end); const params = { Body: chunk, Bucket: bucketName, Key: fileName, PartNumber: partNumber, UploadId: uploadId, s3.uploadPart(params, (err, data) => { if (err) { console.error(err); return; parts[partNumber - 1] = { ETag: data.ETag, PartNumber: partNumber, loaded += chunk.size; if (end === fileSize) { completeUpload(); } else { uploadPart(partNumber + 1, end); function completeUpload() { s3.completeMultipartUpload({ Bucket: bucketName, Key: fileName, UploadId: uploadId, MultipartUpload: { Parts: parts, }, (err, data) => { if (err) { console.error(err); return; localStorage.removeItem('uploadId'); console.log(data); 后端代码: ```java @RestController public class S3Controller { private final AmazonS3 s3Client; @Autowired public S3Controller(AmazonS3 s3Client) { this.s3Client = s3Client; @PostMapping("/upload") public void upload(@RequestParam String fileName, @RequestParam String uploadId, @RequestParam(required = false) Integer partNumber, @RequestParam(required = false) String eTag, @RequestParam(required = false) MultipartFile file) throws IOException { if (file != null && !file.isEmpty()) { s3Client.uploadPart(new UploadPartRequest() .withBucketName("my-bucket") .withKey(fileName) .withPartNumber(partNumber) .withUploadId(uploadId) .withInputStream(file.getInputStream()) .withPartSize(file.getSize())); } else { s3Client.completeMultipartUpload(new CompleteMultipartUploadRequest() .withBucketName("my-bucket") .withKey(fileName) .withUploadId(uploadId) .withPartETags(Collections.singletonList(new PartETag(partNumber, eTag)))); @PostMapping("/create") public String create(@RequestParam String fileName) { InitiateMultipartUploadResult result = s3Client.initiateMultipartUpload(new InitiateMultipartUploadRequest("my-bucket", fileName)); return result.getUploadId(); @GetMapping("/list") public List<MultipartUpload> list() { return s3Client.listMultipartUploads(new ListMultipartUploadsRequest("my-bucket")).getMultipartUploads(); 在这个示例代码中,我们使用AWS SDK for Java操作Amazon S3服务。前端代码负责将文件分成多个部分进行上传,后端代码负责接收并存储每个部分,并在最后将所有部分合并成一个完整的文件。通过使用AWS SDK提供的分块上传接口,我们可以轻松地实现断点续传功能。