AsyncHttpClient And Download Speed Limit
AsyncHttpClient
Official repository and docs: https://github.com/AsyncHttpClient/async-http-client
Maven Dependency
Check the latest version of async-http-client at https://mvnrepository.com/artifact/org.asynchttpclient/async-http-client
<dependency>
<groupId>org.asynchttpclient</groupId>
<artifactId>async-http-client</artifactId>
<version>2.12.3</version>
</dependency>
Usage
Basic Usage
AsyncHttpClient provides 2 APIs for defining requests: bound and unbound. AsyncHttpClient and Dsl` provide methods for standard HTTP methods (POST, PUT, etc)
// bound, define request inline with execute()
Future<Response> whenResponse=asyncHttpClient.prepareGet("http://www.example.com/").execute();
// unbound, define request and execute separately
Request request=get("http://www.example.com/").build();
Future<Response> whenResponse=asyncHttpClient.executeRequest(request);
Client Configuration
Instead of default configuration, create a customized one with Dsl.config()
AsyncHttpClientConfig config = Dsl.config()
.setConnectTimeout(CONN_TIMEOUT)
.setRequestTimeout(REQ_TIMEOUT)
.setMaxRequestRetry(100)
.build();
AsyncHttpClient client = Dsl.asyncHttpClient(config);
Download To File
The default implementation accumulates the HTTP chunks received into an ArrayList. This could lead to high memory consumption, or an OutOfMemory exception when trying to download a large file. Use a FileChannel to write the bytes to our local file directly. We'll use the getBodyByteBuffer() method to access the body part content through a ByteBuffer.
// Open the file, enable append(not necessary in this example)
FileOutputStream os = new FileOutputStream(LOCAL_FILE, true);
// Create a default client
AsyncHttpClient client = Dsl.asyncHttpClient();
// Use Future to block till download complete, just for demostration
ListenableFuture<Response> whenResponse = client.prepareGet(FILE_URL).execute(new AsyncCompletionHandler<Response>() {
@Override
public State onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
os.getChannel().write(bodyPart.getBodyByteBuffer());
return State.CONTINUE;
}
@Override
public Response onCompleted(Response response) throws Exception {
return response;
}
});
Response response=whenResponse.get();
// Thread will never exit if client is not closed
client.close();
You can get the bodyPart length and calculate the totoally received length
private int totalLength;
@Override
public State onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
os.getChannel().write(bodyPart.getBodyByteBuffer());
receivedLength += bodyPart.length();
System.out.println(receivedLength);
if (bodyPart.isLast())
{
System.out.println("last");
return State.ABORT;
}
return State.CONTINUE;
}
Range Requests, Partial Download
You can modify the request header to download part of the file, as long as the server supports range request.
Response DefaultHttpResponse(decodeResult: success, version: HTTP/1.1)
HTTP/1.1 200 OK
Server: openresty
Date: Sun, 04 Dec 2022 13:35:21 GMT
Content-Type: application/octet-stream
Last-Modified: Thu, 21 Apr 2022 17:16:39 GMT
Connection: keep-alive
ETag: "62619177-1b58d5"
Accept-Ranges: bytes <--- this means the server supports range requests
content-length: 1792213
In Java code, create the request like
FileOutputStream os = new FileOutputStream(localFilePath, true);
Request request = Dsl.get(remoteUrl).setHeader(HttpHeaderNames.RANGE, "bytes="+offset+"-"+(offset + length - 1)).build();
ListenableFuture<FragmentResponse> whenResponse = client.executeRequest(request, new AsyncCompletionHandler<>() {
//...
});
You will get the response header as
Response DefaultHttpResponse(decodeResult: success, version: HTTP/1.1)
HTTP/1.1 206 Partial Content
Server: openresty
Date: Sun, 04 Dec 2022 13:27:14 GMT
Content-Type: application/octet-stream
Last-Modified: Fri, 15 Oct 2021 12:25:23 GMT
Connection: keep-alive
ETag: "61697333-268f48e"
X-RateLimit-Byte-Rate: 67108864
Content-Range: bytes 38797312-39845887/40432782
content-length: 1048576
Range Boundary
According to Hypertext Transfer Protocol (HTTP/1.1): Range Requests, the range boundaries are inclusive - inclusive. Examples of byte-ranges-specifier values:
- The first 500 bytes (byte offsets 0-499, inclusive): bytes=0-499
- The second 500 bytes (byte offsets 500-999, inclusive): bytes=500-999
Example Code Of Download Speed Limit
This is an example of controlling download speed by splitting file into fragments -- and download them piece by piece.
public class FragmentableDownloader {
Logger log = LoggerFactory.getLogger(FragmentableDownloader.class);
public static final int CONN_TIMEOUT = 60000;
public static final int REQ_TIMEOUT = 60000;
private AsyncHttpClient client;
/**
* limit bytes per second
*/
private long rateLimit;
/**
* size of each fragment
*/
private long fragmentSize;
private String remoteUrl;
private String localFilePath;
private long fileLength;
private long timestamp;
private long interval;
public FragmentableDownloader(long rateLimit, long fragmentSize, String remoteUrl, String localFilePath) {
this.rateLimit = rateLimit;
this.fragmentSize = fragmentSize;
this.remoteUrl = remoteUrl;
this.localFilePath = localFilePath;
AsyncHttpClientConfig config = Dsl.config()
.setConnectTimeout(CONN_TIMEOUT)
.setRequestTimeout(REQ_TIMEOUT)
.setMaxRequestRetry(100)
.build();
this.client = Dsl.asyncHttpClient(config);
interval = fragmentSize / rateLimit;
}
public FragmentResponse downloadFragment(long offset, long length) throws Exception {
FileOutputStream os = new FileOutputStream(localFilePath, true);
Request request = Dsl.get(remoteUrl).setHeader(HttpHeaderNames.RANGE, "bytes="+offset+"-"+(offset + length - 1)).build();
ListenableFuture<FragmentResponse> whenResponse = client.executeRequest(request, new AsyncCompletionHandler<>() {
private long totalLength;
private int byteTransferred = 0;
@Override
public State onHeadersReceived(HttpHeaders headers) throws Exception {
String length = headers.get(HttpHeaderNames.CONTENT_RANGE);
int pos = length.lastIndexOf('/');
length = length.substring(pos + 1);
this.totalLength = Long.parseLong(length);
return State.CONTINUE;
}
@Override
public State onBodyPartReceived(HttpResponseBodyPart bodyPart) throws Exception {
os.getChannel().write(bodyPart.getBodyByteBuffer());
byteTransferred += bodyPart.length();
//log.info("byteTransferred: {}", byteTransferred);
return State.CONTINUE;
}
@Override
public FragmentResponse onCompleted(Response response) throws Exception {
log.info("complete");
return new FragmentResponse(response.getStatusCode(), totalLength);
}
});
return whenResponse.get();
}
public void download() throws Exception {
Files.deleteIfExists(Path.of(localFilePath));
long offset = 0;
FragmentResponse response = downloadFragment(0, fragmentSize);
offset += fragmentSize;
this.fileLength = response.getTotalLength();
if (this.fileLength <= fragmentSize) {
return;
}
while (offset < fileLength - 1) {
log.info("Offset: {}", offset);
timestamp = System.currentTimeMillis();
response = downloadFragment(offset, fragmentSize);
offset += fragmentSize;
long duration = System.currentTimeMillis() - timestamp;
log.info("file:{}, offset:{}, response: {}, speed:{}", fileLength, offset, response.getStatus(), fragmentSize / duration);
long wait = interval * 1000L - duration;
if (wait > 0) {
log.info("Sleep {} milliseconds for rate limit", wait);
Thread.sleep(wait);
}
}
log.info("Download finished");
client.close();
}
public static void main(String[] args) throws Exception {
String url = "https://mirrors.ustc.edu.cn/ubuntu/dists/jammy/main/signed/linux-amd64/5.13.0-19.19/signed.tar.gz";
String path = "/home/milton/Downloads/signed.tar.gz";
long rateLimit = 500 * 1024L;
long fragmentSize = 10 * 1024 * 1024L;
FragmentableDownloader downloader = new FragmentableDownloader(rateLimit, fragmentSize, url, path);
downloader.download();
}
public static class FragmentResponse {
private int status;
private long totalLength;
public FragmentResponse(int status, long totalLength) {
this.status = status;
this.totalLength = totalLength;
}
public int getStatus() {
return status;
}
public long getTotalLength() {
return totalLength;
}
}
}
