Skip to content

增强 BMCLAPI 低速节点重试与断点续传 #7023

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
wyjh233 wants to merge 2 commits into
base: main
Choose a base branch
from
+678 −35
Open

增强 BMCLAPI 低速节点重试与断点续传 #7023

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
341bfb4
增强 BMCLAPI 低速节点重试与断点续传
wyjh233 Oct 20, 2026
910cb02
修复跨节点续传并按剩余下载时间判断低速
wyjh233 Oct 20, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 5 additions & 2 deletions HMCLCore/src/main/java/org/jackhuang/hmcl/task/CacheFileTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -146,6 +146,7 @@ protected Context getContext(@Nullable UrlResponseInfo response, boolean checkET
throw new IOException("Remote response metadata is unavailable");
}

@Nullable String checksum = expectedSha1 != null ? expectedSha1 : bmclapiHash;
return new Context() {
private final Path temp = Files.createTempFile("hmcl-download-", null);
private final FileChannel fileOutput = FileChannel.open(temp,
Expand Down Expand Up @@ -183,9 +184,11 @@ public void close() throws IOException {
}

try {
if (expectedSha1 != null) {
if (checksum != null) {
ChecksumMismatchException.verifyChecksum(
temp, CacheRepository.SHA1, expectedSha1);
temp, CacheRepository.SHA1, checksum);
}
if (expectedSha1 != null) {
setResult(repository.cacheFile(
temp, CacheRepository.SHA1, expectedSha1));
} else {
Expand Down
137 changes: 104 additions & 33 deletions HMCLCore/src/main/java/org/jackhuang/hmcl/task/FetchTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@
import java.io.*;
import java.net.HttpURLConnection;
import java.net.MalformedURLException;
import java.net.SocketTimeoutException;
import java.net.URLConnection;
import java.nio.file.Path;
import java.time.Duration;
Expand Down Expand Up @@ -83,6 +84,7 @@ private Context getContext() throws IOException {
return getContext(null, false, null);
}

/// Creates a sink that uses the BMCLAPI hash unless an explicit integrity check takes precedence.
protected abstract Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) throws IOException;

@Override
Expand Down Expand Up @@ -142,17 +144,15 @@ public void execute() throws Exception {
}
}

/// Retains byte ranges and file validators independently of the download sink.
private static final class HttpResumeContext {
private static final Pattern CONTENT_RANGE_PATTERN = Pattern.compile("bytes ([0-9]+)-([0-9]+)/([0-9]+)");

static @Nullable FetchTask.HttpResumeContext of(UrlResponseInfo response) throws IOException {
/// Creates resume state for an uncompressed response with a known length and a file validator.
static @Nullable FetchTask.HttpResumeContext of(UrlResponseInfo response, @Nullable String bmclapiHash) throws IOException {
if (response.responseCode() != HttpURLConnection.HTTP_OK)
return null;

boolean acceptRanges = response.headers().firstValue("accept-ranges").orElse("").equalsIgnoreCase("bytes");
if (!acceptRanges)
return null;

var contentEncoding = ContentEncoding.fromHeaders(response.headers());
if (contentEncoding != ContentEncoding.IDENTITY)
return null;
Expand All @@ -161,13 +161,13 @@ private static final class HttpResumeContext {
if (contentLength < 0)
return null;

String eTag = response.headers().firstValue("etag").orElse(null);
String strongETag = isStrongETag(eTag) ? eTag : null;
String lastModified = response.headers().firstValue("last-modified").orElse(null);
if (strongETag == null && StringUtils.isBlank(lastModified))
@Nullable String eTag = response.headers().firstValue("etag").orElse(null);
@Nullable String strongETag = isStrongETag(eTag) ? eTag : null;
@Nullable String lastModified = response.headers().firstValue("last-modified").orElse(null);
if (bmclapiHash == null && strongETag == null && StringUtils.isBlank(lastModified))
return null;

return new HttpResumeContext(response.url(), contentLength, strongETag, lastModified);
return new HttpResumeContext(response.url(), contentLength, strongETag, lastModified, bmclapiHash);
}

/// Final response URL whose partial content is retained for resumption.
Expand All @@ -176,25 +176,34 @@ private static final class HttpResumeContext {
private final @Nullable String strongETag;
private final @Nullable String lastModified;

/// SHA-1 supplied by the redirect chain, which must match when resuming from another node.
private final @Nullable String bmclapiHash;

long countUncompressed;

/// Records the response URL, total byte length, and available validators for resumption.
private HttpResumeContext(WebURL url, long contentLength, @Nullable String strongETag, @Nullable String lastModified) {
/// Records the response URL, total length, and validators for resumption.
private HttpResumeContext(WebURL url, long contentLength, @Nullable String strongETag, @Nullable String lastModified,
@Nullable String bmclapiHash) {
this.url = url;
this.contentLength = contentLength;
this.strongETag = strongETag;
this.lastModified = lastModified;
this.bmclapiHash = bmclapiHash;
}

private static boolean isStrongETag(@Nullable String eTag) {
return StringUtils.isNotBlank(eTag) && !eTag.regionMatches(true, 0, "W/", 0, 2);
}

String ifRange() {
/// Uses content hashes for BMCLAPI resumption instead of sending another node's HTTP validator.
@Nullable String ifRange() {
if (bmclapiHash != null)
return null;
return strongETag != null ? strongETag : Objects.requireNonNull(lastModified);
}

boolean canResume(int statusCode, UrlResponseInfo response) throws IOException {
/// Validates the response range, total length, and file identity before retaining partial content.
boolean canResume(int statusCode, UrlResponseInfo response, @Nullable String bmclapiHash) throws IOException {
if (statusCode != HttpURLConnection.HTTP_PARTIAL)
return false;

Expand All @@ -203,11 +212,14 @@ boolean canResume(int statusCode, UrlResponseInfo response) throws IOException {
return false;

long contentLength = response.headers().firstValueAsLong("content-length").orElse(-1L);
if (this.contentLength != contentLength + this.countUncompressed)
if (contentLength >= 0 && this.contentLength - this.countUncompressed != contentLength)
return false;

if (strongETag != null) {
String eTag = response.headers().firstValue("etag").orElse(null);
if (this.bmclapiHash != null) {
if (!this.bmclapiHash.equalsIgnoreCase(bmclapiHash))
return false;
} else if (strongETag != null) {
@Nullable String eTag = response.headers().firstValue("etag").orElse(null);
if (!strongETag.equals(eTag))
return false;
} else {
Expand All @@ -229,10 +241,7 @@ boolean canResume(int statusCode, UrlResponseInfo response) throws IOException {
long end = Long.parseLong(matcher.group(2));
long total = Long.parseLong(matcher.group(3));

if (start != countUncompressed || end < start || total != this.contentLength)
return false;

return end - start + 1 == contentLength;
return start == countUncompressed && total == this.contentLength && end == total - 1;
} catch (NumberFormatException e) {
return false;
}
Expand All @@ -243,14 +252,17 @@ boolean hasPartialContent() {
}
}

/// Reads the response body, retaining resumable fragments and applying the optional node retry policy.
private void download(Context context,
@Nullable FetchTask.HttpResumeContext resume, InputStream inputStream,
long contentLength,
ContentEncoding contentEncoding) throws IOException, InterruptedException {
ContentEncoding contentEncoding, @Nullable BMCLAPIRetryPolicy retryPolicy) throws IOException, InterruptedException {
boolean success;
try (var counter = new CounterInputStream(inputStream);
var input = contentEncoding.wrap(counter)) {
long lastDownloaded = 0L;
if (retryPolicy != null)
retryPolicy.startAttempt(System.nanoTime());
byte[] buffer = new byte[IOUtils.DEFAULT_BUFFER_SIZE];
while (true) {
if (isCancelled()) break;
Expand All @@ -268,13 +280,18 @@ private void download(Context context,
if (resume != null)
resume.countUncompressed += len;

if (contentLength >= 0) {
if (resume != null) {
updateProgress(resume.countUncompressed, resume.contentLength);
} else if (contentLength >= 0) {
// Update progress information per second
updateProgress(counter.downloaded, contentLength);
}

updateDownloadSpeed(counter.downloaded - lastDownloaded);
lastDownloaded = counter.downloaded;

if (retryPolicy != null && retryPolicy.shouldRetry(counter.downloaded, contentLength, System.nanoTime()))
throw new SocketTimeoutException("BMCLAPI node is too slow to complete the remaining download promptly");
}

if (isCancelled())
Expand Down Expand Up @@ -308,6 +325,7 @@ private void downloadHttp(WebURL url, int retry, Duration connectTimeout, Durati

@Nullable Context context = null;
@Nullable HttpResumeContext resumeContext = null;
BMCLAPIRetryPolicy retryPolicy = new BMCLAPIRetryPolicy();

@Nullable ArrayList<Exception> exceptions = null;

Expand All @@ -322,11 +340,12 @@ private void downloadHttp(WebURL url, int retry, Duration connectTimeout, Durati
@Nullable List<WebURL> redirects = null;
try {
beforeDownload(url);
updateProgress(0);
if (resumeContext == null || !resumeContext.hasPartialContent())
updateProgress(0);

@Nullable HttpURLConnection connection;
UrlResponseInfo responseInfo;
@Nullable String bmclapiHash;
@Nullable String bmclapiHash = null;
int responseCode;

WebURL currentURI = url;
Expand All @@ -338,8 +357,11 @@ private void downloadHttp(WebURL url, int retry, Duration connectTimeout, Durati
if (useCachedResult && checkETag && !resumeRequested)
headers.putAll(repository.injectConnection(url));
if (resumeRequested) {
headers.put("accept-encoding", "identity");
headers.put("range", "bytes=" + resumeContext.countUncompressed + "-");
headers.put("if-range", resumeContext.ifRange());
@Nullable String ifRange = resumeContext.ifRange();
if (ifRange != null)
headers.put("if-range", ifRange);
}

do {
Expand All @@ -352,8 +374,12 @@ private void downloadHttp(WebURL url, int retry, Duration connectTimeout, Durati
responseCode = connection.getResponseCode();
responseInfo = UrlResponseInfo.of(connection);

bmclapiHash = responseInfo.headers().firstValue("x-bmclapi-hash").orElse(null);
if (DigestUtils.isSha1Digest(bmclapiHash)) {
@Nullable String responseHash = responseInfo.headers().firstValue("x-bmclapi-hash").orElse(null);
if (DigestUtils.isSha1Digest(responseHash)) {
if (bmclapiHash != null && !bmclapiHash.equalsIgnoreCase(responseHash))
throw new IOException("Conflicting BMCLAPI hashes in redirect chain");
bmclapiHash = responseHash;
headers.put("accept-encoding", "identity");
Optional<Path> cache = repository.checkExistentFile(null, "SHA-1", bmclapiHash);
if (cache.isPresent()) {
useCachedResult(cache.get());
Expand Down Expand Up @@ -432,11 +458,12 @@ private void downloadHttp(WebURL url, int retry, Duration connectTimeout, Durati

if (context == null) {
context = getContext(responseInfo, checkETag, bmclapiHash);
resumeContext = HttpResumeContext.of(responseInfo);
resumeContext = HttpResumeContext.of(responseInfo, bmclapiHash);
} else if (resumeRequested) {
if (resumeContext.canResume(responseCode, responseInfo)) {
if (resumeContext.canResume(responseCode, responseInfo, bmclapiHash)) {
// Resume download
LOG.info("Resuming " + resumeContext.url + " from " + resumeContext.countUncompressed);
contentLength = resumeContext.contentLength - resumeContext.countUncompressed;
} else {
// Failed to resume download, so we will retry from the beginning
resumeContext = null;
Expand All @@ -448,15 +475,16 @@ private void downloadHttp(WebURL url, int retry, Duration connectTimeout, Durati
} else {
discardContext(context);
context = getContext(responseInfo, checkETag, bmclapiHash);
resumeContext = HttpResumeContext.of(responseInfo);
resumeContext = HttpResumeContext.of(responseInfo, bmclapiHash);
}

try {
inputStream = connection.getInputStream();
download(context,
resumeContext, inputStream,
contentLength,
contentEncoding);
contentEncoding, bmclapiHash != null && redirects != null && attempts < attemptsLimit - 1
? retryPolicy : null);
inputStream = null;
responseBodyConsumed = true;
} catch (IOException | InterruptedException | RuntimeException | Error e) {
Expand Down Expand Up @@ -547,7 +575,7 @@ private void downloadNotHttp(WebURL url, int retry, Duration connectTimeout, Dur
download(context,
null, conn.getInputStream(),
conn.getContentLengthLong(),
ContentEncoding.fromConnection(conn));
ContentEncoding.fromConnection(conn), null);
}
return;
} catch (InterruptedException e) {
Expand Down Expand Up @@ -602,6 +630,49 @@ private static void updateDownloadSpeed(long speed) {
downloadSpeed.addAndGet(speed);
}

/// Retries persistently slow BMCLAPI nodes without exhausting downloads on bandwidth-limited connections.
static final class BMCLAPIRetryPolicy {
/// Maximum slow-node retries within one candidate's existing retry budget.
private static final int MAX_SLOW_RETRIES = 2;
/// Observation window for each connection.
private static final long WINDOW_NANOS = TimeUnit.SECONDS.toNanos(10);
/// Minimum transfer rate in bytes per second before retrying a slow node.
private static final long MIN_BYTES_PER_SECOND = 128 * 1024;
/// Monotonic time at the beginning of the current window.
private long started;
/// Total bytes received before the current window.
private long downloaded;
/// Number of node reselections requested by this policy across all attempts.
private int retries;

/// Starts a fresh observation window while retaining the candidate's slow-node retry count.
void startAttempt(long now) {
this.started = now;
this.downloaded = 0;
}

/// Retries a slow window only when finishing at the current rate would take another full window.
/// Unknown-length responses cannot provide a useful estimate and retain normal timeout handling.
boolean shouldRetry(long downloaded, long contentLength, long now) {
if (retries >= MAX_SLOW_RETRIES || contentLength <= downloaded)
return false;

long elapsed = now - started;
if (elapsed < WINDOW_NANOS)
return false;

double bytesPerSecond = (downloaded - this.downloaded) * (double) TimeUnit.SECONDS.toNanos(1) / elapsed;
this.started = now;
this.downloaded = downloaded;
if (bytesPerSecond >= MIN_BYTES_PER_SECOND
|| contentLength - downloaded <= bytesPerSecond * WINDOW_NANOS / TimeUnit.SECONDS.toNanos(1))
return false;

retries++;
return true;
}
}

private static final class CounterInputStream extends FilterInputStream {
long downloaded;

Expand Down
9 changes: 9 additions & 0 deletions HMCLCore/src/main/java/org/jackhuang/hmcl/task/GetTask.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,9 @@
import com.google.gson.reflect.TypeToken;
import org.glavo.url.WebURL;
import org.jackhuang.hmcl.download.DownloadCandidates;
import org.jackhuang.hmcl.util.DigestUtils;
import org.jackhuang.hmcl.util.gson.JsonUtils;
import org.jackhuang.hmcl.util.io.ChecksumMismatchException;
import org.jackhuang.hmcl.util.io.NetworkUtils;
import org.jackhuang.hmcl.util.io.UrlResponseInfo;
import org.jetbrains.annotations.Nullable;
Expand Down Expand Up @@ -63,6 +65,7 @@ protected void useCachedResult(Path cachedFile) throws IOException {
setResult(Files.readString(cachedFile));
}

/// Creates an in-memory sink, verifying the supplied content hash before decoding or caching text.
@Override
protected Context getContext(@Nullable UrlResponseInfo response, boolean checkETag, @Nullable String bmclapiHash) {
long length = -1;
Expand All @@ -85,6 +88,12 @@ public void write(byte[] buffer, int offset, int len) {
public void close() throws IOException {
if (!isSuccess()) return;

if (bmclapiHash != null) {
String actualHash = DigestUtils.digestToString("SHA-1", baos.toByteArray());
if (!bmclapiHash.equalsIgnoreCase(actualHash))
throw new ChecksumMismatchException("SHA-1", bmclapiHash, actualHash);
}

Charset charset = StandardCharsets.UTF_8;
if (response != null)
charset = NetworkUtils.getCharsetFromContentType(response.headers().firstValue("content-type").orElse(null));
Expand Down
Loading
Loading