Fix delay between URL fetch and image download (#9452)
Fetch each source image URL immediately before downloading each image instead of fetching all URLs and then downloading all images. Source image URLs may change, so the downloader may fail if there is too long a delay between fetching the image URL and downloading the image.
Cette révision appartient à :
Parent
cb2d43c0d1
révision
bbe0ab1dd0
1 fichiers modifiés avec 10 ajouts et 10 suppressions
|
@ -338,21 +338,21 @@ class Downloader(
|
|||
|
||||
download.status = Download.State.DOWNLOADING
|
||||
|
||||
// Get all the URLs to the source images, fetch pages if necessary
|
||||
pageList.filter { it.imageUrl.isNullOrEmpty() }.forEach { page ->
|
||||
page.status = Page.State.LOAD_PAGE
|
||||
try {
|
||||
page.imageUrl = download.source.fetchImageUrl(page).awaitSingle()
|
||||
} catch (e: Throwable) {
|
||||
page.status = Page.State.ERROR
|
||||
}
|
||||
}
|
||||
|
||||
// Start downloading images, consider we can have downloaded images already
|
||||
// Concurrently do 2 pages at a time
|
||||
pageList.asFlow()
|
||||
.flatMapMerge(concurrency = 2) { page ->
|
||||
flow {
|
||||
// Fetch image URL if necessary
|
||||
if (page.imageUrl.isNullOrEmpty()) {
|
||||
page.status = Page.State.LOAD_PAGE
|
||||
try {
|
||||
page.imageUrl = download.source.fetchImageUrl(page).awaitSingle()
|
||||
} catch (e: Throwable) {
|
||||
page.status = Page.State.ERROR
|
||||
}
|
||||
}
|
||||
|
||||
withIOContext { getOrDownloadImage(page, download, tmpDir) }
|
||||
emit(page)
|
||||
}.flowOn(Dispatchers.IO)
|
||||
|
|
Référencer dans un nouveau ticket