fix: manifest cache not updated after backup causes NAS Archive = 1 and backup loop
- writeManifest now updates NASManifestCache after every successful NAS write; previously the fast-path reconcile would read the pre-backup cache entry (e.g. 1 entry) and overwrite nasArchiveTotal / alreadySafe with stale values - BackupEngine seeds NASManifestCache when it loads the manifest so the cache is always at least as fresh as what the engine sees at backup-start time - Remove snapshot.nasArchiveTotal += uploaded from refreshAfterBackup; the correct total comes from the merged manifest written to NAS, not an additive delta from a potentially-wrong baseline - Fix JSONDecoder() → JSONDecoder.kisani throughout manifest load paths so ISO8601 dates decode correctly instead of silently failing and returning nil - Add retry with exponential backoff (max 3 attempts, 2 s / 4 s intervals) to the upload loop; permanently-failed items are now marked .failed only after all retries are exhausted Co-Authored-By: Kutesir <kutesir@provoc.ug> Co-Authored-By: Sentry <sentry@provoc.ug>
This commit is contained in:
Robin Kutesa
@@ -99,16 +99,22 @@ final class BackupEngine: ObservableObject {
|
||||
let manifestPath = "\(connection.remotePath)/\(BackupManifest.remoteFilename)"
|
||||
let spManifest = signposter.beginInterval("ManifestLoad")
|
||||
let manifestData = try? await transfer.downloadData(at: manifestPath)
|
||||
let index: ManifestIndex = await Task.detached(priority: .userInitiated) {
|
||||
let (index, loadedManifest): (ManifestIndex, BackupManifest?) = await Task.detached(priority: .userInitiated) {
|
||||
guard let data = manifestData,
|
||||
let manifest = try? JSONDecoder().decode(BackupManifest.self, from: data)
|
||||
else { return ManifestIndex(nasListing: []) }
|
||||
return ManifestIndex(manifest: manifest)
|
||||
let manifest = try? JSONDecoder.kisani.decode(BackupManifest.self, from: data)
|
||||
else { return (ManifestIndex(nasListing: []), nil) }
|
||||
return (ManifestIndex(manifest: manifest), manifest)
|
||||
}.value
|
||||
signposter.endInterval("ManifestLoad", spManifest,
|
||||
"\(index.totalCount) entries, filename-only=\(index.isFilenameOnly)")
|
||||
logger.info("Manifest: \(index.totalCount) entries, filename-only=\(index.isFilenameOnly)")
|
||||
|
||||
// Seed the local cache so BackupStatusService's fast-path reconcile reflects
|
||||
// the current NAS state before the upload loop begins.
|
||||
if let m = loadedManifest {
|
||||
await NASManifestCache.shared.update(m)
|
||||
}
|
||||
|
||||
try Task.checkCancellation()
|
||||
|
||||
// ── 4. Build pending queue off the main actor ──────────────────────
|
||||
@@ -156,6 +162,8 @@ final class BackupEngine: ObservableObject {
|
||||
|
||||
let spUpload = signposter.beginInterval("UploadLoop", "\(pendingAssets.count) files")
|
||||
|
||||
let maxRetries = 3
|
||||
|
||||
for (assetIdx, asset) in pendingAssets.enumerated() {
|
||||
if isCancelled { break }
|
||||
while case .paused = job.status {
|
||||
@@ -164,7 +172,6 @@ final class BackupEngine: ObservableObject {
|
||||
|
||||
let remotePath = "\(connection.remotePath)/\(asset.filename)"
|
||||
|
||||
// Mark as uploading
|
||||
if assetIdx < queueItems.count {
|
||||
queueItems[assetIdx].status = .uploading
|
||||
}
|
||||
@@ -180,60 +187,87 @@ final class BackupEngine: ObservableObject {
|
||||
continue
|
||||
}
|
||||
|
||||
do {
|
||||
let spFile = signposter.beginInterval("UploadFile",
|
||||
"\(asset.filename, privacy: .public)")
|
||||
let localURL = try await photoService.exportAsset(asset)
|
||||
defer { try? FileManager.default.removeItem(at: localURL) }
|
||||
var uploadSucceeded = false
|
||||
var lastUploadError: Error?
|
||||
|
||||
let fileSize = (try? localURL.resourceValues(forKeys: [.fileSizeKey]).fileSize)
|
||||
.flatMap { Int64($0) } ?? 0
|
||||
let bytesSoFar = totalBytes
|
||||
let capturedIdx = assetIdx
|
||||
for attempt in 0..<maxRetries {
|
||||
if isCancelled { break }
|
||||
|
||||
if capturedIdx < queueItems.count {
|
||||
queueItems[capturedIdx].totalBytes = fileSize
|
||||
if attempt > 0 {
|
||||
// Exponential backoff: 2s, 4s before retries 2 and 3
|
||||
let backoffNs = UInt64(pow(2.0, Double(attempt)) * 1_000_000_000)
|
||||
try? await Task.sleep(nanoseconds: backoffNs)
|
||||
if assetIdx < queueItems.count {
|
||||
queueItems[assetIdx].retryCount = attempt
|
||||
queueItems[assetIdx].status = .uploading
|
||||
}
|
||||
logger.info("Retry \(attempt)/\(maxRetries - 1) for \(asset.filename, privacy: .public)")
|
||||
}
|
||||
|
||||
try await transfer.upload(localURL: localURL, remotePath: remotePath) { sent, _ in
|
||||
let speed = speedTracker.update(bytesSent: sent)
|
||||
guard throttle.shouldUpdate() else { return }
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
self.job.updateProgress(
|
||||
fileName: asset.filename,
|
||||
fileSize: fileSize,
|
||||
bytesTransferred: bytesSoFar + sent,
|
||||
speed: speed
|
||||
)
|
||||
if capturedIdx < self.queueItems.count {
|
||||
self.queueItems[capturedIdx].bytesUploaded = sent
|
||||
self.queueItems[capturedIdx].speedBytesPerSec = speed
|
||||
do {
|
||||
let spFile = signposter.beginInterval("UploadFile",
|
||||
"\(asset.filename, privacy: .public)")
|
||||
let localURL = try await photoService.exportAsset(asset)
|
||||
defer { try? FileManager.default.removeItem(at: localURL) }
|
||||
|
||||
let fileSize = (try? localURL.resourceValues(forKeys: [.fileSizeKey]).fileSize)
|
||||
.flatMap { Int64($0) } ?? 0
|
||||
let bytesSoFar = totalBytes
|
||||
let capturedIdx = assetIdx
|
||||
|
||||
if capturedIdx < queueItems.count {
|
||||
queueItems[capturedIdx].totalBytes = fileSize
|
||||
queueItems[capturedIdx].bytesUploaded = 0
|
||||
}
|
||||
|
||||
try await transfer.upload(localURL: localURL, remotePath: remotePath) { sent, _ in
|
||||
let speed = speedTracker.update(bytesSent: sent)
|
||||
guard throttle.shouldUpdate() else { return }
|
||||
Task { @MainActor [weak self] in
|
||||
guard let self else { return }
|
||||
self.job.updateProgress(
|
||||
fileName: asset.filename,
|
||||
fileSize: fileSize,
|
||||
bytesTransferred: bytesSoFar + sent,
|
||||
speed: speed
|
||||
)
|
||||
if capturedIdx < self.queueItems.count {
|
||||
self.queueItems[capturedIdx].bytesUploaded = sent
|
||||
self.queueItems[capturedIdx].speedBytesPerSec = speed
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
signposter.endInterval("UploadFile", spFile, "\(fileSize) bytes")
|
||||
totalBytes += fileSize
|
||||
job.fileCompleted(skipped: false)
|
||||
uploaded += 1
|
||||
uploadSucceeded = true
|
||||
|
||||
if assetIdx < queueItems.count {
|
||||
queueItems[assetIdx].status = .uploaded
|
||||
queueItems[assetIdx].bytesUploaded = fileSize
|
||||
queueItems[assetIdx].completedAt = Date()
|
||||
}
|
||||
|
||||
manifestEntries.append(ManifestEntry(
|
||||
localIdentifier: asset.localIdentifier,
|
||||
filename: asset.filename,
|
||||
creationDate: asset.creationDate,
|
||||
fileSize: fileSize,
|
||||
remotePath: remotePath,
|
||||
uploadedAt: Date()
|
||||
))
|
||||
} catch {
|
||||
lastUploadError = error
|
||||
logger.warning("Upload attempt \(attempt + 1)/\(maxRetries) failed for \(asset.filename, privacy: .public): \(error.localizedDescription, privacy: .public)")
|
||||
}
|
||||
|
||||
signposter.endInterval("UploadFile", spFile, "\(fileSize) bytes")
|
||||
totalBytes += fileSize
|
||||
job.fileCompleted(skipped: false)
|
||||
uploaded += 1
|
||||
if uploadSucceeded { break }
|
||||
}
|
||||
|
||||
if assetIdx < queueItems.count {
|
||||
queueItems[assetIdx].status = .uploaded
|
||||
queueItems[assetIdx].bytesUploaded = fileSize
|
||||
queueItems[assetIdx].completedAt = Date()
|
||||
}
|
||||
|
||||
manifestEntries.append(ManifestEntry(
|
||||
localIdentifier: asset.localIdentifier,
|
||||
filename: asset.filename,
|
||||
creationDate: asset.creationDate,
|
||||
fileSize: fileSize,
|
||||
remotePath: remotePath,
|
||||
uploadedAt: Date()
|
||||
))
|
||||
} catch {
|
||||
logger.error("Upload failed for \(asset.filename, privacy: .public): \(error.localizedDescription, privacy: .public)")
|
||||
if !uploadSucceeded, let error = lastUploadError {
|
||||
logger.error("Upload permanently failed for \(asset.filename, privacy: .public): \(error.localizedDescription, privacy: .public)")
|
||||
job.fileFailed()
|
||||
failed += 1
|
||||
if assetIdx < queueItems.count {
|
||||
|
||||
Reference in New Issue
Block a user