fix: manifest out of sync with NAS causes gallery to show stale item count
Two root causes: 1. Backup interrupted before loop completion = manifest write never fires, leaving uploaded files as NAS orphans invisible to gallery and dedup. 2. Manifest only written once at loop end, so any crash loses all progress. Fixes: - On each backup run, list the NAS directory and merge any files not tracked in the manifest (orphaned from prior interrupted sessions). The healed manifest is written back to NAS and NASManifestCache before uploads start, so the gallery reflects actual NAS contents immediately on next open. - Incremental checkpoint writes every 25 uploads using the already-open transfer connection — interrupted backups now preserve progress in blocks of 25 instead of losing everything. Co-Authored-By: Kutesir <kutesir@provoc.ug> Co-Authored-By: Sentry <sentry@provoc.ug>
This commit is contained in:
Robin Kutesa
@@ -92,39 +92,75 @@ final class BackupEngine: ObservableObject {
|
||||
|
||||
try Task.checkCancellation()
|
||||
|
||||
// ── 3. Load manifest — parse off the main actor ────────────────────
|
||||
// The download suspends the main actor (good). JSON decode + Set<String>
|
||||
// construction for a large manifest is O(N) CPU work — run it in a
|
||||
// detached task so we don't block touch handling or animations.
|
||||
// ── 3. Load manifest + heal orphaned files ─────────────────────────
|
||||
// After any interrupted backup, files may sit on NAS without a manifest
|
||||
// entry. We list the directory and merge any untracked files so the
|
||||
// gallery and dedup logic reflect actual NAS contents.
|
||||
let manifestPath = "\(connection.remotePath)/\(BackupManifest.remoteFilename)"
|
||||
let spManifest = signposter.beginInterval("ManifestLoad")
|
||||
let manifestData = try? await transfer.downloadData(at: manifestPath)
|
||||
let (index, loadedManifest): (ManifestIndex, BackupManifest?) = await Task.detached(priority: .userInitiated) {
|
||||
guard let data = manifestData,
|
||||
let manifest = try? JSONDecoder.kisani.decode(BackupManifest.self, from: data)
|
||||
else { return (ManifestIndex(nasListing: []), nil) }
|
||||
return (ManifestIndex(manifest: manifest), manifest)
|
||||
}.value
|
||||
signposter.endInterval("ManifestLoad", spManifest,
|
||||
"\(index.totalCount) entries, filename-only=\(index.isFilenameOnly)")
|
||||
logger.info("Manifest: \(index.totalCount) entries, filename-only=\(index.isFilenameOnly)")
|
||||
|
||||
// Seed the local cache so BackupStatusService's fast-path reconcile reflects
|
||||
// the current NAS state before the upload loop begins.
|
||||
if let m = loadedManifest {
|
||||
await NASManifestCache.shared.update(m)
|
||||
let manifestData = try? await transfer.downloadData(at: manifestPath)
|
||||
let nasItems = (try? await transfer.listDirectory(at: connection.remotePath)) ?? []
|
||||
|
||||
// Decode + heal off main actor (CPU-bound work).
|
||||
struct ManifestLoadResult { var index: ManifestIndex; var manifest: BackupManifest; var orphanCount: Int }
|
||||
var loadResult: ManifestLoadResult = await Task.detached(priority: .userInitiated) {
|
||||
var manifest: BackupManifest
|
||||
if let data = manifestData,
|
||||
let decoded = try? JSONDecoder.kisani.decode(BackupManifest.self, from: data) {
|
||||
manifest = decoded
|
||||
} else {
|
||||
manifest = BackupManifest.buildFromNASListing(nasItems)
|
||||
return ManifestLoadResult(index: ManifestIndex(manifest: manifest), manifest: manifest, orphanCount: 0)
|
||||
}
|
||||
|
||||
// Heal: find NAS files not tracked in the manifest (orphans from
|
||||
// backups that were interrupted before the manifest write step).
|
||||
let knownFilenames = Set(manifest.entries.map { $0.filename.lowercased() })
|
||||
let mediaExts: Set<String> = ["jpg","jpeg","png","heic","heif","gif","webp","bmp",
|
||||
"tiff","tif","raw","dng","cr2","nef","arw",
|
||||
"mp4","mov","m4v","avi","mkv"]
|
||||
let orphans: [ManifestEntry] = nasItems.compactMap { item in
|
||||
guard !item.isDirectory,
|
||||
item.name != BackupManifest.remoteFilename,
|
||||
!item.name.hasPrefix("."),
|
||||
mediaExts.contains((item.name as NSString).pathExtension.lowercased()),
|
||||
!knownFilenames.contains(item.name.lowercased()) else { return nil }
|
||||
return ManifestEntry(
|
||||
localIdentifier: "",
|
||||
filename: item.name,
|
||||
creationDate: item.modifiedDate,
|
||||
fileSize: item.size,
|
||||
remotePath: item.path,
|
||||
uploadedAt: item.modifiedDate ?? Date()
|
||||
)
|
||||
}
|
||||
if !orphans.isEmpty { manifest.merge(entries: orphans) }
|
||||
return ManifestLoadResult(index: ManifestIndex(manifest: manifest), manifest: manifest, orphanCount: orphans.count)
|
||||
}.value
|
||||
|
||||
signposter.endInterval("ManifestLoad", spManifest,
|
||||
"\(loadResult.index.totalCount) entries, orphans=\(loadResult.orphanCount)")
|
||||
logger.info("Manifest: \(loadResult.index.totalCount) entries (\(loadResult.orphanCount) orphans healed)")
|
||||
|
||||
// If the manifest gained entries from healing, write it back to NAS now
|
||||
// so the gallery reflects actual NAS contents even before uploads start.
|
||||
if loadResult.orphanCount > 0, let encoded = try? JSONEncoder.kisani.encode(loadResult.manifest) {
|
||||
try? await transfer.writeData(encoded, to: manifestPath)
|
||||
}
|
||||
|
||||
// Always seed local cache with the current (healed) state.
|
||||
await NASManifestCache.shared.update(loadResult.manifest)
|
||||
|
||||
try Task.checkCancellation()
|
||||
|
||||
// ── 4. Build pending queue off the main actor ──────────────────────
|
||||
// Filtering a large [PhotoAsset] is O(N) with two Set lookups per item.
|
||||
// Safe to run in background: both assets and index are value-type safe.
|
||||
let spFilter = signposter.beginInterval("BuildPendingQueue")
|
||||
let currentIndex = loadResult.index
|
||||
let pendingAssets: [PhotoAsset] = await Task.detached(priority: .userInitiated) {
|
||||
assets.filter { asset in
|
||||
!index.matches(localIdentifier: asset.localIdentifier) &&
|
||||
!(index.isFilenameOnly && index.matches(filename: asset.filename))
|
||||
!currentIndex.matches(localIdentifier: asset.localIdentifier) &&
|
||||
!(currentIndex.isFilenameOnly && currentIndex.matches(filename: asset.filename))
|
||||
}
|
||||
}.value
|
||||
signposter.endInterval("BuildPendingQueue", spFilter, "\(pendingAssets.count) pending")
|
||||
@@ -159,6 +195,8 @@ final class BackupEngine: ObservableObject {
|
||||
var throttle = ProgressThrottle(hz: 8)
|
||||
var manifestEntries: [ManifestEntry] = []
|
||||
manifestEntries.reserveCapacity(pendingAssets.count)
|
||||
var lastCheckpointAt = 0 // uploaded count at last manifest checkpoint write
|
||||
let checkpointInterval = 25
|
||||
|
||||
let spUpload = signposter.beginInterval("UploadLoop", "\(pendingAssets.count) files")
|
||||
|
||||
@@ -258,6 +296,19 @@ final class BackupEngine: ObservableObject {
|
||||
remotePath: remotePath,
|
||||
uploadedAt: Date()
|
||||
))
|
||||
|
||||
// Checkpoint write every N uploads — protects against interruption.
|
||||
// Uses the same open connection so no extra connect/disconnect overhead.
|
||||
if uploaded - lastCheckpointAt >= checkpointInterval {
|
||||
lastCheckpointAt = uploaded
|
||||
var checkpoint = loadResult.manifest
|
||||
checkpoint.merge(entries: manifestEntries)
|
||||
if let encoded = try? JSONEncoder.kisani.encode(checkpoint) {
|
||||
try? await transfer.writeData(encoded, to: manifestPath)
|
||||
await NASManifestCache.shared.update(checkpoint)
|
||||
logger.info("Manifest checkpoint: \(checkpoint.entries.count) total entries")
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
lastUploadError = error
|
||||
logger.warning("Upload attempt \(attempt + 1)/\(maxRetries) failed for \(asset.filename, privacy: .public): \(error.localizedDescription, privacy: .public)")
|
||||
|
||||
Reference in New Issue
Block a user